Skip to main content

zeph_durable/backend/
local.rs

1// SPDX-FileCopyrightText: 2026 Andrei G <bug-ops>
2// SPDX-License-Identifier: MIT OR Apache-2.0
3
4//! The always-compiled local journal backend.
5//!
6//! [`LocalBackend`] owns a dedicated [`zeph_db::DbPool`] on its own `durable.db` file (INV-14): a
7//! separate database keeps the high-write journal off the shared application pool, where
8//! `BEGIN IMMEDIATE` contention would otherwise serialize unrelated writers. The schema lives in
9//! `zeph-db/migrations/{sqlite,postgres}/` and is applied via [`zeph_db::run_migrations`]; the
10//! backend owns no `.sql` files of its own.
11//!
12//! # Sealing and integrity
13//!
14//! Payload-bearing entries (currently [`EntryKind::StepResult`]) are AEAD-sealed through the
15//! injected [`PayloadCipher`] before they touch the database, with the entry's location bound as
16//! associated data so a sealed blob cannot be relocated to another step or execution. Control
17//! entries (currently [`EntryKind::EffectIntent`]) carry no payload; when an HMAC key is configured
18//! the backend stamps a keyed BLAKE3 row HMAC over their identity for shared-database deployments,
19//! and every read recomputes and constant-time-verifies that HMAC, failing closed with
20//! [`DurableError::ControlIntegrity`] on a forged or relocated row. When no cipher is injected the
21//! payload is stored verbatim — a development-only posture gated by
22//! [`encryption_gate`](crate::encryption_gate) at startup.
23//!
24//! # Scope
25//!
26//! This revision journals the step-execution entries — [`EntryKind::StepResult`] and
27//! [`EntryKind::EffectIntent`] — that the durable step primitive records, plus execution
28//! lifecycle (open and [`finalize`](Journal::finalize)), the writer's restart anchor (`max_seq`),
29//! and the idempotency-key point lookup
30//! ([`lookup_committed_result`](ExecutionBackend::lookup_committed_result)) that lets a guarded step
31//! recognize an already-committed effect after a replay divergence (INV-13). Promise, timer, and
32//! checkpoint entries are journaled by the promise/timer and retention layers; until then
33//! [`append`](Journal::append) of those kinds fails closed with
34//! [`DurableError::UnsupportedEntryKind`] rather than dropping their state. The retention sweep
35//! ([`prune`](Journal::prune)) is a no-op stub here.
36
37use std::fmt;
38use std::fmt::Write as _;
39use std::path::PathBuf;
40use std::sync::Arc;
41use std::time::{SystemTime, UNIX_EPOCH};
42
43use bytes::Bytes;
44use zeph_db::{DbPool, sql};
45
46use crate::backend::execution_lock::ExecutionLock;
47use crate::backend::{BackendCapabilities, ExecutionBackend, ExecutionSummary, RedactedEntry};
48use crate::cipher::{EntryKindTag, PayloadAad, PayloadCipher, ensure_payload_within_limit};
49use crate::config::RetentionPolicy;
50use crate::error::DurableError;
51use crate::ids::{
52    ExecutionId, ExecutionKind, IdempotencyKey, JournalSeq, PromiseId, StepId, TimerId,
53};
54use crate::journal::{EntryKind, ExecutionStatus, Journal, JournalEntry};
55use crate::promise::PromiseRecord;
56use crate::retention::{CheckpointSnapshot, FoldedStep, decode_checkpoint, encode_checkpoint};
57use crate::waiters::NotifyRegistry;
58use tracing::Instrument as _;
59
60/// Slack added to `max_payload_bytes` for the read-side size guard.
61///
62/// The stored blob carries AEAD framing (key-id, extended nonce, tag) on top of the plaintext, so a
63/// payload accepted at exactly the limit on write is slightly larger on read. The guard exists only
64/// to reject absurdly large rows before allocation/decryption (INV-11), so a small fixed slack
65/// above any real AEAD overhead keeps legitimate near-limit entries readable without weakening the
66/// denial-of-service protection.
67const SEAL_OVERHEAD_SLACK: u64 = 128;
68
69/// Row shape returned by the `list_executions` query.
70type ExecutionRow = (String, String, String, i64, i64, Option<i64>, i64);
71
72/// Row shape returned by the `read_execution_redacted` query.
73type RedactedRow = (
74    i64,
75    i64,
76    String,
77    Option<Vec<u8>>,
78    Option<String>,
79    Option<i64>,
80    i64,
81);
82
83/// Render the first 8 bytes of an idempotency key as a lowercase hex prefix (INV-5).
84fn idem_key_prefix(bytes: &[u8]) -> String {
85    bytes.iter().take(8).fold(String::new(), |mut acc, b| {
86        let _ = write!(acc, "{b:02x}");
87        acc
88    })
89}
90
91/// Outcome of a [`LocalBackend::cancel_execution`] request (#6362).
92///
93/// `Canceled` is the only outcome that wrote to the row; every other variant is a refusal or a
94/// no-op, so a caller can always trust "did this call change the database" from the variant alone.
95#[derive(Debug, Clone, Copy, PartialEq, Eq)]
96pub enum CancelOutcome {
97    /// The execution was `running` with no live owner detected; it is now `canceled` and will
98    /// never be reopened (INV-16′).
99    Canceled,
100    /// The execution was already terminal (`completed`, `failed`, `aborted`, or already
101    /// `canceled`) — idempotent no-op, per NFR-003.
102    AlreadyTerminal {
103        /// The execution's status before (and, since no write happened, after) this call.
104        status: ExecutionStatus,
105    },
106    /// No execution exists for the given id.
107    NotFound,
108    /// Another process holds the execution's [`ExecutionLock`] (SQLite/Unix only). This may be the
109    /// execution's true owner still journaling, or a concurrent maintenance sweep/prune/cancel
110    /// transiently holding the same lock — the flock alone cannot distinguish the two, so no claim
111    /// stronger than "held" is made. The row was not touched; cooperative live-owner cancellation
112    /// (FR-007) is deferred to a follow-up issue.
113    LiveOwner {
114        /// PID of the process currently holding the lock, or `0` if it could not be determined.
115        pid: u32,
116    },
117    /// This backend cannot verify whether a live owner holds the execution (a cross-process
118    /// backend, e.g. Postgres, with no advisory-lock directory to probe). Refusing rather than
119    /// blind-flipping a possibly-live row (F3).
120    LivenessUnverifiable,
121}
122
123/// The always-compiled durable backend that journals to a dedicated `durable.db`.
124///
125/// Construct it from a [`zeph_db::DbPool`] (or open one with [`LocalBackend::open`]), then attach an
126/// optional [`PayloadCipher`] and HMAC key with the builder methods. Call [`LocalBackend::init`]
127/// once before use to apply the schema migrations.
128///
129/// # Examples
130///
131/// ```no_run
132/// # async fn run() -> Result<(), Box<dyn std::error::Error>> {
133/// use zeph_durable::LocalBackend;
134///
135/// // 1 MiB payload ceiling, matching the spec default.
136/// let backend = LocalBackend::open("durable.db", 1_048_576).await?;
137/// backend.init().await?;
138/// # Ok(()) }
139/// ```
140pub struct LocalBackend {
141    pool: DbPool,
142    cipher: Option<Arc<dyn PayloadCipher>>,
143    hmac_key: Option<[u8; 32]>,
144    /// Previous control-entry HMAC key for the rotation window (#6451), mirroring the AEAD
145    /// cipher's `previous` slot. `Some` only while a `zeph durable rotate-key` window is open
146    /// (`config.previous_key_id.is_some()`); `verify_control_hmac` tries this key when a row
147    /// fails to verify under `hmac_key`. Writes always stamp with `hmac_key` only.
148    previous_hmac_key: Option<[u8; 32]>,
149    /// The current high-water-mark key (issue #6360), keyed by its non-secret rotation epoch.
150    /// `None` disables the HWM: no bump on commit/fold, and [`open_execution`](Self::open_execution)
151    /// skips its internal high-water-mark verification entirely on resume. Unlike
152    /// `hmac_key` (shared-DB gated, INV-8), the HWM key is meant to be attached unconditionally
153    /// (FR-009) — it is the only mechanism that detects deletion of a committed `StepResult` row,
154    /// a threat class the AEAD payload seal and the row HMAC do not cover on any deployment,
155    /// single-user local included.
156    hwm_key: Option<HwmKeySlot>,
157    /// A previous HWM key, still accepted for verification during a rotation window (FR-008). Never
158    /// used to sign new writes — every bump/fold always signs under `hwm_key`.
159    hwm_key_previous: Option<HwmKeySlot>,
160    max_payload_bytes: u64,
161    /// In-process wakeup map for parked promise awaits, shared with the resolver path.
162    promise_waiters: NotifyRegistry,
163    /// In-process wakeup map for parked timers, shared with the timer service.
164    timer_waiters: NotifyRegistry,
165    /// Directory for per-`ExecutionId` advisory lock files (INV-15, #6122), used by
166    /// [`open_execution_exclusive`](Self::open_execution_exclusive). `None` when no on-disk path
167    /// is known for this backend — a `:memory:` database, a backend built via
168    /// [`LocalBackend::new`] from a caller-supplied pool, or a non-SQLite (Postgres) deployment,
169    /// where a filesystem lock file cannot express cross-process exclusivity anyway.
170    lock_dir: Option<PathBuf>,
171    /// Set once [`sweep_orphans`](Self::sweep_orphans) has emitted its warn-once log for a
172    /// `lock_dir = None` backend (#6254), so a background retention tick every
173    /// `prune_interval_secs` does not spam the log for the lifetime of the process.
174    orphan_sweep_warned: std::sync::atomic::AtomicBool,
175    /// Vault-sealed integrity marker (issue #6449). `true` only when the *presence* of
176    /// `ZEPH_DURABLE_INTEGRITY_SEALED` in the vault was confirmed at bootstrap — an
177    /// attacker with DB write access cannot set this to `true` (it is never derived from any DB
178    /// column). Once sealed, [`check_high_water_mark`](Self::check_high_water_mark) treats an
179    /// absent integrity row on a keyed, non-grandfathered execution with committed `StepResult`s
180    /// as unconditional tamper, closing the pre-seal migration posture's downgrade lever.
181    integrity_sealed: bool,
182    /// Execution IDs explicitly grandfathered past the seal via `zeph durable seal-integrity
183    /// --grandfather` (issue #6449) — a vault-stored, unforgeable-by-DB-write set. Each entry is
184    /// a **permanent** opt-out for that one execution (not merely a frozen pre-seal snapshot): an
185    /// attacker with DB write access can delete-and-reinsert forged content under the same
186    /// grandfathered `execution_id` and it will still resume unverified. This is an accepted,
187    /// bounded, documented residual of the opt-out — operators should prefer draining a
188    /// resumable execution to a terminal state over grandfathering it.
189    integrity_grandfather: std::collections::HashSet<ExecutionId>,
190}
191
192/// One row-HMAC/high-water-mark key, addressed by its non-secret rotation epoch (FR-008).
193///
194/// The epoch is not sensitive (it is stored in the clear alongside the signed HWM tuple) — it lets
195/// a verifier distinguish "signed under a key I don't currently hold" (re-keyed) from "signed under
196/// my current key but the hash doesn't match" (tampered), per FR-008.
197#[derive(Clone, Copy)]
198struct HwmKeySlot {
199    epoch: u32,
200    key: [u8; 32],
201}
202
203impl fmt::Debug for LocalBackend {
204    /// Redacts the cipher and HMAC/HWM key material — never print key bytes or a cipher handle.
205    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
206        f.debug_struct("LocalBackend")
207            .field("cipher", &self.cipher.as_ref().map(|_| "<cipher>"))
208            .field("hmac_key", &self.hmac_key.as_ref().map(|_| "<redacted>"))
209            .field(
210                "previous_hmac_key",
211                &self.previous_hmac_key.as_ref().map(|_| "<redacted>"),
212            )
213            .field("hwm_key_epoch", &self.hwm_key.as_ref().map(|s| s.epoch))
214            .field("max_payload_bytes", &self.max_payload_bytes)
215            .finish_non_exhaustive()
216    }
217}
218
219impl LocalBackend {
220    /// Wrap an existing [`zeph_db::DbPool`] as a local backend with the given payload ceiling.
221    ///
222    /// Call [`LocalBackend::init`] before any journal operation to apply the schema. Attach a
223    /// cipher and HMAC key with [`with_cipher`](Self::with_cipher) and
224    /// [`with_hmac_key`](Self::with_hmac_key).
225    #[must_use]
226    pub fn new(pool: DbPool, max_payload_bytes: u64) -> Self {
227        Self {
228            pool,
229            cipher: None,
230            hmac_key: None,
231            previous_hmac_key: None,
232            hwm_key: None,
233            hwm_key_previous: None,
234            max_payload_bytes,
235            promise_waiters: NotifyRegistry::default(),
236            timer_waiters: NotifyRegistry::default(),
237            lock_dir: None,
238            orphan_sweep_warned: std::sync::atomic::AtomicBool::new(false),
239            integrity_sealed: false,
240            integrity_grandfather: std::collections::HashSet::new(),
241        }
242    }
243
244    /// Open (or create) a backend on a dedicated `durable.db` file (or `:memory:`).
245    ///
246    /// Connecting also applies the schema migrations, so a freshly opened backend is ready to use;
247    /// [`init`](Self::init) may still be called and is idempotent.
248    ///
249    /// On the `SQLite` backend, also derives the lock directory used by
250    /// [`open_execution_exclusive`](Self::open_execution_exclusive) from `path` (a sibling
251    /// `<path>.locks/` directory), unless `path` is `:memory:`. The Postgres backend never derives
252    /// one — `path` there is a connection URL (which may embed credentials), not a filesystem path.
253    ///
254    /// # Errors
255    ///
256    /// Returns [`DurableError::Storage`] if the pool cannot be opened or migrations fail.
257    pub async fn open(path: &str, max_payload_bytes: u64) -> Result<Self, DurableError> {
258        let pool = zeph_db::DbConfig {
259            url: path.to_string(),
260            pool_size: 5,
261        }
262        .connect()
263        .await
264        .map_err(|e| DurableError::storage("open", e))?;
265        let mut backend = Self::new(pool, max_payload_bytes);
266        backend.lock_dir = lock_dir_for_path(path);
267        Ok(backend)
268    }
269
270    /// Inject the AEAD payload cipher used to seal and open payload-bearing entries.
271    #[must_use]
272    pub fn with_cipher(mut self, cipher: Arc<dyn PayloadCipher>) -> Self {
273        self.cipher = Some(cipher);
274        self
275    }
276
277    /// Configure the keyed-BLAKE3 HMAC key stamped over control entries on shared-database
278    /// deployments, and used to verify them again on every read (INV-8).
279    #[must_use]
280    pub fn with_hmac_key(mut self, key: [u8; 32]) -> Self {
281        self.hmac_key = Some(key);
282        self
283    }
284
285    /// Register a previous control-entry HMAC key for the rotation window (#6451), mirroring the
286    /// AEAD cipher's `with_previous` window mechanism (`zeph_core::durable::XChaCha20Poly1305Cipher`).
287    ///
288    /// The row-HMAC verification path tries this key when a row fails to
289    /// verify under the current [`with_hmac_key`](Self::with_hmac_key) key, so pre-rotation
290    /// `EffectIntent` control entries stay readable until the window is closed with `zeph durable
291    /// rotate-key --drop-previous`. Unlike the AEAD cipher's `key_id`-tagged blob layout, the
292    /// stored `hmac` column carries no key selector — a deliberate divergence, since control rows
293    /// have no payload envelope to carry one; try-both is security-equivalent for a single-slot
294    /// window. Writes always stamp with the current key only, never this one.
295    #[must_use]
296    pub fn with_previous_hmac_key(mut self, key: [u8; 32]) -> Self {
297        self.previous_hmac_key = Some(key);
298        self
299    }
300
301    /// Configure the current high-water-mark key (issue #6360), addressed by its non-secret
302    /// rotation `epoch`.
303    ///
304    /// Unlike [`with_hmac_key`](Self::with_hmac_key), this is meant to be attached unconditionally
305    /// (FR-009) — attach it whenever `ZEPH_DURABLE_KEY` resolves from the vault, regardless of
306    /// `shared_db`. When set, every committed `StepResult` bumps the signed
307    /// `{key_epoch, max_committed_step_id, committed_result_count}` tuple in-transaction, and
308    /// [`open_execution`](Self::open_execution) verifies it on every resume (FR-004, US-003).
309    #[must_use]
310    pub fn with_hwm_key(mut self, epoch: u32, key: [u8; 32]) -> Self {
311        self.hwm_key = Some(HwmKeySlot { epoch, key });
312        self
313    }
314
315    /// Register a previous high-water-mark key for the rotation window (FR-008).
316    ///
317    /// Verification tries [`hwm_key`](Self::with_hwm_key) first by epoch match, then this slot —
318    /// never the reverse. New writes always sign under the current key regardless of this slot.
319    #[must_use]
320    pub fn with_previous_hwm_key(mut self, epoch: u32, key: [u8; 32]) -> Self {
321        self.hwm_key_previous = Some(HwmKeySlot { epoch, key });
322        self
323    }
324
325    /// Configure whether this backend has been sealed against pre-feature integrity-row absence
326    /// (issue #6449). Pass `true` only when the vault-stored `ZEPH_DURABLE_INTEGRITY_SEALED`
327    /// marker's *presence* was confirmed at bootstrap — never derive this from any DB column
328    /// (that was the S1 defeat the vault-sealed design fixes; see `check_high_water_mark`'s
329    /// doc).
330    #[must_use]
331    pub fn with_integrity_sealed(mut self, sealed: bool) -> Self {
332        self.integrity_sealed = sealed;
333        self
334    }
335
336    /// Register the vault-stored set of execution IDs grandfathered past the integrity seal
337    /// (issue #6449). Each grandfathered id is a *permanent* forge-able slot (not merely a
338    /// frozen pre-existing posture): an attacker with DB write access can delete and re-insert
339    /// forged content under the same id. This is an accepted, bounded, documented operator
340    /// opt-out — prefer draining a resumable execution to a terminal status where practical.
341    #[must_use]
342    pub fn with_grandfather(mut self, ids: std::collections::HashSet<ExecutionId>) -> Self {
343        self.integrity_grandfather = ids;
344        self
345    }
346
347    /// Borrow the underlying pool (for tests and adapters that need direct access).
348    #[must_use]
349    pub fn pool(&self) -> &DbPool {
350        &self.pool
351    }
352
353    /// Apply the durable schema migrations to the backing pool.
354    ///
355    /// Idempotent: safe to call repeatedly. The schema is owned by `zeph-db`, not this crate.
356    ///
357    /// # Errors
358    ///
359    /// Returns [`DurableError::Storage`] if a migration fails.
360    pub async fn init(&self) -> Result<(), DurableError> {
361        zeph_db::run_migrations(&self.pool)
362            .await
363            .map_err(|e| DurableError::storage("init", e))?;
364        Ok(())
365    }
366
367    /// List execution summaries for operability surfaces (the `zeph durable` CLI and TUI).
368    ///
369    /// Returns at most `limit` executions, newest first, optionally filtered by `status` and `kind`
370    /// (each is matched against the raw column tag; `None` disables that filter). Only execution-level
371    /// metadata is read — never payload bytes or resolver tokens (INV-5). The per-execution step
372    /// count is the number of journal entries recorded for it.
373    ///
374    /// Span: `durable.backend.list`.
375    ///
376    /// # Errors
377    ///
378    /// Returns [`DurableError::Storage`] if the query fails, or [`DurableError::Decode`] if a stored
379    /// id or status cannot be reconstructed (schema corruption — the `status` column is
380    /// `CHECK`-constrained, so this is a fail-closed guard rather than a routine path).
381    pub async fn list_executions(
382        &self,
383        status: Option<&str>,
384        kind: Option<&str>,
385        limit: i64,
386    ) -> Result<Vec<ExecutionSummary>, DurableError> {
387        let span = tracing::info_span!(
388            "durable.backend.list",
389            status = status.unwrap_or("*"),
390            kind = kind.unwrap_or("*"),
391            count = tracing::field::Empty,
392        );
393        async move {
394            // `COALESCE(?, col)` keeps a single positional bind per filter and lets the column type
395            // drive the bind type, so the same literal works on both SQLite and Postgres without a
396            // cast on the `?` placeholder.
397            let rows: Vec<ExecutionRow> =
398                zeph_db::query_as(sql!(
399                    "SELECT
400                        e.execution_id,
401                        e.kind,
402                        e.status,
403                        e.created_at,
404                        e.updated_at,
405                        e.finalized_at,
406                        (SELECT COUNT(*) FROM durable_journal j WHERE j.execution_id = e.execution_id)
407                     FROM durable_executions e
408                     WHERE e.status = COALESCE(?, e.status)
409                       AND e.kind = COALESCE(?, e.kind)
410                     ORDER BY e.created_at DESC
411                     LIMIT ?"
412                ))
413                .bind(status)
414                .bind(kind)
415                .bind(limit)
416                .fetch_all(&self.pool)
417                .await
418                .map_err(|e| DurableError::storage("list", e))?;
419            tracing::Span::current().record("count", rows.len());
420            rows.into_iter()
421                .map(|(id, kind, status, created, updated, finalized, steps)| {
422                    Ok(ExecutionSummary {
423                        execution_id: parse_execution_id(&id)?,
424                        kind,
425                        status: ExecutionStatus::from_tag(&status).ok_or(DurableError::Decode {
426                            context: "execution status is not a recognized CHECK-constrained value",
427                        })?,
428                        created_at_ms: created,
429                        updated_at_ms: updated,
430                        finalized_at_ms: finalized,
431                        step_count: steps.max(0).cast_unsigned(),
432                    })
433                })
434                .collect()
435        }
436        .instrument(span)
437        .await
438    }
439
440    /// Look up a single execution's current status, without touching journal entries or payloads.
441    ///
442    /// Backs the `zeph durable resume` CLI's canceled-refusal check (FR-011): resume must report a
443    /// `canceled` execution distinctly from "no adapters wired", which requires knowing the status
444    /// before deciding which message to print.
445    ///
446    /// # Errors
447    ///
448    /// Returns [`DurableError::Storage`] if the query fails, or [`DurableError::Decode`] if the
449    /// stored status cannot be reconstructed (schema corruption — the column is `CHECK`-constrained).
450    pub async fn execution_status(
451        &self,
452        id: ExecutionId,
453    ) -> Result<Option<ExecutionStatus>, DurableError> {
454        let row: Option<(String,)> = zeph_db::query_as(sql!(
455            "SELECT status FROM durable_executions WHERE execution_id = ?"
456        ))
457        .bind(id.as_uuid().to_string())
458        .fetch_optional(&self.pool)
459        .await
460        .map_err(|e| DurableError::storage("execution_status", e))?;
461        row.map(|(status,)| {
462            ExecutionStatus::from_tag(&status).ok_or(DurableError::Decode {
463                context: "execution status is not a recognized CHECK-constrained value",
464            })
465        })
466        .transpose()
467    }
468
469    /// Read one execution's journal entries as redaction-safe metadata, without decrypting payloads.
470    ///
471    /// Unlike [`read_execution`](Journal::read_execution), this never touches the cipher, so it works
472    /// against a journal whose AEAD key is unavailable and never exposes plaintext (INV-5). It backs
473    /// the default (redacted) `zeph durable show`/`inspect` output. Entries are returned in append
474    /// order.
475    ///
476    /// Span: `durable.backend.read_redacted`.
477    ///
478    /// # Errors
479    ///
480    /// Returns [`DurableError::Storage`] if the query fails.
481    pub async fn read_execution_redacted(
482        &self,
483        id: ExecutionId,
484    ) -> Result<Vec<RedactedEntry>, DurableError> {
485        let exec = id.as_uuid().to_string();
486        let rows: Vec<RedactedRow> = zeph_db::query_as(sql!(
487            "SELECT seq, step_id, entry_kind, idem_key, effect_class, LENGTH(payload), created_at
488                 FROM durable_journal WHERE execution_id = ? ORDER BY seq"
489        ))
490        .bind(&exec)
491        .fetch_all(&self.pool)
492        .await
493        .map_err(|e| DurableError::storage("read_redacted", e))?;
494        Ok(rows
495            .into_iter()
496            .map(
497                |(seq, step, entry_kind, idem, effect_class, payload_len, created)| RedactedEntry {
498                    seq,
499                    step_id: StepId::new(u32::try_from(step).unwrap_or(0)),
500                    entry_kind,
501                    effect_class,
502                    idem_key_prefix: idem.as_deref().map(idem_key_prefix),
503                    payload_len: payload_len.unwrap_or(0).max(0).cast_unsigned(),
504                    created_at_ms: created,
505                },
506            )
507            .collect())
508    }
509
510    /// Count terminal executions a [`prune`](Journal::prune) sweep would delete under `policy`.
511    ///
512    /// Read-only: backs `zeph durable prune --dry-run`. It applies the same TTL cutoffs as the
513    /// delete path, so the count is exactly what a real sweep would remove now.
514    ///
515    /// # Errors
516    ///
517    /// Returns [`DurableError::Storage`] if the query fails.
518    pub async fn count_prunable(&self, policy: &RetentionPolicy) -> Result<u64, DurableError> {
519        let cutoffs = crate::retention::PruneCutoffs::from_policy(policy, now_unix_millis());
520        let (count,): (i64,) = zeph_db::query_as(sql!(
521            "SELECT COUNT(*) FROM durable_executions
522             WHERE finalized_at IS NOT NULL
523               AND ( (status = 'completed' AND finalized_at <= ?)
524                  OR (status IN ('failed', 'aborted', 'canceled') AND finalized_at <= ?) )"
525        ))
526        .bind(cutoffs.completed_before_ms)
527        .bind(cutoffs.failed_before_ms)
528        .fetch_one(&self.pool)
529        .await
530        .map_err(|e| DurableError::storage("count_prunable", e))?;
531        Ok(count.max(0).cast_unsigned())
532    }
533
534    /// Count crash-orphaned executions a [`sweep_orphans`](Journal::sweep_orphans) sweep would
535    /// abort under `policy` (#6254).
536    ///
537    /// Read-only: backs `zeph durable prune --dry-run`. Mirrors the real sweep's staleness scan
538    /// and INV-15 flock liveness check (acquiring and immediately releasing each candidate's
539    /// `ExecutionLock`, exactly as the real sweep does, so the count reflects genuinely
540    /// unowned rows rather than staleness alone) — but never mutates `status`. Returns `0` when
541    /// the sweep is disabled (`stale_running_after_secs == 0`) or this backend has no `lock_dir`.
542    ///
543    /// # Errors
544    ///
545    /// Returns [`DurableError::Storage`] if the query fails.
546    pub async fn count_orphans(&self, policy: &RetentionPolicy) -> Result<u64, DurableError> {
547        if policy.stale_running_after_secs == 0 {
548            return Ok(0);
549        }
550        let Some(lock_dir) = self.lock_dir.clone() else {
551            return Ok(0);
552        };
553        let cutoff_ms = orphan_cutoff_ms(policy, now_unix_millis());
554        let candidates: Vec<(String,)> = zeph_db::query_as(sql!(
555            "SELECT execution_id FROM durable_executions WHERE status = 'running' AND updated_at <= ?"
556        ))
557        .bind(cutoff_ms)
558        .fetch_all(&self.pool)
559        .await
560        .map_err(|e| DurableError::storage("count_orphans", e))?;
561        let mut count = 0u64;
562        for (exec_str,) in &candidates {
563            let Ok(execution_id) = parse_execution_id(exec_str) else {
564                continue;
565            };
566            if ExecutionLock::acquire(&lock_dir, execution_id).is_ok() {
567                count += 1;
568            }
569        }
570        Ok(count)
571    }
572
573    /// Count sealed-payload rows across `durable_journal` and `durable_promises` whose leading
574    /// on-disk byte — the AEAD key-id selector (`zeph_core::durable::XChaCha20Poly1305Cipher`'s
575    /// `key_id(1) || nonce(24) || ciphertext || tag(16)` layout) — equals `key_id`.
576    ///
577    /// Read-only; backs `zeph durable rotate-key --drop-previous`'s default-on safety scan
578    /// (#6447): a nonzero count means payloads still sealed under the previous key would become
579    /// permanently unreadable (`UnknownKeyId`) if that key were dropped now. Filters `payload IS
580    /// NOT NULL` on both tables — control entries (`EffectIntent`) carry no payload and are
581    /// irrelevant to this scan.
582    ///
583    /// The predicate is dialect-specific because `SQLite`'s `substr` on a `BLOB` returns a 1-byte
584    /// `BLOB` (compared here against a bound single-byte blob) while `PostgreSQL`'s `bytea`
585    /// cannot be compared against an integer at all (`get_byte(payload, 0)` extracts it as an
586    /// `INTEGER` instead).
587    ///
588    /// May over-count in a mixed-mode deployment where some rows were written while
589    /// `encrypt_payload = false` (plaintext, no key-id prefix): a plaintext row's leading byte is
590    /// arbitrary content that can coincidentally equal `key_id`. This is intentionally fail-safe
591    /// — it can only cause an unnecessary refusal (resolved with `--force`), never a missed match
592    /// that would let a genuinely-sealed row be dropped silently.
593    ///
594    /// # Errors
595    ///
596    /// Returns [`DurableError::Storage`] if either query fails.
597    pub async fn count_sealed_under_key_id(&self, key_id: u8) -> Result<u64, DurableError> {
598        #[cfg(feature = "postgres")]
599        {
600            let key_id_param = i32::from(key_id);
601            let (journal_count,): (i64,) = zeph_db::query_as(sql!(
602                "SELECT COUNT(*) FROM durable_journal
603                 WHERE payload IS NOT NULL AND get_byte(payload, 0) = ?"
604            ))
605            .bind(key_id_param)
606            .fetch_one(&self.pool)
607            .await
608            .map_err(|e| DurableError::storage("count_sealed_under_key_id", e))?;
609            let (promises_count,): (i64,) = zeph_db::query_as(sql!(
610                "SELECT COUNT(*) FROM durable_promises
611                 WHERE payload IS NOT NULL AND get_byte(payload, 0) = ?"
612            ))
613            .bind(key_id_param)
614            .fetch_one(&self.pool)
615            .await
616            .map_err(|e| DurableError::storage("count_sealed_under_key_id", e))?;
617            Ok((journal_count.max(0) + promises_count.max(0)).cast_unsigned())
618        }
619        #[cfg(not(feature = "postgres"))]
620        {
621            let key_byte = vec![key_id];
622            let (journal_count,): (i64,) = zeph_db::query_as(sql!(
623                "SELECT COUNT(*) FROM durable_journal
624                 WHERE payload IS NOT NULL AND substr(payload, 1, 1) = ?"
625            ))
626            .bind(key_byte.clone())
627            .fetch_one(&self.pool)
628            .await
629            .map_err(|e| DurableError::storage("count_sealed_under_key_id", e))?;
630            let (promises_count,): (i64,) = zeph_db::query_as(sql!(
631                "SELECT COUNT(*) FROM durable_promises
632                 WHERE payload IS NOT NULL AND substr(payload, 1, 1) = ?"
633            ))
634            .bind(key_byte)
635            .fetch_one(&self.pool)
636            .await
637            .map_err(|e| DurableError::storage("count_sealed_under_key_id", e))?;
638            Ok((journal_count.max(0) + promises_count.max(0)).cast_unsigned())
639        }
640    }
641
642    /// Count `EffectIntent` control entries whose row HMAC (INV-8) verifies **only** under the
643    /// registered [`previous_hmac_key`](Self::with_previous_hmac_key), not the current
644    /// [`hmac_key`](Self::with_hmac_key) (#6451).
645    ///
646    /// The read-side counterpart to [`count_sealed_under_key_id`](Self::count_sealed_under_key_id)
647    /// for the control-entry HMAC's own rotation window, and **not redundant** with it: the AEAD
648    /// blob-scan only sees payload-bearing rows, but a pre-rotation `EffectIntent` whose
649    /// `StepResult` was never committed (a crash between intent and result, in a still-retained
650    /// non-terminal execution) has a previous-key HMAC and no payload at all — the blob-scan
651    /// cannot see it, so dropping the previous key without this scan would silently orphan its
652    /// HMAC verification. Only `EffectIntent` rows carry a persisted+verified HMAC:
653    /// `PromiseCreated`/`TimerArmed`/`TimerFired`/`Checkpoint` all return
654    /// [`DurableError::UnsupportedEntryKind`] in `prepare_row`, and
655    /// `durable_promises` has no `hmac` column.
656    ///
657    /// Backs `zeph durable rotate-key --drop-previous`'s safety scan alongside the AEAD blob-scan
658    /// — refuse the drop while **either** is nonzero. This is a fourth, dedicated key-attach site
659    /// distinct from the three runtime read paths (agent replay, scheduler daemon, CLI read):
660    /// the caller must attach **both** [`with_hmac_key`](Self::with_hmac_key) (current) and
661    /// [`with_previous_hmac_key`](Self::with_previous_hmac_key) (previous) to this backend before
662    /// calling, or every row's HMAC is unrecomputable and this returns
663    /// [`DurableError::ControlIntegrity`] rather than a (silently wrong) count.
664    ///
665    /// Uses the precise variant — recompute-and-compare against both keys — rather than a pure
666    /// "fails under current" fail-safe: a genuinely corrupt/forged row (matches neither key) is
667    /// not counted here, since it is not something dropping the previous key would newly break;
668    /// [`read_execution`](Journal::read_execution) already rejects it on every read regardless of
669    /// which key is dropped.
670    ///
671    /// Cold path (runs only at `--drop-previous`); control rows are sparse.
672    ///
673    /// # Errors
674    ///
675    /// Returns [`DurableError::Storage`] if the query fails, or [`DurableError::ControlIntegrity`]
676    /// if matching control rows exist but this backend is missing the current or previous HMAC
677    /// key needed to recompute them.
678    pub async fn count_control_entries_under_previous_hmac(&self) -> Result<u64, DurableError> {
679        let rows: Vec<ControlHmacScanRow> = zeph_db::query_as(sql!(
680            "SELECT execution_id, step_id, idem_key, hmac
681             FROM durable_journal
682             WHERE entry_kind = 'effect_intent' AND hmac IS NOT NULL"
683        ))
684        .fetch_all(&self.pool)
685        .await
686        .map_err(|e| DurableError::storage("count_control_entries_under_previous_hmac", e))?;
687
688        if rows.is_empty() {
689            return Ok(0);
690        }
691
692        let (Some(current_key), Some(previous_key)) =
693            (self.hmac_key.as_ref(), self.previous_hmac_key.as_ref())
694        else {
695            return Err(DurableError::ControlIntegrity);
696        };
697
698        let mut count = 0u64;
699        for (execution_id_raw, step_id_raw, idem_key_raw, hmac_raw) in rows {
700            let Ok(execution_id) = parse_execution_id(&execution_id_raw) else {
701                continue;
702            };
703            let Ok(step_id_value) = u32::try_from(step_id_raw) else {
704                continue;
705            };
706            let step_id = StepId::new(step_id_value);
707            let idem_key = idem_key_raw
708                .as_deref()
709                .and_then(|b| slice_to_array32(b, "effect_intent idem_key").ok())
710                .map(IdempotencyKey::from_bytes);
711            let Ok(stored) = slice_to_array32(&hmac_raw, "effect_intent hmac") else {
712                continue;
713            };
714
715            let tag = EntryKindTag::EffectIntent.as_str();
716            let expected_current = Self::keyed_control_hmac(
717                current_key,
718                execution_id,
719                step_id,
720                tag,
721                idem_key.as_ref(),
722            );
723            if blake3::Hash::from(expected_current) == blake3::Hash::from(stored) {
724                continue;
725            }
726            let expected_previous = Self::keyed_control_hmac(
727                previous_key,
728                execution_id,
729                step_id,
730                tag,
731                idem_key.as_ref(),
732            );
733            if blake3::Hash::from(expected_previous) == blake3::Hash::from(stored) {
734                count += 1;
735            }
736        }
737        Ok(count)
738    }
739
740    /// Count `durable_execution_integrity` rows whose high-water-mark was signed under `epoch`.
741    ///
742    /// The `--drop-previous` HWM scan (addendum to #6451, spec-081 FR-008): before permanently
743    /// removing the previous rotation key, refuse if any surviving execution's HWM row is still
744    /// addressed to the previous epoch. Unlike
745    /// [`count_control_entries_under_previous_hmac`](Self::count_control_entries_under_previous_hmac),
746    /// the HWM row carries `key_epoch` in the clear, so this is a plain indexed `COUNT` — no key
747    /// material, no per-row recompute. This is also the only one of the three `--drop-previous`
748    /// scans that catches a checkpoint-folded pre-rotation execution: `checkpoint_fold` never
749    /// re-signs the HWM, so a folded execution's integrity row keeps
750    /// `key_epoch = previous_key_id` even though its old-key-id payloads are gone — invisible to
751    /// both the AEAD blob-scan
752    /// ([`count_sealed_under_key_id`](Self::count_sealed_under_key_id)) and the control-HMAC scan
753    /// (`EffectIntent`-only). Terminal-but-unpruned executions are counted too (the row is deleted
754    /// only by the retention prune sweep, never on `finalize`) — fail-safe over-refusal, resolvable
755    /// with `--force`, mirroring the other two scans' coarseness.
756    ///
757    /// Cold path (runs only at `--drop-previous`); integrity rows are sparse (one per execution).
758    ///
759    /// # Errors
760    ///
761    /// Returns [`DurableError::Storage`] if the query fails.
762    pub async fn count_integrity_rows_under_epoch(&self, epoch: u32) -> Result<u64, DurableError> {
763        let count: i64 = zeph_db::query_scalar(sql!(
764            "SELECT COUNT(*) FROM durable_execution_integrity WHERE key_epoch = ?"
765        ))
766        .bind(i64::from(epoch))
767        .fetch_one(&self.pool)
768        .await
769        .map_err(|e| DurableError::storage("count_integrity_rows_under_epoch", e))?;
770        Ok(count.max(0).cast_unsigned())
771    }
772
773    /// Ensure a `durable_executions` row exists for `id`, returning whether this is a resume.
774    ///
775    /// Inserts a fresh `running` row for a new execution (returning `false`) or detects an existing
776    /// row for a resumed one (returning `true`). The journal's foreign key requires this row before
777    /// any entry is appended, so callers open the execution first.
778    ///
779    /// Reopening a row previously [`finalize`](Journal::finalize)d as `completed`, `failed`, or
780    /// `aborted` un-finalizes it: status resets to `running` and `finalized_at` clears (INV-16′,
781    /// #6254). A `canceled` row is the deliberate exception — see the `canceled` branch below
782    /// (INV-16′, #6362). A caller reopening an execution is, by definition, still using it, so the
783    /// retention sweep (gated on `finalized_at`) must not consider it prunable while it does — without this,
784    /// a long-lived execution finalized at one process's graceful shutdown and legitimately resumed
785    /// by a later process (e.g. a per-conversation `AgentTurn` execution) would keep a stale
786    /// `finalized_at` and could be pruned out from under its still-active journal. `aborted` rows
787    /// are included because the crash-orphan sweep (INV-17) makes `aborted` the common outcome of a
788    /// resumable crash: a resumed execution whose row keeps `finalized_at` set is prunable out from
789    /// under the active resume — the exact hazard this un-finalize prevents for `completed`/`failed`.
790    /// This is also strictly safer for the pre-existing divergence-recovery case, which reopens an
791    /// `aborted` row on purpose: it now also protects that fresh re-drive from prune.
792    ///
793    /// The un-finalize is attempted as a single guarded `UPDATE` (no preceding `SELECT`) so there
794    /// is no read-then-write window against a concurrent prune sweep (#6251 critic S1): if the row
795    /// was deleted by `prune` between an earlier observation and this call, the `UPDATE` simply
796    /// matches zero rows rather than silently resurrecting a half-deleted row. A zero-row `UPDATE`
797    /// falls back to checking whether the row exists at all (already `running`/`aborted`, or
798    /// genuinely gone) before deciding between reporting a resume or inserting a fresh execution —
799    /// so this never reports `is_resume = true` for a row that turned out not to exist.
800    ///
801    /// Every path that resolves to `is_resume = true` verifies the signed high-water-mark
802    /// (issue #6360) before returning: this is the single production call site every durable resume goes through (P1 agent-turn,
803    /// P2 orchestration, scheduler, sub-agent), so it is also the one place the HWM check needs to
804    /// live to cover unattended crash-resume (FR-004, US-003) uniformly.
805    ///
806    /// Span: `durable.backend.open`.
807    ///
808    /// # Errors
809    ///
810    /// Returns [`DurableError::Storage`] if the lookup, reset, or insert fails,
811    /// [`DurableError::HighWaterMarkIntegrity`] if a resumed execution's signed high-water-mark
812    /// does not verify — this is a hard abort with no override (FR-004) — or
813    /// [`DurableError::ExecutionCanceled`] if the row is `canceled` (INV-16′, #6362): checked
814    /// before the HWM verification, since a canceled execution must never be resumed regardless
815    /// of whether its journal is otherwise intact.
816    pub async fn open_execution(
817        &self,
818        id: ExecutionId,
819        kind: ExecutionKind,
820    ) -> Result<bool, DurableError> {
821        let span = tracing::info_span!(
822            "durable.backend.open",
823            execution_id = %id.as_uuid(),
824            kind = kind.as_str(),
825            is_resume = tracing::field::Empty,
826        );
827        async move {
828            let exec = id.as_uuid().to_string();
829
830            // Attempt the un-finalize directly, with no preceding SELECT: this is the only write
831            // this call needs to make for an existing terminal row, so there is no window between
832            // "observe completed/failed" and "reset to running" for a concurrent prune to act in.
833            let reopened = zeph_db::query(sql!(
834                "UPDATE durable_executions SET status = 'running', updated_at = ?, finalized_at = NULL
835                 WHERE execution_id = ? AND status IN ('completed', 'failed', 'aborted')"
836            ))
837            .bind(now_unix_millis())
838            .bind(&exec)
839            .execute(&self.pool)
840            .await
841            .map_err(|e| DurableError::storage("open", e))?;
842            if reopened.rows_affected() > 0 {
843                self.verify_high_water_mark(id).await?;
844                tracing::Span::current().record("is_resume", true);
845                return Ok(true);
846            }
847
848            // Zero rows: either the row doesn't exist, or it exists but wasn't terminal (already
849            // `running`, no reset needed — every terminal status is covered by the UPDATE above),
850            // or it is `canceled` — deliberately excluded from the UPDATE's IN-list (INV-16′).
851            // Distinguish the cases — if a concurrent prune deleted a terminal row between any
852            // earlier observation and this check, this SELECT sees the authoritative post-delete
853            // state instead of a stale belief that it's there.
854            let existing: Option<(String,)> = zeph_db::query_as(sql!(
855                "SELECT status FROM durable_executions WHERE execution_id = ?"
856            ))
857            .bind(&exec)
858            .fetch_optional(&self.pool)
859            .await
860            .map_err(|e| DurableError::storage("open", e))?;
861            if let Some((status,)) = existing {
862                if status == "canceled" {
863                    return Err(DurableError::ExecutionCanceled { execution_id: id });
864                }
865                self.verify_high_water_mark(id).await?;
866                tracing::Span::current().record("is_resume", true);
867                return Ok(true);
868            }
869            let now = now_unix_millis();
870            zeph_db::query(sql!(
871                "INSERT INTO durable_executions
872                    (execution_id, kind, status, created_at, updated_at, finalized_at)
873                 VALUES (?, ?, 'running', ?, ?, NULL)"
874            ))
875            .bind(&exec)
876            .bind(kind.as_str())
877            .bind(now)
878            .bind(now)
879            .execute(&self.pool)
880            .await
881            .map_err(|e| DurableError::storage("open", e))?;
882            tracing::Span::current().record("is_resume", false);
883            Ok(false)
884        }
885        .instrument(span)
886        .await
887    }
888
889    /// Like [`open_execution`](Self::open_execution), but additionally takes a non-blocking,
890    /// exclusive, process-scoped advisory lock on `id` before touching the row (INV-15, #6122).
891    ///
892    /// Closes the race two processes deriving the same `ExecutionId` (e.g. two CLI instances
893    /// pointed at the same `memory.sqlite_path` and the same `ConversationId`) would otherwise hit
894    /// in [`open_execution`](Self::open_execution)'s unsynchronized SELECT-then-INSERT: both could
895    /// observe "no existing row", both insert, and both then drive `next_step` from 0 against the
896    /// same journal, corrupting it. The lock is acquired first, so the loser never reaches the
897    /// row check at all.
898    ///
899    /// Returns `(is_resume, lock)`. The caller MUST hold `lock` for as long as it drives the
900    /// execution — dropping it releases the lock and allows another process to open the same
901    /// `id`. `lock` is `None` when this backend has no on-disk lock directory (a `:memory:`
902    /// database, a backend built via [`LocalBackend::new`], or a Postgres deployment), in which
903    /// case process exclusivity is not enforced — the caller degrades the same way it already does
904    /// for `open_execution`'s other failure modes.
905    ///
906    /// # Errors
907    ///
908    /// Returns [`DurableError::ExecutionLocked`] if another process already holds `id`'s lock, or
909    /// any error [`open_execution`](Self::open_execution) can return.
910    pub async fn open_execution_exclusive(
911        &self,
912        id: ExecutionId,
913        kind: ExecutionKind,
914    ) -> Result<(bool, Option<ExecutionLock>), DurableError> {
915        let lock = self
916            .lock_dir
917            .as_deref()
918            .map(|dir| ExecutionLock::acquire(dir, id))
919            .transpose()?;
920        let is_resume = self.open_execution(id, kind).await?;
921        Ok((is_resume, lock))
922    }
923
924    /// Cancel a `running` execution so it is deliberately, permanently stopped and never resumed
925    /// (#6362, FR-003/006/012/014).
926    ///
927    /// Unlike [`finalize`](Journal::finalize), which blindly flips `status` under the caller's
928    /// authority, this is the operator-facing entry point: it first tries to establish that no
929    /// live process still owns the execution, so a cancel never races a genuinely active owner's
930    /// own `finalize` into an inconsistent state.
931    ///
932    /// **Liveness probe (SQLite/Unix only).** When this backend has an on-disk `lock_dir`
933    /// (opened via [`LocalBackend::open`] against a real file), a non-blocking acquire of `id`'s
934    /// [`ExecutionLock`] distinguishes a live owner from a dead one:
935    /// - Lock held by another process → [`CancelOutcome::LiveOwner`], row untouched.
936    /// - Lock free → held across the write below (a restart cannot race in mid-window), then
937    ///   released.
938    ///
939    /// **No `lock_dir` (`:memory:` or a backend built via [`LocalBackend::new`]).** The safety
940    /// argument here rests on [`ExecutionBackend::capabilities`]'s `cross_process` flag, which
941    /// this crate only ever sets from `cfg!(feature = "postgres")` — i.e. it assumes "no
942    /// `lock_dir` on a `SQLite` build" implies "no other process can hold this row", true for
943    /// `:memory:` but **not** for a file-backed pool handed to [`LocalBackend::new`] directly
944    /// (which never derives a `lock_dir`); that programmatic path is not reachable from the CLI
945    /// (which always uses [`LocalBackend::open`]), but a future caller of `::new` on a shared file
946    /// should not assume the immediate-cancel path is probe-safe there.
947    /// - `cross_process == false` → provably single-process; proceed directly to the write.
948    /// - `cross_process == true` (Postgres) → a live owner cannot be ruled out and there is no
949    ///   flock to probe → [`CancelOutcome::LivenessUnverifiable`], row untouched (F3).
950    ///
951    /// **Write.** A conditional `UPDATE … WHERE status = 'running'` (the same single-writer-wins
952    /// pattern as `finalize`) — no read-then-write window (NFR-001). Zero rows affected then
953    /// disambiguates via a follow-up `SELECT` into [`CancelOutcome::NotFound`] or
954    /// [`CancelOutcome::AlreadyTerminal`] (idempotent for an already-`canceled` row, NFR-003).
955    ///
956    /// Span: `durable.backend.cancel`.
957    ///
958    /// # Errors
959    ///
960    /// Returns [`DurableError::Storage`] if a query fails, or propagates any
961    /// [`DurableError`] other than [`DurableError::ExecutionLocked`] from the lock acquisition
962    /// (`ExecutionLocked` itself is caught and converted into [`CancelOutcome::LiveOwner`], never
963    /// surfaced as an `Err`).
964    pub async fn cancel_execution(&self, id: ExecutionId) -> Result<CancelOutcome, DurableError> {
965        let span = tracing::info_span!(
966            "durable.backend.cancel",
967            execution_id = %id.as_uuid(),
968            prior_status = tracing::field::Empty,
969            path = tracing::field::Empty,
970        );
971        async move {
972            let exec = id.as_uuid().to_string();
973
974            if let Some(lock_dir) = self.lock_dir.clone() {
975                let _lock = match ExecutionLock::acquire(&lock_dir, id) {
976                    Ok(lock) => lock,
977                    Err(DurableError::ExecutionLocked { holder_pid, .. }) => {
978                        tracing::Span::current().record("path", "live_owner_refused");
979                        return Ok(CancelOutcome::LiveOwner { pid: holder_pid });
980                    }
981                    Err(e) => return Err(e),
982                };
983                let outcome = self.cancel_write(&exec).await?;
984                tracing::Span::current().record("path", "immediate");
985                record_prior_status(outcome);
986                return Ok(outcome);
987                // `_lock` drops here, after the write commits.
988            }
989
990            if self.capabilities().cross_process {
991                tracing::Span::current().record("path", "unverifiable");
992                return Ok(CancelOutcome::LivenessUnverifiable);
993            }
994
995            tracing::Span::current().record("path", "no_lock_dir_single_process");
996            let outcome = self.cancel_write(&exec).await?;
997            record_prior_status(outcome);
998            Ok(outcome)
999        }
1000        .instrument(span)
1001        .await
1002    }
1003
1004    /// The conditional terminal write behind [`cancel_execution`](Self::cancel_execution),
1005    /// factored out so both the SQLite/Unix (lock-held) and single-process (no-`lock_dir`) paths
1006    /// share one implementation of the race-safe `UPDATE … WHERE status = 'running'` pattern.
1007    async fn cancel_write(&self, exec: &str) -> Result<CancelOutcome, DurableError> {
1008        let now = now_unix_millis();
1009        let mut tx = zeph_db::begin_write(&self.pool)
1010            .await
1011            .map_err(|e| DurableError::storage("cancel", e))?;
1012        let result = zeph_db::query(sql!(
1013            "UPDATE durable_executions SET status = 'canceled', finalized_at = ?, updated_at = ?
1014             WHERE execution_id = ? AND status = 'running'"
1015        ))
1016        .bind(now)
1017        .bind(now)
1018        .bind(exec)
1019        .execute(&mut *tx)
1020        .await
1021        .map_err(|e| DurableError::storage("cancel", e))?;
1022        if result.rows_affected() > 0 {
1023            tx.commit()
1024                .await
1025                .map_err(|e| DurableError::storage("cancel", e))?;
1026            return Ok(CancelOutcome::Canceled);
1027        }
1028
1029        // Zero rows: either no such execution, or it exists but was not `running`. Read the
1030        // current status inside the same transaction so this reflects exactly what the UPDATE
1031        // above saw — no window for a concurrent writer to change the answer in between.
1032        let existing: Option<(String,)> = zeph_db::query_as(sql!(
1033            "SELECT status FROM durable_executions WHERE execution_id = ?"
1034        ))
1035        .bind(exec)
1036        .fetch_optional(&mut *tx)
1037        .await
1038        .map_err(|e| DurableError::storage("cancel", e))?;
1039        tx.commit()
1040            .await
1041            .map_err(|e| DurableError::storage("cancel", e))?;
1042        match existing {
1043            None => Ok(CancelOutcome::NotFound),
1044            Some((status,)) => {
1045                let status = ExecutionStatus::from_tag(&status).ok_or(DurableError::Decode {
1046                    context: "unrecognized durable_executions.status value",
1047                })?;
1048                Ok(CancelOutcome::AlreadyTerminal { status })
1049            }
1050        }
1051    }
1052
1053    /// Group-commit a batch of buffered entries in a single write transaction.
1054    ///
1055    /// Used by the [`JournalWriter`](crate::JournalWriter) to amortize the WAL fsync across all
1056    /// entries accumulated within a flush interval. Sealing and HMAC computation run before the
1057    /// transaction opens, keeping CPU work off the write lock. The whole batch commits atomically;
1058    /// a single malformed entry aborts the batch.
1059    ///
1060    /// # Errors
1061    ///
1062    /// Returns [`DurableError::Storage`] on a database failure, or a per-entry error
1063    /// ([`DurableError::PayloadTooLarge`], [`DurableError::UnsupportedEntryKind`], or a cipher
1064    /// failure) if an entry cannot be prepared.
1065    pub(crate) async fn append_batch(&self, entries: &[JournalEntry]) -> Result<(), DurableError> {
1066        if entries.is_empty() {
1067            return Ok(());
1068        }
1069        let mut rows = Vec::with_capacity(entries.len());
1070        for entry in entries {
1071            rows.push(self.prepare_row(entry)?);
1072        }
1073        // `sql!()` caches its postgres rewrite per call site (see #5431), so hoisting
1074        // this out of the loop below is no longer required to avoid a leak — kept
1075        // anyway since it reads the intent clearly and costs nothing.
1076        let insert = sql!(
1077            "INSERT INTO durable_journal
1078                (execution_id, step_id, entry_kind, idem_key, effect_class, payload, payload_version, hmac, created_at)
1079             VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)"
1080        );
1081        let mut tx = zeph_db::begin_write(&self.pool)
1082            .await
1083            .map_err(|e| DurableError::storage("append_batch", e))?;
1084        for (entry, row) in entries.iter().zip(rows) {
1085            zeph_db::query(insert)
1086                .bind(row.execution_id)
1087                .bind(row.step_id)
1088                .bind(row.entry_kind)
1089                .bind(row.idem_key)
1090                .bind(row.effect_class)
1091                .bind(row.payload)
1092                .bind(row.payload_version)
1093                .bind(row.hmac)
1094                .bind(row.created_at)
1095                .execute(&mut *tx)
1096                .await
1097                .map_err(|e| DurableError::storage("append_batch", e))?;
1098            if matches!(entry.entry, EntryKind::StepResult { .. }) {
1099                self.bump_hwm_for_step_result(&mut tx, entry.execution_id, entry.step_id)
1100                    .await?;
1101            }
1102        }
1103        tx.commit()
1104            .await
1105            .map_err(|e| DurableError::storage("append_batch", e))?;
1106        Ok(())
1107    }
1108
1109    /// Look up a committed `StepResult` anywhere in an execution by its [`IdempotencyKey`].
1110    ///
1111    /// Backs INV-13: a guarded effect that already committed its result must not re-fire after a
1112    /// replay divergence restarts the execution fresh. Returns the (opened) `StepResult` entry when
1113    /// one exists, or `None`. The `idx_durable_journal_idem_key` partial index makes this an
1114    /// `O(log n)` point lookup rather than a scan.
1115    ///
1116    /// Span: `durable.journal.lookup_idem`.
1117    ///
1118    /// # Errors
1119    ///
1120    /// Returns [`DurableError::Storage`] if the query fails, or [`DurableError::Decode`] if the
1121    /// located row cannot be reconstructed.
1122    pub(crate) async fn lookup_committed_result(
1123        &self,
1124        id: ExecutionId,
1125        idem_key: IdempotencyKey,
1126    ) -> Result<Option<JournalEntry>, DurableError> {
1127        let span = tracing::info_span!(
1128            "durable.journal.lookup_idem",
1129            execution_id = %id.as_uuid(),
1130            found = tracing::field::Empty,
1131        );
1132        async move {
1133            let rows: Vec<JournalRowRead> = zeph_db::query_as(sql!(
1134                "SELECT seq, step_id, entry_kind, idem_key, effect_class, payload, payload_version, hmac, created_at
1135                 FROM durable_journal
1136                 WHERE execution_id = ? AND idem_key = ? AND entry_kind = 'step_result'
1137                 ORDER BY seq LIMIT 1"
1138            ))
1139            .bind(id.as_uuid().to_string())
1140            .bind(idem_key.as_bytes().to_vec())
1141            .fetch_all(&self.pool)
1142            .await
1143            .map_err(|e| DurableError::storage("lookup_idem", e))?;
1144            let entry = self.rows_to_entries(id, rows).await?.into_iter().next();
1145            tracing::Span::current().record("found", entry.is_some());
1146            Ok(entry)
1147        }
1148        .instrument(span)
1149        .await
1150    }
1151
1152    /// Read the highest committed [`JournalSeq`], or `None` for an empty journal.
1153    ///
1154    /// The [`JournalWriter`](crate::JournalWriter) calls this on (re)start to anchor itself at the
1155    /// last durably-committed entry (FR-DE-12). Because `seq` is a database-assigned autoincrement,
1156    /// resumed appends continue from `MAX(seq) + 1` with neither gap nor duplication.
1157    ///
1158    /// # Errors
1159    ///
1160    /// Returns [`DurableError::Storage`] if the query fails.
1161    pub(crate) async fn max_seq(&self) -> Result<Option<JournalSeq>, DurableError> {
1162        let max: Option<i64> = zeph_db::query_scalar(sql!("SELECT MAX(seq) FROM durable_journal"))
1163            .fetch_one(&self.pool)
1164            .await
1165            .map_err(|e| DurableError::storage("max_seq", e))?;
1166        Ok(max.map(JournalSeq::new))
1167    }
1168
1169    /// The in-process wakeup registry for parked promise awaits, shared with the resolver path.
1170    pub(crate) fn promise_waiters(&self) -> &NotifyRegistry {
1171        &self.promise_waiters
1172    }
1173
1174    /// The in-process wakeup registry for parked timers, shared with the timer service.
1175    pub(crate) fn timer_waiters(&self) -> &NotifyRegistry {
1176        &self.timer_waiters
1177    }
1178
1179    /// Insert a freshly-created promise row (INV-9: only the resolver-token hash is stored).
1180    ///
1181    /// Called by `promise()` for a brand-new promise; a resumed execution detects the existing row
1182    /// via [`promise_state`](Self::promise_state) and never re-inserts. Span: `durable.promise.create`.
1183    ///
1184    /// # Errors
1185    ///
1186    /// Returns [`DurableError::Storage`] if the insert fails.
1187    pub(crate) async fn insert_promise(
1188        &self,
1189        id: PromiseId,
1190        execution_id: ExecutionId,
1191        resolver_token_hash: [u8; 32],
1192        created_at_ms: i64,
1193    ) -> Result<(), DurableError> {
1194        let span = tracing::info_span!("durable.promise.create", promise_id = %id.as_uuid());
1195        async move {
1196            zeph_db::query(sql!(
1197                "INSERT INTO durable_promises
1198                    (promise_id, execution_id, resolver_token_hash, resolved, payload, created_at, resolved_at)
1199                 VALUES (?, ?, ?, 0, NULL, ?, NULL)"
1200            ))
1201            .bind(id.as_uuid().to_string())
1202            .bind(execution_id.as_uuid().to_string())
1203            .bind(resolver_token_hash.to_vec())
1204            .bind(created_at_ms)
1205            .execute(&self.pool)
1206            .await
1207            .map_err(|e| DurableError::storage("insert_promise", e))?;
1208            Ok(())
1209        }
1210        .instrument(span)
1211        .await
1212    }
1213
1214    /// Read a promise's persisted state, or `None` if it does not exist.
1215    ///
1216    /// # Errors
1217    ///
1218    /// Returns [`DurableError::Storage`] if the query fails, or [`DurableError::Decode`] if a stored
1219    /// field cannot be reconstructed.
1220    pub(crate) async fn promise_state(
1221        &self,
1222        id: PromiseId,
1223    ) -> Result<Option<PromiseRecord>, DurableError> {
1224        let row: Option<PromiseRowRead> = zeph_db::query_as(sql!(
1225            "SELECT execution_id, resolver_token_hash, resolved, payload
1226             FROM durable_promises WHERE promise_id = ?"
1227        ))
1228        .bind(id.as_uuid().to_string())
1229        .fetch_optional(&self.pool)
1230        .await
1231        .map_err(|e| DurableError::storage("promise_state", e))?;
1232        let Some((exec, hash, resolved, payload)) = row else {
1233            return Ok(None);
1234        };
1235        Ok(Some(PromiseRecord {
1236            execution_id: parse_execution_id(&exec)?,
1237            resolver_token_hash: slice_to_array32(&hash, "promise resolver_token_hash")?,
1238            resolved: resolved != 0,
1239            payload,
1240        }))
1241    }
1242
1243    /// Commit a resolved value to a pending promise, returning whether it transitioned.
1244    ///
1245    /// The conditional `WHERE resolved = 0` makes a double-resolve a no-op (returns `false`); the
1246    /// caller has already authenticated the resolver token. On a real transition any in-process
1247    /// waiter is woken. Span: `durable.promise.resolve`.
1248    ///
1249    /// # Errors
1250    ///
1251    /// Returns [`DurableError::PayloadTooLarge`] if the value exceeds the limit, a cipher failure if
1252    /// sealing fails, or [`DurableError::Storage`] on a database error.
1253    pub(crate) async fn resolve_promise(
1254        &self,
1255        id: PromiseId,
1256        execution_id: ExecutionId,
1257        value_plaintext: &[u8],
1258        resolved_at_ms: i64,
1259    ) -> Result<bool, DurableError> {
1260        let span = tracing::info_span!("durable.promise.resolve", promise_id = %id.as_uuid());
1261        async move {
1262            ensure_payload_within_limit(value_plaintext.len(), self.max_payload_bytes)?;
1263            let aad = promise_payload_aad(execution_id, id);
1264            let sealed = self.seal_payload(value_plaintext, &aad)?;
1265            let affected = zeph_db::query(sql!(
1266                "UPDATE durable_promises SET resolved = 1, payload = ?, resolved_at = ?
1267                 WHERE promise_id = ? AND resolved = 0"
1268            ))
1269            .bind(sealed)
1270            .bind(resolved_at_ms)
1271            .bind(id.as_uuid().to_string())
1272            .execute(&self.pool)
1273            .await
1274            .map_err(|e| DurableError::storage("resolve_promise", e))?
1275            .rows_affected();
1276            if affected > 0 {
1277                self.promise_waiters.wake(id.as_uuid());
1278            }
1279            Ok(affected > 0)
1280        }
1281        .instrument(span)
1282        .await
1283    }
1284
1285    /// Claim the one-time replay notification for a promise, returning whether this call won.
1286    ///
1287    /// The conditional `WHERE notified_at IS NULL` makes the claim single-winner: the first caller
1288    /// transitions the row (returns `true`); every later caller is a no-op (returns `false`). This
1289    /// backs #6027 — a resumed foreground sub-agent's replay notice / TUI completion event must fire
1290    /// at most once across repeated parent restarts. Unlike [`resolve_promise`](Self::resolve_promise)
1291    /// it touches only the `notified_at` bookkeeping column and carries no payload, so no sealing /
1292    /// waiter wakeup is involved. Span: `durable.promise.claim_notify`.
1293    ///
1294    /// # Errors
1295    ///
1296    /// Returns [`DurableError::Storage`] on a database error.
1297    pub(crate) async fn claim_promise_notification(
1298        &self,
1299        id: PromiseId,
1300        notified_at_ms: i64,
1301    ) -> Result<bool, DurableError> {
1302        let span = tracing::info_span!("durable.promise.claim_notify", promise_id = %id.as_uuid());
1303        async move {
1304            let affected = zeph_db::query(sql!(
1305                "UPDATE durable_promises SET notified_at = ?
1306                 WHERE promise_id = ? AND notified_at IS NULL"
1307            ))
1308            .bind(notified_at_ms)
1309            .bind(id.as_uuid().to_string())
1310            .execute(&self.pool)
1311            .await
1312            .map_err(|e| DurableError::storage("claim_promise_notification", e))?
1313            .rows_affected();
1314            Ok(affected > 0)
1315        }
1316        .instrument(span)
1317        .await
1318    }
1319
1320    /// Open a promise's sealed resolved payload back to plaintext.
1321    ///
1322    /// # Errors
1323    ///
1324    /// Returns [`DurableError::ReplayIntegrity`] if the sealed blob does not authenticate, or
1325    /// [`DurableError::PayloadTooLarge`] if it exceeds the read-side limit.
1326    pub(crate) fn open_promise_payload(
1327        &self,
1328        id: PromiseId,
1329        execution_id: ExecutionId,
1330        sealed: &[u8],
1331    ) -> Result<Bytes, DurableError> {
1332        ensure_payload_within_limit(
1333            sealed.len(),
1334            self.max_payload_bytes.saturating_add(SEAL_OVERHEAD_SLACK),
1335        )?;
1336        let aad = promise_payload_aad(execution_id, id);
1337        self.open_payload(sealed, &aad)
1338    }
1339
1340    /// Arm a durable timer to fire at `due_at_ms` (a `durable_timers` row).
1341    ///
1342    /// Span: `durable.timer.arm`.
1343    ///
1344    /// # Errors
1345    ///
1346    /// Returns [`DurableError::Storage`] if the insert fails.
1347    pub(crate) async fn arm_timer(
1348        &self,
1349        id: TimerId,
1350        execution_id: ExecutionId,
1351        due_at_ms: i64,
1352        created_at_ms: i64,
1353    ) -> Result<(), DurableError> {
1354        let span = tracing::info_span!("durable.timer.arm", timer_id = %id.as_uuid(), due_at_ms);
1355        async move {
1356            zeph_db::query(sql!(
1357                "INSERT INTO durable_timers (timer_id, execution_id, due_at, fired, created_at)
1358                 VALUES (?, ?, ?, 0, ?)"
1359            ))
1360            .bind(id.as_uuid().to_string())
1361            .bind(execution_id.as_uuid().to_string())
1362            .bind(due_at_ms)
1363            .bind(created_at_ms)
1364            .execute(&self.pool)
1365            .await
1366            .map_err(|e| DurableError::storage("arm_timer", e))?;
1367            Ok(())
1368        }
1369        .instrument(span)
1370        .await
1371    }
1372
1373    /// Read a timer's `(due_at_ms, fired)` state, or `None` if it does not exist.
1374    ///
1375    /// # Errors
1376    ///
1377    /// Returns [`DurableError::Storage`] if the query fails.
1378    pub(crate) async fn timer_state(
1379        &self,
1380        id: TimerId,
1381    ) -> Result<Option<(i64, bool)>, DurableError> {
1382        let row: Option<(i64, i64)> = zeph_db::query_as(sql!(
1383            "SELECT due_at, fired FROM durable_timers WHERE timer_id = ?"
1384        ))
1385        .bind(id.as_uuid().to_string())
1386        .fetch_optional(&self.pool)
1387        .await
1388        .map_err(|e| DurableError::storage("timer_state", e))?;
1389        Ok(row.map(|(due_at, fired)| (due_at, fired != 0)))
1390    }
1391
1392    /// List every unfired timer whose instant is at or before `now_ms`.
1393    ///
1394    /// The `idx_durable_timers_due(fired, due_at)` index makes this a range scan over due, unfired
1395    /// timers rather than a full-table scan.
1396    ///
1397    /// # Errors
1398    ///
1399    /// Returns [`DurableError::Storage`] if the query fails, or [`DurableError::Decode`] on a
1400    /// malformed id.
1401    pub(crate) async fn due_timers(&self, now_ms: i64) -> Result<Vec<TimerId>, DurableError> {
1402        let rows: Vec<(String,)> = zeph_db::query_as(sql!(
1403            "SELECT timer_id FROM durable_timers WHERE fired = 0 AND due_at <= ? ORDER BY due_at"
1404        ))
1405        .bind(now_ms)
1406        .fetch_all(&self.pool)
1407        .await
1408        .map_err(|e| DurableError::storage("due_timers", e))?;
1409        rows.into_iter().map(|(id,)| parse_timer_id(&id)).collect()
1410    }
1411
1412    /// Mark a timer fired, returning whether it transitioned, and wake its parked waiter.
1413    ///
1414    /// Span: `durable.timer.fire`.
1415    ///
1416    /// # Errors
1417    ///
1418    /// Returns [`DurableError::Storage`] if the update fails.
1419    pub(crate) async fn mark_timer_fired(&self, id: TimerId) -> Result<bool, DurableError> {
1420        let span = tracing::info_span!("durable.timer.fire", timer_id = %id.as_uuid());
1421        async move {
1422            let affected = zeph_db::query(sql!(
1423                "UPDATE durable_timers SET fired = 1 WHERE timer_id = ? AND fired = 0"
1424            ))
1425            .bind(id.as_uuid().to_string())
1426            .execute(&self.pool)
1427            .await
1428            .map_err(|e| DurableError::storage("mark_timer_fired", e))?
1429            .rows_affected();
1430            if affected > 0 {
1431                self.timer_waiters.wake(id.as_uuid());
1432            }
1433            Ok(affected > 0)
1434        }
1435        .instrument(span)
1436        .await
1437    }
1438
1439    /// Open each foldable step result's sealed payload into a [`FoldedStep`], in step order.
1440    ///
1441    /// The per-step AAD is reconstructed from the row so the opened plaintext authenticates exactly
1442    /// as it did at rest; the idempotency key is preserved so the replayed-from-snapshot step still
1443    /// satisfies the divergence guard.
1444    fn open_foldable_steps(
1445        &self,
1446        execution_id: ExecutionId,
1447        rows: Vec<FoldableRowRead>,
1448    ) -> Result<Vec<FoldedStep>, DurableError> {
1449        let mut folded = Vec::with_capacity(rows.len());
1450        for (step_raw, idem, version, payload) in rows {
1451            let step = u32::try_from(step_raw).map_err(|_| DurableError::Decode {
1452                context: "checkpoint step_id out of u32 range",
1453            })?;
1454            let idem_bytes = idem.ok_or(DurableError::Decode {
1455                context: "checkpoint step result missing idem_key",
1456            })?;
1457            let idem_key =
1458                IdempotencyKey::from_bytes(slice_to_array32(&idem_bytes, "checkpoint idem_key")?);
1459            let sealed = payload.ok_or(DurableError::Decode {
1460                context: "checkpoint step result missing payload",
1461            })?;
1462            let aad = PayloadAad::new(
1463                execution_id,
1464                StepId::new(step),
1465                EntryKindTag::StepResult,
1466                Some(idem_key),
1467            );
1468            let plaintext = self.open_payload(&sealed, &aad)?;
1469            let payload_version =
1470                u8::try_from(version.unwrap_or(1)).map_err(|_| DurableError::Decode {
1471                    context: "checkpoint payload_version out of u8 range",
1472                })?;
1473            folded.push(FoldedStep {
1474                step_id: step,
1475                idem_key: *idem_key.as_bytes(),
1476                payload_version,
1477                payload: plaintext,
1478            });
1479        }
1480        Ok(folded)
1481    }
1482
1483    /// Fold an execution's committed-idempotent prefix below `up_to_step` into one checkpoint entry.
1484    ///
1485    /// Reads the foldable idempotent step results, packs as many as fit the payload budget into a
1486    /// sealed snapshot, writes a single [`EntryKind::Checkpoint`] entry, and deletes the folded rows
1487    /// — all in one transaction. A resume replays the folded steps from the snapshot (the snapshot
1488    /// preserves each step's idempotency key for the divergence guard) instead of re-running them.
1489    /// Returns the number of steps folded. Runs only on a background task (spec NEVER: not the hot
1490    /// path). Span: `durable.journal.checkpoint`.
1491    ///
1492    /// The checkpoint row also carries the fold's `folded_count` (issue #6360), in the same
1493    /// transaction as the DELETE. The high-water-mark's own `committed_result_count` is
1494    /// deliberately left untouched here: a fold moves committed results from live rows into the
1495    /// checkpoint snapshot net-zero, so the signed count stays valid without a bump — only the
1496    /// resume-time recomputation needs `folded_count` (`count(surviving StepResult) +
1497    /// SUM(folded_count)`) to see past the fold.
1498    ///
1499    /// # Errors
1500    ///
1501    /// Returns [`DurableError::Storage`] on a database error, or a cipher failure if (re)sealing
1502    /// fails.
1503    pub(crate) async fn checkpoint_fold(
1504        &self,
1505        execution_id: ExecutionId,
1506        up_to_step: u32,
1507    ) -> Result<u64, DurableError> {
1508        let span = tracing::info_span!(
1509            "durable.journal.checkpoint",
1510            execution_id = %execution_id.as_uuid(),
1511            folded_count = tracing::field::Empty,
1512        );
1513        async move {
1514            let exec = execution_id.as_uuid().to_string();
1515            let rows: Vec<FoldableRowRead> = zeph_db::query_as(sql!(
1516                "SELECT step_id, idem_key, payload_version, payload FROM durable_journal
1517                 WHERE execution_id = ? AND entry_kind = 'step_result'
1518                   AND effect_class = 'idempotent' AND step_id < ?
1519                 ORDER BY step_id"
1520            ))
1521            .bind(&exec)
1522            .bind(i64::from(up_to_step))
1523            .fetch_all(&self.pool)
1524            .await
1525            .map_err(|e| DurableError::storage("checkpoint", e))?;
1526            if rows.is_empty() {
1527                return Ok(0);
1528            }
1529
1530            // Open each sealed result, then keep the budget-bounded prefix that fits a checkpoint.
1531            let mut folded = self.open_foldable_steps(execution_id, rows)?;
1532            let lens: Vec<usize> = folded.iter().map(|s| s.payload.len()).collect();
1533            let take = crate::retention::fold_prefix_len(
1534                &lens,
1535                crate::retention::checkpoint_budget(self.max_payload_bytes),
1536            );
1537            if take == 0 {
1538                // Not even one result fits the budget; leave the prefix un-folded rather than write
1539                // an over-limit checkpoint.
1540                return Ok(0);
1541            }
1542            folded.truncate(take);
1543            let fold_end = folded.last().map_or(up_to_step, |s| s.step_id.saturating_add(1));
1544
1545            let snapshot = encode_checkpoint(&folded);
1546            let snap_aad =
1547                PayloadAad::new(execution_id, StepId::new(fold_end), EntryKindTag::Checkpoint, None);
1548            let sealed_snapshot = self.seal_payload(&snapshot, &snap_aad)?;
1549
1550            // `folded_count` (issue #6360) is persisted on the checkpoint row itself, in the same
1551            // transaction as the fold's DELETE, so resume can recompute `committed_result_count` as
1552            // `count(surviving StepResult rows) + SUM(folded_count over checkpoints)` without ever
1553            // observing a fold whose DELETE committed but whose count did not (or vice versa).
1554            let count = folded.len() as u64;
1555
1556            let mut tx = zeph_db::begin_write(&self.pool)
1557                .await
1558                .map_err(|e| DurableError::storage("checkpoint", e))?;
1559            zeph_db::query(sql!(
1560                "INSERT INTO durable_journal
1561                    (execution_id, step_id, entry_kind, idem_key, effect_class, payload, payload_version, hmac, created_at, folded_count)
1562                 VALUES (?, ?, 'checkpoint', NULL, NULL, ?, ?, NULL, ?, ?)"
1563            ))
1564            .bind(&exec)
1565            .bind(i64::from(fold_end))
1566            .bind(sealed_snapshot)
1567            .bind(i32::from(crate::step::PAYLOAD_VERSION))
1568            .bind(now_unix_millis())
1569            .bind(i64::try_from(count).unwrap_or(i64::MAX))
1570            .execute(&mut *tx)
1571            .await
1572            .map_err(|e| DurableError::storage("checkpoint", e))?;
1573            zeph_db::query(sql!(
1574                "DELETE FROM durable_journal
1575                 WHERE execution_id = ? AND entry_kind = 'step_result'
1576                   AND effect_class = 'idempotent' AND step_id < ?"
1577            ))
1578            .bind(&exec)
1579            .bind(i64::from(fold_end))
1580            .execute(&mut *tx)
1581            .await
1582            .map_err(|e| DurableError::storage("checkpoint", e))?;
1583            tx.commit()
1584                .await
1585                .map_err(|e| DurableError::storage("checkpoint", e))?;
1586
1587            tracing::Span::current().record("folded_count", count);
1588            Ok(count)
1589        }
1590        .instrument(span)
1591        .await
1592    }
1593
1594    /// Read every checkpoint snapshot for an execution and reconstruct its folded step results.
1595    ///
1596    /// The replay cursor calls this once on resume to preload folded results before walking the
1597    /// surviving journal rows: each returned [`JournalEntry`] is a `StepResult` whose individual row
1598    /// was deleted by the fold but whose replay value (and idempotency key, for the divergence guard)
1599    /// lives in the snapshot. Each snapshot is AEAD-opened with its checkpoint-bound AAD. Returns an
1600    /// empty vector when the execution has never been folded.
1601    ///
1602    /// # Errors
1603    ///
1604    /// Returns [`DurableError::Storage`] on a database error, or a decode/cipher failure if a
1605    /// snapshot is corrupt.
1606    pub(crate) async fn read_checkpoints(
1607        &self,
1608        execution_id: ExecutionId,
1609    ) -> Result<Vec<JournalEntry>, DurableError> {
1610        let rows: Vec<(i64, Option<Vec<u8>>)> = zeph_db::query_as(sql!(
1611            "SELECT step_id, payload FROM durable_journal
1612             WHERE execution_id = ? AND entry_kind = 'checkpoint' ORDER BY step_id"
1613        ))
1614        .bind(execution_id.as_uuid().to_string())
1615        .fetch_all(&self.pool)
1616        .await
1617        .map_err(|e| DurableError::storage("read_checkpoints", e))?;
1618        if rows.is_empty() {
1619            return Ok(Vec::new());
1620        }
1621        let mut folded: CheckpointSnapshot = Vec::new();
1622        for (up_to, payload) in rows {
1623            let up_to = u32::try_from(up_to).map_err(|_| DurableError::Decode {
1624                context: "checkpoint up_to_step out of u32 range",
1625            })?;
1626            let sealed = payload.ok_or(DurableError::Decode {
1627                context: "checkpoint entry missing snapshot payload",
1628            })?;
1629            ensure_payload_within_limit(
1630                sealed.len(),
1631                self.max_payload_bytes.saturating_add(SEAL_OVERHEAD_SLACK),
1632            )?;
1633            let aad = PayloadAad::new(
1634                execution_id,
1635                StepId::new(up_to),
1636                EntryKindTag::Checkpoint,
1637                None,
1638            );
1639            let plaintext = self.open_payload(&sealed, &aad)?;
1640            folded.extend(decode_checkpoint(&plaintext)?);
1641        }
1642        // Reconstruct each folded step as a replayable `StepResult` entry under the real execution
1643        // kind, so the cursor serves it exactly like a surviving row.
1644        let kind = self.lookup_kind(execution_id).await?;
1645        let entries = folded
1646            .into_iter()
1647            .map(|step| JournalEntry {
1648                seq: None,
1649                execution_id,
1650                kind,
1651                step_id: StepId::new(step.step_id),
1652                entry: EntryKind::StepResult {
1653                    idempotency_key: IdempotencyKey::from_bytes(step.idem_key),
1654                    payload: step.payload,
1655                    effect: crate::EffectClass::Idempotent,
1656                    payload_version: step.payload_version,
1657                },
1658                created_at_ms: 0,
1659            })
1660            .collect();
1661        Ok(entries)
1662    }
1663
1664    /// Delete one bounded batch of prunable terminal executions and their child rows.
1665    ///
1666    /// Selects up to `batch` executions past their TTL, then deletes their journal, promise, timer,
1667    /// integrity (issue #6360), and execution rows in a single transaction (children first, to
1668    /// respect the foreign keys). Returns the number of executions removed; the retention loop
1669    /// stops once a batch returns fewer than `batch`.
1670    ///
1671    /// The candidate-selection `SELECT` runs *inside* the same `begin_write` transaction as the
1672    /// deletes (not on the autocommit pool beforehand), closing the race where a concurrent
1673    /// `open_execution` reopen (un-finalize, #6251) lands between "select prunable ids" and
1674    /// "delete them" — without this, a legitimately-resumed execution could be deleted out from
1675    /// under its own reopen. `SQLite`'s `BEGIN IMMEDIATE` (via `begin_write`) already serializes
1676    /// writers at the file level, so the `SELECT` alone is enough there; `PostgreSQL` needs an
1677    /// explicit `SELECT ... FOR UPDATE` first to take row locks on the same candidates before
1678    /// they're read, since a plain `BEGIN` does not otherwise block a concurrent `UPDATE` on those
1679    /// rows (mirrors the `BEGIN IMMEDIATE` / `SELECT FOR UPDATE` split in `goal/store.rs`).
1680    async fn delete_prune_batch(
1681        &self,
1682        cutoffs: crate::retention::PruneCutoffs,
1683        batch: u64,
1684    ) -> Result<u64, DurableError> {
1685        let mut tx = zeph_db::begin_write(&self.pool)
1686            .await
1687            .map_err(|e| DurableError::storage("prune", e))?;
1688
1689        // Postgres only: lock the same candidate rows before reading them, so a concurrent
1690        // `open_execution` reopen UPDATE on one of these rows blocks until this transaction
1691        // commits (and then no longer matches, since the SELECT below re-reads post-commit) or
1692        // this transaction rolls back. Bounded by the same ORDER BY/LIMIT as the real read below
1693        // so the lock's blast radius matches the batch, not the whole prunable backlog.
1694        #[cfg(feature = "postgres")]
1695        zeph_db::query(sql!(
1696            "SELECT execution_id FROM durable_executions
1697             WHERE finalized_at IS NOT NULL
1698               AND ( (status = 'completed' AND finalized_at <= ?)
1699                  OR (status IN ('failed', 'aborted', 'canceled') AND finalized_at <= ?) )
1700             ORDER BY finalized_at LIMIT ?
1701             FOR UPDATE"
1702        ))
1703        .bind(cutoffs.completed_before_ms)
1704        .bind(cutoffs.failed_before_ms)
1705        .bind(i64::try_from(batch).unwrap_or(i64::MAX))
1706        .execute(&mut *tx)
1707        .await
1708        .map_err(|e| DurableError::storage("prune", e))?;
1709
1710        let ids: Vec<(String,)> = zeph_db::query_as(sql!(
1711            "SELECT execution_id FROM durable_executions
1712             WHERE finalized_at IS NOT NULL
1713               AND ( (status = 'completed' AND finalized_at <= ?)
1714                  OR (status IN ('failed', 'aborted', 'canceled') AND finalized_at <= ?) )
1715             ORDER BY finalized_at LIMIT ?"
1716        ))
1717        .bind(cutoffs.completed_before_ms)
1718        .bind(cutoffs.failed_before_ms)
1719        .bind(i64::try_from(batch).unwrap_or(i64::MAX))
1720        .fetch_all(&mut *tx)
1721        .await
1722        .map_err(|e| DurableError::storage("prune", e))?;
1723        if ids.is_empty() {
1724            tx.commit()
1725                .await
1726                .map_err(|e| DurableError::storage("prune", e))?;
1727            return Ok(0);
1728        }
1729        let journal = sql!("DELETE FROM durable_journal WHERE execution_id = ?");
1730        let promises = sql!("DELETE FROM durable_promises WHERE execution_id = ?");
1731        let timers = sql!("DELETE FROM durable_timers WHERE execution_id = ?");
1732        // Issue #6360: `durable_execution_integrity` references `durable_executions` without
1733        // `ON DELETE CASCADE` (same convention as journal/promises/timers), so it must be deleted
1734        // here too — otherwise the `DELETE FROM durable_executions` below violates the FK on every
1735        // backend with FK enforcement on (Postgres always; SQLite via `zeph-db`'s
1736        // `PRAGMA foreign_keys = ON`), rolling back the whole prune batch for any keyed execution
1737        // that ever committed a `StepResult` (`bump_hwm_for_step_result` always writes this row
1738        // when an HWM key is configured). A no-op `DELETE` for an unkeyed/never-committed execution
1739        // (no row present) is fine.
1740        let integrity = sql!("DELETE FROM durable_execution_integrity WHERE execution_id = ?");
1741        // Re-guarded by the same status/finalized_at predicate as the SELECT above (not just
1742        // `execution_id = ?`) — belt and suspenders alongside the transactional read above.
1743        let executions = sql!(
1744            "DELETE FROM durable_executions
1745             WHERE execution_id = ?
1746               AND finalized_at IS NOT NULL
1747               AND ( (status = 'completed' AND finalized_at <= ?)
1748                  OR (status IN ('failed', 'aborted', 'canceled') AND finalized_at <= ?) )"
1749        );
1750        let mut removed = 0u64;
1751        for (id,) in &ids {
1752            for stmt in [journal, promises, timers, integrity] {
1753                zeph_db::query(stmt)
1754                    .bind(id)
1755                    .execute(&mut *tx)
1756                    .await
1757                    .map_err(|e| DurableError::storage("prune", e))?;
1758            }
1759            let result = zeph_db::query(executions)
1760                .bind(id)
1761                .bind(cutoffs.completed_before_ms)
1762                .bind(cutoffs.failed_before_ms)
1763                .execute(&mut *tx)
1764                .await
1765                .map_err(|e| DurableError::storage("prune", e))?;
1766            removed += result.rows_affected();
1767        }
1768        tx.commit()
1769            .await
1770            .map_err(|e| DurableError::storage("prune", e))?;
1771        Ok(removed)
1772    }
1773
1774    /// One batch of the crash-orphan sweep (INV-17, #6254).
1775    ///
1776    /// Selects up to `batch` `status='running'` rows whose `updated_at` is at or before
1777    /// `cutoff_ms`, then for each candidate non-blockingly try-acquires its INV-15
1778    /// `ExecutionLock`: `ExecutionLocked` (a live owner holds it) short-circuits to skip —
1779    /// staleness of `updated_at` alone is never sufficient grounds to abort. Only when the lock is
1780    /// acquired does the guarded `UPDATE` run, still holding the lock, so the abort is race-free
1781    /// against a concurrent `open_execution_exclusive` reopen for the same id (both require the
1782    /// same non-reentrant flock). The lock releases when it drops at the end of each loop
1783    /// iteration.
1784    ///
1785    /// `cursor` is the previous batch's [`SweepCursor`](crate::retention::SweepCursor) (`None` for
1786    /// the first batch); the candidate scan is keyset-paginated strictly past it so a skipped
1787    /// (lock-held) row is never re-selected by a later batch — #6254 C1: without this, a batch
1788    /// consisting entirely of lock-held rows would re-select the identical rows on every
1789    /// iteration and the caller's batch loop would never terminate. Returns the number of rows
1790    /// scanned (for the caller's batch-continuation decision), the number actually aborted, and
1791    /// the cursor to resume from on the next call.
1792    async fn sweep_orphan_batch(
1793        &self,
1794        lock_dir: &std::path::Path,
1795        cutoff_ms: i64,
1796        batch: u64,
1797        cursor: Option<crate::retention::SweepCursor>,
1798    ) -> Result<crate::retention::SweepBatchOutcome, DurableError> {
1799        // Sentinel "no lower bound" cursor: every real `updated_at` (Unix ms) is > i64::MIN, so
1800        // this keyset predicate is a no-op on the first batch while still using one static,
1801        // sql!()-cacheable query for both the first and subsequent calls.
1802        let (after_updated_at, after_exec) = cursor.map_or((i64::MIN, String::new()), |c| {
1803            (c.updated_at_ms, c.execution_id)
1804        });
1805
1806        let candidates: Vec<(String, i64)> = zeph_db::query_as(sql!(
1807            "SELECT execution_id, updated_at FROM durable_executions
1808             WHERE status = 'running' AND updated_at <= ?
1809               AND (updated_at > ? OR (updated_at = ? AND execution_id > ?))
1810             ORDER BY updated_at, execution_id LIMIT ?"
1811        ))
1812        .bind(cutoff_ms)
1813        .bind(after_updated_at)
1814        .bind(after_updated_at)
1815        .bind(&after_exec)
1816        .bind(i64::try_from(batch).unwrap_or(i64::MAX))
1817        .fetch_all(&self.pool)
1818        .await
1819        .map_err(|e| DurableError::storage("sweep_orphans", e))?;
1820
1821        let scanned = u64::try_from(candidates.len()).unwrap_or(u64::MAX);
1822        let next_cursor = candidates
1823            .last()
1824            .map(|(id, updated_at)| crate::retention::SweepCursor {
1825                updated_at_ms: *updated_at,
1826                execution_id: id.clone(),
1827            });
1828
1829        let now = now_unix_millis();
1830        let abort = sql!(
1831            "UPDATE durable_executions SET status = 'aborted', finalized_at = ?, updated_at = ?
1832             WHERE execution_id = ? AND status = 'running' AND finalized_at IS NULL"
1833        );
1834        let mut aborted = 0u64;
1835        for (exec_str, _updated_at) in &candidates {
1836            let Ok(execution_id) = parse_execution_id(exec_str) else {
1837                continue;
1838            };
1839            match ExecutionLock::acquire(lock_dir, execution_id) {
1840                Ok(_lock) => {
1841                    let result = zeph_db::query(abort)
1842                        .bind(now)
1843                        .bind(now)
1844                        .bind(exec_str)
1845                        .execute(&self.pool)
1846                        .await
1847                        .map_err(|e| DurableError::storage("sweep_orphans", e))?;
1848                    aborted += result.rows_affected();
1849                    // `_lock` drops here, releasing the flock for the next holder.
1850                }
1851                Err(DurableError::ExecutionLocked { .. }) => {
1852                    // A live owner holds this execution — never abort on staleness alone (INV-17).
1853                }
1854                Err(e) => return Err(e),
1855            }
1856        }
1857        Ok(crate::retention::SweepBatchOutcome {
1858            scanned,
1859            aborted,
1860            next_cursor,
1861        })
1862    }
1863
1864    /// Seal a plaintext payload, or pass it through verbatim when no cipher is configured.
1865    fn seal_payload(&self, plaintext: &[u8], aad: &PayloadAad) -> Result<Vec<u8>, DurableError> {
1866        match &self.cipher {
1867            Some(cipher) => Ok(cipher.seal(plaintext, aad)?),
1868            None => Ok(plaintext.to_vec()),
1869        }
1870    }
1871
1872    /// Open a sealed payload, or copy it through verbatim when no cipher is configured.
1873    fn open_payload(&self, sealed: &[u8], aad: &PayloadAad) -> Result<Bytes, DurableError> {
1874        match &self.cipher {
1875            Some(cipher) => Ok(Bytes::from(cipher.open(sealed, aad)?)),
1876            None => Ok(Bytes::copy_from_slice(sealed)),
1877        }
1878    }
1879
1880    /// Compute the keyed-BLAKE3 row HMAC over a control entry's identity, when an HMAC key is set.
1881    ///
1882    /// Binds `(execution_id, step_id, entry_kind, idem_key?)` so a control row cannot be forged or
1883    /// relocated on a shared database. Returns `None` when no key is configured (single-user local).
1884    fn control_hmac(
1885        &self,
1886        entry: &JournalEntry,
1887        idem_key: Option<&IdempotencyKey>,
1888    ) -> Option<Vec<u8>> {
1889        self.compute_control_hmac(
1890            entry.execution_id,
1891            entry.step_id,
1892            entry.entry.tag(),
1893            idem_key,
1894        )
1895        .map(|h| h.to_vec())
1896    }
1897
1898    /// Core keyed-BLAKE3 computation shared by [`control_hmac`](Self::control_hmac) (write path,
1899    /// takes a full [`JournalEntry`]) and [`verify_control_hmac`](Self::verify_control_hmac) (read
1900    /// path, which has the row's identity fields but not yet a reconstructed entry). Returns `None`
1901    /// when no HMAC key is configured (single-user local).
1902    fn compute_control_hmac(
1903        &self,
1904        execution_id: ExecutionId,
1905        step_id: StepId,
1906        tag: &'static str,
1907        idem_key: Option<&IdempotencyKey>,
1908    ) -> Option<[u8; 32]> {
1909        let key = self.hmac_key.as_ref()?;
1910        Some(Self::keyed_control_hmac(
1911            key,
1912            execution_id,
1913            step_id,
1914            tag,
1915            idem_key,
1916        ))
1917    }
1918
1919    /// Keyed-BLAKE3 computation over a control entry's identity, parameterized on the key so both
1920    /// the current and previous rotation-window keys (#6451) can be tried against the same input.
1921    fn keyed_control_hmac(
1922        key: &[u8; 32],
1923        execution_id: ExecutionId,
1924        step_id: StepId,
1925        tag: &'static str,
1926        idem_key: Option<&IdempotencyKey>,
1927    ) -> [u8; 32] {
1928        let mut input = Vec::with_capacity(16 + 4 + 16 + 32);
1929        input.extend_from_slice(execution_id.as_bytes());
1930        input.extend_from_slice(&step_id.value().to_le_bytes());
1931        input.extend_from_slice(tag.as_bytes());
1932        if let Some(k) = idem_key {
1933            input.extend_from_slice(k.as_bytes());
1934        }
1935        *blake3::keyed_hash(key, &input).as_bytes()
1936    }
1937
1938    /// Recompute and constant-time-verify a control entry's row HMAC read back from storage
1939    /// (INV-8), trying the previous rotation-window key (#6451) when the current key does not
1940    /// match.
1941    ///
1942    /// A no-op only when no HMAC key is configured **and** the row carries no stored HMAC — the
1943    /// documented single-user local stance where control entries carry no HMAC and none is
1944    /// enforced. If this backend is unkeyed but the row *does* carry a stamped HMAC, that is
1945    /// config drift between the writer and this reader (e.g. `shared_db` toggled, or a reader
1946    /// whose config disagrees with the writer's over the same physical file) and is rejected
1947    /// fail-closed rather than silently trusted, since an `EffectIntent`'s fields are plaintext
1948    /// and an unkeyed reader has no way to tell a genuine stamped row from a forged one. When a
1949    /// key *is* configured, every control row this backend reads must carry a matching HMAC: a
1950    /// missing HMAC, or a mismatch under both the current and any registered
1951    /// [`previous_hmac_key`](Self::with_previous_hmac_key), fails closed with
1952    /// [`DurableError::ControlIntegrity`].
1953    ///
1954    /// Each comparison uses [`blake3::Hash`] equality, which compares in constant time (the same
1955    /// idiom used for the promise resolver-token check in `promise.rs`), so a forged HMAC reveals
1956    /// no timing signal beyond which of the (at most two) legitimate keys, if any, it was written
1957    /// under — already observable via `created_at` relative to the rotation.
1958    fn verify_control_hmac(
1959        &self,
1960        execution_id: ExecutionId,
1961        step_id: StepId,
1962        tag: &'static str,
1963        idem_key: Option<&IdempotencyKey>,
1964        stored: Option<[u8; 32]>,
1965    ) -> Result<(), DurableError> {
1966        let Some(current_key) = self.hmac_key.as_ref() else {
1967            return if stored.is_some() {
1968                Err(DurableError::ControlIntegrity)
1969            } else {
1970                Ok(())
1971            };
1972        };
1973        let Some(stored) = stored else {
1974            return Err(DurableError::ControlIntegrity);
1975        };
1976        let expected_current =
1977            Self::keyed_control_hmac(current_key, execution_id, step_id, tag, idem_key);
1978        if blake3::Hash::from(expected_current) == blake3::Hash::from(stored) {
1979            return Ok(());
1980        }
1981        if let Some(previous_key) = self.previous_hmac_key.as_ref() {
1982            let expected_previous =
1983                Self::keyed_control_hmac(previous_key, execution_id, step_id, tag, idem_key);
1984            if blake3::Hash::from(expected_previous) == blake3::Hash::from(stored) {
1985                return Ok(());
1986            }
1987        }
1988        Err(DurableError::ControlIntegrity)
1989    }
1990
1991    /// Compute the high-water-mark HMAC (issue #6360) over the signed
1992    /// `{execution_id, max_committed_step_id, committed_result_count, key_epoch}` tuple.
1993    ///
1994    /// Domain-separated from [`compute_control_hmac`](Self::compute_control_hmac)'s input by
1995    /// construction — this binds `max_committed_step_id` and `committed_result_count`, fields the
1996    /// control-entry HMAC never includes — so the two mechanisms safely share key material without
1997    /// a cross-mechanism forgery becoming possible.
1998    fn compute_hwm_hmac(
1999        execution_id: ExecutionId,
2000        max_committed_step_id: u32,
2001        committed_result_count: u64,
2002        key_epoch: u32,
2003        key: &[u8; 32],
2004    ) -> [u8; 32] {
2005        let mut input = Vec::with_capacity(16 + 4 + 8 + 4);
2006        input.extend_from_slice(execution_id.as_bytes());
2007        input.extend_from_slice(&max_committed_step_id.to_le_bytes());
2008        input.extend_from_slice(&committed_result_count.to_le_bytes());
2009        input.extend_from_slice(&key_epoch.to_le_bytes());
2010        *blake3::keyed_hash(key, &input).as_bytes()
2011    }
2012
2013    /// Resolve the high-water-mark key registered for `epoch`: the current key first, then the
2014    /// registered previous key (FR-008 rotation window).
2015    ///
2016    /// Returns `None` when `epoch` matches neither slot — an unresolvable epoch on a row that
2017    /// carries HWM metadata, which the caller must treat as fail-closed (NFR-004), never as legacy:
2018    /// only a row's total *absence* is legacy, not a present-but-unverifiable one (closes the
2019    /// downgrade lever where a stripped/forged epoch would otherwise masquerade as "predates the
2020    /// feature").
2021    fn resolve_hwm_key(&self, epoch: u32) -> Option<[u8; 32]> {
2022        if let Some(slot) = &self.hwm_key
2023            && slot.epoch == epoch
2024        {
2025            return Some(slot.key);
2026        }
2027        if let Some(slot) = &self.hwm_key_previous
2028            && slot.epoch == epoch
2029        {
2030            return Some(slot.key);
2031        }
2032        None
2033    }
2034
2035    /// Bump the signed high-water-mark (issue #6360) after committing a `StepResult` row, inside
2036    /// the same transaction as its INSERT. A no-op when no HWM key is configured.
2037    ///
2038    /// Reads the current signed tuple (or starts from zero for a first-ever committed result),
2039    /// increments `committed_result_count` by one, raises `max_committed_step_id` to `step_id` when
2040    /// higher, and re-signs under the current epoch — all inside `tx`, so a `StepResult` can never
2041    /// commit without its HWM update landing atomically alongside it (no TOCTOU gap). Folding
2042    /// (`checkpoint_fold`) never calls this: a fold moves the same committed results from live rows
2043    /// into a checkpoint snapshot net-zero, so `committed_result_count` is invariant across it —
2044    /// only [`checkpoint_fold`](Self::checkpoint_fold)'s own `folded_count` column changes.
2045    async fn bump_hwm_for_step_result(
2046        &self,
2047        tx: &mut zeph_db::DbTransaction<'_>,
2048        execution_id: ExecutionId,
2049        step_id: StepId,
2050    ) -> Result<(), DurableError> {
2051        let Some(slot) = &self.hwm_key else {
2052            return Ok(());
2053        };
2054        let exec = execution_id.as_uuid().to_string();
2055        let existing: Option<(i64, i64)> = zeph_db::query_as(sql!(
2056            "SELECT max_committed_step_id, committed_result_count
2057             FROM durable_execution_integrity WHERE execution_id = ?"
2058        ))
2059        .bind(&exec)
2060        .fetch_optional(&mut **tx)
2061        .await
2062        .map_err(|e| DurableError::storage("hwm_bump", e))?;
2063        let (prev_max, prev_count) = existing.unwrap_or((0, 0));
2064        let new_max = prev_max.max(i64::from(step_id.value()));
2065        let new_count = prev_count.saturating_add(1);
2066        let hmac = Self::compute_hwm_hmac(
2067            execution_id,
2068            u32::try_from(new_max).unwrap_or(u32::MAX),
2069            u64::try_from(new_count).unwrap_or(u64::MAX),
2070            slot.epoch,
2071            &slot.key,
2072        );
2073        zeph_db::query(sql!(
2074            "INSERT INTO durable_execution_integrity
2075                (execution_id, key_epoch, max_committed_step_id, committed_result_count, hwm_hmac, updated_at)
2076             VALUES (?, ?, ?, ?, ?, ?)
2077             ON CONFLICT(execution_id) DO UPDATE SET
2078                key_epoch = excluded.key_epoch,
2079                max_committed_step_id = excluded.max_committed_step_id,
2080                committed_result_count = excluded.committed_result_count,
2081                hwm_hmac = excluded.hwm_hmac,
2082                updated_at = excluded.updated_at"
2083        ))
2084        .bind(&exec)
2085        .bind(i64::from(slot.epoch))
2086        .bind(new_max)
2087        .bind(new_count)
2088        .bind(hmac.to_vec())
2089        .bind(now_unix_millis())
2090        .execute(&mut **tx)
2091        .await
2092        .map_err(|e| DurableError::storage("hwm_bump", e))?;
2093        Ok(())
2094    }
2095
2096    /// Verify the signed high-water-mark (issue #6360) for a resumed execution, and fail closed on
2097    /// any mismatch (FR-004, US-003: the durable resume path never offers an override).
2098    ///
2099    /// A no-op when no HWM key is configured. On any verification failure, best-effort finalizes
2100    /// the execution as `Aborted` (mirroring the step-cap-exceeded path in `handle.rs`) before
2101    /// returning the error, so a corrupted execution does not linger `running` forever waiting for
2102    /// a resume attempt that will keep failing.
2103    async fn verify_high_water_mark(&self, execution_id: ExecutionId) -> Result<(), DurableError> {
2104        if self.hwm_key.is_none() {
2105            return Ok(());
2106        }
2107        if let Err(error) = self.check_high_water_mark(execution_id).await {
2108            if let Err(finalize_error) = self.finalize(execution_id, ExecutionStatus::Aborted).await
2109            {
2110                tracing::warn!(
2111                    error = %finalize_error,
2112                    "failed to mark HWM-integrity-failed execution aborted"
2113                );
2114            }
2115            return Err(error);
2116        }
2117        Ok(())
2118    }
2119
2120    /// Find every **resumable** (`status = 'running'`) execution that has committed at least one
2121    /// `StepResult` but carries no `durable_execution_integrity` row (issue #6449).
2122    ///
2123    /// This is the drain-before-seal precondition scan for `zeph durable seal-integrity`: the
2124    /// returned set is exactly the executions that would be silently downgraded to
2125    /// unconditional-tamper the moment this backend seals, unless drained to a terminal status
2126    /// first or explicitly grandfathered. A non-resumable (terminal) execution missing its row is
2127    /// not a concern — it can never be resumed again, sealed or not.
2128    ///
2129    /// # Errors
2130    ///
2131    /// Returns [`DurableError::Storage`] if the query fails.
2132    pub async fn find_unsealed_resumable_executions(
2133        &self,
2134    ) -> Result<Vec<ExecutionId>, DurableError> {
2135        let rows: Vec<(String,)> = zeph_db::query_as(sql!(
2136            "SELECT e.execution_id FROM durable_executions e
2137             WHERE e.status = 'running'
2138               AND NOT EXISTS (
2139                 SELECT 1 FROM durable_execution_integrity i WHERE i.execution_id = e.execution_id
2140               )
2141               AND (
2142                 EXISTS (
2143                   SELECT 1 FROM durable_journal j
2144                   WHERE j.execution_id = e.execution_id AND j.entry_kind = 'step_result'
2145                 )
2146                 OR EXISTS (
2147                   SELECT 1 FROM durable_journal j
2148                   WHERE j.execution_id = e.execution_id AND j.entry_kind = 'checkpoint'
2149                     AND j.folded_count > 0
2150                 )
2151               )"
2152        ))
2153        .fetch_all(&self.pool)
2154        .await
2155        .map_err(|e| DurableError::storage("seal_integrity_scan", e))?;
2156
2157        rows.into_iter()
2158            .map(|(id,)| {
2159                ExecutionId::parse_str(&id).map_err(|_| DurableError::Decode {
2160                    context: "malformed execution_id in durable_executions",
2161                })
2162            })
2163            .collect()
2164    }
2165
2166    /// Recompute the number of committed `StepResult`s for `execution_id` directly from the
2167    /// journal: surviving `step_result` rows plus every checkpoint's `folded_count` (a fold moves
2168    /// committed results into a checkpoint snapshot net-zero, so this sum is invariant across
2169    /// folding). Shared by [`check_high_water_mark`](Self::check_high_water_mark)'s present-row
2170    /// recomputation and its post-seal absent-row check (issue #6449).
2171    async fn committed_step_result_count(
2172        &self,
2173        execution_id: ExecutionId,
2174    ) -> Result<u64, DurableError> {
2175        let exec = execution_id.as_uuid().to_string();
2176        let live_count: i64 = zeph_db::query_scalar(sql!(
2177            "SELECT COUNT(*) FROM durable_journal
2178             WHERE execution_id = ? AND entry_kind = 'step_result'"
2179        ))
2180        .bind(&exec)
2181        .fetch_one(&self.pool)
2182        .await
2183        .map_err(|e| DurableError::storage("hwm_verify", e))?;
2184        let folded_sum: i64 = zeph_db::query_scalar(sql!(
2185            "SELECT COALESCE(SUM(folded_count), 0) FROM durable_journal
2186             WHERE execution_id = ? AND entry_kind = 'checkpoint'"
2187        ))
2188        .bind(&exec)
2189        .fetch_one(&self.pool)
2190        .await
2191        .map_err(|e| DurableError::storage("hwm_verify", e))?;
2192        Ok(u64::try_from(live_count.saturating_add(folded_sum)).unwrap_or(0))
2193    }
2194
2195    /// The comparison half of `verify_high_water_mark`.
2196    ///
2197    /// Absent a stored `durable_execution_integrity` row: **pre-seal** (or unkeyed), this
2198    /// execution predates the feature or has committed no `StepResult` yet — nothing to compare
2199    /// against, so it is accepted (migration posture: only a row's total absence is legacy,
2200    /// mirroring the JSONL side's "no chain metadata at all" lane). **Post-seal** (issue #6449 —
2201    /// `integrity_sealed == true`, confirmed via the vault-stored `ZEPH_DURABLE_INTEGRITY_SEALED`
2202    /// marker, never a DB column), a keyed, non-grandfathered execution with at least one
2203    /// committed `StepResult` but no integrity row is unconditional tamper: the drain-before-seal
2204    /// precondition on `zeph durable seal-integrity` guarantees no execution can reach this state
2205    /// legitimately once sealed (the keyed integrity-row write is atomic-in-transaction with the
2206    /// `StepResult` commit, so "committed result present, row absent" cannot occur for anything
2207    /// that started after the vault key was attached). A *present* row is always fully verified:
2208    /// an unresolvable `key_epoch`, an HMAC that does not authenticate, or a recomputed
2209    /// `committed_result_count` that disagrees with the signed value are each a distinct
2210    /// fail-closed [`DurableError::HighWaterMarkIntegrity`].
2211    async fn check_high_water_mark(&self, execution_id: ExecutionId) -> Result<(), DurableError> {
2212        let exec = execution_id.as_uuid().to_string();
2213        let stored: Option<(i64, i64, i64, Vec<u8>)> = zeph_db::query_as(sql!(
2214            "SELECT key_epoch, max_committed_step_id, committed_result_count, hwm_hmac
2215             FROM durable_execution_integrity WHERE execution_id = ?"
2216        ))
2217        .bind(&exec)
2218        .fetch_optional(&self.pool)
2219        .await
2220        .map_err(|e| DurableError::storage("hwm_verify", e))?;
2221        let Some((epoch_raw, max_step_raw, count_raw, hmac)) = stored else {
2222            if self.hwm_key.is_some()
2223                && self.integrity_sealed
2224                && !self.integrity_grandfather.contains(&execution_id)
2225                && self.committed_step_result_count(execution_id).await? >= 1
2226            {
2227                return Err(DurableError::HighWaterMarkIntegrity {
2228                    execution_id,
2229                    reason: "integrity_row_absent_post_seal",
2230                    hint: "TAMPER: this backend is sealed against pre-feature integrity-row \
2231                           absence, this execution is keyed and not grandfathered, and it has \
2232                           committed StepResults — a legitimate keyed execution can never reach \
2233                           this state (the integrity row is written atomically with its first \
2234                           committed StepResult), so an absent row here means the row was \
2235                           deleted outside the write path",
2236                });
2237            }
2238            return Ok(());
2239        };
2240
2241        // Per FR-008, the operator-facing `hint` distinguishes "possibly re-keyed" (a legitimate
2242        // rotation this backend cannot resolve) from "TAMPER" (content that did not authenticate)
2243        // — the durable resume path stays fail-closed either way (FR-004), but the two cases call
2244        // for different operator follow-up, so they must not read the same in the logs.
2245        let fail =
2246            |reason: &'static str, hint: &'static str| DurableError::HighWaterMarkIntegrity {
2247                execution_id,
2248                reason,
2249                hint,
2250            };
2251        let tamper = |reason: &'static str| {
2252            fail(
2253                reason,
2254                "TAMPER: the signed high-water-mark did not authenticate under any key this \
2255                 backend holds for the recorded epoch",
2256            )
2257        };
2258
2259        let epoch = u32::try_from(epoch_raw).map_err(|_| tamper("hmac_mismatch"))?;
2260        let Some(key) = self.resolve_hwm_key(epoch) else {
2261            return Err(fail(
2262                "key_epoch_unresolvable",
2263                "possibly re-keyed: this execution's signed key_epoch is neither the current key \
2264                 nor a registered previous rotation key — if ZEPH_DURABLE_KEY was recently \
2265                 rotated, ensure the rotation window is still open (ZEPH_DURABLE_KEY_PREVIOUS \
2266                 present and [durable] previous_key_id set); the window is closed permanently by \
2267                 `zeph durable rotate-key --drop-previous`. The durable resume path cannot \
2268                 proceed without it (no interactive override)",
2269            ));
2270        };
2271        let stored_hmac =
2272            <[u8; 32]>::try_from(hmac.as_slice()).map_err(|_| tamper("hmac_mismatch"))?;
2273        let max_step = u32::try_from(max_step_raw).unwrap_or(u32::MAX);
2274        let count = u64::try_from(count_raw).unwrap_or(u64::MAX);
2275        let expected = Self::compute_hwm_hmac(execution_id, max_step, count, epoch, &key);
2276        if blake3::Hash::from(expected) != blake3::Hash::from(stored_hmac) {
2277            return Err(tamper("hmac_mismatch"));
2278        }
2279
2280        let recomputed = self.committed_step_result_count(execution_id).await?;
2281        if recomputed != count {
2282            return Err(fail(
2283                "count_mismatch",
2284                "TAMPER: the recomputed committed-result count (surviving StepResult rows plus \
2285                 every checkpoint's folded_count) disagrees with the signed value — a committed \
2286                 result was likely deleted outside the write path",
2287            ));
2288        }
2289        Ok(())
2290    }
2291
2292    /// Derive the persisted column values for an entry, sealing payloads and stamping HMACs.
2293    fn prepare_row(&self, entry: &JournalEntry) -> Result<JournalRow, DurableError> {
2294        let execution_id = entry.execution_id.as_uuid().to_string();
2295        let step_id = i64::from(entry.step_id.value());
2296        let created_at = entry.created_at_ms;
2297        let entry_kind = entry.entry.tag();
2298        match &entry.entry {
2299            EntryKind::StepResult {
2300                idempotency_key,
2301                payload,
2302                effect,
2303                payload_version,
2304            } => {
2305                ensure_payload_within_limit(payload.len(), self.max_payload_bytes)?;
2306                let aad = PayloadAad::new(
2307                    entry.execution_id,
2308                    entry.step_id,
2309                    EntryKindTag::StepResult,
2310                    Some(*idempotency_key),
2311                );
2312                let sealed = self.seal_payload(payload.as_ref(), &aad)?;
2313                Ok(JournalRow {
2314                    execution_id,
2315                    step_id,
2316                    entry_kind,
2317                    idem_key: Some(idempotency_key.as_bytes().to_vec()),
2318                    effect_class: Some(effect.as_str()),
2319                    payload: Some(sealed),
2320                    payload_version: Some(i32::from(*payload_version)),
2321                    hmac: None,
2322                    created_at,
2323                })
2324            }
2325            EntryKind::EffectIntent {
2326                idempotency_key,
2327                effect,
2328                hmac: _,
2329            } => {
2330                // The backend is the HMAC keyholder; it stamps the row HMAC itself when configured
2331                // and ignores any caller-supplied value.
2332                let hmac = self.control_hmac(entry, Some(idempotency_key));
2333                Ok(JournalRow {
2334                    execution_id,
2335                    step_id,
2336                    entry_kind,
2337                    idem_key: Some(idempotency_key.as_bytes().to_vec()),
2338                    effect_class: Some(effect.as_str()),
2339                    payload: None,
2340                    payload_version: None,
2341                    hmac,
2342                    created_at,
2343                })
2344            }
2345            EntryKind::PromiseCreated { .. }
2346            | EntryKind::PromiseResolved { .. }
2347            | EntryKind::TimerArmed { .. }
2348            | EntryKind::TimerFired { .. }
2349            | EntryKind::Checkpoint { .. } => {
2350                Err(DurableError::UnsupportedEntryKind { kind: entry_kind })
2351            }
2352        }
2353    }
2354
2355    /// Look up the owning execution's kind for read-time entry reconstruction.
2356    async fn lookup_kind(&self, id: ExecutionId) -> Result<ExecutionKind, DurableError> {
2357        let kind: Option<String> = zeph_db::query_scalar(sql!(
2358            "SELECT kind FROM durable_executions WHERE execution_id = ?"
2359        ))
2360        .bind(id.as_uuid().to_string())
2361        .fetch_optional(&self.pool)
2362        .await
2363        .map_err(|e| DurableError::storage("read", e))?;
2364        let kind = kind.ok_or(DurableError::Decode {
2365            context: "journaled entries reference a missing execution row",
2366        })?;
2367        ExecutionKind::from_tag(&kind).ok_or(DurableError::Decode {
2368            context: "execution kind is not reconstructible (custom kind read-back unsupported)",
2369        })
2370    }
2371
2372    /// Reconstruct a [`JournalEntry`] from a stored row, opening sealed payloads.
2373    fn row_to_entry(
2374        &self,
2375        id: ExecutionId,
2376        kind: ExecutionKind,
2377        row: JournalRowRead,
2378    ) -> Result<JournalEntry, DurableError> {
2379        let (
2380            seq,
2381            step_id_raw,
2382            entry_kind,
2383            idem_key,
2384            effect_class,
2385            payload,
2386            payload_version,
2387            hmac,
2388            created_at,
2389        ) = row;
2390        let step_id =
2391            StepId::new(
2392                u32::try_from(step_id_raw).map_err(|_| DurableError::Decode {
2393                    context: "step_id out of u32 range",
2394                })?,
2395            );
2396        let entry = match entry_kind.as_str() {
2397            "step_result" => {
2398                let idem_bytes = idem_key.ok_or(DurableError::Decode {
2399                    context: "step_result idem_key missing",
2400                })?;
2401                let idem_key = IdempotencyKey::from_bytes(slice_to_array32(
2402                    &idem_bytes,
2403                    "step_result idem_key",
2404                )?);
2405                let effect = effect_class
2406                    .as_deref()
2407                    .and_then(crate::EffectClass::from_tag)
2408                    .ok_or(DurableError::Decode {
2409                        context: "step_result effect_class missing or invalid",
2410                    })?;
2411                let sealed = payload.ok_or(DurableError::Decode {
2412                    context: "step_result payload missing",
2413                })?;
2414                ensure_payload_within_limit(
2415                    sealed.len(),
2416                    self.max_payload_bytes.saturating_add(SEAL_OVERHEAD_SLACK),
2417                )?;
2418                let aad = PayloadAad::new(id, step_id, EntryKindTag::StepResult, Some(idem_key));
2419                let opened = self.open_payload(&sealed, &aad)?;
2420                let version = u8::try_from(payload_version.unwrap_or(1)).map_err(|_| {
2421                    DurableError::Decode {
2422                        context: "payload_version out of u8 range",
2423                    }
2424                })?;
2425                EntryKind::StepResult {
2426                    idempotency_key: idem_key,
2427                    payload: opened,
2428                    effect,
2429                    payload_version: version,
2430                }
2431            }
2432            "effect_intent" => {
2433                let idem_bytes = idem_key.ok_or(DurableError::Decode {
2434                    context: "effect_intent idem_key missing",
2435                })?;
2436                let idem_key = IdempotencyKey::from_bytes(slice_to_array32(
2437                    &idem_bytes,
2438                    "effect_intent idem_key",
2439                )?);
2440                let effect = effect_class
2441                    .as_deref()
2442                    .and_then(crate::EffectClass::from_tag)
2443                    .ok_or(DurableError::Decode {
2444                        context: "effect_intent effect_class missing or invalid",
2445                    })?;
2446                let hmac = hmac
2447                    .map(|bytes| slice_to_array32(&bytes, "effect_intent hmac"))
2448                    .transpose()?;
2449                self.verify_control_hmac(
2450                    id,
2451                    step_id,
2452                    EntryKindTag::EffectIntent.as_str(),
2453                    Some(&idem_key),
2454                    hmac,
2455                )?;
2456                EntryKind::EffectIntent {
2457                    idempotency_key: idem_key,
2458                    effect,
2459                    hmac,
2460                }
2461            }
2462            "checkpoint" => self.checkpoint_entry(id, step_id, payload)?,
2463            other => {
2464                return Err(DurableError::UnsupportedEntryKind {
2465                    kind: static_entry_tag(other),
2466                });
2467            }
2468        };
2469        Ok(JournalEntry {
2470            seq: Some(JournalSeq::new(seq)),
2471            execution_id: id,
2472            kind,
2473            step_id,
2474            entry,
2475            created_at_ms: created_at,
2476        })
2477    }
2478
2479    /// Reconstruct a [`EntryKind::Checkpoint`] from a stored row, opening its sealed snapshot.
2480    ///
2481    /// `step_id` carries the checkpoint's `up_to_step` (the fold boundary); the snapshot is bound to
2482    /// it in the AAD so a checkpoint blob cannot be relocated to a different fold boundary.
2483    fn checkpoint_entry(
2484        &self,
2485        id: ExecutionId,
2486        step_id: StepId,
2487        payload: Option<Vec<u8>>,
2488    ) -> Result<EntryKind, DurableError> {
2489        let sealed = payload.ok_or(DurableError::Decode {
2490            context: "checkpoint entry missing snapshot payload",
2491        })?;
2492        ensure_payload_within_limit(
2493            sealed.len(),
2494            self.max_payload_bytes.saturating_add(SEAL_OVERHEAD_SLACK),
2495        )?;
2496        let aad = PayloadAad::new(id, step_id, EntryKindTag::Checkpoint, None);
2497        let snapshot = self.open_payload(&sealed, &aad)?;
2498        Ok(EntryKind::Checkpoint {
2499            up_to_step: step_id.value(),
2500            snapshot,
2501        })
2502    }
2503
2504    /// Reconstruct every entry from a fetched row set, sharing one kind lookup.
2505    async fn rows_to_entries(
2506        &self,
2507        id: ExecutionId,
2508        rows: Vec<JournalRowRead>,
2509    ) -> Result<Vec<JournalEntry>, DurableError> {
2510        if rows.is_empty() {
2511            return Ok(Vec::new());
2512        }
2513        let kind = self.lookup_kind(id).await?;
2514        let mut entries = Vec::with_capacity(rows.len());
2515        for row in rows {
2516            entries.push(self.row_to_entry(id, kind, row)?);
2517        }
2518        Ok(entries)
2519    }
2520}
2521
2522impl Journal for LocalBackend {
2523    async fn append(&self, entry: JournalEntry) -> Result<JournalSeq, DurableError> {
2524        let span = tracing::info_span!(
2525            "durable.journal.append",
2526            execution_id = %entry.execution_id.as_uuid(),
2527            step_id = entry.step_id.value(),
2528            entry_kind = entry.entry.tag(),
2529        );
2530        async move {
2531            let row = self.prepare_row(&entry)?;
2532            let insert = sql!(
2533                "INSERT INTO durable_journal
2534                    (execution_id, step_id, entry_kind, idem_key, effect_class, payload, payload_version, hmac, created_at)
2535                 VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)
2536                 RETURNING seq"
2537            );
2538            // A `StepResult` needs its HWM bump (issue #6360) committed atomically alongside the
2539            // INSERT, so it runs inside a transaction; every other entry kind keeps the direct
2540            // autocommit path (unchanged from before this feature).
2541            let seq: i64 = if matches!(entry.entry, EntryKind::StepResult { .. }) {
2542                let mut tx = zeph_db::begin_write(&self.pool)
2543                    .await
2544                    .map_err(|e| DurableError::storage("append", e))?;
2545                let (seq,): (i64,) = zeph_db::query_as(insert)
2546                    .bind(row.execution_id)
2547                    .bind(row.step_id)
2548                    .bind(row.entry_kind)
2549                    .bind(row.idem_key)
2550                    .bind(row.effect_class)
2551                    .bind(row.payload)
2552                    .bind(row.payload_version)
2553                    .bind(row.hmac)
2554                    .bind(row.created_at)
2555                    .fetch_one(&mut *tx)
2556                    .await
2557                    .map_err(|e| DurableError::storage("append", e))?;
2558                self.bump_hwm_for_step_result(&mut tx, entry.execution_id, entry.step_id)
2559                    .await?;
2560                tx.commit()
2561                    .await
2562                    .map_err(|e| DurableError::storage("append", e))?;
2563                seq
2564            } else {
2565                let (seq,): (i64,) = zeph_db::query_as(insert)
2566                    .bind(row.execution_id)
2567                    .bind(row.step_id)
2568                    .bind(row.entry_kind)
2569                    .bind(row.idem_key)
2570                    .bind(row.effect_class)
2571                    .bind(row.payload)
2572                    .bind(row.payload_version)
2573                    .bind(row.hmac)
2574                    .bind(row.created_at)
2575                    .fetch_one(&self.pool)
2576                    .await
2577                    .map_err(|e| DurableError::storage("append", e))?;
2578                seq
2579            };
2580            Ok(JournalSeq::new(seq))
2581        }
2582        .instrument(span)
2583        .await
2584    }
2585
2586    async fn read_execution(&self, id: ExecutionId) -> Result<Vec<JournalEntry>, DurableError> {
2587        let span = tracing::info_span!(
2588            "durable.journal.read",
2589            execution_id = %id.as_uuid(),
2590            step_count = tracing::field::Empty,
2591        );
2592        async move {
2593            let rows: Vec<JournalRowRead> = zeph_db::query_as(sql!(
2594                "SELECT seq, step_id, entry_kind, idem_key, effect_class, payload, payload_version, hmac, created_at
2595                 FROM durable_journal WHERE execution_id = ? ORDER BY seq"
2596            ))
2597            .bind(id.as_uuid().to_string())
2598            .fetch_all(&self.pool)
2599            .await
2600            .map_err(|e| DurableError::storage("read", e))?;
2601            let entries = self.rows_to_entries(id, rows).await?;
2602            tracing::Span::current().record("step_count", entries.len());
2603            Ok(entries)
2604        }
2605        .instrument(span)
2606        .await
2607    }
2608
2609    async fn read_execution_range(
2610        &self,
2611        id: ExecutionId,
2612        from_step_id: u32,
2613        limit: usize,
2614    ) -> Result<Vec<JournalEntry>, DurableError> {
2615        let span = tracing::info_span!(
2616            "durable.journal.read_segment",
2617            execution_id = %id.as_uuid(),
2618            from_step_id,
2619            count = tracing::field::Empty,
2620        );
2621        async move {
2622            let rows: Vec<JournalRowRead> = zeph_db::query_as(sql!(
2623                "SELECT seq, step_id, entry_kind, idem_key, effect_class, payload, payload_version, hmac, created_at
2624                 FROM durable_journal WHERE execution_id = ? AND step_id >= ? ORDER BY step_id, seq LIMIT ?"
2625            ))
2626            .bind(id.as_uuid().to_string())
2627            .bind(i64::from(from_step_id))
2628            .bind(i64::try_from(limit).unwrap_or(i64::MAX))
2629            .fetch_all(&self.pool)
2630            .await
2631            .map_err(|e| DurableError::storage("read_segment", e))?;
2632            let entries = self.rows_to_entries(id, rows).await?;
2633            tracing::Span::current().record("count", entries.len());
2634            Ok(entries)
2635        }
2636        .instrument(span)
2637        .await
2638    }
2639
2640    async fn finalize(&self, id: ExecutionId, status: ExecutionStatus) -> Result<(), DurableError> {
2641        let span = tracing::info_span!(
2642            "durable.journal.finalize",
2643            execution_id = %id.as_uuid(),
2644            status = status.as_str(),
2645        );
2646        async move {
2647            let now = now_unix_millis();
2648            let finalized_at = (!status.is_running()).then_some(now);
2649            let mut tx = zeph_db::begin_write(&self.pool)
2650                .await
2651                .map_err(|e| DurableError::storage("finalize", e))?;
2652            // `AND status = 'running'` makes this a one-shot transition: whichever of a concurrent
2653            // divergence-triggered `Aborted` or a caller's `Completed`/`Failed` commits first wins,
2654            // and the loser's UPDATE affects zero rows instead of clobbering the winner's terminal
2655            // status (finalize is otherwise safe to call more than once per execution).
2656            zeph_db::query(sql!(
2657                "UPDATE durable_executions SET status = ?, updated_at = ?, finalized_at = ?
2658                 WHERE execution_id = ? AND status = 'running'"
2659            ))
2660            .bind(status.as_str())
2661            .bind(now)
2662            .bind(finalized_at)
2663            .bind(id.as_uuid().to_string())
2664            .execute(&mut *tx)
2665            .await
2666            .map_err(|e| DurableError::storage("finalize", e))?;
2667            tx.commit()
2668                .await
2669                .map_err(|e| DurableError::storage("finalize", e))?;
2670            Ok(())
2671        }
2672        .instrument(span)
2673        .await
2674    }
2675
2676    async fn prune(&self, policy: &RetentionPolicy) -> Result<u64, DurableError> {
2677        let now = now_unix_millis();
2678        crate::retention::prune_in_batches(policy, now, |cutoffs, batch| {
2679            self.delete_prune_batch(cutoffs, batch)
2680        })
2681        .await
2682    }
2683
2684    /// Crash-orphan reclamation (INV-17, #6254). See [`Journal::sweep_orphans`] for the contract.
2685    async fn sweep_orphans(&self, policy: &RetentionPolicy) -> Result<u64, DurableError> {
2686        if policy.stale_running_after_secs == 0 {
2687            return Ok(0);
2688        }
2689        let Some(lock_dir) = self.lock_dir.clone() else {
2690            if !self
2691                .orphan_sweep_warned
2692                .swap(true, std::sync::atomic::Ordering::Relaxed)
2693            {
2694                tracing::warn!(
2695                    "durable: crash-orphan sweep requires an on-disk advisory-lock dir; orphan \
2696                     reclamation disabled for this backend (Postgres/:memory:/non-Unix)"
2697                );
2698            }
2699            return Ok(0);
2700        };
2701        let cutoff_ms = orphan_cutoff_ms(policy, now_unix_millis());
2702        crate::retention::sweep_orphans_in_batches(
2703            policy.prune_batch_size,
2704            cutoff_ms,
2705            |cutoff, batch, cursor| self.sweep_orphan_batch(&lock_dir, cutoff, batch, cursor),
2706        )
2707        .await
2708    }
2709}
2710
2711impl crate::sealed::Sealed for LocalBackend {}
2712
2713impl ExecutionBackend for LocalBackend {
2714    fn capabilities(&self) -> BackendCapabilities {
2715        BackendCapabilities {
2716            parallel_steps: true,
2717            // The local backend is in-process on SQLite; a Postgres build talks to a shared server.
2718            cross_process: cfg!(feature = "postgres"),
2719            max_payload: usize::try_from(self.max_payload_bytes).unwrap_or(usize::MAX),
2720        }
2721    }
2722
2723    async fn lookup_committed_result(
2724        &self,
2725        id: ExecutionId,
2726        idem_key: IdempotencyKey,
2727    ) -> Result<Option<JournalEntry>, DurableError> {
2728        LocalBackend::lookup_committed_result(self, id, idem_key).await
2729    }
2730}
2731
2732/// Column values for a single `durable_journal` row, ready to bind.
2733struct JournalRow {
2734    execution_id: String,
2735    step_id: i64,
2736    entry_kind: &'static str,
2737    idem_key: Option<Vec<u8>>,
2738    effect_class: Option<&'static str>,
2739    payload: Option<Vec<u8>>,
2740    payload_version: Option<i32>,
2741    hmac: Option<Vec<u8>>,
2742    created_at: i64,
2743}
2744
2745/// A `durable_journal` row read back from storage, decoded dialect-agnostically.
2746///
2747/// Columns are read as a positional tuple (the convention for crates that depend on `zeph-db` but
2748/// not `sqlx` directly, mirroring `zeph-scheduler`): integers decode as `i64`/`i32` and blobs as
2749/// `Vec<u8>`, which both backends satisfy through the same `sql!()`-rewritten query. The
2750/// field order matches the `SELECT` column list:
2751/// `(seq, step_id, entry_kind, idem_key, effect_class, payload, payload_version, hmac, created_at)`.
2752type JournalRowRead = (
2753    i64,
2754    i64,
2755    String,
2756    Option<Vec<u8>>,
2757    Option<String>,
2758    Option<Vec<u8>>,
2759    Option<i32>,
2760    Option<Vec<u8>>,
2761    i64,
2762);
2763
2764/// A `durable_journal` row read for [`LocalBackend::count_control_entries_under_previous_hmac`],
2765/// in `SELECT` column order: `(execution_id, step_id, idem_key, hmac)`.
2766type ControlHmacScanRow = (String, i64, Option<Vec<u8>>, Vec<u8>);
2767
2768/// A `durable_promises` row read back from storage, in `SELECT` column order:
2769/// `(execution_id, resolver_token_hash, resolved, payload)`.
2770type PromiseRowRead = (String, Vec<u8>, i64, Option<Vec<u8>>);
2771
2772/// A foldable `durable_journal` step-result row, in `SELECT` column order:
2773/// `(step_id, idem_key, payload_version, payload)`.
2774type FoldableRowRead = (i64, Option<Vec<u8>>, Option<i32>, Option<Vec<u8>>);
2775
2776/// Derive the per-execution lock directory sibling to a `path` passed to
2777/// [`LocalBackend::open`], `None` for `:memory:`.
2778///
2779/// Feature-gated to the `SQLite` backend only (INV-15): under `postgres`, `path` is a connection
2780/// URL that may embed credentials, and appending a suffix to mint a directory name would risk
2781/// creating a secret-bearing path component on disk.
2782#[cfg(feature = "sqlite")]
2783fn lock_dir_for_path(path: &str) -> Option<std::path::PathBuf> {
2784    (path != ":memory:").then(|| std::path::PathBuf::from(format!("{path}.locks")))
2785}
2786
2787#[cfg(not(feature = "sqlite"))]
2788fn lock_dir_for_path(_path: &str) -> Option<std::path::PathBuf> {
2789    None
2790}
2791
2792/// Regression coverage for the `postgres`-only branch of [`lock_dir_for_path`] (INV-15): a
2793/// connection URL — which may embed credentials — must never be used to mint an on-disk lock
2794/// directory name. The main `mod tests` block below is gated on `feature = "sqlite"` and so never
2795/// exercises this branch; run with `cargo nextest run -p zeph-durable --no-default-features
2796/// --features postgres`.
2797#[cfg(all(test, not(feature = "sqlite")))]
2798mod postgres_lock_dir_tests {
2799    use super::lock_dir_for_path;
2800
2801    #[test]
2802    fn postgres_url_never_derives_a_lock_dir() {
2803        assert_eq!(
2804            lock_dir_for_path("postgres://user:secret@host/db"),
2805            None,
2806            "a Postgres connection URL (which may embed credentials) must never be used to mint \
2807             an on-disk lock directory name"
2808        );
2809        assert_eq!(lock_dir_for_path(":memory:"), None);
2810    }
2811}
2812
2813/// Current Unix time in milliseconds, clamped into `i64` and never panicking.
2814pub(crate) fn now_unix_millis() -> i64 {
2815    SystemTime::now()
2816        .duration_since(UNIX_EPOCH)
2817        .map_or(0, |d| i64::try_from(d.as_millis()).unwrap_or(i64::MAX))
2818}
2819
2820/// Record `cancel_execution`'s `prior_status` span field for a write-path outcome.
2821///
2822/// `Canceled` was `running` immediately before this call (that is the only status the guarded
2823/// `UPDATE` matches); `AlreadyTerminal` already carries its own prior status. `NotFound` leaves
2824/// the field unset — there was no row to have had a status.
2825fn record_prior_status(outcome: CancelOutcome) {
2826    match outcome {
2827        CancelOutcome::Canceled => {
2828            tracing::Span::current().record("prior_status", "running");
2829        }
2830        CancelOutcome::AlreadyTerminal { status } => {
2831            tracing::Span::current().record("prior_status", status.as_str());
2832        }
2833        CancelOutcome::NotFound
2834        | CancelOutcome::LiveOwner { .. }
2835        | CancelOutcome::LivenessUnverifiable => {}
2836    }
2837}
2838
2839/// The absolute `updated_at` cutoff (Unix ms) at or before which a `status='running'` row becomes
2840/// a crash-orphan sweep candidate (INV-17, #6254).
2841fn orphan_cutoff_ms(policy: &RetentionPolicy, now_ms: i64) -> i64 {
2842    let threshold =
2843        i64::try_from(policy.stale_running_after_secs.saturating_mul(1000)).unwrap_or(i64::MAX);
2844    now_ms.saturating_sub(threshold)
2845}
2846
2847/// Decode a stored blob into a fixed 32-byte array, failing closed on the wrong length.
2848fn slice_to_array32(bytes: &[u8], field: &'static str) -> Result<[u8; 32], DurableError> {
2849    <[u8; 32]>::try_from(bytes).map_err(|_| DurableError::Decode { context: field })
2850}
2851
2852/// Parse a stored `execution_id` TEXT column back into an [`ExecutionId`], failing closed.
2853fn parse_execution_id(text: &str) -> Result<ExecutionId, DurableError> {
2854    uuid::Uuid::parse_str(text)
2855        .map(ExecutionId::from_uuid)
2856        .map_err(|_| DurableError::Decode {
2857            context: "execution_id is not a valid UUID",
2858        })
2859}
2860
2861/// Parse a stored `timer_id` TEXT column back into a [`TimerId`], failing closed.
2862fn parse_timer_id(text: &str) -> Result<TimerId, DurableError> {
2863    uuid::Uuid::parse_str(text)
2864        .map(TimerId::from_uuid)
2865        .map_err(|_| DurableError::Decode {
2866            context: "timer_id is not a valid UUID",
2867        })
2868}
2869
2870/// The AAD binding a promise's resolved payload to `(execution_id, promise_id)`.
2871///
2872/// A promise has no [`StepId`], so the promise id is folded into the AAD's idempotency-key slot:
2873/// a payload sealed for one promise cannot be opened as another's (fail-closed on relocation).
2874fn promise_payload_aad(execution_id: ExecutionId, promise_id: PromiseId) -> PayloadAad {
2875    let binding = IdempotencyKey::derive(
2876        execution_id,
2877        StepId::new(0),
2878        promise_id.as_uuid().as_bytes(),
2879    );
2880    PayloadAad::new(
2881        execution_id,
2882        StepId::new(0),
2883        EntryKindTag::PromiseResolved,
2884        Some(binding),
2885    )
2886}
2887
2888/// Map a database `entry_kind` string to a `'static` tag for [`DurableError::UnsupportedEntryKind`].
2889fn static_entry_tag(tag: &str) -> &'static str {
2890    match tag {
2891        "promise_created" => "promise_created",
2892        "promise_resolved" => "promise_resolved",
2893        "timer_armed" => "timer_armed",
2894        "timer_fired" => "timer_fired",
2895        "checkpoint" => "checkpoint",
2896        _ => "unknown",
2897    }
2898}
2899
2900// Backend tests open a real pool, so they run under the SQLite build (mirroring `zeph-scheduler`,
2901// whose `:memory:` pool is SQLite-specific). The dialect-agnostic `sql!()` SQL and `i64`/`Vec<u8>`
2902// column types are verified to compile under the Postgres feature; live Postgres parity is exercised
2903// by the `#[ignore]`d integration test below.
2904#[cfg(all(test, feature = "sqlite"))]
2905mod tests {
2906    use std::assert_matches;
2907
2908    use super::*;
2909    use crate::cipher::CipherError;
2910    use crate::effect::EffectClass;
2911
2912    /// An AAD-authenticated test cipher: a BLAKE3 tag over the AAD prefixes an XOR-masked payload,
2913    /// so opening with a relocated/forged AAD fails authentication exactly like the real cipher.
2914    struct XorCipher;
2915    const XOR_MASK: u8 = 0x5A;
2916
2917    impl PayloadCipher for XorCipher {
2918        fn seal(&self, plaintext: &[u8], aad: &PayloadAad) -> Result<Vec<u8>, CipherError> {
2919            let tag = blake3::hash(&aad.canonical_bytes());
2920            let mut out = tag.as_bytes()[..8].to_vec();
2921            out.extend(plaintext.iter().map(|b| b ^ XOR_MASK));
2922            Ok(out)
2923        }
2924
2925        fn open(&self, sealed: &[u8], aad: &PayloadAad) -> Result<Vec<u8>, CipherError> {
2926            if sealed.len() < 8 {
2927                return Err(CipherError::Malformed {
2928                    context: "sealed blob shorter than the aad tag",
2929                });
2930            }
2931            let expected = blake3::hash(&aad.canonical_bytes());
2932            if sealed[..8] != expected.as_bytes()[..8] {
2933                return Err(CipherError::Authentication);
2934            }
2935            Ok(sealed[8..].iter().map(|b| b ^ XOR_MASK).collect())
2936        }
2937    }
2938
2939    /// A test cipher double that embeds a `key_id` leading byte (mirroring the production
2940    /// `key_id(1) || nonce || ciphertext || tag` contract documented on [`PayloadCipher`]) and,
2941    /// like the real `XChaCha20Poly1305Cipher::with_previous`, can still decrypt a payload sealed
2942    /// under a registered `previous_id` while always sealing new writes under `current_id` — the
2943    /// minimal shape needed to exercise `checkpoint_fold`'s reseal-under-current behavior across a
2944    /// simulated rotation window, without depending on the real AEAD cipher (out of scope for
2945    /// `zeph-durable`, INV-1).
2946    struct RotatingKeyedCipher {
2947        current_id: u8,
2948        previous_id: Option<u8>,
2949    }
2950
2951    impl PayloadCipher for RotatingKeyedCipher {
2952        fn seal(&self, plaintext: &[u8], aad: &PayloadAad) -> Result<Vec<u8>, CipherError> {
2953            let tag = blake3::hash(&aad.canonical_bytes());
2954            let mut out = vec![self.current_id];
2955            out.extend_from_slice(&tag.as_bytes()[..8]);
2956            out.extend(plaintext.iter().map(|b| b ^ XOR_MASK));
2957            Ok(out)
2958        }
2959
2960        fn open(&self, sealed: &[u8], aad: &PayloadAad) -> Result<Vec<u8>, CipherError> {
2961            if sealed.len() < 9 {
2962                return Err(CipherError::Malformed {
2963                    context: "sealed blob shorter than the key-id + aad tag prefix",
2964                });
2965            }
2966            let id = sealed[0];
2967            if id != self.current_id && Some(id) != self.previous_id {
2968                return Err(CipherError::UnknownKeyId { key_id: id });
2969            }
2970            let expected = blake3::hash(&aad.canonical_bytes());
2971            if sealed[1..9] != expected.as_bytes()[..8] {
2972                return Err(CipherError::Authentication);
2973            }
2974            Ok(sealed[9..].iter().map(|b| b ^ XOR_MASK).collect())
2975        }
2976    }
2977
2978    async fn mem_backend(max_payload_bytes: u64) -> LocalBackend {
2979        let backend = LocalBackend::open(":memory:", max_payload_bytes)
2980            .await
2981            .expect("open in-memory backend");
2982        backend.init().await.expect("apply migrations");
2983        backend
2984    }
2985
2986    fn step_result(exec: ExecutionId, step: u32, payload: &[u8]) -> JournalEntry {
2987        let step_id = StepId::new(step);
2988        JournalEntry {
2989            seq: None,
2990            execution_id: exec,
2991            kind: ExecutionKind::AgentTurn,
2992            step_id,
2993            entry: EntryKind::StepResult {
2994                idempotency_key: IdempotencyKey::derive(exec, step_id, b"tool:read"),
2995                payload: Bytes::copy_from_slice(payload),
2996                effect: EffectClass::Idempotent,
2997                payload_version: 1,
2998            },
2999            created_at_ms: 100,
3000        }
3001    }
3002
3003    fn effect_intent(exec: ExecutionId, step: u32) -> JournalEntry {
3004        let step_id = StepId::new(step);
3005        JournalEntry {
3006            seq: None,
3007            execution_id: exec,
3008            kind: ExecutionKind::AgentTurn,
3009            step_id,
3010            entry: EntryKind::EffectIntent {
3011                idempotency_key: IdempotencyKey::derive(exec, step_id, b"transfer"),
3012                effect: EffectClass::ExactlyOnceGuarded,
3013                hmac: None,
3014            },
3015            created_at_ms: 100,
3016        }
3017    }
3018
3019    /// Regression for #6447: `count_sealed_under_key_id` scans `durable_journal.payload` by its
3020    /// leading byte, ignores control entries (`payload IS NULL`), and never matches an unrelated
3021    /// key-id. No cipher is attached, so `seal_payload` stores the plaintext verbatim (local.rs
3022    /// `seal_payload`'s `None => Ok(plaintext.to_vec())` branch) — the first byte of the crafted
3023    /// payload lands on disk unchanged, letting the test control it directly without depending on
3024    /// the real AEAD cipher (out of scope for `zeph-durable`, INV-1).
3025    #[tokio::test]
3026    async fn count_sealed_under_key_id_counts_matching_journal_rows_and_excludes_control_entries() {
3027        let backend = mem_backend(1_048_576).await;
3028        let exec = ExecutionId::new();
3029        backend
3030            .open_execution(exec, ExecutionKind::AgentTurn)
3031            .await
3032            .unwrap();
3033
3034        backend
3035            .append(step_result(exec, 0, &[5, 0, 0]))
3036            .await
3037            .unwrap();
3038        backend
3039            .append(step_result(exec, 1, &[6, 0, 0]))
3040            .await
3041            .unwrap();
3042        // A control entry carries no payload and must never be counted, regardless of key_id.
3043        backend.append(effect_intent(exec, 2)).await.unwrap();
3044
3045        assert_eq!(backend.count_sealed_under_key_id(5).await.unwrap(), 1);
3046        assert_eq!(backend.count_sealed_under_key_id(6).await.unwrap(), 1);
3047        assert_eq!(backend.count_sealed_under_key_id(7).await.unwrap(), 0);
3048    }
3049
3050    /// Regression for #6447: the scan also covers `durable_promises.payload`, not just the
3051    /// journal — a promise resolved under the previous key must count too, or `--drop-previous`
3052    /// could silently orphan it.
3053    #[tokio::test]
3054    async fn count_sealed_under_key_id_counts_matching_promise_rows() {
3055        let backend = mem_backend(1_048_576).await;
3056        let exec = ExecutionId::new();
3057        backend
3058            .open_execution(exec, ExecutionKind::AgentTurn)
3059            .await
3060            .unwrap();
3061        let promise_id = PromiseId::new();
3062        backend
3063            .insert_promise(promise_id, exec, [0u8; 32], 100)
3064            .await
3065            .unwrap();
3066        // Unresolved promise row: payload is still NULL, must not be counted.
3067        assert_eq!(backend.count_sealed_under_key_id(9).await.unwrap(), 0);
3068
3069        backend
3070            .resolve_promise(promise_id, exec, &[9, 1, 2, 3], 200)
3071            .await
3072            .unwrap();
3073
3074        assert_eq!(backend.count_sealed_under_key_id(9).await.unwrap(), 1);
3075        assert_eq!(backend.count_sealed_under_key_id(10).await.unwrap(), 0);
3076    }
3077
3078    #[tokio::test]
3079    async fn open_execution_is_fresh_then_resume() {
3080        let backend = mem_backend(1_048_576).await;
3081        let exec = ExecutionId::new();
3082        assert!(
3083            !backend
3084                .open_execution(exec, ExecutionKind::AgentTurn)
3085                .await
3086                .unwrap()
3087        );
3088        assert!(
3089            backend
3090                .open_execution(exec, ExecutionKind::AgentTurn)
3091                .await
3092                .unwrap()
3093        );
3094    }
3095
3096    #[tokio::test]
3097    async fn open_execution_exclusive_is_fresh_then_resume() {
3098        // A file-backed (not `:memory:`) backend is required: only `LocalBackend::open` with a
3099        // real on-disk path derives a `lock_dir` (#6122).
3100        let dir = tempfile::tempdir().unwrap();
3101        let db_path = dir.path().join("durable.db");
3102        let backend = LocalBackend::open(&db_path.to_string_lossy(), 1_048_576)
3103            .await
3104            .unwrap();
3105        backend.init().await.unwrap();
3106
3107        let exec = ExecutionId::new();
3108        let (is_resume, lock) = backend
3109            .open_execution_exclusive(exec, ExecutionKind::AgentTurn)
3110            .await
3111            .unwrap();
3112        assert!(!is_resume);
3113        assert!(lock.is_some(), "a file-backed backend must derive a lock");
3114        drop(lock);
3115
3116        let (is_resume, _lock) = backend
3117            .open_execution_exclusive(exec, ExecutionKind::AgentTurn)
3118            .await
3119            .unwrap();
3120        assert!(is_resume);
3121    }
3122
3123    /// Regression test for #6122: two `LocalBackend` handles onto the same on-disk journal (as
3124    /// two independent CLI processes sharing `memory.sqlite_path` would each construct) must not
3125    /// both be able to hold `open_execution_exclusive` for the same colliding `ExecutionId`
3126    /// concurrently.
3127    #[tokio::test]
3128    async fn open_execution_exclusive_rejects_concurrent_second_holder() {
3129        let dir = tempfile::tempdir().unwrap();
3130        let db_path = dir.path().join("durable.db");
3131        let url = db_path.to_string_lossy().into_owned();
3132
3133        let backend_a = LocalBackend::open(&url, 1_048_576).await.unwrap();
3134        backend_a.init().await.unwrap();
3135        let backend_b = LocalBackend::open(&url, 1_048_576).await.unwrap();
3136
3137        let exec = ExecutionId::new();
3138        let (_, _lock_a) = backend_a
3139            .open_execution_exclusive(exec, ExecutionKind::AgentTurn)
3140            .await
3141            .unwrap();
3142
3143        let err = backend_b
3144            .open_execution_exclusive(exec, ExecutionKind::AgentTurn)
3145            .await
3146            .expect_err("a second concurrent holder must be rejected");
3147        assert!(
3148            matches!(err, DurableError::ExecutionLocked { execution_id, .. } if execution_id == exec),
3149            "expected ExecutionLocked, got {err:?}"
3150        );
3151    }
3152
3153    #[tokio::test]
3154    async fn open_execution_exclusive_on_memory_backend_returns_no_lock() {
3155        // `:memory:` has no on-disk directory to lock, so it degrades to unenforced exclusivity —
3156        // consistent with `SessionEventLog::open_exclusive`'s non-Unix degrade.
3157        let backend = mem_backend(1_048_576).await;
3158        let exec = ExecutionId::new();
3159        let (is_resume, lock) = backend
3160            .open_execution_exclusive(exec, ExecutionKind::AgentTurn)
3161            .await
3162            .unwrap();
3163        assert!(!is_resume);
3164        assert!(lock.is_none());
3165    }
3166
3167    #[tokio::test]
3168    async fn list_executions_summarizes_and_filters() {
3169        let backend = mem_backend(1_048_576).await;
3170        let turn = ExecutionId::new();
3171        let dag = ExecutionId::new();
3172        backend
3173            .open_execution(turn, ExecutionKind::AgentTurn)
3174            .await
3175            .unwrap();
3176        backend
3177            .open_execution(dag, ExecutionKind::DagRun)
3178            .await
3179            .unwrap();
3180        backend.append(step_result(turn, 0, b"a")).await.unwrap();
3181        backend.append(step_result(turn, 1, b"b")).await.unwrap();
3182        backend.append(step_result(dag, 0, b"c")).await.unwrap();
3183        backend
3184            .finalize(turn, ExecutionStatus::Completed)
3185            .await
3186            .unwrap();
3187
3188        // Unfiltered: both executions with their per-execution step counts.
3189        let all = backend.list_executions(None, None, 10).await.unwrap();
3190        assert_eq!(all.len(), 2);
3191
3192        let turn_row = all
3193            .iter()
3194            .find(|e| e.execution_id == turn)
3195            .expect("turn present");
3196        assert_eq!(turn_row.kind, "agent_turn");
3197        assert_eq!(turn_row.status, ExecutionStatus::Completed);
3198        assert_eq!(turn_row.step_count, 2);
3199        assert!(turn_row.finalized_at_ms.is_some());
3200
3201        let dag_row = all
3202            .iter()
3203            .find(|e| e.execution_id == dag)
3204            .expect("dag present");
3205        assert_eq!(dag_row.status, ExecutionStatus::Running);
3206        assert_eq!(dag_row.step_count, 1);
3207        assert!(dag_row.finalized_at_ms.is_none());
3208
3209        // Status filter narrows to the still-running execution.
3210        let running = backend
3211            .list_executions(Some("running"), None, 10)
3212            .await
3213            .unwrap();
3214        assert_eq!(running.len(), 1);
3215        assert_eq!(running[0].execution_id, dag);
3216
3217        // Kind filter narrows to the DAG execution.
3218        let dags = backend
3219            .list_executions(None, Some("dag_run"), 10)
3220            .await
3221            .unwrap();
3222        assert_eq!(dags.len(), 1);
3223        assert_eq!(dags[0].execution_id, dag);
3224
3225        // Limit caps the result set.
3226        let one = backend.list_executions(None, None, 1).await.unwrap();
3227        assert_eq!(one.len(), 1);
3228    }
3229
3230    #[tokio::test]
3231    async fn append_and_read_round_trips_step_result() {
3232        let backend = mem_backend(1_048_576).await;
3233        let exec = ExecutionId::new();
3234        backend
3235            .open_execution(exec, ExecutionKind::AgentTurn)
3236            .await
3237            .unwrap();
3238
3239        let seq = backend
3240            .append(step_result(exec, 0, b"hello"))
3241            .await
3242            .unwrap();
3243        assert_eq!(seq.value(), 1, "first append takes seq 1");
3244
3245        let entries = backend.read_execution(exec).await.unwrap();
3246        assert_eq!(entries.len(), 1);
3247        match &entries[0].entry {
3248            EntryKind::StepResult {
3249                payload, effect, ..
3250            } => {
3251                assert_eq!(payload.as_ref(), b"hello");
3252                assert_eq!(*effect, EffectClass::Idempotent);
3253            }
3254            other => panic!("unexpected entry kind: {other:?}"),
3255        }
3256        assert_eq!(entries[0].seq, Some(seq));
3257    }
3258
3259    #[tokio::test]
3260    async fn cipher_seals_payload_at_rest_but_round_trips() {
3261        let backend = mem_backend(1_048_576)
3262            .await
3263            .with_cipher(Arc::new(XorCipher));
3264        let exec = ExecutionId::new();
3265        backend
3266            .open_execution(exec, ExecutionKind::AgentTurn)
3267            .await
3268            .unwrap();
3269        backend
3270            .append(step_result(exec, 0, b"secret-payload"))
3271            .await
3272            .unwrap();
3273
3274        // The stored column is sealed, never the plaintext.
3275        let (stored,): (Option<Vec<u8>>,) = zeph_db::query_as(sql!(
3276            "SELECT payload FROM durable_journal WHERE execution_id = ?"
3277        ))
3278        .bind(exec.as_uuid().to_string())
3279        .fetch_one(backend.pool())
3280        .await
3281        .unwrap();
3282        let stored = stored.expect("payload present");
3283        assert_ne!(
3284            stored.as_slice(),
3285            b"secret-payload",
3286            "payload must be sealed at rest"
3287        );
3288
3289        // Reading opens it back to the original plaintext.
3290        let entries = backend.read_execution(exec).await.unwrap();
3291        match &entries[0].entry {
3292            EntryKind::StepResult { payload, .. } => {
3293                assert_eq!(payload.as_ref(), b"secret-payload");
3294            }
3295            other => panic!("unexpected entry kind: {other:?}"),
3296        }
3297    }
3298
3299    #[tokio::test]
3300    async fn control_entry_hmac_is_stamped_only_when_keyed() {
3301        let exec = ExecutionId::new();
3302
3303        let unkeyed = mem_backend(1_048_576).await;
3304        unkeyed
3305            .open_execution(exec, ExecutionKind::AgentTurn)
3306            .await
3307            .unwrap();
3308        unkeyed.append(effect_intent(exec, 0)).await.unwrap();
3309        match &unkeyed.read_execution(exec).await.unwrap()[0].entry {
3310            EntryKind::EffectIntent { hmac, .. } => assert!(hmac.is_none()),
3311            other => panic!("unexpected entry kind: {other:?}"),
3312        }
3313
3314        let keyed = mem_backend(1_048_576).await.with_hmac_key([7u8; 32]);
3315        let exec2 = ExecutionId::new();
3316        keyed
3317            .open_execution(exec2, ExecutionKind::AgentTurn)
3318            .await
3319            .unwrap();
3320        keyed.append(effect_intent(exec2, 0)).await.unwrap();
3321        match &keyed.read_execution(exec2).await.unwrap()[0].entry {
3322            EntryKind::EffectIntent { hmac, .. } => {
3323                assert!(
3324                    hmac.is_some(),
3325                    "keyed backend stamps a row HMAC over control entries"
3326                );
3327            }
3328            other => panic!("unexpected entry kind: {other:?}"),
3329        }
3330    }
3331
3332    /// Regression for #6043/#6044: a control entry written under one HMAC key must fail closed
3333    /// with [`DurableError::ControlIntegrity`] when read back under a *different* key — the
3334    /// forged/relocated-row rejection the row HMAC exists to provide. Both backends share the
3335    /// same underlying pool (a second `LocalBackend` handle over the same connection), so this
3336    /// exercises the read path's recompute-and-compare, not just a difference in whether a key is
3337    /// configured at all.
3338    #[tokio::test]
3339    async fn read_execution_rejects_control_hmac_under_wrong_key() {
3340        let writer = mem_backend(1_048_576).await.with_hmac_key([1u8; 32]);
3341        let exec = ExecutionId::new();
3342        writer
3343            .open_execution(exec, ExecutionKind::AgentTurn)
3344            .await
3345            .unwrap();
3346        writer.append(effect_intent(exec, 0)).await.unwrap();
3347
3348        let wrong_key_reader =
3349            LocalBackend::new(writer.pool().clone(), 1_048_576).with_hmac_key([2u8; 32]);
3350        assert_matches!(
3351            wrong_key_reader.read_execution(exec).await,
3352            Err(DurableError::ControlIntegrity)
3353        );
3354
3355        // Reading under the correct key still succeeds.
3356        let right_key_reader =
3357            LocalBackend::new(writer.pool().clone(), 1_048_576).with_hmac_key([1u8; 32]);
3358        assert!(right_key_reader.read_execution(exec).await.is_ok());
3359    }
3360
3361    /// Regression for #6043/#6044: a control entry written by an *unkeyed* backend (`hmac =
3362    /// NULL`) must fail closed when later read by a keyed backend — a keyed backend enforces that
3363    /// every control row it reads carries a matching HMAC, so a missing HMAC is treated the same
3364    /// as a mismatched one rather than silently passing through unverified.
3365    #[tokio::test]
3366    async fn read_execution_rejects_missing_hmac_on_keyed_backend() {
3367        let writer = mem_backend(1_048_576).await;
3368        let exec = ExecutionId::new();
3369        writer
3370            .open_execution(exec, ExecutionKind::AgentTurn)
3371            .await
3372            .unwrap();
3373        writer.append(effect_intent(exec, 0)).await.unwrap();
3374
3375        let keyed_reader =
3376            LocalBackend::new(writer.pool().clone(), 1_048_576).with_hmac_key([3u8; 32]);
3377        assert_matches!(
3378            keyed_reader.read_execution(exec).await,
3379            Err(DurableError::ControlIntegrity)
3380        );
3381    }
3382
3383    /// Regression for #6043/#6044 (review S1): a control entry written by a *keyed* backend must
3384    /// fail closed when later read by an *unkeyed* backend, rather than silently trusting the
3385    /// stamped HMAC as an ordinary (unverified) plaintext field. Before this fix,
3386    /// `verify_control_hmac` returned `Ok(())` unconditionally whenever the reader had no HMAC
3387    /// key, regardless of whether the stored row carried one — so config drift between a keyed
3388    /// writer and an unkeyed reader over the same physical file (e.g. `shared_db` toggled, or a
3389    /// reader whose config disagrees with the writer's) let a stamped row through unverified,
3390    /// which is exactly the forgery-acceptance gap #6043 says the row HMAC closes.
3391    #[tokio::test]
3392    async fn read_execution_rejects_stamped_hmac_on_unkeyed_backend() {
3393        let writer = mem_backend(1_048_576).await.with_hmac_key([4u8; 32]);
3394        let exec = ExecutionId::new();
3395        writer
3396            .open_execution(exec, ExecutionKind::AgentTurn)
3397            .await
3398            .unwrap();
3399        writer.append(effect_intent(exec, 0)).await.unwrap();
3400
3401        let unkeyed_reader = LocalBackend::new(writer.pool().clone(), 1_048_576);
3402        assert_matches!(
3403            unkeyed_reader.read_execution(exec).await,
3404            Err(DurableError::ControlIntegrity)
3405        );
3406    }
3407
3408    /// #6451: a control entry written under the pre-rotation key must still verify once a reader
3409    /// registers that key as `previous_hmac_key`, even though its own `hmac_key` has moved on to
3410    /// the new (post-rotation) key — the try-both rotation window, symmetric to the AEAD cipher's
3411    /// `with_previous`. This is also the payload-less "crash-orphan" shape the drop-scan exists
3412    /// for: `effect_intent` entries never carry a payload, so this row has a previous-key HMAC
3413    /// with nothing for the AEAD blob-scan to see.
3414    #[tokio::test]
3415    async fn verify_control_hmac_accepts_row_under_previous_key_during_window() {
3416        let writer = mem_backend(1_048_576).await.with_hmac_key([1u8; 32]);
3417        let exec = ExecutionId::new();
3418        writer
3419            .open_execution(exec, ExecutionKind::AgentTurn)
3420            .await
3421            .unwrap();
3422        writer.append(effect_intent(exec, 0)).await.unwrap();
3423
3424        // Post-rotation reader: current key is the new key [2u8; 32], previous is the pre-rotation
3425        // key [1u8; 32] that actually stamped the row.
3426        let post_rotation_reader = LocalBackend::new(writer.pool().clone(), 1_048_576)
3427            .with_hmac_key([2u8; 32])
3428            .with_previous_hmac_key([1u8; 32]);
3429        assert!(
3430            post_rotation_reader.read_execution(exec).await.is_ok(),
3431            "a row stamped under the previous key must verify during the rotation window"
3432        );
3433
3434        // A fresh row written by the post-rotation writer stamps under the current key only, and
3435        // must verify without needing the previous slot.
3436        let post_rotation_writer = LocalBackend::new(writer.pool().clone(), 1_048_576)
3437            .with_hmac_key([2u8; 32])
3438            .with_previous_hmac_key([1u8; 32]);
3439        post_rotation_writer
3440            .append(effect_intent(exec, 1))
3441            .await
3442            .unwrap();
3443        assert!(post_rotation_writer.read_execution(exec).await.is_ok());
3444    }
3445
3446    /// #6451: a row that matches neither the current nor the registered previous key must still
3447    /// fail closed — the rotation window widens acceptance to exactly two legitimate keys, never
3448    /// to "any key".
3449    #[tokio::test]
3450    async fn verify_control_hmac_rejects_row_under_neither_current_nor_previous_key() {
3451        let writer = mem_backend(1_048_576).await.with_hmac_key([9u8; 32]);
3452        let exec = ExecutionId::new();
3453        writer
3454            .open_execution(exec, ExecutionKind::AgentTurn)
3455            .await
3456            .unwrap();
3457        writer.append(effect_intent(exec, 0)).await.unwrap();
3458
3459        let unrelated_reader = LocalBackend::new(writer.pool().clone(), 1_048_576)
3460            .with_hmac_key([2u8; 32])
3461            .with_previous_hmac_key([3u8; 32]);
3462        assert_matches!(
3463            unrelated_reader.read_execution(exec).await,
3464            Err(DurableError::ControlIntegrity)
3465        );
3466    }
3467
3468    /// #6451: `count_control_entries_under_previous_hmac` is the drop-scan gate for
3469    /// `--drop-previous` — it must count a row that verifies only under the previous key, and
3470    /// must not count a row that still verifies under the current key (no false refusal once the
3471    /// row has actually been re-keyed).
3472    #[tokio::test]
3473    async fn count_control_entries_under_previous_hmac_counts_previous_only_rows() {
3474        let writer = mem_backend(1_048_576).await.with_hmac_key([1u8; 32]);
3475        let exec = ExecutionId::new();
3476        writer
3477            .open_execution(exec, ExecutionKind::AgentTurn)
3478            .await
3479            .unwrap();
3480        // Pre-rotation row: stamped under [1u8; 32], the soon-to-be-previous key.
3481        writer.append(effect_intent(exec, 0)).await.unwrap();
3482
3483        let scanner_mid_window = LocalBackend::new(writer.pool().clone(), 1_048_576)
3484            .with_hmac_key([2u8; 32])
3485            .with_previous_hmac_key([1u8; 32]);
3486        assert_eq!(
3487            scanner_mid_window
3488                .count_control_entries_under_previous_hmac()
3489                .await
3490                .unwrap(),
3491            1,
3492            "a row stamped under the previous key only must be counted"
3493        );
3494
3495        // A post-rotation row, stamped under the new current key, must not be counted.
3496        let post_rotation_writer = LocalBackend::new(writer.pool().clone(), 1_048_576)
3497            .with_hmac_key([2u8; 32])
3498            .with_previous_hmac_key([1u8; 32]);
3499        post_rotation_writer
3500            .append(effect_intent(exec, 1))
3501            .await
3502            .unwrap();
3503        assert_eq!(
3504            post_rotation_writer
3505                .count_control_entries_under_previous_hmac()
3506                .await
3507                .unwrap(),
3508            1,
3509            "the post-rotation row (verifies under current) must not add to the count"
3510        );
3511    }
3512
3513    /// #6451: once every previous-key row has been superseded (or there were none), the scan
3514    /// reports zero without requiring any rows to exist at all — the clean `--drop-previous`
3515    /// no-op/success path.
3516    #[tokio::test]
3517    async fn count_control_entries_under_previous_hmac_is_zero_on_empty_journal() {
3518        let backend = mem_backend(1_048_576).await;
3519        assert_eq!(
3520            backend
3521                .count_control_entries_under_previous_hmac()
3522                .await
3523                .unwrap(),
3524            0
3525        );
3526    }
3527
3528    /// #6451 critic finding 1: the scan cannot be trusted without both keys attached — a caller
3529    /// that opens the backend unkeyed (as the pre-fix `--drop-previous` scan site did) must get a
3530    /// hard error, not a silently-wrong count that could let `--drop-previous` refuse forever (or
3531    /// worse, proceed unsafely).
3532    #[tokio::test]
3533    async fn count_control_entries_under_previous_hmac_errors_when_keys_missing() {
3534        let writer = mem_backend(1_048_576).await.with_hmac_key([1u8; 32]);
3535        let exec = ExecutionId::new();
3536        writer
3537            .open_execution(exec, ExecutionKind::AgentTurn)
3538            .await
3539            .unwrap();
3540        writer.append(effect_intent(exec, 0)).await.unwrap();
3541
3542        let unkeyed_scanner = LocalBackend::new(writer.pool().clone(), 1_048_576);
3543        assert_matches!(
3544            unkeyed_scanner
3545                .count_control_entries_under_previous_hmac()
3546                .await,
3547            Err(DurableError::ControlIntegrity)
3548        );
3549
3550        let current_only_scanner =
3551            LocalBackend::new(writer.pool().clone(), 1_048_576).with_hmac_key([1u8; 32]);
3552        assert_matches!(
3553            current_only_scanner
3554                .count_control_entries_under_previous_hmac()
3555                .await,
3556            Err(DurableError::ControlIntegrity)
3557        );
3558    }
3559
3560    #[tokio::test]
3561    async fn promise_and_timer_entries_fail_closed() {
3562        let backend = mem_backend(1_048_576).await;
3563        let exec = ExecutionId::new();
3564        backend
3565            .open_execution(exec, ExecutionKind::AgentTurn)
3566            .await
3567            .unwrap();
3568        let timer = JournalEntry {
3569            seq: None,
3570            execution_id: exec,
3571            kind: ExecutionKind::AgentTurn,
3572            step_id: StepId::new(0),
3573            entry: EntryKind::TimerArmed {
3574                timer_id: crate::TimerId::new(),
3575                due_at_ms: 1_000,
3576                hmac: None,
3577            },
3578            created_at_ms: 0,
3579        };
3580        assert_matches!(
3581            backend.append(timer).await,
3582            Err(DurableError::UnsupportedEntryKind {
3583                kind: "timer_armed"
3584            })
3585        );
3586    }
3587
3588    #[tokio::test]
3589    async fn payload_over_limit_is_rejected_fail_closed() {
3590        let backend = mem_backend(8).await;
3591        let exec = ExecutionId::new();
3592        backend
3593            .open_execution(exec, ExecutionKind::AgentTurn)
3594            .await
3595            .unwrap();
3596        let big = vec![0u8; 64];
3597        assert_matches!(
3598            backend.append(step_result(exec, 0, &big)).await,
3599            Err(DurableError::PayloadTooLarge { .. })
3600        );
3601    }
3602
3603    #[tokio::test]
3604    async fn finalize_marks_terminal_status_and_time() {
3605        let backend = mem_backend(1_048_576).await;
3606        let exec = ExecutionId::new();
3607        backend
3608            .open_execution(exec, ExecutionKind::AgentTurn)
3609            .await
3610            .unwrap();
3611        backend
3612            .finalize(exec, ExecutionStatus::Completed)
3613            .await
3614            .unwrap();
3615
3616        let (status, finalized): (String, Option<i64>) = zeph_db::query_as(sql!(
3617            "SELECT status, finalized_at FROM durable_executions WHERE execution_id = ?"
3618        ))
3619        .bind(exec.as_uuid().to_string())
3620        .fetch_one(backend.pool())
3621        .await
3622        .unwrap();
3623        assert_eq!(status, "completed");
3624        assert!(finalized.is_some(), "a terminal status stamps finalized_at");
3625    }
3626
3627    #[tokio::test]
3628    async fn finalize_is_a_noop_once_already_terminal() {
3629        // #6251: finalize must be safe to call more than once (e.g. a caller's own `Completed`
3630        // racing the divergence guard's `Aborted`) — whichever status lands first wins.
3631        let backend = mem_backend(1_048_576).await;
3632        let exec = ExecutionId::new();
3633        backend
3634            .open_execution(exec, ExecutionKind::AgentTurn)
3635            .await
3636            .unwrap();
3637        backend
3638            .finalize(exec, ExecutionStatus::Completed)
3639            .await
3640            .unwrap();
3641
3642        // A later call with a different terminal status must not overwrite the first.
3643        backend
3644            .finalize(exec, ExecutionStatus::Failed)
3645            .await
3646            .unwrap();
3647
3648        let (status,): (String,) = zeph_db::query_as(sql!(
3649            "SELECT status FROM durable_executions WHERE execution_id = ?"
3650        ))
3651        .bind(exec.as_uuid().to_string())
3652        .fetch_one(backend.pool())
3653        .await
3654        .unwrap();
3655        assert_eq!(
3656            status, "completed",
3657            "the first terminal status must stick; a later finalize call is a no-op"
3658        );
3659    }
3660
3661    #[tokio::test]
3662    async fn finalize_after_abort_is_a_noop() {
3663        // #6251: the reverse direction of the divergence race — the internal `Aborted` transition
3664        // (replay-divergence guard) commits first, so a consumer's later own `Completed`/`Failed`
3665        // call must be a no-op rather than resurrecting the row out of its aborted state.
3666        let backend = mem_backend(1_048_576).await;
3667        let exec = ExecutionId::new();
3668        backend
3669            .open_execution(exec, ExecutionKind::AgentTurn)
3670            .await
3671            .unwrap();
3672        backend
3673            .finalize(exec, ExecutionStatus::Aborted)
3674            .await
3675            .unwrap();
3676
3677        backend
3678            .finalize(exec, ExecutionStatus::Completed)
3679            .await
3680            .unwrap();
3681
3682        let (status,): (String,) = zeph_db::query_as(sql!(
3683            "SELECT status FROM durable_executions WHERE execution_id = ?"
3684        ))
3685        .bind(exec.as_uuid().to_string())
3686        .fetch_one(backend.pool())
3687        .await
3688        .unwrap();
3689        assert_eq!(
3690            status, "aborted",
3691            "an aborted execution must not be overwritten by a later Completed/Failed call"
3692        );
3693    }
3694
3695    #[tokio::test]
3696    async fn reopening_a_finalized_execution_resets_it_to_running() {
3697        // #6251: a finalized execution that is legitimately reopened (e.g. a resumed conversation)
3698        // must not keep a stale `finalized_at` — otherwise the retention sweep could prune a row
3699        // that is still receiving new journal writes.
3700        let backend = mem_backend(1_048_576).await;
3701        let exec = ExecutionId::new();
3702        backend
3703            .open_execution(exec, ExecutionKind::AgentTurn)
3704            .await
3705            .unwrap();
3706        backend
3707            .finalize(exec, ExecutionStatus::Completed)
3708            .await
3709            .unwrap();
3710
3711        let is_resume = backend
3712            .open_execution(exec, ExecutionKind::AgentTurn)
3713            .await
3714            .unwrap();
3715        assert!(is_resume, "the row already existed, so this is a resume");
3716
3717        let (status, finalized): (String, Option<i64>) = zeph_db::query_as(sql!(
3718            "SELECT status, finalized_at FROM durable_executions WHERE execution_id = ?"
3719        ))
3720        .bind(exec.as_uuid().to_string())
3721        .fetch_one(backend.pool())
3722        .await
3723        .unwrap();
3724        assert_eq!(
3725            status, "running",
3726            "reopening a completed execution must un-finalize it"
3727        );
3728        assert!(
3729            finalized.is_none(),
3730            "reopening must clear the stale finalized_at"
3731        );
3732    }
3733
3734    #[tokio::test]
3735    async fn reopening_a_failed_execution_resets_it_to_running() {
3736        // #6251: same guarantee as `reopening_a_finalized_execution_resets_it_to_running`, but for
3737        // the `Failed` terminal status — e.g. a scheduler retry of the same (job_name, slot_ms)
3738        // after the previous fire failed must not orphan a `Failed` row.
3739        let backend = mem_backend(1_048_576).await;
3740        let exec = ExecutionId::new();
3741        backend
3742            .open_execution(exec, ExecutionKind::AgentTurn)
3743            .await
3744            .unwrap();
3745        backend
3746            .finalize(exec, ExecutionStatus::Failed)
3747            .await
3748            .unwrap();
3749
3750        let is_resume = backend
3751            .open_execution(exec, ExecutionKind::AgentTurn)
3752            .await
3753            .unwrap();
3754        assert!(is_resume, "the row already existed, so this is a resume");
3755
3756        let (status, finalized): (String, Option<i64>) = zeph_db::query_as(sql!(
3757            "SELECT status, finalized_at FROM durable_executions WHERE execution_id = ?"
3758        ))
3759        .bind(exec.as_uuid().to_string())
3760        .fetch_one(backend.pool())
3761        .await
3762        .unwrap();
3763        assert_eq!(
3764            status, "running",
3765            "reopening a failed execution must un-finalize it"
3766        );
3767        assert!(
3768            finalized.is_none(),
3769            "reopening must clear the stale finalized_at"
3770        );
3771    }
3772
3773    #[tokio::test]
3774    async fn reopening_an_aborted_execution_un_finalizes_it() {
3775        // INV-16 (#6254): reopening a row in ANY terminal status — including `aborted` — must
3776        // un-finalize it back to `running` with `finalized_at` cleared. This covers both the
3777        // pre-existing divergence-recovery reopen (which starts a fresh replay cursor on
3778        // purpose) and the new crash-orphan sweep (INV-17), which makes `aborted` the common
3779        // outcome of a resumable crash: a resumed execution whose row keeps `finalized_at` set
3780        // would otherwise be prunable out from under the active resume.
3781        let backend = mem_backend(1_048_576).await;
3782        let exec = ExecutionId::new();
3783        backend
3784            .open_execution(exec, ExecutionKind::AgentTurn)
3785            .await
3786            .unwrap();
3787        backend
3788            .finalize(exec, ExecutionStatus::Aborted)
3789            .await
3790            .unwrap();
3791
3792        let is_resume = backend
3793            .open_execution(exec, ExecutionKind::AgentTurn)
3794            .await
3795            .unwrap();
3796        assert!(is_resume, "the row already existed, so this is a resume");
3797
3798        let (status, finalized): (String, Option<i64>) = zeph_db::query_as(sql!(
3799            "SELECT status, finalized_at FROM durable_executions WHERE execution_id = ?"
3800        ))
3801        .bind(exec.as_uuid().to_string())
3802        .fetch_one(backend.pool())
3803        .await
3804        .unwrap();
3805        assert_eq!(
3806            status, "running",
3807            "reopening an aborted execution must un-finalize it (INV-16)"
3808        );
3809        assert!(
3810            finalized.is_none(),
3811            "reopening must clear the stale finalized_at"
3812        );
3813    }
3814
3815    #[tokio::test]
3816    async fn cancel_execution_with_no_live_owner_cancels_immediately() {
3817        // `:memory:` has `lock_dir = None` and `cross_process = false` (sqlite build), so this
3818        // exercises the provably-safe single-process direct-write path (F3).
3819        let backend = mem_backend(1_048_576).await;
3820        let exec = ExecutionId::new();
3821        backend
3822            .open_execution(exec, ExecutionKind::AgentTurn)
3823            .await
3824            .unwrap();
3825
3826        let outcome = backend.cancel_execution(exec).await.unwrap();
3827        assert_eq!(outcome, CancelOutcome::Canceled);
3828
3829        let (status, finalized): (String, Option<i64>) = zeph_db::query_as(sql!(
3830            "SELECT status, finalized_at FROM durable_executions WHERE execution_id = ?"
3831        ))
3832        .bind(exec.as_uuid().to_string())
3833        .fetch_one(backend.pool())
3834        .await
3835        .unwrap();
3836        assert_eq!(status, "canceled");
3837        assert!(finalized.is_some(), "a terminal status stamps finalized_at");
3838    }
3839
3840    #[tokio::test]
3841    async fn cancel_execution_with_no_live_owner_on_file_backed_pool_cancels_immediately() {
3842        // The SQLite/Unix lock-probe path: no lock is held, so the probe succeeds and the write
3843        // proceeds while the lock is held across it, then releases.
3844        let dir = tempfile::tempdir().unwrap();
3845        let backend =
3846            LocalBackend::open(&dir.path().join("durable.db").to_string_lossy(), 1_048_576)
3847                .await
3848                .unwrap();
3849        backend.init().await.unwrap();
3850
3851        let exec = ExecutionId::new();
3852        backend
3853            .open_execution(exec, ExecutionKind::AgentTurn)
3854            .await
3855            .unwrap();
3856
3857        let outcome = backend.cancel_execution(exec).await.unwrap();
3858        assert_eq!(outcome, CancelOutcome::Canceled);
3859
3860        // The lock must have been released after the write: a fresh acquire succeeds.
3861        let lock_dir = backend.lock_dir.clone().unwrap();
3862        assert!(ExecutionLock::acquire(&lock_dir, exec).is_ok());
3863    }
3864
3865    #[tokio::test]
3866    async fn cancel_execution_refuses_a_live_owner_without_touching_the_row() {
3867        // FR-006/FR-007 refusal: a live owner's held flock must short-circuit cancel to
3868        // `LiveOwner`, and the row must be left completely untouched.
3869        let dir = tempfile::tempdir().unwrap();
3870        let db_path = dir.path().join("durable.db");
3871        let url = db_path.to_string_lossy().into_owned();
3872
3873        let owner = LocalBackend::open(&url, 1_048_576).await.unwrap();
3874        owner.init().await.unwrap();
3875        let canceler = LocalBackend::open(&url, 1_048_576).await.unwrap();
3876
3877        let exec = ExecutionId::new();
3878        let (_, _lock) = owner
3879            .open_execution_exclusive(exec, ExecutionKind::AgentTurn)
3880            .await
3881            .unwrap();
3882
3883        let outcome = canceler.cancel_execution(exec).await.unwrap();
3884        assert!(
3885            matches!(outcome, CancelOutcome::LiveOwner { pid } if pid == std::process::id()),
3886            "expected LiveOwner{{pid: {}}}, got {outcome:?}",
3887            std::process::id()
3888        );
3889
3890        let (status,): (String,) = zeph_db::query_as(sql!(
3891            "SELECT status FROM durable_executions WHERE execution_id = ?"
3892        ))
3893        .bind(exec.as_uuid().to_string())
3894        .fetch_one(owner.pool())
3895        .await
3896        .unwrap();
3897        assert_eq!(status, "running", "a live-owned row must never be touched");
3898    }
3899
3900    #[tokio::test]
3901    async fn cancel_execution_is_idempotent_on_a_second_call() {
3902        // NFR-003: canceling an already-canceled row is a no-op, not an error.
3903        let backend = mem_backend(1_048_576).await;
3904        let exec = ExecutionId::new();
3905        backend
3906            .open_execution(exec, ExecutionKind::AgentTurn)
3907            .await
3908            .unwrap();
3909
3910        assert_eq!(
3911            backend.cancel_execution(exec).await.unwrap(),
3912            CancelOutcome::Canceled
3913        );
3914        let second = backend.cancel_execution(exec).await.unwrap();
3915        assert_eq!(
3916            second,
3917            CancelOutcome::AlreadyTerminal {
3918                status: ExecutionStatus::Canceled
3919            }
3920        );
3921    }
3922
3923    #[tokio::test]
3924    async fn cancel_execution_on_each_other_terminal_status_is_already_terminal() {
3925        for status in [
3926            ExecutionStatus::Completed,
3927            ExecutionStatus::Failed,
3928            ExecutionStatus::Aborted,
3929        ] {
3930            let backend = mem_backend(1_048_576).await;
3931            let exec = ExecutionId::new();
3932            backend
3933                .open_execution(exec, ExecutionKind::AgentTurn)
3934                .await
3935                .unwrap();
3936            backend.finalize(exec, status).await.unwrap();
3937
3938            let outcome = backend.cancel_execution(exec).await.unwrap();
3939            assert_eq!(
3940                outcome,
3941                CancelOutcome::AlreadyTerminal { status },
3942                "canceling a {status:?} execution must be a no-op reporting its own status"
3943            );
3944        }
3945    }
3946
3947    #[tokio::test]
3948    async fn cancel_execution_on_unknown_id_returns_not_found() {
3949        let backend = mem_backend(1_048_576).await;
3950        let outcome = backend.cancel_execution(ExecutionId::new()).await.unwrap();
3951        assert_eq!(outcome, CancelOutcome::NotFound);
3952    }
3953
3954    #[tokio::test]
3955    async fn cancel_execution_races_finalize_exactly_one_terminal_status_wins() {
3956        // SC-003: concurrent `cancel_execution` and `finalize(Completed)` — the guarded
3957        // `UPDATE ... WHERE status = 'running'` pattern shared by both means whichever commits
3958        // first wins, and the loser's write is simply a no-op rather than clobbering the winner.
3959        // Drives the two as genuinely concurrent tasks against a real multi-connection pool
3960        // (file-backed — `:memory:` forces a single connection, per `zeph-db/src/pool.rs`'s
3961        // `connect_sqlite`, which would serialize the two calls trivially and prove nothing),
3962        // across many trials so both orderings are exercised without artificial delay injection —
3963        // mirrors `concurrent_prune_and_reopen_never_lose_or_corrupt_the_row`'s pattern.
3964        let dir = tempfile::tempdir().unwrap();
3965        let db_url = dir.path().join("durable.db").to_string_lossy().into_owned();
3966        let backend = Arc::new(LocalBackend::open(&db_url, 1_048_576).await.unwrap());
3967        backend.init().await.unwrap();
3968
3969        for _ in 0..20 {
3970            let exec = ExecutionId::new();
3971            backend
3972                .open_execution(exec, ExecutionKind::AgentTurn)
3973                .await
3974                .unwrap();
3975
3976            let cancel_backend = backend.clone();
3977            let cancel = tokio::spawn(async move { cancel_backend.cancel_execution(exec).await });
3978            let finalize_backend = backend.clone();
3979            let finalize = tokio::spawn(async move {
3980                finalize_backend
3981                    .finalize(exec, ExecutionStatus::Completed)
3982                    .await
3983            });
3984
3985            let (cancel_result, finalize_result) = tokio::join!(cancel, finalize);
3986            let cancel_outcome = cancel_result
3987                .expect("cancel task must not panic")
3988                .expect("cancel_execution must not error under a concurrent finalize");
3989            finalize_result
3990                .expect("finalize task must not panic")
3991                .expect("finalize must not error under a concurrent cancel");
3992
3993            let (status,): (String,) = zeph_db::query_as(sql!(
3994                "SELECT status FROM durable_executions WHERE execution_id = ?"
3995            ))
3996            .bind(exec.as_uuid().to_string())
3997            .fetch_one(backend.pool())
3998            .await
3999            .unwrap();
4000
4001            // Whichever guarded UPDATE committed first wins; the loser's is a no-op. Both
4002            // outcomes are legitimate depending on scheduling — the invariant is that exactly one
4003            // terminal status is recorded, matching whichever `cancel_execution` outcome resulted.
4004            match cancel_outcome {
4005                CancelOutcome::Canceled => assert_eq!(
4006                    status, "canceled",
4007                    "cancel_execution won the race — the row must be canceled"
4008                ),
4009                CancelOutcome::AlreadyTerminal {
4010                    status: ExecutionStatus::Completed,
4011                } => assert_eq!(
4012                    status, "completed",
4013                    "finalize won the race — the row must be completed, and cancel's own \
4014                     guarded UPDATE must have found it already non-running"
4015                ),
4016                other => panic!(
4017                    "cancel_execution must only ever win or lose cleanly against a concurrent \
4018                     finalize, got {other:?}"
4019                ),
4020            }
4021        }
4022    }
4023
4024    #[tokio::test]
4025    async fn cancel_execution_races_sweep_orphans_exactly_one_of_canceled_or_aborted_wins() {
4026        // SC-004: concurrent `cancel_execution` and `sweep_orphans` on the same stale `running`
4027        // row. Both probe the same INV-15 `ExecutionLock` before writing, so this race has two
4028        // layers: whichever task wins the flock is the only one that ever attempts a write (the
4029        // loser either gets `LiveOwner` immediately without touching the row, or skips the
4030        // candidate without aborting it — INV-17's "never abort on staleness alone" rule already
4031        // covers a live-held lock). Drives the two as genuinely concurrent tasks against a real
4032        // multi-connection pool, across many trials so both lock-acquisition orderings are
4033        // exercised — mirrors `concurrent_sweep_and_reopen_race_never_corrupts_the_row`'s pattern.
4034        let dir = tempfile::tempdir().unwrap();
4035        let db_url = dir.path().join("durable.db").to_string_lossy().into_owned();
4036        let backend = Arc::new(LocalBackend::open(&db_url, 1_048_576).await.unwrap());
4037        backend.init().await.unwrap();
4038
4039        let policy = RetentionPolicy {
4040            stale_running_after_secs: 1,
4041            prune_batch_size: 10,
4042            ..RetentionPolicy::default()
4043        };
4044
4045        for _ in 0..20 {
4046            let exec = ExecutionId::new();
4047            backend
4048                .open_execution(exec, ExecutionKind::AgentTurn)
4049                .await
4050                .unwrap();
4051            backdate_updated_at(&backend, exec, 0).await;
4052
4053            let cancel_backend = backend.clone();
4054            let cancel = tokio::spawn(async move { cancel_backend.cancel_execution(exec).await });
4055            let sweep_backend = backend.clone();
4056            let policy_for_task = policy.clone();
4057            let sweep =
4058                tokio::spawn(async move { sweep_backend.sweep_orphans(&policy_for_task).await });
4059
4060            let (cancel_result, sweep_result) = tokio::join!(cancel, sweep);
4061            let cancel_outcome = cancel_result
4062                .expect("cancel task must not panic")
4063                .expect("cancel_execution must not error under a concurrent sweep");
4064            let aborted = sweep_result
4065                .expect("sweep task must not panic")
4066                .expect("sweep_orphans must not error under a concurrent cancel");
4067
4068            let (status,): (String,) = zeph_db::query_as(sql!(
4069                "SELECT status FROM durable_executions WHERE execution_id = ?"
4070            ))
4071            .bind(exec.as_uuid().to_string())
4072            .fetch_one(backend.pool())
4073            .await
4074            .unwrap();
4075
4076            match cancel_outcome {
4077                CancelOutcome::Canceled => {
4078                    assert_eq!(aborted, 0, "cancel won the lock — sweep must skip this row");
4079                    assert_eq!(status, "canceled");
4080                }
4081                CancelOutcome::LiveOwner { .. } => {
4082                    assert_eq!(aborted, 1, "sweep won the lock — it must abort this row");
4083                    assert_eq!(status, "aborted");
4084                }
4085                other => panic!(
4086                    "cancel_execution must only ever win the lock (Canceled) or lose it \
4087                     (LiveOwner) against a concurrent sweep, got {other:?}"
4088                ),
4089            }
4090        }
4091    }
4092
4093    #[tokio::test]
4094    async fn open_execution_on_canceled_row_fails_closed_and_never_resumes() {
4095        // INV-16′ (#6362), mirroring spec-064 scenario #13: unlike `completed`/`failed`/`aborted`,
4096        // a `canceled` row is the one deliberate carve-out — reopening it must fail closed with
4097        // `ExecutionCanceled` rather than un-finalizing it back to `running`.
4098        let backend = mem_backend(1_048_576).await;
4099        let exec = ExecutionId::new();
4100        backend
4101            .open_execution(exec, ExecutionKind::AgentTurn)
4102            .await
4103            .unwrap();
4104        let outcome = backend.cancel_execution(exec).await.unwrap();
4105        assert_eq!(outcome, CancelOutcome::Canceled);
4106
4107        let err = backend
4108            .open_execution(exec, ExecutionKind::AgentTurn)
4109            .await
4110            .expect_err("reopening a canceled execution must fail closed");
4111        assert!(
4112            matches!(err, DurableError::ExecutionCanceled { execution_id } if execution_id == exec),
4113            "expected ExecutionCanceled, got {err:?}"
4114        );
4115
4116        let (status,): (String,) = zeph_db::query_as(sql!(
4117            "SELECT status FROM durable_executions WHERE execution_id = ?"
4118        ))
4119        .bind(exec.as_uuid().to_string())
4120        .fetch_one(backend.pool())
4121        .await
4122        .unwrap();
4123        assert_eq!(
4124            status, "canceled",
4125            "the row must never be reset to running by a reopen attempt"
4126        );
4127    }
4128
4129    #[tokio::test]
4130    async fn open_execution_exclusive_on_canceled_row_fails_closed_with_lock_released() {
4131        // Same INV-16′ guarantee via the exclusive entry point; the flock guard must still be
4132        // released normally (no lock leak) when the call returns an error.
4133        let dir = tempfile::tempdir().unwrap();
4134        let db_path = dir.path().join("durable.db");
4135        let backend = LocalBackend::open(&db_path.to_string_lossy(), 1_048_576)
4136            .await
4137            .unwrap();
4138        backend.init().await.unwrap();
4139
4140        let exec = ExecutionId::new();
4141        backend
4142            .open_execution(exec, ExecutionKind::AgentTurn)
4143            .await
4144            .unwrap();
4145        assert_eq!(
4146            backend.cancel_execution(exec).await.unwrap(),
4147            CancelOutcome::Canceled
4148        );
4149
4150        let err = backend
4151            .open_execution_exclusive(exec, ExecutionKind::AgentTurn)
4152            .await
4153            .expect_err("reopening a canceled execution exclusively must fail closed");
4154        assert!(matches!(err, DurableError::ExecutionCanceled { .. }));
4155
4156        // The lock must have been released (no leak): a fresh acquire on the same id succeeds.
4157        let dir2 = backend.lock_dir.clone().unwrap();
4158        assert!(ExecutionLock::acquire(&dir2, exec).is_ok());
4159    }
4160
4161    #[tokio::test]
4162    async fn reopen_of_a_row_deleted_out_from_under_it_starts_fresh() {
4163        // #6251 critic S1: simulates the tail of the prune-vs-reopen race — a concurrent prune
4164        // sweep deletes the row entirely before the reopen's guarded UPDATE runs. The guarded
4165        // UPDATE must match zero rows (not resurrect a half-deleted row), and the existence
4166        // fallback must see the row is genuinely gone and start a fresh execution rather than
4167        // falsely reporting `is_resume = true` for a row that no longer exists.
4168        let backend = mem_backend(1_048_576).await;
4169        let exec = ExecutionId::new();
4170        backend
4171            .open_execution(exec, ExecutionKind::AgentTurn)
4172            .await
4173            .unwrap();
4174        backend
4175            .finalize(exec, ExecutionStatus::Completed)
4176            .await
4177            .unwrap();
4178
4179        // Simulate the prune sweep's delete completing before the reopen runs.
4180        zeph_db::query(sql!(
4181            "DELETE FROM durable_executions WHERE execution_id = ?"
4182        ))
4183        .bind(exec.as_uuid().to_string())
4184        .execute(backend.pool())
4185        .await
4186        .unwrap();
4187
4188        let is_resume = backend
4189            .open_execution(exec, ExecutionKind::AgentTurn)
4190            .await
4191            .unwrap();
4192        assert!(
4193            !is_resume,
4194            "a row deleted by a concurrent prune must be reported as a fresh execution, not a resume"
4195        );
4196
4197        let (status,): (String,) = zeph_db::query_as(sql!(
4198            "SELECT status FROM durable_executions WHERE execution_id = ?"
4199        ))
4200        .bind(exec.as_uuid().to_string())
4201        .fetch_one(backend.pool())
4202        .await
4203        .unwrap();
4204        assert_eq!(status, "running", "the fresh row starts running");
4205    }
4206
4207    #[tokio::test]
4208    async fn prune_does_not_delete_a_row_reopened_since_it_was_finalized() {
4209        // #6251 critic S1: a row finalized, then legitimately reopened (un-finalized back to
4210        // running) before prune runs, must not be deleted even though prune's cutoff would have
4211        // matched its now-stale-if-it-were-still-finalized state.
4212        let backend = mem_backend(1_048_576).await;
4213        let exec = ExecutionId::new();
4214        backend
4215            .open_execution(exec, ExecutionKind::AgentTurn)
4216            .await
4217            .unwrap();
4218        backend.append(step_result(exec, 0, b"x")).await.unwrap();
4219        zeph_db::query(sql!(
4220            "UPDATE durable_executions SET status = 'completed', finalized_at = 1000 WHERE execution_id = ?"
4221        ))
4222        .bind(exec.as_uuid().to_string())
4223        .execute(backend.pool())
4224        .await
4225        .unwrap();
4226
4227        // A legitimate resume reopens and un-finalizes it before the prune sweep runs.
4228        let is_resume = backend
4229            .open_execution(exec, ExecutionKind::AgentTurn)
4230            .await
4231            .unwrap();
4232        assert!(is_resume);
4233
4234        let policy = RetentionPolicy {
4235            ttl_completed_secs: 1,
4236            prune_batch_size: 10,
4237            ..RetentionPolicy::default()
4238        };
4239        let deleted = backend.prune(&policy).await.unwrap();
4240        assert_eq!(
4241            deleted, 0,
4242            "a reopened (un-finalized) execution must not be pruned"
4243        );
4244        assert_eq!(
4245            backend.read_execution(exec).await.unwrap().len(),
4246            1,
4247            "the execution's journal must survive"
4248        );
4249    }
4250
4251    #[tokio::test]
4252    async fn concurrent_prune_and_reopen_never_lose_or_corrupt_the_row() {
4253        // #6251 critic S1: the deterministic tests above exercise each ordering of the prune-vs-
4254        // reopen race one step at a time; this test drives the two operations as genuinely
4255        // concurrent tasks against a real multi-connection pool (file-backed — `:memory:` forces
4256        // a single connection, per `zeph-db/src/pool.rs`'s `connect_sqlite`, which would serialize
4257        // the two calls trivially and prove nothing about the locking fix). Runs many trials with
4258        // fresh executions so the two tasks' actual scheduling order varies across iterations,
4259        // covering both "prune's tx starts first" and "reopen's UPDATE starts first" without
4260        // needing artificial delay injection into the DB layer.
4261        //
4262        // Invariant checked every trial, regardless of which task wins: neither operation errors,
4263        // and the row is never lost — it either stays `running` (reopen won, or ran after prune's
4264        // read already excluded it) or is deleted and then reinserted fresh by reopen's
4265        // does-not-exist fallback (prune won). It must never end up half-deleted (FK violation on
4266        // a later journal append) or stuck `completed` with a live journal.
4267        let dir = tempfile::tempdir().unwrap();
4268        let db_url = dir.path().join("durable.db").to_string_lossy().into_owned();
4269        let backend = Arc::new(LocalBackend::open(&db_url, 1_048_576).await.unwrap());
4270        backend.init().await.unwrap();
4271
4272        let policy = RetentionPolicy {
4273            ttl_completed_secs: 1,
4274            prune_batch_size: 10,
4275            ..RetentionPolicy::default()
4276        };
4277
4278        for _ in 0..20 {
4279            let exec = ExecutionId::new();
4280            backend
4281                .open_execution(exec, ExecutionKind::AgentTurn)
4282                .await
4283                .unwrap();
4284            backend.append(step_result(exec, 0, b"x")).await.unwrap();
4285            // Backdate finalized_at so this row is immediately prune-eligible.
4286            zeph_db::query(sql!(
4287                "UPDATE durable_executions SET status = 'completed', finalized_at = 1000 WHERE execution_id = ?"
4288            ))
4289            .bind(exec.as_uuid().to_string())
4290            .execute(backend.pool())
4291            .await
4292            .unwrap();
4293
4294            let reopen_backend = backend.clone();
4295            let reopen = tokio::spawn(async move {
4296                reopen_backend
4297                    .open_execution(exec, ExecutionKind::AgentTurn)
4298                    .await
4299            });
4300            let prune_backend = backend.clone();
4301            let policy_for_task = policy.clone();
4302            let prune = tokio::spawn(async move { prune_backend.prune(&policy_for_task).await });
4303
4304            let (reopen_result, prune_result) = tokio::join!(reopen, prune);
4305            reopen_result
4306                .expect("reopen task must not panic")
4307                .expect("reopen must not error under concurrent prune");
4308            prune_result
4309                .expect("prune task must not panic")
4310                .expect("prune must not error under a concurrent reopen");
4311
4312            let (status,): (String,) = zeph_db::query_as(sql!(
4313                "SELECT status FROM durable_executions WHERE execution_id = ?"
4314            ))
4315            .bind(exec.as_uuid().to_string())
4316            .fetch_one(backend.pool())
4317            .await
4318            .expect(
4319                "the row must exist under either race outcome — reopened-running, or \
4320                 deleted-then-reinserted-fresh-running by reopen's fallback",
4321            );
4322            assert_eq!(
4323                status, "running",
4324                "whichever task wins, the row must end up running — never left completed \
4325                 (orphaned from a live journal) or absent"
4326            );
4327        }
4328    }
4329
4330    #[tokio::test]
4331    async fn max_seq_reflects_committed_appends() {
4332        let backend = mem_backend(1_048_576).await;
4333        assert_eq!(
4334            backend.max_seq().await.unwrap(),
4335            None,
4336            "empty journal has no max seq"
4337        );
4338
4339        let exec = ExecutionId::new();
4340        backend
4341            .open_execution(exec, ExecutionKind::AgentTurn)
4342            .await
4343            .unwrap();
4344        for step in 0..3 {
4345            backend.append(step_result(exec, step, b"x")).await.unwrap();
4346        }
4347        assert_eq!(backend.max_seq().await.unwrap(), Some(JournalSeq::new(3)));
4348    }
4349
4350    #[tokio::test]
4351    async fn append_batch_group_commits_every_entry() {
4352        let backend = mem_backend(1_048_576).await;
4353        let exec = ExecutionId::new();
4354        backend
4355            .open_execution(exec, ExecutionKind::AgentTurn)
4356            .await
4357            .unwrap();
4358        let batch = vec![
4359            step_result(exec, 0, b"a"),
4360            step_result(exec, 1, b"b"),
4361            step_result(exec, 2, b"c"),
4362        ];
4363        backend.append_batch(&batch).await.unwrap();
4364        assert_eq!(backend.read_execution(exec).await.unwrap().len(), 3);
4365    }
4366
4367    #[tokio::test]
4368    async fn read_execution_range_bounds_the_segment() {
4369        let backend = mem_backend(1_048_576).await;
4370        let exec = ExecutionId::new();
4371        backend
4372            .open_execution(exec, ExecutionKind::AgentTurn)
4373            .await
4374            .unwrap();
4375        for step in 0..5 {
4376            backend.append(step_result(exec, step, b"x")).await.unwrap();
4377        }
4378        let segment = backend.read_execution_range(exec, 2, 2).await.unwrap();
4379        assert_eq!(segment.len(), 2);
4380        assert_eq!(segment[0].step_id, StepId::new(2));
4381        assert_eq!(segment[1].step_id, StepId::new(3));
4382    }
4383
4384    #[tokio::test]
4385    async fn lookup_committed_result_finds_by_idem_key() {
4386        let backend = mem_backend(1_048_576).await;
4387        let exec = ExecutionId::new();
4388        backend
4389            .open_execution(exec, ExecutionKind::AgentTurn)
4390            .await
4391            .unwrap();
4392        let entry = step_result(exec, 0, b"committed");
4393        let idem_key = match &entry.entry {
4394            EntryKind::StepResult {
4395                idempotency_key, ..
4396            } => *idempotency_key,
4397            other => panic!("unexpected entry kind: {other:?}"),
4398        };
4399        backend.append(entry).await.unwrap();
4400
4401        let found = backend
4402            .lookup_committed_result(exec, idem_key)
4403            .await
4404            .unwrap()
4405            .expect("committed result is located by its idempotency key");
4406        match &found.entry {
4407            EntryKind::StepResult { payload, .. } => assert_eq!(payload.as_ref(), b"committed"),
4408            other => panic!("unexpected entry kind: {other:?}"),
4409        }
4410
4411        // A key that was never committed yields nothing rather than erroring.
4412        let absent = IdempotencyKey::derive(exec, StepId::new(99), b"never");
4413        assert!(
4414            backend
4415                .lookup_committed_result(exec, absent)
4416                .await
4417                .unwrap()
4418                .is_none()
4419        );
4420    }
4421
4422    #[tokio::test]
4423    async fn capabilities_describe_the_local_profile() {
4424        let backend = mem_backend(4096).await;
4425        let caps = backend.capabilities();
4426        assert!(caps.parallel_steps);
4427        assert!(
4428            !caps.cross_process,
4429            "the SQLite local backend is in-process"
4430        );
4431        assert_eq!(caps.max_payload, 4096);
4432    }
4433
4434    #[tokio::test]
4435    async fn promise_insert_state_and_resolve_round_trip() {
4436        let backend = mem_backend(1_048_576)
4437            .await
4438            .with_cipher(Arc::new(XorCipher));
4439        let exec = ExecutionId::new();
4440        backend
4441            .open_execution(exec, ExecutionKind::AgentTurn)
4442            .await
4443            .unwrap();
4444        let promise = PromiseId::derive(exec, StepId::new(0));
4445        backend
4446            .insert_promise(promise, exec, [9u8; 32], 100)
4447            .await
4448            .unwrap();
4449
4450        let pending = backend.promise_state(promise).await.unwrap().unwrap();
4451        assert!(!pending.resolved);
4452        assert_eq!(pending.execution_id, exec);
4453        assert_eq!(pending.resolver_token_hash, [9u8; 32]);
4454
4455        // Resolve seals the value at rest; a second resolve is a no-op.
4456        assert!(
4457            backend
4458                .resolve_promise(promise, exec, b"answer", 200)
4459                .await
4460                .unwrap()
4461        );
4462        assert!(
4463            !backend
4464                .resolve_promise(promise, exec, b"again", 300)
4465                .await
4466                .unwrap()
4467        );
4468
4469        let resolved = backend.promise_state(promise).await.unwrap().unwrap();
4470        assert!(resolved.resolved);
4471        let sealed = resolved.payload.expect("resolved payload present");
4472        assert_ne!(sealed.as_slice(), b"answer", "payload is sealed at rest");
4473        let opened = backend
4474            .open_promise_payload(promise, exec, &sealed)
4475            .unwrap();
4476        assert_eq!(opened.as_ref(), b"answer");
4477    }
4478
4479    #[tokio::test]
4480    async fn claim_promise_notification_is_single_winner() {
4481        let backend = mem_backend(1_048_576).await;
4482        let exec = ExecutionId::new();
4483        backend
4484            .open_execution(exec, ExecutionKind::AgentTurn)
4485            .await
4486            .unwrap();
4487        let promise = PromiseId::derive(exec, StepId::new(0));
4488        backend
4489            .insert_promise(promise, exec, [9u8; 32], 100)
4490            .await
4491            .unwrap();
4492
4493        // First claim wins (transitions notified_at from NULL).
4494        assert!(
4495            backend
4496                .claim_promise_notification(promise, 200)
4497                .await
4498                .unwrap()
4499        );
4500        // Every later claim on the same promise is a no-op.
4501        assert!(
4502            !backend
4503                .claim_promise_notification(promise, 300)
4504                .await
4505                .unwrap()
4506        );
4507    }
4508
4509    #[tokio::test]
4510    async fn timer_arm_due_and_fire() {
4511        let backend = mem_backend(1_048_576).await;
4512        let exec = ExecutionId::new();
4513        backend
4514            .open_execution(exec, ExecutionKind::AgentTurn)
4515            .await
4516            .unwrap();
4517        let past = TimerId::derive(exec, StepId::new(0));
4518        let future = TimerId::derive(exec, StepId::new(1));
4519        backend.arm_timer(past, exec, 1_000, 0).await.unwrap();
4520        backend
4521            .arm_timer(future, exec, 9_000_000_000_000, 0)
4522            .await
4523            .unwrap();
4524
4525        // Only the past-due timer is returned at now = 5000.
4526        let due = backend.due_timers(5_000).await.unwrap();
4527        assert_eq!(due, vec![past]);
4528
4529        assert!(backend.mark_timer_fired(past).await.unwrap());
4530        assert!(
4531            !backend.mark_timer_fired(past).await.unwrap(),
4532            "second fire is a no-op"
4533        );
4534        assert_eq!(
4535            backend.timer_state(past).await.unwrap(),
4536            Some((1_000, true))
4537        );
4538        // The fired timer no longer appears as due.
4539        assert!(backend.due_timers(5_000).await.unwrap().is_empty());
4540    }
4541
4542    #[tokio::test]
4543    async fn prune_deletes_terminal_executions_past_ttl() {
4544        let backend = mem_backend(1_048_576).await;
4545        // An old completed execution (finalized long ago) and a fresh running one.
4546        let old = ExecutionId::new();
4547        backend
4548            .open_execution(old, ExecutionKind::AgentTurn)
4549            .await
4550            .unwrap();
4551        backend.append(step_result(old, 0, b"x")).await.unwrap();
4552        // Backdate its finalized_at far into the past.
4553        zeph_db::query(sql!(
4554            "UPDATE durable_executions SET status = 'completed', finalized_at = 1000 WHERE execution_id = ?"
4555        ))
4556        .bind(old.as_uuid().to_string())
4557        .execute(backend.pool())
4558        .await
4559        .unwrap();
4560
4561        let live = ExecutionId::new();
4562        backend
4563            .open_execution(live, ExecutionKind::AgentTurn)
4564            .await
4565            .unwrap();
4566        backend.append(step_result(live, 0, b"y")).await.unwrap();
4567
4568        let policy = RetentionPolicy {
4569            ttl_completed_secs: 1,
4570            prune_batch_size: 10,
4571            ..RetentionPolicy::default()
4572        };
4573        let deleted = backend.prune(&policy).await.unwrap();
4574        assert_eq!(deleted, 1, "only the aged terminal execution is pruned");
4575
4576        // The old execution and its journal are gone; the live one survives.
4577        assert!(backend.read_execution(old).await.unwrap().is_empty());
4578        assert!(
4579            backend
4580                .promise_state(PromiseId::derive(old, StepId::new(0)))
4581                .await
4582                .unwrap()
4583                .is_none()
4584        );
4585        assert_eq!(backend.read_execution(live).await.unwrap().len(), 1);
4586    }
4587
4588    #[tokio::test]
4589    async fn count_prunable_and_prune_include_canceled_executions_past_ttl() {
4590        // FR-013 (#6362): a canceled row groups with failed/aborted for the retention TTL cutoff
4591        // — without this, canceled rows would never be pruned and would accumulate forever.
4592        let backend = mem_backend(1_048_576).await;
4593        let exec = ExecutionId::new();
4594        backend
4595            .open_execution(exec, ExecutionKind::AgentTurn)
4596            .await
4597            .unwrap();
4598        assert_eq!(
4599            backend.cancel_execution(exec).await.unwrap(),
4600            CancelOutcome::Canceled
4601        );
4602        // Backdate finalized_at far into the past so it is past the failed/aborted TTL cutoff.
4603        zeph_db::query(sql!(
4604            "UPDATE durable_executions SET finalized_at = 1000 WHERE execution_id = ?"
4605        ))
4606        .bind(exec.as_uuid().to_string())
4607        .execute(backend.pool())
4608        .await
4609        .unwrap();
4610
4611        let policy = RetentionPolicy {
4612            ttl_failed_secs: 1,
4613            prune_batch_size: 10,
4614            ..RetentionPolicy::default()
4615        };
4616        let prunable = backend.count_prunable(&policy).await.unwrap();
4617        assert_eq!(
4618            prunable, 1,
4619            "an aged canceled row must be counted as prunable"
4620        );
4621
4622        let deleted = backend.prune(&policy).await.unwrap();
4623        assert_eq!(deleted, 1, "an aged canceled row must actually be pruned");
4624        assert!(backend.read_execution(exec).await.unwrap().is_empty());
4625    }
4626
4627    /// Regression for issue #6360 (critic B1): a keyed backend's `durable_execution_integrity` row
4628    /// (created by `bump_hwm_for_step_result` for every committed `StepResult`) references
4629    /// `durable_executions` without `ON DELETE CASCADE` — the same convention as
4630    /// `durable_journal`/`durable_promises`/`durable_timers`, which `delete_prune_batch` deletes
4631    /// manually before the parent row. Before the fix, the integrity row was never included in that
4632    /// manual delete, so `DELETE FROM durable_executions` violated the FK under `SQLite`'s
4633    /// `PRAGMA foreign_keys = ON` (and unconditionally on `PostgreSQL`), rolling back the whole
4634    /// prune batch for every keyed execution — retention silently stopped working on any real
4635    /// (`ZEPH_DURABLE_KEY`-configured) deployment. Exercises the previously-untested path: all
4636    /// prior prune tests used unkeyed backends, which never create an integrity row and so never
4637    /// hit the FK.
4638    #[tokio::test]
4639    async fn prune_deletes_a_keyed_execution_and_its_integrity_row() {
4640        let backend = mem_backend(1_048_576).await.with_hwm_key(0, [42u8; 32]);
4641        let old = ExecutionId::new();
4642        backend
4643            .open_execution(old, ExecutionKind::AgentTurn)
4644            .await
4645            .unwrap();
4646        backend.append(step_result(old, 0, b"x")).await.unwrap();
4647
4648        // The committed StepResult must have created an integrity row.
4649        let before: (i64,) = zeph_db::query_as(sql!(
4650            "SELECT COUNT(*) FROM durable_execution_integrity WHERE execution_id = ?"
4651        ))
4652        .bind(old.as_uuid().to_string())
4653        .fetch_one(backend.pool())
4654        .await
4655        .unwrap();
4656        assert_eq!(
4657            before.0, 1,
4658            "a committed StepResult must create an integrity row"
4659        );
4660
4661        zeph_db::query(sql!(
4662            "UPDATE durable_executions SET status = 'completed', finalized_at = 1000 WHERE execution_id = ?"
4663        ))
4664        .bind(old.as_uuid().to_string())
4665        .execute(backend.pool())
4666        .await
4667        .unwrap();
4668
4669        let policy = RetentionPolicy {
4670            ttl_completed_secs: 1,
4671            prune_batch_size: 10,
4672            ..RetentionPolicy::default()
4673        };
4674        let deleted = backend
4675            .prune(&policy)
4676            .await
4677            .expect("prune must not fail closed on a keyed execution's FK");
4678        assert_eq!(deleted, 1, "the keyed execution is pruned like any other");
4679
4680        assert!(backend.read_execution(old).await.unwrap().is_empty());
4681        let after: (i64,) = zeph_db::query_as(sql!(
4682            "SELECT COUNT(*) FROM durable_execution_integrity WHERE execution_id = ?"
4683        ))
4684        .bind(old.as_uuid().to_string())
4685        .fetch_one(backend.pool())
4686        .await
4687        .unwrap();
4688        assert_eq!(
4689            after.0, 0,
4690            "the integrity row must be pruned alongside its execution"
4691        );
4692    }
4693
4694    /// Backdate a `durable_executions` row's `updated_at` so it becomes a sweep candidate.
4695    async fn backdate_updated_at(backend: &LocalBackend, id: ExecutionId, updated_at_ms: i64) {
4696        zeph_db::query(sql!(
4697            "UPDATE durable_executions SET updated_at = ? WHERE execution_id = ?"
4698        ))
4699        .bind(updated_at_ms)
4700        .bind(id.as_uuid().to_string())
4701        .execute(backend.pool())
4702        .await
4703        .unwrap();
4704    }
4705
4706    #[tokio::test]
4707    async fn sweep_orphans_disabled_when_threshold_is_zero() {
4708        // A file-backed backend so the sweep would otherwise have a lock_dir to work with;
4709        // stale_running_after_secs = 0 must short-circuit before any scan.
4710        let dir = tempfile::tempdir().unwrap();
4711        let backend =
4712            LocalBackend::open(&dir.path().join("durable.db").to_string_lossy(), 1_048_576)
4713                .await
4714                .unwrap();
4715        backend.init().await.unwrap();
4716
4717        let exec = ExecutionId::new();
4718        backend
4719            .open_execution(exec, ExecutionKind::AgentTurn)
4720            .await
4721            .unwrap();
4722        backdate_updated_at(&backend, exec, 0).await;
4723
4724        let policy = RetentionPolicy {
4725            stale_running_after_secs: 0,
4726            ..RetentionPolicy::default()
4727        };
4728        let aborted = backend.sweep_orphans(&policy).await.unwrap();
4729        assert_eq!(
4730            aborted, 0,
4731            "stale_running_after_secs = 0 disables the sweep"
4732        );
4733
4734        let (status,): (String,) = zeph_db::query_as(sql!(
4735            "SELECT status FROM durable_executions WHERE execution_id = ?"
4736        ))
4737        .bind(exec.as_uuid().to_string())
4738        .fetch_one(backend.pool())
4739        .await
4740        .unwrap();
4741        assert_eq!(status, "running");
4742    }
4743
4744    #[tokio::test]
4745    async fn sweep_orphans_is_a_documented_no_op_on_memory_backend() {
4746        // `:memory:` has no on-disk lock_dir (INV-15 degrade), so the sweep must never abort on
4747        // staleness alone — FR-DE-19.
4748        let backend = mem_backend(1_048_576).await;
4749        let exec = ExecutionId::new();
4750        backend
4751            .open_execution(exec, ExecutionKind::AgentTurn)
4752            .await
4753            .unwrap();
4754        backdate_updated_at(&backend, exec, 0).await;
4755
4756        let policy = RetentionPolicy {
4757            stale_running_after_secs: 1,
4758            ..RetentionPolicy::default()
4759        };
4760        let aborted = backend.sweep_orphans(&policy).await.unwrap();
4761        assert_eq!(
4762            aborted, 0,
4763            "a lock_dir=None backend must never abort on staleness alone"
4764        );
4765
4766        let (status,): (String,) = zeph_db::query_as(sql!(
4767            "SELECT status FROM durable_executions WHERE execution_id = ?"
4768        ))
4769        .bind(exec.as_uuid().to_string())
4770        .fetch_one(backend.pool())
4771        .await
4772        .unwrap();
4773        assert_eq!(status, "running");
4774    }
4775
4776    #[tokio::test]
4777    async fn sweep_orphans_aborts_a_stale_running_execution_with_no_live_owner() {
4778        // FR-DE-16/17: a stale `running` row whose lock is free (no live owner) is hard-aborted.
4779        let dir = tempfile::tempdir().unwrap();
4780        let backend =
4781            LocalBackend::open(&dir.path().join("durable.db").to_string_lossy(), 1_048_576)
4782                .await
4783                .unwrap();
4784        backend.init().await.unwrap();
4785
4786        let exec = ExecutionId::new();
4787        backend
4788            .open_execution(exec, ExecutionKind::AgentTurn)
4789            .await
4790            .unwrap();
4791        // Nothing holds this execution's ExecutionLock — `open_execution` (not `_exclusive`)
4792        // never acquires one, simulating a crashed owner whose flock released on process exit.
4793        backdate_updated_at(&backend, exec, 0).await;
4794
4795        let policy = RetentionPolicy {
4796            stale_running_after_secs: 1,
4797            ..RetentionPolicy::default()
4798        };
4799        let aborted = backend.sweep_orphans(&policy).await.unwrap();
4800        assert_eq!(aborted, 1);
4801
4802        let (status, finalized): (String, Option<i64>) = zeph_db::query_as(sql!(
4803            "SELECT status, finalized_at FROM durable_executions WHERE execution_id = ?"
4804        ))
4805        .bind(exec.as_uuid().to_string())
4806        .fetch_one(backend.pool())
4807        .await
4808        .unwrap();
4809        assert_eq!(status, "aborted");
4810        assert!(finalized.is_some());
4811    }
4812
4813    #[tokio::test]
4814    async fn sweep_orphans_skips_an_execution_whose_lock_is_held_by_a_live_owner() {
4815        // INV-17: staleness of `updated_at` alone is never sufficient — a stale-but-alive
4816        // execution (long single step, parked HITL promise, multi-hour job) must survive the
4817        // sweep as long as its owner still holds the INV-15 flock.
4818        let dir = tempfile::tempdir().unwrap();
4819        let db_path = dir.path().join("durable.db");
4820        let url = db_path.to_string_lossy().into_owned();
4821
4822        let owner = LocalBackend::open(&url, 1_048_576).await.unwrap();
4823        owner.init().await.unwrap();
4824        let sweeper = LocalBackend::open(&url, 1_048_576).await.unwrap();
4825
4826        let exec = ExecutionId::new();
4827        let (_, _lock) = owner
4828            .open_execution_exclusive(exec, ExecutionKind::AgentTurn)
4829            .await
4830            .unwrap();
4831        backdate_updated_at(&owner, exec, 0).await;
4832
4833        let policy = RetentionPolicy {
4834            stale_running_after_secs: 1,
4835            ..RetentionPolicy::default()
4836        };
4837        let aborted = sweeper.sweep_orphans(&policy).await.unwrap();
4838        assert_eq!(aborted, 0, "a live-held lock must never be swept");
4839
4840        let (status,): (String,) = zeph_db::query_as(sql!(
4841            "SELECT status FROM durable_executions WHERE execution_id = ?"
4842        ))
4843        .bind(exec.as_uuid().to_string())
4844        .fetch_one(owner.pool())
4845        .await
4846        .unwrap();
4847        assert_eq!(status, "running");
4848    }
4849
4850    #[tokio::test]
4851    async fn sweep_orphans_leaves_a_fresh_running_execution_untouched() {
4852        // A recently-updated `running` row is not yet a sweep candidate at all.
4853        let dir = tempfile::tempdir().unwrap();
4854        let backend =
4855            LocalBackend::open(&dir.path().join("durable.db").to_string_lossy(), 1_048_576)
4856                .await
4857                .unwrap();
4858        backend.init().await.unwrap();
4859
4860        let exec = ExecutionId::new();
4861        backend
4862            .open_execution(exec, ExecutionKind::AgentTurn)
4863            .await
4864            .unwrap();
4865
4866        let policy = RetentionPolicy {
4867            stale_running_after_secs: 3600,
4868            ..RetentionPolicy::default()
4869        };
4870        let aborted = backend.sweep_orphans(&policy).await.unwrap();
4871        assert_eq!(aborted, 0);
4872    }
4873
4874    #[tokio::test]
4875    async fn sweep_orphans_never_touches_a_stale_canceled_row() {
4876        // FR-008 regression (#6362): `sweep_orphan_batch` only ever candidate-selects
4877        // `status = 'running'` rows, so a canceled row — even a stale one — must never be
4878        // resurrected or otherwise touched, across repeated sweep cycles.
4879        let dir = tempfile::tempdir().unwrap();
4880        let backend =
4881            LocalBackend::open(&dir.path().join("durable.db").to_string_lossy(), 1_048_576)
4882                .await
4883                .unwrap();
4884        backend.init().await.unwrap();
4885
4886        let exec = ExecutionId::new();
4887        backend
4888            .open_execution(exec, ExecutionKind::AgentTurn)
4889            .await
4890            .unwrap();
4891        assert_eq!(
4892            backend.cancel_execution(exec).await.unwrap(),
4893            CancelOutcome::Canceled
4894        );
4895        backdate_updated_at(&backend, exec, 0).await;
4896
4897        let policy = RetentionPolicy {
4898            stale_running_after_secs: 1,
4899            ..RetentionPolicy::default()
4900        };
4901        for _ in 0..3 {
4902            let aborted = backend.sweep_orphans(&policy).await.unwrap();
4903            assert_eq!(aborted, 0, "a canceled row must never be swept");
4904        }
4905
4906        let (status,): (String,) = zeph_db::query_as(sql!(
4907            "SELECT status FROM durable_executions WHERE execution_id = ?"
4908        ))
4909        .bind(exec.as_uuid().to_string())
4910        .fetch_one(backend.pool())
4911        .await
4912        .unwrap();
4913        assert_eq!(
4914            status, "canceled",
4915            "sweep must never resurrect a canceled row"
4916        );
4917    }
4918
4919    #[tokio::test]
4920    async fn count_orphans_matches_sweep_without_mutating() {
4921        let dir = tempfile::tempdir().unwrap();
4922        let backend =
4923            LocalBackend::open(&dir.path().join("durable.db").to_string_lossy(), 1_048_576)
4924                .await
4925                .unwrap();
4926        backend.init().await.unwrap();
4927
4928        let exec = ExecutionId::new();
4929        backend
4930            .open_execution(exec, ExecutionKind::AgentTurn)
4931            .await
4932            .unwrap();
4933        backdate_updated_at(&backend, exec, 0).await;
4934
4935        let policy = RetentionPolicy {
4936            stale_running_after_secs: 1,
4937            ..RetentionPolicy::default()
4938        };
4939        let counted = backend.count_orphans(&policy).await.unwrap();
4940        assert_eq!(counted, 1);
4941
4942        // count_orphans must not have mutated the row.
4943        let (status,): (String,) = zeph_db::query_as(sql!(
4944            "SELECT status FROM durable_executions WHERE execution_id = ?"
4945        ))
4946        .bind(exec.as_uuid().to_string())
4947        .fetch_one(backend.pool())
4948        .await
4949        .unwrap();
4950        assert_eq!(status, "running");
4951
4952        let aborted = backend.sweep_orphans(&policy).await.unwrap();
4953        assert_eq!(
4954            aborted, counted,
4955            "sweep must abort exactly what count_orphans counted"
4956        );
4957    }
4958
4959    /// Batching-boundary regression: a candidate set straddling `prune_batch_size` (one more row
4960    /// than a single batch) must be fully processed across multiple batches, not just the first
4961    /// one. Exercises the real `sweep_orphan_batch`/`sweep_orphans_in_batches` composition end to
4962    /// end (not the pure-logic unit test in `retention.rs`), so the SQL `LIMIT` and the
4963    /// `scanned`-driven continuation check are both proven against a real DB.
4964    #[tokio::test]
4965    async fn sweep_orphans_processes_every_batch_when_candidates_straddle_the_batch_size() {
4966        let dir = tempfile::tempdir().unwrap();
4967        let backend =
4968            LocalBackend::open(&dir.path().join("durable.db").to_string_lossy(), 1_048_576)
4969                .await
4970                .unwrap();
4971        backend.init().await.unwrap();
4972
4973        let batch_size = 2u64;
4974        let candidate_count = batch_size + 1; // straddles the batch boundary
4975        let mut execs = Vec::new();
4976        for _ in 0..candidate_count {
4977            let exec = ExecutionId::new();
4978            backend
4979                .open_execution(exec, ExecutionKind::AgentTurn)
4980                .await
4981                .unwrap();
4982            backdate_updated_at(&backend, exec, 0).await;
4983            execs.push(exec);
4984        }
4985
4986        let policy = RetentionPolicy {
4987            stale_running_after_secs: 1,
4988            prune_batch_size: batch_size,
4989            ..RetentionPolicy::default()
4990        };
4991        let aborted = backend.sweep_orphans(&policy).await.unwrap();
4992        assert_eq!(
4993            aborted, candidate_count,
4994            "every candidate must be aborted, including the one past the first batch"
4995        );
4996
4997        for exec in execs {
4998            let (status,): (String,) = zeph_db::query_as(sql!(
4999                "SELECT status FROM durable_executions WHERE execution_id = ?"
5000            ))
5001            .bind(exec.as_uuid().to_string())
5002            .fetch_one(backend.pool())
5003            .await
5004            .unwrap();
5005            assert_eq!(status, "aborted");
5006        }
5007    }
5008
5009    /// #6254 C1 regression: when the count of stale-but-live (lock-held) candidates is `>=
5010    /// prune_batch_size`, the sweep must still terminate rather than looping forever re-selecting
5011    /// the same lock-held rows. Before the keyset-pagination fix, `sweep_orphan_batch`'s candidate
5012    /// `SELECT` had no offset/cursor, so a batch consisting entirely of lock-held rows (which the
5013    /// sweep never deletes, mutates, or otherwise removes from the `status='running'` candidate
5014    /// set) would re-select the identical rows on every iteration: `scanned` would stay `==
5015    /// batch` and `aborted` would stay `0` forever, so `sweep_orphans_in_batches`'s `scanned <
5016    /// batch` continuation check would never trip. Exercises the real DB-backed
5017    /// `sweep_orphan_batch`/`sweep_orphans_in_batches` composition (not the pure-logic
5018    /// simulation in `retention.rs`) with more lock-held candidates than `prune_batch_size`, so a
5019    /// naive single-batch-worth-of-locks reproduction would not have caught a bug that only
5020    /// manifests once the candidate set spans multiple batches.
5021    #[tokio::test]
5022    async fn sweep_orphans_terminates_when_lock_held_candidates_exceed_batch_size() {
5023        let dir = tempfile::tempdir().unwrap();
5024        let db_url = dir.path().join("durable.db").to_string_lossy().into_owned();
5025
5026        let owner = LocalBackend::open(&db_url, 1_048_576).await.unwrap();
5027        owner.init().await.unwrap();
5028        let sweeper = LocalBackend::open(&db_url, 1_048_576).await.unwrap();
5029
5030        let batch_size = 2u64;
5031        let candidate_count = batch_size * 2 + 1; // spans at least three batches, all lock-held
5032        let mut locks = Vec::new();
5033        for _ in 0..candidate_count {
5034            let exec = ExecutionId::new();
5035            let (_, lock) = owner
5036                .open_execution_exclusive(exec, ExecutionKind::AgentTurn)
5037                .await
5038                .unwrap();
5039            backdate_updated_at(&owner, exec, 0).await;
5040            locks.push(lock); // held for the whole test — every candidate stays lock-held
5041        }
5042
5043        let policy = RetentionPolicy {
5044            stale_running_after_secs: 1,
5045            prune_batch_size: batch_size,
5046            ..RetentionPolicy::default()
5047        };
5048
5049        let aborted = tokio::time::timeout(
5050            std::time::Duration::from_secs(10),
5051            sweeper.sweep_orphans(&policy),
5052        )
5053        .await
5054        .expect(
5055            "sweep_orphans must terminate even when lock-held candidates exceed prune_batch_size \
5056             (#6254 C1) — it hung instead of returning",
5057        )
5058        .unwrap();
5059
5060        assert_eq!(aborted, 0, "every candidate's lock is held by a live owner");
5061        drop(locks);
5062    }
5063
5064    /// INV-17: the sweep's guarded abort `UPDATE` runs only while holding the same non-reentrant
5065    /// flock a concurrent `open_execution_exclusive` reopen for the same execution id requires, so
5066    /// the two can never both mutate the row at once. Drives them as genuinely concurrent tasks
5067    /// against a real multi-connection pool (file-backed — `:memory:` forces a single connection,
5068    /// which would serialize the two calls trivially and prove nothing) across many trials so both
5069    /// orderings ("sweep acquires the lock first" and "reopen acquires the lock first") are
5070    /// exercised without artificial delay injection, mirroring the #6251
5071    /// `concurrent_prune_and_reopen_never_lose_or_corrupt_the_row` pattern above.
5072    #[tokio::test]
5073    async fn concurrent_sweep_and_reopen_race_never_corrupts_the_row() {
5074        let dir = tempfile::tempdir().unwrap();
5075        let db_url = dir.path().join("durable.db").to_string_lossy().into_owned();
5076        let backend = Arc::new(LocalBackend::open(&db_url, 1_048_576).await.unwrap());
5077        backend.init().await.unwrap();
5078
5079        let policy = RetentionPolicy {
5080            stale_running_after_secs: 1,
5081            prune_batch_size: 10,
5082            ..RetentionPolicy::default()
5083        };
5084
5085        for _ in 0..20 {
5086            let exec = ExecutionId::new();
5087            backend
5088                .open_execution(exec, ExecutionKind::AgentTurn)
5089                .await
5090                .unwrap();
5091            backdate_updated_at(&backend, exec, 0).await;
5092
5093            let sweep_backend = backend.clone();
5094            let policy_for_task = policy.clone();
5095            let sweep =
5096                tokio::spawn(async move { sweep_backend.sweep_orphans(&policy_for_task).await });
5097
5098            let reopen_backend = backend.clone();
5099            let reopen = tokio::spawn(async move {
5100                reopen_backend
5101                    .open_execution_exclusive(exec, ExecutionKind::AgentTurn)
5102                    .await
5103            });
5104
5105            let (sweep_result, reopen_result) = tokio::join!(sweep, reopen);
5106            let aborted = sweep_result
5107                .expect("sweep task must not panic")
5108                .expect("sweep must not error under a concurrent reopen");
5109            assert!(aborted <= 1, "at most one candidate row exists per trial");
5110
5111            match reopen_result.expect("reopen task must not panic") {
5112                Ok((_is_resume, _lock)) => {
5113                    // reopen won the race for the lock (either before the sweep even tried, or
5114                    // after the sweep aborted the row and released) — the row must be `running`
5115                    // with `finalized_at` cleared either way (INV-16 un-finalizes `aborted` too).
5116                    let (status, finalized): (String, Option<i64>) = zeph_db::query_as(sql!(
5117                        "SELECT status, finalized_at FROM durable_executions WHERE execution_id = ?"
5118                    ))
5119                    .bind(exec.as_uuid().to_string())
5120                    .fetch_one(backend.pool())
5121                    .await
5122                    .unwrap();
5123                    assert_eq!(status, "running");
5124                    assert!(finalized.is_none());
5125                    // Finalize before the next trial: when reopen wins because the row was
5126                    // already `running` (not terminal) at the time it checked, `open_execution`'s
5127                    // existing-row branch never bumps `updated_at` — left alone, this row would
5128                    // stay a stale `running` candidate forever and pollute a later trial's
5129                    // `aborted` count (the `assert!(aborted <= 1, ...)` above would then see more
5130                    // than this trial's own row). Each trial must start with a clean slate of
5131                    // exactly its own candidate.
5132                    backend
5133                        .finalize(exec, ExecutionStatus::Completed)
5134                        .await
5135                        .unwrap();
5136                }
5137                Err(DurableError::ExecutionLocked { .. }) => {
5138                    // The sweep held the lock at the moment reopen tried — expected under the race.
5139                }
5140                Err(e) => panic!(
5141                    "reopen must only ever fail with ExecutionLocked under this race, got {e:?}"
5142                ),
5143            }
5144        }
5145    }
5146
5147    #[tokio::test]
5148    async fn checkpoint_fold_compacts_idempotent_prefix_and_replays() {
5149        let backend = mem_backend(1_048_576)
5150            .await
5151            .with_cipher(Arc::new(XorCipher));
5152        let exec = ExecutionId::new();
5153        backend
5154            .open_execution(exec, ExecutionKind::AgentTurn)
5155            .await
5156            .unwrap();
5157        for step in 0..5 {
5158            backend
5159                .append(step_result(exec, step, format!("v{step}").as_bytes()))
5160                .await
5161                .unwrap();
5162        }
5163
5164        // Fold steps 0..3 into a checkpoint.
5165        let folded = backend.checkpoint_fold(exec, 3).await.unwrap();
5166        assert_eq!(folded, 3);
5167
5168        // The individual rows for the folded steps are gone; steps 3 and 4 remain, plus a checkpoint.
5169        let remaining = backend.read_execution(exec).await.unwrap();
5170        let step_results: Vec<u32> = remaining
5171            .iter()
5172            .filter(|e| matches!(e.entry, EntryKind::StepResult { .. }))
5173            .map(|e| e.step_id.value())
5174            .collect();
5175        assert_eq!(step_results, vec![3, 4], "folded step rows are deleted");
5176        assert!(
5177            remaining
5178                .iter()
5179                .any(|e| matches!(e.entry, EntryKind::Checkpoint { .. })),
5180            "a checkpoint entry replaces the folded prefix"
5181        );
5182
5183        // The reconstructed folded results carry the original values and idempotency keys.
5184        let preloaded = backend.read_checkpoints(exec).await.unwrap();
5185        assert_eq!(preloaded.len(), 3);
5186        for (i, entry) in preloaded.iter().enumerate() {
5187            let step = u32::try_from(i).unwrap();
5188            assert_eq!(entry.step_id, StepId::new(step));
5189            match &entry.entry {
5190                EntryKind::StepResult {
5191                    payload,
5192                    idempotency_key,
5193                    ..
5194                } => {
5195                    assert_eq!(payload.as_ref(), format!("v{step}").as_bytes());
5196                    assert_eq!(
5197                        *idempotency_key,
5198                        IdempotencyKey::derive(exec, StepId::new(step), b"tool:read")
5199                    );
5200                }
5201                other => panic!("unexpected folded entry: {other:?}"),
5202            }
5203        }
5204    }
5205
5206    // High-water-mark tests (issue #6360). `mem_backend` opens a fresh `:memory:` pool per call, so
5207    // these tests share a pool via `LocalBackend::new(backend.pool().clone(), ...)` when they need a
5208    // second backend handle (a different key, or unkeyed) reading the same journal — mirroring the
5209    // existing `read_execution_rejects_control_hmac_under_wrong_key` pattern above.
5210
5211    #[tokio::test]
5212    async fn hwm_is_a_no_op_when_unkeyed() {
5213        let backend = mem_backend(1_048_576).await;
5214        let exec = ExecutionId::new();
5215        assert!(
5216            !backend
5217                .open_execution(exec, ExecutionKind::AgentTurn)
5218                .await
5219                .unwrap()
5220        );
5221        backend.append(step_result(exec, 0, b"v0")).await.unwrap();
5222        // No integrity row should exist, and resume must still succeed.
5223        assert!(
5224            backend
5225                .open_execution(exec, ExecutionKind::AgentTurn)
5226                .await
5227                .unwrap()
5228        );
5229    }
5230
5231    #[tokio::test]
5232    async fn hwm_verifies_on_resume_after_single_append_and_batch_append() {
5233        let backend = mem_backend(1_048_576).await.with_hwm_key(0, [1u8; 32]);
5234        let exec = ExecutionId::new();
5235        assert!(
5236            !backend
5237                .open_execution(exec, ExecutionKind::AgentTurn)
5238                .await
5239                .unwrap()
5240        );
5241        backend.append(step_result(exec, 0, b"v0")).await.unwrap();
5242        backend
5243            .append_batch(&[step_result(exec, 1, b"v1"), step_result(exec, 2, b"v2")])
5244            .await
5245            .unwrap();
5246
5247        assert!(
5248            backend
5249                .open_execution(exec, ExecutionKind::AgentTurn)
5250                .await
5251                .unwrap(),
5252            "resume must succeed when the recomputed count matches the signed HWM"
5253        );
5254    }
5255
5256    #[tokio::test]
5257    async fn hwm_detects_deletion_of_a_committed_step_result() {
5258        let backend = mem_backend(1_048_576).await.with_hwm_key(0, [2u8; 32]);
5259        let exec = ExecutionId::new();
5260        backend
5261            .open_execution(exec, ExecutionKind::AgentTurn)
5262            .await
5263            .unwrap();
5264        backend.append(step_result(exec, 0, b"v0")).await.unwrap();
5265        backend.append(step_result(exec, 1, b"v1")).await.unwrap();
5266
5267        // Simulate an attacker (or a bug) deleting a committed result without going through the
5268        // legitimate `checkpoint_fold` path, which would have kept `folded_count` in sync.
5269        zeph_db::query(sql!(
5270            "DELETE FROM durable_journal WHERE execution_id = ? AND step_id = 1"
5271        ))
5272        .bind(exec.as_uuid().to_string())
5273        .execute(backend.pool())
5274        .await
5275        .unwrap();
5276
5277        let err = backend
5278            .open_execution(exec, ExecutionKind::AgentTurn)
5279            .await
5280            .unwrap_err();
5281        assert_matches!(
5282            err,
5283            DurableError::HighWaterMarkIntegrity {
5284                reason: "count_mismatch",
5285                ..
5286            }
5287        );
5288
5289        // The execution must be finalized Aborted, not left running for a retry loop to keep
5290        // tripping the same check.
5291        let summaries = backend.list_executions(None, None, 10).await.unwrap();
5292        let summary = summaries.iter().find(|s| s.execution_id == exec).unwrap();
5293        assert_eq!(summary.status, ExecutionStatus::Aborted);
5294    }
5295
5296    #[tokio::test]
5297    async fn hwm_survives_a_legitimate_checkpoint_fold() {
5298        let backend = mem_backend(1_048_576)
5299            .await
5300            .with_cipher(Arc::new(XorCipher))
5301            .with_hwm_key(0, [3u8; 32]);
5302        let exec = ExecutionId::new();
5303        backend
5304            .open_execution(exec, ExecutionKind::AgentTurn)
5305            .await
5306            .unwrap();
5307        for step in 0..5 {
5308            backend
5309                .append(step_result(exec, step, format!("v{step}").as_bytes()))
5310                .await
5311                .unwrap();
5312        }
5313
5314        let folded = backend.checkpoint_fold(exec, 3).await.unwrap();
5315        assert_eq!(folded, 3);
5316
5317        assert!(
5318            backend
5319                .open_execution(exec, ExecutionKind::AgentTurn)
5320                .await
5321                .unwrap(),
5322            "a legitimate fold must not trip the HWM check: committed_result_count is invariant \
5323             across it (folded_count restores what the DELETE removed)"
5324        );
5325    }
5326
5327    /// S1 regression (addendum to #6451, spec-081 FR-008): a pre-rotation execution whose
5328    /// `StepResult`s are checkpoint-folded post-rotation reseals its checkpoint snapshot under
5329    /// the NEW `key_id` and DELETEs every old-key-id `StepResult` row it folds, but
5330    /// `checkpoint_fold` never re-signs the HWM (`committed_result_count` is deliberately
5331    /// invariant across a fold — see the doc on `checkpoint_fold`). So the integrity row keeps
5332    /// `key_epoch = previous_key_id` even once no old-key-id payload survives at all. This
5333    /// execution has `StepResult`s only (no `EffectIntent`), so both the AEAD blob-scan and the
5334    /// control-HMAC scan see nothing — `count_integrity_rows_under_epoch` is the only one of the
5335    /// three `--drop-previous` scans that catches it.
5336    #[tokio::test]
5337    async fn count_integrity_rows_under_epoch_catches_a_checkpoint_folded_pre_rotation_execution() {
5338        let pre_rotation = mem_backend(1_048_576)
5339            .await
5340            .with_cipher(Arc::new(RotatingKeyedCipher {
5341                current_id: 0,
5342                previous_id: None,
5343            }))
5344            .with_hwm_key(0, [20u8; 32]);
5345        let exec = ExecutionId::new();
5346        pre_rotation
5347            .open_execution(exec, ExecutionKind::AgentTurn)
5348            .await
5349            .unwrap();
5350        for step in 0..3 {
5351            pre_rotation
5352                .append(step_result(exec, step, format!("v{step}").as_bytes()))
5353                .await
5354                .unwrap();
5355        }
5356
5357        // Rotate: a fresh handle over the same journal now speaks the NEW current epoch/key-id
5358        // (1), with the old one (0) registered as previous for both the cipher (so fold can still
5359        // decrypt the not-yet-folded rows) and the HWM (the rotation window) — exactly mirroring
5360        // a real `zeph durable rotate-key` followed by a background fold.
5361        let post_rotation = LocalBackend::new(pre_rotation.pool().clone(), 1_048_576)
5362            .with_cipher(Arc::new(RotatingKeyedCipher {
5363                current_id: 1,
5364                previous_id: Some(0),
5365            }))
5366            .with_hwm_key(1, [21u8; 32])
5367            .with_previous_hwm_key(0, [20u8; 32]);
5368
5369        let folded = post_rotation.checkpoint_fold(exec, 3).await.unwrap();
5370        assert_eq!(
5371            folded, 3,
5372            "fold must compact every committed StepResult, leaving none live"
5373        );
5374
5375        assert_eq!(
5376            post_rotation.count_sealed_under_key_id(0).await.unwrap(),
5377            0,
5378            "every pre-rotation payload was folded away and resealed under the new key_id; the \
5379             AEAD scan sees nothing left sealed under the previous key_id"
5380        );
5381        assert_eq!(
5382            post_rotation
5383                .count_integrity_rows_under_epoch(0)
5384                .await
5385                .unwrap(),
5386            1,
5387            "the folded execution's HWM row still carries the previous epoch -- checkpoint_fold \
5388             never re-signs it (S1)"
5389        );
5390        assert_eq!(
5391            post_rotation
5392                .count_integrity_rows_under_epoch(1)
5393                .await
5394                .unwrap(),
5395            0,
5396            "the row has not migrated to the current epoch -- only a fresh StepResult commit \
5397             after resume would bump it"
5398        );
5399
5400        // The folded execution is not corrupted — it must still resume cleanly through the open
5401        // rotation window (this addendum's epoch=key_id design), it is just still dependent on
5402        // the previous HWM key until `--drop-previous` (which S1's fix now correctly refuses).
5403        assert!(
5404            post_rotation
5405                .open_execution(exec, ExecutionKind::AgentTurn)
5406                .await
5407                .unwrap(),
5408            "a folded pre-rotation execution must still resume through the open rotation window"
5409        );
5410    }
5411
5412    #[tokio::test]
5413    async fn hwm_detects_deletion_that_a_fold_does_not_cover() {
5414        let backend = mem_backend(1_048_576)
5415            .await
5416            .with_cipher(Arc::new(XorCipher))
5417            .with_hwm_key(0, [4u8; 32]);
5418        let exec = ExecutionId::new();
5419        backend
5420            .open_execution(exec, ExecutionKind::AgentTurn)
5421            .await
5422            .unwrap();
5423        for step in 0..5 {
5424            backend
5425                .append(step_result(exec, step, format!("v{step}").as_bytes()))
5426                .await
5427                .unwrap();
5428        }
5429        backend.checkpoint_fold(exec, 3).await.unwrap();
5430
5431        // Delete one of the *surviving* (non-folded) rows outside the write path.
5432        zeph_db::query(sql!(
5433            "DELETE FROM durable_journal WHERE execution_id = ? AND step_id = 4 AND entry_kind = 'step_result'"
5434        ))
5435        .bind(exec.as_uuid().to_string())
5436        .execute(backend.pool())
5437        .await
5438        .unwrap();
5439
5440        assert_matches!(
5441            backend
5442                .open_execution(exec, ExecutionKind::AgentTurn)
5443                .await
5444                .unwrap_err(),
5445            DurableError::HighWaterMarkIntegrity {
5446                reason: "count_mismatch",
5447                ..
5448            }
5449        );
5450    }
5451
5452    #[tokio::test]
5453    async fn hwm_unresolvable_key_epoch_fails_closed_not_legacy() {
5454        let writer = mem_backend(1_048_576).await.with_hwm_key(0, [5u8; 32]);
5455        let exec = ExecutionId::new();
5456        writer
5457            .open_execution(exec, ExecutionKind::AgentTurn)
5458            .await
5459            .unwrap();
5460        writer.append(step_result(exec, 0, b"v0")).await.unwrap();
5461
5462        // A different backend over the same journal, current epoch 9, no previous slot registered
5463        // for epoch 0 — the stored row's epoch is unresolvable. Per NFR-004/S-new-2 this must fail
5464        // closed, never silently degrade to "legacy" just because the row's epoch is unknown here.
5465        let reader = LocalBackend::new(writer.pool().clone(), 1_048_576).with_hwm_key(9, [6u8; 32]);
5466        assert_matches!(
5467            reader
5468                .open_execution(exec, ExecutionKind::AgentTurn)
5469                .await
5470                .unwrap_err(),
5471            DurableError::HighWaterMarkIntegrity {
5472                reason: "key_epoch_unresolvable",
5473                ..
5474            }
5475        );
5476    }
5477
5478    #[tokio::test]
5479    async fn hwm_previous_epoch_key_resolves_as_rekeyed_not_tampered() {
5480        let writer = mem_backend(1_048_576).await.with_hwm_key(0, [7u8; 32]);
5481        let exec = ExecutionId::new();
5482        writer
5483            .open_execution(exec, ExecutionKind::AgentTurn)
5484            .await
5485            .unwrap();
5486        writer.append(step_result(exec, 0, b"v0")).await.unwrap();
5487
5488        // A rotated backend: current epoch 1 under a new key, but the old epoch-0 key is still
5489        // registered as `previous` for the rotation window (FR-008). Verification must succeed via
5490        // the previous slot rather than reporting tamper.
5491        let reader = LocalBackend::new(writer.pool().clone(), 1_048_576)
5492            .with_hwm_key(1, [8u8; 32])
5493            .with_previous_hwm_key(0, [7u8; 32]);
5494        assert!(
5495            reader
5496                .open_execution(exec, ExecutionKind::AgentTurn)
5497                .await
5498                .unwrap(),
5499            "a row signed under a registered previous epoch must verify, not fail as tampered"
5500        );
5501    }
5502
5503    #[tokio::test]
5504    async fn hwm_wrong_key_under_the_same_epoch_is_tamper() {
5505        let writer = mem_backend(1_048_576).await.with_hwm_key(0, [9u8; 32]);
5506        let exec = ExecutionId::new();
5507        writer
5508            .open_execution(exec, ExecutionKind::AgentTurn)
5509            .await
5510            .unwrap();
5511        writer.append(step_result(exec, 0, b"v0")).await.unwrap();
5512
5513        let reader =
5514            LocalBackend::new(writer.pool().clone(), 1_048_576).with_hwm_key(0, [10u8; 32]);
5515        assert_matches!(
5516            reader
5517                .open_execution(exec, ExecutionKind::AgentTurn)
5518                .await
5519                .unwrap_err(),
5520            DurableError::HighWaterMarkIntegrity {
5521                reason: "hmac_mismatch",
5522                ..
5523            }
5524        );
5525    }
5526
5527    #[tokio::test]
5528    async fn hwm_accepts_a_legacy_execution_with_no_integrity_row() {
5529        // Entries written by an unkeyed backend leave no `durable_execution_integrity` row at all —
5530        // the genuine "predates this feature" case, distinct from a row that exists but is
5531        // unresolvable. A keyed backend resuming it must accept it (migration posture), not fail.
5532        let unkeyed_writer = mem_backend(1_048_576).await;
5533        let exec = ExecutionId::new();
5534        unkeyed_writer
5535            .open_execution(exec, ExecutionKind::AgentTurn)
5536            .await
5537            .unwrap();
5538        unkeyed_writer
5539            .append(step_result(exec, 0, b"v0"))
5540            .await
5541            .unwrap();
5542
5543        let keyed_reader =
5544            LocalBackend::new(unkeyed_writer.pool().clone(), 1_048_576).with_hwm_key(0, [11u8; 32]);
5545        assert!(
5546            keyed_reader
5547                .open_execution(exec, ExecutionKind::AgentTurn)
5548                .await
5549                .unwrap(),
5550            "an execution with no integrity row at all is legacy, not tampered"
5551        );
5552    }
5553
5554    // --- Vault-sealed integrity boundary tests (issue #6449) ---
5555
5556    #[tokio::test]
5557    async fn hwm_unsealed_absent_row_after_deletion_is_still_ok() {
5558        // A keyed but *unsealed* backend (the pre-#6449-cutover posture): even after a committed
5559        // StepResult's integrity row is deleted, resume must still succeed — the migration
5560        // posture unless/until an operator explicitly seals.
5561        let backend = mem_backend(1_048_576).await.with_hwm_key(0, [30u8; 32]);
5562        let exec = ExecutionId::new();
5563        backend
5564            .open_execution(exec, ExecutionKind::AgentTurn)
5565            .await
5566            .unwrap();
5567        backend.append(step_result(exec, 0, b"v0")).await.unwrap();
5568
5569        zeph_db::query(sql!(
5570            "DELETE FROM durable_execution_integrity WHERE execution_id = ?"
5571        ))
5572        .bind(exec.as_uuid().to_string())
5573        .execute(backend.pool())
5574        .await
5575        .unwrap();
5576
5577        assert!(
5578            backend
5579                .open_execution(exec, ExecutionKind::AgentTurn)
5580                .await
5581                .unwrap(),
5582            "unsealed backend must not treat an absent integrity row as tamper"
5583        );
5584    }
5585
5586    #[tokio::test]
5587    async fn hwm_post_seal_absent_row_with_committed_results_is_tamper() {
5588        let backend = mem_backend(1_048_576)
5589            .await
5590            .with_hwm_key(0, [31u8; 32])
5591            .with_integrity_sealed(true);
5592        let exec = ExecutionId::new();
5593        backend
5594            .open_execution(exec, ExecutionKind::AgentTurn)
5595            .await
5596            .unwrap();
5597        backend.append(step_result(exec, 0, b"v0")).await.unwrap();
5598
5599        // Attacker (DB write access) deletes the integrity row, keeping the committed
5600        // StepResult in place to replay it.
5601        zeph_db::query(sql!(
5602            "DELETE FROM durable_execution_integrity WHERE execution_id = ?"
5603        ))
5604        .bind(exec.as_uuid().to_string())
5605        .execute(backend.pool())
5606        .await
5607        .unwrap();
5608
5609        let err = backend
5610            .open_execution(exec, ExecutionKind::AgentTurn)
5611            .await
5612            .unwrap_err();
5613        assert_matches!(
5614            err,
5615            DurableError::HighWaterMarkIntegrity {
5616                reason: "integrity_row_absent_post_seal",
5617                ..
5618            }
5619        );
5620    }
5621
5622    #[tokio::test]
5623    async fn hwm_post_seal_forged_created_at_does_not_evade_the_seal() {
5624        // Proves S1 is fully closed: the boundary no longer consults `created_at` at all, so
5625        // an attacker forging it (the rev1 defeat) has no effect once sealed.
5626        let backend = mem_backend(1_048_576)
5627            .await
5628            .with_hwm_key(0, [32u8; 32])
5629            .with_integrity_sealed(true);
5630        let exec = ExecutionId::new();
5631        backend
5632            .open_execution(exec, ExecutionKind::AgentTurn)
5633            .await
5634            .unwrap();
5635        backend.append(step_result(exec, 0, b"v0")).await.unwrap();
5636
5637        zeph_db::query(sql!(
5638            "UPDATE durable_executions SET created_at = 0 WHERE execution_id = ?"
5639        ))
5640        .bind(exec.as_uuid().to_string())
5641        .execute(backend.pool())
5642        .await
5643        .unwrap();
5644        zeph_db::query(sql!(
5645            "DELETE FROM durable_execution_integrity WHERE execution_id = ?"
5646        ))
5647        .bind(exec.as_uuid().to_string())
5648        .execute(backend.pool())
5649        .await
5650        .unwrap();
5651
5652        let err = backend
5653            .open_execution(exec, ExecutionKind::AgentTurn)
5654            .await
5655            .unwrap_err();
5656        assert_matches!(
5657            err,
5658            DurableError::HighWaterMarkIntegrity {
5659                reason: "integrity_row_absent_post_seal",
5660                ..
5661            },
5662            "forging created_at must not evade the seal — it is never consulted"
5663        );
5664    }
5665
5666    #[tokio::test]
5667    async fn hwm_grandfathered_execution_absent_row_is_ok() {
5668        let exec = ExecutionId::new();
5669        let writer = mem_backend(1_048_576).await.with_hwm_key(0, [33u8; 32]);
5670        writer
5671            .open_execution(exec, ExecutionKind::AgentTurn)
5672            .await
5673            .unwrap();
5674        writer.append(step_result(exec, 0, b"v0")).await.unwrap();
5675        zeph_db::query(sql!(
5676            "DELETE FROM durable_execution_integrity WHERE execution_id = ?"
5677        ))
5678        .bind(exec.as_uuid().to_string())
5679        .execute(writer.pool())
5680        .await
5681        .unwrap();
5682
5683        let sealed_but_grandfathered = LocalBackend::new(writer.pool().clone(), 1_048_576)
5684            .with_hwm_key(0, [33u8; 32])
5685            .with_integrity_sealed(true)
5686            .with_grandfather(std::collections::HashSet::from([exec]));
5687
5688        assert!(
5689            sealed_but_grandfathered
5690                .open_execution(exec, ExecutionKind::AgentTurn)
5691                .await
5692                .unwrap(),
5693            "a grandfathered execution_id must resume despite the seal"
5694        );
5695    }
5696
5697    #[tokio::test]
5698    async fn find_unsealed_resumable_executions_finds_only_the_offending_set() {
5699        let backend = mem_backend(1_048_576).await.with_hwm_key(0, [35u8; 32]);
5700
5701        // (a) running, keyed, committed StepResult, integrity row deleted — the offending case.
5702        let offending = ExecutionId::new();
5703        backend
5704            .open_execution(offending, ExecutionKind::AgentTurn)
5705            .await
5706            .unwrap();
5707        backend
5708            .append(step_result(offending, 0, b"v0"))
5709            .await
5710            .unwrap();
5711        zeph_db::query(sql!(
5712            "DELETE FROM durable_execution_integrity WHERE execution_id = ?"
5713        ))
5714        .bind(offending.as_uuid().to_string())
5715        .execute(backend.pool())
5716        .await
5717        .unwrap();
5718
5719        // (b) running, keyed, has an intact integrity row — not offending.
5720        let intact = ExecutionId::new();
5721        backend
5722            .open_execution(intact, ExecutionKind::AgentTurn)
5723            .await
5724            .unwrap();
5725        backend.append(step_result(intact, 0, b"v0")).await.unwrap();
5726
5727        // (c) running, no committed results at all — not offending (nothing to smuggle).
5728        let empty = ExecutionId::new();
5729        backend
5730            .open_execution(empty, ExecutionKind::AgentTurn)
5731            .await
5732            .unwrap();
5733
5734        // (d) terminal (finalized), integrity row absent — not offending (can never resume again).
5735        let terminal = ExecutionId::new();
5736        backend
5737            .open_execution(terminal, ExecutionKind::AgentTurn)
5738            .await
5739            .unwrap();
5740        backend
5741            .append(step_result(terminal, 0, b"v0"))
5742            .await
5743            .unwrap();
5744        zeph_db::query(sql!(
5745            "DELETE FROM durable_execution_integrity WHERE execution_id = ?"
5746        ))
5747        .bind(terminal.as_uuid().to_string())
5748        .execute(backend.pool())
5749        .await
5750        .unwrap();
5751        backend
5752            .finalize(terminal, ExecutionStatus::Completed)
5753            .await
5754            .unwrap();
5755
5756        let found = backend.find_unsealed_resumable_executions().await.unwrap();
5757        assert_eq!(
5758            found,
5759            vec![offending],
5760            "only the truly offending execution must be returned"
5761        );
5762    }
5763
5764    #[tokio::test]
5765    async fn hwm_post_seal_absent_row_with_zero_committed_results_is_ok() {
5766        // A sealed backend with no committed StepResult at all (e.g. an execution that was
5767        // opened but never produced a result) has nothing to smuggle — accepted even post-seal.
5768        let backend = mem_backend(1_048_576)
5769            .await
5770            .with_hwm_key(0, [34u8; 32])
5771            .with_integrity_sealed(true);
5772        let exec = ExecutionId::new();
5773        backend
5774            .open_execution(exec, ExecutionKind::AgentTurn)
5775            .await
5776            .unwrap();
5777
5778        assert!(
5779            backend
5780                .open_execution(exec, ExecutionKind::AgentTurn)
5781                .await
5782                .unwrap(),
5783            "zero committed results, post-seal, must not be treated as tamper"
5784        );
5785    }
5786
5787    #[tokio::test]
5788    async fn hwm_ignores_effect_intent_and_control_entries() {
5789        // Only `StepResult` rows count toward `committed_result_count` (S-new-1) — an EffectIntent
5790        // must not bump the HWM, and its presence alone must not trip verification.
5791        let backend = mem_backend(1_048_576).await.with_hwm_key(0, [12u8; 32]);
5792        let exec = ExecutionId::new();
5793        backend
5794            .open_execution(exec, ExecutionKind::AgentTurn)
5795            .await
5796            .unwrap();
5797        backend.append(effect_intent(exec, 0)).await.unwrap();
5798        backend.append(step_result(exec, 1, b"v1")).await.unwrap();
5799
5800        let stored: (i64,) = zeph_db::query_as(sql!(
5801            "SELECT committed_result_count FROM durable_execution_integrity WHERE execution_id = ?"
5802        ))
5803        .bind(exec.as_uuid().to_string())
5804        .fetch_one(backend.pool())
5805        .await
5806        .unwrap();
5807        assert_eq!(
5808            stored.0, 1,
5809            "only the StepResult row counts, not the EffectIntent"
5810        );
5811
5812        assert!(
5813            backend
5814                .open_execution(exec, ExecutionKind::AgentTurn)
5815                .await
5816                .unwrap()
5817        );
5818    }
5819}