Skip to main content

zeph_durable/backend/
local.rs

1// SPDX-FileCopyrightText: 2026 Andrei G <bug-ops>
2// SPDX-License-Identifier: MIT OR Apache-2.0
3
4//! The always-compiled local journal backend.
5//!
6//! [`LocalBackend`] owns a dedicated [`zeph_db::DbPool`] on its own `durable.db` file (INV-14): a
7//! separate database keeps the high-write journal off the shared application pool, where
8//! `BEGIN IMMEDIATE` contention would otherwise serialize unrelated writers. The schema lives in
9//! `zeph-db/migrations/{sqlite,postgres}/` and is applied via [`zeph_db::run_migrations`]; the
10//! backend owns no `.sql` files of its own.
11//!
12//! # Sealing and integrity
13//!
14//! Payload-bearing entries (currently [`EntryKind::StepResult`]) are AEAD-sealed through the
15//! injected [`PayloadCipher`] before they touch the database, with the entry's location bound as
16//! associated data so a sealed blob cannot be relocated to another step or execution. Control
17//! entries (currently [`EntryKind::EffectIntent`]) carry no payload; when an HMAC key is configured
18//! the backend stamps a keyed BLAKE3 row HMAC over their identity for shared-database deployments,
19//! and every read recomputes and constant-time-verifies that HMAC, failing closed with
20//! [`DurableError::ControlIntegrity`] on a forged or relocated row. When no cipher is injected the
21//! payload is stored verbatim — a development-only posture gated by
22//! [`encryption_gate`](crate::encryption_gate) at startup.
23//!
24//! # Scope
25//!
26//! This revision journals the step-execution entries — [`EntryKind::StepResult`] and
27//! [`EntryKind::EffectIntent`] — that the durable step primitive records, plus execution
28//! lifecycle (open and [`finalize`](Journal::finalize)), the writer's restart anchor (`max_seq`),
29//! and the idempotency-key point lookup
30//! ([`lookup_committed_result`](ExecutionBackend::lookup_committed_result)) that lets a guarded step
31//! recognize an already-committed effect after a replay divergence (INV-13). Promise, timer, and
32//! checkpoint entries are journaled by the promise/timer and retention layers; until then
33//! [`append`](Journal::append) of those kinds fails closed with
34//! [`DurableError::UnsupportedEntryKind`] rather than dropping their state. The retention sweep
35//! ([`prune`](Journal::prune)) is a no-op stub here.
36
37use std::fmt;
38use std::fmt::Write as _;
39use std::path::PathBuf;
40use std::sync::Arc;
41use std::time::{SystemTime, UNIX_EPOCH};
42
43use bytes::Bytes;
44use zeph_db::{DbPool, sql};
45
46use crate::backend::execution_lock::ExecutionLock;
47use crate::backend::{BackendCapabilities, ExecutionBackend, ExecutionSummary, RedactedEntry};
48use crate::cipher::{EntryKindTag, PayloadAad, PayloadCipher, ensure_payload_within_limit};
49use crate::config::RetentionPolicy;
50use crate::error::DurableError;
51use crate::ids::{
52    ExecutionId, ExecutionKind, IdempotencyKey, JournalSeq, PromiseId, StepId, TimerId,
53};
54use crate::journal::{EntryKind, ExecutionStatus, Journal, JournalEntry};
55use crate::promise::PromiseRecord;
56use crate::retention::{CheckpointSnapshot, FoldedStep, decode_checkpoint, encode_checkpoint};
57use crate::waiters::NotifyRegistry;
58use tracing::Instrument as _;
59
60/// Slack added to `max_payload_bytes` for the read-side size guard.
61///
62/// The stored blob carries AEAD framing (key-id, extended nonce, tag) on top of the plaintext, so a
63/// payload accepted at exactly the limit on write is slightly larger on read. The guard exists only
64/// to reject absurdly large rows before allocation/decryption (INV-11), so a small fixed slack
65/// above any real AEAD overhead keeps legitimate near-limit entries readable without weakening the
66/// denial-of-service protection.
67const SEAL_OVERHEAD_SLACK: u64 = 128;
68
69/// Row shape returned by the `list_executions` query.
70type ExecutionRow = (String, String, String, i64, i64, Option<i64>, i64);
71
72/// Row shape returned by the `read_execution_redacted` query.
73type RedactedRow = (
74    i64,
75    i64,
76    String,
77    Option<Vec<u8>>,
78    Option<String>,
79    Option<i64>,
80    i64,
81);
82
83/// Render the first 8 bytes of an idempotency key as a lowercase hex prefix (INV-5).
84fn idem_key_prefix(bytes: &[u8]) -> String {
85    bytes.iter().take(8).fold(String::new(), |mut acc, b| {
86        let _ = write!(acc, "{b:02x}");
87        acc
88    })
89}
90
91/// Outcome of a [`LocalBackend::cancel_execution`] request (#6362).
92///
93/// `Canceled` is the only outcome that wrote to the row; every other variant is a refusal or a
94/// no-op, so a caller can always trust "did this call change the database" from the variant alone.
95#[derive(Debug, Clone, Copy, PartialEq, Eq)]
96pub enum CancelOutcome {
97    /// The execution was `running` with no live owner detected; it is now `canceled` and will
98    /// never be reopened (INV-16′).
99    Canceled,
100    /// The execution was already terminal (`completed`, `failed`, `aborted`, or already
101    /// `canceled`) — idempotent no-op, per NFR-003.
102    AlreadyTerminal {
103        /// The execution's status before (and, since no write happened, after) this call.
104        status: ExecutionStatus,
105    },
106    /// No execution exists for the given id.
107    NotFound,
108    /// Another process holds the execution's [`ExecutionLock`] (SQLite/Unix only). This may be the
109    /// execution's true owner still journaling, or a concurrent maintenance sweep/prune/cancel
110    /// transiently holding the same lock — the flock alone cannot distinguish the two, so no claim
111    /// stronger than "held" is made. The row was not touched; cooperative live-owner cancellation
112    /// (FR-007) is deferred to a follow-up issue.
113    LiveOwner {
114        /// PID of the process currently holding the lock, or `0` if it could not be determined.
115        pid: u32,
116    },
117    /// This backend cannot verify whether a live owner holds the execution (a cross-process
118    /// backend, e.g. Postgres, with no advisory-lock directory to probe). Refusing rather than
119    /// blind-flipping a possibly-live row (F3).
120    LivenessUnverifiable,
121}
122
123/// The always-compiled durable backend that journals to a dedicated `durable.db`.
124///
125/// Construct it from a [`zeph_db::DbPool`] (or open one with [`LocalBackend::open`]), then attach an
126/// optional [`PayloadCipher`] and HMAC key with the builder methods. Call [`LocalBackend::init`]
127/// once before use to apply the schema migrations.
128///
129/// # Examples
130///
131/// ```no_run
132/// # async fn run() -> Result<(), Box<dyn std::error::Error>> {
133/// use zeph_durable::LocalBackend;
134///
135/// // 1 MiB payload ceiling, matching the spec default.
136/// let backend = LocalBackend::open("durable.db", 1_048_576).await?;
137/// backend.init().await?;
138/// # Ok(()) }
139/// ```
140pub struct LocalBackend {
141    pool: DbPool,
142    cipher: Option<Arc<dyn PayloadCipher>>,
143    hmac_key: Option<[u8; 32]>,
144    /// Previous control-entry HMAC key for the rotation window (#6451), mirroring the AEAD
145    /// cipher's `previous` slot. `Some` only while a `zeph durable rotate-key` window is open
146    /// (`config.previous_key_id.is_some()`); `verify_control_hmac` tries this key when a row
147    /// fails to verify under `hmac_key`. Writes always stamp with `hmac_key` only.
148    previous_hmac_key: Option<[u8; 32]>,
149    /// The current high-water-mark key (issue #6360), keyed by its non-secret rotation epoch.
150    /// `None` disables the HWM: no bump on commit/fold, and [`open_execution`](Self::open_execution)
151    /// skips its internal high-water-mark verification entirely on resume. Unlike
152    /// `hmac_key` (shared-DB gated, INV-8), the HWM key is meant to be attached unconditionally
153    /// (FR-009) — it is the only mechanism that detects deletion of a committed `StepResult` row,
154    /// a threat class the AEAD payload seal and the row HMAC do not cover on any deployment,
155    /// single-user local included.
156    hwm_key: Option<HwmKeySlot>,
157    /// A previous HWM key, still accepted for verification during a rotation window (FR-008). Never
158    /// used to sign new writes — every bump/fold always signs under `hwm_key`.
159    hwm_key_previous: Option<HwmKeySlot>,
160    max_payload_bytes: u64,
161    /// In-process wakeup map for parked promise awaits, shared with the resolver path.
162    promise_waiters: NotifyRegistry,
163    /// In-process wakeup map for parked timers, shared with the timer service.
164    timer_waiters: NotifyRegistry,
165    /// Directory for per-`ExecutionId` advisory lock files (INV-15, #6122), used by
166    /// [`open_execution_exclusive`](Self::open_execution_exclusive). `None` when no on-disk path
167    /// is known for this backend — a `:memory:` database, a backend built via
168    /// [`LocalBackend::new`] from a caller-supplied pool, or a non-SQLite (Postgres) deployment,
169    /// where a filesystem lock file cannot express cross-process exclusivity anyway.
170    lock_dir: Option<PathBuf>,
171    /// Set once [`sweep_orphans`](Self::sweep_orphans) has emitted its warn-once log for a
172    /// `lock_dir = None` backend (#6254), so a background retention tick every
173    /// `prune_interval_secs` does not spam the log for the lifetime of the process.
174    orphan_sweep_warned: std::sync::atomic::AtomicBool,
175    /// Vault-sealed integrity marker (issue #6449). `true` only when the *presence* of
176    /// `ZEPH_DURABLE_INTEGRITY_SEALED` in the vault was confirmed at bootstrap — an
177    /// attacker with DB write access cannot set this to `true` (it is never derived from any DB
178    /// column). Once sealed, [`check_high_water_mark`](Self::check_high_water_mark) treats an
179    /// absent integrity row on a keyed, non-grandfathered execution with committed `StepResult`s
180    /// as unconditional tamper, closing the pre-seal migration posture's downgrade lever.
181    integrity_sealed: bool,
182    /// Execution IDs explicitly grandfathered past the seal via `zeph durable seal-integrity
183    /// --grandfather` (issue #6449) — a vault-stored, unforgeable-by-DB-write set. Each entry is
184    /// a **permanent** opt-out for that one execution (not merely a frozen pre-seal snapshot): an
185    /// attacker with DB write access can delete-and-reinsert forged content under the same
186    /// grandfathered `execution_id` and it will still resume unverified. This is an accepted,
187    /// bounded, documented residual of the opt-out — operators should prefer draining a
188    /// resumable execution to a terminal state over grandfathering it.
189    integrity_grandfather: std::collections::HashSet<ExecutionId>,
190}
191
192/// One row-HMAC/high-water-mark key, addressed by its non-secret rotation epoch (FR-008).
193///
194/// The epoch is not sensitive (it is stored in the clear alongside the signed HWM tuple) — it lets
195/// a verifier distinguish "signed under a key I don't currently hold" (re-keyed) from "signed under
196/// my current key but the hash doesn't match" (tampered), per FR-008.
197#[derive(Clone, Copy)]
198struct HwmKeySlot {
199    epoch: u32,
200    key: [u8; 32],
201}
202
203impl fmt::Debug for LocalBackend {
204    /// Redacts the cipher and HMAC/HWM key material — never print key bytes or a cipher handle.
205    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
206        f.debug_struct("LocalBackend")
207            .field("cipher", &self.cipher.as_ref().map(|_| "<cipher>"))
208            .field("hmac_key", &self.hmac_key.as_ref().map(|_| "<redacted>"))
209            .field(
210                "previous_hmac_key",
211                &self.previous_hmac_key.as_ref().map(|_| "<redacted>"),
212            )
213            .field("hwm_key_epoch", &self.hwm_key.as_ref().map(|s| s.epoch))
214            .field("max_payload_bytes", &self.max_payload_bytes)
215            .finish_non_exhaustive()
216    }
217}
218
219/// Construction, builders, and accessors.
220impl LocalBackend {
221    /// Wrap an existing [`zeph_db::DbPool`] as a local backend with the given payload ceiling.
222    ///
223    /// Call [`LocalBackend::init`] before any journal operation to apply the schema. Attach a
224    /// cipher and HMAC key with [`with_cipher`](Self::with_cipher) and
225    /// [`with_hmac_key`](Self::with_hmac_key).
226    #[must_use]
227    pub fn new(pool: DbPool, max_payload_bytes: u64) -> Self {
228        Self {
229            pool,
230            cipher: None,
231            hmac_key: None,
232            previous_hmac_key: None,
233            hwm_key: None,
234            hwm_key_previous: None,
235            max_payload_bytes,
236            promise_waiters: NotifyRegistry::default(),
237            timer_waiters: NotifyRegistry::default(),
238            lock_dir: None,
239            orphan_sweep_warned: std::sync::atomic::AtomicBool::new(false),
240            integrity_sealed: false,
241            integrity_grandfather: std::collections::HashSet::new(),
242        }
243    }
244
245    /// Open (or create) a backend on a dedicated `durable.db` file (or `:memory:`).
246    ///
247    /// Connecting also applies the schema migrations, so a freshly opened backend is ready to use;
248    /// [`init`](Self::init) may still be called and is idempotent.
249    ///
250    /// On the `SQLite` backend, also derives the lock directory used by
251    /// [`open_execution_exclusive`](Self::open_execution_exclusive) from `path` (a sibling
252    /// `<path>.locks/` directory), unless `path` is `:memory:`. The Postgres backend never derives
253    /// one — `path` there is a connection URL (which may embed credentials), not a filesystem path.
254    ///
255    /// # Errors
256    ///
257    /// Returns [`DurableError::Storage`] if the pool cannot be opened or migrations fail.
258    pub async fn open(path: &str, max_payload_bytes: u64) -> Result<Self, DurableError> {
259        let pool = zeph_db::DbConfig {
260            url: path.to_string(),
261            pool_size: 5,
262        }
263        .connect()
264        .await
265        .map_err(|e| DurableError::storage("open", e))?;
266        let mut backend = Self::new(pool, max_payload_bytes);
267        backend.lock_dir = lock_dir_for_path(path);
268        Ok(backend)
269    }
270
271    /// Inject the AEAD payload cipher used to seal and open payload-bearing entries.
272    #[must_use]
273    pub fn with_cipher(mut self, cipher: Arc<dyn PayloadCipher>) -> Self {
274        self.cipher = Some(cipher);
275        self
276    }
277
278    /// Configure the keyed-BLAKE3 HMAC key stamped over control entries on shared-database
279    /// deployments, and used to verify them again on every read (INV-8).
280    #[must_use]
281    pub fn with_hmac_key(mut self, key: [u8; 32]) -> Self {
282        self.hmac_key = Some(key);
283        self
284    }
285
286    /// Register a previous control-entry HMAC key for the rotation window (#6451), mirroring the
287    /// AEAD cipher's `with_previous` window mechanism (`zeph_core::durable::XChaCha20Poly1305Cipher`).
288    ///
289    /// The row-HMAC verification path tries this key when a row fails to
290    /// verify under the current [`with_hmac_key`](Self::with_hmac_key) key, so pre-rotation
291    /// `EffectIntent` control entries stay readable until the window is closed with `zeph durable
292    /// rotate-key --drop-previous`. Unlike the AEAD cipher's `key_id`-tagged blob layout, the
293    /// stored `hmac` column carries no key selector — a deliberate divergence, since control rows
294    /// have no payload envelope to carry one; try-both is security-equivalent for a single-slot
295    /// window. Writes always stamp with the current key only, never this one.
296    #[must_use]
297    pub fn with_previous_hmac_key(mut self, key: [u8; 32]) -> Self {
298        self.previous_hmac_key = Some(key);
299        self
300    }
301
302    /// Configure the current high-water-mark key (issue #6360), addressed by its non-secret
303    /// rotation `epoch`.
304    ///
305    /// Unlike [`with_hmac_key`](Self::with_hmac_key), this is meant to be attached unconditionally
306    /// (FR-009) — attach it whenever `ZEPH_DURABLE_KEY` resolves from the vault, regardless of
307    /// `shared_db`. When set, every committed `StepResult` bumps the signed
308    /// `{key_epoch, max_committed_step_id, committed_result_count}` tuple in-transaction, and
309    /// [`open_execution`](Self::open_execution) verifies it on every resume (FR-004, US-003).
310    #[must_use]
311    pub fn with_hwm_key(mut self, epoch: u32, key: [u8; 32]) -> Self {
312        self.hwm_key = Some(HwmKeySlot { epoch, key });
313        self
314    }
315
316    /// Register a previous high-water-mark key for the rotation window (FR-008).
317    ///
318    /// Verification tries [`hwm_key`](Self::with_hwm_key) first by epoch match, then this slot —
319    /// never the reverse. New writes always sign under the current key regardless of this slot.
320    #[must_use]
321    pub fn with_previous_hwm_key(mut self, epoch: u32, key: [u8; 32]) -> Self {
322        self.hwm_key_previous = Some(HwmKeySlot { epoch, key });
323        self
324    }
325
326    /// Configure whether this backend has been sealed against pre-feature integrity-row absence
327    /// (issue #6449). Pass `true` only when the vault-stored `ZEPH_DURABLE_INTEGRITY_SEALED`
328    /// marker's *presence* was confirmed at bootstrap — never derive this from any DB column
329    /// (that was the S1 defeat the vault-sealed design fixes; see `check_high_water_mark`'s
330    /// doc).
331    #[must_use]
332    pub fn with_integrity_sealed(mut self, sealed: bool) -> Self {
333        self.integrity_sealed = sealed;
334        self
335    }
336
337    /// Register the vault-stored set of execution IDs grandfathered past the integrity seal
338    /// (issue #6449). Each grandfathered id is a *permanent* forge-able slot (not merely a
339    /// frozen pre-existing posture): an attacker with DB write access can delete and re-insert
340    /// forged content under the same id. This is an accepted, bounded, documented operator
341    /// opt-out — prefer draining a resumable execution to a terminal status where practical.
342    #[must_use]
343    pub fn with_grandfather(mut self, ids: std::collections::HashSet<ExecutionId>) -> Self {
344        self.integrity_grandfather = ids;
345        self
346    }
347
348    /// Borrow the underlying pool (for tests and adapters that need direct access).
349    #[must_use]
350    pub fn pool(&self) -> &DbPool {
351        &self.pool
352    }
353
354    /// Apply the durable schema migrations to the backing pool.
355    ///
356    /// Idempotent: safe to call repeatedly. The schema is owned by `zeph-db`, not this crate.
357    ///
358    /// # Errors
359    ///
360    /// Returns [`DurableError::Storage`] if a migration fails.
361    pub async fn init(&self) -> Result<(), DurableError> {
362        zeph_db::run_migrations(&self.pool)
363            .await
364            .map_err(|e| DurableError::storage("init", e))?;
365        Ok(())
366    }
367}
368
369/// Execution lifecycle: CRUD, journal-row mapping, checkpoints, and promise/timer resolution.
370impl LocalBackend {
371    /// List execution summaries for operability surfaces (the `zeph durable` CLI and TUI).
372    ///
373    /// Returns at most `limit` executions, newest first, optionally filtered by `status` and `kind`
374    /// (each is matched against the raw column tag; `None` disables that filter). Only execution-level
375    /// metadata is read — never payload bytes or resolver tokens (INV-5). The per-execution step
376    /// count is the number of journal entries recorded for it.
377    ///
378    /// Span: `durable.backend.list`.
379    ///
380    /// # Errors
381    ///
382    /// Returns [`DurableError::Storage`] if the query fails, or [`DurableError::Decode`] if a stored
383    /// id or status cannot be reconstructed (schema corruption — the `status` column is
384    /// `CHECK`-constrained, so this is a fail-closed guard rather than a routine path).
385    pub async fn list_executions(
386        &self,
387        status: Option<&str>,
388        kind: Option<&str>,
389        limit: i64,
390    ) -> Result<Vec<ExecutionSummary>, DurableError> {
391        let span = tracing::info_span!(
392            "durable.backend.list",
393            status = status.unwrap_or("*"),
394            kind = kind.unwrap_or("*"),
395            count = tracing::field::Empty,
396        );
397        async move {
398            // `COALESCE(?, col)` keeps a single positional bind per filter and lets the column type
399            // drive the bind type, so the same literal works on both SQLite and Postgres without a
400            // cast on the `?` placeholder.
401            let rows: Vec<ExecutionRow> =
402                zeph_db::query_as(sql!(
403                    "SELECT
404                        e.execution_id,
405                        e.kind,
406                        e.status,
407                        e.created_at,
408                        e.updated_at,
409                        e.finalized_at,
410                        (SELECT COUNT(*) FROM durable_journal j WHERE j.execution_id = e.execution_id)
411                     FROM durable_executions e
412                     WHERE e.status = COALESCE(?, e.status)
413                       AND e.kind = COALESCE(?, e.kind)
414                     ORDER BY e.created_at DESC
415                     LIMIT ?"
416                ))
417                .bind(status)
418                .bind(kind)
419                .bind(limit)
420                .fetch_all(&self.pool)
421                .await
422                .map_err(|e| DurableError::storage("list", e))?;
423            tracing::Span::current().record("count", rows.len());
424            rows.into_iter()
425                .map(|(id, kind, status, created, updated, finalized, steps)| {
426                    Ok(ExecutionSummary {
427                        execution_id: parse_execution_id(&id)?,
428                        kind,
429                        status: ExecutionStatus::from_tag(&status).ok_or(DurableError::Decode {
430                            context: "execution status is not a recognized CHECK-constrained value",
431                        })?,
432                        created_at_ms: created,
433                        updated_at_ms: updated,
434                        finalized_at_ms: finalized,
435                        step_count: steps.max(0).cast_unsigned(),
436                    })
437                })
438                .collect()
439        }
440        .instrument(span)
441        .await
442    }
443
444    /// Look up a single execution's current status, without touching journal entries or payloads.
445    ///
446    /// Backs the `zeph durable resume` CLI's canceled-refusal check (FR-011): resume must report a
447    /// `canceled` execution distinctly from "no adapters wired", which requires knowing the status
448    /// before deciding which message to print.
449    ///
450    /// # Errors
451    ///
452    /// Returns [`DurableError::Storage`] if the query fails, or [`DurableError::Decode`] if the
453    /// stored status cannot be reconstructed (schema corruption — the column is `CHECK`-constrained).
454    pub async fn execution_status(
455        &self,
456        id: ExecutionId,
457    ) -> Result<Option<ExecutionStatus>, DurableError> {
458        let row: Option<(String,)> = zeph_db::query_as(sql!(
459            "SELECT status FROM durable_executions WHERE execution_id = ?"
460        ))
461        .bind(id.as_uuid().to_string())
462        .fetch_optional(&self.pool)
463        .await
464        .map_err(|e| DurableError::storage("execution_status", e))?;
465        row.map(|(status,)| {
466            ExecutionStatus::from_tag(&status).ok_or(DurableError::Decode {
467                context: "execution status is not a recognized CHECK-constrained value",
468            })
469        })
470        .transpose()
471    }
472
473    /// Read one execution's journal entries as redaction-safe metadata, without decrypting payloads.
474    ///
475    /// Unlike [`read_execution`](Journal::read_execution), this never touches the cipher, so it works
476    /// against a journal whose AEAD key is unavailable and never exposes plaintext (INV-5). It backs
477    /// the default (redacted) `zeph durable show`/`inspect` output. Entries are returned in append
478    /// order.
479    ///
480    /// Span: `durable.backend.read_redacted`.
481    ///
482    /// # Errors
483    ///
484    /// Returns [`DurableError::Storage`] if the query fails.
485    pub async fn read_execution_redacted(
486        &self,
487        id: ExecutionId,
488    ) -> Result<Vec<RedactedEntry>, DurableError> {
489        let exec = id.as_uuid().to_string();
490        let rows: Vec<RedactedRow> = zeph_db::query_as(sql!(
491            "SELECT seq, step_id, entry_kind, idem_key, effect_class, LENGTH(payload), created_at
492                 FROM durable_journal WHERE execution_id = ? ORDER BY seq"
493        ))
494        .bind(&exec)
495        .fetch_all(&self.pool)
496        .await
497        .map_err(|e| DurableError::storage("read_redacted", e))?;
498        Ok(rows
499            .into_iter()
500            .map(
501                |(seq, step, entry_kind, idem, effect_class, payload_len, created)| RedactedEntry {
502                    seq,
503                    step_id: StepId::new(u32::try_from(step).unwrap_or(0)),
504                    entry_kind,
505                    effect_class,
506                    idem_key_prefix: idem.as_deref().map(idem_key_prefix),
507                    payload_len: payload_len.unwrap_or(0).max(0).cast_unsigned(),
508                    created_at_ms: created,
509                },
510            )
511            .collect())
512    }
513
514    /// Ensure a `durable_executions` row exists for `id`, returning whether this is a resume.
515    ///
516    /// Inserts a fresh `running` row for a new execution (returning `false`) or detects an existing
517    /// row for a resumed one (returning `true`). The journal's foreign key requires this row before
518    /// any entry is appended, so callers open the execution first.
519    ///
520    /// Reopening a row previously [`finalize`](Journal::finalize)d as `completed`, `failed`, or
521    /// `aborted` un-finalizes it: status resets to `running` and `finalized_at` clears (INV-16′,
522    /// #6254). A `canceled` row is the deliberate exception — see the `canceled` branch below
523    /// (INV-16′, #6362). A caller reopening an execution is, by definition, still using it, so the
524    /// retention sweep (gated on `finalized_at`) must not consider it prunable while it does — without this,
525    /// a long-lived execution finalized at one process's graceful shutdown and legitimately resumed
526    /// by a later process (e.g. a per-conversation `AgentTurn` execution) would keep a stale
527    /// `finalized_at` and could be pruned out from under its still-active journal. `aborted` rows
528    /// are included because the crash-orphan sweep (INV-17) makes `aborted` the common outcome of a
529    /// resumable crash: a resumed execution whose row keeps `finalized_at` set is prunable out from
530    /// under the active resume — the exact hazard this un-finalize prevents for `completed`/`failed`.
531    /// This is also strictly safer for the pre-existing divergence-recovery case, which reopens an
532    /// `aborted` row on purpose: it now also protects that fresh re-drive from prune.
533    ///
534    /// The un-finalize is attempted as a single guarded `UPDATE` (no preceding `SELECT`) so there
535    /// is no read-then-write window against a concurrent prune sweep (#6251 critic S1): if the row
536    /// was deleted by `prune` between an earlier observation and this call, the `UPDATE` simply
537    /// matches zero rows rather than silently resurrecting a half-deleted row. A zero-row `UPDATE`
538    /// falls back to checking whether the row exists at all (already `running`/`aborted`, or
539    /// genuinely gone) before deciding between reporting a resume or inserting a fresh execution —
540    /// so this never reports `is_resume = true` for a row that turned out not to exist.
541    ///
542    /// Every path that resolves to `is_resume = true` verifies the signed high-water-mark
543    /// (issue #6360) before returning: this is the single production call site every durable resume goes through (P1 agent-turn,
544    /// P2 orchestration, scheduler, sub-agent), so it is also the one place the HWM check needs to
545    /// live to cover unattended crash-resume (FR-004, US-003) uniformly.
546    ///
547    /// Span: `durable.backend.open`.
548    ///
549    /// # Errors
550    ///
551    /// Returns [`DurableError::Storage`] if the lookup, reset, or insert fails,
552    /// [`DurableError::HighWaterMarkIntegrity`] if a resumed execution's signed high-water-mark
553    /// does not verify — this is a hard abort with no override (FR-004) — or
554    /// [`DurableError::ExecutionCanceled`] if the row is `canceled` (INV-16′, #6362): checked
555    /// before the HWM verification, since a canceled execution must never be resumed regardless
556    /// of whether its journal is otherwise intact.
557    pub async fn open_execution(
558        &self,
559        id: ExecutionId,
560        kind: ExecutionKind,
561    ) -> Result<bool, DurableError> {
562        let span = tracing::info_span!(
563            "durable.backend.open",
564            execution_id = %id.as_uuid(),
565            kind = kind.as_str(),
566            is_resume = tracing::field::Empty,
567        );
568        async move {
569            let exec = id.as_uuid().to_string();
570
571            // Attempt the un-finalize directly, with no preceding SELECT: this is the only write
572            // this call needs to make for an existing terminal row, so there is no window between
573            // "observe completed/failed" and "reset to running" for a concurrent prune to act in.
574            let reopened = zeph_db::query(sql!(
575                "UPDATE durable_executions SET status = 'running', updated_at = ?, finalized_at = NULL
576                 WHERE execution_id = ? AND status IN ('completed', 'failed', 'aborted')"
577            ))
578            .bind(now_unix_millis())
579            .bind(&exec)
580            .execute(&self.pool)
581            .await
582            .map_err(|e| DurableError::storage("open", e))?;
583            if reopened.rows_affected() > 0 {
584                self.verify_high_water_mark(id).await?;
585                tracing::Span::current().record("is_resume", true);
586                return Ok(true);
587            }
588
589            // Zero rows: either the row doesn't exist, or it exists but wasn't terminal (already
590            // `running`, no reset needed — every terminal status is covered by the UPDATE above),
591            // or it is `canceled` — deliberately excluded from the UPDATE's IN-list (INV-16′).
592            // Distinguish the cases — if a concurrent prune deleted a terminal row between any
593            // earlier observation and this check, this SELECT sees the authoritative post-delete
594            // state instead of a stale belief that it's there.
595            let existing: Option<(String,)> = zeph_db::query_as(sql!(
596                "SELECT status FROM durable_executions WHERE execution_id = ?"
597            ))
598            .bind(&exec)
599            .fetch_optional(&self.pool)
600            .await
601            .map_err(|e| DurableError::storage("open", e))?;
602            if let Some((status,)) = existing {
603                if status == "canceled" {
604                    return Err(DurableError::ExecutionCanceled { execution_id: id });
605                }
606                self.verify_high_water_mark(id).await?;
607                tracing::Span::current().record("is_resume", true);
608                return Ok(true);
609            }
610            let now = now_unix_millis();
611            zeph_db::query(sql!(
612                "INSERT INTO durable_executions
613                    (execution_id, kind, status, created_at, updated_at, finalized_at)
614                 VALUES (?, ?, 'running', ?, ?, NULL)"
615            ))
616            .bind(&exec)
617            .bind(kind.as_str())
618            .bind(now)
619            .bind(now)
620            .execute(&self.pool)
621            .await
622            .map_err(|e| DurableError::storage("open", e))?;
623            tracing::Span::current().record("is_resume", false);
624            Ok(false)
625        }
626        .instrument(span)
627        .await
628    }
629
630    /// Like [`open_execution`](Self::open_execution), but additionally takes a non-blocking,
631    /// exclusive, process-scoped advisory lock on `id` before touching the row (INV-15, #6122).
632    ///
633    /// Closes the race two processes deriving the same `ExecutionId` (e.g. two CLI instances
634    /// pointed at the same `memory.sqlite_path` and the same `ConversationId`) would otherwise hit
635    /// in [`open_execution`](Self::open_execution)'s unsynchronized SELECT-then-INSERT: both could
636    /// observe "no existing row", both insert, and both then drive `next_step` from 0 against the
637    /// same journal, corrupting it. The lock is acquired first, so the loser never reaches the
638    /// row check at all.
639    ///
640    /// Returns `(is_resume, lock)`. The caller MUST hold `lock` for as long as it drives the
641    /// execution — dropping it releases the lock and allows another process to open the same
642    /// `id`. `lock` is `None` when this backend has no on-disk lock directory (a `:memory:`
643    /// database, a backend built via [`LocalBackend::new`], or a Postgres deployment), in which
644    /// case process exclusivity is not enforced — the caller degrades the same way it already does
645    /// for `open_execution`'s other failure modes.
646    ///
647    /// # Errors
648    ///
649    /// Returns [`DurableError::ExecutionLocked`] if another process already holds `id`'s lock, or
650    /// any error [`open_execution`](Self::open_execution) can return.
651    pub async fn open_execution_exclusive(
652        &self,
653        id: ExecutionId,
654        kind: ExecutionKind,
655    ) -> Result<(bool, Option<ExecutionLock>), DurableError> {
656        let lock = self
657            .lock_dir
658            .as_deref()
659            .map(|dir| ExecutionLock::acquire(dir, id))
660            .transpose()?;
661        let is_resume = self.open_execution(id, kind).await?;
662        Ok((is_resume, lock))
663    }
664
665    /// Cancel a `running` execution so it is deliberately, permanently stopped and never resumed
666    /// (#6362, FR-003/006/012/014).
667    ///
668    /// Unlike [`finalize`](Journal::finalize), which blindly flips `status` under the caller's
669    /// authority, this is the operator-facing entry point: it first tries to establish that no
670    /// live process still owns the execution, so a cancel never races a genuinely active owner's
671    /// own `finalize` into an inconsistent state.
672    ///
673    /// **Liveness probe (SQLite/Unix only).** When this backend has an on-disk `lock_dir`
674    /// (opened via [`LocalBackend::open`] against a real file), a non-blocking acquire of `id`'s
675    /// [`ExecutionLock`] distinguishes a live owner from a dead one:
676    /// - Lock held by another process → [`CancelOutcome::LiveOwner`], row untouched.
677    /// - Lock free → held across the write below (a restart cannot race in mid-window), then
678    ///   released.
679    ///
680    /// **No `lock_dir` (`:memory:` or a backend built via [`LocalBackend::new`]).** The safety
681    /// argument here rests on [`ExecutionBackend::capabilities`]'s `cross_process` flag, which
682    /// this crate only ever sets from `cfg!(feature = "postgres")` — i.e. it assumes "no
683    /// `lock_dir` on a `SQLite` build" implies "no other process can hold this row", true for
684    /// `:memory:` but **not** for a file-backed pool handed to [`LocalBackend::new`] directly
685    /// (which never derives a `lock_dir`); that programmatic path is not reachable from the CLI
686    /// (which always uses [`LocalBackend::open`]), but a future caller of `::new` on a shared file
687    /// should not assume the immediate-cancel path is probe-safe there.
688    /// - `cross_process == false` → provably single-process; proceed directly to the write.
689    /// - `cross_process == true` (Postgres) → a live owner cannot be ruled out and there is no
690    ///   flock to probe → [`CancelOutcome::LivenessUnverifiable`], row untouched (F3).
691    ///
692    /// **Write.** A conditional `UPDATE … WHERE status = 'running'` (the same single-writer-wins
693    /// pattern as `finalize`) — no read-then-write window (NFR-001). Zero rows affected then
694    /// disambiguates via a follow-up `SELECT` into [`CancelOutcome::NotFound`] or
695    /// [`CancelOutcome::AlreadyTerminal`] (idempotent for an already-`canceled` row, NFR-003).
696    ///
697    /// Span: `durable.backend.cancel`.
698    ///
699    /// # Errors
700    ///
701    /// Returns [`DurableError::Storage`] if a query fails, or propagates any
702    /// [`DurableError`] other than [`DurableError::ExecutionLocked`] from the lock acquisition
703    /// (`ExecutionLocked` itself is caught and converted into [`CancelOutcome::LiveOwner`], never
704    /// surfaced as an `Err`).
705    pub async fn cancel_execution(&self, id: ExecutionId) -> Result<CancelOutcome, DurableError> {
706        let span = tracing::info_span!(
707            "durable.backend.cancel",
708            execution_id = %id.as_uuid(),
709            prior_status = tracing::field::Empty,
710            path = tracing::field::Empty,
711        );
712        async move {
713            let exec = id.as_uuid().to_string();
714
715            if let Some(lock_dir) = self.lock_dir.clone() {
716                let _lock = match ExecutionLock::acquire(&lock_dir, id) {
717                    Ok(lock) => lock,
718                    Err(DurableError::ExecutionLocked { holder_pid, .. }) => {
719                        tracing::Span::current().record("path", "live_owner_refused");
720                        return Ok(CancelOutcome::LiveOwner { pid: holder_pid });
721                    }
722                    Err(e) => return Err(e),
723                };
724                let outcome = self.cancel_write(&exec).await?;
725                tracing::Span::current().record("path", "immediate");
726                record_prior_status(outcome);
727                return Ok(outcome);
728                // `_lock` drops here, after the write commits.
729            }
730
731            if self.capabilities().cross_process {
732                tracing::Span::current().record("path", "unverifiable");
733                return Ok(CancelOutcome::LivenessUnverifiable);
734            }
735
736            tracing::Span::current().record("path", "no_lock_dir_single_process");
737            let outcome = self.cancel_write(&exec).await?;
738            record_prior_status(outcome);
739            Ok(outcome)
740        }
741        .instrument(span)
742        .await
743    }
744
745    /// The conditional terminal write behind [`cancel_execution`](Self::cancel_execution),
746    /// factored out so both the SQLite/Unix (lock-held) and single-process (no-`lock_dir`) paths
747    /// share one implementation of the race-safe `UPDATE … WHERE status = 'running'` pattern.
748    async fn cancel_write(&self, exec: &str) -> Result<CancelOutcome, DurableError> {
749        let now = now_unix_millis();
750        let mut tx = zeph_db::begin_write(&self.pool)
751            .await
752            .map_err(|e| DurableError::storage("cancel", e))?;
753        let result = zeph_db::query(sql!(
754            "UPDATE durable_executions SET status = 'canceled', finalized_at = ?, updated_at = ?
755             WHERE execution_id = ? AND status = 'running'"
756        ))
757        .bind(now)
758        .bind(now)
759        .bind(exec)
760        .execute(&mut *tx)
761        .await
762        .map_err(|e| DurableError::storage("cancel", e))?;
763        if result.rows_affected() > 0 {
764            tx.commit()
765                .await
766                .map_err(|e| DurableError::storage("cancel", e))?;
767            return Ok(CancelOutcome::Canceled);
768        }
769
770        // Zero rows: either no such execution, or it exists but was not `running`. Read the
771        // current status inside the same transaction so this reflects exactly what the UPDATE
772        // above saw — no window for a concurrent writer to change the answer in between.
773        let existing: Option<(String,)> = zeph_db::query_as(sql!(
774            "SELECT status FROM durable_executions WHERE execution_id = ?"
775        ))
776        .bind(exec)
777        .fetch_optional(&mut *tx)
778        .await
779        .map_err(|e| DurableError::storage("cancel", e))?;
780        tx.commit()
781            .await
782            .map_err(|e| DurableError::storage("cancel", e))?;
783        match existing {
784            None => Ok(CancelOutcome::NotFound),
785            Some((status,)) => {
786                let status = ExecutionStatus::from_tag(&status).ok_or(DurableError::Decode {
787                    context: "unrecognized durable_executions.status value",
788                })?;
789                Ok(CancelOutcome::AlreadyTerminal { status })
790            }
791        }
792    }
793
794    /// Group-commit a batch of buffered entries in a single write transaction.
795    ///
796    /// Used by the [`JournalWriter`](crate::JournalWriter) to amortize the WAL fsync across all
797    /// entries accumulated within a flush interval. Sealing and HMAC computation run before the
798    /// transaction opens, keeping CPU work off the write lock. The whole batch commits atomically;
799    /// a single malformed entry aborts the batch.
800    ///
801    /// # Errors
802    ///
803    /// Returns [`DurableError::Storage`] on a database failure, or a per-entry error
804    /// ([`DurableError::PayloadTooLarge`], [`DurableError::UnsupportedEntryKind`], or a cipher
805    /// failure) if an entry cannot be prepared.
806    pub(crate) async fn append_batch(&self, entries: &[JournalEntry]) -> Result<(), DurableError> {
807        if entries.is_empty() {
808            return Ok(());
809        }
810        let mut rows = Vec::with_capacity(entries.len());
811        for entry in entries {
812            rows.push(self.prepare_row(entry)?);
813        }
814        // `sql!()` caches its postgres rewrite per call site (see #5431), so hoisting
815        // this out of the loop below is no longer required to avoid a leak — kept
816        // anyway since it reads the intent clearly and costs nothing.
817        let insert = sql!(
818            "INSERT INTO durable_journal
819                (execution_id, step_id, entry_kind, idem_key, effect_class, payload, payload_version, hmac, created_at)
820             VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)"
821        );
822        let mut tx = zeph_db::begin_write(&self.pool)
823            .await
824            .map_err(|e| DurableError::storage("append_batch", e))?;
825        for (entry, row) in entries.iter().zip(rows) {
826            zeph_db::query(insert)
827                .bind(row.execution_id)
828                .bind(row.step_id)
829                .bind(row.entry_kind)
830                .bind(row.idem_key)
831                .bind(row.effect_class)
832                .bind(row.payload)
833                .bind(row.payload_version)
834                .bind(row.hmac)
835                .bind(row.created_at)
836                .execute(&mut *tx)
837                .await
838                .map_err(|e| DurableError::storage("append_batch", e))?;
839            if matches!(entry.entry, EntryKind::StepResult { .. }) {
840                self.bump_hwm_for_step_result(&mut tx, entry.execution_id, entry.step_id)
841                    .await?;
842            }
843        }
844        tx.commit()
845            .await
846            .map_err(|e| DurableError::storage("append_batch", e))?;
847        Ok(())
848    }
849
850    /// Look up a committed `StepResult` anywhere in an execution by its [`IdempotencyKey`].
851    ///
852    /// Backs INV-13: a guarded effect that already committed its result must not re-fire after a
853    /// replay divergence restarts the execution fresh. Returns the (opened) `StepResult` entry when
854    /// one exists, or `None`. The `idx_durable_journal_idem_key` partial index makes this an
855    /// `O(log n)` point lookup rather than a scan.
856    ///
857    /// Span: `durable.journal.lookup_idem`.
858    ///
859    /// # Errors
860    ///
861    /// Returns [`DurableError::Storage`] if the query fails, or [`DurableError::Decode`] if the
862    /// located row cannot be reconstructed.
863    pub(crate) async fn lookup_committed_result(
864        &self,
865        id: ExecutionId,
866        idem_key: IdempotencyKey,
867    ) -> Result<Option<JournalEntry>, DurableError> {
868        let span = tracing::info_span!(
869            "durable.journal.lookup_idem",
870            execution_id = %id.as_uuid(),
871            found = tracing::field::Empty,
872        );
873        async move {
874            let rows: Vec<JournalRowRead> = zeph_db::query_as(sql!(
875                "SELECT seq, step_id, entry_kind, idem_key, effect_class, payload, payload_version, hmac, created_at
876                 FROM durable_journal
877                 WHERE execution_id = ? AND idem_key = ? AND entry_kind = 'step_result'
878                 ORDER BY seq LIMIT 1"
879            ))
880            .bind(id.as_uuid().to_string())
881            .bind(idem_key.as_bytes().to_vec())
882            .fetch_all(&self.pool)
883            .await
884            .map_err(|e| DurableError::storage("lookup_idem", e))?;
885            let entry = self.rows_to_entries(id, rows).await?.into_iter().next();
886            tracing::Span::current().record("found", entry.is_some());
887            Ok(entry)
888        }
889        .instrument(span)
890        .await
891    }
892
893    /// Read the highest committed [`JournalSeq`], or `None` for an empty journal.
894    ///
895    /// The [`JournalWriter`](crate::JournalWriter) calls this on (re)start to anchor itself at the
896    /// last durably-committed entry (FR-DE-12). Because `seq` is a database-assigned autoincrement,
897    /// resumed appends continue from `MAX(seq) + 1` with neither gap nor duplication.
898    ///
899    /// # Errors
900    ///
901    /// Returns [`DurableError::Storage`] if the query fails.
902    pub(crate) async fn max_seq(&self) -> Result<Option<JournalSeq>, DurableError> {
903        let max: Option<i64> = zeph_db::query_scalar(sql!("SELECT MAX(seq) FROM durable_journal"))
904            .fetch_one(&self.pool)
905            .await
906            .map_err(|e| DurableError::storage("max_seq", e))?;
907        Ok(max.map(JournalSeq::new))
908    }
909
910    /// The in-process wakeup registry for parked promise awaits, shared with the resolver path.
911    pub(crate) fn promise_waiters(&self) -> &NotifyRegistry {
912        &self.promise_waiters
913    }
914
915    /// The in-process wakeup registry for parked timers, shared with the timer service.
916    pub(crate) fn timer_waiters(&self) -> &NotifyRegistry {
917        &self.timer_waiters
918    }
919
920    /// Insert a freshly-created promise row (INV-9: only the resolver-token hash is stored).
921    ///
922    /// Called by `promise()` for a brand-new promise; a resumed execution detects the existing row
923    /// via [`promise_state`](Self::promise_state) and never re-inserts. Span: `durable.promise.create`.
924    ///
925    /// # Errors
926    ///
927    /// Returns [`DurableError::Storage`] if the insert fails.
928    pub(crate) async fn insert_promise(
929        &self,
930        id: PromiseId,
931        execution_id: ExecutionId,
932        resolver_token_hash: [u8; 32],
933        created_at_ms: i64,
934    ) -> Result<(), DurableError> {
935        let span = tracing::info_span!("durable.promise.create", promise_id = %id.as_uuid());
936        async move {
937            zeph_db::query(sql!(
938                "INSERT INTO durable_promises
939                    (promise_id, execution_id, resolver_token_hash, resolved, payload, created_at, resolved_at)
940                 VALUES (?, ?, ?, 0, NULL, ?, NULL)"
941            ))
942            .bind(id.as_uuid().to_string())
943            .bind(execution_id.as_uuid().to_string())
944            .bind(resolver_token_hash.to_vec())
945            .bind(created_at_ms)
946            .execute(&self.pool)
947            .await
948            .map_err(|e| DurableError::storage("insert_promise", e))?;
949            Ok(())
950        }
951        .instrument(span)
952        .await
953    }
954
955    /// Read a promise's persisted state, or `None` if it does not exist.
956    ///
957    /// # Errors
958    ///
959    /// Returns [`DurableError::Storage`] if the query fails, or [`DurableError::Decode`] if a stored
960    /// field cannot be reconstructed.
961    pub(crate) async fn promise_state(
962        &self,
963        id: PromiseId,
964    ) -> Result<Option<PromiseRecord>, DurableError> {
965        let row: Option<PromiseRowRead> = zeph_db::query_as(sql!(
966            "SELECT execution_id, resolver_token_hash, resolved, payload
967             FROM durable_promises WHERE promise_id = ?"
968        ))
969        .bind(id.as_uuid().to_string())
970        .fetch_optional(&self.pool)
971        .await
972        .map_err(|e| DurableError::storage("promise_state", e))?;
973        let Some((exec, hash, resolved, payload)) = row else {
974            return Ok(None);
975        };
976        Ok(Some(PromiseRecord {
977            execution_id: parse_execution_id(&exec)?,
978            resolver_token_hash: slice_to_array32(&hash, "promise resolver_token_hash")?,
979            resolved: resolved != 0,
980            payload,
981        }))
982    }
983
984    /// Commit a resolved value to a pending promise, returning whether it transitioned.
985    ///
986    /// The conditional `WHERE resolved = 0` makes a double-resolve a no-op (returns `false`); the
987    /// caller has already authenticated the resolver token. On a real transition any in-process
988    /// waiter is woken. Span: `durable.promise.resolve`.
989    ///
990    /// # Errors
991    ///
992    /// Returns [`DurableError::PayloadTooLarge`] if the value exceeds the limit, a cipher failure if
993    /// sealing fails, or [`DurableError::Storage`] on a database error.
994    pub(crate) async fn resolve_promise(
995        &self,
996        id: PromiseId,
997        execution_id: ExecutionId,
998        value_plaintext: &[u8],
999        resolved_at_ms: i64,
1000    ) -> Result<bool, DurableError> {
1001        let span = tracing::info_span!("durable.promise.resolve", promise_id = %id.as_uuid());
1002        async move {
1003            ensure_payload_within_limit(value_plaintext.len(), self.max_payload_bytes)?;
1004            let aad = promise_payload_aad(execution_id, id);
1005            let sealed = self.seal_payload(value_plaintext, &aad)?;
1006            let affected = zeph_db::query(sql!(
1007                "UPDATE durable_promises SET resolved = 1, payload = ?, resolved_at = ?
1008                 WHERE promise_id = ? AND resolved = 0"
1009            ))
1010            .bind(sealed)
1011            .bind(resolved_at_ms)
1012            .bind(id.as_uuid().to_string())
1013            .execute(&self.pool)
1014            .await
1015            .map_err(|e| DurableError::storage("resolve_promise", e))?
1016            .rows_affected();
1017            if affected > 0 {
1018                self.promise_waiters.wake(id.as_uuid());
1019            }
1020            Ok(affected > 0)
1021        }
1022        .instrument(span)
1023        .await
1024    }
1025
1026    /// Claim the one-time replay notification for a promise, returning whether this call won.
1027    ///
1028    /// The conditional `WHERE notified_at IS NULL` makes the claim single-winner: the first caller
1029    /// transitions the row (returns `true`); every later caller is a no-op (returns `false`). This
1030    /// backs #6027 — a resumed foreground sub-agent's replay notice / TUI completion event must fire
1031    /// at most once across repeated parent restarts. Unlike [`resolve_promise`](Self::resolve_promise)
1032    /// it touches only the `notified_at` bookkeeping column and carries no payload, so no sealing /
1033    /// waiter wakeup is involved. Span: `durable.promise.claim_notify`.
1034    ///
1035    /// # Errors
1036    ///
1037    /// Returns [`DurableError::Storage`] on a database error.
1038    pub(crate) async fn claim_promise_notification(
1039        &self,
1040        id: PromiseId,
1041        notified_at_ms: i64,
1042    ) -> Result<bool, DurableError> {
1043        let span = tracing::info_span!("durable.promise.claim_notify", promise_id = %id.as_uuid());
1044        async move {
1045            let affected = zeph_db::query(sql!(
1046                "UPDATE durable_promises SET notified_at = ?
1047                 WHERE promise_id = ? AND notified_at IS NULL"
1048            ))
1049            .bind(notified_at_ms)
1050            .bind(id.as_uuid().to_string())
1051            .execute(&self.pool)
1052            .await
1053            .map_err(|e| DurableError::storage("claim_promise_notification", e))?
1054            .rows_affected();
1055            Ok(affected > 0)
1056        }
1057        .instrument(span)
1058        .await
1059    }
1060
1061    /// Open a promise's sealed resolved payload back to plaintext.
1062    ///
1063    /// # Errors
1064    ///
1065    /// Returns [`DurableError::ReplayIntegrity`] if the sealed blob does not authenticate, or
1066    /// [`DurableError::PayloadTooLarge`] if it exceeds the read-side limit.
1067    pub(crate) fn open_promise_payload(
1068        &self,
1069        id: PromiseId,
1070        execution_id: ExecutionId,
1071        sealed: &[u8],
1072    ) -> Result<Bytes, DurableError> {
1073        ensure_payload_within_limit(
1074            sealed.len(),
1075            self.max_payload_bytes.saturating_add(SEAL_OVERHEAD_SLACK),
1076        )?;
1077        let aad = promise_payload_aad(execution_id, id);
1078        self.open_payload(sealed, &aad)
1079    }
1080
1081    /// Arm a durable timer to fire at `due_at_ms` (a `durable_timers` row).
1082    ///
1083    /// Span: `durable.timer.arm`.
1084    ///
1085    /// # Errors
1086    ///
1087    /// Returns [`DurableError::Storage`] if the insert fails.
1088    pub(crate) async fn arm_timer(
1089        &self,
1090        id: TimerId,
1091        execution_id: ExecutionId,
1092        due_at_ms: i64,
1093        created_at_ms: i64,
1094    ) -> Result<(), DurableError> {
1095        let span = tracing::info_span!("durable.timer.arm", timer_id = %id.as_uuid(), due_at_ms);
1096        async move {
1097            zeph_db::query(sql!(
1098                "INSERT INTO durable_timers (timer_id, execution_id, due_at, fired, created_at)
1099                 VALUES (?, ?, ?, 0, ?)"
1100            ))
1101            .bind(id.as_uuid().to_string())
1102            .bind(execution_id.as_uuid().to_string())
1103            .bind(due_at_ms)
1104            .bind(created_at_ms)
1105            .execute(&self.pool)
1106            .await
1107            .map_err(|e| DurableError::storage("arm_timer", e))?;
1108            Ok(())
1109        }
1110        .instrument(span)
1111        .await
1112    }
1113
1114    /// Read a timer's `(due_at_ms, fired)` state, or `None` if it does not exist.
1115    ///
1116    /// # Errors
1117    ///
1118    /// Returns [`DurableError::Storage`] if the query fails.
1119    pub(crate) async fn timer_state(
1120        &self,
1121        id: TimerId,
1122    ) -> Result<Option<(i64, bool)>, DurableError> {
1123        let row: Option<(i64, i64)> = zeph_db::query_as(sql!(
1124            "SELECT due_at, fired FROM durable_timers WHERE timer_id = ?"
1125        ))
1126        .bind(id.as_uuid().to_string())
1127        .fetch_optional(&self.pool)
1128        .await
1129        .map_err(|e| DurableError::storage("timer_state", e))?;
1130        Ok(row.map(|(due_at, fired)| (due_at, fired != 0)))
1131    }
1132
1133    /// List every unfired timer whose instant is at or before `now_ms`.
1134    ///
1135    /// The `idx_durable_timers_due(fired, due_at)` index makes this a range scan over due, unfired
1136    /// timers rather than a full-table scan.
1137    ///
1138    /// # Errors
1139    ///
1140    /// Returns [`DurableError::Storage`] if the query fails, or [`DurableError::Decode`] on a
1141    /// malformed id.
1142    pub(crate) async fn due_timers(&self, now_ms: i64) -> Result<Vec<TimerId>, DurableError> {
1143        let rows: Vec<(String,)> = zeph_db::query_as(sql!(
1144            "SELECT timer_id FROM durable_timers WHERE fired = 0 AND due_at <= ? ORDER BY due_at"
1145        ))
1146        .bind(now_ms)
1147        .fetch_all(&self.pool)
1148        .await
1149        .map_err(|e| DurableError::storage("due_timers", e))?;
1150        rows.into_iter().map(|(id,)| parse_timer_id(&id)).collect()
1151    }
1152
1153    /// Mark a timer fired, returning whether it transitioned, and wake its parked waiter.
1154    ///
1155    /// Span: `durable.timer.fire`.
1156    ///
1157    /// # Errors
1158    ///
1159    /// Returns [`DurableError::Storage`] if the update fails.
1160    pub(crate) async fn mark_timer_fired(&self, id: TimerId) -> Result<bool, DurableError> {
1161        let span = tracing::info_span!("durable.timer.fire", timer_id = %id.as_uuid());
1162        async move {
1163            let affected = zeph_db::query(sql!(
1164                "UPDATE durable_timers SET fired = 1 WHERE timer_id = ? AND fired = 0"
1165            ))
1166            .bind(id.as_uuid().to_string())
1167            .execute(&self.pool)
1168            .await
1169            .map_err(|e| DurableError::storage("mark_timer_fired", e))?
1170            .rows_affected();
1171            if affected > 0 {
1172                self.timer_waiters.wake(id.as_uuid());
1173            }
1174            Ok(affected > 0)
1175        }
1176        .instrument(span)
1177        .await
1178    }
1179
1180    /// Open each foldable step result's sealed payload into a [`FoldedStep`], in step order.
1181    ///
1182    /// The per-step AAD is reconstructed from the row so the opened plaintext authenticates exactly
1183    /// as it did at rest; the idempotency key is preserved so the replayed-from-snapshot step still
1184    /// satisfies the divergence guard.
1185    fn open_foldable_steps(
1186        &self,
1187        execution_id: ExecutionId,
1188        rows: Vec<FoldableRowRead>,
1189    ) -> Result<Vec<FoldedStep>, DurableError> {
1190        let mut folded = Vec::with_capacity(rows.len());
1191        for (step_raw, idem, version, payload) in rows {
1192            let step = u32::try_from(step_raw).map_err(|_| DurableError::Decode {
1193                context: "checkpoint step_id out of u32 range",
1194            })?;
1195            let idem_bytes = idem.ok_or(DurableError::Decode {
1196                context: "checkpoint step result missing idem_key",
1197            })?;
1198            let idem_key =
1199                IdempotencyKey::from_bytes(slice_to_array32(&idem_bytes, "checkpoint idem_key")?);
1200            let sealed = payload.ok_or(DurableError::Decode {
1201                context: "checkpoint step result missing payload",
1202            })?;
1203            let aad = PayloadAad::new(
1204                execution_id,
1205                StepId::new(step),
1206                EntryKindTag::StepResult,
1207                Some(idem_key),
1208            );
1209            let plaintext = self.open_payload(&sealed, &aad)?;
1210            let payload_version =
1211                u8::try_from(version.unwrap_or(1)).map_err(|_| DurableError::Decode {
1212                    context: "checkpoint payload_version out of u8 range",
1213                })?;
1214            folded.push(FoldedStep {
1215                step_id: step,
1216                idem_key: *idem_key.as_bytes(),
1217                payload_version,
1218                payload: plaintext,
1219            });
1220        }
1221        Ok(folded)
1222    }
1223
1224    /// Fold an execution's committed-idempotent prefix below `up_to_step` into one checkpoint entry.
1225    ///
1226    /// Reads the foldable idempotent step results, packs as many as fit the payload budget into a
1227    /// sealed snapshot, writes a single [`EntryKind::Checkpoint`] entry, and deletes the folded rows
1228    /// — all in one transaction. A resume replays the folded steps from the snapshot (the snapshot
1229    /// preserves each step's idempotency key for the divergence guard) instead of re-running them.
1230    /// Returns the number of steps folded. Runs only on a background task (spec NEVER: not the hot
1231    /// path). Span: `durable.journal.checkpoint`.
1232    ///
1233    /// The checkpoint row also carries the fold's `folded_count` (issue #6360), in the same
1234    /// transaction as the DELETE. The high-water-mark's own `committed_result_count` is
1235    /// deliberately left untouched here: a fold moves committed results from live rows into the
1236    /// checkpoint snapshot net-zero, so the signed count stays valid without a bump — only the
1237    /// resume-time recomputation needs `folded_count` (`count(surviving StepResult) +
1238    /// SUM(folded_count)`) to see past the fold.
1239    ///
1240    /// # Errors
1241    ///
1242    /// Returns [`DurableError::Storage`] on a database error, or a cipher failure if (re)sealing
1243    /// fails.
1244    pub(crate) async fn checkpoint_fold(
1245        &self,
1246        execution_id: ExecutionId,
1247        up_to_step: u32,
1248    ) -> Result<u64, DurableError> {
1249        let span = tracing::info_span!(
1250            "durable.journal.checkpoint",
1251            execution_id = %execution_id.as_uuid(),
1252            folded_count = tracing::field::Empty,
1253        );
1254        async move {
1255            let exec = execution_id.as_uuid().to_string();
1256            let rows: Vec<FoldableRowRead> = zeph_db::query_as(sql!(
1257                "SELECT step_id, idem_key, payload_version, payload FROM durable_journal
1258                 WHERE execution_id = ? AND entry_kind = 'step_result'
1259                   AND effect_class = 'idempotent' AND step_id < ?
1260                 ORDER BY step_id"
1261            ))
1262            .bind(&exec)
1263            .bind(i64::from(up_to_step))
1264            .fetch_all(&self.pool)
1265            .await
1266            .map_err(|e| DurableError::storage("checkpoint", e))?;
1267            if rows.is_empty() {
1268                return Ok(0);
1269            }
1270
1271            // Open each sealed result, then keep the budget-bounded prefix that fits a checkpoint.
1272            let mut folded = self.open_foldable_steps(execution_id, rows)?;
1273            let lens: Vec<usize> = folded.iter().map(|s| s.payload.len()).collect();
1274            let take = crate::retention::fold_prefix_len(
1275                &lens,
1276                crate::retention::checkpoint_budget(self.max_payload_bytes),
1277            );
1278            if take == 0 {
1279                // Not even one result fits the budget; leave the prefix un-folded rather than write
1280                // an over-limit checkpoint.
1281                return Ok(0);
1282            }
1283            folded.truncate(take);
1284            let fold_end = folded.last().map_or(up_to_step, |s| s.step_id.saturating_add(1));
1285
1286            let snapshot = encode_checkpoint(&folded);
1287            let snap_aad =
1288                PayloadAad::new(execution_id, StepId::new(fold_end), EntryKindTag::Checkpoint, None);
1289            let sealed_snapshot = self.seal_payload(&snapshot, &snap_aad)?;
1290
1291            // `folded_count` (issue #6360) is persisted on the checkpoint row itself, in the same
1292            // transaction as the fold's DELETE, so resume can recompute `committed_result_count` as
1293            // `count(surviving StepResult rows) + SUM(folded_count over checkpoints)` without ever
1294            // observing a fold whose DELETE committed but whose count did not (or vice versa).
1295            let count = folded.len() as u64;
1296
1297            let mut tx = zeph_db::begin_write(&self.pool)
1298                .await
1299                .map_err(|e| DurableError::storage("checkpoint", e))?;
1300            zeph_db::query(sql!(
1301                "INSERT INTO durable_journal
1302                    (execution_id, step_id, entry_kind, idem_key, effect_class, payload, payload_version, hmac, created_at, folded_count)
1303                 VALUES (?, ?, 'checkpoint', NULL, NULL, ?, ?, NULL, ?, ?)"
1304            ))
1305            .bind(&exec)
1306            .bind(i64::from(fold_end))
1307            .bind(sealed_snapshot)
1308            .bind(i32::from(crate::step::PAYLOAD_VERSION))
1309            .bind(now_unix_millis())
1310            .bind(i64::try_from(count).unwrap_or(i64::MAX))
1311            .execute(&mut *tx)
1312            .await
1313            .map_err(|e| DurableError::storage("checkpoint", e))?;
1314            zeph_db::query(sql!(
1315                "DELETE FROM durable_journal
1316                 WHERE execution_id = ? AND entry_kind = 'step_result'
1317                   AND effect_class = 'idempotent' AND step_id < ?"
1318            ))
1319            .bind(&exec)
1320            .bind(i64::from(fold_end))
1321            .execute(&mut *tx)
1322            .await
1323            .map_err(|e| DurableError::storage("checkpoint", e))?;
1324            tx.commit()
1325                .await
1326                .map_err(|e| DurableError::storage("checkpoint", e))?;
1327
1328            tracing::Span::current().record("folded_count", count);
1329            Ok(count)
1330        }
1331        .instrument(span)
1332        .await
1333    }
1334
1335    /// Read every checkpoint snapshot for an execution and reconstruct its folded step results.
1336    ///
1337    /// The replay cursor calls this once on resume to preload folded results before walking the
1338    /// surviving journal rows: each returned [`JournalEntry`] is a `StepResult` whose individual row
1339    /// was deleted by the fold but whose replay value (and idempotency key, for the divergence guard)
1340    /// lives in the snapshot. Each snapshot is AEAD-opened with its checkpoint-bound AAD. Returns an
1341    /// empty vector when the execution has never been folded.
1342    ///
1343    /// # Errors
1344    ///
1345    /// Returns [`DurableError::Storage`] on a database error, or a decode/cipher failure if a
1346    /// snapshot is corrupt.
1347    pub(crate) async fn read_checkpoints(
1348        &self,
1349        execution_id: ExecutionId,
1350    ) -> Result<Vec<JournalEntry>, DurableError> {
1351        let rows: Vec<(i64, Option<Vec<u8>>)> = zeph_db::query_as(sql!(
1352            "SELECT step_id, payload FROM durable_journal
1353             WHERE execution_id = ? AND entry_kind = 'checkpoint' ORDER BY step_id"
1354        ))
1355        .bind(execution_id.as_uuid().to_string())
1356        .fetch_all(&self.pool)
1357        .await
1358        .map_err(|e| DurableError::storage("read_checkpoints", e))?;
1359        if rows.is_empty() {
1360            return Ok(Vec::new());
1361        }
1362        let mut folded: CheckpointSnapshot = Vec::new();
1363        for (up_to, payload) in rows {
1364            let up_to = u32::try_from(up_to).map_err(|_| DurableError::Decode {
1365                context: "checkpoint up_to_step out of u32 range",
1366            })?;
1367            let sealed = payload.ok_or(DurableError::Decode {
1368                context: "checkpoint entry missing snapshot payload",
1369            })?;
1370            ensure_payload_within_limit(
1371                sealed.len(),
1372                self.max_payload_bytes.saturating_add(SEAL_OVERHEAD_SLACK),
1373            )?;
1374            let aad = PayloadAad::new(
1375                execution_id,
1376                StepId::new(up_to),
1377                EntryKindTag::Checkpoint,
1378                None,
1379            );
1380            let plaintext = self.open_payload(&sealed, &aad)?;
1381            folded.extend(decode_checkpoint(&plaintext)?);
1382        }
1383        // Reconstruct each folded step as a replayable `StepResult` entry under the real execution
1384        // kind, so the cursor serves it exactly like a surviving row.
1385        let kind = self.lookup_kind(execution_id).await?;
1386        let entries = folded
1387            .into_iter()
1388            .map(|step| JournalEntry {
1389                seq: None,
1390                execution_id,
1391                kind,
1392                step_id: StepId::new(step.step_id),
1393                entry: EntryKind::StepResult {
1394                    idempotency_key: IdempotencyKey::from_bytes(step.idem_key),
1395                    payload: step.payload,
1396                    effect: crate::EffectClass::Idempotent,
1397                    payload_version: step.payload_version,
1398                },
1399                created_at_ms: 0,
1400            })
1401            .collect();
1402        Ok(entries)
1403    }
1404
1405    /// Find every **resumable** (`status = 'running'`) execution that has committed at least one
1406    /// `StepResult` but carries no `durable_execution_integrity` row (issue #6449).
1407    ///
1408    /// This is the drain-before-seal precondition scan for `zeph durable seal-integrity`: the
1409    /// returned set is exactly the executions that would be silently downgraded to
1410    /// unconditional-tamper the moment this backend seals, unless drained to a terminal status
1411    /// first or explicitly grandfathered. A non-resumable (terminal) execution missing its row is
1412    /// not a concern — it can never be resumed again, sealed or not.
1413    ///
1414    /// # Errors
1415    ///
1416    /// Returns [`DurableError::Storage`] if the query fails.
1417    pub async fn find_unsealed_resumable_executions(
1418        &self,
1419    ) -> Result<Vec<ExecutionId>, DurableError> {
1420        let rows: Vec<(String,)> = zeph_db::query_as(sql!(
1421            "SELECT e.execution_id FROM durable_executions e
1422             WHERE e.status = 'running'
1423               AND NOT EXISTS (
1424                 SELECT 1 FROM durable_execution_integrity i WHERE i.execution_id = e.execution_id
1425               )
1426               AND (
1427                 EXISTS (
1428                   SELECT 1 FROM durable_journal j
1429                   WHERE j.execution_id = e.execution_id AND j.entry_kind = 'step_result'
1430                 )
1431                 OR EXISTS (
1432                   SELECT 1 FROM durable_journal j
1433                   WHERE j.execution_id = e.execution_id AND j.entry_kind = 'checkpoint'
1434                     AND j.folded_count > 0
1435                 )
1436               )"
1437        ))
1438        .fetch_all(&self.pool)
1439        .await
1440        .map_err(|e| DurableError::storage("seal_integrity_scan", e))?;
1441
1442        rows.into_iter()
1443            .map(|(id,)| {
1444                ExecutionId::parse_str(&id).map_err(|_| DurableError::Decode {
1445                    context: "malformed execution_id in durable_executions",
1446                })
1447            })
1448            .collect()
1449    }
1450
1451    /// Recompute the number of committed `StepResult`s for `execution_id` directly from the
1452    /// journal: surviving `step_result` rows plus every checkpoint's `folded_count` (a fold moves
1453    /// committed results into a checkpoint snapshot net-zero, so this sum is invariant across
1454    /// folding). Shared by [`check_high_water_mark`](Self::check_high_water_mark)'s present-row
1455    /// recomputation and its post-seal absent-row check (issue #6449).
1456    async fn committed_step_result_count(
1457        &self,
1458        execution_id: ExecutionId,
1459    ) -> Result<u64, DurableError> {
1460        let exec = execution_id.as_uuid().to_string();
1461        let live_count: i64 = zeph_db::query_scalar(sql!(
1462            "SELECT COUNT(*) FROM durable_journal
1463             WHERE execution_id = ? AND entry_kind = 'step_result'"
1464        ))
1465        .bind(&exec)
1466        .fetch_one(&self.pool)
1467        .await
1468        .map_err(|e| DurableError::storage("hwm_verify", e))?;
1469        let folded_sum: i64 = zeph_db::query_scalar(sql!(
1470            "SELECT COALESCE(SUM(folded_count), 0) FROM durable_journal
1471             WHERE execution_id = ? AND entry_kind = 'checkpoint'"
1472        ))
1473        .bind(&exec)
1474        .fetch_one(&self.pool)
1475        .await
1476        .map_err(|e| DurableError::storage("hwm_verify", e))?;
1477        Ok(u64::try_from(live_count.saturating_add(folded_sum)).unwrap_or(0))
1478    }
1479
1480    /// Derive the persisted column values for an entry, sealing payloads and stamping HMACs.
1481    fn prepare_row(&self, entry: &JournalEntry) -> Result<JournalRow, DurableError> {
1482        let execution_id = entry.execution_id.as_uuid().to_string();
1483        let step_id = i64::from(entry.step_id.value());
1484        let created_at = entry.created_at_ms;
1485        let entry_kind = entry.entry.tag();
1486        match &entry.entry {
1487            EntryKind::StepResult {
1488                idempotency_key,
1489                payload,
1490                effect,
1491                payload_version,
1492            } => {
1493                ensure_payload_within_limit(payload.len(), self.max_payload_bytes)?;
1494                let aad = PayloadAad::new(
1495                    entry.execution_id,
1496                    entry.step_id,
1497                    EntryKindTag::StepResult,
1498                    Some(*idempotency_key),
1499                );
1500                let sealed = self.seal_payload(payload.as_ref(), &aad)?;
1501                Ok(JournalRow {
1502                    execution_id,
1503                    step_id,
1504                    entry_kind,
1505                    idem_key: Some(idempotency_key.as_bytes().to_vec()),
1506                    effect_class: Some(effect.as_str()),
1507                    payload: Some(sealed),
1508                    payload_version: Some(i32::from(*payload_version)),
1509                    hmac: None,
1510                    created_at,
1511                })
1512            }
1513            EntryKind::EffectIntent {
1514                idempotency_key,
1515                effect,
1516                hmac: _,
1517            } => {
1518                // The backend is the HMAC keyholder; it stamps the row HMAC itself when configured
1519                // and ignores any caller-supplied value.
1520                let hmac = self.control_hmac(entry, Some(idempotency_key));
1521                Ok(JournalRow {
1522                    execution_id,
1523                    step_id,
1524                    entry_kind,
1525                    idem_key: Some(idempotency_key.as_bytes().to_vec()),
1526                    effect_class: Some(effect.as_str()),
1527                    payload: None,
1528                    payload_version: None,
1529                    hmac,
1530                    created_at,
1531                })
1532            }
1533            EntryKind::PromiseCreated { .. }
1534            | EntryKind::PromiseResolved { .. }
1535            | EntryKind::TimerArmed { .. }
1536            | EntryKind::TimerFired { .. }
1537            | EntryKind::Checkpoint { .. } => {
1538                Err(DurableError::UnsupportedEntryKind { kind: entry_kind })
1539            }
1540        }
1541    }
1542
1543    /// Look up the owning execution's kind for read-time entry reconstruction.
1544    async fn lookup_kind(&self, id: ExecutionId) -> Result<ExecutionKind, DurableError> {
1545        let kind: Option<String> = zeph_db::query_scalar(sql!(
1546            "SELECT kind FROM durable_executions WHERE execution_id = ?"
1547        ))
1548        .bind(id.as_uuid().to_string())
1549        .fetch_optional(&self.pool)
1550        .await
1551        .map_err(|e| DurableError::storage("read", e))?;
1552        let kind = kind.ok_or(DurableError::Decode {
1553            context: "journaled entries reference a missing execution row",
1554        })?;
1555        ExecutionKind::from_tag(&kind).ok_or(DurableError::Decode {
1556            context: "execution kind is not reconstructible (custom kind read-back unsupported)",
1557        })
1558    }
1559
1560    /// Reconstruct a [`JournalEntry`] from a stored row, opening sealed payloads.
1561    fn row_to_entry(
1562        &self,
1563        id: ExecutionId,
1564        kind: ExecutionKind,
1565        row: JournalRowRead,
1566    ) -> Result<JournalEntry, DurableError> {
1567        let (
1568            seq,
1569            step_id_raw,
1570            entry_kind,
1571            idem_key,
1572            effect_class,
1573            payload,
1574            payload_version,
1575            hmac,
1576            created_at,
1577        ) = row;
1578        let step_id =
1579            StepId::new(
1580                u32::try_from(step_id_raw).map_err(|_| DurableError::Decode {
1581                    context: "step_id out of u32 range",
1582                })?,
1583            );
1584        let entry = match entry_kind.as_str() {
1585            "step_result" => {
1586                let idem_bytes = idem_key.ok_or(DurableError::Decode {
1587                    context: "step_result idem_key missing",
1588                })?;
1589                let idem_key = IdempotencyKey::from_bytes(slice_to_array32(
1590                    &idem_bytes,
1591                    "step_result idem_key",
1592                )?);
1593                let effect = effect_class
1594                    .as_deref()
1595                    .and_then(crate::EffectClass::from_tag)
1596                    .ok_or(DurableError::Decode {
1597                        context: "step_result effect_class missing or invalid",
1598                    })?;
1599                let sealed = payload.ok_or(DurableError::Decode {
1600                    context: "step_result payload missing",
1601                })?;
1602                ensure_payload_within_limit(
1603                    sealed.len(),
1604                    self.max_payload_bytes.saturating_add(SEAL_OVERHEAD_SLACK),
1605                )?;
1606                let aad = PayloadAad::new(id, step_id, EntryKindTag::StepResult, Some(idem_key));
1607                let opened = self.open_payload(&sealed, &aad)?;
1608                let version = u8::try_from(payload_version.unwrap_or(1)).map_err(|_| {
1609                    DurableError::Decode {
1610                        context: "payload_version out of u8 range",
1611                    }
1612                })?;
1613                EntryKind::StepResult {
1614                    idempotency_key: idem_key,
1615                    payload: opened,
1616                    effect,
1617                    payload_version: version,
1618                }
1619            }
1620            "effect_intent" => {
1621                let idem_bytes = idem_key.ok_or(DurableError::Decode {
1622                    context: "effect_intent idem_key missing",
1623                })?;
1624                let idem_key = IdempotencyKey::from_bytes(slice_to_array32(
1625                    &idem_bytes,
1626                    "effect_intent idem_key",
1627                )?);
1628                let effect = effect_class
1629                    .as_deref()
1630                    .and_then(crate::EffectClass::from_tag)
1631                    .ok_or(DurableError::Decode {
1632                        context: "effect_intent effect_class missing or invalid",
1633                    })?;
1634                let hmac = hmac
1635                    .map(|bytes| slice_to_array32(&bytes, "effect_intent hmac"))
1636                    .transpose()?;
1637                self.verify_control_hmac(
1638                    id,
1639                    step_id,
1640                    EntryKindTag::EffectIntent.as_str(),
1641                    Some(&idem_key),
1642                    hmac,
1643                )?;
1644                EntryKind::EffectIntent {
1645                    idempotency_key: idem_key,
1646                    effect,
1647                    hmac,
1648                }
1649            }
1650            "checkpoint" => self.checkpoint_entry(id, step_id, payload)?,
1651            other => {
1652                return Err(DurableError::UnsupportedEntryKind {
1653                    kind: static_entry_tag(other),
1654                });
1655            }
1656        };
1657        Ok(JournalEntry {
1658            seq: Some(JournalSeq::new(seq)),
1659            execution_id: id,
1660            kind,
1661            step_id,
1662            entry,
1663            created_at_ms: created_at,
1664        })
1665    }
1666
1667    /// Reconstruct a [`EntryKind::Checkpoint`] from a stored row, opening its sealed snapshot.
1668    ///
1669    /// `step_id` carries the checkpoint's `up_to_step` (the fold boundary); the snapshot is bound to
1670    /// it in the AAD so a checkpoint blob cannot be relocated to a different fold boundary.
1671    fn checkpoint_entry(
1672        &self,
1673        id: ExecutionId,
1674        step_id: StepId,
1675        payload: Option<Vec<u8>>,
1676    ) -> Result<EntryKind, DurableError> {
1677        let sealed = payload.ok_or(DurableError::Decode {
1678            context: "checkpoint entry missing snapshot payload",
1679        })?;
1680        ensure_payload_within_limit(
1681            sealed.len(),
1682            self.max_payload_bytes.saturating_add(SEAL_OVERHEAD_SLACK),
1683        )?;
1684        let aad = PayloadAad::new(id, step_id, EntryKindTag::Checkpoint, None);
1685        let snapshot = self.open_payload(&sealed, &aad)?;
1686        Ok(EntryKind::Checkpoint {
1687            up_to_step: step_id.value(),
1688            snapshot,
1689        })
1690    }
1691
1692    /// Reconstruct every entry from a fetched row set, sharing one kind lookup.
1693    async fn rows_to_entries(
1694        &self,
1695        id: ExecutionId,
1696        rows: Vec<JournalRowRead>,
1697    ) -> Result<Vec<JournalEntry>, DurableError> {
1698        if rows.is_empty() {
1699            return Ok(Vec::new());
1700        }
1701        let kind = self.lookup_kind(id).await?;
1702        let mut entries = Vec::with_capacity(rows.len());
1703        for row in rows {
1704            entries.push(self.row_to_entry(id, kind, row)?);
1705        }
1706        Ok(entries)
1707    }
1708}
1709
1710/// Control-entry HMAC and high-water-mark crypto.
1711impl LocalBackend {
1712    /// Compute the keyed-BLAKE3 row HMAC over a control entry's identity, when an HMAC key is set.
1713    ///
1714    /// Binds `(execution_id, step_id, entry_kind, idem_key?)` so a control row cannot be forged or
1715    /// relocated on a shared database. Returns `None` when no key is configured (single-user local).
1716    fn control_hmac(
1717        &self,
1718        entry: &JournalEntry,
1719        idem_key: Option<&IdempotencyKey>,
1720    ) -> Option<Vec<u8>> {
1721        self.compute_control_hmac(
1722            entry.execution_id,
1723            entry.step_id,
1724            entry.entry.tag(),
1725            idem_key,
1726        )
1727        .map(|h| h.to_vec())
1728    }
1729
1730    /// Core keyed-BLAKE3 computation shared by [`control_hmac`](Self::control_hmac) (write path,
1731    /// takes a full [`JournalEntry`]) and [`verify_control_hmac`](Self::verify_control_hmac) (read
1732    /// path, which has the row's identity fields but not yet a reconstructed entry). Returns `None`
1733    /// when no HMAC key is configured (single-user local).
1734    fn compute_control_hmac(
1735        &self,
1736        execution_id: ExecutionId,
1737        step_id: StepId,
1738        tag: &'static str,
1739        idem_key: Option<&IdempotencyKey>,
1740    ) -> Option<[u8; 32]> {
1741        let key = self.hmac_key.as_ref()?;
1742        Some(Self::keyed_control_hmac(
1743            key,
1744            execution_id,
1745            step_id,
1746            tag,
1747            idem_key,
1748        ))
1749    }
1750
1751    /// Keyed-BLAKE3 computation over a control entry's identity, parameterized on the key so both
1752    /// the current and previous rotation-window keys (#6451) can be tried against the same input.
1753    fn keyed_control_hmac(
1754        key: &[u8; 32],
1755        execution_id: ExecutionId,
1756        step_id: StepId,
1757        tag: &'static str,
1758        idem_key: Option<&IdempotencyKey>,
1759    ) -> [u8; 32] {
1760        let mut input = Vec::with_capacity(16 + 4 + 16 + 32);
1761        input.extend_from_slice(execution_id.as_bytes());
1762        input.extend_from_slice(&step_id.value().to_le_bytes());
1763        input.extend_from_slice(tag.as_bytes());
1764        if let Some(k) = idem_key {
1765            input.extend_from_slice(k.as_bytes());
1766        }
1767        *blake3::keyed_hash(key, &input).as_bytes()
1768    }
1769
1770    /// Recompute and constant-time-verify a control entry's row HMAC read back from storage
1771    /// (INV-8), trying the previous rotation-window key (#6451) when the current key does not
1772    /// match.
1773    ///
1774    /// A no-op only when no HMAC key is configured **and** the row carries no stored HMAC — the
1775    /// documented single-user local stance where control entries carry no HMAC and none is
1776    /// enforced. If this backend is unkeyed but the row *does* carry a stamped HMAC, that is
1777    /// config drift between the writer and this reader (e.g. `shared_db` toggled, or a reader
1778    /// whose config disagrees with the writer's over the same physical file) and is rejected
1779    /// fail-closed rather than silently trusted, since an `EffectIntent`'s fields are plaintext
1780    /// and an unkeyed reader has no way to tell a genuine stamped row from a forged one. When a
1781    /// key *is* configured, every control row this backend reads must carry a matching HMAC: a
1782    /// missing HMAC, or a mismatch under both the current and any registered
1783    /// [`previous_hmac_key`](Self::with_previous_hmac_key), fails closed with
1784    /// [`DurableError::ControlIntegrity`].
1785    ///
1786    /// Each comparison uses [`blake3::Hash`] equality, which compares in constant time (the same
1787    /// idiom used for the promise resolver-token check in `promise.rs`), so a forged HMAC reveals
1788    /// no timing signal beyond which of the (at most two) legitimate keys, if any, it was written
1789    /// under — already observable via `created_at` relative to the rotation.
1790    fn verify_control_hmac(
1791        &self,
1792        execution_id: ExecutionId,
1793        step_id: StepId,
1794        tag: &'static str,
1795        idem_key: Option<&IdempotencyKey>,
1796        stored: Option<[u8; 32]>,
1797    ) -> Result<(), DurableError> {
1798        let Some(current_key) = self.hmac_key.as_ref() else {
1799            return if stored.is_some() {
1800                Err(DurableError::ControlIntegrity)
1801            } else {
1802                Ok(())
1803            };
1804        };
1805        let Some(stored) = stored else {
1806            return Err(DurableError::ControlIntegrity);
1807        };
1808        let expected_current =
1809            Self::keyed_control_hmac(current_key, execution_id, step_id, tag, idem_key);
1810        if blake3::Hash::from(expected_current) == blake3::Hash::from(stored) {
1811            return Ok(());
1812        }
1813        if let Some(previous_key) = self.previous_hmac_key.as_ref() {
1814            let expected_previous =
1815                Self::keyed_control_hmac(previous_key, execution_id, step_id, tag, idem_key);
1816            if blake3::Hash::from(expected_previous) == blake3::Hash::from(stored) {
1817                return Ok(());
1818            }
1819        }
1820        Err(DurableError::ControlIntegrity)
1821    }
1822
1823    /// Compute the high-water-mark HMAC (issue #6360) over the signed
1824    /// `{execution_id, max_committed_step_id, committed_result_count, key_epoch}` tuple.
1825    ///
1826    /// Domain-separated from [`compute_control_hmac`](Self::compute_control_hmac)'s input by
1827    /// construction — this binds `max_committed_step_id` and `committed_result_count`, fields the
1828    /// control-entry HMAC never includes — so the two mechanisms safely share key material without
1829    /// a cross-mechanism forgery becoming possible.
1830    fn compute_hwm_hmac(
1831        execution_id: ExecutionId,
1832        max_committed_step_id: u32,
1833        committed_result_count: u64,
1834        key_epoch: u32,
1835        key: &[u8; 32],
1836    ) -> [u8; 32] {
1837        let mut input = Vec::with_capacity(16 + 4 + 8 + 4);
1838        input.extend_from_slice(execution_id.as_bytes());
1839        input.extend_from_slice(&max_committed_step_id.to_le_bytes());
1840        input.extend_from_slice(&committed_result_count.to_le_bytes());
1841        input.extend_from_slice(&key_epoch.to_le_bytes());
1842        *blake3::keyed_hash(key, &input).as_bytes()
1843    }
1844
1845    /// Resolve the high-water-mark key registered for `epoch`: the current key first, then the
1846    /// registered previous key (FR-008 rotation window).
1847    ///
1848    /// Returns `None` when `epoch` matches neither slot — an unresolvable epoch on a row that
1849    /// carries HWM metadata, which the caller must treat as fail-closed (NFR-004), never as legacy:
1850    /// only a row's total *absence* is legacy, not a present-but-unverifiable one (closes the
1851    /// downgrade lever where a stripped/forged epoch would otherwise masquerade as "predates the
1852    /// feature").
1853    fn resolve_hwm_key(&self, epoch: u32) -> Option<[u8; 32]> {
1854        if let Some(slot) = &self.hwm_key
1855            && slot.epoch == epoch
1856        {
1857            return Some(slot.key);
1858        }
1859        if let Some(slot) = &self.hwm_key_previous
1860            && slot.epoch == epoch
1861        {
1862            return Some(slot.key);
1863        }
1864        None
1865    }
1866
1867    /// Bump the signed high-water-mark (issue #6360) after committing a `StepResult` row, inside
1868    /// the same transaction as its INSERT. A no-op when no HWM key is configured.
1869    ///
1870    /// Reads the current signed tuple (or starts from zero for a first-ever committed result),
1871    /// increments `committed_result_count` by one, raises `max_committed_step_id` to `step_id` when
1872    /// higher, and re-signs under the current epoch — all inside `tx`, so a `StepResult` can never
1873    /// commit without its HWM update landing atomically alongside it (no TOCTOU gap). Folding
1874    /// (`checkpoint_fold`) never calls this: a fold moves the same committed results from live rows
1875    /// into a checkpoint snapshot net-zero, so `committed_result_count` is invariant across it —
1876    /// only [`checkpoint_fold`](Self::checkpoint_fold)'s own `folded_count` column changes.
1877    async fn bump_hwm_for_step_result(
1878        &self,
1879        tx: &mut zeph_db::DbTransaction<'_>,
1880        execution_id: ExecutionId,
1881        step_id: StepId,
1882    ) -> Result<(), DurableError> {
1883        let Some(slot) = &self.hwm_key else {
1884            return Ok(());
1885        };
1886        let exec = execution_id.as_uuid().to_string();
1887        let existing: Option<(i64, i64)> = zeph_db::query_as(sql!(
1888            "SELECT max_committed_step_id, committed_result_count
1889             FROM durable_execution_integrity WHERE execution_id = ?"
1890        ))
1891        .bind(&exec)
1892        .fetch_optional(&mut **tx)
1893        .await
1894        .map_err(|e| DurableError::storage("hwm_bump", e))?;
1895        let (prev_max, prev_count) = existing.unwrap_or((0, 0));
1896        let new_max = prev_max.max(i64::from(step_id.value()));
1897        let new_count = prev_count.saturating_add(1);
1898        let hmac = Self::compute_hwm_hmac(
1899            execution_id,
1900            u32::try_from(new_max).unwrap_or(u32::MAX),
1901            u64::try_from(new_count).unwrap_or(u64::MAX),
1902            slot.epoch,
1903            &slot.key,
1904        );
1905        zeph_db::query(sql!(
1906            "INSERT INTO durable_execution_integrity
1907                (execution_id, key_epoch, max_committed_step_id, committed_result_count, hwm_hmac, updated_at)
1908             VALUES (?, ?, ?, ?, ?, ?)
1909             ON CONFLICT(execution_id) DO UPDATE SET
1910                key_epoch = excluded.key_epoch,
1911                max_committed_step_id = excluded.max_committed_step_id,
1912                committed_result_count = excluded.committed_result_count,
1913                hwm_hmac = excluded.hwm_hmac,
1914                updated_at = excluded.updated_at"
1915        ))
1916        .bind(&exec)
1917        .bind(i64::from(slot.epoch))
1918        .bind(new_max)
1919        .bind(new_count)
1920        .bind(hmac.to_vec())
1921        .bind(now_unix_millis())
1922        .execute(&mut **tx)
1923        .await
1924        .map_err(|e| DurableError::storage("hwm_bump", e))?;
1925        Ok(())
1926    }
1927
1928    /// Verify the signed high-water-mark (issue #6360) for a resumed execution, and fail closed on
1929    /// any mismatch (FR-004, US-003: the durable resume path never offers an override).
1930    ///
1931    /// A no-op when no HWM key is configured. On any verification failure, best-effort finalizes
1932    /// the execution as `Aborted` (mirroring the step-cap-exceeded path in `handle.rs`) before
1933    /// returning the error, so a corrupted execution does not linger `running` forever waiting for
1934    /// a resume attempt that will keep failing.
1935    async fn verify_high_water_mark(&self, execution_id: ExecutionId) -> Result<(), DurableError> {
1936        if self.hwm_key.is_none() {
1937            return Ok(());
1938        }
1939        if let Err(error) = self.check_high_water_mark(execution_id).await {
1940            if let Err(finalize_error) = self.finalize(execution_id, ExecutionStatus::Aborted).await
1941            {
1942                tracing::warn!(
1943                    error = %finalize_error,
1944                    "failed to mark HWM-integrity-failed execution aborted"
1945                );
1946            }
1947            return Err(error);
1948        }
1949        Ok(())
1950    }
1951
1952    /// The comparison half of `verify_high_water_mark`.
1953    ///
1954    /// Absent a stored `durable_execution_integrity` row: **pre-seal** (or unkeyed), this
1955    /// execution predates the feature or has committed no `StepResult` yet — nothing to compare
1956    /// against, so it is accepted (migration posture: only a row's total absence is legacy,
1957    /// mirroring the JSONL side's "no chain metadata at all" lane). **Post-seal** (issue #6449 —
1958    /// `integrity_sealed == true`, confirmed via the vault-stored `ZEPH_DURABLE_INTEGRITY_SEALED`
1959    /// marker, never a DB column), a keyed, non-grandfathered execution with at least one
1960    /// committed `StepResult` but no integrity row is unconditional tamper: the drain-before-seal
1961    /// precondition on `zeph durable seal-integrity` guarantees no execution can reach this state
1962    /// legitimately once sealed (the keyed integrity-row write is atomic-in-transaction with the
1963    /// `StepResult` commit, so "committed result present, row absent" cannot occur for anything
1964    /// that started after the vault key was attached). A *present* row is always fully verified:
1965    /// an unresolvable `key_epoch`, an HMAC that does not authenticate, or a recomputed
1966    /// `committed_result_count` that disagrees with the signed value are each a distinct
1967    /// fail-closed [`DurableError::HighWaterMarkIntegrity`].
1968    async fn check_high_water_mark(&self, execution_id: ExecutionId) -> Result<(), DurableError> {
1969        let exec = execution_id.as_uuid().to_string();
1970        let stored: Option<(i64, i64, i64, Vec<u8>)> = zeph_db::query_as(sql!(
1971            "SELECT key_epoch, max_committed_step_id, committed_result_count, hwm_hmac
1972             FROM durable_execution_integrity WHERE execution_id = ?"
1973        ))
1974        .bind(&exec)
1975        .fetch_optional(&self.pool)
1976        .await
1977        .map_err(|e| DurableError::storage("hwm_verify", e))?;
1978        let Some((epoch_raw, max_step_raw, count_raw, hmac)) = stored else {
1979            if self.hwm_key.is_some()
1980                && self.integrity_sealed
1981                && !self.integrity_grandfather.contains(&execution_id)
1982                && self.committed_step_result_count(execution_id).await? >= 1
1983            {
1984                return Err(DurableError::HighWaterMarkIntegrity {
1985                    execution_id,
1986                    reason: "integrity_row_absent_post_seal",
1987                    hint: "TAMPER: this backend is sealed against pre-feature integrity-row \
1988                           absence, this execution is keyed and not grandfathered, and it has \
1989                           committed StepResults — a legitimate keyed execution can never reach \
1990                           this state (the integrity row is written atomically with its first \
1991                           committed StepResult), so an absent row here means the row was \
1992                           deleted outside the write path",
1993                });
1994            }
1995            return Ok(());
1996        };
1997
1998        // Per FR-008, the operator-facing `hint` distinguishes "possibly re-keyed" (a legitimate
1999        // rotation this backend cannot resolve) from "TAMPER" (content that did not authenticate)
2000        // — the durable resume path stays fail-closed either way (FR-004), but the two cases call
2001        // for different operator follow-up, so they must not read the same in the logs.
2002        let fail =
2003            |reason: &'static str, hint: &'static str| DurableError::HighWaterMarkIntegrity {
2004                execution_id,
2005                reason,
2006                hint,
2007            };
2008        let tamper = |reason: &'static str| {
2009            fail(
2010                reason,
2011                "TAMPER: the signed high-water-mark did not authenticate under any key this \
2012                 backend holds for the recorded epoch",
2013            )
2014        };
2015
2016        let epoch = u32::try_from(epoch_raw).map_err(|_| tamper("hmac_mismatch"))?;
2017        let Some(key) = self.resolve_hwm_key(epoch) else {
2018            return Err(fail(
2019                "key_epoch_unresolvable",
2020                "possibly re-keyed: this execution's signed key_epoch is neither the current key \
2021                 nor a registered previous rotation key — if ZEPH_DURABLE_KEY was recently \
2022                 rotated, ensure the rotation window is still open (ZEPH_DURABLE_KEY_PREVIOUS \
2023                 present and [durable] previous_key_id set); the window is closed permanently by \
2024                 `zeph durable rotate-key --drop-previous`. The durable resume path cannot \
2025                 proceed without it (no interactive override)",
2026            ));
2027        };
2028        let stored_hmac =
2029            <[u8; 32]>::try_from(hmac.as_slice()).map_err(|_| tamper("hmac_mismatch"))?;
2030        let max_step = u32::try_from(max_step_raw).unwrap_or(u32::MAX);
2031        let count = u64::try_from(count_raw).unwrap_or(u64::MAX);
2032        let expected = Self::compute_hwm_hmac(execution_id, max_step, count, epoch, &key);
2033        if blake3::Hash::from(expected) != blake3::Hash::from(stored_hmac) {
2034            return Err(tamper("hmac_mismatch"));
2035        }
2036
2037        let recomputed = self.committed_step_result_count(execution_id).await?;
2038        if recomputed != count {
2039            return Err(fail(
2040                "count_mismatch",
2041                "TAMPER: the recomputed committed-result count (surviving StepResult rows plus \
2042                 every checkpoint's folded_count) disagrees with the signed value — a committed \
2043                 result was likely deleted outside the write path",
2044            ));
2045        }
2046        Ok(())
2047    }
2048}
2049
2050/// Payload sealing and opening.
2051impl LocalBackend {
2052    /// Seal a plaintext payload, or pass it through verbatim when no cipher is configured.
2053    fn seal_payload(&self, plaintext: &[u8], aad: &PayloadAad) -> Result<Vec<u8>, DurableError> {
2054        match &self.cipher {
2055            Some(cipher) => Ok(cipher.seal(plaintext, aad)?),
2056            None => Ok(plaintext.to_vec()),
2057        }
2058    }
2059
2060    /// Open a sealed payload, or copy it through verbatim when no cipher is configured.
2061    fn open_payload(&self, sealed: &[u8], aad: &PayloadAad) -> Result<Bytes, DurableError> {
2062        match &self.cipher {
2063            Some(cipher) => Ok(Bytes::from(cipher.open(sealed, aad)?)),
2064            None => Ok(Bytes::copy_from_slice(sealed)),
2065        }
2066    }
2067}
2068
2069/// Retention: prunable/orphan counting and batch pruning.
2070impl LocalBackend {
2071    /// Count terminal executions a [`prune`](Journal::prune) sweep would delete under `policy`.
2072    ///
2073    /// Read-only: backs `zeph durable prune --dry-run`. It applies the same TTL cutoffs as the
2074    /// delete path, so the count is exactly what a real sweep would remove now.
2075    ///
2076    /// # Errors
2077    ///
2078    /// Returns [`DurableError::Storage`] if the query fails.
2079    pub async fn count_prunable(&self, policy: &RetentionPolicy) -> Result<u64, DurableError> {
2080        let cutoffs = crate::retention::PruneCutoffs::from_policy(policy, now_unix_millis());
2081        let (count,): (i64,) = zeph_db::query_as(sql!(
2082            "SELECT COUNT(*) FROM durable_executions
2083             WHERE finalized_at IS NOT NULL
2084               AND ( (status = 'completed' AND finalized_at <= ?)
2085                  OR (status IN ('failed', 'aborted', 'canceled') AND finalized_at <= ?) )"
2086        ))
2087        .bind(cutoffs.completed_before_ms)
2088        .bind(cutoffs.failed_before_ms)
2089        .fetch_one(&self.pool)
2090        .await
2091        .map_err(|e| DurableError::storage("count_prunable", e))?;
2092        Ok(count.max(0).cast_unsigned())
2093    }
2094
2095    /// Count crash-orphaned executions a [`sweep_orphans`](Journal::sweep_orphans) sweep would
2096    /// abort under `policy` (#6254).
2097    ///
2098    /// Read-only: backs `zeph durable prune --dry-run`. Mirrors the real sweep's staleness scan
2099    /// and INV-15 flock liveness check (acquiring and immediately releasing each candidate's
2100    /// `ExecutionLock`, exactly as the real sweep does, so the count reflects genuinely
2101    /// unowned rows rather than staleness alone) — but never mutates `status`. Returns `0` when
2102    /// the sweep is disabled (`stale_running_after_secs == 0`) or this backend has no `lock_dir`.
2103    ///
2104    /// # Errors
2105    ///
2106    /// Returns [`DurableError::Storage`] if the query fails.
2107    pub async fn count_orphans(&self, policy: &RetentionPolicy) -> Result<u64, DurableError> {
2108        if policy.stale_running_after_secs == 0 {
2109            return Ok(0);
2110        }
2111        let Some(lock_dir) = self.lock_dir.clone() else {
2112            return Ok(0);
2113        };
2114        let cutoff_ms = orphan_cutoff_ms(policy, now_unix_millis());
2115        let candidates: Vec<(String,)> = zeph_db::query_as(sql!(
2116            "SELECT execution_id FROM durable_executions WHERE status = 'running' AND updated_at <= ?"
2117        ))
2118        .bind(cutoff_ms)
2119        .fetch_all(&self.pool)
2120        .await
2121        .map_err(|e| DurableError::storage("count_orphans", e))?;
2122        let mut count = 0u64;
2123        for (exec_str,) in &candidates {
2124            let Ok(execution_id) = parse_execution_id(exec_str) else {
2125                continue;
2126            };
2127            if ExecutionLock::acquire(&lock_dir, execution_id).is_ok() {
2128                count += 1;
2129            }
2130        }
2131        Ok(count)
2132    }
2133
2134    /// Count sealed-payload rows across `durable_journal` and `durable_promises` whose leading
2135    /// on-disk byte — the AEAD key-id selector (`zeph_core::durable::XChaCha20Poly1305Cipher`'s
2136    /// `key_id(1) || nonce(24) || ciphertext || tag(16)` layout) — equals `key_id`.
2137    ///
2138    /// Read-only; backs `zeph durable rotate-key --drop-previous`'s default-on safety scan
2139    /// (#6447): a nonzero count means payloads still sealed under the previous key would become
2140    /// permanently unreadable (`UnknownKeyId`) if that key were dropped now. Filters `payload IS
2141    /// NOT NULL` on both tables — control entries (`EffectIntent`) carry no payload and are
2142    /// irrelevant to this scan.
2143    ///
2144    /// The predicate is dialect-specific because `SQLite`'s `substr` on a `BLOB` returns a 1-byte
2145    /// `BLOB` (compared here against a bound single-byte blob) while `PostgreSQL`'s `bytea`
2146    /// cannot be compared against an integer at all (`get_byte(payload, 0)` extracts it as an
2147    /// `INTEGER` instead).
2148    ///
2149    /// May over-count in a mixed-mode deployment where some rows were written while
2150    /// `encrypt_payload = false` (plaintext, no key-id prefix): a plaintext row's leading byte is
2151    /// arbitrary content that can coincidentally equal `key_id`. This is intentionally fail-safe
2152    /// — it can only cause an unnecessary refusal (resolved with `--force`), never a missed match
2153    /// that would let a genuinely-sealed row be dropped silently.
2154    ///
2155    /// # Errors
2156    ///
2157    /// Returns [`DurableError::Storage`] if either query fails.
2158    pub async fn count_sealed_under_key_id(&self, key_id: u8) -> Result<u64, DurableError> {
2159        #[cfg(feature = "postgres")]
2160        {
2161            let key_id_param = i32::from(key_id);
2162            let (journal_count,): (i64,) = zeph_db::query_as(sql!(
2163                "SELECT COUNT(*) FROM durable_journal
2164                 WHERE payload IS NOT NULL AND get_byte(payload, 0) = ?"
2165            ))
2166            .bind(key_id_param)
2167            .fetch_one(&self.pool)
2168            .await
2169            .map_err(|e| DurableError::storage("count_sealed_under_key_id", e))?;
2170            let (promises_count,): (i64,) = zeph_db::query_as(sql!(
2171                "SELECT COUNT(*) FROM durable_promises
2172                 WHERE payload IS NOT NULL AND get_byte(payload, 0) = ?"
2173            ))
2174            .bind(key_id_param)
2175            .fetch_one(&self.pool)
2176            .await
2177            .map_err(|e| DurableError::storage("count_sealed_under_key_id", e))?;
2178            Ok((journal_count.max(0) + promises_count.max(0)).cast_unsigned())
2179        }
2180        #[cfg(not(feature = "postgres"))]
2181        {
2182            let key_byte = vec![key_id];
2183            let (journal_count,): (i64,) = zeph_db::query_as(sql!(
2184                "SELECT COUNT(*) FROM durable_journal
2185                 WHERE payload IS NOT NULL AND substr(payload, 1, 1) = ?"
2186            ))
2187            .bind(key_byte.clone())
2188            .fetch_one(&self.pool)
2189            .await
2190            .map_err(|e| DurableError::storage("count_sealed_under_key_id", e))?;
2191            let (promises_count,): (i64,) = zeph_db::query_as(sql!(
2192                "SELECT COUNT(*) FROM durable_promises
2193                 WHERE payload IS NOT NULL AND substr(payload, 1, 1) = ?"
2194            ))
2195            .bind(key_byte)
2196            .fetch_one(&self.pool)
2197            .await
2198            .map_err(|e| DurableError::storage("count_sealed_under_key_id", e))?;
2199            Ok((journal_count.max(0) + promises_count.max(0)).cast_unsigned())
2200        }
2201    }
2202
2203    /// Count `EffectIntent` control entries whose row HMAC (INV-8) verifies **only** under the
2204    /// registered [`previous_hmac_key`](Self::with_previous_hmac_key), not the current
2205    /// [`hmac_key`](Self::with_hmac_key) (#6451).
2206    ///
2207    /// The read-side counterpart to [`count_sealed_under_key_id`](Self::count_sealed_under_key_id)
2208    /// for the control-entry HMAC's own rotation window, and **not redundant** with it: the AEAD
2209    /// blob-scan only sees payload-bearing rows, but a pre-rotation `EffectIntent` whose
2210    /// `StepResult` was never committed (a crash between intent and result, in a still-retained
2211    /// non-terminal execution) has a previous-key HMAC and no payload at all — the blob-scan
2212    /// cannot see it, so dropping the previous key without this scan would silently orphan its
2213    /// HMAC verification. Only `EffectIntent` rows carry a persisted+verified HMAC:
2214    /// `PromiseCreated`/`TimerArmed`/`TimerFired`/`Checkpoint` all return
2215    /// [`DurableError::UnsupportedEntryKind`] in `prepare_row`, and
2216    /// `durable_promises` has no `hmac` column.
2217    ///
2218    /// Backs `zeph durable rotate-key --drop-previous`'s safety scan alongside the AEAD blob-scan
2219    /// — refuse the drop while **either** is nonzero. This is a fourth, dedicated key-attach site
2220    /// distinct from the three runtime read paths (agent replay, scheduler daemon, CLI read):
2221    /// the caller must attach **both** [`with_hmac_key`](Self::with_hmac_key) (current) and
2222    /// [`with_previous_hmac_key`](Self::with_previous_hmac_key) (previous) to this backend before
2223    /// calling, or every row's HMAC is unrecomputable and this returns
2224    /// [`DurableError::ControlIntegrity`] rather than a (silently wrong) count.
2225    ///
2226    /// Uses the precise variant — recompute-and-compare against both keys — rather than a pure
2227    /// "fails under current" fail-safe: a genuinely corrupt/forged row (matches neither key) is
2228    /// not counted here, since it is not something dropping the previous key would newly break;
2229    /// [`read_execution`](Journal::read_execution) already rejects it on every read regardless of
2230    /// which key is dropped.
2231    ///
2232    /// Cold path (runs only at `--drop-previous`); control rows are sparse.
2233    ///
2234    /// # Errors
2235    ///
2236    /// Returns [`DurableError::Storage`] if the query fails, or [`DurableError::ControlIntegrity`]
2237    /// if matching control rows exist but this backend is missing the current or previous HMAC
2238    /// key needed to recompute them.
2239    pub async fn count_control_entries_under_previous_hmac(&self) -> Result<u64, DurableError> {
2240        let rows: Vec<ControlHmacScanRow> = zeph_db::query_as(sql!(
2241            "SELECT execution_id, step_id, idem_key, hmac
2242             FROM durable_journal
2243             WHERE entry_kind = 'effect_intent' AND hmac IS NOT NULL"
2244        ))
2245        .fetch_all(&self.pool)
2246        .await
2247        .map_err(|e| DurableError::storage("count_control_entries_under_previous_hmac", e))?;
2248
2249        if rows.is_empty() {
2250            return Ok(0);
2251        }
2252
2253        let (Some(current_key), Some(previous_key)) =
2254            (self.hmac_key.as_ref(), self.previous_hmac_key.as_ref())
2255        else {
2256            return Err(DurableError::ControlIntegrity);
2257        };
2258
2259        let mut count = 0u64;
2260        for (execution_id_raw, step_id_raw, idem_key_raw, hmac_raw) in rows {
2261            let Ok(execution_id) = parse_execution_id(&execution_id_raw) else {
2262                continue;
2263            };
2264            let Ok(step_id_value) = u32::try_from(step_id_raw) else {
2265                continue;
2266            };
2267            let step_id = StepId::new(step_id_value);
2268            let idem_key = idem_key_raw
2269                .as_deref()
2270                .and_then(|b| slice_to_array32(b, "effect_intent idem_key").ok())
2271                .map(IdempotencyKey::from_bytes);
2272            let Ok(stored) = slice_to_array32(&hmac_raw, "effect_intent hmac") else {
2273                continue;
2274            };
2275
2276            let tag = EntryKindTag::EffectIntent.as_str();
2277            let expected_current = Self::keyed_control_hmac(
2278                current_key,
2279                execution_id,
2280                step_id,
2281                tag,
2282                idem_key.as_ref(),
2283            );
2284            if blake3::Hash::from(expected_current) == blake3::Hash::from(stored) {
2285                continue;
2286            }
2287            let expected_previous = Self::keyed_control_hmac(
2288                previous_key,
2289                execution_id,
2290                step_id,
2291                tag,
2292                idem_key.as_ref(),
2293            );
2294            if blake3::Hash::from(expected_previous) == blake3::Hash::from(stored) {
2295                count += 1;
2296            }
2297        }
2298        Ok(count)
2299    }
2300
2301    /// Count `durable_execution_integrity` rows whose high-water-mark was signed under `epoch`.
2302    ///
2303    /// The `--drop-previous` HWM scan (addendum to #6451, spec-081 FR-008): before permanently
2304    /// removing the previous rotation key, refuse if any surviving execution's HWM row is still
2305    /// addressed to the previous epoch. Unlike
2306    /// [`count_control_entries_under_previous_hmac`](Self::count_control_entries_under_previous_hmac),
2307    /// the HWM row carries `key_epoch` in the clear, so this is a plain indexed `COUNT` — no key
2308    /// material, no per-row recompute. This is also the only one of the three `--drop-previous`
2309    /// scans that catches a checkpoint-folded pre-rotation execution: `checkpoint_fold` never
2310    /// re-signs the HWM, so a folded execution's integrity row keeps
2311    /// `key_epoch = previous_key_id` even though its old-key-id payloads are gone — invisible to
2312    /// both the AEAD blob-scan
2313    /// ([`count_sealed_under_key_id`](Self::count_sealed_under_key_id)) and the control-HMAC scan
2314    /// (`EffectIntent`-only). Terminal-but-unpruned executions are counted too (the row is deleted
2315    /// only by the retention prune sweep, never on `finalize`) — fail-safe over-refusal, resolvable
2316    /// with `--force`, mirroring the other two scans' coarseness.
2317    ///
2318    /// Cold path (runs only at `--drop-previous`); integrity rows are sparse (one per execution).
2319    ///
2320    /// # Errors
2321    ///
2322    /// Returns [`DurableError::Storage`] if the query fails.
2323    pub async fn count_integrity_rows_under_epoch(&self, epoch: u32) -> Result<u64, DurableError> {
2324        let count: i64 = zeph_db::query_scalar(sql!(
2325            "SELECT COUNT(*) FROM durable_execution_integrity WHERE key_epoch = ?"
2326        ))
2327        .bind(i64::from(epoch))
2328        .fetch_one(&self.pool)
2329        .await
2330        .map_err(|e| DurableError::storage("count_integrity_rows_under_epoch", e))?;
2331        Ok(count.max(0).cast_unsigned())
2332    }
2333
2334    /// Delete one bounded batch of prunable terminal executions and their child rows.
2335    ///
2336    /// Selects up to `batch` executions past their TTL, then deletes their journal, promise, timer,
2337    /// integrity (issue #6360), and execution rows in a single transaction (children first, to
2338    /// respect the foreign keys). Returns the number of executions removed; the retention loop
2339    /// stops once a batch returns fewer than `batch`.
2340    ///
2341    /// The candidate-selection `SELECT` runs *inside* the same `begin_write` transaction as the
2342    /// deletes (not on the autocommit pool beforehand), closing the race where a concurrent
2343    /// `open_execution` reopen (un-finalize, #6251) lands between "select prunable ids" and
2344    /// "delete them" — without this, a legitimately-resumed execution could be deleted out from
2345    /// under its own reopen. `SQLite`'s `BEGIN IMMEDIATE` (via `begin_write`) already serializes
2346    /// writers at the file level, so the `SELECT` alone is enough there; `PostgreSQL` needs an
2347    /// explicit `SELECT ... FOR UPDATE` first to take row locks on the same candidates before
2348    /// they're read, since a plain `BEGIN` does not otherwise block a concurrent `UPDATE` on those
2349    /// rows (mirrors the `BEGIN IMMEDIATE` / `SELECT FOR UPDATE` split in `goal/store.rs`).
2350    async fn delete_prune_batch(
2351        &self,
2352        cutoffs: crate::retention::PruneCutoffs,
2353        batch: u64,
2354    ) -> Result<u64, DurableError> {
2355        let mut tx = zeph_db::begin_write(&self.pool)
2356            .await
2357            .map_err(|e| DurableError::storage("prune", e))?;
2358
2359        // Postgres only: lock the same candidate rows before reading them, so a concurrent
2360        // `open_execution` reopen UPDATE on one of these rows blocks until this transaction
2361        // commits (and then no longer matches, since the SELECT below re-reads post-commit) or
2362        // this transaction rolls back. Bounded by the same ORDER BY/LIMIT as the real read below
2363        // so the lock's blast radius matches the batch, not the whole prunable backlog.
2364        #[cfg(feature = "postgres")]
2365        zeph_db::query(sql!(
2366            "SELECT execution_id FROM durable_executions
2367             WHERE finalized_at IS NOT NULL
2368               AND ( (status = 'completed' AND finalized_at <= ?)
2369                  OR (status IN ('failed', 'aborted', 'canceled') AND finalized_at <= ?) )
2370             ORDER BY finalized_at LIMIT ?
2371             FOR UPDATE"
2372        ))
2373        .bind(cutoffs.completed_before_ms)
2374        .bind(cutoffs.failed_before_ms)
2375        .bind(i64::try_from(batch).unwrap_or(i64::MAX))
2376        .execute(&mut *tx)
2377        .await
2378        .map_err(|e| DurableError::storage("prune", e))?;
2379
2380        let ids: Vec<(String,)> = zeph_db::query_as(sql!(
2381            "SELECT execution_id FROM durable_executions
2382             WHERE finalized_at IS NOT NULL
2383               AND ( (status = 'completed' AND finalized_at <= ?)
2384                  OR (status IN ('failed', 'aborted', 'canceled') AND finalized_at <= ?) )
2385             ORDER BY finalized_at LIMIT ?"
2386        ))
2387        .bind(cutoffs.completed_before_ms)
2388        .bind(cutoffs.failed_before_ms)
2389        .bind(i64::try_from(batch).unwrap_or(i64::MAX))
2390        .fetch_all(&mut *tx)
2391        .await
2392        .map_err(|e| DurableError::storage("prune", e))?;
2393        if ids.is_empty() {
2394            tx.commit()
2395                .await
2396                .map_err(|e| DurableError::storage("prune", e))?;
2397            return Ok(0);
2398        }
2399        let journal = sql!("DELETE FROM durable_journal WHERE execution_id = ?");
2400        let promises = sql!("DELETE FROM durable_promises WHERE execution_id = ?");
2401        let timers = sql!("DELETE FROM durable_timers WHERE execution_id = ?");
2402        // Issue #6360: `durable_execution_integrity` references `durable_executions` without
2403        // `ON DELETE CASCADE` (same convention as journal/promises/timers), so it must be deleted
2404        // here too — otherwise the `DELETE FROM durable_executions` below violates the FK on every
2405        // backend with FK enforcement on (Postgres always; SQLite via `zeph-db`'s
2406        // `PRAGMA foreign_keys = ON`), rolling back the whole prune batch for any keyed execution
2407        // that ever committed a `StepResult` (`bump_hwm_for_step_result` always writes this row
2408        // when an HWM key is configured). A no-op `DELETE` for an unkeyed/never-committed execution
2409        // (no row present) is fine.
2410        let integrity = sql!("DELETE FROM durable_execution_integrity WHERE execution_id = ?");
2411        // Re-guarded by the same status/finalized_at predicate as the SELECT above (not just
2412        // `execution_id = ?`) — belt and suspenders alongside the transactional read above.
2413        let executions = sql!(
2414            "DELETE FROM durable_executions
2415             WHERE execution_id = ?
2416               AND finalized_at IS NOT NULL
2417               AND ( (status = 'completed' AND finalized_at <= ?)
2418                  OR (status IN ('failed', 'aborted', 'canceled') AND finalized_at <= ?) )"
2419        );
2420        let mut removed = 0u64;
2421        for (id,) in &ids {
2422            for stmt in [journal, promises, timers, integrity] {
2423                zeph_db::query(stmt)
2424                    .bind(id)
2425                    .execute(&mut *tx)
2426                    .await
2427                    .map_err(|e| DurableError::storage("prune", e))?;
2428            }
2429            let result = zeph_db::query(executions)
2430                .bind(id)
2431                .bind(cutoffs.completed_before_ms)
2432                .bind(cutoffs.failed_before_ms)
2433                .execute(&mut *tx)
2434                .await
2435                .map_err(|e| DurableError::storage("prune", e))?;
2436            removed += result.rows_affected();
2437        }
2438        tx.commit()
2439            .await
2440            .map_err(|e| DurableError::storage("prune", e))?;
2441        Ok(removed)
2442    }
2443
2444    /// One batch of the crash-orphan sweep (INV-17, #6254).
2445    ///
2446    /// Selects up to `batch` `status='running'` rows whose `updated_at` is at or before
2447    /// `cutoff_ms`, then for each candidate non-blockingly try-acquires its INV-15
2448    /// `ExecutionLock`: `ExecutionLocked` (a live owner holds it) short-circuits to skip —
2449    /// staleness of `updated_at` alone is never sufficient grounds to abort. Only when the lock is
2450    /// acquired does the guarded `UPDATE` run, still holding the lock, so the abort is race-free
2451    /// against a concurrent `open_execution_exclusive` reopen for the same id (both require the
2452    /// same non-reentrant flock). The lock releases when it drops at the end of each loop
2453    /// iteration.
2454    ///
2455    /// `cursor` is the previous batch's [`SweepCursor`](crate::retention::SweepCursor) (`None` for
2456    /// the first batch); the candidate scan is keyset-paginated strictly past it so a skipped
2457    /// (lock-held) row is never re-selected by a later batch — #6254 C1: without this, a batch
2458    /// consisting entirely of lock-held rows would re-select the identical rows on every
2459    /// iteration and the caller's batch loop would never terminate. Returns the number of rows
2460    /// scanned (for the caller's batch-continuation decision), the number actually aborted, and
2461    /// the cursor to resume from on the next call.
2462    async fn sweep_orphan_batch(
2463        &self,
2464        lock_dir: &std::path::Path,
2465        cutoff_ms: i64,
2466        batch: u64,
2467        cursor: Option<crate::retention::SweepCursor>,
2468    ) -> Result<crate::retention::SweepBatchOutcome, DurableError> {
2469        // Sentinel "no lower bound" cursor: every real `updated_at` (Unix ms) is > i64::MIN, so
2470        // this keyset predicate is a no-op on the first batch while still using one static,
2471        // sql!()-cacheable query for both the first and subsequent calls.
2472        let (after_updated_at, after_exec) = cursor.map_or((i64::MIN, String::new()), |c| {
2473            (c.updated_at_ms, c.execution_id)
2474        });
2475
2476        let candidates: Vec<(String, i64)> = zeph_db::query_as(sql!(
2477            "SELECT execution_id, updated_at FROM durable_executions
2478             WHERE status = 'running' AND updated_at <= ?
2479               AND (updated_at > ? OR (updated_at = ? AND execution_id > ?))
2480             ORDER BY updated_at, execution_id LIMIT ?"
2481        ))
2482        .bind(cutoff_ms)
2483        .bind(after_updated_at)
2484        .bind(after_updated_at)
2485        .bind(&after_exec)
2486        .bind(i64::try_from(batch).unwrap_or(i64::MAX))
2487        .fetch_all(&self.pool)
2488        .await
2489        .map_err(|e| DurableError::storage("sweep_orphans", e))?;
2490
2491        let scanned = u64::try_from(candidates.len()).unwrap_or(u64::MAX);
2492        let next_cursor = candidates
2493            .last()
2494            .map(|(id, updated_at)| crate::retention::SweepCursor {
2495                updated_at_ms: *updated_at,
2496                execution_id: id.clone(),
2497            });
2498
2499        let now = now_unix_millis();
2500        let abort = sql!(
2501            "UPDATE durable_executions SET status = 'aborted', finalized_at = ?, updated_at = ?
2502             WHERE execution_id = ? AND status = 'running' AND finalized_at IS NULL"
2503        );
2504        let mut aborted = 0u64;
2505        for (exec_str, _updated_at) in &candidates {
2506            let Ok(execution_id) = parse_execution_id(exec_str) else {
2507                continue;
2508            };
2509            match ExecutionLock::acquire(lock_dir, execution_id) {
2510                Ok(_lock) => {
2511                    let result = zeph_db::query(abort)
2512                        .bind(now)
2513                        .bind(now)
2514                        .bind(exec_str)
2515                        .execute(&self.pool)
2516                        .await
2517                        .map_err(|e| DurableError::storage("sweep_orphans", e))?;
2518                    aborted += result.rows_affected();
2519                    // `_lock` drops here, releasing the flock for the next holder.
2520                }
2521                Err(DurableError::ExecutionLocked { .. }) => {
2522                    // A live owner holds this execution — never abort on staleness alone (INV-17).
2523                }
2524                Err(e) => return Err(e),
2525            }
2526        }
2527        Ok(crate::retention::SweepBatchOutcome {
2528            scanned,
2529            aborted,
2530            next_cursor,
2531        })
2532    }
2533}
2534
2535impl Journal for LocalBackend {
2536    async fn append(&self, entry: JournalEntry) -> Result<JournalSeq, DurableError> {
2537        let span = tracing::info_span!(
2538            "durable.journal.append",
2539            execution_id = %entry.execution_id.as_uuid(),
2540            step_id = entry.step_id.value(),
2541            entry_kind = entry.entry.tag(),
2542        );
2543        async move {
2544            let row = self.prepare_row(&entry)?;
2545            let insert = sql!(
2546                "INSERT INTO durable_journal
2547                    (execution_id, step_id, entry_kind, idem_key, effect_class, payload, payload_version, hmac, created_at)
2548                 VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)
2549                 RETURNING seq"
2550            );
2551            // A `StepResult` needs its HWM bump (issue #6360) committed atomically alongside the
2552            // INSERT, so it runs inside a transaction; every other entry kind keeps the direct
2553            // autocommit path (unchanged from before this feature).
2554            let seq: i64 = if matches!(entry.entry, EntryKind::StepResult { .. }) {
2555                let mut tx = zeph_db::begin_write(&self.pool)
2556                    .await
2557                    .map_err(|e| DurableError::storage("append", e))?;
2558                let (seq,): (i64,) = zeph_db::query_as(insert)
2559                    .bind(row.execution_id)
2560                    .bind(row.step_id)
2561                    .bind(row.entry_kind)
2562                    .bind(row.idem_key)
2563                    .bind(row.effect_class)
2564                    .bind(row.payload)
2565                    .bind(row.payload_version)
2566                    .bind(row.hmac)
2567                    .bind(row.created_at)
2568                    .fetch_one(&mut *tx)
2569                    .await
2570                    .map_err(|e| DurableError::storage("append", e))?;
2571                self.bump_hwm_for_step_result(&mut tx, entry.execution_id, entry.step_id)
2572                    .await?;
2573                tx.commit()
2574                    .await
2575                    .map_err(|e| DurableError::storage("append", e))?;
2576                seq
2577            } else {
2578                let (seq,): (i64,) = zeph_db::query_as(insert)
2579                    .bind(row.execution_id)
2580                    .bind(row.step_id)
2581                    .bind(row.entry_kind)
2582                    .bind(row.idem_key)
2583                    .bind(row.effect_class)
2584                    .bind(row.payload)
2585                    .bind(row.payload_version)
2586                    .bind(row.hmac)
2587                    .bind(row.created_at)
2588                    .fetch_one(&self.pool)
2589                    .await
2590                    .map_err(|e| DurableError::storage("append", e))?;
2591                seq
2592            };
2593            Ok(JournalSeq::new(seq))
2594        }
2595        .instrument(span)
2596        .await
2597    }
2598
2599    async fn read_execution(&self, id: ExecutionId) -> Result<Vec<JournalEntry>, DurableError> {
2600        let span = tracing::info_span!(
2601            "durable.journal.read",
2602            execution_id = %id.as_uuid(),
2603            step_count = tracing::field::Empty,
2604        );
2605        async move {
2606            let rows: Vec<JournalRowRead> = zeph_db::query_as(sql!(
2607                "SELECT seq, step_id, entry_kind, idem_key, effect_class, payload, payload_version, hmac, created_at
2608                 FROM durable_journal WHERE execution_id = ? ORDER BY seq"
2609            ))
2610            .bind(id.as_uuid().to_string())
2611            .fetch_all(&self.pool)
2612            .await
2613            .map_err(|e| DurableError::storage("read", e))?;
2614            let entries = self.rows_to_entries(id, rows).await?;
2615            tracing::Span::current().record("step_count", entries.len());
2616            Ok(entries)
2617        }
2618        .instrument(span)
2619        .await
2620    }
2621
2622    async fn read_execution_range(
2623        &self,
2624        id: ExecutionId,
2625        from_step_id: u32,
2626        limit: usize,
2627    ) -> Result<Vec<JournalEntry>, DurableError> {
2628        let span = tracing::info_span!(
2629            "durable.journal.read_segment",
2630            execution_id = %id.as_uuid(),
2631            from_step_id,
2632            count = tracing::field::Empty,
2633        );
2634        async move {
2635            let rows: Vec<JournalRowRead> = zeph_db::query_as(sql!(
2636                "SELECT seq, step_id, entry_kind, idem_key, effect_class, payload, payload_version, hmac, created_at
2637                 FROM durable_journal WHERE execution_id = ? AND step_id >= ? ORDER BY step_id, seq LIMIT ?"
2638            ))
2639            .bind(id.as_uuid().to_string())
2640            .bind(i64::from(from_step_id))
2641            .bind(i64::try_from(limit).unwrap_or(i64::MAX))
2642            .fetch_all(&self.pool)
2643            .await
2644            .map_err(|e| DurableError::storage("read_segment", e))?;
2645            let entries = self.rows_to_entries(id, rows).await?;
2646            tracing::Span::current().record("count", entries.len());
2647            Ok(entries)
2648        }
2649        .instrument(span)
2650        .await
2651    }
2652
2653    async fn finalize(&self, id: ExecutionId, status: ExecutionStatus) -> Result<(), DurableError> {
2654        let span = tracing::info_span!(
2655            "durable.journal.finalize",
2656            execution_id = %id.as_uuid(),
2657            status = status.as_str(),
2658        );
2659        async move {
2660            let now = now_unix_millis();
2661            let finalized_at = (!status.is_running()).then_some(now);
2662            let mut tx = zeph_db::begin_write(&self.pool)
2663                .await
2664                .map_err(|e| DurableError::storage("finalize", e))?;
2665            // `AND status = 'running'` makes this a one-shot transition: whichever of a concurrent
2666            // divergence-triggered `Aborted` or a caller's `Completed`/`Failed` commits first wins,
2667            // and the loser's UPDATE affects zero rows instead of clobbering the winner's terminal
2668            // status (finalize is otherwise safe to call more than once per execution).
2669            zeph_db::query(sql!(
2670                "UPDATE durable_executions SET status = ?, updated_at = ?, finalized_at = ?
2671                 WHERE execution_id = ? AND status = 'running'"
2672            ))
2673            .bind(status.as_str())
2674            .bind(now)
2675            .bind(finalized_at)
2676            .bind(id.as_uuid().to_string())
2677            .execute(&mut *tx)
2678            .await
2679            .map_err(|e| DurableError::storage("finalize", e))?;
2680            tx.commit()
2681                .await
2682                .map_err(|e| DurableError::storage("finalize", e))?;
2683            Ok(())
2684        }
2685        .instrument(span)
2686        .await
2687    }
2688
2689    async fn prune(&self, policy: &RetentionPolicy) -> Result<u64, DurableError> {
2690        let now = now_unix_millis();
2691        crate::retention::prune_in_batches(policy, now, |cutoffs, batch| {
2692            self.delete_prune_batch(cutoffs, batch)
2693        })
2694        .await
2695    }
2696
2697    /// Crash-orphan reclamation (INV-17, #6254). See [`Journal::sweep_orphans`] for the contract.
2698    async fn sweep_orphans(&self, policy: &RetentionPolicy) -> Result<u64, DurableError> {
2699        if policy.stale_running_after_secs == 0 {
2700            return Ok(0);
2701        }
2702        let Some(lock_dir) = self.lock_dir.clone() else {
2703            if !self
2704                .orphan_sweep_warned
2705                .swap(true, std::sync::atomic::Ordering::Relaxed)
2706            {
2707                tracing::warn!(
2708                    "durable: crash-orphan sweep requires an on-disk advisory-lock dir; orphan \
2709                     reclamation disabled for this backend (Postgres/:memory:/non-Unix)"
2710                );
2711            }
2712            return Ok(0);
2713        };
2714        let cutoff_ms = orphan_cutoff_ms(policy, now_unix_millis());
2715        crate::retention::sweep_orphans_in_batches(
2716            policy.prune_batch_size,
2717            cutoff_ms,
2718            |cutoff, batch, cursor| self.sweep_orphan_batch(&lock_dir, cutoff, batch, cursor),
2719        )
2720        .await
2721    }
2722}
2723
2724impl crate::sealed::Sealed for LocalBackend {}
2725
2726impl ExecutionBackend for LocalBackend {
2727    fn capabilities(&self) -> BackendCapabilities {
2728        BackendCapabilities {
2729            parallel_steps: true,
2730            // The local backend is in-process on SQLite; a Postgres build talks to a shared server.
2731            cross_process: cfg!(feature = "postgres"),
2732            max_payload: usize::try_from(self.max_payload_bytes).unwrap_or(usize::MAX),
2733        }
2734    }
2735
2736    async fn lookup_committed_result(
2737        &self,
2738        id: ExecutionId,
2739        idem_key: IdempotencyKey,
2740    ) -> Result<Option<JournalEntry>, DurableError> {
2741        LocalBackend::lookup_committed_result(self, id, idem_key).await
2742    }
2743}
2744
2745/// Column values for a single `durable_journal` row, ready to bind.
2746struct JournalRow {
2747    execution_id: String,
2748    step_id: i64,
2749    entry_kind: &'static str,
2750    idem_key: Option<Vec<u8>>,
2751    effect_class: Option<&'static str>,
2752    payload: Option<Vec<u8>>,
2753    payload_version: Option<i32>,
2754    hmac: Option<Vec<u8>>,
2755    created_at: i64,
2756}
2757
2758/// A `durable_journal` row read back from storage, decoded dialect-agnostically.
2759///
2760/// Columns are read as a positional tuple (the convention for crates that depend on `zeph-db` but
2761/// not `sqlx` directly, mirroring `zeph-scheduler`): integers decode as `i64`/`i32` and blobs as
2762/// `Vec<u8>`, which both backends satisfy through the same `sql!()`-rewritten query. The
2763/// field order matches the `SELECT` column list:
2764/// `(seq, step_id, entry_kind, idem_key, effect_class, payload, payload_version, hmac, created_at)`.
2765type JournalRowRead = (
2766    i64,
2767    i64,
2768    String,
2769    Option<Vec<u8>>,
2770    Option<String>,
2771    Option<Vec<u8>>,
2772    Option<i32>,
2773    Option<Vec<u8>>,
2774    i64,
2775);
2776
2777/// A `durable_journal` row read for [`LocalBackend::count_control_entries_under_previous_hmac`],
2778/// in `SELECT` column order: `(execution_id, step_id, idem_key, hmac)`.
2779type ControlHmacScanRow = (String, i64, Option<Vec<u8>>, Vec<u8>);
2780
2781/// A `durable_promises` row read back from storage, in `SELECT` column order:
2782/// `(execution_id, resolver_token_hash, resolved, payload)`.
2783type PromiseRowRead = (String, Vec<u8>, i64, Option<Vec<u8>>);
2784
2785/// A foldable `durable_journal` step-result row, in `SELECT` column order:
2786/// `(step_id, idem_key, payload_version, payload)`.
2787type FoldableRowRead = (i64, Option<Vec<u8>>, Option<i32>, Option<Vec<u8>>);
2788
2789/// Derive the per-execution lock directory sibling to a `path` passed to
2790/// [`LocalBackend::open`], `None` for `:memory:`.
2791///
2792/// Feature-gated to the `SQLite` backend only (INV-15): under `postgres`, `path` is a connection
2793/// URL that may embed credentials, and appending a suffix to mint a directory name would risk
2794/// creating a secret-bearing path component on disk.
2795#[cfg(feature = "sqlite")]
2796fn lock_dir_for_path(path: &str) -> Option<std::path::PathBuf> {
2797    (path != ":memory:").then(|| std::path::PathBuf::from(format!("{path}.locks")))
2798}
2799
2800#[cfg(not(feature = "sqlite"))]
2801fn lock_dir_for_path(_path: &str) -> Option<std::path::PathBuf> {
2802    None
2803}
2804
2805/// Regression coverage for the `postgres`-only branch of [`lock_dir_for_path`] (INV-15): a
2806/// connection URL — which may embed credentials — must never be used to mint an on-disk lock
2807/// directory name. The main `mod tests` block below is gated on `feature = "sqlite"` and so never
2808/// exercises this branch; run with `cargo nextest run -p zeph-durable --no-default-features
2809/// --features postgres`.
2810#[cfg(all(test, not(feature = "sqlite")))]
2811mod postgres_lock_dir_tests {
2812    use super::lock_dir_for_path;
2813
2814    #[test]
2815    fn postgres_url_never_derives_a_lock_dir() {
2816        assert_eq!(
2817            lock_dir_for_path("postgres://user:secret@host/db"),
2818            None,
2819            "a Postgres connection URL (which may embed credentials) must never be used to mint \
2820             an on-disk lock directory name"
2821        );
2822        assert_eq!(lock_dir_for_path(":memory:"), None);
2823    }
2824}
2825
2826/// Current Unix time in milliseconds, clamped into `i64` and never panicking.
2827pub(crate) fn now_unix_millis() -> i64 {
2828    SystemTime::now()
2829        .duration_since(UNIX_EPOCH)
2830        .map_or(0, |d| i64::try_from(d.as_millis()).unwrap_or(i64::MAX))
2831}
2832
2833/// Record `cancel_execution`'s `prior_status` span field for a write-path outcome.
2834///
2835/// `Canceled` was `running` immediately before this call (that is the only status the guarded
2836/// `UPDATE` matches); `AlreadyTerminal` already carries its own prior status. `NotFound` leaves
2837/// the field unset — there was no row to have had a status.
2838fn record_prior_status(outcome: CancelOutcome) {
2839    match outcome {
2840        CancelOutcome::Canceled => {
2841            tracing::Span::current().record("prior_status", "running");
2842        }
2843        CancelOutcome::AlreadyTerminal { status } => {
2844            tracing::Span::current().record("prior_status", status.as_str());
2845        }
2846        CancelOutcome::NotFound
2847        | CancelOutcome::LiveOwner { .. }
2848        | CancelOutcome::LivenessUnverifiable => {}
2849    }
2850}
2851
2852/// The absolute `updated_at` cutoff (Unix ms) at or before which a `status='running'` row becomes
2853/// a crash-orphan sweep candidate (INV-17, #6254).
2854fn orphan_cutoff_ms(policy: &RetentionPolicy, now_ms: i64) -> i64 {
2855    let threshold =
2856        i64::try_from(policy.stale_running_after_secs.saturating_mul(1000)).unwrap_or(i64::MAX);
2857    now_ms.saturating_sub(threshold)
2858}
2859
2860/// Decode a stored blob into a fixed 32-byte array, failing closed on the wrong length.
2861fn slice_to_array32(bytes: &[u8], field: &'static str) -> Result<[u8; 32], DurableError> {
2862    <[u8; 32]>::try_from(bytes).map_err(|_| DurableError::Decode { context: field })
2863}
2864
2865/// Parse a stored `execution_id` TEXT column back into an [`ExecutionId`], failing closed.
2866fn parse_execution_id(text: &str) -> Result<ExecutionId, DurableError> {
2867    uuid::Uuid::parse_str(text)
2868        .map(ExecutionId::from_uuid)
2869        .map_err(|_| DurableError::Decode {
2870            context: "execution_id is not a valid UUID",
2871        })
2872}
2873
2874/// Parse a stored `timer_id` TEXT column back into a [`TimerId`], failing closed.
2875fn parse_timer_id(text: &str) -> Result<TimerId, DurableError> {
2876    uuid::Uuid::parse_str(text)
2877        .map(TimerId::from_uuid)
2878        .map_err(|_| DurableError::Decode {
2879            context: "timer_id is not a valid UUID",
2880        })
2881}
2882
2883/// The AAD binding a promise's resolved payload to `(execution_id, promise_id)`.
2884///
2885/// A promise has no [`StepId`], so the promise id is folded into the AAD's idempotency-key slot:
2886/// a payload sealed for one promise cannot be opened as another's (fail-closed on relocation).
2887fn promise_payload_aad(execution_id: ExecutionId, promise_id: PromiseId) -> PayloadAad {
2888    let binding = IdempotencyKey::derive(
2889        execution_id,
2890        StepId::new(0),
2891        promise_id.as_uuid().as_bytes(),
2892    );
2893    PayloadAad::new(
2894        execution_id,
2895        StepId::new(0),
2896        EntryKindTag::PromiseResolved,
2897        Some(binding),
2898    )
2899}
2900
2901/// Map a database `entry_kind` string to a `'static` tag for [`DurableError::UnsupportedEntryKind`].
2902fn static_entry_tag(tag: &str) -> &'static str {
2903    match tag {
2904        "promise_created" => "promise_created",
2905        "promise_resolved" => "promise_resolved",
2906        "timer_armed" => "timer_armed",
2907        "timer_fired" => "timer_fired",
2908        "checkpoint" => "checkpoint",
2909        _ => "unknown",
2910    }
2911}
2912
2913// Backend tests open a real pool, so they run under the SQLite build (mirroring `zeph-scheduler`,
2914// whose `:memory:` pool is SQLite-specific). The dialect-agnostic `sql!()` SQL and `i64`/`Vec<u8>`
2915// column types are verified to compile under the Postgres feature; live Postgres parity is exercised
2916// by the `#[ignore]`d integration test below.
2917#[cfg(all(test, feature = "sqlite"))]
2918mod tests {
2919    use std::assert_matches;
2920
2921    use super::*;
2922    use crate::cipher::CipherError;
2923    use crate::effect::EffectClass;
2924
2925    /// An AAD-authenticated test cipher: a BLAKE3 tag over the AAD prefixes an XOR-masked payload,
2926    /// so opening with a relocated/forged AAD fails authentication exactly like the real cipher.
2927    struct XorCipher;
2928    const XOR_MASK: u8 = 0x5A;
2929
2930    impl PayloadCipher for XorCipher {
2931        fn seal(&self, plaintext: &[u8], aad: &PayloadAad) -> Result<Vec<u8>, CipherError> {
2932            let tag = blake3::hash(&aad.canonical_bytes());
2933            let mut out = tag.as_bytes()[..8].to_vec();
2934            out.extend(plaintext.iter().map(|b| b ^ XOR_MASK));
2935            Ok(out)
2936        }
2937
2938        fn open(&self, sealed: &[u8], aad: &PayloadAad) -> Result<Vec<u8>, CipherError> {
2939            if sealed.len() < 8 {
2940                return Err(CipherError::Malformed {
2941                    context: "sealed blob shorter than the aad tag",
2942                });
2943            }
2944            let expected = blake3::hash(&aad.canonical_bytes());
2945            if sealed[..8] != expected.as_bytes()[..8] {
2946                return Err(CipherError::Authentication);
2947            }
2948            Ok(sealed[8..].iter().map(|b| b ^ XOR_MASK).collect())
2949        }
2950    }
2951
2952    /// A test cipher double that embeds a `key_id` leading byte (mirroring the production
2953    /// `key_id(1) || nonce || ciphertext || tag` contract documented on [`PayloadCipher`]) and,
2954    /// like the real `XChaCha20Poly1305Cipher::with_previous`, can still decrypt a payload sealed
2955    /// under a registered `previous_id` while always sealing new writes under `current_id` — the
2956    /// minimal shape needed to exercise `checkpoint_fold`'s reseal-under-current behavior across a
2957    /// simulated rotation window, without depending on the real AEAD cipher (out of scope for
2958    /// `zeph-durable`, INV-1).
2959    struct RotatingKeyedCipher {
2960        current_id: u8,
2961        previous_id: Option<u8>,
2962    }
2963
2964    impl PayloadCipher for RotatingKeyedCipher {
2965        fn seal(&self, plaintext: &[u8], aad: &PayloadAad) -> Result<Vec<u8>, CipherError> {
2966            let tag = blake3::hash(&aad.canonical_bytes());
2967            let mut out = vec![self.current_id];
2968            out.extend_from_slice(&tag.as_bytes()[..8]);
2969            out.extend(plaintext.iter().map(|b| b ^ XOR_MASK));
2970            Ok(out)
2971        }
2972
2973        fn open(&self, sealed: &[u8], aad: &PayloadAad) -> Result<Vec<u8>, CipherError> {
2974            if sealed.len() < 9 {
2975                return Err(CipherError::Malformed {
2976                    context: "sealed blob shorter than the key-id + aad tag prefix",
2977                });
2978            }
2979            let id = sealed[0];
2980            if id != self.current_id && Some(id) != self.previous_id {
2981                return Err(CipherError::UnknownKeyId { key_id: id });
2982            }
2983            let expected = blake3::hash(&aad.canonical_bytes());
2984            if sealed[1..9] != expected.as_bytes()[..8] {
2985                return Err(CipherError::Authentication);
2986            }
2987            Ok(sealed[9..].iter().map(|b| b ^ XOR_MASK).collect())
2988        }
2989    }
2990
2991    async fn mem_backend(max_payload_bytes: u64) -> LocalBackend {
2992        let backend = LocalBackend::open(":memory:", max_payload_bytes)
2993            .await
2994            .expect("open in-memory backend");
2995        backend.init().await.expect("apply migrations");
2996        backend
2997    }
2998
2999    fn step_result(exec: ExecutionId, step: u32, payload: &[u8]) -> JournalEntry {
3000        let step_id = StepId::new(step);
3001        JournalEntry {
3002            seq: None,
3003            execution_id: exec,
3004            kind: ExecutionKind::AgentTurn,
3005            step_id,
3006            entry: EntryKind::StepResult {
3007                idempotency_key: IdempotencyKey::derive(exec, step_id, b"tool:read"),
3008                payload: Bytes::copy_from_slice(payload),
3009                effect: EffectClass::Idempotent,
3010                payload_version: 1,
3011            },
3012            created_at_ms: 100,
3013        }
3014    }
3015
3016    fn effect_intent(exec: ExecutionId, step: u32) -> JournalEntry {
3017        let step_id = StepId::new(step);
3018        JournalEntry {
3019            seq: None,
3020            execution_id: exec,
3021            kind: ExecutionKind::AgentTurn,
3022            step_id,
3023            entry: EntryKind::EffectIntent {
3024                idempotency_key: IdempotencyKey::derive(exec, step_id, b"transfer"),
3025                effect: EffectClass::ExactlyOnceGuarded,
3026                hmac: None,
3027            },
3028            created_at_ms: 100,
3029        }
3030    }
3031
3032    /// Regression for #6447: `count_sealed_under_key_id` scans `durable_journal.payload` by its
3033    /// leading byte, ignores control entries (`payload IS NULL`), and never matches an unrelated
3034    /// key-id. No cipher is attached, so `seal_payload` stores the plaintext verbatim (local.rs
3035    /// `seal_payload`'s `None => Ok(plaintext.to_vec())` branch) — the first byte of the crafted
3036    /// payload lands on disk unchanged, letting the test control it directly without depending on
3037    /// the real AEAD cipher (out of scope for `zeph-durable`, INV-1).
3038    #[tokio::test]
3039    async fn count_sealed_under_key_id_counts_matching_journal_rows_and_excludes_control_entries() {
3040        let backend = mem_backend(1_048_576).await;
3041        let exec = ExecutionId::new();
3042        backend
3043            .open_execution(exec, ExecutionKind::AgentTurn)
3044            .await
3045            .unwrap();
3046
3047        backend
3048            .append(step_result(exec, 0, &[5, 0, 0]))
3049            .await
3050            .unwrap();
3051        backend
3052            .append(step_result(exec, 1, &[6, 0, 0]))
3053            .await
3054            .unwrap();
3055        // A control entry carries no payload and must never be counted, regardless of key_id.
3056        backend.append(effect_intent(exec, 2)).await.unwrap();
3057
3058        assert_eq!(backend.count_sealed_under_key_id(5).await.unwrap(), 1);
3059        assert_eq!(backend.count_sealed_under_key_id(6).await.unwrap(), 1);
3060        assert_eq!(backend.count_sealed_under_key_id(7).await.unwrap(), 0);
3061    }
3062
3063    /// Regression for #6447: the scan also covers `durable_promises.payload`, not just the
3064    /// journal — a promise resolved under the previous key must count too, or `--drop-previous`
3065    /// could silently orphan it.
3066    #[tokio::test]
3067    async fn count_sealed_under_key_id_counts_matching_promise_rows() {
3068        let backend = mem_backend(1_048_576).await;
3069        let exec = ExecutionId::new();
3070        backend
3071            .open_execution(exec, ExecutionKind::AgentTurn)
3072            .await
3073            .unwrap();
3074        let promise_id = PromiseId::new();
3075        backend
3076            .insert_promise(promise_id, exec, [0u8; 32], 100)
3077            .await
3078            .unwrap();
3079        // Unresolved promise row: payload is still NULL, must not be counted.
3080        assert_eq!(backend.count_sealed_under_key_id(9).await.unwrap(), 0);
3081
3082        backend
3083            .resolve_promise(promise_id, exec, &[9, 1, 2, 3], 200)
3084            .await
3085            .unwrap();
3086
3087        assert_eq!(backend.count_sealed_under_key_id(9).await.unwrap(), 1);
3088        assert_eq!(backend.count_sealed_under_key_id(10).await.unwrap(), 0);
3089    }
3090
3091    #[tokio::test]
3092    async fn open_execution_is_fresh_then_resume() {
3093        let backend = mem_backend(1_048_576).await;
3094        let exec = ExecutionId::new();
3095        assert!(
3096            !backend
3097                .open_execution(exec, ExecutionKind::AgentTurn)
3098                .await
3099                .unwrap()
3100        );
3101        assert!(
3102            backend
3103                .open_execution(exec, ExecutionKind::AgentTurn)
3104                .await
3105                .unwrap()
3106        );
3107    }
3108
3109    #[tokio::test]
3110    async fn open_execution_exclusive_is_fresh_then_resume() {
3111        // A file-backed (not `:memory:`) backend is required: only `LocalBackend::open` with a
3112        // real on-disk path derives a `lock_dir` (#6122).
3113        let dir = tempfile::tempdir().unwrap();
3114        let db_path = dir.path().join("durable.db");
3115        let backend = LocalBackend::open(&db_path.to_string_lossy(), 1_048_576)
3116            .await
3117            .unwrap();
3118        backend.init().await.unwrap();
3119
3120        let exec = ExecutionId::new();
3121        let (is_resume, lock) = backend
3122            .open_execution_exclusive(exec, ExecutionKind::AgentTurn)
3123            .await
3124            .unwrap();
3125        assert!(!is_resume);
3126        assert!(lock.is_some(), "a file-backed backend must derive a lock");
3127        drop(lock);
3128
3129        let (is_resume, _lock) = backend
3130            .open_execution_exclusive(exec, ExecutionKind::AgentTurn)
3131            .await
3132            .unwrap();
3133        assert!(is_resume);
3134    }
3135
3136    /// Regression test for #6122: two `LocalBackend` handles onto the same on-disk journal (as
3137    /// two independent CLI processes sharing `memory.sqlite_path` would each construct) must not
3138    /// both be able to hold `open_execution_exclusive` for the same colliding `ExecutionId`
3139    /// concurrently.
3140    #[tokio::test]
3141    async fn open_execution_exclusive_rejects_concurrent_second_holder() {
3142        let dir = tempfile::tempdir().unwrap();
3143        let db_path = dir.path().join("durable.db");
3144        let url = db_path.to_string_lossy().into_owned();
3145
3146        let backend_a = LocalBackend::open(&url, 1_048_576).await.unwrap();
3147        backend_a.init().await.unwrap();
3148        let backend_b = LocalBackend::open(&url, 1_048_576).await.unwrap();
3149
3150        let exec = ExecutionId::new();
3151        let (_, _lock_a) = backend_a
3152            .open_execution_exclusive(exec, ExecutionKind::AgentTurn)
3153            .await
3154            .unwrap();
3155
3156        let err = backend_b
3157            .open_execution_exclusive(exec, ExecutionKind::AgentTurn)
3158            .await
3159            .expect_err("a second concurrent holder must be rejected");
3160        assert!(
3161            matches!(err, DurableError::ExecutionLocked { execution_id, .. } if execution_id == exec),
3162            "expected ExecutionLocked, got {err:?}"
3163        );
3164    }
3165
3166    #[tokio::test]
3167    async fn open_execution_exclusive_on_memory_backend_returns_no_lock() {
3168        // `:memory:` has no on-disk directory to lock, so it degrades to unenforced exclusivity —
3169        // consistent with `SessionEventLog::open_exclusive`'s non-Unix degrade.
3170        let backend = mem_backend(1_048_576).await;
3171        let exec = ExecutionId::new();
3172        let (is_resume, lock) = backend
3173            .open_execution_exclusive(exec, ExecutionKind::AgentTurn)
3174            .await
3175            .unwrap();
3176        assert!(!is_resume);
3177        assert!(lock.is_none());
3178    }
3179
3180    #[tokio::test]
3181    async fn list_executions_summarizes_and_filters() {
3182        let backend = mem_backend(1_048_576).await;
3183        let turn = ExecutionId::new();
3184        let dag = ExecutionId::new();
3185        backend
3186            .open_execution(turn, ExecutionKind::AgentTurn)
3187            .await
3188            .unwrap();
3189        backend
3190            .open_execution(dag, ExecutionKind::DagRun)
3191            .await
3192            .unwrap();
3193        backend.append(step_result(turn, 0, b"a")).await.unwrap();
3194        backend.append(step_result(turn, 1, b"b")).await.unwrap();
3195        backend.append(step_result(dag, 0, b"c")).await.unwrap();
3196        backend
3197            .finalize(turn, ExecutionStatus::Completed)
3198            .await
3199            .unwrap();
3200
3201        // Unfiltered: both executions with their per-execution step counts.
3202        let all = backend.list_executions(None, None, 10).await.unwrap();
3203        assert_eq!(all.len(), 2);
3204
3205        let turn_row = all
3206            .iter()
3207            .find(|e| e.execution_id == turn)
3208            .expect("turn present");
3209        assert_eq!(turn_row.kind, "agent_turn");
3210        assert_eq!(turn_row.status, ExecutionStatus::Completed);
3211        assert_eq!(turn_row.step_count, 2);
3212        assert!(turn_row.finalized_at_ms.is_some());
3213
3214        let dag_row = all
3215            .iter()
3216            .find(|e| e.execution_id == dag)
3217            .expect("dag present");
3218        assert_eq!(dag_row.status, ExecutionStatus::Running);
3219        assert_eq!(dag_row.step_count, 1);
3220        assert!(dag_row.finalized_at_ms.is_none());
3221
3222        // Status filter narrows to the still-running execution.
3223        let running = backend
3224            .list_executions(Some("running"), None, 10)
3225            .await
3226            .unwrap();
3227        assert_eq!(running.len(), 1);
3228        assert_eq!(running[0].execution_id, dag);
3229
3230        // Kind filter narrows to the DAG execution.
3231        let dags = backend
3232            .list_executions(None, Some("dag_run"), 10)
3233            .await
3234            .unwrap();
3235        assert_eq!(dags.len(), 1);
3236        assert_eq!(dags[0].execution_id, dag);
3237
3238        // Limit caps the result set.
3239        let one = backend.list_executions(None, None, 1).await.unwrap();
3240        assert_eq!(one.len(), 1);
3241    }
3242
3243    #[tokio::test]
3244    async fn append_and_read_round_trips_step_result() {
3245        let backend = mem_backend(1_048_576).await;
3246        let exec = ExecutionId::new();
3247        backend
3248            .open_execution(exec, ExecutionKind::AgentTurn)
3249            .await
3250            .unwrap();
3251
3252        let seq = backend
3253            .append(step_result(exec, 0, b"hello"))
3254            .await
3255            .unwrap();
3256        assert_eq!(seq.value(), 1, "first append takes seq 1");
3257
3258        let entries = backend.read_execution(exec).await.unwrap();
3259        assert_eq!(entries.len(), 1);
3260        match &entries[0].entry {
3261            EntryKind::StepResult {
3262                payload, effect, ..
3263            } => {
3264                assert_eq!(payload.as_ref(), b"hello");
3265                assert_eq!(*effect, EffectClass::Idempotent);
3266            }
3267            other => panic!("unexpected entry kind: {other:?}"),
3268        }
3269        assert_eq!(entries[0].seq, Some(seq));
3270    }
3271
3272    #[tokio::test]
3273    async fn cipher_seals_payload_at_rest_but_round_trips() {
3274        let backend = mem_backend(1_048_576)
3275            .await
3276            .with_cipher(Arc::new(XorCipher));
3277        let exec = ExecutionId::new();
3278        backend
3279            .open_execution(exec, ExecutionKind::AgentTurn)
3280            .await
3281            .unwrap();
3282        backend
3283            .append(step_result(exec, 0, b"secret-payload"))
3284            .await
3285            .unwrap();
3286
3287        // The stored column is sealed, never the plaintext.
3288        let (stored,): (Option<Vec<u8>>,) = zeph_db::query_as(sql!(
3289            "SELECT payload FROM durable_journal WHERE execution_id = ?"
3290        ))
3291        .bind(exec.as_uuid().to_string())
3292        .fetch_one(backend.pool())
3293        .await
3294        .unwrap();
3295        let stored = stored.expect("payload present");
3296        assert_ne!(
3297            stored.as_slice(),
3298            b"secret-payload",
3299            "payload must be sealed at rest"
3300        );
3301
3302        // Reading opens it back to the original plaintext.
3303        let entries = backend.read_execution(exec).await.unwrap();
3304        match &entries[0].entry {
3305            EntryKind::StepResult { payload, .. } => {
3306                assert_eq!(payload.as_ref(), b"secret-payload");
3307            }
3308            other => panic!("unexpected entry kind: {other:?}"),
3309        }
3310    }
3311
3312    #[tokio::test]
3313    async fn control_entry_hmac_is_stamped_only_when_keyed() {
3314        let exec = ExecutionId::new();
3315
3316        let unkeyed = mem_backend(1_048_576).await;
3317        unkeyed
3318            .open_execution(exec, ExecutionKind::AgentTurn)
3319            .await
3320            .unwrap();
3321        unkeyed.append(effect_intent(exec, 0)).await.unwrap();
3322        match &unkeyed.read_execution(exec).await.unwrap()[0].entry {
3323            EntryKind::EffectIntent { hmac, .. } => assert!(hmac.is_none()),
3324            other => panic!("unexpected entry kind: {other:?}"),
3325        }
3326
3327        let keyed = mem_backend(1_048_576).await.with_hmac_key([7u8; 32]);
3328        let exec2 = ExecutionId::new();
3329        keyed
3330            .open_execution(exec2, ExecutionKind::AgentTurn)
3331            .await
3332            .unwrap();
3333        keyed.append(effect_intent(exec2, 0)).await.unwrap();
3334        match &keyed.read_execution(exec2).await.unwrap()[0].entry {
3335            EntryKind::EffectIntent { hmac, .. } => {
3336                assert!(
3337                    hmac.is_some(),
3338                    "keyed backend stamps a row HMAC over control entries"
3339                );
3340            }
3341            other => panic!("unexpected entry kind: {other:?}"),
3342        }
3343    }
3344
3345    /// Regression for #6043/#6044: a control entry written under one HMAC key must fail closed
3346    /// with [`DurableError::ControlIntegrity`] when read back under a *different* key — the
3347    /// forged/relocated-row rejection the row HMAC exists to provide. Both backends share the
3348    /// same underlying pool (a second `LocalBackend` handle over the same connection), so this
3349    /// exercises the read path's recompute-and-compare, not just a difference in whether a key is
3350    /// configured at all.
3351    #[tokio::test]
3352    async fn read_execution_rejects_control_hmac_under_wrong_key() {
3353        let writer = mem_backend(1_048_576).await.with_hmac_key([1u8; 32]);
3354        let exec = ExecutionId::new();
3355        writer
3356            .open_execution(exec, ExecutionKind::AgentTurn)
3357            .await
3358            .unwrap();
3359        writer.append(effect_intent(exec, 0)).await.unwrap();
3360
3361        let wrong_key_reader =
3362            LocalBackend::new(writer.pool().clone(), 1_048_576).with_hmac_key([2u8; 32]);
3363        assert_matches!(
3364            wrong_key_reader.read_execution(exec).await,
3365            Err(DurableError::ControlIntegrity)
3366        );
3367
3368        // Reading under the correct key still succeeds.
3369        let right_key_reader =
3370            LocalBackend::new(writer.pool().clone(), 1_048_576).with_hmac_key([1u8; 32]);
3371        assert!(right_key_reader.read_execution(exec).await.is_ok());
3372    }
3373
3374    /// Regression for #6043/#6044: a control entry written by an *unkeyed* backend (`hmac =
3375    /// NULL`) must fail closed when later read by a keyed backend — a keyed backend enforces that
3376    /// every control row it reads carries a matching HMAC, so a missing HMAC is treated the same
3377    /// as a mismatched one rather than silently passing through unverified.
3378    #[tokio::test]
3379    async fn read_execution_rejects_missing_hmac_on_keyed_backend() {
3380        let writer = mem_backend(1_048_576).await;
3381        let exec = ExecutionId::new();
3382        writer
3383            .open_execution(exec, ExecutionKind::AgentTurn)
3384            .await
3385            .unwrap();
3386        writer.append(effect_intent(exec, 0)).await.unwrap();
3387
3388        let keyed_reader =
3389            LocalBackend::new(writer.pool().clone(), 1_048_576).with_hmac_key([3u8; 32]);
3390        assert_matches!(
3391            keyed_reader.read_execution(exec).await,
3392            Err(DurableError::ControlIntegrity)
3393        );
3394    }
3395
3396    /// Regression for #6043/#6044 (review S1): a control entry written by a *keyed* backend must
3397    /// fail closed when later read by an *unkeyed* backend, rather than silently trusting the
3398    /// stamped HMAC as an ordinary (unverified) plaintext field. Before this fix,
3399    /// `verify_control_hmac` returned `Ok(())` unconditionally whenever the reader had no HMAC
3400    /// key, regardless of whether the stored row carried one — so config drift between a keyed
3401    /// writer and an unkeyed reader over the same physical file (e.g. `shared_db` toggled, or a
3402    /// reader whose config disagrees with the writer's) let a stamped row through unverified,
3403    /// which is exactly the forgery-acceptance gap #6043 says the row HMAC closes.
3404    #[tokio::test]
3405    async fn read_execution_rejects_stamped_hmac_on_unkeyed_backend() {
3406        let writer = mem_backend(1_048_576).await.with_hmac_key([4u8; 32]);
3407        let exec = ExecutionId::new();
3408        writer
3409            .open_execution(exec, ExecutionKind::AgentTurn)
3410            .await
3411            .unwrap();
3412        writer.append(effect_intent(exec, 0)).await.unwrap();
3413
3414        let unkeyed_reader = LocalBackend::new(writer.pool().clone(), 1_048_576);
3415        assert_matches!(
3416            unkeyed_reader.read_execution(exec).await,
3417            Err(DurableError::ControlIntegrity)
3418        );
3419    }
3420
3421    /// #6451: a control entry written under the pre-rotation key must still verify once a reader
3422    /// registers that key as `previous_hmac_key`, even though its own `hmac_key` has moved on to
3423    /// the new (post-rotation) key — the try-both rotation window, symmetric to the AEAD cipher's
3424    /// `with_previous`. This is also the payload-less "crash-orphan" shape the drop-scan exists
3425    /// for: `effect_intent` entries never carry a payload, so this row has a previous-key HMAC
3426    /// with nothing for the AEAD blob-scan to see.
3427    #[tokio::test]
3428    async fn verify_control_hmac_accepts_row_under_previous_key_during_window() {
3429        let writer = mem_backend(1_048_576).await.with_hmac_key([1u8; 32]);
3430        let exec = ExecutionId::new();
3431        writer
3432            .open_execution(exec, ExecutionKind::AgentTurn)
3433            .await
3434            .unwrap();
3435        writer.append(effect_intent(exec, 0)).await.unwrap();
3436
3437        // Post-rotation reader: current key is the new key [2u8; 32], previous is the pre-rotation
3438        // key [1u8; 32] that actually stamped the row.
3439        let post_rotation_reader = LocalBackend::new(writer.pool().clone(), 1_048_576)
3440            .with_hmac_key([2u8; 32])
3441            .with_previous_hmac_key([1u8; 32]);
3442        assert!(
3443            post_rotation_reader.read_execution(exec).await.is_ok(),
3444            "a row stamped under the previous key must verify during the rotation window"
3445        );
3446
3447        // A fresh row written by the post-rotation writer stamps under the current key only, and
3448        // must verify without needing the previous slot.
3449        let post_rotation_writer = LocalBackend::new(writer.pool().clone(), 1_048_576)
3450            .with_hmac_key([2u8; 32])
3451            .with_previous_hmac_key([1u8; 32]);
3452        post_rotation_writer
3453            .append(effect_intent(exec, 1))
3454            .await
3455            .unwrap();
3456        assert!(post_rotation_writer.read_execution(exec).await.is_ok());
3457    }
3458
3459    /// #6451: a row that matches neither the current nor the registered previous key must still
3460    /// fail closed — the rotation window widens acceptance to exactly two legitimate keys, never
3461    /// to "any key".
3462    #[tokio::test]
3463    async fn verify_control_hmac_rejects_row_under_neither_current_nor_previous_key() {
3464        let writer = mem_backend(1_048_576).await.with_hmac_key([9u8; 32]);
3465        let exec = ExecutionId::new();
3466        writer
3467            .open_execution(exec, ExecutionKind::AgentTurn)
3468            .await
3469            .unwrap();
3470        writer.append(effect_intent(exec, 0)).await.unwrap();
3471
3472        let unrelated_reader = LocalBackend::new(writer.pool().clone(), 1_048_576)
3473            .with_hmac_key([2u8; 32])
3474            .with_previous_hmac_key([3u8; 32]);
3475        assert_matches!(
3476            unrelated_reader.read_execution(exec).await,
3477            Err(DurableError::ControlIntegrity)
3478        );
3479    }
3480
3481    /// #6451: `count_control_entries_under_previous_hmac` is the drop-scan gate for
3482    /// `--drop-previous` — it must count a row that verifies only under the previous key, and
3483    /// must not count a row that still verifies under the current key (no false refusal once the
3484    /// row has actually been re-keyed).
3485    #[tokio::test]
3486    async fn count_control_entries_under_previous_hmac_counts_previous_only_rows() {
3487        let writer = mem_backend(1_048_576).await.with_hmac_key([1u8; 32]);
3488        let exec = ExecutionId::new();
3489        writer
3490            .open_execution(exec, ExecutionKind::AgentTurn)
3491            .await
3492            .unwrap();
3493        // Pre-rotation row: stamped under [1u8; 32], the soon-to-be-previous key.
3494        writer.append(effect_intent(exec, 0)).await.unwrap();
3495
3496        let scanner_mid_window = LocalBackend::new(writer.pool().clone(), 1_048_576)
3497            .with_hmac_key([2u8; 32])
3498            .with_previous_hmac_key([1u8; 32]);
3499        assert_eq!(
3500            scanner_mid_window
3501                .count_control_entries_under_previous_hmac()
3502                .await
3503                .unwrap(),
3504            1,
3505            "a row stamped under the previous key only must be counted"
3506        );
3507
3508        // A post-rotation row, stamped under the new current key, must not be counted.
3509        let post_rotation_writer = LocalBackend::new(writer.pool().clone(), 1_048_576)
3510            .with_hmac_key([2u8; 32])
3511            .with_previous_hmac_key([1u8; 32]);
3512        post_rotation_writer
3513            .append(effect_intent(exec, 1))
3514            .await
3515            .unwrap();
3516        assert_eq!(
3517            post_rotation_writer
3518                .count_control_entries_under_previous_hmac()
3519                .await
3520                .unwrap(),
3521            1,
3522            "the post-rotation row (verifies under current) must not add to the count"
3523        );
3524    }
3525
3526    /// #6451: once every previous-key row has been superseded (or there were none), the scan
3527    /// reports zero without requiring any rows to exist at all — the clean `--drop-previous`
3528    /// no-op/success path.
3529    #[tokio::test]
3530    async fn count_control_entries_under_previous_hmac_is_zero_on_empty_journal() {
3531        let backend = mem_backend(1_048_576).await;
3532        assert_eq!(
3533            backend
3534                .count_control_entries_under_previous_hmac()
3535                .await
3536                .unwrap(),
3537            0
3538        );
3539    }
3540
3541    /// #6451 critic finding 1: the scan cannot be trusted without both keys attached — a caller
3542    /// that opens the backend unkeyed (as the pre-fix `--drop-previous` scan site did) must get a
3543    /// hard error, not a silently-wrong count that could let `--drop-previous` refuse forever (or
3544    /// worse, proceed unsafely).
3545    #[tokio::test]
3546    async fn count_control_entries_under_previous_hmac_errors_when_keys_missing() {
3547        let writer = mem_backend(1_048_576).await.with_hmac_key([1u8; 32]);
3548        let exec = ExecutionId::new();
3549        writer
3550            .open_execution(exec, ExecutionKind::AgentTurn)
3551            .await
3552            .unwrap();
3553        writer.append(effect_intent(exec, 0)).await.unwrap();
3554
3555        let unkeyed_scanner = LocalBackend::new(writer.pool().clone(), 1_048_576);
3556        assert_matches!(
3557            unkeyed_scanner
3558                .count_control_entries_under_previous_hmac()
3559                .await,
3560            Err(DurableError::ControlIntegrity)
3561        );
3562
3563        let current_only_scanner =
3564            LocalBackend::new(writer.pool().clone(), 1_048_576).with_hmac_key([1u8; 32]);
3565        assert_matches!(
3566            current_only_scanner
3567                .count_control_entries_under_previous_hmac()
3568                .await,
3569            Err(DurableError::ControlIntegrity)
3570        );
3571    }
3572
3573    #[tokio::test]
3574    async fn promise_and_timer_entries_fail_closed() {
3575        let backend = mem_backend(1_048_576).await;
3576        let exec = ExecutionId::new();
3577        backend
3578            .open_execution(exec, ExecutionKind::AgentTurn)
3579            .await
3580            .unwrap();
3581        let timer = JournalEntry {
3582            seq: None,
3583            execution_id: exec,
3584            kind: ExecutionKind::AgentTurn,
3585            step_id: StepId::new(0),
3586            entry: EntryKind::TimerArmed {
3587                timer_id: crate::TimerId::new(),
3588                due_at_ms: 1_000,
3589                hmac: None,
3590            },
3591            created_at_ms: 0,
3592        };
3593        assert_matches!(
3594            backend.append(timer).await,
3595            Err(DurableError::UnsupportedEntryKind {
3596                kind: "timer_armed"
3597            })
3598        );
3599    }
3600
3601    #[tokio::test]
3602    async fn payload_over_limit_is_rejected_fail_closed() {
3603        let backend = mem_backend(8).await;
3604        let exec = ExecutionId::new();
3605        backend
3606            .open_execution(exec, ExecutionKind::AgentTurn)
3607            .await
3608            .unwrap();
3609        let big = vec![0u8; 64];
3610        assert_matches!(
3611            backend.append(step_result(exec, 0, &big)).await,
3612            Err(DurableError::PayloadTooLarge { .. })
3613        );
3614    }
3615
3616    #[tokio::test]
3617    async fn finalize_marks_terminal_status_and_time() {
3618        let backend = mem_backend(1_048_576).await;
3619        let exec = ExecutionId::new();
3620        backend
3621            .open_execution(exec, ExecutionKind::AgentTurn)
3622            .await
3623            .unwrap();
3624        backend
3625            .finalize(exec, ExecutionStatus::Completed)
3626            .await
3627            .unwrap();
3628
3629        let (status, finalized): (String, Option<i64>) = zeph_db::query_as(sql!(
3630            "SELECT status, finalized_at FROM durable_executions WHERE execution_id = ?"
3631        ))
3632        .bind(exec.as_uuid().to_string())
3633        .fetch_one(backend.pool())
3634        .await
3635        .unwrap();
3636        assert_eq!(status, "completed");
3637        assert!(finalized.is_some(), "a terminal status stamps finalized_at");
3638    }
3639
3640    #[tokio::test]
3641    async fn finalize_is_a_noop_once_already_terminal() {
3642        // #6251: finalize must be safe to call more than once (e.g. a caller's own `Completed`
3643        // racing the divergence guard's `Aborted`) — whichever status lands first wins.
3644        let backend = mem_backend(1_048_576).await;
3645        let exec = ExecutionId::new();
3646        backend
3647            .open_execution(exec, ExecutionKind::AgentTurn)
3648            .await
3649            .unwrap();
3650        backend
3651            .finalize(exec, ExecutionStatus::Completed)
3652            .await
3653            .unwrap();
3654
3655        // A later call with a different terminal status must not overwrite the first.
3656        backend
3657            .finalize(exec, ExecutionStatus::Failed)
3658            .await
3659            .unwrap();
3660
3661        let (status,): (String,) = zeph_db::query_as(sql!(
3662            "SELECT status FROM durable_executions WHERE execution_id = ?"
3663        ))
3664        .bind(exec.as_uuid().to_string())
3665        .fetch_one(backend.pool())
3666        .await
3667        .unwrap();
3668        assert_eq!(
3669            status, "completed",
3670            "the first terminal status must stick; a later finalize call is a no-op"
3671        );
3672    }
3673
3674    #[tokio::test]
3675    async fn finalize_after_abort_is_a_noop() {
3676        // #6251: the reverse direction of the divergence race — the internal `Aborted` transition
3677        // (replay-divergence guard) commits first, so a consumer's later own `Completed`/`Failed`
3678        // call must be a no-op rather than resurrecting the row out of its aborted state.
3679        let backend = mem_backend(1_048_576).await;
3680        let exec = ExecutionId::new();
3681        backend
3682            .open_execution(exec, ExecutionKind::AgentTurn)
3683            .await
3684            .unwrap();
3685        backend
3686            .finalize(exec, ExecutionStatus::Aborted)
3687            .await
3688            .unwrap();
3689
3690        backend
3691            .finalize(exec, ExecutionStatus::Completed)
3692            .await
3693            .unwrap();
3694
3695        let (status,): (String,) = zeph_db::query_as(sql!(
3696            "SELECT status FROM durable_executions WHERE execution_id = ?"
3697        ))
3698        .bind(exec.as_uuid().to_string())
3699        .fetch_one(backend.pool())
3700        .await
3701        .unwrap();
3702        assert_eq!(
3703            status, "aborted",
3704            "an aborted execution must not be overwritten by a later Completed/Failed call"
3705        );
3706    }
3707
3708    #[tokio::test]
3709    async fn reopening_a_finalized_execution_resets_it_to_running() {
3710        // #6251: a finalized execution that is legitimately reopened (e.g. a resumed conversation)
3711        // must not keep a stale `finalized_at` — otherwise the retention sweep could prune a row
3712        // that is still receiving new journal writes.
3713        let backend = mem_backend(1_048_576).await;
3714        let exec = ExecutionId::new();
3715        backend
3716            .open_execution(exec, ExecutionKind::AgentTurn)
3717            .await
3718            .unwrap();
3719        backend
3720            .finalize(exec, ExecutionStatus::Completed)
3721            .await
3722            .unwrap();
3723
3724        let is_resume = backend
3725            .open_execution(exec, ExecutionKind::AgentTurn)
3726            .await
3727            .unwrap();
3728        assert!(is_resume, "the row already existed, so this is a resume");
3729
3730        let (status, finalized): (String, Option<i64>) = zeph_db::query_as(sql!(
3731            "SELECT status, finalized_at FROM durable_executions WHERE execution_id = ?"
3732        ))
3733        .bind(exec.as_uuid().to_string())
3734        .fetch_one(backend.pool())
3735        .await
3736        .unwrap();
3737        assert_eq!(
3738            status, "running",
3739            "reopening a completed execution must un-finalize it"
3740        );
3741        assert!(
3742            finalized.is_none(),
3743            "reopening must clear the stale finalized_at"
3744        );
3745    }
3746
3747    #[tokio::test]
3748    async fn reopening_a_failed_execution_resets_it_to_running() {
3749        // #6251: same guarantee as `reopening_a_finalized_execution_resets_it_to_running`, but for
3750        // the `Failed` terminal status — e.g. a scheduler retry of the same (job_name, slot_ms)
3751        // after the previous fire failed must not orphan a `Failed` row.
3752        let backend = mem_backend(1_048_576).await;
3753        let exec = ExecutionId::new();
3754        backend
3755            .open_execution(exec, ExecutionKind::AgentTurn)
3756            .await
3757            .unwrap();
3758        backend
3759            .finalize(exec, ExecutionStatus::Failed)
3760            .await
3761            .unwrap();
3762
3763        let is_resume = backend
3764            .open_execution(exec, ExecutionKind::AgentTurn)
3765            .await
3766            .unwrap();
3767        assert!(is_resume, "the row already existed, so this is a resume");
3768
3769        let (status, finalized): (String, Option<i64>) = zeph_db::query_as(sql!(
3770            "SELECT status, finalized_at FROM durable_executions WHERE execution_id = ?"
3771        ))
3772        .bind(exec.as_uuid().to_string())
3773        .fetch_one(backend.pool())
3774        .await
3775        .unwrap();
3776        assert_eq!(
3777            status, "running",
3778            "reopening a failed execution must un-finalize it"
3779        );
3780        assert!(
3781            finalized.is_none(),
3782            "reopening must clear the stale finalized_at"
3783        );
3784    }
3785
3786    #[tokio::test]
3787    async fn reopening_an_aborted_execution_un_finalizes_it() {
3788        // INV-16 (#6254): reopening a row in ANY terminal status — including `aborted` — must
3789        // un-finalize it back to `running` with `finalized_at` cleared. This covers both the
3790        // pre-existing divergence-recovery reopen (which starts a fresh replay cursor on
3791        // purpose) and the new crash-orphan sweep (INV-17), which makes `aborted` the common
3792        // outcome of a resumable crash: a resumed execution whose row keeps `finalized_at` set
3793        // would otherwise be prunable out from under the active resume.
3794        let backend = mem_backend(1_048_576).await;
3795        let exec = ExecutionId::new();
3796        backend
3797            .open_execution(exec, ExecutionKind::AgentTurn)
3798            .await
3799            .unwrap();
3800        backend
3801            .finalize(exec, ExecutionStatus::Aborted)
3802            .await
3803            .unwrap();
3804
3805        let is_resume = backend
3806            .open_execution(exec, ExecutionKind::AgentTurn)
3807            .await
3808            .unwrap();
3809        assert!(is_resume, "the row already existed, so this is a resume");
3810
3811        let (status, finalized): (String, Option<i64>) = zeph_db::query_as(sql!(
3812            "SELECT status, finalized_at FROM durable_executions WHERE execution_id = ?"
3813        ))
3814        .bind(exec.as_uuid().to_string())
3815        .fetch_one(backend.pool())
3816        .await
3817        .unwrap();
3818        assert_eq!(
3819            status, "running",
3820            "reopening an aborted execution must un-finalize it (INV-16)"
3821        );
3822        assert!(
3823            finalized.is_none(),
3824            "reopening must clear the stale finalized_at"
3825        );
3826    }
3827
3828    #[tokio::test]
3829    async fn cancel_execution_with_no_live_owner_cancels_immediately() {
3830        // `:memory:` has `lock_dir = None` and `cross_process = false` (sqlite build), so this
3831        // exercises the provably-safe single-process direct-write path (F3).
3832        let backend = mem_backend(1_048_576).await;
3833        let exec = ExecutionId::new();
3834        backend
3835            .open_execution(exec, ExecutionKind::AgentTurn)
3836            .await
3837            .unwrap();
3838
3839        let outcome = backend.cancel_execution(exec).await.unwrap();
3840        assert_eq!(outcome, CancelOutcome::Canceled);
3841
3842        let (status, finalized): (String, Option<i64>) = zeph_db::query_as(sql!(
3843            "SELECT status, finalized_at FROM durable_executions WHERE execution_id = ?"
3844        ))
3845        .bind(exec.as_uuid().to_string())
3846        .fetch_one(backend.pool())
3847        .await
3848        .unwrap();
3849        assert_eq!(status, "canceled");
3850        assert!(finalized.is_some(), "a terminal status stamps finalized_at");
3851    }
3852
3853    #[tokio::test]
3854    async fn cancel_execution_with_no_live_owner_on_file_backed_pool_cancels_immediately() {
3855        // The SQLite/Unix lock-probe path: no lock is held, so the probe succeeds and the write
3856        // proceeds while the lock is held across it, then releases.
3857        let dir = tempfile::tempdir().unwrap();
3858        let backend =
3859            LocalBackend::open(&dir.path().join("durable.db").to_string_lossy(), 1_048_576)
3860                .await
3861                .unwrap();
3862        backend.init().await.unwrap();
3863
3864        let exec = ExecutionId::new();
3865        backend
3866            .open_execution(exec, ExecutionKind::AgentTurn)
3867            .await
3868            .unwrap();
3869
3870        let outcome = backend.cancel_execution(exec).await.unwrap();
3871        assert_eq!(outcome, CancelOutcome::Canceled);
3872
3873        // The lock must have been released after the write: a fresh acquire succeeds.
3874        let lock_dir = backend.lock_dir.clone().unwrap();
3875        assert!(ExecutionLock::acquire(&lock_dir, exec).is_ok());
3876    }
3877
3878    #[tokio::test]
3879    async fn cancel_execution_refuses_a_live_owner_without_touching_the_row() {
3880        // FR-006/FR-007 refusal: a live owner's held flock must short-circuit cancel to
3881        // `LiveOwner`, and the row must be left completely untouched.
3882        let dir = tempfile::tempdir().unwrap();
3883        let db_path = dir.path().join("durable.db");
3884        let url = db_path.to_string_lossy().into_owned();
3885
3886        let owner = LocalBackend::open(&url, 1_048_576).await.unwrap();
3887        owner.init().await.unwrap();
3888        let canceler = LocalBackend::open(&url, 1_048_576).await.unwrap();
3889
3890        let exec = ExecutionId::new();
3891        let (_, _lock) = owner
3892            .open_execution_exclusive(exec, ExecutionKind::AgentTurn)
3893            .await
3894            .unwrap();
3895
3896        let outcome = canceler.cancel_execution(exec).await.unwrap();
3897        assert!(
3898            matches!(outcome, CancelOutcome::LiveOwner { pid } if pid == std::process::id()),
3899            "expected LiveOwner{{pid: {}}}, got {outcome:?}",
3900            std::process::id()
3901        );
3902
3903        let (status,): (String,) = zeph_db::query_as(sql!(
3904            "SELECT status FROM durable_executions WHERE execution_id = ?"
3905        ))
3906        .bind(exec.as_uuid().to_string())
3907        .fetch_one(owner.pool())
3908        .await
3909        .unwrap();
3910        assert_eq!(status, "running", "a live-owned row must never be touched");
3911    }
3912
3913    #[tokio::test]
3914    async fn cancel_execution_is_idempotent_on_a_second_call() {
3915        // NFR-003: canceling an already-canceled row is a no-op, not an error.
3916        let backend = mem_backend(1_048_576).await;
3917        let exec = ExecutionId::new();
3918        backend
3919            .open_execution(exec, ExecutionKind::AgentTurn)
3920            .await
3921            .unwrap();
3922
3923        assert_eq!(
3924            backend.cancel_execution(exec).await.unwrap(),
3925            CancelOutcome::Canceled
3926        );
3927        let second = backend.cancel_execution(exec).await.unwrap();
3928        assert_eq!(
3929            second,
3930            CancelOutcome::AlreadyTerminal {
3931                status: ExecutionStatus::Canceled
3932            }
3933        );
3934    }
3935
3936    #[tokio::test]
3937    async fn cancel_execution_on_each_other_terminal_status_is_already_terminal() {
3938        for status in [
3939            ExecutionStatus::Completed,
3940            ExecutionStatus::Failed,
3941            ExecutionStatus::Aborted,
3942        ] {
3943            let backend = mem_backend(1_048_576).await;
3944            let exec = ExecutionId::new();
3945            backend
3946                .open_execution(exec, ExecutionKind::AgentTurn)
3947                .await
3948                .unwrap();
3949            backend.finalize(exec, status).await.unwrap();
3950
3951            let outcome = backend.cancel_execution(exec).await.unwrap();
3952            assert_eq!(
3953                outcome,
3954                CancelOutcome::AlreadyTerminal { status },
3955                "canceling a {status:?} execution must be a no-op reporting its own status"
3956            );
3957        }
3958    }
3959
3960    #[tokio::test]
3961    async fn cancel_execution_on_unknown_id_returns_not_found() {
3962        let backend = mem_backend(1_048_576).await;
3963        let outcome = backend.cancel_execution(ExecutionId::new()).await.unwrap();
3964        assert_eq!(outcome, CancelOutcome::NotFound);
3965    }
3966
3967    #[tokio::test]
3968    async fn cancel_execution_races_finalize_exactly_one_terminal_status_wins() {
3969        // SC-003: concurrent `cancel_execution` and `finalize(Completed)` — the guarded
3970        // `UPDATE ... WHERE status = 'running'` pattern shared by both means whichever commits
3971        // first wins, and the loser's write is simply a no-op rather than clobbering the winner.
3972        // Drives the two as genuinely concurrent tasks against a real multi-connection pool
3973        // (file-backed — `:memory:` forces a single connection, per `zeph-db/src/pool.rs`'s
3974        // `connect_sqlite`, which would serialize the two calls trivially and prove nothing),
3975        // across many trials so both orderings are exercised without artificial delay injection —
3976        // mirrors `concurrent_prune_and_reopen_never_lose_or_corrupt_the_row`'s pattern.
3977        let dir = tempfile::tempdir().unwrap();
3978        let db_url = dir.path().join("durable.db").to_string_lossy().into_owned();
3979        let backend = Arc::new(LocalBackend::open(&db_url, 1_048_576).await.unwrap());
3980        backend.init().await.unwrap();
3981
3982        for _ in 0..20 {
3983            let exec = ExecutionId::new();
3984            backend
3985                .open_execution(exec, ExecutionKind::AgentTurn)
3986                .await
3987                .unwrap();
3988
3989            let cancel_backend = backend.clone();
3990            let cancel = tokio::spawn(async move { cancel_backend.cancel_execution(exec).await });
3991            let finalize_backend = backend.clone();
3992            let finalize = tokio::spawn(async move {
3993                finalize_backend
3994                    .finalize(exec, ExecutionStatus::Completed)
3995                    .await
3996            });
3997
3998            let (cancel_result, finalize_result) = tokio::join!(cancel, finalize);
3999            let cancel_outcome = cancel_result
4000                .expect("cancel task must not panic")
4001                .expect("cancel_execution must not error under a concurrent finalize");
4002            finalize_result
4003                .expect("finalize task must not panic")
4004                .expect("finalize must not error under a concurrent cancel");
4005
4006            let (status,): (String,) = zeph_db::query_as(sql!(
4007                "SELECT status FROM durable_executions WHERE execution_id = ?"
4008            ))
4009            .bind(exec.as_uuid().to_string())
4010            .fetch_one(backend.pool())
4011            .await
4012            .unwrap();
4013
4014            // Whichever guarded UPDATE committed first wins; the loser's is a no-op. Both
4015            // outcomes are legitimate depending on scheduling — the invariant is that exactly one
4016            // terminal status is recorded, matching whichever `cancel_execution` outcome resulted.
4017            match cancel_outcome {
4018                CancelOutcome::Canceled => assert_eq!(
4019                    status, "canceled",
4020                    "cancel_execution won the race — the row must be canceled"
4021                ),
4022                CancelOutcome::AlreadyTerminal {
4023                    status: ExecutionStatus::Completed,
4024                } => assert_eq!(
4025                    status, "completed",
4026                    "finalize won the race — the row must be completed, and cancel's own \
4027                     guarded UPDATE must have found it already non-running"
4028                ),
4029                other => panic!(
4030                    "cancel_execution must only ever win or lose cleanly against a concurrent \
4031                     finalize, got {other:?}"
4032                ),
4033            }
4034        }
4035    }
4036
4037    #[tokio::test]
4038    async fn cancel_execution_races_sweep_orphans_exactly_one_of_canceled_or_aborted_wins() {
4039        // SC-004: concurrent `cancel_execution` and `sweep_orphans` on the same stale `running`
4040        // row. Both probe the same INV-15 `ExecutionLock` before writing, so this race has two
4041        // layers: whichever task wins the flock is the only one that ever attempts a write (the
4042        // loser either gets `LiveOwner` immediately without touching the row, or skips the
4043        // candidate without aborting it — INV-17's "never abort on staleness alone" rule already
4044        // covers a live-held lock). Drives the two as genuinely concurrent tasks against a real
4045        // multi-connection pool, across many trials so both lock-acquisition orderings are
4046        // exercised — mirrors `concurrent_sweep_and_reopen_race_never_corrupts_the_row`'s pattern.
4047        let dir = tempfile::tempdir().unwrap();
4048        let db_url = dir.path().join("durable.db").to_string_lossy().into_owned();
4049        let backend = Arc::new(LocalBackend::open(&db_url, 1_048_576).await.unwrap());
4050        backend.init().await.unwrap();
4051
4052        let policy = RetentionPolicy {
4053            stale_running_after_secs: 1,
4054            prune_batch_size: 10,
4055            ..RetentionPolicy::default()
4056        };
4057
4058        for _ in 0..20 {
4059            let exec = ExecutionId::new();
4060            backend
4061                .open_execution(exec, ExecutionKind::AgentTurn)
4062                .await
4063                .unwrap();
4064            backdate_updated_at(&backend, exec, 0).await;
4065
4066            let cancel_backend = backend.clone();
4067            let cancel = tokio::spawn(async move { cancel_backend.cancel_execution(exec).await });
4068            let sweep_backend = backend.clone();
4069            let policy_for_task = policy.clone();
4070            let sweep =
4071                tokio::spawn(async move { sweep_backend.sweep_orphans(&policy_for_task).await });
4072
4073            let (cancel_result, sweep_result) = tokio::join!(cancel, sweep);
4074            let cancel_outcome = cancel_result
4075                .expect("cancel task must not panic")
4076                .expect("cancel_execution must not error under a concurrent sweep");
4077            let aborted = sweep_result
4078                .expect("sweep task must not panic")
4079                .expect("sweep_orphans must not error under a concurrent cancel");
4080
4081            let (status,): (String,) = zeph_db::query_as(sql!(
4082                "SELECT status FROM durable_executions WHERE execution_id = ?"
4083            ))
4084            .bind(exec.as_uuid().to_string())
4085            .fetch_one(backend.pool())
4086            .await
4087            .unwrap();
4088
4089            match cancel_outcome {
4090                CancelOutcome::Canceled => {
4091                    assert_eq!(aborted, 0, "cancel won the lock — sweep must skip this row");
4092                    assert_eq!(status, "canceled");
4093                }
4094                CancelOutcome::LiveOwner { .. } => {
4095                    assert_eq!(aborted, 1, "sweep won the lock — it must abort this row");
4096                    assert_eq!(status, "aborted");
4097                }
4098                other => panic!(
4099                    "cancel_execution must only ever win the lock (Canceled) or lose it \
4100                     (LiveOwner) against a concurrent sweep, got {other:?}"
4101                ),
4102            }
4103        }
4104    }
4105
4106    #[tokio::test]
4107    async fn open_execution_on_canceled_row_fails_closed_and_never_resumes() {
4108        // INV-16′ (#6362), mirroring spec-064 scenario #13: unlike `completed`/`failed`/`aborted`,
4109        // a `canceled` row is the one deliberate carve-out — reopening it must fail closed with
4110        // `ExecutionCanceled` rather than un-finalizing it back to `running`.
4111        let backend = mem_backend(1_048_576).await;
4112        let exec = ExecutionId::new();
4113        backend
4114            .open_execution(exec, ExecutionKind::AgentTurn)
4115            .await
4116            .unwrap();
4117        let outcome = backend.cancel_execution(exec).await.unwrap();
4118        assert_eq!(outcome, CancelOutcome::Canceled);
4119
4120        let err = backend
4121            .open_execution(exec, ExecutionKind::AgentTurn)
4122            .await
4123            .expect_err("reopening a canceled execution must fail closed");
4124        assert!(
4125            matches!(err, DurableError::ExecutionCanceled { execution_id } if execution_id == exec),
4126            "expected ExecutionCanceled, got {err:?}"
4127        );
4128
4129        let (status,): (String,) = zeph_db::query_as(sql!(
4130            "SELECT status FROM durable_executions WHERE execution_id = ?"
4131        ))
4132        .bind(exec.as_uuid().to_string())
4133        .fetch_one(backend.pool())
4134        .await
4135        .unwrap();
4136        assert_eq!(
4137            status, "canceled",
4138            "the row must never be reset to running by a reopen attempt"
4139        );
4140    }
4141
4142    #[tokio::test]
4143    async fn open_execution_exclusive_on_canceled_row_fails_closed_with_lock_released() {
4144        // Same INV-16′ guarantee via the exclusive entry point; the flock guard must still be
4145        // released normally (no lock leak) when the call returns an error.
4146        let dir = tempfile::tempdir().unwrap();
4147        let db_path = dir.path().join("durable.db");
4148        let backend = LocalBackend::open(&db_path.to_string_lossy(), 1_048_576)
4149            .await
4150            .unwrap();
4151        backend.init().await.unwrap();
4152
4153        let exec = ExecutionId::new();
4154        backend
4155            .open_execution(exec, ExecutionKind::AgentTurn)
4156            .await
4157            .unwrap();
4158        assert_eq!(
4159            backend.cancel_execution(exec).await.unwrap(),
4160            CancelOutcome::Canceled
4161        );
4162
4163        let err = backend
4164            .open_execution_exclusive(exec, ExecutionKind::AgentTurn)
4165            .await
4166            .expect_err("reopening a canceled execution exclusively must fail closed");
4167        assert!(matches!(err, DurableError::ExecutionCanceled { .. }));
4168
4169        // The lock must have been released (no leak): a fresh acquire on the same id succeeds.
4170        let dir2 = backend.lock_dir.clone().unwrap();
4171        assert!(ExecutionLock::acquire(&dir2, exec).is_ok());
4172    }
4173
4174    #[tokio::test]
4175    async fn reopen_of_a_row_deleted_out_from_under_it_starts_fresh() {
4176        // #6251 critic S1: simulates the tail of the prune-vs-reopen race — a concurrent prune
4177        // sweep deletes the row entirely before the reopen's guarded UPDATE runs. The guarded
4178        // UPDATE must match zero rows (not resurrect a half-deleted row), and the existence
4179        // fallback must see the row is genuinely gone and start a fresh execution rather than
4180        // falsely reporting `is_resume = true` for a row that no longer exists.
4181        let backend = mem_backend(1_048_576).await;
4182        let exec = ExecutionId::new();
4183        backend
4184            .open_execution(exec, ExecutionKind::AgentTurn)
4185            .await
4186            .unwrap();
4187        backend
4188            .finalize(exec, ExecutionStatus::Completed)
4189            .await
4190            .unwrap();
4191
4192        // Simulate the prune sweep's delete completing before the reopen runs.
4193        zeph_db::query(sql!(
4194            "DELETE FROM durable_executions WHERE execution_id = ?"
4195        ))
4196        .bind(exec.as_uuid().to_string())
4197        .execute(backend.pool())
4198        .await
4199        .unwrap();
4200
4201        let is_resume = backend
4202            .open_execution(exec, ExecutionKind::AgentTurn)
4203            .await
4204            .unwrap();
4205        assert!(
4206            !is_resume,
4207            "a row deleted by a concurrent prune must be reported as a fresh execution, not a resume"
4208        );
4209
4210        let (status,): (String,) = zeph_db::query_as(sql!(
4211            "SELECT status FROM durable_executions WHERE execution_id = ?"
4212        ))
4213        .bind(exec.as_uuid().to_string())
4214        .fetch_one(backend.pool())
4215        .await
4216        .unwrap();
4217        assert_eq!(status, "running", "the fresh row starts running");
4218    }
4219
4220    #[tokio::test]
4221    async fn prune_does_not_delete_a_row_reopened_since_it_was_finalized() {
4222        // #6251 critic S1: a row finalized, then legitimately reopened (un-finalized back to
4223        // running) before prune runs, must not be deleted even though prune's cutoff would have
4224        // matched its now-stale-if-it-were-still-finalized state.
4225        let backend = mem_backend(1_048_576).await;
4226        let exec = ExecutionId::new();
4227        backend
4228            .open_execution(exec, ExecutionKind::AgentTurn)
4229            .await
4230            .unwrap();
4231        backend.append(step_result(exec, 0, b"x")).await.unwrap();
4232        zeph_db::query(sql!(
4233            "UPDATE durable_executions SET status = 'completed', finalized_at = 1000 WHERE execution_id = ?"
4234        ))
4235        .bind(exec.as_uuid().to_string())
4236        .execute(backend.pool())
4237        .await
4238        .unwrap();
4239
4240        // A legitimate resume reopens and un-finalizes it before the prune sweep runs.
4241        let is_resume = backend
4242            .open_execution(exec, ExecutionKind::AgentTurn)
4243            .await
4244            .unwrap();
4245        assert!(is_resume);
4246
4247        let policy = RetentionPolicy {
4248            ttl_completed_secs: 1,
4249            prune_batch_size: 10,
4250            ..RetentionPolicy::default()
4251        };
4252        let deleted = backend.prune(&policy).await.unwrap();
4253        assert_eq!(
4254            deleted, 0,
4255            "a reopened (un-finalized) execution must not be pruned"
4256        );
4257        assert_eq!(
4258            backend.read_execution(exec).await.unwrap().len(),
4259            1,
4260            "the execution's journal must survive"
4261        );
4262    }
4263
4264    #[tokio::test]
4265    async fn concurrent_prune_and_reopen_never_lose_or_corrupt_the_row() {
4266        // #6251 critic S1: the deterministic tests above exercise each ordering of the prune-vs-
4267        // reopen race one step at a time; this test drives the two operations as genuinely
4268        // concurrent tasks against a real multi-connection pool (file-backed — `:memory:` forces
4269        // a single connection, per `zeph-db/src/pool.rs`'s `connect_sqlite`, which would serialize
4270        // the two calls trivially and prove nothing about the locking fix). Runs many trials with
4271        // fresh executions so the two tasks' actual scheduling order varies across iterations,
4272        // covering both "prune's tx starts first" and "reopen's UPDATE starts first" without
4273        // needing artificial delay injection into the DB layer.
4274        //
4275        // Invariant checked every trial, regardless of which task wins: neither operation errors,
4276        // and the row is never lost — it either stays `running` (reopen won, or ran after prune's
4277        // read already excluded it) or is deleted and then reinserted fresh by reopen's
4278        // does-not-exist fallback (prune won). It must never end up half-deleted (FK violation on
4279        // a later journal append) or stuck `completed` with a live journal.
4280        let dir = tempfile::tempdir().unwrap();
4281        let db_url = dir.path().join("durable.db").to_string_lossy().into_owned();
4282        let backend = Arc::new(LocalBackend::open(&db_url, 1_048_576).await.unwrap());
4283        backend.init().await.unwrap();
4284
4285        let policy = RetentionPolicy {
4286            ttl_completed_secs: 1,
4287            prune_batch_size: 10,
4288            ..RetentionPolicy::default()
4289        };
4290
4291        for _ in 0..20 {
4292            let exec = ExecutionId::new();
4293            backend
4294                .open_execution(exec, ExecutionKind::AgentTurn)
4295                .await
4296                .unwrap();
4297            backend.append(step_result(exec, 0, b"x")).await.unwrap();
4298            // Backdate finalized_at so this row is immediately prune-eligible.
4299            zeph_db::query(sql!(
4300                "UPDATE durable_executions SET status = 'completed', finalized_at = 1000 WHERE execution_id = ?"
4301            ))
4302            .bind(exec.as_uuid().to_string())
4303            .execute(backend.pool())
4304            .await
4305            .unwrap();
4306
4307            let reopen_backend = backend.clone();
4308            let reopen = tokio::spawn(async move {
4309                reopen_backend
4310                    .open_execution(exec, ExecutionKind::AgentTurn)
4311                    .await
4312            });
4313            let prune_backend = backend.clone();
4314            let policy_for_task = policy.clone();
4315            let prune = tokio::spawn(async move { prune_backend.prune(&policy_for_task).await });
4316
4317            let (reopen_result, prune_result) = tokio::join!(reopen, prune);
4318            reopen_result
4319                .expect("reopen task must not panic")
4320                .expect("reopen must not error under concurrent prune");
4321            prune_result
4322                .expect("prune task must not panic")
4323                .expect("prune must not error under a concurrent reopen");
4324
4325            let (status,): (String,) = zeph_db::query_as(sql!(
4326                "SELECT status FROM durable_executions WHERE execution_id = ?"
4327            ))
4328            .bind(exec.as_uuid().to_string())
4329            .fetch_one(backend.pool())
4330            .await
4331            .expect(
4332                "the row must exist under either race outcome — reopened-running, or \
4333                 deleted-then-reinserted-fresh-running by reopen's fallback",
4334            );
4335            assert_eq!(
4336                status, "running",
4337                "whichever task wins, the row must end up running — never left completed \
4338                 (orphaned from a live journal) or absent"
4339            );
4340        }
4341    }
4342
4343    #[tokio::test]
4344    async fn max_seq_reflects_committed_appends() {
4345        let backend = mem_backend(1_048_576).await;
4346        assert_eq!(
4347            backend.max_seq().await.unwrap(),
4348            None,
4349            "empty journal has no max seq"
4350        );
4351
4352        let exec = ExecutionId::new();
4353        backend
4354            .open_execution(exec, ExecutionKind::AgentTurn)
4355            .await
4356            .unwrap();
4357        for step in 0..3 {
4358            backend.append(step_result(exec, step, b"x")).await.unwrap();
4359        }
4360        assert_eq!(backend.max_seq().await.unwrap(), Some(JournalSeq::new(3)));
4361    }
4362
4363    #[tokio::test]
4364    async fn append_batch_group_commits_every_entry() {
4365        let backend = mem_backend(1_048_576).await;
4366        let exec = ExecutionId::new();
4367        backend
4368            .open_execution(exec, ExecutionKind::AgentTurn)
4369            .await
4370            .unwrap();
4371        let batch = vec![
4372            step_result(exec, 0, b"a"),
4373            step_result(exec, 1, b"b"),
4374            step_result(exec, 2, b"c"),
4375        ];
4376        backend.append_batch(&batch).await.unwrap();
4377        assert_eq!(backend.read_execution(exec).await.unwrap().len(), 3);
4378    }
4379
4380    #[tokio::test]
4381    async fn read_execution_range_bounds_the_segment() {
4382        let backend = mem_backend(1_048_576).await;
4383        let exec = ExecutionId::new();
4384        backend
4385            .open_execution(exec, ExecutionKind::AgentTurn)
4386            .await
4387            .unwrap();
4388        for step in 0..5 {
4389            backend.append(step_result(exec, step, b"x")).await.unwrap();
4390        }
4391        let segment = backend.read_execution_range(exec, 2, 2).await.unwrap();
4392        assert_eq!(segment.len(), 2);
4393        assert_eq!(segment[0].step_id, StepId::new(2));
4394        assert_eq!(segment[1].step_id, StepId::new(3));
4395    }
4396
4397    #[tokio::test]
4398    async fn lookup_committed_result_finds_by_idem_key() {
4399        let backend = mem_backend(1_048_576).await;
4400        let exec = ExecutionId::new();
4401        backend
4402            .open_execution(exec, ExecutionKind::AgentTurn)
4403            .await
4404            .unwrap();
4405        let entry = step_result(exec, 0, b"committed");
4406        let idem_key = match &entry.entry {
4407            EntryKind::StepResult {
4408                idempotency_key, ..
4409            } => *idempotency_key,
4410            other => panic!("unexpected entry kind: {other:?}"),
4411        };
4412        backend.append(entry).await.unwrap();
4413
4414        let found = backend
4415            .lookup_committed_result(exec, idem_key)
4416            .await
4417            .unwrap()
4418            .expect("committed result is located by its idempotency key");
4419        match &found.entry {
4420            EntryKind::StepResult { payload, .. } => assert_eq!(payload.as_ref(), b"committed"),
4421            other => panic!("unexpected entry kind: {other:?}"),
4422        }
4423
4424        // A key that was never committed yields nothing rather than erroring.
4425        let absent = IdempotencyKey::derive(exec, StepId::new(99), b"never");
4426        assert!(
4427            backend
4428                .lookup_committed_result(exec, absent)
4429                .await
4430                .unwrap()
4431                .is_none()
4432        );
4433    }
4434
4435    #[tokio::test]
4436    async fn capabilities_describe_the_local_profile() {
4437        let backend = mem_backend(4096).await;
4438        let caps = backend.capabilities();
4439        assert!(caps.parallel_steps);
4440        assert!(
4441            !caps.cross_process,
4442            "the SQLite local backend is in-process"
4443        );
4444        assert_eq!(caps.max_payload, 4096);
4445    }
4446
4447    #[tokio::test]
4448    async fn promise_insert_state_and_resolve_round_trip() {
4449        let backend = mem_backend(1_048_576)
4450            .await
4451            .with_cipher(Arc::new(XorCipher));
4452        let exec = ExecutionId::new();
4453        backend
4454            .open_execution(exec, ExecutionKind::AgentTurn)
4455            .await
4456            .unwrap();
4457        let promise = PromiseId::derive(exec, StepId::new(0));
4458        backend
4459            .insert_promise(promise, exec, [9u8; 32], 100)
4460            .await
4461            .unwrap();
4462
4463        let pending = backend.promise_state(promise).await.unwrap().unwrap();
4464        assert!(!pending.resolved);
4465        assert_eq!(pending.execution_id, exec);
4466        assert_eq!(pending.resolver_token_hash, [9u8; 32]);
4467
4468        // Resolve seals the value at rest; a second resolve is a no-op.
4469        assert!(
4470            backend
4471                .resolve_promise(promise, exec, b"answer", 200)
4472                .await
4473                .unwrap()
4474        );
4475        assert!(
4476            !backend
4477                .resolve_promise(promise, exec, b"again", 300)
4478                .await
4479                .unwrap()
4480        );
4481
4482        let resolved = backend.promise_state(promise).await.unwrap().unwrap();
4483        assert!(resolved.resolved);
4484        let sealed = resolved.payload.expect("resolved payload present");
4485        assert_ne!(sealed.as_slice(), b"answer", "payload is sealed at rest");
4486        let opened = backend
4487            .open_promise_payload(promise, exec, &sealed)
4488            .unwrap();
4489        assert_eq!(opened.as_ref(), b"answer");
4490    }
4491
4492    #[tokio::test]
4493    async fn claim_promise_notification_is_single_winner() {
4494        let backend = mem_backend(1_048_576).await;
4495        let exec = ExecutionId::new();
4496        backend
4497            .open_execution(exec, ExecutionKind::AgentTurn)
4498            .await
4499            .unwrap();
4500        let promise = PromiseId::derive(exec, StepId::new(0));
4501        backend
4502            .insert_promise(promise, exec, [9u8; 32], 100)
4503            .await
4504            .unwrap();
4505
4506        // First claim wins (transitions notified_at from NULL).
4507        assert!(
4508            backend
4509                .claim_promise_notification(promise, 200)
4510                .await
4511                .unwrap()
4512        );
4513        // Every later claim on the same promise is a no-op.
4514        assert!(
4515            !backend
4516                .claim_promise_notification(promise, 300)
4517                .await
4518                .unwrap()
4519        );
4520    }
4521
4522    #[tokio::test]
4523    async fn timer_arm_due_and_fire() {
4524        let backend = mem_backend(1_048_576).await;
4525        let exec = ExecutionId::new();
4526        backend
4527            .open_execution(exec, ExecutionKind::AgentTurn)
4528            .await
4529            .unwrap();
4530        let past = TimerId::derive(exec, StepId::new(0));
4531        let future = TimerId::derive(exec, StepId::new(1));
4532        backend.arm_timer(past, exec, 1_000, 0).await.unwrap();
4533        backend
4534            .arm_timer(future, exec, 9_000_000_000_000, 0)
4535            .await
4536            .unwrap();
4537
4538        // Only the past-due timer is returned at now = 5000.
4539        let due = backend.due_timers(5_000).await.unwrap();
4540        assert_eq!(due, vec![past]);
4541
4542        assert!(backend.mark_timer_fired(past).await.unwrap());
4543        assert!(
4544            !backend.mark_timer_fired(past).await.unwrap(),
4545            "second fire is a no-op"
4546        );
4547        assert_eq!(
4548            backend.timer_state(past).await.unwrap(),
4549            Some((1_000, true))
4550        );
4551        // The fired timer no longer appears as due.
4552        assert!(backend.due_timers(5_000).await.unwrap().is_empty());
4553    }
4554
4555    #[tokio::test]
4556    async fn prune_deletes_terminal_executions_past_ttl() {
4557        let backend = mem_backend(1_048_576).await;
4558        // An old completed execution (finalized long ago) and a fresh running one.
4559        let old = ExecutionId::new();
4560        backend
4561            .open_execution(old, ExecutionKind::AgentTurn)
4562            .await
4563            .unwrap();
4564        backend.append(step_result(old, 0, b"x")).await.unwrap();
4565        // Backdate its finalized_at far into the past.
4566        zeph_db::query(sql!(
4567            "UPDATE durable_executions SET status = 'completed', finalized_at = 1000 WHERE execution_id = ?"
4568        ))
4569        .bind(old.as_uuid().to_string())
4570        .execute(backend.pool())
4571        .await
4572        .unwrap();
4573
4574        let live = ExecutionId::new();
4575        backend
4576            .open_execution(live, ExecutionKind::AgentTurn)
4577            .await
4578            .unwrap();
4579        backend.append(step_result(live, 0, b"y")).await.unwrap();
4580
4581        let policy = RetentionPolicy {
4582            ttl_completed_secs: 1,
4583            prune_batch_size: 10,
4584            ..RetentionPolicy::default()
4585        };
4586        let deleted = backend.prune(&policy).await.unwrap();
4587        assert_eq!(deleted, 1, "only the aged terminal execution is pruned");
4588
4589        // The old execution and its journal are gone; the live one survives.
4590        assert!(backend.read_execution(old).await.unwrap().is_empty());
4591        assert!(
4592            backend
4593                .promise_state(PromiseId::derive(old, StepId::new(0)))
4594                .await
4595                .unwrap()
4596                .is_none()
4597        );
4598        assert_eq!(backend.read_execution(live).await.unwrap().len(), 1);
4599    }
4600
4601    #[tokio::test]
4602    async fn count_prunable_and_prune_include_canceled_executions_past_ttl() {
4603        // FR-013 (#6362): a canceled row groups with failed/aborted for the retention TTL cutoff
4604        // — without this, canceled rows would never be pruned and would accumulate forever.
4605        let backend = mem_backend(1_048_576).await;
4606        let exec = ExecutionId::new();
4607        backend
4608            .open_execution(exec, ExecutionKind::AgentTurn)
4609            .await
4610            .unwrap();
4611        assert_eq!(
4612            backend.cancel_execution(exec).await.unwrap(),
4613            CancelOutcome::Canceled
4614        );
4615        // Backdate finalized_at far into the past so it is past the failed/aborted TTL cutoff.
4616        zeph_db::query(sql!(
4617            "UPDATE durable_executions SET finalized_at = 1000 WHERE execution_id = ?"
4618        ))
4619        .bind(exec.as_uuid().to_string())
4620        .execute(backend.pool())
4621        .await
4622        .unwrap();
4623
4624        let policy = RetentionPolicy {
4625            ttl_failed_secs: 1,
4626            prune_batch_size: 10,
4627            ..RetentionPolicy::default()
4628        };
4629        let prunable = backend.count_prunable(&policy).await.unwrap();
4630        assert_eq!(
4631            prunable, 1,
4632            "an aged canceled row must be counted as prunable"
4633        );
4634
4635        let deleted = backend.prune(&policy).await.unwrap();
4636        assert_eq!(deleted, 1, "an aged canceled row must actually be pruned");
4637        assert!(backend.read_execution(exec).await.unwrap().is_empty());
4638    }
4639
4640    /// Regression for issue #6360 (critic B1): a keyed backend's `durable_execution_integrity` row
4641    /// (created by `bump_hwm_for_step_result` for every committed `StepResult`) references
4642    /// `durable_executions` without `ON DELETE CASCADE` — the same convention as
4643    /// `durable_journal`/`durable_promises`/`durable_timers`, which `delete_prune_batch` deletes
4644    /// manually before the parent row. Before the fix, the integrity row was never included in that
4645    /// manual delete, so `DELETE FROM durable_executions` violated the FK under `SQLite`'s
4646    /// `PRAGMA foreign_keys = ON` (and unconditionally on `PostgreSQL`), rolling back the whole
4647    /// prune batch for every keyed execution — retention silently stopped working on any real
4648    /// (`ZEPH_DURABLE_KEY`-configured) deployment. Exercises the previously-untested path: all
4649    /// prior prune tests used unkeyed backends, which never create an integrity row and so never
4650    /// hit the FK.
4651    #[tokio::test]
4652    async fn prune_deletes_a_keyed_execution_and_its_integrity_row() {
4653        let backend = mem_backend(1_048_576).await.with_hwm_key(0, [42u8; 32]);
4654        let old = ExecutionId::new();
4655        backend
4656            .open_execution(old, ExecutionKind::AgentTurn)
4657            .await
4658            .unwrap();
4659        backend.append(step_result(old, 0, b"x")).await.unwrap();
4660
4661        // The committed StepResult must have created an integrity row.
4662        let before: (i64,) = zeph_db::query_as(sql!(
4663            "SELECT COUNT(*) FROM durable_execution_integrity WHERE execution_id = ?"
4664        ))
4665        .bind(old.as_uuid().to_string())
4666        .fetch_one(backend.pool())
4667        .await
4668        .unwrap();
4669        assert_eq!(
4670            before.0, 1,
4671            "a committed StepResult must create an integrity row"
4672        );
4673
4674        zeph_db::query(sql!(
4675            "UPDATE durable_executions SET status = 'completed', finalized_at = 1000 WHERE execution_id = ?"
4676        ))
4677        .bind(old.as_uuid().to_string())
4678        .execute(backend.pool())
4679        .await
4680        .unwrap();
4681
4682        let policy = RetentionPolicy {
4683            ttl_completed_secs: 1,
4684            prune_batch_size: 10,
4685            ..RetentionPolicy::default()
4686        };
4687        let deleted = backend
4688            .prune(&policy)
4689            .await
4690            .expect("prune must not fail closed on a keyed execution's FK");
4691        assert_eq!(deleted, 1, "the keyed execution is pruned like any other");
4692
4693        assert!(backend.read_execution(old).await.unwrap().is_empty());
4694        let after: (i64,) = zeph_db::query_as(sql!(
4695            "SELECT COUNT(*) FROM durable_execution_integrity WHERE execution_id = ?"
4696        ))
4697        .bind(old.as_uuid().to_string())
4698        .fetch_one(backend.pool())
4699        .await
4700        .unwrap();
4701        assert_eq!(
4702            after.0, 0,
4703            "the integrity row must be pruned alongside its execution"
4704        );
4705    }
4706
4707    /// Backdate a `durable_executions` row's `updated_at` so it becomes a sweep candidate.
4708    async fn backdate_updated_at(backend: &LocalBackend, id: ExecutionId, updated_at_ms: i64) {
4709        zeph_db::query(sql!(
4710            "UPDATE durable_executions SET updated_at = ? WHERE execution_id = ?"
4711        ))
4712        .bind(updated_at_ms)
4713        .bind(id.as_uuid().to_string())
4714        .execute(backend.pool())
4715        .await
4716        .unwrap();
4717    }
4718
4719    #[tokio::test]
4720    async fn sweep_orphans_disabled_when_threshold_is_zero() {
4721        // A file-backed backend so the sweep would otherwise have a lock_dir to work with;
4722        // stale_running_after_secs = 0 must short-circuit before any scan.
4723        let dir = tempfile::tempdir().unwrap();
4724        let backend =
4725            LocalBackend::open(&dir.path().join("durable.db").to_string_lossy(), 1_048_576)
4726                .await
4727                .unwrap();
4728        backend.init().await.unwrap();
4729
4730        let exec = ExecutionId::new();
4731        backend
4732            .open_execution(exec, ExecutionKind::AgentTurn)
4733            .await
4734            .unwrap();
4735        backdate_updated_at(&backend, exec, 0).await;
4736
4737        let policy = RetentionPolicy {
4738            stale_running_after_secs: 0,
4739            ..RetentionPolicy::default()
4740        };
4741        let aborted = backend.sweep_orphans(&policy).await.unwrap();
4742        assert_eq!(
4743            aborted, 0,
4744            "stale_running_after_secs = 0 disables the sweep"
4745        );
4746
4747        let (status,): (String,) = zeph_db::query_as(sql!(
4748            "SELECT status FROM durable_executions WHERE execution_id = ?"
4749        ))
4750        .bind(exec.as_uuid().to_string())
4751        .fetch_one(backend.pool())
4752        .await
4753        .unwrap();
4754        assert_eq!(status, "running");
4755    }
4756
4757    #[tokio::test]
4758    async fn sweep_orphans_is_a_documented_no_op_on_memory_backend() {
4759        // `:memory:` has no on-disk lock_dir (INV-15 degrade), so the sweep must never abort on
4760        // staleness alone — FR-DE-19.
4761        let backend = mem_backend(1_048_576).await;
4762        let exec = ExecutionId::new();
4763        backend
4764            .open_execution(exec, ExecutionKind::AgentTurn)
4765            .await
4766            .unwrap();
4767        backdate_updated_at(&backend, exec, 0).await;
4768
4769        let policy = RetentionPolicy {
4770            stale_running_after_secs: 1,
4771            ..RetentionPolicy::default()
4772        };
4773        let aborted = backend.sweep_orphans(&policy).await.unwrap();
4774        assert_eq!(
4775            aborted, 0,
4776            "a lock_dir=None backend must never abort on staleness alone"
4777        );
4778
4779        let (status,): (String,) = zeph_db::query_as(sql!(
4780            "SELECT status FROM durable_executions WHERE execution_id = ?"
4781        ))
4782        .bind(exec.as_uuid().to_string())
4783        .fetch_one(backend.pool())
4784        .await
4785        .unwrap();
4786        assert_eq!(status, "running");
4787    }
4788
4789    #[tokio::test]
4790    async fn sweep_orphans_aborts_a_stale_running_execution_with_no_live_owner() {
4791        // FR-DE-16/17: a stale `running` row whose lock is free (no live owner) is hard-aborted.
4792        let dir = tempfile::tempdir().unwrap();
4793        let backend =
4794            LocalBackend::open(&dir.path().join("durable.db").to_string_lossy(), 1_048_576)
4795                .await
4796                .unwrap();
4797        backend.init().await.unwrap();
4798
4799        let exec = ExecutionId::new();
4800        backend
4801            .open_execution(exec, ExecutionKind::AgentTurn)
4802            .await
4803            .unwrap();
4804        // Nothing holds this execution's ExecutionLock — `open_execution` (not `_exclusive`)
4805        // never acquires one, simulating a crashed owner whose flock released on process exit.
4806        backdate_updated_at(&backend, exec, 0).await;
4807
4808        let policy = RetentionPolicy {
4809            stale_running_after_secs: 1,
4810            ..RetentionPolicy::default()
4811        };
4812        let aborted = backend.sweep_orphans(&policy).await.unwrap();
4813        assert_eq!(aborted, 1);
4814
4815        let (status, finalized): (String, Option<i64>) = zeph_db::query_as(sql!(
4816            "SELECT status, finalized_at FROM durable_executions WHERE execution_id = ?"
4817        ))
4818        .bind(exec.as_uuid().to_string())
4819        .fetch_one(backend.pool())
4820        .await
4821        .unwrap();
4822        assert_eq!(status, "aborted");
4823        assert!(finalized.is_some());
4824    }
4825
4826    #[tokio::test]
4827    async fn sweep_orphans_skips_an_execution_whose_lock_is_held_by_a_live_owner() {
4828        // INV-17: staleness of `updated_at` alone is never sufficient — a stale-but-alive
4829        // execution (long single step, parked HITL promise, multi-hour job) must survive the
4830        // sweep as long as its owner still holds the INV-15 flock.
4831        let dir = tempfile::tempdir().unwrap();
4832        let db_path = dir.path().join("durable.db");
4833        let url = db_path.to_string_lossy().into_owned();
4834
4835        let owner = LocalBackend::open(&url, 1_048_576).await.unwrap();
4836        owner.init().await.unwrap();
4837        let sweeper = LocalBackend::open(&url, 1_048_576).await.unwrap();
4838
4839        let exec = ExecutionId::new();
4840        let (_, _lock) = owner
4841            .open_execution_exclusive(exec, ExecutionKind::AgentTurn)
4842            .await
4843            .unwrap();
4844        backdate_updated_at(&owner, exec, 0).await;
4845
4846        let policy = RetentionPolicy {
4847            stale_running_after_secs: 1,
4848            ..RetentionPolicy::default()
4849        };
4850        let aborted = sweeper.sweep_orphans(&policy).await.unwrap();
4851        assert_eq!(aborted, 0, "a live-held lock must never be swept");
4852
4853        let (status,): (String,) = zeph_db::query_as(sql!(
4854            "SELECT status FROM durable_executions WHERE execution_id = ?"
4855        ))
4856        .bind(exec.as_uuid().to_string())
4857        .fetch_one(owner.pool())
4858        .await
4859        .unwrap();
4860        assert_eq!(status, "running");
4861    }
4862
4863    #[tokio::test]
4864    async fn sweep_orphans_leaves_a_fresh_running_execution_untouched() {
4865        // A recently-updated `running` row is not yet a sweep candidate at all.
4866        let dir = tempfile::tempdir().unwrap();
4867        let backend =
4868            LocalBackend::open(&dir.path().join("durable.db").to_string_lossy(), 1_048_576)
4869                .await
4870                .unwrap();
4871        backend.init().await.unwrap();
4872
4873        let exec = ExecutionId::new();
4874        backend
4875            .open_execution(exec, ExecutionKind::AgentTurn)
4876            .await
4877            .unwrap();
4878
4879        let policy = RetentionPolicy {
4880            stale_running_after_secs: 3600,
4881            ..RetentionPolicy::default()
4882        };
4883        let aborted = backend.sweep_orphans(&policy).await.unwrap();
4884        assert_eq!(aborted, 0);
4885    }
4886
4887    #[tokio::test]
4888    async fn sweep_orphans_never_touches_a_stale_canceled_row() {
4889        // FR-008 regression (#6362): `sweep_orphan_batch` only ever candidate-selects
4890        // `status = 'running'` rows, so a canceled row — even a stale one — must never be
4891        // resurrected or otherwise touched, across repeated sweep cycles.
4892        let dir = tempfile::tempdir().unwrap();
4893        let backend =
4894            LocalBackend::open(&dir.path().join("durable.db").to_string_lossy(), 1_048_576)
4895                .await
4896                .unwrap();
4897        backend.init().await.unwrap();
4898
4899        let exec = ExecutionId::new();
4900        backend
4901            .open_execution(exec, ExecutionKind::AgentTurn)
4902            .await
4903            .unwrap();
4904        assert_eq!(
4905            backend.cancel_execution(exec).await.unwrap(),
4906            CancelOutcome::Canceled
4907        );
4908        backdate_updated_at(&backend, exec, 0).await;
4909
4910        let policy = RetentionPolicy {
4911            stale_running_after_secs: 1,
4912            ..RetentionPolicy::default()
4913        };
4914        for _ in 0..3 {
4915            let aborted = backend.sweep_orphans(&policy).await.unwrap();
4916            assert_eq!(aborted, 0, "a canceled row must never be swept");
4917        }
4918
4919        let (status,): (String,) = zeph_db::query_as(sql!(
4920            "SELECT status FROM durable_executions WHERE execution_id = ?"
4921        ))
4922        .bind(exec.as_uuid().to_string())
4923        .fetch_one(backend.pool())
4924        .await
4925        .unwrap();
4926        assert_eq!(
4927            status, "canceled",
4928            "sweep must never resurrect a canceled row"
4929        );
4930    }
4931
4932    #[tokio::test]
4933    async fn count_orphans_matches_sweep_without_mutating() {
4934        let dir = tempfile::tempdir().unwrap();
4935        let backend =
4936            LocalBackend::open(&dir.path().join("durable.db").to_string_lossy(), 1_048_576)
4937                .await
4938                .unwrap();
4939        backend.init().await.unwrap();
4940
4941        let exec = ExecutionId::new();
4942        backend
4943            .open_execution(exec, ExecutionKind::AgentTurn)
4944            .await
4945            .unwrap();
4946        backdate_updated_at(&backend, exec, 0).await;
4947
4948        let policy = RetentionPolicy {
4949            stale_running_after_secs: 1,
4950            ..RetentionPolicy::default()
4951        };
4952        let counted = backend.count_orphans(&policy).await.unwrap();
4953        assert_eq!(counted, 1);
4954
4955        // count_orphans must not have mutated the row.
4956        let (status,): (String,) = zeph_db::query_as(sql!(
4957            "SELECT status FROM durable_executions WHERE execution_id = ?"
4958        ))
4959        .bind(exec.as_uuid().to_string())
4960        .fetch_one(backend.pool())
4961        .await
4962        .unwrap();
4963        assert_eq!(status, "running");
4964
4965        let aborted = backend.sweep_orphans(&policy).await.unwrap();
4966        assert_eq!(
4967            aborted, counted,
4968            "sweep must abort exactly what count_orphans counted"
4969        );
4970    }
4971
4972    /// Batching-boundary regression: a candidate set straddling `prune_batch_size` (one more row
4973    /// than a single batch) must be fully processed across multiple batches, not just the first
4974    /// one. Exercises the real `sweep_orphan_batch`/`sweep_orphans_in_batches` composition end to
4975    /// end (not the pure-logic unit test in `retention.rs`), so the SQL `LIMIT` and the
4976    /// `scanned`-driven continuation check are both proven against a real DB.
4977    #[tokio::test]
4978    async fn sweep_orphans_processes_every_batch_when_candidates_straddle_the_batch_size() {
4979        let dir = tempfile::tempdir().unwrap();
4980        let backend =
4981            LocalBackend::open(&dir.path().join("durable.db").to_string_lossy(), 1_048_576)
4982                .await
4983                .unwrap();
4984        backend.init().await.unwrap();
4985
4986        let batch_size = 2u64;
4987        let candidate_count = batch_size + 1; // straddles the batch boundary
4988        let mut execs = Vec::new();
4989        for _ in 0..candidate_count {
4990            let exec = ExecutionId::new();
4991            backend
4992                .open_execution(exec, ExecutionKind::AgentTurn)
4993                .await
4994                .unwrap();
4995            backdate_updated_at(&backend, exec, 0).await;
4996            execs.push(exec);
4997        }
4998
4999        let policy = RetentionPolicy {
5000            stale_running_after_secs: 1,
5001            prune_batch_size: batch_size,
5002            ..RetentionPolicy::default()
5003        };
5004        let aborted = backend.sweep_orphans(&policy).await.unwrap();
5005        assert_eq!(
5006            aborted, candidate_count,
5007            "every candidate must be aborted, including the one past the first batch"
5008        );
5009
5010        for exec in execs {
5011            let (status,): (String,) = zeph_db::query_as(sql!(
5012                "SELECT status FROM durable_executions WHERE execution_id = ?"
5013            ))
5014            .bind(exec.as_uuid().to_string())
5015            .fetch_one(backend.pool())
5016            .await
5017            .unwrap();
5018            assert_eq!(status, "aborted");
5019        }
5020    }
5021
5022    /// #6254 C1 regression: when the count of stale-but-live (lock-held) candidates is `>=
5023    /// prune_batch_size`, the sweep must still terminate rather than looping forever re-selecting
5024    /// the same lock-held rows. Before the keyset-pagination fix, `sweep_orphan_batch`'s candidate
5025    /// `SELECT` had no offset/cursor, so a batch consisting entirely of lock-held rows (which the
5026    /// sweep never deletes, mutates, or otherwise removes from the `status='running'` candidate
5027    /// set) would re-select the identical rows on every iteration: `scanned` would stay `==
5028    /// batch` and `aborted` would stay `0` forever, so `sweep_orphans_in_batches`'s `scanned <
5029    /// batch` continuation check would never trip. Exercises the real DB-backed
5030    /// `sweep_orphan_batch`/`sweep_orphans_in_batches` composition (not the pure-logic
5031    /// simulation in `retention.rs`) with more lock-held candidates than `prune_batch_size`, so a
5032    /// naive single-batch-worth-of-locks reproduction would not have caught a bug that only
5033    /// manifests once the candidate set spans multiple batches.
5034    #[tokio::test]
5035    async fn sweep_orphans_terminates_when_lock_held_candidates_exceed_batch_size() {
5036        let dir = tempfile::tempdir().unwrap();
5037        let db_url = dir.path().join("durable.db").to_string_lossy().into_owned();
5038
5039        let owner = LocalBackend::open(&db_url, 1_048_576).await.unwrap();
5040        owner.init().await.unwrap();
5041        let sweeper = LocalBackend::open(&db_url, 1_048_576).await.unwrap();
5042
5043        let batch_size = 2u64;
5044        let candidate_count = batch_size * 2 + 1; // spans at least three batches, all lock-held
5045        let mut locks = Vec::new();
5046        for _ in 0..candidate_count {
5047            let exec = ExecutionId::new();
5048            let (_, lock) = owner
5049                .open_execution_exclusive(exec, ExecutionKind::AgentTurn)
5050                .await
5051                .unwrap();
5052            backdate_updated_at(&owner, exec, 0).await;
5053            locks.push(lock); // held for the whole test — every candidate stays lock-held
5054        }
5055
5056        let policy = RetentionPolicy {
5057            stale_running_after_secs: 1,
5058            prune_batch_size: batch_size,
5059            ..RetentionPolicy::default()
5060        };
5061
5062        let aborted = tokio::time::timeout(
5063            std::time::Duration::from_secs(10),
5064            sweeper.sweep_orphans(&policy),
5065        )
5066        .await
5067        .expect(
5068            "sweep_orphans must terminate even when lock-held candidates exceed prune_batch_size \
5069             (#6254 C1) — it hung instead of returning",
5070        )
5071        .unwrap();
5072
5073        assert_eq!(aborted, 0, "every candidate's lock is held by a live owner");
5074        drop(locks);
5075    }
5076
5077    /// INV-17: the sweep's guarded abort `UPDATE` runs only while holding the same non-reentrant
5078    /// flock a concurrent `open_execution_exclusive` reopen for the same execution id requires, so
5079    /// the two can never both mutate the row at once. Drives them as genuinely concurrent tasks
5080    /// against a real multi-connection pool (file-backed — `:memory:` forces a single connection,
5081    /// which would serialize the two calls trivially and prove nothing) across many trials so both
5082    /// orderings ("sweep acquires the lock first" and "reopen acquires the lock first") are
5083    /// exercised without artificial delay injection, mirroring the #6251
5084    /// `concurrent_prune_and_reopen_never_lose_or_corrupt_the_row` pattern above.
5085    #[tokio::test]
5086    async fn concurrent_sweep_and_reopen_race_never_corrupts_the_row() {
5087        let dir = tempfile::tempdir().unwrap();
5088        let db_url = dir.path().join("durable.db").to_string_lossy().into_owned();
5089        let backend = Arc::new(LocalBackend::open(&db_url, 1_048_576).await.unwrap());
5090        backend.init().await.unwrap();
5091
5092        let policy = RetentionPolicy {
5093            stale_running_after_secs: 1,
5094            prune_batch_size: 10,
5095            ..RetentionPolicy::default()
5096        };
5097
5098        for _ in 0..20 {
5099            let exec = ExecutionId::new();
5100            backend
5101                .open_execution(exec, ExecutionKind::AgentTurn)
5102                .await
5103                .unwrap();
5104            backdate_updated_at(&backend, exec, 0).await;
5105
5106            let sweep_backend = backend.clone();
5107            let policy_for_task = policy.clone();
5108            let sweep =
5109                tokio::spawn(async move { sweep_backend.sweep_orphans(&policy_for_task).await });
5110
5111            let reopen_backend = backend.clone();
5112            let reopen = tokio::spawn(async move {
5113                reopen_backend
5114                    .open_execution_exclusive(exec, ExecutionKind::AgentTurn)
5115                    .await
5116            });
5117
5118            let (sweep_result, reopen_result) = tokio::join!(sweep, reopen);
5119            let aborted = sweep_result
5120                .expect("sweep task must not panic")
5121                .expect("sweep must not error under a concurrent reopen");
5122            assert!(aborted <= 1, "at most one candidate row exists per trial");
5123
5124            match reopen_result.expect("reopen task must not panic") {
5125                Ok((_is_resume, _lock)) => {
5126                    // reopen won the race for the lock (either before the sweep even tried, or
5127                    // after the sweep aborted the row and released) — the row must be `running`
5128                    // with `finalized_at` cleared either way (INV-16 un-finalizes `aborted` too).
5129                    let (status, finalized): (String, Option<i64>) = zeph_db::query_as(sql!(
5130                        "SELECT status, finalized_at FROM durable_executions WHERE execution_id = ?"
5131                    ))
5132                    .bind(exec.as_uuid().to_string())
5133                    .fetch_one(backend.pool())
5134                    .await
5135                    .unwrap();
5136                    assert_eq!(status, "running");
5137                    assert!(finalized.is_none());
5138                    // Finalize before the next trial: when reopen wins because the row was
5139                    // already `running` (not terminal) at the time it checked, `open_execution`'s
5140                    // existing-row branch never bumps `updated_at` — left alone, this row would
5141                    // stay a stale `running` candidate forever and pollute a later trial's
5142                    // `aborted` count (the `assert!(aborted <= 1, ...)` above would then see more
5143                    // than this trial's own row). Each trial must start with a clean slate of
5144                    // exactly its own candidate.
5145                    backend
5146                        .finalize(exec, ExecutionStatus::Completed)
5147                        .await
5148                        .unwrap();
5149                }
5150                Err(DurableError::ExecutionLocked { .. }) => {
5151                    // The sweep held the lock at the moment reopen tried — expected under the race.
5152                }
5153                Err(e) => panic!(
5154                    "reopen must only ever fail with ExecutionLocked under this race, got {e:?}"
5155                ),
5156            }
5157        }
5158    }
5159
5160    #[tokio::test]
5161    async fn checkpoint_fold_compacts_idempotent_prefix_and_replays() {
5162        let backend = mem_backend(1_048_576)
5163            .await
5164            .with_cipher(Arc::new(XorCipher));
5165        let exec = ExecutionId::new();
5166        backend
5167            .open_execution(exec, ExecutionKind::AgentTurn)
5168            .await
5169            .unwrap();
5170        for step in 0..5 {
5171            backend
5172                .append(step_result(exec, step, format!("v{step}").as_bytes()))
5173                .await
5174                .unwrap();
5175        }
5176
5177        // Fold steps 0..3 into a checkpoint.
5178        let folded = backend.checkpoint_fold(exec, 3).await.unwrap();
5179        assert_eq!(folded, 3);
5180
5181        // The individual rows for the folded steps are gone; steps 3 and 4 remain, plus a checkpoint.
5182        let remaining = backend.read_execution(exec).await.unwrap();
5183        let step_results: Vec<u32> = remaining
5184            .iter()
5185            .filter(|e| matches!(e.entry, EntryKind::StepResult { .. }))
5186            .map(|e| e.step_id.value())
5187            .collect();
5188        assert_eq!(step_results, vec![3, 4], "folded step rows are deleted");
5189        assert!(
5190            remaining
5191                .iter()
5192                .any(|e| matches!(e.entry, EntryKind::Checkpoint { .. })),
5193            "a checkpoint entry replaces the folded prefix"
5194        );
5195
5196        // The reconstructed folded results carry the original values and idempotency keys.
5197        let preloaded = backend.read_checkpoints(exec).await.unwrap();
5198        assert_eq!(preloaded.len(), 3);
5199        for (i, entry) in preloaded.iter().enumerate() {
5200            let step = u32::try_from(i).unwrap();
5201            assert_eq!(entry.step_id, StepId::new(step));
5202            match &entry.entry {
5203                EntryKind::StepResult {
5204                    payload,
5205                    idempotency_key,
5206                    ..
5207                } => {
5208                    assert_eq!(payload.as_ref(), format!("v{step}").as_bytes());
5209                    assert_eq!(
5210                        *idempotency_key,
5211                        IdempotencyKey::derive(exec, StepId::new(step), b"tool:read")
5212                    );
5213                }
5214                other => panic!("unexpected folded entry: {other:?}"),
5215            }
5216        }
5217    }
5218
5219    // High-water-mark tests (issue #6360). `mem_backend` opens a fresh `:memory:` pool per call, so
5220    // these tests share a pool via `LocalBackend::new(backend.pool().clone(), ...)` when they need a
5221    // second backend handle (a different key, or unkeyed) reading the same journal — mirroring the
5222    // existing `read_execution_rejects_control_hmac_under_wrong_key` pattern above.
5223
5224    #[tokio::test]
5225    async fn hwm_is_a_no_op_when_unkeyed() {
5226        let backend = mem_backend(1_048_576).await;
5227        let exec = ExecutionId::new();
5228        assert!(
5229            !backend
5230                .open_execution(exec, ExecutionKind::AgentTurn)
5231                .await
5232                .unwrap()
5233        );
5234        backend.append(step_result(exec, 0, b"v0")).await.unwrap();
5235        // No integrity row should exist, and resume must still succeed.
5236        assert!(
5237            backend
5238                .open_execution(exec, ExecutionKind::AgentTurn)
5239                .await
5240                .unwrap()
5241        );
5242    }
5243
5244    #[tokio::test]
5245    async fn hwm_verifies_on_resume_after_single_append_and_batch_append() {
5246        let backend = mem_backend(1_048_576).await.with_hwm_key(0, [1u8; 32]);
5247        let exec = ExecutionId::new();
5248        assert!(
5249            !backend
5250                .open_execution(exec, ExecutionKind::AgentTurn)
5251                .await
5252                .unwrap()
5253        );
5254        backend.append(step_result(exec, 0, b"v0")).await.unwrap();
5255        backend
5256            .append_batch(&[step_result(exec, 1, b"v1"), step_result(exec, 2, b"v2")])
5257            .await
5258            .unwrap();
5259
5260        assert!(
5261            backend
5262                .open_execution(exec, ExecutionKind::AgentTurn)
5263                .await
5264                .unwrap(),
5265            "resume must succeed when the recomputed count matches the signed HWM"
5266        );
5267    }
5268
5269    #[tokio::test]
5270    async fn hwm_detects_deletion_of_a_committed_step_result() {
5271        let backend = mem_backend(1_048_576).await.with_hwm_key(0, [2u8; 32]);
5272        let exec = ExecutionId::new();
5273        backend
5274            .open_execution(exec, ExecutionKind::AgentTurn)
5275            .await
5276            .unwrap();
5277        backend.append(step_result(exec, 0, b"v0")).await.unwrap();
5278        backend.append(step_result(exec, 1, b"v1")).await.unwrap();
5279
5280        // Simulate an attacker (or a bug) deleting a committed result without going through the
5281        // legitimate `checkpoint_fold` path, which would have kept `folded_count` in sync.
5282        zeph_db::query(sql!(
5283            "DELETE FROM durable_journal WHERE execution_id = ? AND step_id = 1"
5284        ))
5285        .bind(exec.as_uuid().to_string())
5286        .execute(backend.pool())
5287        .await
5288        .unwrap();
5289
5290        let err = backend
5291            .open_execution(exec, ExecutionKind::AgentTurn)
5292            .await
5293            .unwrap_err();
5294        assert_matches!(
5295            err,
5296            DurableError::HighWaterMarkIntegrity {
5297                reason: "count_mismatch",
5298                ..
5299            }
5300        );
5301
5302        // The execution must be finalized Aborted, not left running for a retry loop to keep
5303        // tripping the same check.
5304        let summaries = backend.list_executions(None, None, 10).await.unwrap();
5305        let summary = summaries.iter().find(|s| s.execution_id == exec).unwrap();
5306        assert_eq!(summary.status, ExecutionStatus::Aborted);
5307    }
5308
5309    #[tokio::test]
5310    async fn hwm_survives_a_legitimate_checkpoint_fold() {
5311        let backend = mem_backend(1_048_576)
5312            .await
5313            .with_cipher(Arc::new(XorCipher))
5314            .with_hwm_key(0, [3u8; 32]);
5315        let exec = ExecutionId::new();
5316        backend
5317            .open_execution(exec, ExecutionKind::AgentTurn)
5318            .await
5319            .unwrap();
5320        for step in 0..5 {
5321            backend
5322                .append(step_result(exec, step, format!("v{step}").as_bytes()))
5323                .await
5324                .unwrap();
5325        }
5326
5327        let folded = backend.checkpoint_fold(exec, 3).await.unwrap();
5328        assert_eq!(folded, 3);
5329
5330        assert!(
5331            backend
5332                .open_execution(exec, ExecutionKind::AgentTurn)
5333                .await
5334                .unwrap(),
5335            "a legitimate fold must not trip the HWM check: committed_result_count is invariant \
5336             across it (folded_count restores what the DELETE removed)"
5337        );
5338    }
5339
5340    /// S1 regression (addendum to #6451, spec-081 FR-008): a pre-rotation execution whose
5341    /// `StepResult`s are checkpoint-folded post-rotation reseals its checkpoint snapshot under
5342    /// the NEW `key_id` and DELETEs every old-key-id `StepResult` row it folds, but
5343    /// `checkpoint_fold` never re-signs the HWM (`committed_result_count` is deliberately
5344    /// invariant across a fold — see the doc on `checkpoint_fold`). So the integrity row keeps
5345    /// `key_epoch = previous_key_id` even once no old-key-id payload survives at all. This
5346    /// execution has `StepResult`s only (no `EffectIntent`), so both the AEAD blob-scan and the
5347    /// control-HMAC scan see nothing — `count_integrity_rows_under_epoch` is the only one of the
5348    /// three `--drop-previous` scans that catches it.
5349    #[tokio::test]
5350    async fn count_integrity_rows_under_epoch_catches_a_checkpoint_folded_pre_rotation_execution() {
5351        let pre_rotation = mem_backend(1_048_576)
5352            .await
5353            .with_cipher(Arc::new(RotatingKeyedCipher {
5354                current_id: 0,
5355                previous_id: None,
5356            }))
5357            .with_hwm_key(0, [20u8; 32]);
5358        let exec = ExecutionId::new();
5359        pre_rotation
5360            .open_execution(exec, ExecutionKind::AgentTurn)
5361            .await
5362            .unwrap();
5363        for step in 0..3 {
5364            pre_rotation
5365                .append(step_result(exec, step, format!("v{step}").as_bytes()))
5366                .await
5367                .unwrap();
5368        }
5369
5370        // Rotate: a fresh handle over the same journal now speaks the NEW current epoch/key-id
5371        // (1), with the old one (0) registered as previous for both the cipher (so fold can still
5372        // decrypt the not-yet-folded rows) and the HWM (the rotation window) — exactly mirroring
5373        // a real `zeph durable rotate-key` followed by a background fold.
5374        let post_rotation = LocalBackend::new(pre_rotation.pool().clone(), 1_048_576)
5375            .with_cipher(Arc::new(RotatingKeyedCipher {
5376                current_id: 1,
5377                previous_id: Some(0),
5378            }))
5379            .with_hwm_key(1, [21u8; 32])
5380            .with_previous_hwm_key(0, [20u8; 32]);
5381
5382        let folded = post_rotation.checkpoint_fold(exec, 3).await.unwrap();
5383        assert_eq!(
5384            folded, 3,
5385            "fold must compact every committed StepResult, leaving none live"
5386        );
5387
5388        assert_eq!(
5389            post_rotation.count_sealed_under_key_id(0).await.unwrap(),
5390            0,
5391            "every pre-rotation payload was folded away and resealed under the new key_id; the \
5392             AEAD scan sees nothing left sealed under the previous key_id"
5393        );
5394        assert_eq!(
5395            post_rotation
5396                .count_integrity_rows_under_epoch(0)
5397                .await
5398                .unwrap(),
5399            1,
5400            "the folded execution's HWM row still carries the previous epoch -- checkpoint_fold \
5401             never re-signs it (S1)"
5402        );
5403        assert_eq!(
5404            post_rotation
5405                .count_integrity_rows_under_epoch(1)
5406                .await
5407                .unwrap(),
5408            0,
5409            "the row has not migrated to the current epoch -- only a fresh StepResult commit \
5410             after resume would bump it"
5411        );
5412
5413        // The folded execution is not corrupted — it must still resume cleanly through the open
5414        // rotation window (this addendum's epoch=key_id design), it is just still dependent on
5415        // the previous HWM key until `--drop-previous` (which S1's fix now correctly refuses).
5416        assert!(
5417            post_rotation
5418                .open_execution(exec, ExecutionKind::AgentTurn)
5419                .await
5420                .unwrap(),
5421            "a folded pre-rotation execution must still resume through the open rotation window"
5422        );
5423    }
5424
5425    #[tokio::test]
5426    async fn hwm_detects_deletion_that_a_fold_does_not_cover() {
5427        let backend = mem_backend(1_048_576)
5428            .await
5429            .with_cipher(Arc::new(XorCipher))
5430            .with_hwm_key(0, [4u8; 32]);
5431        let exec = ExecutionId::new();
5432        backend
5433            .open_execution(exec, ExecutionKind::AgentTurn)
5434            .await
5435            .unwrap();
5436        for step in 0..5 {
5437            backend
5438                .append(step_result(exec, step, format!("v{step}").as_bytes()))
5439                .await
5440                .unwrap();
5441        }
5442        backend.checkpoint_fold(exec, 3).await.unwrap();
5443
5444        // Delete one of the *surviving* (non-folded) rows outside the write path.
5445        zeph_db::query(sql!(
5446            "DELETE FROM durable_journal WHERE execution_id = ? AND step_id = 4 AND entry_kind = 'step_result'"
5447        ))
5448        .bind(exec.as_uuid().to_string())
5449        .execute(backend.pool())
5450        .await
5451        .unwrap();
5452
5453        assert_matches!(
5454            backend
5455                .open_execution(exec, ExecutionKind::AgentTurn)
5456                .await
5457                .unwrap_err(),
5458            DurableError::HighWaterMarkIntegrity {
5459                reason: "count_mismatch",
5460                ..
5461            }
5462        );
5463    }
5464
5465    #[tokio::test]
5466    async fn hwm_unresolvable_key_epoch_fails_closed_not_legacy() {
5467        let writer = mem_backend(1_048_576).await.with_hwm_key(0, [5u8; 32]);
5468        let exec = ExecutionId::new();
5469        writer
5470            .open_execution(exec, ExecutionKind::AgentTurn)
5471            .await
5472            .unwrap();
5473        writer.append(step_result(exec, 0, b"v0")).await.unwrap();
5474
5475        // A different backend over the same journal, current epoch 9, no previous slot registered
5476        // for epoch 0 — the stored row's epoch is unresolvable. Per NFR-004/S-new-2 this must fail
5477        // closed, never silently degrade to "legacy" just because the row's epoch is unknown here.
5478        let reader = LocalBackend::new(writer.pool().clone(), 1_048_576).with_hwm_key(9, [6u8; 32]);
5479        assert_matches!(
5480            reader
5481                .open_execution(exec, ExecutionKind::AgentTurn)
5482                .await
5483                .unwrap_err(),
5484            DurableError::HighWaterMarkIntegrity {
5485                reason: "key_epoch_unresolvable",
5486                ..
5487            }
5488        );
5489    }
5490
5491    #[tokio::test]
5492    async fn hwm_previous_epoch_key_resolves_as_rekeyed_not_tampered() {
5493        let writer = mem_backend(1_048_576).await.with_hwm_key(0, [7u8; 32]);
5494        let exec = ExecutionId::new();
5495        writer
5496            .open_execution(exec, ExecutionKind::AgentTurn)
5497            .await
5498            .unwrap();
5499        writer.append(step_result(exec, 0, b"v0")).await.unwrap();
5500
5501        // A rotated backend: current epoch 1 under a new key, but the old epoch-0 key is still
5502        // registered as `previous` for the rotation window (FR-008). Verification must succeed via
5503        // the previous slot rather than reporting tamper.
5504        let reader = LocalBackend::new(writer.pool().clone(), 1_048_576)
5505            .with_hwm_key(1, [8u8; 32])
5506            .with_previous_hwm_key(0, [7u8; 32]);
5507        assert!(
5508            reader
5509                .open_execution(exec, ExecutionKind::AgentTurn)
5510                .await
5511                .unwrap(),
5512            "a row signed under a registered previous epoch must verify, not fail as tampered"
5513        );
5514    }
5515
5516    #[tokio::test]
5517    async fn hwm_wrong_key_under_the_same_epoch_is_tamper() {
5518        let writer = mem_backend(1_048_576).await.with_hwm_key(0, [9u8; 32]);
5519        let exec = ExecutionId::new();
5520        writer
5521            .open_execution(exec, ExecutionKind::AgentTurn)
5522            .await
5523            .unwrap();
5524        writer.append(step_result(exec, 0, b"v0")).await.unwrap();
5525
5526        let reader =
5527            LocalBackend::new(writer.pool().clone(), 1_048_576).with_hwm_key(0, [10u8; 32]);
5528        assert_matches!(
5529            reader
5530                .open_execution(exec, ExecutionKind::AgentTurn)
5531                .await
5532                .unwrap_err(),
5533            DurableError::HighWaterMarkIntegrity {
5534                reason: "hmac_mismatch",
5535                ..
5536            }
5537        );
5538    }
5539
5540    #[tokio::test]
5541    async fn hwm_accepts_a_legacy_execution_with_no_integrity_row() {
5542        // Entries written by an unkeyed backend leave no `durable_execution_integrity` row at all —
5543        // the genuine "predates this feature" case, distinct from a row that exists but is
5544        // unresolvable. A keyed backend resuming it must accept it (migration posture), not fail.
5545        let unkeyed_writer = mem_backend(1_048_576).await;
5546        let exec = ExecutionId::new();
5547        unkeyed_writer
5548            .open_execution(exec, ExecutionKind::AgentTurn)
5549            .await
5550            .unwrap();
5551        unkeyed_writer
5552            .append(step_result(exec, 0, b"v0"))
5553            .await
5554            .unwrap();
5555
5556        let keyed_reader =
5557            LocalBackend::new(unkeyed_writer.pool().clone(), 1_048_576).with_hwm_key(0, [11u8; 32]);
5558        assert!(
5559            keyed_reader
5560                .open_execution(exec, ExecutionKind::AgentTurn)
5561                .await
5562                .unwrap(),
5563            "an execution with no integrity row at all is legacy, not tampered"
5564        );
5565    }
5566
5567    // --- Vault-sealed integrity boundary tests (issue #6449) ---
5568
5569    #[tokio::test]
5570    async fn hwm_unsealed_absent_row_after_deletion_is_still_ok() {
5571        // A keyed but *unsealed* backend (the pre-#6449-cutover posture): even after a committed
5572        // StepResult's integrity row is deleted, resume must still succeed — the migration
5573        // posture unless/until an operator explicitly seals.
5574        let backend = mem_backend(1_048_576).await.with_hwm_key(0, [30u8; 32]);
5575        let exec = ExecutionId::new();
5576        backend
5577            .open_execution(exec, ExecutionKind::AgentTurn)
5578            .await
5579            .unwrap();
5580        backend.append(step_result(exec, 0, b"v0")).await.unwrap();
5581
5582        zeph_db::query(sql!(
5583            "DELETE FROM durable_execution_integrity WHERE execution_id = ?"
5584        ))
5585        .bind(exec.as_uuid().to_string())
5586        .execute(backend.pool())
5587        .await
5588        .unwrap();
5589
5590        assert!(
5591            backend
5592                .open_execution(exec, ExecutionKind::AgentTurn)
5593                .await
5594                .unwrap(),
5595            "unsealed backend must not treat an absent integrity row as tamper"
5596        );
5597    }
5598
5599    #[tokio::test]
5600    async fn hwm_post_seal_absent_row_with_committed_results_is_tamper() {
5601        let backend = mem_backend(1_048_576)
5602            .await
5603            .with_hwm_key(0, [31u8; 32])
5604            .with_integrity_sealed(true);
5605        let exec = ExecutionId::new();
5606        backend
5607            .open_execution(exec, ExecutionKind::AgentTurn)
5608            .await
5609            .unwrap();
5610        backend.append(step_result(exec, 0, b"v0")).await.unwrap();
5611
5612        // Attacker (DB write access) deletes the integrity row, keeping the committed
5613        // StepResult in place to replay it.
5614        zeph_db::query(sql!(
5615            "DELETE FROM durable_execution_integrity WHERE execution_id = ?"
5616        ))
5617        .bind(exec.as_uuid().to_string())
5618        .execute(backend.pool())
5619        .await
5620        .unwrap();
5621
5622        let err = backend
5623            .open_execution(exec, ExecutionKind::AgentTurn)
5624            .await
5625            .unwrap_err();
5626        assert_matches!(
5627            err,
5628            DurableError::HighWaterMarkIntegrity {
5629                reason: "integrity_row_absent_post_seal",
5630                ..
5631            }
5632        );
5633    }
5634
5635    #[tokio::test]
5636    async fn hwm_post_seal_forged_created_at_does_not_evade_the_seal() {
5637        // Proves S1 is fully closed: the boundary no longer consults `created_at` at all, so
5638        // an attacker forging it (the rev1 defeat) has no effect once sealed.
5639        let backend = mem_backend(1_048_576)
5640            .await
5641            .with_hwm_key(0, [32u8; 32])
5642            .with_integrity_sealed(true);
5643        let exec = ExecutionId::new();
5644        backend
5645            .open_execution(exec, ExecutionKind::AgentTurn)
5646            .await
5647            .unwrap();
5648        backend.append(step_result(exec, 0, b"v0")).await.unwrap();
5649
5650        zeph_db::query(sql!(
5651            "UPDATE durable_executions SET created_at = 0 WHERE execution_id = ?"
5652        ))
5653        .bind(exec.as_uuid().to_string())
5654        .execute(backend.pool())
5655        .await
5656        .unwrap();
5657        zeph_db::query(sql!(
5658            "DELETE FROM durable_execution_integrity WHERE execution_id = ?"
5659        ))
5660        .bind(exec.as_uuid().to_string())
5661        .execute(backend.pool())
5662        .await
5663        .unwrap();
5664
5665        let err = backend
5666            .open_execution(exec, ExecutionKind::AgentTurn)
5667            .await
5668            .unwrap_err();
5669        assert_matches!(
5670            err,
5671            DurableError::HighWaterMarkIntegrity {
5672                reason: "integrity_row_absent_post_seal",
5673                ..
5674            },
5675            "forging created_at must not evade the seal — it is never consulted"
5676        );
5677    }
5678
5679    #[tokio::test]
5680    async fn hwm_grandfathered_execution_absent_row_is_ok() {
5681        let exec = ExecutionId::new();
5682        let writer = mem_backend(1_048_576).await.with_hwm_key(0, [33u8; 32]);
5683        writer
5684            .open_execution(exec, ExecutionKind::AgentTurn)
5685            .await
5686            .unwrap();
5687        writer.append(step_result(exec, 0, b"v0")).await.unwrap();
5688        zeph_db::query(sql!(
5689            "DELETE FROM durable_execution_integrity WHERE execution_id = ?"
5690        ))
5691        .bind(exec.as_uuid().to_string())
5692        .execute(writer.pool())
5693        .await
5694        .unwrap();
5695
5696        let sealed_but_grandfathered = LocalBackend::new(writer.pool().clone(), 1_048_576)
5697            .with_hwm_key(0, [33u8; 32])
5698            .with_integrity_sealed(true)
5699            .with_grandfather(std::collections::HashSet::from([exec]));
5700
5701        assert!(
5702            sealed_but_grandfathered
5703                .open_execution(exec, ExecutionKind::AgentTurn)
5704                .await
5705                .unwrap(),
5706            "a grandfathered execution_id must resume despite the seal"
5707        );
5708    }
5709
5710    #[tokio::test]
5711    async fn find_unsealed_resumable_executions_finds_only_the_offending_set() {
5712        let backend = mem_backend(1_048_576).await.with_hwm_key(0, [35u8; 32]);
5713
5714        // (a) running, keyed, committed StepResult, integrity row deleted — the offending case.
5715        let offending = ExecutionId::new();
5716        backend
5717            .open_execution(offending, ExecutionKind::AgentTurn)
5718            .await
5719            .unwrap();
5720        backend
5721            .append(step_result(offending, 0, b"v0"))
5722            .await
5723            .unwrap();
5724        zeph_db::query(sql!(
5725            "DELETE FROM durable_execution_integrity WHERE execution_id = ?"
5726        ))
5727        .bind(offending.as_uuid().to_string())
5728        .execute(backend.pool())
5729        .await
5730        .unwrap();
5731
5732        // (b) running, keyed, has an intact integrity row — not offending.
5733        let intact = ExecutionId::new();
5734        backend
5735            .open_execution(intact, ExecutionKind::AgentTurn)
5736            .await
5737            .unwrap();
5738        backend.append(step_result(intact, 0, b"v0")).await.unwrap();
5739
5740        // (c) running, no committed results at all — not offending (nothing to smuggle).
5741        let empty = ExecutionId::new();
5742        backend
5743            .open_execution(empty, ExecutionKind::AgentTurn)
5744            .await
5745            .unwrap();
5746
5747        // (d) terminal (finalized), integrity row absent — not offending (can never resume again).
5748        let terminal = ExecutionId::new();
5749        backend
5750            .open_execution(terminal, ExecutionKind::AgentTurn)
5751            .await
5752            .unwrap();
5753        backend
5754            .append(step_result(terminal, 0, b"v0"))
5755            .await
5756            .unwrap();
5757        zeph_db::query(sql!(
5758            "DELETE FROM durable_execution_integrity WHERE execution_id = ?"
5759        ))
5760        .bind(terminal.as_uuid().to_string())
5761        .execute(backend.pool())
5762        .await
5763        .unwrap();
5764        backend
5765            .finalize(terminal, ExecutionStatus::Completed)
5766            .await
5767            .unwrap();
5768
5769        let found = backend.find_unsealed_resumable_executions().await.unwrap();
5770        assert_eq!(
5771            found,
5772            vec![offending],
5773            "only the truly offending execution must be returned"
5774        );
5775    }
5776
5777    #[tokio::test]
5778    async fn hwm_post_seal_absent_row_with_zero_committed_results_is_ok() {
5779        // A sealed backend with no committed StepResult at all (e.g. an execution that was
5780        // opened but never produced a result) has nothing to smuggle — accepted even post-seal.
5781        let backend = mem_backend(1_048_576)
5782            .await
5783            .with_hwm_key(0, [34u8; 32])
5784            .with_integrity_sealed(true);
5785        let exec = ExecutionId::new();
5786        backend
5787            .open_execution(exec, ExecutionKind::AgentTurn)
5788            .await
5789            .unwrap();
5790
5791        assert!(
5792            backend
5793                .open_execution(exec, ExecutionKind::AgentTurn)
5794                .await
5795                .unwrap(),
5796            "zero committed results, post-seal, must not be treated as tamper"
5797        );
5798    }
5799
5800    #[tokio::test]
5801    async fn hwm_ignores_effect_intent_and_control_entries() {
5802        // Only `StepResult` rows count toward `committed_result_count` (S-new-1) — an EffectIntent
5803        // must not bump the HWM, and its presence alone must not trip verification.
5804        let backend = mem_backend(1_048_576).await.with_hwm_key(0, [12u8; 32]);
5805        let exec = ExecutionId::new();
5806        backend
5807            .open_execution(exec, ExecutionKind::AgentTurn)
5808            .await
5809            .unwrap();
5810        backend.append(effect_intent(exec, 0)).await.unwrap();
5811        backend.append(step_result(exec, 1, b"v1")).await.unwrap();
5812
5813        let stored: (i64,) = zeph_db::query_as(sql!(
5814            "SELECT committed_result_count FROM durable_execution_integrity WHERE execution_id = ?"
5815        ))
5816        .bind(exec.as_uuid().to_string())
5817        .fetch_one(backend.pool())
5818        .await
5819        .unwrap();
5820        assert_eq!(
5821            stored.0, 1,
5822            "only the StepResult row counts, not the EffectIntent"
5823        );
5824
5825        assert!(
5826            backend
5827                .open_execution(exec, ExecutionKind::AgentTurn)
5828                .await
5829                .unwrap()
5830        );
5831    }
5832}