Skip to main content

faucet_cli/serve/history/
sql.rs

1//! Shared SQL run-history machinery for the Postgres and SQLite backends
2//! (Phase 5, #127). Both backends are identical except for the connection setup
3//! and the placeholder dialect (`$n` vs `?`), so the schema, prepared-statement
4//! text, pure helpers, and the entire `RunHistory` impl live here once and are
5//! instantiated for each concrete `sqlx` pool via the `impl_sql_history!` macro.
6//!
7//! **Portability:** every column is `TEXT` (timestamps are stored as fixed-width
8//! RFC3339 with nanosecond precision + `Z`, which sorts lexicographically in
9//! chronological order, so keyset pagination and expiry comparisons work without
10//! any database date type — and thus without the `sqlx` `chrono` feature). The
11//! whole `RunRecord` is serialized into the `body` column (the source of truth on
12//! read); the dedicated columns exist only for filtering, ordering, and expiry.
13//!
14//! **Idempotency** lives in a separate `faucet_serve_idem` table whose `key`
15//! primary key is the required unique index (spec §10/§11). The claim is atomic
16//! via `INSERT … ON CONFLICT DO NOTHING` plus an optimistic, expiry-guarded
17//! takeover `UPDATE`, mirroring the memory backend's shard-locked semantics.
18
19use super::{HistoryError, RunRecord, RunStatus};
20use chrono::{DateTime, Utc};
21use std::time::Duration;
22
23/// DDL run at connect time. Valid verbatim on both Postgres and SQLite (only
24/// `TEXT` columns, `IF NOT EXISTS`, and standard indexes).
25pub const DDL: &[&str] = &[
26    // `owner` is the id of the serve instance that owns the run; `lease_expires_at`
27    // is the RFC3339 instant past which that ownership is presumed dead. Together
28    // they fence orphan recovery: an instance only fails a non-terminal run whose
29    // lease has expired, never another live instance's heartbeated runs (#146 H7).
30    "CREATE TABLE IF NOT EXISTS faucet_serve_runs (\
31        run_id TEXT PRIMARY KEY,\
32        name TEXT,\
33        status TEXT NOT NULL,\
34        submitted_at TEXT NOT NULL,\
35        finished_at TEXT,\
36        idempotency_key TEXT,\
37        owner TEXT,\
38        lease_expires_at TEXT,\
39        cancel_requested TEXT,\
40        body TEXT NOT NULL)",
41    "CREATE INDEX IF NOT EXISTS faucet_serve_runs_submitted_idx \
42        ON faucet_serve_runs (submitted_at)",
43    // Speeds the per-tick orphan scan / lease renewal, which filter on
44    // (status, owner, lease_expires_at).
45    "CREATE INDEX IF NOT EXISTS faucet_serve_runs_status_lease_idx \
46        ON faucet_serve_runs (status, lease_expires_at)",
47    // Speeds the cluster dispatcher's pending-run query (ordered by submitted_at).
48    "CREATE INDEX IF NOT EXISTS faucet_serve_runs_pending_idx \
49        ON faucet_serve_runs (status, submitted_at)",
50    "CREATE TABLE IF NOT EXISTS faucet_serve_instances (\
51        instance_id TEXT PRIMARY KEY,\
52        started_at TEXT NOT NULL,\
53        last_heartbeat TEXT NOT NULL,\
54        listen TEXT,\
55        max_concurrent TEXT,\
56        in_flight TEXT)",
57    "CREATE INDEX IF NOT EXISTS faucet_serve_instances_hb_idx \
58        ON faucet_serve_instances (last_heartbeat)",
59    "CREATE TABLE IF NOT EXISTS faucet_serve_idem (\
60        key TEXT PRIMARY KEY,\
61        run_id TEXT NOT NULL,\
62        fingerprint TEXT NOT NULL,\
63        claimed_at TEXT NOT NULL)",
64    // Source shards for clustered Mode B (#230). One row per (run, shard);
65    // `owner`/`lease_expires_at`/`attempt` reuse Mode A's lease-fencing semantics
66    // at shard granularity. `size_estimate` (an integer stored as TEXT) drives
67    // skew-aware, largest-first claiming. `descriptor` is the opaque connector
68    // shard spec, replayed to the worker that claims the shard.
69    "CREATE TABLE IF NOT EXISTS faucet_serve_shards (\
70        run_id TEXT NOT NULL,\
71        shard_id TEXT NOT NULL,\
72        descriptor TEXT NOT NULL,\
73        size_estimate TEXT,\
74        status TEXT NOT NULL,\
75        owner TEXT,\
76        lease_expires_at TEXT,\
77        attempt TEXT NOT NULL,\
78        finished_at TEXT,\
79        PRIMARY KEY (run_id, shard_id))",
80    "CREATE INDEX IF NOT EXISTS faucet_serve_shards_claim_idx \
81        ON faucet_serve_shards (status, lease_expires_at)",
82    // Audit log for RBAC (#205). One row per mutating (or denied) control-plane
83    // action. `id` is a time-ordered UUIDv7; `ts` is fixed-width RFC3339 so the
84    // newest-first ordering and retention purge sort lexicographically.
85    "CREATE TABLE IF NOT EXISTS faucet_serve_audit (\
86        id TEXT PRIMARY KEY,\
87        ts TEXT NOT NULL,\
88        principal TEXT NOT NULL,\
89        role TEXT NOT NULL,\
90        action TEXT NOT NULL,\
91        run_id TEXT,\
92        config_fingerprint TEXT,\
93        source_ip TEXT,\
94        result TEXT NOT NULL)",
95    "CREATE INDEX IF NOT EXISTS faucet_serve_audit_ts_idx \
96        ON faucet_serve_audit (ts)",
97    // Persistent run logs (#529). `seq` is a zero-padded fixed-width string so it
98    // sorts lexically = numerically; purged on its own retention window (`ts`).
99    "CREATE TABLE IF NOT EXISTS faucet_serve_run_logs (\
100        run_id TEXT NOT NULL,\
101        seq TEXT NOT NULL,\
102        ts TEXT NOT NULL,\
103        level TEXT NOT NULL,\
104        line TEXT NOT NULL,\
105        PRIMARY KEY (run_id, seq))",
106    "CREATE INDEX IF NOT EXISTS faucet_serve_run_logs_ts_idx \
107        ON faucet_serve_run_logs (ts)",
108    // Data Movement Catalog (#279). Accumulating cross-run state, deliberately
109    // NOT covered by `purge_expired` (the history is the value). Same
110    // TEXT-columns + JSON `body` convention as the run tables: the dedicated
111    // columns exist for filtering only; `body` is the source of truth on read.
112    "CREATE TABLE IF NOT EXISTS faucet_catalog_datasets (\
113        id TEXT PRIMARY KEY,\
114        uri TEXT NOT NULL,\
115        kind TEXT NOT NULL,\
116        last_seen TEXT NOT NULL,\
117        body TEXT NOT NULL)",
118    // One row per (dataset, schema version); appended only on content change.
119    // `version` is an integer stored as TEXT (cast on ORDER BY), matching the
120    // shard table's `size_estimate` convention.
121    "CREATE TABLE IF NOT EXISTS faucet_catalog_schema_versions (\
122        dataset_id TEXT NOT NULL,\
123        version TEXT NOT NULL,\
124        recorded_at TEXT NOT NULL,\
125        body TEXT NOT NULL,\
126        PRIMARY KEY (dataset_id, version))",
127    // One row per (source dataset, sink dataset) lineage edge.
128    "CREATE TABLE IF NOT EXISTS faucet_catalog_edges (\
129        src_id TEXT NOT NULL,\
130        dst_id TEXT NOT NULL,\
131        last_seen TEXT NOT NULL,\
132        body TEXT NOT NULL,\
133        PRIMARY KEY (src_id, dst_id))",
134    // Per-run volume points, capped per dataset at `catalog::STATS_RETAIN`.
135    "CREATE TABLE IF NOT EXISTS faucet_catalog_stats (\
136        dataset_id TEXT NOT NULL,\
137        recorded_at TEXT NOT NULL,\
138        run_id TEXT NOT NULL,\
139        records TEXT NOT NULL,\
140        PRIMARY KEY (dataset_id, recorded_at))",
141    // Resolved+expanded config snapshots for `faucet plan --diff` (#374). One
142    // row per pipeline (latest-wins upsert); `body` is the redacted
143    // `ConfigSnapshot` JSON — no secret material is ever stored.
144    "CREATE TABLE IF NOT EXISTS faucet_config_snapshots (\
145        pipeline TEXT PRIMARY KEY,\
146        recorded_at TEXT NOT NULL,\
147        faucet_version TEXT NOT NULL,\
148        body TEXT NOT NULL)",
149    // Registered pipeline templates (#444), one row per (id, version). `body` is
150    // the full `TemplateRecord` JSON — including the config document *verbatim*,
151    // so `${env:…}` / `${vault:…}` stay unresolved tokens and no secret material
152    // is ever persisted. `version` is an integer stored as TEXT (cast on
153    // ORDER BY), matching the schema-version / shard-estimate convention.
154    // Deliberately NOT purged by run retention: a template outlives its runs.
155    "CREATE TABLE IF NOT EXISTS faucet_templates (\
156        id TEXT NOT NULL,\
157        version TEXT NOT NULL,\
158        name TEXT,\
159        created_at TEXT NOT NULL,\
160        body TEXT NOT NULL,\
161        PRIMARY KEY (id, version))",
162    // Named channel pointers (#444): one row per (template, channel), each
163    // aiming at a numeric version. `latest` is derived from `faucet_templates`
164    // and never stored here. Deleting a version drops the channels aimed at it,
165    // so a pointer can never dangle.
166    "CREATE TABLE IF NOT EXISTS faucet_template_tags (\
167        id TEXT NOT NULL,\
168        tag TEXT NOT NULL,\
169        version TEXT NOT NULL,\
170        updated_at TEXT NOT NULL,\
171        PRIMARY KEY (id, tag))",
172    // Append-only launch log (#444): the source of truth for `stable` (newest
173    // entry) and `previous` (the one before it), the derived template status, and
174    // the launch/rollback audit trail. `seq` is an integer stored as TEXT (cast on
175    // ORDER BY), matching the version/estimate convention elsewhere.
176    "CREATE TABLE IF NOT EXISTS faucet_template_launches (\
177        id TEXT NOT NULL,\
178        seq TEXT NOT NULL,\
179        version TEXT NOT NULL,\
180        launched_at TEXT NOT NULL,\
181        launched_by TEXT,\
182        PRIMARY KEY (id, seq))",
183    // Deprecation markers — the only *stored* part of a template's lifecycle
184    // status (`draft` vs `launched` derives from the launch log).
185    "CREATE TABLE IF NOT EXISTS faucet_template_deprecations (\
186        id TEXT PRIMARY KEY,\
187        deprecated_at TEXT NOT NULL,\
188        deprecated_by TEXT,\
189        reason TEXT)",
190];
191
192/// SQL placeholder dialect.
193#[derive(Clone, Copy, Debug)]
194pub enum Dialect {
195    Postgres,
196    Sqlite,
197}
198
199/// Prepared-statement text for a backend, built once per dialect at connect time.
200pub struct Stmts {
201    /// (`cancel_requested` is intentionally NOT written by `upsert` — it is set
202    /// only via `request_cancel` and cleared by `reclaim_requeue`; it defaults to
203    /// NULL on insert.)
204    pub upsert: String,
205    pub select_body: String,
206    pub select_status: String,
207    pub select_submitted: String,
208    pub delete: String,
209    pub list: String,
210    pub purge_runs: String,
211    pub purge_idem: String,
212    /// Select non-terminal runs whose owning instance's lease has expired (or
213    /// is unset) — the orphans this instance may safely fail. Param: `now`.
214    pub select_orphans: String,
215    /// Extend the lease of this instance's own non-terminal runs (heartbeat).
216    /// Params: `new_lease_expiry`, `instance_id`.
217    pub renew_leases: String,
218    pub insert_idem: String,
219    pub select_idem: String,
220    pub takeover_idem: String,
221    /// Delete the idempotency claim(s) that point at a given run — used when a
222    /// run is deleted so a replay of the key starts fresh rather than 404-ing
223    /// on the missing record (#146 M8). Scoped by `run_id`, so a newer run that
224    /// re-claimed the same key keeps its claim.
225    pub delete_idem_by_run: String,
226    /// Cluster dispatcher: fetch oldest pending runs up to a given limit.
227    pub select_pending: String,
228    /// Cluster dispatcher: atomically claim a pending run (set owner + running).
229    pub claim_one: String,
230    /// Cluster reclaimer: select expired running runs for requeue/fail evaluation.
231    /// NOTE: `'queued'` is the single-instance status; cluster runs flow
232    /// `pending → running`, so the failover reclaimer covers `'running'` only.
233    pub reclaim_select: String,
234    /// Cluster reclaimer: requeue an expired running run back to pending.
235    pub reclaim_requeue: String,
236    /// Cluster reclaimer: fail an expired running run that cannot be requeued.
237    pub reclaim_fail: String,
238    /// Finalize a run owned by this instance (terminal status update).
239    pub finalize_owned: String,
240    /// Cancel a pending run directly (transition pending → cancelled).
241    pub cancel_pending: String,
242    /// Request cancellation of an in-flight run owned by another instance.
243    pub request_cancel: String,
244    /// List run IDs owned by this instance that have a pending cancellation request.
245    pub pending_cancellations: String,
246    /// Upsert this instance's membership heartbeat into `faucet_serve_instances`.
247    pub heartbeat_instance: String,
248    /// List instances whose last heartbeat is at or after a given threshold.
249    pub live_instances: String,
250    /// Prune instances whose last heartbeat is before a given threshold.
251    pub prune_instances: String,
252    // ── Source shards (Mode B, #230) ─────────────────────────────────────────
253    /// Idempotent shard insert (`ON CONFLICT (run_id, shard_id) DO NOTHING`).
254    pub insert_shard: String,
255    /// Select claimable pending shards joined to their run body, largest first.
256    pub claim_shards_select: String,
257    /// Atomically claim one pending shard for this instance.
258    pub claim_shard_one: String,
259    /// Heartbeat this instance's running shards.
260    pub renew_shard_leases: String,
261    /// Select expired-lease running shards for requeue/fail evaluation.
262    pub reclaim_shards_select: String,
263    /// Requeue an expired running shard back to pending (attempt++).
264    pub reclaim_shard_requeue: String,
265    /// Fail an expired running shard that exhausted its attempts (poison).
266    pub reclaim_shard_fail: String,
267    /// Owner-fenced terminal write for one shard.
268    pub finalize_shard: String,
269    /// Status counts for a run's shards.
270    pub shard_progress: String,
271    /// Distinct run_ids for which THIS instance owns a `running` shard whose
272    /// parent run has a pending cancellation request (cross-instance shard
273    /// cancel, F10). Param: `instance_id`.
274    pub pending_shard_cancellations: String,
275    /// Select run_ids of `sharded` parents (candidates to finalize once all
276    /// their shards are terminal, F11).
277    pub select_sharded_parents: String,
278    /// Status-fenced terminal write for a `sharded` parent (F11). A benign
279    /// double-finalize across instances is a no-op: the guard requires the
280    /// parent to still be `sharded`. Does NOT re-arm owner/lease.
281    pub finalize_sharded_parent: String,
282    /// Delete a run's shard rows (paired with [`delete`](Self::delete) so a
283    /// deleted run leaves no orphaned shard rows behind, F25). Param: `run_id`.
284    pub delete_shards_by_run: String,
285    /// Purge shard rows whose parent run no longer exists (run-record purged by
286    /// retention, F25). No params — a set-difference against `faucet_serve_runs`.
287    pub purge_orphan_shards: String,
288    // ── Audit log (RBAC, #205) ───────────────────────────────────────────────
289    /// Append one audit record.
290    pub insert_audit: String,
291    /// Newest-first audit records matching the (nullable) filters. Param order:
292    /// principal, action, since, until, limit.
293    pub list_audit: String,
294    /// Purge audit records older than a threshold (retention).
295    pub purge_audit: String,
296    // ── Persistent run logs (#529) ────────────────────────────────────────────
297    /// Insert one run-log line. Params: run_id, seq, ts, level, line.
298    pub insert_run_log: String,
299    /// A run's log lines with `seq > after` (excluding the truncation sentinel),
300    /// oldest-first, capped. Param order: run_id, after_seq, limit.
301    pub list_run_logs: String,
302    /// Whether a run has a truncation sentinel row. Param: run_id.
303    pub run_log_truncated: String,
304    /// Purge run-log lines older than a threshold (retention).
305    pub purge_run_logs: String,
306    // ── Data Movement Catalog (#279) ─────────────────────────────────────────
307    /// One dataset body by id (the merge read + the detail head).
308    pub catalog_select_dataset: String,
309    /// Upsert one dataset row (filter columns + body). Params: id, uri, kind,
310    /// last_seen, body.
311    pub catalog_upsert_dataset: String,
312    /// Every dataset body — filtering/ordering happens in shared pure code
313    /// ([`catalog::filter_datasets`](super::catalog::filter_datasets)), so the
314    /// memory and SQL backends can never disagree on semantics.
315    pub catalog_select_datasets: String,
316    /// Append one schema-timeline entry; `ON CONFLICT DO NOTHING` so a cluster
317    /// replay of the same (dataset, version) is idempotent.
318    pub catalog_insert_schema_version: String,
319    /// A dataset's schema timeline, oldest first.
320    pub catalog_select_schema_versions: String,
321    /// Upsert one lineage edge. Params: src_id, dst_id, last_seen, body.
322    pub catalog_upsert_edge: String,
323    /// Every edge body, newest activity first.
324    pub catalog_select_edges: String,
325    /// Append one volume point. Params: dataset_id, recorded_at, run_id, records.
326    pub catalog_insert_stat: String,
327    /// A dataset's most recent volume points. Params: dataset_id, limit.
328    pub catalog_select_stats: String,
329    /// Drop volume points beyond the newest `STATS_RETAIN` for one dataset.
330    /// Params: dataset_id, dataset_id, keep-limit.
331    pub catalog_prune_stats: String,
332    /// Upsert the latest config snapshot for a pipeline (#374).
333    /// Params: pipeline, recorded_at, faucet_version, body.
334    pub catalog_upsert_config_snapshot: String,
335    /// The latest config snapshot body for a pipeline. Param: pipeline.
336    pub catalog_select_config_snapshot: String,
337    // ── Pipeline templates (#444) ────────────────────────────────────────────
338    /// Highest existing version for a template id (0 when new). Param: id.
339    pub template_max_version: String,
340    /// Insert one template version. Params: id, version, name, created_at, body.
341    pub template_insert: String,
342    /// One template version's body. Params: id, version.
343    pub template_select_version: String,
344    /// The latest version's body for an id. Param: id.
345    pub template_select_latest: String,
346    /// Every template body (latest-per-id folding happens in shared pure code,
347    /// so the memory and SQL backends can never disagree).
348    pub template_select_all: String,
349    /// Version numbers for one id, newest first. Param: id.
350    pub template_versions: String,
351    /// Delete one version. Params: id, version.
352    pub template_delete_version: String,
353    /// Delete every version of an id. Param: id.
354    pub template_delete_all: String,
355    /// Upsert one channel pointer. Params: id, tag, version, updated_at.
356    pub template_upsert_tag: String,
357    /// Every channel pointer for an id. Param: id.
358    pub template_select_tags: String,
359    /// Delete one channel pointer. Params: id, tag.
360    pub template_delete_tag: String,
361    /// Delete every channel pointer for an id. Param: id.
362    pub template_delete_tags_all: String,
363    /// Delete the channel pointers aimed at one version. Params: id, version.
364    pub template_delete_tags_for_version: String,
365    /// Highest launch seq for a template (0 when never launched). Param: id.
366    pub template_max_launch_seq: String,
367    /// Append one launch entry. Params: id, seq, version, launched_at, launched_by.
368    pub template_insert_launch: String,
369    /// The launch log for a template, newest first. Param: id.
370    pub template_select_launches: String,
371    /// Delete every launch entry for a template. Param: id.
372    pub template_delete_launches_all: String,
373    /// Delete the launch entries naming one version. Params: id, version.
374    pub template_delete_launches_for_version: String,
375    /// Upsert the deprecation marker. Params: id, deprecated_at, deprecated_by, reason.
376    pub template_upsert_deprecation: String,
377    /// Read the deprecation marker. Param: id.
378    pub template_select_deprecation: String,
379    /// Clear the deprecation marker. Param: id.
380    pub template_delete_deprecation: String,
381}
382
383impl Stmts {
384    pub fn new(dialect: Dialect) -> Self {
385        match dialect {
386            Dialect::Postgres => Self::postgres(),
387            Dialect::Sqlite => Self::sqlite(),
388        }
389    }
390
391    fn postgres() -> Self {
392        Self {
393            upsert: "INSERT INTO faucet_serve_runs \
394                (run_id,name,status,submitted_at,finished_at,idempotency_key,owner,lease_expires_at,body) \
395                VALUES ($1,$2,$3,$4,$5,$6,$7,$8,$9) \
396                ON CONFLICT (run_id) DO UPDATE SET \
397                name=excluded.name,status=excluded.status,submitted_at=excluded.submitted_at,\
398                finished_at=excluded.finished_at,idempotency_key=excluded.idempotency_key,\
399                owner=excluded.owner,lease_expires_at=excluded.lease_expires_at,\
400                body=excluded.body"
401                .into(),
402            select_body: "SELECT body FROM faucet_serve_runs WHERE run_id=$1".into(),
403            select_status: "SELECT status FROM faucet_serve_runs WHERE run_id=$1".into(),
404            select_submitted: "SELECT submitted_at FROM faucet_serve_runs WHERE run_id=$1".into(),
405            delete: "DELETE FROM faucet_serve_runs WHERE run_id=$1".into(),
406            // Casts make the parameter types explicit so `$n IS NULL` cannot trip
407            // Postgres' "could not determine data type of parameter" check.
408            list: "SELECT body FROM faucet_serve_runs \
409                WHERE ($1::text IS NULL OR status = $2::text) \
410                AND ($3::text IS NULL OR name = $4::text) \
411                AND ($5::text IS NULL OR submitted_at >= $6::text) \
412                AND ($7::text IS NULL OR submitted_at <= $8::text) \
413                AND ($9::text IS NULL OR (submitted_at < $10::text \
414                    OR (submitted_at = $11::text AND run_id < $12::text))) \
415                ORDER BY submitted_at DESC, run_id DESC LIMIT $13"
416                .into(),
417            purge_runs: "DELETE FROM faucet_serve_runs \
418                WHERE status IN ('completed','failed','cancelled') \
419                AND finished_at IS NOT NULL AND finished_at < $1"
420                .into(),
421            purge_idem: "DELETE FROM faucet_serve_idem WHERE claimed_at < $1".into(),
422            select_orphans: "SELECT body FROM faucet_serve_runs \
423                WHERE status IN ('queued','running') \
424                AND (lease_expires_at IS NULL OR lease_expires_at < $1)"
425                .into(),
426            renew_leases: "UPDATE faucet_serve_runs SET lease_expires_at = $1 \
427                WHERE owner = $2 AND status IN ('queued','running')"
428                .into(),
429            insert_idem: "INSERT INTO faucet_serve_idem (key,run_id,fingerprint,claimed_at) \
430                VALUES ($1,$2,$3,$4) ON CONFLICT (key) DO NOTHING"
431                .into(),
432            select_idem: "SELECT run_id,fingerprint,claimed_at FROM faucet_serve_idem WHERE key=$1"
433                .into(),
434            takeover_idem: "UPDATE faucet_serve_idem \
435                SET run_id=$1,fingerprint=$2,claimed_at=$3 WHERE key=$4 AND claimed_at=$5"
436                .into(),
437            delete_idem_by_run: "DELETE FROM faucet_serve_idem WHERE run_id=$1".into(),
438            select_pending: "SELECT run_id, body FROM faucet_serve_runs \
439                WHERE status = 'pending' ORDER BY submitted_at ASC LIMIT $1"
440                .into(),
441            claim_one: "UPDATE faucet_serve_runs \
442                SET owner = $1, status = 'running', lease_expires_at = $2, body = $3 \
443                WHERE run_id = $4 AND status = 'pending'"
444                .into(),
445            reclaim_select: "SELECT body FROM faucet_serve_runs \
446                WHERE status = 'running' \
447                AND (lease_expires_at IS NULL OR lease_expires_at < $1)"
448                .into(),
449            // Preserve `cancel_requested` across a requeue (audit #321 M7): a
450            // cross-instance cancel acknowledged while the owner was partitioned
451            // must survive re-queueing so the next owner still honours it, rather
452            // than the run silently running to completion despite the cancel.
453            reclaim_requeue: "UPDATE faucet_serve_runs \
454                SET status = 'pending', owner = NULL, lease_expires_at = NULL, \
455                    body = $1 \
456                WHERE run_id = $2 AND status = 'running' \
457                AND (lease_expires_at IS NULL OR lease_expires_at < $3)"
458                .into(),
459            reclaim_fail: "UPDATE faucet_serve_runs \
460                SET status = 'failed', finished_at = $1, body = $2, owner = NULL \
461                WHERE run_id = $3 AND status = 'running' \
462                AND (lease_expires_at IS NULL OR lease_expires_at < $4)"
463                .into(),
464            // Status-fenced (audit #321 L5): only finalize a still-non-terminal
465            // record, so a stale zombie execution that shares the same owner (a
466            // lease-lapse re-claim by the same instance) can never overwrite the
467            // terminal record written by the live execution. First finalizer wins.
468            finalize_owned: "UPDATE faucet_serve_runs \
469                SET status = $1, finished_at = $2, lease_expires_at = $3, body = $4 \
470                WHERE run_id = $5 AND owner = $6 \
471                AND status NOT IN ('completed','failed','cancelled')"
472                .into(),
473            cancel_pending: "UPDATE faucet_serve_runs \
474                SET status = 'cancelled', finished_at = $1, body = $2 \
475                WHERE run_id = $3 AND status = 'pending'"
476                .into(),
477            request_cancel: "UPDATE faucet_serve_runs \
478                SET cancel_requested = $1 WHERE run_id = $2 AND status IN ('running','sharded')"
479                .into(),
480            pending_cancellations: "SELECT run_id FROM faucet_serve_runs \
481                WHERE status = 'running' AND owner = $1 AND cancel_requested IS NOT NULL"
482                .into(),
483            heartbeat_instance: "INSERT INTO faucet_serve_instances \
484                (instance_id, started_at, last_heartbeat, listen, max_concurrent, in_flight) \
485                VALUES ($1,$2,$3,$4,$5,$6) \
486                ON CONFLICT (instance_id) DO UPDATE SET \
487                last_heartbeat = excluded.last_heartbeat, listen = excluded.listen, \
488                max_concurrent = excluded.max_concurrent, in_flight = excluded.in_flight"
489                .into(),
490            live_instances: "SELECT instance_id, started_at, last_heartbeat, listen, \
491                max_concurrent, in_flight FROM faucet_serve_instances \
492                WHERE last_heartbeat >= $1"
493                .into(),
494            prune_instances: "DELETE FROM faucet_serve_instances WHERE last_heartbeat < $1".into(),
495            insert_shard: "INSERT INTO faucet_serve_shards \
496                (run_id, shard_id, descriptor, size_estimate, status, attempt) \
497                VALUES ($1,$2,$3,$4,'pending','0') \
498                ON CONFLICT (run_id, shard_id) DO NOTHING"
499                .into(),
500            claim_shards_select: "SELECT s.run_id, s.shard_id, s.descriptor, r.body \
501                FROM faucet_serve_shards s JOIN faucet_serve_runs r ON r.run_id = s.run_id \
502                WHERE s.status = 'pending' \
503                ORDER BY CAST(COALESCE(s.size_estimate, '0') AS BIGINT) DESC, s.run_id, s.shard_id \
504                LIMIT $1"
505                .into(),
506            claim_shard_one: "UPDATE faucet_serve_shards \
507                SET owner = $1, status = 'running', lease_expires_at = $2 \
508                WHERE run_id = $3 AND shard_id = $4 AND status = 'pending'"
509                .into(),
510            renew_shard_leases: "UPDATE faucet_serve_shards SET lease_expires_at = $1 \
511                WHERE owner = $2 AND status = 'running'"
512                .into(),
513            reclaim_shards_select: "SELECT run_id, shard_id, attempt FROM faucet_serve_shards \
514                WHERE status = 'running' \
515                AND (lease_expires_at IS NULL OR lease_expires_at < $1)"
516                .into(),
517            reclaim_shard_requeue: "UPDATE faucet_serve_shards \
518                SET status = 'pending', owner = NULL, lease_expires_at = NULL, attempt = $1 \
519                WHERE run_id = $2 AND shard_id = $3 AND status = 'running' \
520                AND (lease_expires_at IS NULL OR lease_expires_at < $4)"
521                .into(),
522            reclaim_shard_fail: "UPDATE faucet_serve_shards \
523                SET status = 'failed', finished_at = $1, owner = NULL \
524                WHERE run_id = $2 AND shard_id = $3 AND status = 'running' \
525                AND (lease_expires_at IS NULL OR lease_expires_at < $4)"
526                .into(),
527            finalize_shard: "UPDATE faucet_serve_shards \
528                SET status = $1, finished_at = $2 \
529                WHERE run_id = $3 AND shard_id = $4 AND owner = $5 AND status = 'running'"
530                .into(),
531            shard_progress: "SELECT status, COUNT(*) AS n FROM faucet_serve_shards \
532                WHERE run_id = $1 GROUP BY status"
533                .into(),
534            pending_shard_cancellations: "SELECT DISTINCT s.run_id \
535                FROM faucet_serve_shards s \
536                JOIN faucet_serve_runs r ON r.run_id = s.run_id \
537                WHERE s.owner = $1 AND s.status = 'running' \
538                AND r.cancel_requested IS NOT NULL"
539                .into(),
540            select_sharded_parents: "SELECT run_id FROM faucet_serve_runs \
541                WHERE status = 'sharded'"
542                .into(),
543            finalize_sharded_parent: "UPDATE faucet_serve_runs \
544                SET status = $1, finished_at = $2, body = $3 \
545                WHERE run_id = $4 AND status = 'sharded'"
546                .into(),
547            delete_shards_by_run: "DELETE FROM faucet_serve_shards WHERE run_id = $1".into(),
548            purge_orphan_shards: "DELETE FROM faucet_serve_shards \
549                WHERE run_id NOT IN (SELECT run_id FROM faucet_serve_runs)"
550                .into(),
551            insert_audit: "INSERT INTO faucet_serve_audit \
552                (id, ts, principal, role, action, run_id, config_fingerprint, source_ip, result) \
553                VALUES ($1,$2,$3,$4,$5,$6,$7,$8,$9)"
554                .into(),
555            list_audit: "SELECT id, ts, principal, role, action, run_id, config_fingerprint, \
556                source_ip, result FROM faucet_serve_audit \
557                WHERE ($1::text IS NULL OR principal = $2::text) \
558                AND ($3::text IS NULL OR action = $4::text) \
559                AND ($5::text IS NULL OR ts >= $6::text) \
560                AND ($7::text IS NULL OR ts <= $8::text) \
561                ORDER BY ts DESC, id DESC LIMIT $9"
562                .into(),
563            purge_audit: "DELETE FROM faucet_serve_audit WHERE ts < $1".into(),
564            insert_run_log: "INSERT INTO faucet_serve_run_logs \
565                (run_id, seq, ts, level, line) VALUES ($1,$2,$3,$4,$5) \
566                ON CONFLICT (run_id, seq) DO NOTHING"
567                .into(),
568            list_run_logs: "SELECT seq, ts, level, line FROM faucet_serve_run_logs \
569                WHERE run_id = $1 AND seq <> $2 AND ($3::text IS NULL OR seq > $4::text) \
570                ORDER BY seq ASC LIMIT $5"
571                .into(),
572            run_log_truncated: "SELECT 1 FROM faucet_serve_run_logs \
573                WHERE run_id = $1 AND seq = $2 LIMIT 1"
574                .into(),
575            purge_run_logs: "DELETE FROM faucet_serve_run_logs WHERE ts < $1".into(),
576            catalog_select_dataset: "SELECT body FROM faucet_catalog_datasets WHERE id=$1".into(),
577            catalog_upsert_dataset: "INSERT INTO faucet_catalog_datasets \
578                (id, uri, kind, last_seen, body) VALUES ($1,$2,$3,$4,$5) \
579                ON CONFLICT (id) DO UPDATE SET uri=excluded.uri, kind=excluded.kind, \
580                last_seen=excluded.last_seen, body=excluded.body"
581                .into(),
582            catalog_select_datasets: "SELECT body FROM faucet_catalog_datasets".into(),
583            catalog_insert_schema_version: "INSERT INTO faucet_catalog_schema_versions \
584                (dataset_id, version, recorded_at, body) VALUES ($1,$2,$3,$4) \
585                ON CONFLICT (dataset_id, version) DO NOTHING"
586                .into(),
587            catalog_select_schema_versions: "SELECT body FROM faucet_catalog_schema_versions \
588                WHERE dataset_id=$1 ORDER BY CAST(version AS BIGINT) ASC"
589                .into(),
590            catalog_upsert_edge: "INSERT INTO faucet_catalog_edges \
591                (src_id, dst_id, last_seen, body) VALUES ($1,$2,$3,$4) \
592                ON CONFLICT (src_id, dst_id) DO UPDATE SET \
593                last_seen=excluded.last_seen, body=excluded.body"
594                .into(),
595            catalog_select_edges: "SELECT body FROM faucet_catalog_edges \
596                ORDER BY last_seen DESC, src_id, dst_id"
597                .into(),
598            catalog_insert_stat: "INSERT INTO faucet_catalog_stats \
599                (dataset_id, recorded_at, run_id, records) VALUES ($1,$2,$3,$4) \
600                ON CONFLICT (dataset_id, recorded_at) DO NOTHING"
601                .into(),
602            catalog_select_stats: "SELECT recorded_at, run_id, records \
603                FROM faucet_catalog_stats WHERE dataset_id=$1 \
604                ORDER BY recorded_at DESC LIMIT $2"
605                .into(),
606            catalog_prune_stats: "DELETE FROM faucet_catalog_stats \
607                WHERE dataset_id=$1 AND recorded_at NOT IN (\
608                    SELECT recorded_at FROM faucet_catalog_stats WHERE dataset_id=$2 \
609                    ORDER BY recorded_at DESC LIMIT $3)"
610                .into(),
611            catalog_upsert_config_snapshot: "INSERT INTO faucet_config_snapshots \
612                (pipeline, recorded_at, faucet_version, body) VALUES ($1,$2,$3,$4) \
613                ON CONFLICT (pipeline) DO UPDATE SET recorded_at=excluded.recorded_at, \
614                faucet_version=excluded.faucet_version, body=excluded.body"
615                .into(),
616            catalog_select_config_snapshot:
617                "SELECT body FROM faucet_config_snapshots WHERE pipeline=$1".into(),
618            template_max_version: "SELECT COALESCE(MAX(CAST(version AS BIGINT)), 0) AS v \
619                FROM faucet_templates WHERE id=$1"
620                .into(),
621            template_insert: "INSERT INTO faucet_templates \
622                (id, version, name, created_at, body) VALUES ($1,$2,$3,$4,$5)"
623                .into(),
624            template_select_version:
625                "SELECT body FROM faucet_templates WHERE id=$1 AND version=$2".into(),
626            template_select_latest: "SELECT body FROM faucet_templates WHERE id=$1 \
627                ORDER BY CAST(version AS BIGINT) DESC LIMIT 1"
628                .into(),
629            template_select_all: "SELECT body FROM faucet_templates".into(),
630            template_versions: "SELECT version FROM faucet_templates WHERE id=$1 \
631                ORDER BY CAST(version AS BIGINT) DESC"
632                .into(),
633            template_delete_version: "DELETE FROM faucet_templates WHERE id=$1 AND version=$2"
634                .into(),
635            template_delete_all: "DELETE FROM faucet_templates WHERE id=$1".into(),
636            template_upsert_tag: "INSERT INTO faucet_template_tags \
637                (id, tag, version, updated_at) VALUES ($1,$2,$3,$4) \
638                ON CONFLICT (id, tag) DO UPDATE SET version=excluded.version, \
639                updated_at=excluded.updated_at"
640                .into(),
641            template_select_tags: "SELECT tag, version FROM faucet_template_tags \
642                WHERE id=$1 ORDER BY tag"
643                .into(),
644            template_delete_tag: "DELETE FROM faucet_template_tags WHERE id=$1 AND tag=$2".into(),
645            template_delete_tags_all: "DELETE FROM faucet_template_tags WHERE id=$1".into(),
646            template_delete_tags_for_version:
647                "DELETE FROM faucet_template_tags WHERE id=$1 AND version=$2".into(),
648            template_max_launch_seq: "SELECT COALESCE(MAX(CAST(seq AS BIGINT)), 0) AS v \
649                FROM faucet_template_launches WHERE id=$1"
650                .into(),
651            template_insert_launch: "INSERT INTO faucet_template_launches \
652                (id, seq, version, launched_at, launched_by) VALUES ($1,$2,$3,$4,$5)"
653                .into(),
654            template_select_launches: "SELECT seq, version, launched_at, launched_by \
655                FROM faucet_template_launches WHERE id=$1 ORDER BY CAST(seq AS BIGINT) DESC"
656                .into(),
657            template_delete_launches_all: "DELETE FROM faucet_template_launches WHERE id=$1".into(),
658            template_delete_launches_for_version:
659                "DELETE FROM faucet_template_launches WHERE id=$1 AND version=$2".into(),
660            template_upsert_deprecation: "INSERT INTO faucet_template_deprecations \
661                (id, deprecated_at, deprecated_by, reason) VALUES ($1,$2,$3,$4) \
662                ON CONFLICT (id) DO UPDATE SET deprecated_at=excluded.deprecated_at, \
663                deprecated_by=excluded.deprecated_by, reason=excluded.reason"
664                .into(),
665            template_select_deprecation: "SELECT deprecated_at, deprecated_by, reason \
666                FROM faucet_template_deprecations WHERE id=$1"
667                .into(),
668            template_delete_deprecation: "DELETE FROM faucet_template_deprecations WHERE id=$1"
669                .into(),
670        }
671    }
672
673    fn sqlite() -> Self {
674        Self {
675            upsert: "INSERT INTO faucet_serve_runs \
676                (run_id,name,status,submitted_at,finished_at,idempotency_key,owner,lease_expires_at,body) \
677                VALUES (?,?,?,?,?,?,?,?,?) \
678                ON CONFLICT (run_id) DO UPDATE SET \
679                name=excluded.name,status=excluded.status,submitted_at=excluded.submitted_at,\
680                finished_at=excluded.finished_at,idempotency_key=excluded.idempotency_key,\
681                owner=excluded.owner,lease_expires_at=excluded.lease_expires_at,\
682                body=excluded.body"
683                .into(),
684            select_body: "SELECT body FROM faucet_serve_runs WHERE run_id=?".into(),
685            select_status: "SELECT status FROM faucet_serve_runs WHERE run_id=?".into(),
686            select_submitted: "SELECT submitted_at FROM faucet_serve_runs WHERE run_id=?".into(),
687            delete: "DELETE FROM faucet_serve_runs WHERE run_id=?".into(),
688            list: "SELECT body FROM faucet_serve_runs \
689                WHERE (? IS NULL OR status = ?) \
690                AND (? IS NULL OR name = ?) \
691                AND (? IS NULL OR submitted_at >= ?) \
692                AND (? IS NULL OR submitted_at <= ?) \
693                AND (? IS NULL OR (submitted_at < ? \
694                    OR (submitted_at = ? AND run_id < ?))) \
695                ORDER BY submitted_at DESC, run_id DESC LIMIT ?"
696                .into(),
697            purge_runs: "DELETE FROM faucet_serve_runs \
698                WHERE status IN ('completed','failed','cancelled') \
699                AND finished_at IS NOT NULL AND finished_at < ?"
700                .into(),
701            purge_idem: "DELETE FROM faucet_serve_idem WHERE claimed_at < ?".into(),
702            select_orphans: "SELECT body FROM faucet_serve_runs \
703                WHERE status IN ('queued','running') \
704                AND (lease_expires_at IS NULL OR lease_expires_at < ?)"
705                .into(),
706            renew_leases: "UPDATE faucet_serve_runs SET lease_expires_at = ? \
707                WHERE owner = ? AND status IN ('queued','running')"
708                .into(),
709            insert_idem: "INSERT INTO faucet_serve_idem (key,run_id,fingerprint,claimed_at) \
710                VALUES (?,?,?,?) ON CONFLICT (key) DO NOTHING"
711                .into(),
712            select_idem: "SELECT run_id,fingerprint,claimed_at FROM faucet_serve_idem WHERE key=?"
713                .into(),
714            takeover_idem: "UPDATE faucet_serve_idem \
715                SET run_id=?,fingerprint=?,claimed_at=? WHERE key=? AND claimed_at=?"
716                .into(),
717            delete_idem_by_run: "DELETE FROM faucet_serve_idem WHERE run_id=?".into(),
718            select_pending: "SELECT run_id, body FROM faucet_serve_runs \
719                WHERE status = 'pending' ORDER BY submitted_at ASC LIMIT ?"
720                .into(),
721            claim_one: "UPDATE faucet_serve_runs \
722                SET owner = ?, status = 'running', lease_expires_at = ?, body = ? \
723                WHERE run_id = ? AND status = 'pending'"
724                .into(),
725            reclaim_select: "SELECT body FROM faucet_serve_runs \
726                WHERE status = 'running' \
727                AND (lease_expires_at IS NULL OR lease_expires_at < ?)"
728                .into(),
729            // Preserve `cancel_requested` across a requeue (audit #321 M7).
730            reclaim_requeue: "UPDATE faucet_serve_runs \
731                SET status = 'pending', owner = NULL, lease_expires_at = NULL, \
732                    body = ? \
733                WHERE run_id = ? AND status = 'running' \
734                AND (lease_expires_at IS NULL OR lease_expires_at < ?)"
735                .into(),
736            reclaim_fail: "UPDATE faucet_serve_runs \
737                SET status = 'failed', finished_at = ?, body = ?, owner = NULL \
738                WHERE run_id = ? AND status = 'running' \
739                AND (lease_expires_at IS NULL OR lease_expires_at < ?)"
740                .into(),
741            // Status-fenced (audit #321 L5): first finalizer wins.
742            finalize_owned: "UPDATE faucet_serve_runs \
743                SET status = ?, finished_at = ?, lease_expires_at = ?, body = ? \
744                WHERE run_id = ? AND owner = ? \
745                AND status NOT IN ('completed','failed','cancelled')"
746                .into(),
747            cancel_pending: "UPDATE faucet_serve_runs \
748                SET status = 'cancelled', finished_at = ?, body = ? \
749                WHERE run_id = ? AND status = 'pending'"
750                .into(),
751            request_cancel: "UPDATE faucet_serve_runs \
752                SET cancel_requested = ? WHERE run_id = ? AND status IN ('running','sharded')"
753                .into(),
754            pending_cancellations: "SELECT run_id FROM faucet_serve_runs \
755                WHERE status = 'running' AND owner = ? AND cancel_requested IS NOT NULL"
756                .into(),
757            heartbeat_instance: "INSERT INTO faucet_serve_instances \
758                (instance_id, started_at, last_heartbeat, listen, max_concurrent, in_flight) \
759                VALUES (?,?,?,?,?,?) \
760                ON CONFLICT (instance_id) DO UPDATE SET \
761                last_heartbeat = excluded.last_heartbeat, listen = excluded.listen, \
762                max_concurrent = excluded.max_concurrent, in_flight = excluded.in_flight"
763                .into(),
764            live_instances: "SELECT instance_id, started_at, last_heartbeat, listen, \
765                max_concurrent, in_flight FROM faucet_serve_instances \
766                WHERE last_heartbeat >= ?"
767                .into(),
768            prune_instances: "DELETE FROM faucet_serve_instances WHERE last_heartbeat < ?".into(),
769            insert_shard: "INSERT INTO faucet_serve_shards \
770                (run_id, shard_id, descriptor, size_estimate, status, attempt) \
771                VALUES (?,?,?,?,'pending','0') \
772                ON CONFLICT (run_id, shard_id) DO NOTHING"
773                .into(),
774            claim_shards_select: "SELECT s.run_id, s.shard_id, s.descriptor, r.body \
775                FROM faucet_serve_shards s JOIN faucet_serve_runs r ON r.run_id = s.run_id \
776                WHERE s.status = 'pending' \
777                ORDER BY CAST(COALESCE(s.size_estimate, '0') AS INTEGER) DESC, s.run_id, s.shard_id \
778                LIMIT ?"
779                .into(),
780            claim_shard_one: "UPDATE faucet_serve_shards \
781                SET owner = ?, status = 'running', lease_expires_at = ? \
782                WHERE run_id = ? AND shard_id = ? AND status = 'pending'"
783                .into(),
784            renew_shard_leases: "UPDATE faucet_serve_shards SET lease_expires_at = ? \
785                WHERE owner = ? AND status = 'running'"
786                .into(),
787            reclaim_shards_select: "SELECT run_id, shard_id, attempt FROM faucet_serve_shards \
788                WHERE status = 'running' \
789                AND (lease_expires_at IS NULL OR lease_expires_at < ?)"
790                .into(),
791            reclaim_shard_requeue: "UPDATE faucet_serve_shards \
792                SET status = 'pending', owner = NULL, lease_expires_at = NULL, attempt = ? \
793                WHERE run_id = ? AND shard_id = ? AND status = 'running' \
794                AND (lease_expires_at IS NULL OR lease_expires_at < ?)"
795                .into(),
796            reclaim_shard_fail: "UPDATE faucet_serve_shards \
797                SET status = 'failed', finished_at = ?, owner = NULL \
798                WHERE run_id = ? AND shard_id = ? AND status = 'running' \
799                AND (lease_expires_at IS NULL OR lease_expires_at < ?)"
800                .into(),
801            finalize_shard: "UPDATE faucet_serve_shards \
802                SET status = ?, finished_at = ? \
803                WHERE run_id = ? AND shard_id = ? AND owner = ? AND status = 'running'"
804                .into(),
805            shard_progress: "SELECT status, COUNT(*) AS n FROM faucet_serve_shards \
806                WHERE run_id = ? GROUP BY status"
807                .into(),
808            pending_shard_cancellations: "SELECT DISTINCT s.run_id \
809                FROM faucet_serve_shards s \
810                JOIN faucet_serve_runs r ON r.run_id = s.run_id \
811                WHERE s.owner = ? AND s.status = 'running' \
812                AND r.cancel_requested IS NOT NULL"
813                .into(),
814            select_sharded_parents: "SELECT run_id FROM faucet_serve_runs \
815                WHERE status = 'sharded'"
816                .into(),
817            finalize_sharded_parent: "UPDATE faucet_serve_runs \
818                SET status = ?, finished_at = ?, body = ? \
819                WHERE run_id = ? AND status = 'sharded'"
820                .into(),
821            delete_shards_by_run: "DELETE FROM faucet_serve_shards WHERE run_id = ?".into(),
822            purge_orphan_shards: "DELETE FROM faucet_serve_shards \
823                WHERE run_id NOT IN (SELECT run_id FROM faucet_serve_runs)"
824                .into(),
825            insert_audit: "INSERT INTO faucet_serve_audit \
826                (id, ts, principal, role, action, run_id, config_fingerprint, source_ip, result) \
827                VALUES (?,?,?,?,?,?,?,?,?)"
828                .into(),
829            list_audit: "SELECT id, ts, principal, role, action, run_id, config_fingerprint, \
830                source_ip, result FROM faucet_serve_audit \
831                WHERE (? IS NULL OR principal = ?) \
832                AND (? IS NULL OR action = ?) \
833                AND (? IS NULL OR ts >= ?) \
834                AND (? IS NULL OR ts <= ?) \
835                ORDER BY ts DESC, id DESC LIMIT ?"
836                .into(),
837            purge_audit: "DELETE FROM faucet_serve_audit WHERE ts < ?".into(),
838            insert_run_log: "INSERT INTO faucet_serve_run_logs \
839                (run_id, seq, ts, level, line) VALUES (?,?,?,?,?) \
840                ON CONFLICT (run_id, seq) DO NOTHING"
841                .into(),
842            list_run_logs: "SELECT seq, ts, level, line FROM faucet_serve_run_logs \
843                WHERE run_id = ? AND seq <> ? AND (? IS NULL OR seq > ?) \
844                ORDER BY seq ASC LIMIT ?"
845                .into(),
846            run_log_truncated: "SELECT 1 FROM faucet_serve_run_logs \
847                WHERE run_id = ? AND seq = ? LIMIT 1"
848                .into(),
849            purge_run_logs: "DELETE FROM faucet_serve_run_logs WHERE ts < ?".into(),
850            catalog_select_dataset: "SELECT body FROM faucet_catalog_datasets WHERE id=?".into(),
851            catalog_upsert_dataset: "INSERT INTO faucet_catalog_datasets \
852                (id, uri, kind, last_seen, body) VALUES (?,?,?,?,?) \
853                ON CONFLICT (id) DO UPDATE SET uri=excluded.uri, kind=excluded.kind, \
854                last_seen=excluded.last_seen, body=excluded.body"
855                .into(),
856            catalog_select_datasets: "SELECT body FROM faucet_catalog_datasets".into(),
857            catalog_insert_schema_version: "INSERT INTO faucet_catalog_schema_versions \
858                (dataset_id, version, recorded_at, body) VALUES (?,?,?,?) \
859                ON CONFLICT (dataset_id, version) DO NOTHING"
860                .into(),
861            catalog_select_schema_versions: "SELECT body FROM faucet_catalog_schema_versions \
862                WHERE dataset_id=? ORDER BY CAST(version AS INTEGER) ASC"
863                .into(),
864            catalog_upsert_edge: "INSERT INTO faucet_catalog_edges \
865                (src_id, dst_id, last_seen, body) VALUES (?,?,?,?) \
866                ON CONFLICT (src_id, dst_id) DO UPDATE SET \
867                last_seen=excluded.last_seen, body=excluded.body"
868                .into(),
869            catalog_select_edges: "SELECT body FROM faucet_catalog_edges \
870                ORDER BY last_seen DESC, src_id, dst_id"
871                .into(),
872            catalog_insert_stat: "INSERT INTO faucet_catalog_stats \
873                (dataset_id, recorded_at, run_id, records) VALUES (?,?,?,?) \
874                ON CONFLICT (dataset_id, recorded_at) DO NOTHING"
875                .into(),
876            catalog_select_stats: "SELECT recorded_at, run_id, records \
877                FROM faucet_catalog_stats WHERE dataset_id=? \
878                ORDER BY recorded_at DESC LIMIT ?"
879                .into(),
880            catalog_prune_stats: "DELETE FROM faucet_catalog_stats \
881                WHERE dataset_id=? AND recorded_at NOT IN (\
882                    SELECT recorded_at FROM faucet_catalog_stats WHERE dataset_id=? \
883                    ORDER BY recorded_at DESC LIMIT ?)"
884                .into(),
885            catalog_upsert_config_snapshot: "INSERT INTO faucet_config_snapshots \
886                (pipeline, recorded_at, faucet_version, body) VALUES (?,?,?,?) \
887                ON CONFLICT (pipeline) DO UPDATE SET recorded_at=excluded.recorded_at, \
888                faucet_version=excluded.faucet_version, body=excluded.body"
889                .into(),
890            catalog_select_config_snapshot:
891                "SELECT body FROM faucet_config_snapshots WHERE pipeline=?".into(),
892            template_max_version: "SELECT COALESCE(MAX(CAST(version AS INTEGER)), 0) AS v \
893                FROM faucet_templates WHERE id=?"
894                .into(),
895            template_insert: "INSERT INTO faucet_templates \
896                (id, version, name, created_at, body) VALUES (?,?,?,?,?)"
897                .into(),
898            template_select_version: "SELECT body FROM faucet_templates WHERE id=? AND version=?"
899                .into(),
900            template_select_latest: "SELECT body FROM faucet_templates WHERE id=? \
901                ORDER BY CAST(version AS INTEGER) DESC LIMIT 1"
902                .into(),
903            template_select_all: "SELECT body FROM faucet_templates".into(),
904            template_versions: "SELECT version FROM faucet_templates WHERE id=? \
905                ORDER BY CAST(version AS INTEGER) DESC"
906                .into(),
907            template_delete_version: "DELETE FROM faucet_templates WHERE id=? AND version=?".into(),
908            template_delete_all: "DELETE FROM faucet_templates WHERE id=?".into(),
909            template_upsert_tag: "INSERT INTO faucet_template_tags \
910                (id, tag, version, updated_at) VALUES (?,?,?,?) \
911                ON CONFLICT (id, tag) DO UPDATE SET version=excluded.version, \
912                updated_at=excluded.updated_at"
913                .into(),
914            template_select_tags: "SELECT tag, version FROM faucet_template_tags \
915                WHERE id=? ORDER BY tag"
916                .into(),
917            template_delete_tag: "DELETE FROM faucet_template_tags WHERE id=? AND tag=?".into(),
918            template_delete_tags_all: "DELETE FROM faucet_template_tags WHERE id=?".into(),
919            template_delete_tags_for_version:
920                "DELETE FROM faucet_template_tags WHERE id=? AND version=?".into(),
921            template_max_launch_seq: "SELECT COALESCE(MAX(CAST(seq AS INTEGER)), 0) AS v \
922                FROM faucet_template_launches WHERE id=?"
923                .into(),
924            template_insert_launch: "INSERT INTO faucet_template_launches \
925                (id, seq, version, launched_at, launched_by) VALUES (?,?,?,?,?)"
926                .into(),
927            template_select_launches: "SELECT seq, version, launched_at, launched_by \
928                FROM faucet_template_launches WHERE id=? ORDER BY CAST(seq AS INTEGER) DESC"
929                .into(),
930            template_delete_launches_all: "DELETE FROM faucet_template_launches WHERE id=?".into(),
931            template_delete_launches_for_version:
932                "DELETE FROM faucet_template_launches WHERE id=? AND version=?".into(),
933            template_upsert_deprecation: "INSERT INTO faucet_template_deprecations \
934                (id, deprecated_at, deprecated_by, reason) VALUES (?,?,?,?) \
935                ON CONFLICT (id) DO UPDATE SET deprecated_at=excluded.deprecated_at, \
936                deprecated_by=excluded.deprecated_by, reason=excluded.reason"
937                .into(),
938            template_select_deprecation: "SELECT deprecated_at, deprecated_by, reason \
939                FROM faucet_template_deprecations WHERE id=?"
940                .into(),
941            template_delete_deprecation: "DELETE FROM faucet_template_deprecations WHERE id=?"
942                .into(),
943        }
944    }
945}
946
947/// Bounded retry count for the atomic idempotency claim (handles a claim being
948/// purged concurrently between the insert attempt and the read-back) and for the
949/// read-max-then-insert paths (template versions, launch-log seqs).
950///
951/// Sized for *contention*, not just for a lost race. SQLite serializes writers
952/// and answers a write-write overlap on a deferred transaction with `database is
953/// locked` **immediately** (waiting would deadlock), so `busy_timeout` does not
954/// help there and the attempt budget is the only thing standing between a
955/// concurrent writer and a surfaced error. With N concurrent writers each needing
956/// its own turn, a budget of 4 is thinner than it looks: six writers on a loaded
957/// machine exhausted it and failed a register (#457 CI).
958pub const CLAIM_ATTEMPTS: usize = 8;
959
960/// Distinguishes concurrent retriers so their backoffs do not re-collide.
961///
962/// Every waiter sleeping the *same* duration just reproduces the same race one
963/// beat later. A per-call sequence number is a dependency-free way to stagger
964/// them (the alternative, a random jitter, would mean pulling in an RNG here).
965static RETRY_SEQ: std::sync::atomic::AtomicUsize = std::sync::atomic::AtomicUsize::new(0);
966
967/// Sleep before a read-max-then-insert retry (no-op on the first attempt).
968///
969/// Exponential (5ms, 10ms, 20ms, …, capped) plus a per-caller stagger, so a set
970/// of writers that collided on attempt 1 spreads out instead of colliding again.
971/// Worst case across the whole budget is a few hundred milliseconds — cheap next
972/// to failing a write the caller expected to succeed.
973pub async fn retry_backoff(attempt: usize) {
974    if attempt <= 1 {
975        return;
976    }
977    const BASE_MS: u64 = 5;
978    const CAP_MS: u64 = 160;
979    let exp = BASE_MS
980        .saturating_mul(1u64 << (attempt - 2).min(6))
981        .min(CAP_MS);
982    // 0..BASE_MS of per-caller offset, so equal-length sleeps do not re-align.
983    let stagger =
984        (RETRY_SEQ.fetch_add(1, std::sync::atomic::Ordering::Relaxed) as u64) % (BASE_MS + 1);
985    tokio::time::sleep(std::time::Duration::from_millis(exp + stagger)).await;
986}
987
988/// Fixed-width RFC3339 (nanoseconds + `Z`) — lexicographically sortable.
989pub fn fmt_ts(dt: DateTime<Utc>) -> String {
990    dt.to_rfc3339_opts(chrono::SecondsFormat::Nanos, true)
991}
992
993/// Zero-pad a run-log sequence to a fixed 20 digits (the width of `u64::MAX`) so
994/// it sorts lexically = numerically as a TEXT column (#529).
995pub fn pad_seq(seq: u64) -> String {
996    format!("{seq:020}")
997}
998
999/// Inverse of [`pad_seq`]; a malformed value falls back to 0.
1000pub fn unpad_seq(raw: &str) -> u64 {
1001    raw.trim_start_matches('0').parse().unwrap_or(0)
1002}
1003
1004/// Inverse of [`fmt_ts`]. An unparseable value falls back to *now* rather than
1005/// failing the read: a single corrupt timestamp must not make a whole template or
1006/// audit row unusable, and every caller uses the value for display/ordering only.
1007pub fn parse_ts(raw: &str) -> DateTime<Utc> {
1008    DateTime::parse_from_rfc3339(raw)
1009        .map(|d| d.to_utc())
1010        .unwrap_or_else(|_| Utc::now())
1011}
1012
1013/// True when a claim timestamped `claimed_at` (RFC3339) is older than `window`.
1014/// An unparseable or future timestamp is treated as **not** expired (safe: it
1015/// won't be silently re-claimed).
1016pub fn is_expired(claimed_at: &str, now: DateTime<Utc>, window: Duration) -> bool {
1017    match DateTime::parse_from_rfc3339(claimed_at) {
1018        Ok(t) => now
1019            .signed_duration_since(t.with_timezone(&Utc))
1020            .to_std()
1021            .map(|age| age >= window)
1022            .unwrap_or(false),
1023        Err(_) => false,
1024    }
1025}
1026
1027/// RFC3339 timestamp `window` before `now` (the purge / expiry threshold).
1028pub fn threshold(now: DateTime<Utc>, window: Duration) -> String {
1029    let delta =
1030        chrono::Duration::from_std(window).unwrap_or_else(|_| chrono::Duration::days(36_500));
1031    fmt_ts(now - delta)
1032}
1033
1034pub fn encode_body(rec: &RunRecord) -> Result<String, HistoryError> {
1035    serde_json::to_string(rec).map_err(|e| HistoryError::Backend(format!("encode run record: {e}")))
1036}
1037
1038pub fn decode_body(body: &str) -> Result<RunRecord, HistoryError> {
1039    serde_json::from_str(body).map_err(|e| HistoryError::Backend(format!("decode run record: {e}")))
1040}
1041
1042/// Generic body (de)serialization for the catalog tables (#279).
1043pub fn encode_json<T: serde::Serialize>(value: &T, what: &str) -> Result<String, HistoryError> {
1044    serde_json::to_string(value).map_err(|e| HistoryError::Backend(format!("encode {what}: {e}")))
1045}
1046
1047pub fn decode_json<T: serde::de::DeserializeOwned>(
1048    body: &str,
1049    what: &str,
1050) -> Result<T, HistoryError> {
1051    serde_json::from_str(body).map_err(|e| HistoryError::Backend(format!("decode {what}: {e}")))
1052}
1053
1054pub fn parse_status(s: &str) -> RunStatus {
1055    match s {
1056        "queued" => RunStatus::Queued,
1057        "pending" => RunStatus::Pending,
1058        "running" => RunStatus::Running,
1059        "sharded" => RunStatus::Sharded,
1060        "completed" => RunStatus::Completed,
1061        "cancelled" => RunStatus::Cancelled,
1062        _ => RunStatus::Failed,
1063    }
1064}
1065
1066/// Generate a concrete `RunHistory` implementation over a specific `sqlx` pool.
1067/// `$name` is the backend struct, `$pool` its `sqlx` pool type. The struct holds
1068/// the pool, the idempotency retention window, and the dialect's [`Stmts`].
1069macro_rules! impl_sql_history {
1070    ($name:ident, $pool:ty) => {
1071        /// SQL-backed [`RunHistory`](crate::serve::history::RunHistory). See
1072        /// [`crate::serve::history::sql`] for the shared schema + semantics.
1073        pub struct $name {
1074            pool: $pool,
1075            idem_retention: std::time::Duration,
1076            /// This serve instance's id, stamped as `owner` on every upsert.
1077            instance_id: String,
1078            /// How far ahead each upsert / heartbeat pushes a run's lease.
1079            lease_ttl: std::time::Duration,
1080            stmts: $crate::serve::history::sql::Stmts,
1081        }
1082
1083        impl $name {
1084            /// Assemble from an already-connected pool (used by `connect`).
1085            pub fn from_parts(
1086                pool: $pool,
1087                idem_retention: std::time::Duration,
1088                lease_ttl: std::time::Duration,
1089                instance_id: String,
1090                stmts: $crate::serve::history::sql::Stmts,
1091            ) -> Self {
1092                Self {
1093                    pool,
1094                    idem_retention,
1095                    instance_id,
1096                    lease_ttl,
1097                    stmts,
1098                }
1099            }
1100
1101            /// Borrow the underlying pool (tests close it to exercise fallback).
1102            pub fn pool(&self) -> &$pool {
1103                &self.pool
1104            }
1105        }
1106
1107        #[async_trait::async_trait]
1108        impl $crate::serve::history::RunHistory for $name {
1109            async fn claim_idempotency(
1110                &self,
1111                key: &str,
1112                fingerprint: &str,
1113                run_id: &str,
1114                window: std::time::Duration,
1115            ) -> Result<$crate::serve::history::Claim, $crate::serve::history::HistoryError> {
1116                use sqlx::Row as _;
1117                use $crate::serve::history::Claim;
1118                use $crate::serve::history::HistoryError;
1119                use $crate::serve::history::sql;
1120
1121                let now = chrono::Utc::now();
1122                let now_s = sql::fmt_ts(now);
1123                let backend = |e: sqlx::Error| HistoryError::Backend(e.to_string());
1124
1125                for _ in 0..sql::CLAIM_ATTEMPTS {
1126                    // 1) Atomic first-claim: the winner inserts exactly one row.
1127                    let inserted = sqlx::query(&self.stmts.insert_idem)
1128                        .bind(key)
1129                        .bind(run_id)
1130                        .bind(fingerprint)
1131                        .bind(&now_s)
1132                        .execute(&self.pool)
1133                        .await
1134                        .map_err(backend)?
1135                        .rows_affected();
1136                    if inserted == 1 {
1137                        return Ok(Claim::Fresh);
1138                    }
1139                    // 2) Conflict: inspect the existing claim.
1140                    let Some(row) = sqlx::query(&self.stmts.select_idem)
1141                        .bind(key)
1142                        .fetch_optional(&self.pool)
1143                        .await
1144                        .map_err(backend)?
1145                    else {
1146                        // Vanished between the insert and the read — retry.
1147                        continue;
1148                    };
1149                    let existing_run: String = row.try_get("run_id").map_err(backend)?;
1150                    let existing_fp: String = row.try_get("fingerprint").map_err(backend)?;
1151                    let claimed_at: String = row.try_get("claimed_at").map_err(backend)?;
1152
1153                    if sql::is_expired(&claimed_at, now, window) {
1154                        // 3) Optimistic, expiry-guarded takeover: only the request
1155                        // that still sees `claimed_at` succeeds.
1156                        let took = sqlx::query(&self.stmts.takeover_idem)
1157                            .bind(run_id)
1158                            .bind(fingerprint)
1159                            .bind(&now_s)
1160                            .bind(key)
1161                            .bind(&claimed_at)
1162                            .execute(&self.pool)
1163                            .await
1164                            .map_err(backend)?
1165                            .rows_affected();
1166                        if took == 1 {
1167                            return Ok(Claim::Fresh);
1168                        }
1169                        continue; // lost the race; re-evaluate
1170                    }
1171                    return Ok(if existing_fp == fingerprint {
1172                        Claim::Replay(existing_run)
1173                    } else {
1174                        Claim::Conflict
1175                    });
1176                }
1177                // Pathological contention only. Conservative: a 409 is safer than
1178                // risking a duplicate run.
1179                tracing::warn!(
1180                    key,
1181                    "idempotency claim exhausted retries; reporting conflict"
1182                );
1183                Ok(Claim::Conflict)
1184            }
1185
1186            async fn upsert(
1187                &self,
1188                rec: &$crate::serve::history::RunRecord,
1189            ) -> Result<(), $crate::serve::history::HistoryError> {
1190                use $crate::serve::history::HistoryError;
1191                use $crate::serve::history::sql;
1192                let body = sql::encode_body(rec)?;
1193                let submitted = sql::fmt_ts(rec.submitted_at);
1194                let finished = rec.finished_at.map(sql::fmt_ts);
1195                // Stamp this instance as the owner and start/renew the lease.
1196                // The owner/lease are SQL-column-only (never in the record body),
1197                // so the heartbeat can extend a lease without a body read-modify-
1198                // write race (#146 H7).
1199                let lease = sql::fmt_ts(chrono::Utc::now() + self.lease_ttl);
1200                sqlx::query(&self.stmts.upsert)
1201                    .bind(&rec.run_id)
1202                    .bind(rec.name.as_deref())
1203                    .bind(rec.status.as_str())
1204                    .bind(&submitted)
1205                    .bind(finished.as_deref())
1206                    .bind(rec.idempotency_key.as_deref())
1207                    .bind(&self.instance_id)
1208                    .bind(&lease)
1209                    .bind(&body)
1210                    .execute(&self.pool)
1211                    .await
1212                    .map_err(|e| HistoryError::Backend(e.to_string()))?;
1213                Ok(())
1214            }
1215
1216            async fn get(
1217                &self,
1218                id: &str,
1219            ) -> Result<
1220                Option<$crate::serve::history::RunRecord>,
1221                $crate::serve::history::HistoryError,
1222            > {
1223                use sqlx::Row as _;
1224                use $crate::serve::history::HistoryError;
1225                use $crate::serve::history::sql;
1226                let row = sqlx::query(&self.stmts.select_body)
1227                    .bind(id)
1228                    .fetch_optional(&self.pool)
1229                    .await
1230                    .map_err(|e| HistoryError::Backend(e.to_string()))?;
1231                match row {
1232                    None => Ok(None),
1233                    Some(r) => {
1234                        let body: String = r
1235                            .try_get("body")
1236                            .map_err(|e| HistoryError::Backend(e.to_string()))?;
1237                        Ok(Some(sql::decode_body(&body)?))
1238                    }
1239                }
1240            }
1241
1242            async fn list(
1243                &self,
1244                filter: &$crate::serve::history::ListFilter,
1245            ) -> Result<$crate::serve::history::ListPage, $crate::serve::history::HistoryError>
1246            {
1247                use sqlx::Row as _;
1248                use $crate::serve::history::HistoryError;
1249                use $crate::serve::history::ListPage;
1250                use $crate::serve::history::sql;
1251                let backend = |e: sqlx::Error| HistoryError::Backend(e.to_string());
1252
1253                // Resolve the cursor's submitted_at for keyset pagination. An
1254                // unknown cursor is ignored (page starts from the top), matching
1255                // the memory backend.
1256                let cursor_ts: Option<String> = match &filter.cursor {
1257                    None => None,
1258                    Some(c) => sqlx::query(&self.stmts.select_submitted)
1259                        .bind(c)
1260                        .fetch_optional(&self.pool)
1261                        .await
1262                        .map_err(backend)?
1263                        .map(|r| r.try_get::<String, _>("submitted_at"))
1264                        .transpose()
1265                        .map_err(backend)?,
1266                };
1267                let cur_id = if cursor_ts.is_some() {
1268                    filter.cursor.as_deref()
1269                } else {
1270                    None
1271                };
1272
1273                let status_s = filter.status.map(|s| s.as_str());
1274                let name_s = filter.name.as_deref();
1275                let since_s = filter.since.map(sql::fmt_ts);
1276                let until_s = filter.until.map(sql::fmt_ts);
1277                let limit = filter.limit.max(1);
1278                let fetch_n = limit as i64 + 1; // +1 to detect a next page
1279
1280                let rows = sqlx::query(&self.stmts.list)
1281                    .bind(status_s)
1282                    .bind(status_s)
1283                    .bind(name_s)
1284                    .bind(name_s)
1285                    .bind(since_s.as_deref())
1286                    .bind(since_s.as_deref())
1287                    .bind(until_s.as_deref())
1288                    .bind(until_s.as_deref())
1289                    .bind(cursor_ts.as_deref())
1290                    .bind(cursor_ts.as_deref())
1291                    .bind(cursor_ts.as_deref())
1292                    .bind(cur_id)
1293                    .bind(fetch_n)
1294                    .fetch_all(&self.pool)
1295                    .await
1296                    .map_err(backend)?;
1297
1298                let mut runs = Vec::with_capacity(rows.len());
1299                for r in &rows {
1300                    let body: String = r.try_get("body").map_err(backend)?;
1301                    runs.push(sql::decode_body(&body)?);
1302                }
1303                let next_cursor = if runs.len() > limit {
1304                    Some(runs[limit - 1].run_id.clone())
1305                } else {
1306                    None
1307                };
1308                runs.truncate(limit);
1309                Ok(ListPage { runs, next_cursor })
1310            }
1311
1312            async fn delete(
1313                &self,
1314                id: &str,
1315            ) -> Result<$crate::serve::history::DeleteOutcome, $crate::serve::history::HistoryError>
1316            {
1317                use sqlx::Row as _;
1318                use $crate::serve::history::DeleteOutcome;
1319                use $crate::serve::history::HistoryError;
1320                use $crate::serve::history::sql;
1321                let backend = |e: sqlx::Error| HistoryError::Backend(e.to_string());
1322                let status: Option<String> = sqlx::query(&self.stmts.select_status)
1323                    .bind(id)
1324                    .fetch_optional(&self.pool)
1325                    .await
1326                    .map_err(backend)?
1327                    .map(|r| r.try_get::<String, _>("status"))
1328                    .transpose()
1329                    .map_err(backend)?;
1330                match status {
1331                    None => Ok(DeleteOutcome::NotFound),
1332                    Some(s) if !sql::parse_status(&s).is_terminal() => {
1333                        Ok(DeleteOutcome::StillRunning)
1334                    }
1335                    Some(_) => {
1336                        sqlx::query(&self.stmts.delete)
1337                            .bind(id)
1338                            .execute(&self.pool)
1339                            .await
1340                            .map_err(backend)?;
1341                        // Drop the run's idempotency claim too, so a replay of
1342                        // the key starts fresh instead of 404-ing on the deleted
1343                        // record until the claim self-expires (#146 M8). Scoped
1344                        // by run_id, so a newer run that re-claimed the same key
1345                        // keeps its claim.
1346                        sqlx::query(&self.stmts.delete_idem_by_run)
1347                            .bind(id)
1348                            .execute(&self.pool)
1349                            .await
1350                            .map_err(backend)?;
1351                        // Drop the run's shard rows too (Mode B, #230), so a
1352                        // deleted run leaves no orphaned shard rows that would
1353                        // otherwise leak unboundedly (F25).
1354                        sqlx::query(&self.stmts.delete_shards_by_run)
1355                            .bind(id)
1356                            .execute(&self.pool)
1357                            .await
1358                            .map_err(backend)?;
1359                        Ok(DeleteOutcome::Deleted)
1360                    }
1361                }
1362            }
1363
1364            async fn release_idempotency(
1365                &self,
1366                run_id: &str,
1367            ) -> Result<(), $crate::serve::history::HistoryError> {
1368                use $crate::serve::history::HistoryError;
1369                sqlx::query(&self.stmts.delete_idem_by_run)
1370                    .bind(run_id)
1371                    .execute(&self.pool)
1372                    .await
1373                    .map_err(|e| HistoryError::Backend(e.to_string()))?;
1374                Ok(())
1375            }
1376
1377            async fn purge_expired(
1378                &self,
1379                retain_for: std::time::Duration,
1380            ) -> Result<usize, $crate::serve::history::HistoryError> {
1381                use $crate::serve::history::HistoryError;
1382                use $crate::serve::history::sql;
1383                let backend = |e: sqlx::Error| HistoryError::Backend(e.to_string());
1384                let now = chrono::Utc::now();
1385                let removed = sqlx::query(&self.stmts.purge_runs)
1386                    .bind(sql::threshold(now, retain_for))
1387                    .execute(&self.pool)
1388                    .await
1389                    .map_err(backend)?
1390                    .rows_affected() as usize;
1391                // Drop expired idempotency claims too (best-effort).
1392                let _ = sqlx::query(&self.stmts.purge_idem)
1393                    .bind(sql::threshold(now, self.idem_retention))
1394                    .execute(&self.pool)
1395                    .await;
1396                // Drop membership rows that have not heartbeated within the
1397                // run-retention window (far longer than the lease, so this never
1398                // prunes a live member — that's `live_instances(ttl)`'s job).
1399                let _ = sqlx::query(&self.stmts.prune_instances)
1400                    .bind(sql::threshold(now, retain_for))
1401                    .execute(&self.pool)
1402                    .await;
1403                // Reclaim shard rows whose parent run was just purged (F25):
1404                // `purge_runs` removed the expired terminal records above, so any
1405                // shard row no longer matching a run is orphaned. Best-effort.
1406                let _ = sqlx::query(&self.stmts.purge_orphan_shards)
1407                    .execute(&self.pool)
1408                    .await;
1409                // Drop audit records older than the run-retention window (#205).
1410                let _ = sqlx::query(&self.stmts.purge_audit)
1411                    .bind(sql::threshold(now, retain_for))
1412                    .execute(&self.pool)
1413                    .await;
1414                Ok(removed)
1415            }
1416
1417            async fn recover_orphans(&self) -> Result<usize, $crate::serve::history::HistoryError> {
1418                use sqlx::Row as _;
1419                use $crate::serve::history::HistoryError;
1420                use $crate::serve::history::RunStatus;
1421                use $crate::serve::history::sql;
1422                let backend = |e: sqlx::Error| HistoryError::Backend(e.to_string());
1423                let now = chrono::Utc::now();
1424                // Only non-terminal runs whose lease has expired (the owning
1425                // instance is presumed dead). A live instance heartbeats its
1426                // runs' leases into the future, so this never fails another
1427                // healthy instance's in-flight runs (#146 H7).
1428                let rows = sqlx::query(&self.stmts.select_orphans)
1429                    .bind(sql::fmt_ts(now))
1430                    .fetch_all(&self.pool)
1431                    .await
1432                    .map_err(backend)?;
1433                let mut count = 0usize;
1434                for r in &rows {
1435                    let body: String = r.try_get("body").map_err(backend)?;
1436                    let mut rec = sql::decode_body(&body)?;
1437                    rec.status = RunStatus::Failed;
1438                    rec.finished_at = Some(now);
1439                    rec.error = Some(
1440                        "owning serve instance's lease expired before the run finished".into(),
1441                    );
1442                    if rec.elapsed_secs.is_none()
1443                        && let Some(started) = rec.started_at
1444                    {
1445                        rec.elapsed_secs = (now - started).to_std().ok().map(|d| d.as_secs_f64());
1446                    }
1447                    self.upsert(&rec).await?;
1448                    count += 1;
1449                }
1450                Ok(count)
1451            }
1452
1453            async fn renew_leases(&self) -> Result<usize, $crate::serve::history::HistoryError> {
1454                use $crate::serve::history::HistoryError;
1455                use $crate::serve::history::sql;
1456                let backend = |e: sqlx::Error| HistoryError::Backend(e.to_string());
1457                let new_lease = sql::fmt_ts(chrono::Utc::now() + self.lease_ttl);
1458                let renewed = sqlx::query(&self.stmts.renew_leases)
1459                    .bind(&new_lease)
1460                    .bind(&self.instance_id)
1461                    .execute(&self.pool)
1462                    .await
1463                    .map_err(backend)?
1464                    .rows_affected() as usize;
1465                Ok(renewed)
1466            }
1467
1468            async fn claim_pending(
1469                &self,
1470                limit: usize,
1471            ) -> Result<Vec<$crate::serve::history::RunRecord>, $crate::serve::history::HistoryError>
1472            {
1473                use sqlx::Row as _;
1474                use $crate::serve::history::HistoryError;
1475                use $crate::serve::history::RunStatus;
1476                use $crate::serve::history::sql;
1477                let backend = |e: sqlx::Error| HistoryError::Backend(e.to_string());
1478                if limit == 0 {
1479                    return Ok(Vec::new());
1480                }
1481                let now = chrono::Utc::now();
1482                let lease = sql::fmt_ts(now + self.lease_ttl);
1483
1484                // 1. Candidate pending runs (oldest first), with their bodies.
1485                let rows = sqlx::query(&self.stmts.select_pending)
1486                    .bind(limit as i64)
1487                    .fetch_all(&self.pool)
1488                    .await
1489                    .map_err(backend)?;
1490
1491                // Per-row conditional claim (1 SELECT + N guarded UPDATEs). The
1492                // batch is bounded by the caller's free permits (small), and this
1493                // is portable across Postgres + SQLite — deliberately NOT a
1494                // Postgres-only `FOR UPDATE SKIP LOCKED`.
1495                let mut claimed = Vec::new();
1496                for row in &rows {
1497                    let run_id: String = row.try_get("run_id").map_err(backend)?;
1498                    let body: String = row.try_get("body").map_err(backend)?;
1499                    // Flip the record to Running and rewrite the body so the column
1500                    // and the (source-of-truth) body stay consistent — a GET right
1501                    // after the claim must not show a stale `pending`.
1502                    let mut r = sql::decode_body(&body)?;
1503                    r.status = RunStatus::Running;
1504                    let new_body = sql::encode_body(&r)?;
1505                    // 2. Conditional claim — only the first committer wins.
1506                    let won = sqlx::query(&self.stmts.claim_one)
1507                        .bind(&self.instance_id)
1508                        .bind(&lease)
1509                        .bind(&new_body)
1510                        .bind(&run_id)
1511                        .execute(&self.pool)
1512                        .await
1513                        .map_err(backend)?
1514                        .rows_affected();
1515                    if won == 1 {
1516                        claimed.push(r);
1517                    }
1518                }
1519                Ok(claimed)
1520            }
1521
1522            async fn reclaim_orphans(
1523                &self,
1524                max_attempts: u32,
1525            ) -> Result<$crate::serve::history::ReclaimReport, $crate::serve::history::HistoryError>
1526            {
1527                use sqlx::Row as _;
1528                use $crate::serve::history::HistoryError;
1529                use $crate::serve::history::ReclaimReport;
1530                use $crate::serve::history::RunStatus;
1531                use $crate::serve::history::sql;
1532                let backend = |e: sqlx::Error| HistoryError::Backend(e.to_string());
1533                let now = chrono::Utc::now();
1534                let now_s = sql::fmt_ts(now);
1535
1536                let rows = sqlx::query(&self.stmts.reclaim_select)
1537                    .bind(&now_s)
1538                    .fetch_all(&self.pool)
1539                    .await
1540                    .map_err(backend)?;
1541
1542                let mut report = ReclaimReport::default();
1543                for row in &rows {
1544                    let body: String = row.try_get("body").map_err(backend)?;
1545                    let mut rec = sql::decode_body(&body)?;
1546                    let next_attempt = rec.attempt + 1;
1547                    // Cap is on the attempts already made: a run that has been
1548                    // reclaimed fewer than `max_attempts` times gets another try;
1549                    // once it reaches the cap it is poisoned.
1550                    if rec.attempt < max_attempts {
1551                        // Re-queue for another instance to re-run.
1552                        rec.attempt = next_attempt;
1553                        rec.status = RunStatus::Pending;
1554                        let new_body = sql::encode_body(&rec)?;
1555                        let n = sqlx::query(&self.stmts.reclaim_requeue)
1556                            .bind(&new_body)
1557                            .bind(&rec.run_id)
1558                            .bind(&now_s)
1559                            .execute(&self.pool)
1560                            .await
1561                            .map_err(backend)?
1562                            .rows_affected();
1563                        if n == 1 {
1564                            report.requeued += 1;
1565                        }
1566                    } else {
1567                        // Poison: too many attempts.
1568                        rec.attempt = next_attempt;
1569                        rec.status = RunStatus::Failed;
1570                        rec.finished_at = Some(now);
1571                        rec.error = Some(format!(
1572                            "run reclaimed {next_attempt} times after its owning instance's \
1573                             lease expired; giving up (poison run)"
1574                        ));
1575                        if rec.elapsed_secs.is_none()
1576                            && let Some(started) = rec.started_at
1577                        {
1578                            rec.elapsed_secs =
1579                                (now - started).to_std().ok().map(|d| d.as_secs_f64());
1580                        }
1581                        let new_body = sql::encode_body(&rec)?;
1582                        let n = sqlx::query(&self.stmts.reclaim_fail)
1583                            .bind(&now_s)
1584                            .bind(&new_body)
1585                            .bind(&rec.run_id)
1586                            .bind(&now_s)
1587                            .execute(&self.pool)
1588                            .await
1589                            .map_err(backend)?
1590                            .rows_affected();
1591                        if n == 1 {
1592                            report.failed += 1;
1593                        }
1594                    }
1595                }
1596                Ok(report)
1597            }
1598
1599            async fn finalize_owned(
1600                &self,
1601                rec: &$crate::serve::history::RunRecord,
1602            ) -> Result<bool, $crate::serve::history::HistoryError> {
1603                use $crate::serve::history::HistoryError;
1604                use $crate::serve::history::sql;
1605                let backend = |e: sqlx::Error| HistoryError::Backend(e.to_string());
1606                // Defensive: a terminal record must carry finished_at, or
1607                // purge_runs (which requires finished_at IS NOT NULL) can never
1608                // reclaim it. Stamp it if a caller left it unset.
1609                let mut rec = rec.clone();
1610                if rec.status.is_terminal() && rec.finished_at.is_none() {
1611                    rec.finished_at = Some(chrono::Utc::now());
1612                }
1613                let body = sql::encode_body(&rec)?;
1614                let finished = rec.finished_at.map(sql::fmt_ts);
1615                let lease = sql::fmt_ts(chrono::Utc::now() + self.lease_ttl);
1616                let n = sqlx::query(&self.stmts.finalize_owned)
1617                    .bind(rec.status.as_str())
1618                    .bind(finished.as_deref())
1619                    .bind(&lease)
1620                    .bind(&body)
1621                    .bind(&rec.run_id)
1622                    .bind(&self.instance_id)
1623                    .execute(&self.pool)
1624                    .await
1625                    .map_err(backend)?
1626                    .rows_affected();
1627                Ok(n == 1)
1628            }
1629
1630            async fn finalize_sharded_parent(
1631                &self,
1632                run_id: &str,
1633                status: $crate::serve::history::RunStatus,
1634                finished_at: chrono::DateTime<chrono::Utc>,
1635                error: Option<String>,
1636            ) -> Result<bool, $crate::serve::history::HistoryError> {
1637                use sqlx::Row as _;
1638                use $crate::serve::history::HistoryError;
1639                use $crate::serve::history::RunStatus;
1640                use $crate::serve::history::sql;
1641                let backend = |e: sqlx::Error| HistoryError::Backend(e.to_string());
1642                // Read the parent body, apply the terminal status, and write back
1643                // conditional on it still being `sharded` — so a concurrent
1644                // double-finalize from two instances has exactly one winner and
1645                // neither re-stamps owner/lease on the terminal record (F45).
1646                let Some(row) = sqlx::query(&self.stmts.select_body)
1647                    .bind(run_id)
1648                    .fetch_optional(&self.pool)
1649                    .await
1650                    .map_err(backend)?
1651                else {
1652                    return Ok(false);
1653                };
1654                let body: String = row.try_get("body").map_err(backend)?;
1655                let mut rec = sql::decode_body(&body)?;
1656                if rec.status != RunStatus::Sharded {
1657                    return Ok(false);
1658                }
1659                rec.status = status;
1660                rec.finished_at = Some(finished_at);
1661                rec.error = error;
1662                let new_body = sql::encode_body(&rec)?;
1663                let n = sqlx::query(&self.stmts.finalize_sharded_parent)
1664                    .bind(status.as_str())
1665                    .bind(sql::fmt_ts(finished_at))
1666                    .bind(&new_body)
1667                    .bind(run_id)
1668                    .execute(&self.pool)
1669                    .await
1670                    .map_err(backend)?
1671                    .rows_affected();
1672                Ok(n == 1)
1673            }
1674
1675            async fn cancel_pending(
1676                &self,
1677                run_id: &str,
1678            ) -> Result<bool, $crate::serve::history::HistoryError> {
1679                use sqlx::Row as _;
1680                use $crate::serve::history::HistoryError;
1681                use $crate::serve::history::RunStatus;
1682                use $crate::serve::history::sql;
1683                let backend = |e: sqlx::Error| HistoryError::Backend(e.to_string());
1684                // Read the pending run's body, flip it to Cancelled, and write back
1685                // conditional on it still being pending (loses the race to a claim).
1686                let Some(row) = sqlx::query(&self.stmts.select_body)
1687                    .bind(run_id)
1688                    .fetch_optional(&self.pool)
1689                    .await
1690                    .map_err(backend)?
1691                else {
1692                    return Ok(false);
1693                };
1694                let body: String = row.try_get("body").map_err(backend)?;
1695                let mut rec = sql::decode_body(&body)?;
1696                if rec.status != RunStatus::Pending {
1697                    return Ok(false);
1698                }
1699                let now = chrono::Utc::now();
1700                rec.status = RunStatus::Cancelled;
1701                rec.finished_at = Some(now);
1702                let new_body = sql::encode_body(&rec)?;
1703                let n = sqlx::query(&self.stmts.cancel_pending)
1704                    .bind(sql::fmt_ts(now))
1705                    .bind(&new_body)
1706                    .bind(run_id)
1707                    .execute(&self.pool)
1708                    .await
1709                    .map_err(backend)?
1710                    .rows_affected();
1711                Ok(n == 1)
1712            }
1713
1714            async fn request_cancel(
1715                &self,
1716                run_id: &str,
1717            ) -> Result<(), $crate::serve::history::HistoryError> {
1718                use $crate::serve::history::HistoryError;
1719                use $crate::serve::history::sql;
1720                let backend = |e: sqlx::Error| HistoryError::Backend(e.to_string());
1721                sqlx::query(&self.stmts.request_cancel)
1722                    .bind(sql::fmt_ts(chrono::Utc::now()))
1723                    .bind(run_id)
1724                    .execute(&self.pool)
1725                    .await
1726                    .map_err(backend)?;
1727                Ok(())
1728            }
1729
1730            async fn pending_cancellations(
1731                &self,
1732            ) -> Result<Vec<String>, $crate::serve::history::HistoryError> {
1733                use sqlx::Row as _;
1734                use $crate::serve::history::HistoryError;
1735                let backend = |e: sqlx::Error| HistoryError::Backend(e.to_string());
1736                let rows = sqlx::query(&self.stmts.pending_cancellations)
1737                    .bind(&self.instance_id)
1738                    .fetch_all(&self.pool)
1739                    .await
1740                    .map_err(backend)?;
1741                let mut ids = Vec::with_capacity(rows.len());
1742                for r in &rows {
1743                    ids.push(r.try_get::<String, _>("run_id").map_err(backend)?);
1744                }
1745                Ok(ids)
1746            }
1747
1748            async fn heartbeat_instance(
1749                &self,
1750                beat: &$crate::serve::history::InstanceHeartbeat,
1751            ) -> Result<(), $crate::serve::history::HistoryError> {
1752                use $crate::serve::history::HistoryError;
1753                use $crate::serve::history::sql;
1754                let backend = |e: sqlx::Error| HistoryError::Backend(e.to_string());
1755                let now = sql::fmt_ts(chrono::Utc::now());
1756                sqlx::query(&self.stmts.heartbeat_instance)
1757                    .bind(&self.instance_id)
1758                    .bind(sql::fmt_ts(beat.started_at))
1759                    .bind(&now)
1760                    .bind(beat.listen.as_deref())
1761                    .bind(beat.max_concurrent.to_string())
1762                    .bind(beat.in_flight.to_string())
1763                    .execute(&self.pool)
1764                    .await
1765                    .map_err(backend)?;
1766                Ok(())
1767            }
1768
1769            async fn live_instances(
1770                &self,
1771                ttl: std::time::Duration,
1772            ) -> Result<Vec<$crate::serve::history::InstanceRecord>, $crate::serve::history::HistoryError>
1773            {
1774                use sqlx::Row as _;
1775                use $crate::serve::history::HistoryError;
1776                use $crate::serve::history::InstanceRecord;
1777                use $crate::serve::history::sql;
1778                let backend = |e: sqlx::Error| HistoryError::Backend(e.to_string());
1779                let now = chrono::Utc::now();
1780                let rows = sqlx::query(&self.stmts.live_instances)
1781                    .bind(sql::threshold(now, ttl))
1782                    .fetch_all(&self.pool)
1783                    .await
1784                    .map_err(backend)?;
1785                let parse_dt = |s: &str| {
1786                    chrono::DateTime::parse_from_rfc3339(s)
1787                        .map(|d| d.to_utc())
1788                        .unwrap_or(now)
1789                };
1790                let mut out = Vec::with_capacity(rows.len());
1791                for r in &rows {
1792                    let started: String = r.try_get("started_at").map_err(backend)?;
1793                    let hb: String = r.try_get("last_heartbeat").map_err(backend)?;
1794                    let mc: Option<String> = r.try_get("max_concurrent").map_err(backend)?;
1795                    let inf: Option<String> = r.try_get("in_flight").map_err(backend)?;
1796                    out.push(InstanceRecord {
1797                        instance_id: r.try_get("instance_id").map_err(backend)?,
1798                        started_at: parse_dt(&started),
1799                        last_heartbeat: parse_dt(&hb),
1800                        listen: r.try_get("listen").map_err(backend)?,
1801                        max_concurrent: mc.and_then(|s| s.parse().ok()).unwrap_or(0),
1802                        in_flight: inf.and_then(|s| s.parse().ok()).unwrap_or(0),
1803                    });
1804                }
1805                Ok(out)
1806            }
1807
1808            // ── Source shards (Mode B, #230) ─────────────────────────────────
1809
1810            async fn insert_shards(
1811                &self,
1812                run_id: &str,
1813                shards: &[$crate::serve::history::ShardInsert],
1814            ) -> Result<usize, $crate::serve::history::HistoryError> {
1815                use $crate::serve::history::HistoryError;
1816                let backend = |e: sqlx::Error| HistoryError::Backend(e.to_string());
1817                let mut inserted = 0usize;
1818                for s in shards {
1819                    let descriptor = serde_json::to_string(&s.descriptor).map_err(|e| {
1820                        HistoryError::Backend(format!("encode shard descriptor: {e}"))
1821                    })?;
1822                    let size = s.size_estimate.map(|n| n.to_string());
1823                    let n = sqlx::query(&self.stmts.insert_shard)
1824                        .bind(run_id)
1825                        .bind(&s.shard_id)
1826                        .bind(&descriptor)
1827                        .bind(size.as_deref())
1828                        .execute(&self.pool)
1829                        .await
1830                        .map_err(backend)?
1831                        .rows_affected();
1832                    inserted += n as usize;
1833                }
1834                Ok(inserted)
1835            }
1836
1837            async fn claim_shards(
1838                &self,
1839                limit: usize,
1840            ) -> Result<
1841                Vec<$crate::serve::history::ClaimedShard>,
1842                $crate::serve::history::HistoryError,
1843            > {
1844                use sqlx::Row as _;
1845                use $crate::serve::history::ClaimedShard;
1846                use $crate::serve::history::HistoryError;
1847                use $crate::serve::history::sql;
1848                let backend = |e: sqlx::Error| HistoryError::Backend(e.to_string());
1849                if limit == 0 {
1850                    return Ok(Vec::new());
1851                }
1852                let lease = sql::fmt_ts(chrono::Utc::now() + self.lease_ttl);
1853
1854                // 1. Candidate pending shards (largest estimated size first),
1855                //    joined to their parent run body.
1856                let rows = sqlx::query(&self.stmts.claim_shards_select)
1857                    .bind(limit as i64)
1858                    .fetch_all(&self.pool)
1859                    .await
1860                    .map_err(backend)?;
1861
1862                // 2. Per-row conditional claim (portable; not FOR UPDATE SKIP LOCKED).
1863                let mut claimed = Vec::new();
1864                for row in &rows {
1865                    let run_id: String = row.try_get("run_id").map_err(backend)?;
1866                    let shard_id: String = row.try_get("shard_id").map_err(backend)?;
1867                    let descriptor_s: String = row.try_get("descriptor").map_err(backend)?;
1868                    let body: String = row.try_get("body").map_err(backend)?;
1869                    let won = sqlx::query(&self.stmts.claim_shard_one)
1870                        .bind(&self.instance_id)
1871                        .bind(&lease)
1872                        .bind(&run_id)
1873                        .bind(&shard_id)
1874                        .execute(&self.pool)
1875                        .await
1876                        .map_err(backend)?
1877                        .rows_affected();
1878                    if won == 1 {
1879                        let descriptor: serde_json::Value = serde_json::from_str(&descriptor_s)
1880                            .map_err(|e| {
1881                                HistoryError::Backend(format!("decode shard descriptor: {e}"))
1882                            })?;
1883                        let run = sql::decode_body(&body)?;
1884                        claimed.push(ClaimedShard {
1885                            run_id,
1886                            shard_id,
1887                            descriptor,
1888                            run,
1889                        });
1890                    }
1891                }
1892                Ok(claimed)
1893            }
1894
1895            async fn renew_shard_leases(
1896                &self,
1897            ) -> Result<usize, $crate::serve::history::HistoryError> {
1898                use $crate::serve::history::HistoryError;
1899                use $crate::serve::history::sql;
1900                let backend = |e: sqlx::Error| HistoryError::Backend(e.to_string());
1901                let lease = sql::fmt_ts(chrono::Utc::now() + self.lease_ttl);
1902                let n = sqlx::query(&self.stmts.renew_shard_leases)
1903                    .bind(&lease)
1904                    .bind(&self.instance_id)
1905                    .execute(&self.pool)
1906                    .await
1907                    .map_err(backend)?
1908                    .rows_affected() as usize;
1909                Ok(n)
1910            }
1911
1912            async fn reclaim_shards(
1913                &self,
1914                max_attempts: u32,
1915            ) -> Result<$crate::serve::history::ReclaimReport, $crate::serve::history::HistoryError>
1916            {
1917                use sqlx::Row as _;
1918                use $crate::serve::history::HistoryError;
1919                use $crate::serve::history::ReclaimReport;
1920                use $crate::serve::history::sql;
1921                let backend = |e: sqlx::Error| HistoryError::Backend(e.to_string());
1922                let now_s = sql::fmt_ts(chrono::Utc::now());
1923
1924                let rows = sqlx::query(&self.stmts.reclaim_shards_select)
1925                    .bind(&now_s)
1926                    .fetch_all(&self.pool)
1927                    .await
1928                    .map_err(backend)?;
1929
1930                let mut report = ReclaimReport::default();
1931                for row in &rows {
1932                    let run_id: String = row.try_get("run_id").map_err(backend)?;
1933                    let shard_id: String = row.try_get("shard_id").map_err(backend)?;
1934                    let attempt_s: String = row.try_get("attempt").map_err(backend)?;
1935                    let attempt: u32 = attempt_s.parse().unwrap_or(0);
1936                    if attempt < max_attempts {
1937                        let next = (attempt + 1).to_string();
1938                        let n = sqlx::query(&self.stmts.reclaim_shard_requeue)
1939                            .bind(&next)
1940                            .bind(&run_id)
1941                            .bind(&shard_id)
1942                            .bind(&now_s)
1943                            .execute(&self.pool)
1944                            .await
1945                            .map_err(backend)?
1946                            .rows_affected();
1947                        if n == 1 {
1948                            report.requeued += 1;
1949                        }
1950                    } else {
1951                        let n = sqlx::query(&self.stmts.reclaim_shard_fail)
1952                            .bind(&now_s)
1953                            .bind(&run_id)
1954                            .bind(&shard_id)
1955                            .bind(&now_s)
1956                            .execute(&self.pool)
1957                            .await
1958                            .map_err(backend)?
1959                            .rows_affected();
1960                        if n == 1 {
1961                            report.failed += 1;
1962                        }
1963                    }
1964                }
1965                Ok(report)
1966            }
1967
1968            async fn finalize_shard(
1969                &self,
1970                run_id: &str,
1971                shard_id: &str,
1972                success: bool,
1973            ) -> Result<bool, $crate::serve::history::HistoryError> {
1974                use $crate::serve::history::HistoryError;
1975                use $crate::serve::history::sql;
1976                let backend = |e: sqlx::Error| HistoryError::Backend(e.to_string());
1977                let status = if success { "completed" } else { "failed" };
1978                let now_s = sql::fmt_ts(chrono::Utc::now());
1979                let n = sqlx::query(&self.stmts.finalize_shard)
1980                    .bind(status)
1981                    .bind(&now_s)
1982                    .bind(run_id)
1983                    .bind(shard_id)
1984                    .bind(&self.instance_id)
1985                    .execute(&self.pool)
1986                    .await
1987                    .map_err(backend)?
1988                    .rows_affected();
1989                Ok(n == 1)
1990            }
1991
1992            async fn shard_progress(
1993                &self,
1994                run_id: &str,
1995            ) -> Result<$crate::serve::history::ShardProgress, $crate::serve::history::HistoryError>
1996            {
1997                use sqlx::Row as _;
1998                use $crate::serve::history::HistoryError;
1999                use $crate::serve::history::ShardProgress;
2000                let backend = |e: sqlx::Error| HistoryError::Backend(e.to_string());
2001                let rows = sqlx::query(&self.stmts.shard_progress)
2002                    .bind(run_id)
2003                    .fetch_all(&self.pool)
2004                    .await
2005                    .map_err(backend)?;
2006                let mut p = ShardProgress::default();
2007                for row in &rows {
2008                    let status: String = row.try_get("status").map_err(backend)?;
2009                    let n: i64 = row.try_get("n").map_err(backend)?;
2010                    let n = n.max(0) as usize;
2011                    p.total += n;
2012                    match status.as_str() {
2013                        "completed" => p.completed += n,
2014                        "failed" => p.failed += n,
2015                        "running" => p.running += n,
2016                        _ => p.pending += n,
2017                    }
2018                }
2019                Ok(p)
2020            }
2021
2022            async fn pending_shard_cancellations(
2023                &self,
2024            ) -> Result<Vec<String>, $crate::serve::history::HistoryError> {
2025                use sqlx::Row as _;
2026                use $crate::serve::history::HistoryError;
2027                let backend = |e: sqlx::Error| HistoryError::Backend(e.to_string());
2028                let rows = sqlx::query(&self.stmts.pending_shard_cancellations)
2029                    .bind(&self.instance_id)
2030                    .fetch_all(&self.pool)
2031                    .await
2032                    .map_err(backend)?;
2033                let mut ids = Vec::with_capacity(rows.len());
2034                for r in &rows {
2035                    ids.push(r.try_get::<String, _>("run_id").map_err(backend)?);
2036                }
2037                Ok(ids)
2038            }
2039
2040            async fn finalize_completed_sharded_parents(
2041                &self,
2042            ) -> Result<usize, $crate::serve::history::HistoryError> {
2043                use sqlx::Row as _;
2044                use $crate::serve::history::HistoryError;
2045                use $crate::serve::history::RunStatus;
2046                use $crate::serve::history::sql;
2047                let backend = |e: sqlx::Error| HistoryError::Backend(e.to_string());
2048
2049                // Candidate `sharded` parents — finalize each whose shards are all
2050                // terminal. The status-fenced UPDATE makes a concurrent finalize
2051                // (here or in `maybe_finalize_parent`) a benign no-op.
2052                let rows = sqlx::query(&self.stmts.select_sharded_parents)
2053                    .fetch_all(&self.pool)
2054                    .await
2055                    .map_err(backend)?;
2056
2057                let mut finalized = 0usize;
2058                for row in &rows {
2059                    let run_id: String = row.try_get("run_id").map_err(backend)?;
2060                    let progress = self.shard_progress(&run_id).await?;
2061                    if !progress.all_terminal() {
2062                        continue;
2063                    }
2064                    let success = progress.failed == 0;
2065                    // Read-modify-write the body so the surfaced record stays
2066                    // consistent (status, finished_at, error) with the column.
2067                    let Some(body_row) = sqlx::query(&self.stmts.select_body)
2068                        .bind(&run_id)
2069                        .fetch_optional(&self.pool)
2070                        .await
2071                        .map_err(backend)?
2072                    else {
2073                        continue;
2074                    };
2075                    let body: String = body_row.try_get("body").map_err(backend)?;
2076                    let mut rec = sql::decode_body(&body)?;
2077                    // Skip if it raced to terminal already (column says sharded but
2078                    // the body was just updated). The fenced UPDATE is the real guard.
2079                    if rec.status != RunStatus::Sharded {
2080                        continue;
2081                    }
2082                    let now = chrono::Utc::now();
2083                    rec.status = if success {
2084                        RunStatus::Completed
2085                    } else {
2086                        RunStatus::Failed
2087                    };
2088                    rec.finished_at = Some(now);
2089                    if !success {
2090                        rec.error = Some(format!(
2091                            "{}/{} shard(s) failed",
2092                            progress.failed, progress.total
2093                        ));
2094                    }
2095                    let new_body = sql::encode_body(&rec)?;
2096                    let n = sqlx::query(&self.stmts.finalize_sharded_parent)
2097                        .bind(rec.status.as_str())
2098                        .bind(sql::fmt_ts(now))
2099                        .bind(&new_body)
2100                        .bind(&run_id)
2101                        .execute(&self.pool)
2102                        .await
2103                        .map_err(backend)?
2104                        .rows_affected();
2105                    if n == 1 {
2106                        finalized += 1;
2107                        $crate::serve::metrics::record_run_finished(
2108                            rec.status,
2109                            if success { "ok" } else { "error" },
2110                        );
2111                        tracing::info!(
2112                            run_id,
2113                            shards = progress.total,
2114                            failed = progress.failed,
2115                            "sharded run finalized by sweep (F11)"
2116                        );
2117                    }
2118                }
2119                Ok(finalized)
2120            }
2121
2122            // ── Audit log (RBAC, #205) ───────────────────────────────────────
2123
2124            async fn record_audit(
2125                &self,
2126                entry: &$crate::serve::history::AuditEntry,
2127            ) -> Result<(), $crate::serve::history::HistoryError> {
2128                use $crate::serve::history::HistoryError;
2129                use $crate::serve::history::sql;
2130                let backend = |e: sqlx::Error| HistoryError::Backend(e.to_string());
2131                sqlx::query(&self.stmts.insert_audit)
2132                    .bind(&entry.id)
2133                    .bind(sql::fmt_ts(entry.timestamp))
2134                    .bind(&entry.principal)
2135                    .bind(&entry.role)
2136                    .bind(&entry.action)
2137                    .bind(entry.run_id.as_deref())
2138                    .bind(entry.config_fingerprint.as_deref())
2139                    .bind(entry.source_ip.as_deref())
2140                    .bind(&entry.result)
2141                    .execute(&self.pool)
2142                    .await
2143                    .map_err(backend)?;
2144                Ok(())
2145            }
2146
2147            async fn list_audit(
2148                &self,
2149                filter: &$crate::serve::history::AuditFilter,
2150            ) -> Result<
2151                Vec<$crate::serve::history::AuditEntry>,
2152                $crate::serve::history::HistoryError,
2153            > {
2154                use sqlx::Row as _;
2155                use $crate::serve::history::AuditEntry;
2156                use $crate::serve::history::HistoryError;
2157                use $crate::serve::history::sql;
2158                let backend = |e: sqlx::Error| HistoryError::Backend(e.to_string());
2159                let principal = filter.principal.as_deref();
2160                let action = filter.action.as_deref();
2161                let since = filter.since.map(sql::fmt_ts);
2162                let until = filter.until.map(sql::fmt_ts);
2163                let limit = filter.limit.max(1) as i64;
2164                let rows = sqlx::query(&self.stmts.list_audit)
2165                    .bind(principal)
2166                    .bind(principal)
2167                    .bind(action)
2168                    .bind(action)
2169                    .bind(since.as_deref())
2170                    .bind(since.as_deref())
2171                    .bind(until.as_deref())
2172                    .bind(until.as_deref())
2173                    .bind(limit)
2174                    .fetch_all(&self.pool)
2175                    .await
2176                    .map_err(backend)?;
2177                let mut out = Vec::with_capacity(rows.len());
2178                for r in &rows {
2179                    let ts: String = r.try_get("ts").map_err(backend)?;
2180                    let timestamp = $crate::serve::history::sql::parse_ts(&ts);
2181                    out.push(AuditEntry {
2182                        id: r.try_get("id").map_err(backend)?,
2183                        timestamp,
2184                        principal: r.try_get("principal").map_err(backend)?,
2185                        role: r.try_get("role").map_err(backend)?,
2186                        action: r.try_get("action").map_err(backend)?,
2187                        run_id: r.try_get("run_id").map_err(backend)?,
2188                        config_fingerprint: r.try_get("config_fingerprint").map_err(backend)?,
2189                        source_ip: r.try_get("source_ip").map_err(backend)?,
2190                        result: r.try_get("result").map_err(backend)?,
2191                    });
2192                }
2193                Ok(out)
2194            }
2195
2196            // ── Persistent run logs (#529) ────────────────────────────────────
2197
2198            async fn record_run_logs(
2199                &self,
2200                run_id: &str,
2201                lines: &[$crate::serve::history::RunLogLine],
2202            ) -> Result<(), $crate::serve::history::HistoryError> {
2203                use $crate::serve::history::HistoryError;
2204                use $crate::serve::history::sql;
2205                if lines.is_empty() {
2206                    return Ok(());
2207                }
2208                let backend = |e: sqlx::Error| HistoryError::Backend(e.to_string());
2209                let mut tx = self.pool.begin().await.map_err(backend)?;
2210                for l in lines {
2211                    sqlx::query(&self.stmts.insert_run_log)
2212                        .bind(run_id)
2213                        .bind(sql::pad_seq(l.seq))
2214                        .bind(&l.ts)
2215                        .bind(&l.level)
2216                        .bind(&l.line)
2217                        .execute(&mut *tx)
2218                        .await
2219                        .map_err(backend)?;
2220                }
2221                tx.commit().await.map_err(backend)?;
2222                Ok(())
2223            }
2224
2225            async fn list_run_logs(
2226                &self,
2227                run_id: &str,
2228                after_seq: Option<u64>,
2229                limit: usize,
2230            ) -> Result<
2231                $crate::serve::history::RunLogPage,
2232                $crate::serve::history::HistoryError,
2233            > {
2234                use sqlx::Row as _;
2235                use $crate::serve::history::HistoryError;
2236                use $crate::serve::history::sql;
2237                use $crate::serve::history::{RunLogLine, RunLogPage, RUN_LOG_TRUNCATED_SEQ};
2238                let backend = |e: sqlx::Error| HistoryError::Backend(e.to_string());
2239                let sentinel = sql::pad_seq(RUN_LOG_TRUNCATED_SEQ);
2240                let after = after_seq.map(sql::pad_seq);
2241                let limit = limit.max(1) as i64;
2242                let rows = sqlx::query(&self.stmts.list_run_logs)
2243                    .bind(run_id)
2244                    .bind(&sentinel)
2245                    .bind(after.as_deref())
2246                    .bind(after.as_deref())
2247                    .bind(limit)
2248                    .fetch_all(&self.pool)
2249                    .await
2250                    .map_err(backend)?;
2251                let mut out = Vec::with_capacity(rows.len());
2252                for r in &rows {
2253                    let seq: String = r.try_get("seq").map_err(backend)?;
2254                    out.push(RunLogLine {
2255                        seq: sql::unpad_seq(&seq),
2256                        ts: r.try_get("ts").map_err(backend)?,
2257                        level: r.try_get("level").map_err(backend)?,
2258                        line: r.try_get("line").map_err(backend)?,
2259                    });
2260                }
2261                let truncated = sqlx::query(&self.stmts.run_log_truncated)
2262                    .bind(run_id)
2263                    .bind(&sentinel)
2264                    .fetch_optional(&self.pool)
2265                    .await
2266                    .map_err(backend)?
2267                    .is_some();
2268                Ok(RunLogPage { lines: out, truncated })
2269            }
2270
2271            async fn purge_run_logs(
2272                &self,
2273                older_than: std::time::Duration,
2274            ) -> Result<usize, $crate::serve::history::HistoryError> {
2275                use $crate::serve::history::HistoryError;
2276                use $crate::serve::history::sql;
2277                let backend = |e: sqlx::Error| HistoryError::Backend(e.to_string());
2278                let cutoff = sql::threshold(chrono::Utc::now(), older_than);
2279                let res = sqlx::query(&self.stmts.purge_run_logs)
2280                    .bind(cutoff)
2281                    .execute(&self.pool)
2282                    .await
2283                    .map_err(backend)?;
2284                Ok(res.rows_affected() as usize)
2285            }
2286
2287            // ── Data Movement Catalog (#279) ─────────────────────────────────
2288
2289            async fn catalog_record(
2290                &self,
2291                update: &$crate::serve::history::catalog::CatalogUpdate,
2292            ) -> Result<(), $crate::serve::history::HistoryError> {
2293                use sqlx::Row as _;
2294                use $crate::serve::history::HistoryError;
2295                use $crate::serve::history::catalog;
2296                use $crate::serve::history::sql;
2297                let backend = |e: sqlx::Error| HistoryError::Backend(e.to_string());
2298                let now_s = sql::fmt_ts(update.recorded_at);
2299
2300                for obs in update.sources.iter().chain(std::iter::once(&update.sink)) {
2301                    let id = catalog::dataset_id(&obs.uri);
2302                    // Read-merge-write; last-write-wins under cluster concurrency
2303                    // (counters may undercount on a race — acceptable for
2304                    // operational stats, never for correctness).
2305                    let existing = sqlx::query(&self.stmts.catalog_select_dataset)
2306                        .bind(&id)
2307                        .fetch_optional(&self.pool)
2308                        .await
2309                        .map_err(backend)?
2310                        .map(|r| r.try_get::<String, _>("body"))
2311                        .transpose()
2312                        .map_err(backend)?
2313                        .map(|b| {
2314                            sql::decode_json::<catalog::CatalogDataset>(&b, "catalog dataset")
2315                        })
2316                        .transpose()?;
2317                    let (ds, new_version) = catalog::apply_observation(
2318                        existing.as_ref(),
2319                        obs,
2320                        &update.run_id,
2321                        &update.pipeline,
2322                        &update.row,
2323                        update.recorded_at,
2324                    );
2325                    sqlx::query(&self.stmts.catalog_upsert_dataset)
2326                        .bind(&ds.id)
2327                        .bind(&ds.uri)
2328                        .bind(&ds.kind)
2329                        .bind(&now_s)
2330                        .bind(sql::encode_json(&ds, "catalog dataset")?)
2331                        .execute(&self.pool)
2332                        .await
2333                        .map_err(backend)?;
2334                    if let Some(v) = new_version {
2335                        sqlx::query(&self.stmts.catalog_insert_schema_version)
2336                            .bind(&v.dataset_id)
2337                            .bind(v.version.to_string())
2338                            .bind(sql::fmt_ts(v.recorded_at))
2339                            .bind(sql::encode_json(&v, "catalog schema version")?)
2340                            .execute(&self.pool)
2341                            .await
2342                            .map_err(backend)?;
2343                    }
2344                    sqlx::query(&self.stmts.catalog_insert_stat)
2345                        .bind(&id)
2346                        .bind(&now_s)
2347                        .bind(&update.run_id)
2348                        .bind(obs.records.to_string())
2349                        .execute(&self.pool)
2350                        .await
2351                        .map_err(backend)?;
2352                    sqlx::query(&self.stmts.catalog_prune_stats)
2353                        .bind(&id)
2354                        .bind(&id)
2355                        .bind(catalog::STATS_RETAIN as i64)
2356                        .execute(&self.pool)
2357                        .await
2358                        .map_err(backend)?;
2359                }
2360
2361                // One edge per input dataset — a merge/join sink has several
2362                // (#459). Fetched once and reused across the inputs.
2363                let dst_id = catalog::dataset_id(&update.sink.uri);
2364                let existing_edges = self.catalog_all_edges().await?;
2365                for source in &update.sources {
2366                    let src_id = catalog::dataset_id(&source.uri);
2367                    let existing = existing_edges
2368                        .iter()
2369                        .find(|e| e.src_id == src_id && e.dst_id == dst_id);
2370                    let edge = catalog::apply_edge(existing, update, source);
2371                    sqlx::query(&self.stmts.catalog_upsert_edge)
2372                        .bind(&edge.src_id)
2373                        .bind(&edge.dst_id)
2374                        .bind(&now_s)
2375                        .bind(sql::encode_json(&edge, "catalog edge")?)
2376                        .execute(&self.pool)
2377                        .await
2378                        .map_err(backend)?;
2379                }
2380                Ok(())
2381            }
2382
2383            async fn catalog_list_datasets(
2384                &self,
2385                filter: &$crate::serve::history::catalog::CatalogListFilter,
2386            ) -> Result<
2387                $crate::serve::history::catalog::CatalogDatasetPage,
2388                $crate::serve::history::HistoryError,
2389            > {
2390                use sqlx::Row as _;
2391                use $crate::serve::history::HistoryError;
2392                use $crate::serve::history::catalog;
2393                use $crate::serve::history::sql;
2394                let backend = |e: sqlx::Error| HistoryError::Backend(e.to_string());
2395                let rows = sqlx::query(&self.stmts.catalog_select_datasets)
2396                    .fetch_all(&self.pool)
2397                    .await
2398                    .map_err(backend)?;
2399                let mut all = Vec::with_capacity(rows.len());
2400                for r in &rows {
2401                    let body: String = r.try_get("body").map_err(backend)?;
2402                    all.push(sql::decode_json(&body, "catalog dataset")?);
2403                }
2404                Ok(catalog::filter_datasets(all, filter))
2405            }
2406
2407            async fn catalog_get_dataset(
2408                &self,
2409                id: &str,
2410            ) -> Result<
2411                Option<$crate::serve::history::catalog::CatalogDatasetDetail>,
2412                $crate::serve::history::HistoryError,
2413            > {
2414                use sqlx::Row as _;
2415                use $crate::serve::history::HistoryError;
2416                use $crate::serve::history::catalog;
2417                use $crate::serve::history::sql;
2418                let backend = |e: sqlx::Error| HistoryError::Backend(e.to_string());
2419                let Some(row) = sqlx::query(&self.stmts.catalog_select_dataset)
2420                    .bind(id)
2421                    .fetch_optional(&self.pool)
2422                    .await
2423                    .map_err(backend)?
2424                else {
2425                    return Ok(None);
2426                };
2427                let body: String = row.try_get("body").map_err(backend)?;
2428                let dataset: catalog::CatalogDataset =
2429                    sql::decode_json(&body, "catalog dataset")?;
2430
2431                let rows = sqlx::query(&self.stmts.catalog_select_schema_versions)
2432                    .bind(id)
2433                    .fetch_all(&self.pool)
2434                    .await
2435                    .map_err(backend)?;
2436                let mut schema_timeline = Vec::with_capacity(rows.len());
2437                for r in &rows {
2438                    let body: String = r.try_get("body").map_err(backend)?;
2439                    schema_timeline.push(sql::decode_json(&body, "catalog schema version")?);
2440                }
2441
2442                let rows = sqlx::query(&self.stmts.catalog_select_stats)
2443                    .bind(id)
2444                    .bind(catalog::STATS_DETAIL_LIMIT as i64)
2445                    .fetch_all(&self.pool)
2446                    .await
2447                    .map_err(backend)?;
2448                let mut stats = Vec::with_capacity(rows.len());
2449                for r in &rows {
2450                    let recorded: String = r.try_get("recorded_at").map_err(backend)?;
2451                    let run_id: String = r.try_get("run_id").map_err(backend)?;
2452                    let records: String = r.try_get("records").map_err(backend)?;
2453                    stats.push(catalog::CatalogStatsPoint {
2454                        recorded_at: chrono::DateTime::parse_from_rfc3339(&recorded)
2455                            .map(|d| d.to_utc())
2456                            .unwrap_or_else(|_| chrono::Utc::now()),
2457                        run_id,
2458                        records: records.parse().unwrap_or(0),
2459                    });
2460                }
2461
2462                let edges = self.catalog_all_edges().await?;
2463                let (downstream, rest): (Vec<_>, Vec<_>) =
2464                    edges.into_iter().partition(|e| e.src_id == id);
2465                let upstream = rest.into_iter().filter(|e| e.dst_id == id).collect();
2466                Ok(Some(catalog::CatalogDatasetDetail {
2467                    dataset,
2468                    schema_timeline,
2469                    stats,
2470                    upstream,
2471                    downstream,
2472                }))
2473            }
2474
2475            async fn catalog_lineage(
2476                &self,
2477                root: Option<&str>,
2478                depth: u32,
2479            ) -> Result<
2480                Vec<$crate::serve::history::catalog::CatalogLineageEdge>,
2481                $crate::serve::history::HistoryError,
2482            > {
2483                use $crate::serve::history::catalog;
2484                let edges = self.catalog_all_edges().await?;
2485                Ok(catalog::lineage_slice(edges, root, depth))
2486            }
2487
2488            async fn catalog_record_config_snapshot(
2489                &self,
2490                snapshot: &$crate::serve::history::catalog::ConfigSnapshot,
2491            ) -> Result<(), $crate::serve::history::HistoryError> {
2492                use $crate::serve::history::HistoryError;
2493                use $crate::serve::history::sql;
2494                let backend = |e: sqlx::Error| HistoryError::Backend(e.to_string());
2495                sqlx::query(&self.stmts.catalog_upsert_config_snapshot)
2496                    .bind(&snapshot.pipeline)
2497                    .bind(sql::fmt_ts(snapshot.recorded_at))
2498                    .bind(&snapshot.faucet_version)
2499                    .bind(sql::encode_json(snapshot, "config snapshot")?)
2500                    .execute(&self.pool)
2501                    .await
2502                    .map_err(backend)?;
2503                Ok(())
2504            }
2505
2506            async fn catalog_last_config_snapshot(
2507                &self,
2508                pipeline: &str,
2509            ) -> Result<
2510                Option<$crate::serve::history::catalog::ConfigSnapshot>,
2511                $crate::serve::history::HistoryError,
2512            > {
2513                use sqlx::Row as _;
2514                use $crate::serve::history::HistoryError;
2515                use $crate::serve::history::sql;
2516                let backend = |e: sqlx::Error| HistoryError::Backend(e.to_string());
2517                let Some(row) = sqlx::query(&self.stmts.catalog_select_config_snapshot)
2518                    .bind(pipeline)
2519                    .fetch_optional(&self.pool)
2520                    .await
2521                    .map_err(backend)?
2522                else {
2523                    return Ok(None);
2524                };
2525                let body: String = row.try_get("body").map_err(backend)?;
2526                Ok(Some(sql::decode_json(&body, "config snapshot")?))
2527            }
2528
2529            // ── Pipeline-template registry (#444) ────────────────────────────
2530
2531            async fn template_register(
2532                &self,
2533                draft: &$crate::serve::history::templates::TemplateDraft,
2534            ) -> Result<
2535                $crate::serve::history::templates::TemplateRecord,
2536                $crate::serve::history::HistoryError,
2537            > {
2538                use sqlx::Row as _;
2539                use $crate::serve::history::HistoryError;
2540                use $crate::serve::history::{sql, templates};
2541                let backend = |e: sqlx::Error| HistoryError::Backend(e.to_string());
2542                let id = draft.id.to_string();
2543
2544                // Read-max-then-insert inside a transaction. Two concurrent
2545                // registers can still both read the same max under READ
2546                // COMMITTED; the primary key then rejects one of them, and the
2547                // retry picks up the winner's version — so a register never
2548                // silently overwrites another (F: lost-update).
2549                for attempt in 1..=sql::CLAIM_ATTEMPTS {
2550                    sql::retry_backoff(attempt).await;
2551                    // Opening the transaction and reading the current max are as
2552                    // contention-prone as the insert itself (SQLite answers a
2553                    // write-write overlap with `database is locked` immediately,
2554                    // since waiting would deadlock). Feed those failures into the
2555                    // same retry rather than aborting the register.
2556                    let mut tx = match self.pool.begin().await {
2557                        Ok(tx) => tx,
2558                        Err(e) if attempt < sql::CLAIM_ATTEMPTS => {
2559                            tracing::debug!(
2560                                template = %id, attempt, error = %e,
2561                                "template version transaction lost a race; retrying"
2562                            );
2563                            continue;
2564                        }
2565                        Err(e) => return Err(backend(e)),
2566                    };
2567                    let row = match sqlx::query(&self.stmts.template_max_version)
2568                        .bind(&id)
2569                        .fetch_one(&mut *tx)
2570                        .await
2571                    {
2572                        Ok(row) => row,
2573                        Err(e) if attempt < sql::CLAIM_ATTEMPTS => {
2574                            let _ = tx.rollback().await;
2575                            tracing::debug!(
2576                                template = %id, attempt, error = %e,
2577                                "template version read lost a race; retrying"
2578                            );
2579                            continue;
2580                        }
2581                        Err(e) => {
2582                            let _ = tx.rollback().await;
2583                            return Err(backend(e));
2584                        }
2585                    };
2586                    let max: i64 = row.try_get("v").map_err(backend)?;
2587                    let next = (max as u32).saturating_add(1);
2588                    let record = templates::TemplateRecord {
2589                        id: id.clone(),
2590                        version: next,
2591                        name: draft.name.clone(),
2592                        description: draft.description.clone(),
2593                        body: draft.body.clone(),
2594                        format: draft.format,
2595                        params: draft.params.clone(),
2596                        created_at: chrono::Utc::now(),
2597                        created_by: draft.created_by.clone(),
2598                    };
2599                    let insert = sqlx::query(&self.stmts.template_insert)
2600                        .bind(&id)
2601                        .bind(next.to_string())
2602                        .bind(&record.name)
2603                        .bind(sql::fmt_ts(record.created_at))
2604                        .bind(sql::encode_json(&record, "pipeline template")?)
2605                        .execute(&mut *tx)
2606                        .await;
2607                    match insert {
2608                        Ok(_) => {
2609                            tx.commit().await.map_err(backend)?;
2610                            // Bound the version history so a template
2611                            // re-registered on every deploy can't grow forever.
2612                            let keep = self.template_versions(&id).await?;
2613                            for stale in templates::versions_to_prune(keep) {
2614                                let _ = sqlx::query(&self.stmts.template_delete_version)
2615                                    .bind(&id)
2616                                    .bind(stale.to_string())
2617                                    .execute(&self.pool)
2618                                    .await;
2619                            }
2620                            return Ok(record);
2621                        }
2622                        Err(e) if attempt < sql::CLAIM_ATTEMPTS => {
2623                            let _ = tx.rollback().await;
2624                            tracing::debug!(
2625                                template = %id, attempt, error = %e,
2626                                "template version insert lost a race; retrying with the next version"
2627                            );
2628                        }
2629                        Err(e) => {
2630                            let _ = tx.rollback().await;
2631                            return Err(backend(e));
2632                        }
2633                    }
2634                }
2635                Err(HistoryError::Backend(format!(
2636                    "could not assign a version for template '{id}' after {} attempts",
2637                    sql::CLAIM_ATTEMPTS
2638                )))
2639            }
2640
2641            async fn template_get(
2642                &self,
2643                id: &str,
2644                version: Option<u32>,
2645            ) -> Result<
2646                Option<$crate::serve::history::templates::TemplateRecord>,
2647                $crate::serve::history::HistoryError,
2648            > {
2649                use sqlx::Row as _;
2650                use $crate::serve::history::HistoryError;
2651                use $crate::serve::history::sql;
2652                let backend = |e: sqlx::Error| HistoryError::Backend(e.to_string());
2653                let row = match version {
2654                    Some(v) => sqlx::query(&self.stmts.template_select_version)
2655                        .bind(id)
2656                        .bind(v.to_string())
2657                        .fetch_optional(&self.pool)
2658                        .await,
2659                    None => sqlx::query(&self.stmts.template_select_latest)
2660                        .bind(id)
2661                        .fetch_optional(&self.pool)
2662                        .await,
2663                }
2664                .map_err(backend)?;
2665                let Some(row) = row else {
2666                    return Ok(None);
2667                };
2668                let body: String = row.try_get("body").map_err(backend)?;
2669                Ok(Some(sql::decode_json(&body, "pipeline template")?))
2670            }
2671
2672            async fn template_list(
2673                &self,
2674            ) -> Result<
2675                Vec<$crate::serve::history::templates::TemplateSummary>,
2676                $crate::serve::history::HistoryError,
2677            > {
2678                use sqlx::Row as _;
2679                use $crate::serve::history::HistoryError;
2680                use $crate::serve::history::{sql, templates};
2681                let backend = |e: sqlx::Error| HistoryError::Backend(e.to_string());
2682                let rows = sqlx::query(&self.stmts.template_select_all)
2683                    .fetch_all(&self.pool)
2684                    .await
2685                    .map_err(backend)?;
2686                let mut all = Vec::with_capacity(rows.len());
2687                for r in &rows {
2688                    let body: String = r.try_get("body").map_err(backend)?;
2689                    all.push(sql::decode_json(&body, "pipeline template")?);
2690                }
2691                Ok(templates::latest_per_id(all))
2692            }
2693
2694            async fn template_versions(
2695                &self,
2696                id: &str,
2697            ) -> Result<Vec<u32>, $crate::serve::history::HistoryError> {
2698                use sqlx::Row as _;
2699                use $crate::serve::history::HistoryError;
2700                let backend = |e: sqlx::Error| HistoryError::Backend(e.to_string());
2701                let rows = sqlx::query(&self.stmts.template_versions)
2702                    .bind(id)
2703                    .fetch_all(&self.pool)
2704                    .await
2705                    .map_err(backend)?;
2706                let mut out = Vec::with_capacity(rows.len());
2707                for r in &rows {
2708                    let v: String = r.try_get("version").map_err(backend)?;
2709                    if let Ok(n) = v.parse::<u32>() {
2710                        out.push(n);
2711                    }
2712                }
2713                Ok(out)
2714            }
2715
2716            async fn template_delete(
2717                &self,
2718                id: &str,
2719                version: Option<u32>,
2720            ) -> Result<usize, $crate::serve::history::HistoryError> {
2721                use $crate::serve::history::HistoryError;
2722                let backend = |e: sqlx::Error| HistoryError::Backend(e.to_string());
2723                let result = match version {
2724                    Some(v) => {
2725                        // Drop channels + launch entries aimed at this version
2726                        // first, so no pointer can outlive what it points at.
2727                        sqlx::query(&self.stmts.template_delete_tags_for_version)
2728                            .bind(id)
2729                            .bind(v.to_string())
2730                            .execute(&self.pool)
2731                            .await
2732                            .map_err(backend)?;
2733                        sqlx::query(&self.stmts.template_delete_launches_for_version)
2734                            .bind(id)
2735                            .bind(v.to_string())
2736                            .execute(&self.pool)
2737                            .await
2738                            .map_err(backend)?;
2739                        sqlx::query(&self.stmts.template_delete_version)
2740                            .bind(id)
2741                            .bind(v.to_string())
2742                            .execute(&self.pool)
2743                            .await
2744                    }
2745                    None => {
2746                        for stmt in [
2747                            &self.stmts.template_delete_tags_all,
2748                            &self.stmts.template_delete_launches_all,
2749                            &self.stmts.template_delete_deprecation,
2750                        ] {
2751                            sqlx::query(stmt)
2752                                .bind(id)
2753                                .execute(&self.pool)
2754                                .await
2755                                .map_err(backend)?;
2756                        }
2757                        sqlx::query(&self.stmts.template_delete_all)
2758                            .bind(id)
2759                            .execute(&self.pool)
2760                            .await
2761                    }
2762                }
2763                .map_err(backend)?;
2764                Ok(result.rows_affected() as usize)
2765            }
2766
2767            async fn template_set_tag(
2768                &self,
2769                id: &str,
2770                tag: &str,
2771                version: u32,
2772            ) -> Result<(), $crate::serve::history::HistoryError> {
2773                use $crate::serve::history::HistoryError;
2774                use $crate::serve::history::sql;
2775                let backend = |e: sqlx::Error| HistoryError::Backend(e.to_string());
2776                sqlx::query(&self.stmts.template_upsert_tag)
2777                    .bind(id)
2778                    .bind(tag)
2779                    .bind(version.to_string())
2780                    .bind(sql::fmt_ts(chrono::Utc::now()))
2781                    .execute(&self.pool)
2782                    .await
2783                    .map_err(backend)?;
2784                Ok(())
2785            }
2786
2787            async fn template_tags(
2788                &self,
2789                id: &str,
2790            ) -> Result<
2791                std::collections::BTreeMap<String, u32>,
2792                $crate::serve::history::HistoryError,
2793            > {
2794                use sqlx::Row as _;
2795                use $crate::serve::history::HistoryError;
2796                let backend = |e: sqlx::Error| HistoryError::Backend(e.to_string());
2797                let rows = sqlx::query(&self.stmts.template_select_tags)
2798                    .bind(id)
2799                    .fetch_all(&self.pool)
2800                    .await
2801                    .map_err(backend)?;
2802                let mut out = std::collections::BTreeMap::new();
2803                for r in &rows {
2804                    let tag: String = r.try_get("tag").map_err(backend)?;
2805                    let version: String = r.try_get("version").map_err(backend)?;
2806                    if let Ok(n) = version.parse::<u32>() {
2807                        out.insert(tag, n);
2808                    }
2809                }
2810                Ok(out)
2811            }
2812
2813            async fn template_delete_tag(
2814                &self,
2815                id: &str,
2816                tag: &str,
2817            ) -> Result<bool, $crate::serve::history::HistoryError> {
2818                use $crate::serve::history::HistoryError;
2819                let backend = |e: sqlx::Error| HistoryError::Backend(e.to_string());
2820                let result = sqlx::query(&self.stmts.template_delete_tag)
2821                    .bind(id)
2822                    .bind(tag)
2823                    .execute(&self.pool)
2824                    .await
2825                    .map_err(backend)?;
2826                Ok(result.rows_affected() > 0)
2827            }
2828
2829            async fn template_launch(
2830                &self,
2831                id: &str,
2832                version: u32,
2833                launched_by: Option<&str>,
2834            ) -> Result<Option<u32>, $crate::serve::history::HistoryError> {
2835                use sqlx::Row as _;
2836                use $crate::serve::history::HistoryError;
2837                use $crate::serve::history::{sql, templates};
2838                let backend = |e: sqlx::Error| HistoryError::Backend(e.to_string());
2839
2840                // Re-launching what is already stable is a no-op: appending would
2841                // make `previous` a duplicate of `stable` and destroy the rollback
2842                // target.
2843                let existing = self.template_launches(id).await?;
2844                if templates::stable_version(&existing) == Some(version) {
2845                    return Ok(None);
2846                }
2847                // Read-max-then-insert with a bounded PK-conflict retry, exactly
2848                // like version assignment: two concurrent launches must produce
2849                // two distinct entries, never a lost write.
2850                for attempt in 1..=sql::CLAIM_ATTEMPTS {
2851                    sql::retry_backoff(attempt).await;
2852                    // As in `template_register`: a contended *read* is transient,
2853                    // so let the loop retry instead of failing the launch.
2854                    let row = match sqlx::query(&self.stmts.template_max_launch_seq)
2855                        .bind(id)
2856                        .fetch_one(&self.pool)
2857                        .await
2858                    {
2859                        Ok(row) => row,
2860                        Err(e) if attempt < sql::CLAIM_ATTEMPTS => {
2861                            tracing::debug!(
2862                                template = %id, attempt, error = %e,
2863                                "launch-log seq read lost a race; retrying"
2864                            );
2865                            continue;
2866                        }
2867                        Err(e) => return Err(backend(e)),
2868                    };
2869                    let max: i64 = row.try_get("v").map_err(backend)?;
2870                    let seq = (max as u32).saturating_add(1);
2871                    let insert = sqlx::query(&self.stmts.template_insert_launch)
2872                        .bind(id)
2873                        .bind(seq.to_string())
2874                        .bind(version.to_string())
2875                        .bind(sql::fmt_ts(chrono::Utc::now()))
2876                        .bind(launched_by)
2877                        .execute(&self.pool)
2878                        .await;
2879                    match insert {
2880                        Ok(_) => return Ok(Some(seq)),
2881                        Err(e) if attempt < sql::CLAIM_ATTEMPTS => {
2882                            tracing::debug!(
2883                                template = %id, attempt, error = %e,
2884                                "launch-log insert lost a race; retrying with the next seq"
2885                            );
2886                        }
2887                        Err(e) => return Err(backend(e)),
2888                    }
2889                }
2890                Err(HistoryError::Backend(format!(
2891                    "could not append a launch for template '{id}' after {} attempts",
2892                    sql::CLAIM_ATTEMPTS
2893                )))
2894            }
2895
2896            async fn template_launches(
2897                &self,
2898                id: &str,
2899            ) -> Result<
2900                Vec<$crate::serve::history::templates::LaunchRecord>,
2901                $crate::serve::history::HistoryError,
2902            > {
2903                use sqlx::Row as _;
2904                use $crate::serve::history::HistoryError;
2905                use $crate::serve::history::{sql, templates};
2906                let backend = |e: sqlx::Error| HistoryError::Backend(e.to_string());
2907                let rows = sqlx::query(&self.stmts.template_select_launches)
2908                    .bind(id)
2909                    .fetch_all(&self.pool)
2910                    .await
2911                    .map_err(backend)?;
2912                let mut out = Vec::with_capacity(rows.len());
2913                for r in &rows {
2914                    let seq: String = r.try_get("seq").map_err(backend)?;
2915                    let version: String = r.try_get("version").map_err(backend)?;
2916                    let launched_at: String = r.try_get("launched_at").map_err(backend)?;
2917                    let launched_by: Option<String> = r.try_get("launched_by").map_err(backend)?;
2918                    // Skip an unparseable row rather than failing the whole read —
2919                    // a corrupt entry must not make a template unusable.
2920                    let (Ok(seq), Ok(version)) = (seq.parse::<u32>(), version.parse::<u32>()) else {
2921                        continue;
2922                    };
2923                    out.push(templates::LaunchRecord {
2924                        seq,
2925                        version,
2926                        launched_at: sql::parse_ts(&launched_at),
2927                        launched_by,
2928                    });
2929                }
2930                Ok(out)
2931            }
2932
2933            async fn template_set_deprecation(
2934                &self,
2935                id: &str,
2936                record: Option<&$crate::serve::history::templates::DeprecationRecord>,
2937            ) -> Result<(), $crate::serve::history::HistoryError> {
2938                use $crate::serve::history::HistoryError;
2939                use $crate::serve::history::sql;
2940                let backend = |e: sqlx::Error| HistoryError::Backend(e.to_string());
2941                match record {
2942                    Some(r) => {
2943                        sqlx::query(&self.stmts.template_upsert_deprecation)
2944                            .bind(id)
2945                            .bind(sql::fmt_ts(r.deprecated_at))
2946                            .bind(r.deprecated_by.as_deref())
2947                            .bind(r.reason.as_deref())
2948                            .execute(&self.pool)
2949                            .await
2950                            .map_err(backend)?;
2951                    }
2952                    None => {
2953                        sqlx::query(&self.stmts.template_delete_deprecation)
2954                            .bind(id)
2955                            .execute(&self.pool)
2956                            .await
2957                            .map_err(backend)?;
2958                    }
2959                }
2960                Ok(())
2961            }
2962
2963            async fn template_deprecation(
2964                &self,
2965                id: &str,
2966            ) -> Result<
2967                Option<$crate::serve::history::templates::DeprecationRecord>,
2968                $crate::serve::history::HistoryError,
2969            > {
2970                use sqlx::Row as _;
2971                use $crate::serve::history::HistoryError;
2972                use $crate::serve::history::{sql, templates};
2973                let backend = |e: sqlx::Error| HistoryError::Backend(e.to_string());
2974                let Some(row) = sqlx::query(&self.stmts.template_select_deprecation)
2975                    .bind(id)
2976                    .fetch_optional(&self.pool)
2977                    .await
2978                    .map_err(backend)?
2979                else {
2980                    return Ok(None);
2981                };
2982                let at: String = row.try_get("deprecated_at").map_err(backend)?;
2983                Ok(Some(templates::DeprecationRecord {
2984                    deprecated_at: sql::parse_ts(&at),
2985                    deprecated_by: row.try_get("deprecated_by").map_err(backend)?,
2986                    reason: row.try_get("reason").map_err(backend)?,
2987                }))
2988            }
2989
2990            fn degraded(&self) -> bool {
2991                // A live SQL backend is never self-degraded; the FallbackHistory
2992                // wrapper owns degradation when the backend becomes unreachable.
2993                false
2994            }
2995        }
2996
2997        impl $name {
2998            /// Every catalog lineage edge, newest activity first (#279).
2999            async fn catalog_all_edges(
3000                &self,
3001            ) -> Result<
3002                Vec<$crate::serve::history::catalog::CatalogLineageEdge>,
3003                $crate::serve::history::HistoryError,
3004            > {
3005                use sqlx::Row as _;
3006                use $crate::serve::history::HistoryError;
3007                use $crate::serve::history::sql;
3008                let backend = |e: sqlx::Error| HistoryError::Backend(e.to_string());
3009                let rows = sqlx::query(&self.stmts.catalog_select_edges)
3010                    .fetch_all(&self.pool)
3011                    .await
3012                    .map_err(backend)?;
3013                let mut edges = Vec::with_capacity(rows.len());
3014                for r in &rows {
3015                    let body: String = r.try_get("body").map_err(backend)?;
3016                    edges.push(sql::decode_json(&body, "catalog edge")?);
3017                }
3018                Ok(edges)
3019            }
3020        }
3021    };
3022}
3023
3024pub(crate) use impl_sql_history;
3025
3026#[cfg(test)]
3027mod tests {
3028    use super::*;
3029
3030    #[test]
3031    fn postgres_shard_statements_are_built() {
3032        // SQLite tests only build the Sqlite statement set; exercise the
3033        // Postgres shard-statement construction too (Mode B, #230).
3034        let s = Stmts::new(Dialect::Postgres);
3035        assert!(s.insert_shard.contains("faucet_serve_shards"));
3036        assert!(s.insert_shard.contains("ON CONFLICT"));
3037        assert!(s.claim_shards_select.contains("JOIN faucet_serve_runs"));
3038        assert!(s.claim_shard_one.contains("'running'"));
3039        assert!(s.renew_shard_leases.contains("lease_expires_at"));
3040        assert!(s.reclaim_shards_select.contains("'running'"));
3041        assert!(s.reclaim_shard_requeue.contains("'pending'"));
3042        assert!(s.reclaim_shard_fail.contains("'failed'"));
3043        assert!(s.finalize_shard.contains("owner"));
3044        assert!(s.shard_progress.contains("GROUP BY"));
3045    }
3046
3047    #[test]
3048    fn fmt_ts_is_fixed_width_and_sortable() {
3049        let a = fmt_ts(
3050            DateTime::parse_from_rfc3339("2026-01-01T00:00:00Z")
3051                .unwrap()
3052                .to_utc(),
3053        );
3054        let b = fmt_ts(
3055            DateTime::parse_from_rfc3339("2026-01-01T00:00:01Z")
3056                .unwrap()
3057                .to_utc(),
3058        );
3059        assert!(a.ends_with('Z'));
3060        assert_eq!(a.len(), b.len(), "fixed width");
3061        assert!(a < b, "lexicographic order matches chronological order");
3062    }
3063
3064    #[test]
3065    fn is_expired_respects_window() {
3066        let now = Utc::now();
3067        let old = fmt_ts(now - chrono::Duration::seconds(120));
3068        assert!(is_expired(&old, now, Duration::from_secs(60)));
3069        assert!(!is_expired(&old, now, Duration::from_secs(600)));
3070        // Unparseable → not expired (conservative).
3071        assert!(!is_expired("not-a-timestamp", now, Duration::ZERO));
3072    }
3073
3074    #[test]
3075    fn parse_status_round_trips_known_and_defaults_failed() {
3076        for s in [
3077            RunStatus::Queued,
3078            RunStatus::Pending,
3079            RunStatus::Running,
3080            RunStatus::Completed,
3081            RunStatus::Failed,
3082            RunStatus::Cancelled,
3083        ] {
3084            assert_eq!(parse_status(s.as_str()), s);
3085        }
3086        assert_eq!(parse_status("garbage"), RunStatus::Failed);
3087    }
3088
3089    #[test]
3090    fn body_round_trips() {
3091        let rec = RunRecord::queued(
3092            "r1".into(),
3093            Some("n".into()),
3094            Default::default(),
3095            Some("idem".into()),
3096            Utc::now(),
3097        );
3098        let encoded = encode_body(&rec).unwrap();
3099        let decoded = decode_body(&encoded).unwrap();
3100        assert_eq!(decoded.run_id, "r1");
3101        assert_eq!(decoded.idempotency_key.as_deref(), Some("idem"));
3102    }
3103
3104    #[test]
3105    fn postgres_and_sqlite_statements_differ_only_in_placeholders() {
3106        let pg = Stmts::new(Dialect::Postgres);
3107        let lite = Stmts::new(Dialect::Sqlite);
3108        assert!(pg.upsert.contains("$1") && lite.upsert.contains('?'));
3109        assert!(pg.list.contains("$13") && lite.list.contains('?'));
3110        // Both target the same tables / conflict targets.
3111        assert!(pg.insert_idem.contains("ON CONFLICT (key) DO NOTHING"));
3112        assert!(lite.insert_idem.contains("ON CONFLICT (key) DO NOTHING"));
3113        assert!(pg.claim_one.contains("$3") && lite.claim_one.contains('?'));
3114        assert!(pg.heartbeat_instance.contains("faucet_serve_instances"));
3115        assert!(lite.heartbeat_instance.contains("faucet_serve_instances"));
3116    }
3117}