Skip to main content

faucet_cli/serve/history/
sql.rs

1//! Shared SQL run-history machinery for the Postgres and SQLite backends
2//! (Phase 5, #127). Both backends are identical except for the connection setup
3//! and the placeholder dialect (`$n` vs `?`), so the schema, prepared-statement
4//! text, pure helpers, and the entire `RunHistory` impl live here once and are
5//! instantiated for each concrete `sqlx` pool via the `impl_sql_history!` macro.
6//!
7//! **Portability:** every column is `TEXT` (timestamps are stored as fixed-width
8//! RFC3339 with nanosecond precision + `Z`, which sorts lexicographically in
9//! chronological order, so keyset pagination and expiry comparisons work without
10//! any database date type — and thus without the `sqlx` `chrono` feature). The
11//! whole `RunRecord` is serialized into the `body` column (the source of truth on
12//! read); the dedicated columns exist only for filtering, ordering, and expiry.
13//!
14//! **Idempotency** lives in a separate `faucet_serve_idem` table whose `key`
15//! primary key is the required unique index (spec §10/§11). The claim is atomic
16//! via `INSERT … ON CONFLICT DO NOTHING` plus an optimistic, expiry-guarded
17//! takeover `UPDATE`, mirroring the memory backend's shard-locked semantics.
18
19use super::{HistoryError, RunRecord, RunStatus};
20use chrono::{DateTime, Utc};
21use std::time::Duration;
22
23/// DDL run at connect time. Valid verbatim on both Postgres and SQLite (only
24/// `TEXT` columns, `IF NOT EXISTS`, and standard indexes).
25pub const DDL: &[&str] = &[
26    // `owner` is the id of the serve instance that owns the run; `lease_expires_at`
27    // is the RFC3339 instant past which that ownership is presumed dead. Together
28    // they fence orphan recovery: an instance only fails a non-terminal run whose
29    // lease has expired, never another live instance's heartbeated runs (#146 H7).
30    "CREATE TABLE IF NOT EXISTS faucet_serve_runs (\
31        run_id TEXT PRIMARY KEY,\
32        name TEXT,\
33        status TEXT NOT NULL,\
34        submitted_at TEXT NOT NULL,\
35        finished_at TEXT,\
36        idempotency_key TEXT,\
37        owner TEXT,\
38        lease_expires_at TEXT,\
39        cancel_requested TEXT,\
40        body TEXT NOT NULL)",
41    "CREATE INDEX IF NOT EXISTS faucet_serve_runs_submitted_idx \
42        ON faucet_serve_runs (submitted_at)",
43    // Speeds the per-tick orphan scan / lease renewal, which filter on
44    // (status, owner, lease_expires_at).
45    "CREATE INDEX IF NOT EXISTS faucet_serve_runs_status_lease_idx \
46        ON faucet_serve_runs (status, lease_expires_at)",
47    // Speeds the cluster dispatcher's pending-run query (ordered by submitted_at).
48    "CREATE INDEX IF NOT EXISTS faucet_serve_runs_pending_idx \
49        ON faucet_serve_runs (status, submitted_at)",
50    "CREATE TABLE IF NOT EXISTS faucet_serve_instances (\
51        instance_id TEXT PRIMARY KEY,\
52        started_at TEXT NOT NULL,\
53        last_heartbeat TEXT NOT NULL,\
54        listen TEXT,\
55        max_concurrent TEXT,\
56        in_flight TEXT)",
57    "CREATE INDEX IF NOT EXISTS faucet_serve_instances_hb_idx \
58        ON faucet_serve_instances (last_heartbeat)",
59    "CREATE TABLE IF NOT EXISTS faucet_serve_idem (\
60        key TEXT PRIMARY KEY,\
61        run_id TEXT NOT NULL,\
62        fingerprint TEXT NOT NULL,\
63        claimed_at TEXT NOT NULL)",
64    // Source shards for clustered Mode B (#230). One row per (run, shard);
65    // `owner`/`lease_expires_at`/`attempt` reuse Mode A's lease-fencing semantics
66    // at shard granularity. `size_estimate` (an integer stored as TEXT) drives
67    // skew-aware, largest-first claiming. `descriptor` is the opaque connector
68    // shard spec, replayed to the worker that claims the shard.
69    "CREATE TABLE IF NOT EXISTS faucet_serve_shards (\
70        run_id TEXT NOT NULL,\
71        shard_id TEXT NOT NULL,\
72        descriptor TEXT NOT NULL,\
73        size_estimate TEXT,\
74        status TEXT NOT NULL,\
75        owner TEXT,\
76        lease_expires_at TEXT,\
77        attempt TEXT NOT NULL,\
78        finished_at TEXT,\
79        PRIMARY KEY (run_id, shard_id))",
80    "CREATE INDEX IF NOT EXISTS faucet_serve_shards_claim_idx \
81        ON faucet_serve_shards (status, lease_expires_at)",
82    // Audit log for RBAC (#205). One row per mutating (or denied) control-plane
83    // action. `id` is a time-ordered UUIDv7; `ts` is fixed-width RFC3339 so the
84    // newest-first ordering and retention purge sort lexicographically.
85    "CREATE TABLE IF NOT EXISTS faucet_serve_audit (\
86        id TEXT PRIMARY KEY,\
87        ts TEXT NOT NULL,\
88        principal TEXT NOT NULL,\
89        role TEXT NOT NULL,\
90        action TEXT NOT NULL,\
91        run_id TEXT,\
92        config_fingerprint TEXT,\
93        source_ip TEXT,\
94        result TEXT NOT NULL)",
95    "CREATE INDEX IF NOT EXISTS faucet_serve_audit_ts_idx \
96        ON faucet_serve_audit (ts)",
97    // Data Movement Catalog (#279). Accumulating cross-run state, deliberately
98    // NOT covered by `purge_expired` (the history is the value). Same
99    // TEXT-columns + JSON `body` convention as the run tables: the dedicated
100    // columns exist for filtering only; `body` is the source of truth on read.
101    "CREATE TABLE IF NOT EXISTS faucet_catalog_datasets (\
102        id TEXT PRIMARY KEY,\
103        uri TEXT NOT NULL,\
104        kind TEXT NOT NULL,\
105        last_seen TEXT NOT NULL,\
106        body TEXT NOT NULL)",
107    // One row per (dataset, schema version); appended only on content change.
108    // `version` is an integer stored as TEXT (cast on ORDER BY), matching the
109    // shard table's `size_estimate` convention.
110    "CREATE TABLE IF NOT EXISTS faucet_catalog_schema_versions (\
111        dataset_id TEXT NOT NULL,\
112        version TEXT NOT NULL,\
113        recorded_at TEXT NOT NULL,\
114        body TEXT NOT NULL,\
115        PRIMARY KEY (dataset_id, version))",
116    // One row per (source dataset, sink dataset) lineage edge.
117    "CREATE TABLE IF NOT EXISTS faucet_catalog_edges (\
118        src_id TEXT NOT NULL,\
119        dst_id TEXT NOT NULL,\
120        last_seen TEXT NOT NULL,\
121        body TEXT NOT NULL,\
122        PRIMARY KEY (src_id, dst_id))",
123    // Per-run volume points, capped per dataset at `catalog::STATS_RETAIN`.
124    "CREATE TABLE IF NOT EXISTS faucet_catalog_stats (\
125        dataset_id TEXT NOT NULL,\
126        recorded_at TEXT NOT NULL,\
127        run_id TEXT NOT NULL,\
128        records TEXT NOT NULL,\
129        PRIMARY KEY (dataset_id, recorded_at))",
130    // Resolved+expanded config snapshots for `faucet plan --diff` (#374). One
131    // row per pipeline (latest-wins upsert); `body` is the redacted
132    // `ConfigSnapshot` JSON — no secret material is ever stored.
133    "CREATE TABLE IF NOT EXISTS faucet_config_snapshots (\
134        pipeline TEXT PRIMARY KEY,\
135        recorded_at TEXT NOT NULL,\
136        faucet_version TEXT NOT NULL,\
137        body TEXT NOT NULL)",
138    // Registered pipeline templates (#444), one row per (id, version). `body` is
139    // the full `TemplateRecord` JSON — including the config document *verbatim*,
140    // so `${env:…}` / `${vault:…}` stay unresolved tokens and no secret material
141    // is ever persisted. `version` is an integer stored as TEXT (cast on
142    // ORDER BY), matching the schema-version / shard-estimate convention.
143    // Deliberately NOT purged by run retention: a template outlives its runs.
144    "CREATE TABLE IF NOT EXISTS faucet_templates (\
145        id TEXT NOT NULL,\
146        version TEXT NOT NULL,\
147        name TEXT,\
148        created_at TEXT NOT NULL,\
149        body TEXT NOT NULL,\
150        PRIMARY KEY (id, version))",
151    // Named channel pointers (#444): one row per (template, channel), each
152    // aiming at a numeric version. `latest` is derived from `faucet_templates`
153    // and never stored here. Deleting a version drops the channels aimed at it,
154    // so a pointer can never dangle.
155    "CREATE TABLE IF NOT EXISTS faucet_template_tags (\
156        id TEXT NOT NULL,\
157        tag TEXT NOT NULL,\
158        version TEXT NOT NULL,\
159        updated_at TEXT NOT NULL,\
160        PRIMARY KEY (id, tag))",
161    // Append-only launch log (#444): the source of truth for `stable` (newest
162    // entry) and `previous` (the one before it), the derived template status, and
163    // the launch/rollback audit trail. `seq` is an integer stored as TEXT (cast on
164    // ORDER BY), matching the version/estimate convention elsewhere.
165    "CREATE TABLE IF NOT EXISTS faucet_template_launches (\
166        id TEXT NOT NULL,\
167        seq TEXT NOT NULL,\
168        version TEXT NOT NULL,\
169        launched_at TEXT NOT NULL,\
170        launched_by TEXT,\
171        PRIMARY KEY (id, seq))",
172    // Deprecation markers — the only *stored* part of a template's lifecycle
173    // status (`draft` vs `launched` derives from the launch log).
174    "CREATE TABLE IF NOT EXISTS faucet_template_deprecations (\
175        id TEXT PRIMARY KEY,\
176        deprecated_at TEXT NOT NULL,\
177        deprecated_by TEXT,\
178        reason TEXT)",
179];
180
181/// SQL placeholder dialect.
182#[derive(Clone, Copy, Debug)]
183pub enum Dialect {
184    Postgres,
185    Sqlite,
186}
187
188/// Prepared-statement text for a backend, built once per dialect at connect time.
189pub struct Stmts {
190    /// (`cancel_requested` is intentionally NOT written by `upsert` — it is set
191    /// only via `request_cancel` and cleared by `reclaim_requeue`; it defaults to
192    /// NULL on insert.)
193    pub upsert: String,
194    pub select_body: String,
195    pub select_status: String,
196    pub select_submitted: String,
197    pub delete: String,
198    pub list: String,
199    pub purge_runs: String,
200    pub purge_idem: String,
201    /// Select non-terminal runs whose owning instance's lease has expired (or
202    /// is unset) — the orphans this instance may safely fail. Param: `now`.
203    pub select_orphans: String,
204    /// Extend the lease of this instance's own non-terminal runs (heartbeat).
205    /// Params: `new_lease_expiry`, `instance_id`.
206    pub renew_leases: String,
207    pub insert_idem: String,
208    pub select_idem: String,
209    pub takeover_idem: String,
210    /// Delete the idempotency claim(s) that point at a given run — used when a
211    /// run is deleted so a replay of the key starts fresh rather than 404-ing
212    /// on the missing record (#146 M8). Scoped by `run_id`, so a newer run that
213    /// re-claimed the same key keeps its claim.
214    pub delete_idem_by_run: String,
215    /// Cluster dispatcher: fetch oldest pending runs up to a given limit.
216    pub select_pending: String,
217    /// Cluster dispatcher: atomically claim a pending run (set owner + running).
218    pub claim_one: String,
219    /// Cluster reclaimer: select expired running runs for requeue/fail evaluation.
220    /// NOTE: `'queued'` is the single-instance status; cluster runs flow
221    /// `pending → running`, so the failover reclaimer covers `'running'` only.
222    pub reclaim_select: String,
223    /// Cluster reclaimer: requeue an expired running run back to pending.
224    pub reclaim_requeue: String,
225    /// Cluster reclaimer: fail an expired running run that cannot be requeued.
226    pub reclaim_fail: String,
227    /// Finalize a run owned by this instance (terminal status update).
228    pub finalize_owned: String,
229    /// Cancel a pending run directly (transition pending → cancelled).
230    pub cancel_pending: String,
231    /// Request cancellation of an in-flight run owned by another instance.
232    pub request_cancel: String,
233    /// List run IDs owned by this instance that have a pending cancellation request.
234    pub pending_cancellations: String,
235    /// Upsert this instance's membership heartbeat into `faucet_serve_instances`.
236    pub heartbeat_instance: String,
237    /// List instances whose last heartbeat is at or after a given threshold.
238    pub live_instances: String,
239    /// Prune instances whose last heartbeat is before a given threshold.
240    pub prune_instances: String,
241    // ── Source shards (Mode B, #230) ─────────────────────────────────────────
242    /// Idempotent shard insert (`ON CONFLICT (run_id, shard_id) DO NOTHING`).
243    pub insert_shard: String,
244    /// Select claimable pending shards joined to their run body, largest first.
245    pub claim_shards_select: String,
246    /// Atomically claim one pending shard for this instance.
247    pub claim_shard_one: String,
248    /// Heartbeat this instance's running shards.
249    pub renew_shard_leases: String,
250    /// Select expired-lease running shards for requeue/fail evaluation.
251    pub reclaim_shards_select: String,
252    /// Requeue an expired running shard back to pending (attempt++).
253    pub reclaim_shard_requeue: String,
254    /// Fail an expired running shard that exhausted its attempts (poison).
255    pub reclaim_shard_fail: String,
256    /// Owner-fenced terminal write for one shard.
257    pub finalize_shard: String,
258    /// Status counts for a run's shards.
259    pub shard_progress: String,
260    /// Distinct run_ids for which THIS instance owns a `running` shard whose
261    /// parent run has a pending cancellation request (cross-instance shard
262    /// cancel, F10). Param: `instance_id`.
263    pub pending_shard_cancellations: String,
264    /// Select run_ids of `sharded` parents (candidates to finalize once all
265    /// their shards are terminal, F11).
266    pub select_sharded_parents: String,
267    /// Status-fenced terminal write for a `sharded` parent (F11). A benign
268    /// double-finalize across instances is a no-op: the guard requires the
269    /// parent to still be `sharded`. Does NOT re-arm owner/lease.
270    pub finalize_sharded_parent: String,
271    /// Delete a run's shard rows (paired with [`delete`](Self::delete) so a
272    /// deleted run leaves no orphaned shard rows behind, F25). Param: `run_id`.
273    pub delete_shards_by_run: String,
274    /// Purge shard rows whose parent run no longer exists (run-record purged by
275    /// retention, F25). No params — a set-difference against `faucet_serve_runs`.
276    pub purge_orphan_shards: String,
277    // ── Audit log (RBAC, #205) ───────────────────────────────────────────────
278    /// Append one audit record.
279    pub insert_audit: String,
280    /// Newest-first audit records matching the (nullable) filters. Param order:
281    /// principal, action, since, until, limit.
282    pub list_audit: String,
283    /// Purge audit records older than a threshold (retention).
284    pub purge_audit: String,
285    // ── Data Movement Catalog (#279) ─────────────────────────────────────────
286    /// One dataset body by id (the merge read + the detail head).
287    pub catalog_select_dataset: String,
288    /// Upsert one dataset row (filter columns + body). Params: id, uri, kind,
289    /// last_seen, body.
290    pub catalog_upsert_dataset: String,
291    /// Every dataset body — filtering/ordering happens in shared pure code
292    /// ([`catalog::filter_datasets`](super::catalog::filter_datasets)), so the
293    /// memory and SQL backends can never disagree on semantics.
294    pub catalog_select_datasets: String,
295    /// Append one schema-timeline entry; `ON CONFLICT DO NOTHING` so a cluster
296    /// replay of the same (dataset, version) is idempotent.
297    pub catalog_insert_schema_version: String,
298    /// A dataset's schema timeline, oldest first.
299    pub catalog_select_schema_versions: String,
300    /// Upsert one lineage edge. Params: src_id, dst_id, last_seen, body.
301    pub catalog_upsert_edge: String,
302    /// Every edge body, newest activity first.
303    pub catalog_select_edges: String,
304    /// Append one volume point. Params: dataset_id, recorded_at, run_id, records.
305    pub catalog_insert_stat: String,
306    /// A dataset's most recent volume points. Params: dataset_id, limit.
307    pub catalog_select_stats: String,
308    /// Drop volume points beyond the newest `STATS_RETAIN` for one dataset.
309    /// Params: dataset_id, dataset_id, keep-limit.
310    pub catalog_prune_stats: String,
311    /// Upsert the latest config snapshot for a pipeline (#374).
312    /// Params: pipeline, recorded_at, faucet_version, body.
313    pub catalog_upsert_config_snapshot: String,
314    /// The latest config snapshot body for a pipeline. Param: pipeline.
315    pub catalog_select_config_snapshot: String,
316    // ── Pipeline templates (#444) ────────────────────────────────────────────
317    /// Highest existing version for a template id (0 when new). Param: id.
318    pub template_max_version: String,
319    /// Insert one template version. Params: id, version, name, created_at, body.
320    pub template_insert: String,
321    /// One template version's body. Params: id, version.
322    pub template_select_version: String,
323    /// The latest version's body for an id. Param: id.
324    pub template_select_latest: String,
325    /// Every template body (latest-per-id folding happens in shared pure code,
326    /// so the memory and SQL backends can never disagree).
327    pub template_select_all: String,
328    /// Version numbers for one id, newest first. Param: id.
329    pub template_versions: String,
330    /// Delete one version. Params: id, version.
331    pub template_delete_version: String,
332    /// Delete every version of an id. Param: id.
333    pub template_delete_all: String,
334    /// Upsert one channel pointer. Params: id, tag, version, updated_at.
335    pub template_upsert_tag: String,
336    /// Every channel pointer for an id. Param: id.
337    pub template_select_tags: String,
338    /// Delete one channel pointer. Params: id, tag.
339    pub template_delete_tag: String,
340    /// Delete every channel pointer for an id. Param: id.
341    pub template_delete_tags_all: String,
342    /// Delete the channel pointers aimed at one version. Params: id, version.
343    pub template_delete_tags_for_version: String,
344    /// Highest launch seq for a template (0 when never launched). Param: id.
345    pub template_max_launch_seq: String,
346    /// Append one launch entry. Params: id, seq, version, launched_at, launched_by.
347    pub template_insert_launch: String,
348    /// The launch log for a template, newest first. Param: id.
349    pub template_select_launches: String,
350    /// Delete every launch entry for a template. Param: id.
351    pub template_delete_launches_all: String,
352    /// Delete the launch entries naming one version. Params: id, version.
353    pub template_delete_launches_for_version: String,
354    /// Upsert the deprecation marker. Params: id, deprecated_at, deprecated_by, reason.
355    pub template_upsert_deprecation: String,
356    /// Read the deprecation marker. Param: id.
357    pub template_select_deprecation: String,
358    /// Clear the deprecation marker. Param: id.
359    pub template_delete_deprecation: String,
360}
361
362impl Stmts {
363    pub fn new(dialect: Dialect) -> Self {
364        match dialect {
365            Dialect::Postgres => Self::postgres(),
366            Dialect::Sqlite => Self::sqlite(),
367        }
368    }
369
370    fn postgres() -> Self {
371        Self {
372            upsert: "INSERT INTO faucet_serve_runs \
373                (run_id,name,status,submitted_at,finished_at,idempotency_key,owner,lease_expires_at,body) \
374                VALUES ($1,$2,$3,$4,$5,$6,$7,$8,$9) \
375                ON CONFLICT (run_id) DO UPDATE SET \
376                name=excluded.name,status=excluded.status,submitted_at=excluded.submitted_at,\
377                finished_at=excluded.finished_at,idempotency_key=excluded.idempotency_key,\
378                owner=excluded.owner,lease_expires_at=excluded.lease_expires_at,\
379                body=excluded.body"
380                .into(),
381            select_body: "SELECT body FROM faucet_serve_runs WHERE run_id=$1".into(),
382            select_status: "SELECT status FROM faucet_serve_runs WHERE run_id=$1".into(),
383            select_submitted: "SELECT submitted_at FROM faucet_serve_runs WHERE run_id=$1".into(),
384            delete: "DELETE FROM faucet_serve_runs WHERE run_id=$1".into(),
385            // Casts make the parameter types explicit so `$n IS NULL` cannot trip
386            // Postgres' "could not determine data type of parameter" check.
387            list: "SELECT body FROM faucet_serve_runs \
388                WHERE ($1::text IS NULL OR status = $2::text) \
389                AND ($3::text IS NULL OR name = $4::text) \
390                AND ($5::text IS NULL OR submitted_at >= $6::text) \
391                AND ($7::text IS NULL OR submitted_at <= $8::text) \
392                AND ($9::text IS NULL OR (submitted_at < $10::text \
393                    OR (submitted_at = $11::text AND run_id < $12::text))) \
394                ORDER BY submitted_at DESC, run_id DESC LIMIT $13"
395                .into(),
396            purge_runs: "DELETE FROM faucet_serve_runs \
397                WHERE status IN ('completed','failed','cancelled') \
398                AND finished_at IS NOT NULL AND finished_at < $1"
399                .into(),
400            purge_idem: "DELETE FROM faucet_serve_idem WHERE claimed_at < $1".into(),
401            select_orphans: "SELECT body FROM faucet_serve_runs \
402                WHERE status IN ('queued','running') \
403                AND (lease_expires_at IS NULL OR lease_expires_at < $1)"
404                .into(),
405            renew_leases: "UPDATE faucet_serve_runs SET lease_expires_at = $1 \
406                WHERE owner = $2 AND status IN ('queued','running')"
407                .into(),
408            insert_idem: "INSERT INTO faucet_serve_idem (key,run_id,fingerprint,claimed_at) \
409                VALUES ($1,$2,$3,$4) ON CONFLICT (key) DO NOTHING"
410                .into(),
411            select_idem: "SELECT run_id,fingerprint,claimed_at FROM faucet_serve_idem WHERE key=$1"
412                .into(),
413            takeover_idem: "UPDATE faucet_serve_idem \
414                SET run_id=$1,fingerprint=$2,claimed_at=$3 WHERE key=$4 AND claimed_at=$5"
415                .into(),
416            delete_idem_by_run: "DELETE FROM faucet_serve_idem WHERE run_id=$1".into(),
417            select_pending: "SELECT run_id, body FROM faucet_serve_runs \
418                WHERE status = 'pending' ORDER BY submitted_at ASC LIMIT $1"
419                .into(),
420            claim_one: "UPDATE faucet_serve_runs \
421                SET owner = $1, status = 'running', lease_expires_at = $2, body = $3 \
422                WHERE run_id = $4 AND status = 'pending'"
423                .into(),
424            reclaim_select: "SELECT body FROM faucet_serve_runs \
425                WHERE status = 'running' \
426                AND (lease_expires_at IS NULL OR lease_expires_at < $1)"
427                .into(),
428            // Preserve `cancel_requested` across a requeue (audit #321 M7): a
429            // cross-instance cancel acknowledged while the owner was partitioned
430            // must survive re-queueing so the next owner still honours it, rather
431            // than the run silently running to completion despite the cancel.
432            reclaim_requeue: "UPDATE faucet_serve_runs \
433                SET status = 'pending', owner = NULL, lease_expires_at = NULL, \
434                    body = $1 \
435                WHERE run_id = $2 AND status = 'running' \
436                AND (lease_expires_at IS NULL OR lease_expires_at < $3)"
437                .into(),
438            reclaim_fail: "UPDATE faucet_serve_runs \
439                SET status = 'failed', finished_at = $1, body = $2, owner = NULL \
440                WHERE run_id = $3 AND status = 'running' \
441                AND (lease_expires_at IS NULL OR lease_expires_at < $4)"
442                .into(),
443            // Status-fenced (audit #321 L5): only finalize a still-non-terminal
444            // record, so a stale zombie execution that shares the same owner (a
445            // lease-lapse re-claim by the same instance) can never overwrite the
446            // terminal record written by the live execution. First finalizer wins.
447            finalize_owned: "UPDATE faucet_serve_runs \
448                SET status = $1, finished_at = $2, lease_expires_at = $3, body = $4 \
449                WHERE run_id = $5 AND owner = $6 \
450                AND status NOT IN ('completed','failed','cancelled')"
451                .into(),
452            cancel_pending: "UPDATE faucet_serve_runs \
453                SET status = 'cancelled', finished_at = $1, body = $2 \
454                WHERE run_id = $3 AND status = 'pending'"
455                .into(),
456            request_cancel: "UPDATE faucet_serve_runs \
457                SET cancel_requested = $1 WHERE run_id = $2 AND status IN ('running','sharded')"
458                .into(),
459            pending_cancellations: "SELECT run_id FROM faucet_serve_runs \
460                WHERE status = 'running' AND owner = $1 AND cancel_requested IS NOT NULL"
461                .into(),
462            heartbeat_instance: "INSERT INTO faucet_serve_instances \
463                (instance_id, started_at, last_heartbeat, listen, max_concurrent, in_flight) \
464                VALUES ($1,$2,$3,$4,$5,$6) \
465                ON CONFLICT (instance_id) DO UPDATE SET \
466                last_heartbeat = excluded.last_heartbeat, listen = excluded.listen, \
467                max_concurrent = excluded.max_concurrent, in_flight = excluded.in_flight"
468                .into(),
469            live_instances: "SELECT instance_id, started_at, last_heartbeat, listen, \
470                max_concurrent, in_flight FROM faucet_serve_instances \
471                WHERE last_heartbeat >= $1"
472                .into(),
473            prune_instances: "DELETE FROM faucet_serve_instances WHERE last_heartbeat < $1".into(),
474            insert_shard: "INSERT INTO faucet_serve_shards \
475                (run_id, shard_id, descriptor, size_estimate, status, attempt) \
476                VALUES ($1,$2,$3,$4,'pending','0') \
477                ON CONFLICT (run_id, shard_id) DO NOTHING"
478                .into(),
479            claim_shards_select: "SELECT s.run_id, s.shard_id, s.descriptor, r.body \
480                FROM faucet_serve_shards s JOIN faucet_serve_runs r ON r.run_id = s.run_id \
481                WHERE s.status = 'pending' \
482                ORDER BY CAST(COALESCE(s.size_estimate, '0') AS BIGINT) DESC, s.run_id, s.shard_id \
483                LIMIT $1"
484                .into(),
485            claim_shard_one: "UPDATE faucet_serve_shards \
486                SET owner = $1, status = 'running', lease_expires_at = $2 \
487                WHERE run_id = $3 AND shard_id = $4 AND status = 'pending'"
488                .into(),
489            renew_shard_leases: "UPDATE faucet_serve_shards SET lease_expires_at = $1 \
490                WHERE owner = $2 AND status = 'running'"
491                .into(),
492            reclaim_shards_select: "SELECT run_id, shard_id, attempt FROM faucet_serve_shards \
493                WHERE status = 'running' \
494                AND (lease_expires_at IS NULL OR lease_expires_at < $1)"
495                .into(),
496            reclaim_shard_requeue: "UPDATE faucet_serve_shards \
497                SET status = 'pending', owner = NULL, lease_expires_at = NULL, attempt = $1 \
498                WHERE run_id = $2 AND shard_id = $3 AND status = 'running' \
499                AND (lease_expires_at IS NULL OR lease_expires_at < $4)"
500                .into(),
501            reclaim_shard_fail: "UPDATE faucet_serve_shards \
502                SET status = 'failed', finished_at = $1, owner = NULL \
503                WHERE run_id = $2 AND shard_id = $3 AND status = 'running' \
504                AND (lease_expires_at IS NULL OR lease_expires_at < $4)"
505                .into(),
506            finalize_shard: "UPDATE faucet_serve_shards \
507                SET status = $1, finished_at = $2 \
508                WHERE run_id = $3 AND shard_id = $4 AND owner = $5 AND status = 'running'"
509                .into(),
510            shard_progress: "SELECT status, COUNT(*) AS n FROM faucet_serve_shards \
511                WHERE run_id = $1 GROUP BY status"
512                .into(),
513            pending_shard_cancellations: "SELECT DISTINCT s.run_id \
514                FROM faucet_serve_shards s \
515                JOIN faucet_serve_runs r ON r.run_id = s.run_id \
516                WHERE s.owner = $1 AND s.status = 'running' \
517                AND r.cancel_requested IS NOT NULL"
518                .into(),
519            select_sharded_parents: "SELECT run_id FROM faucet_serve_runs \
520                WHERE status = 'sharded'"
521                .into(),
522            finalize_sharded_parent: "UPDATE faucet_serve_runs \
523                SET status = $1, finished_at = $2, body = $3 \
524                WHERE run_id = $4 AND status = 'sharded'"
525                .into(),
526            delete_shards_by_run: "DELETE FROM faucet_serve_shards WHERE run_id = $1".into(),
527            purge_orphan_shards: "DELETE FROM faucet_serve_shards \
528                WHERE run_id NOT IN (SELECT run_id FROM faucet_serve_runs)"
529                .into(),
530            insert_audit: "INSERT INTO faucet_serve_audit \
531                (id, ts, principal, role, action, run_id, config_fingerprint, source_ip, result) \
532                VALUES ($1,$2,$3,$4,$5,$6,$7,$8,$9)"
533                .into(),
534            list_audit: "SELECT id, ts, principal, role, action, run_id, config_fingerprint, \
535                source_ip, result FROM faucet_serve_audit \
536                WHERE ($1::text IS NULL OR principal = $2::text) \
537                AND ($3::text IS NULL OR action = $4::text) \
538                AND ($5::text IS NULL OR ts >= $6::text) \
539                AND ($7::text IS NULL OR ts <= $8::text) \
540                ORDER BY ts DESC, id DESC LIMIT $9"
541                .into(),
542            purge_audit: "DELETE FROM faucet_serve_audit WHERE ts < $1".into(),
543            catalog_select_dataset: "SELECT body FROM faucet_catalog_datasets WHERE id=$1".into(),
544            catalog_upsert_dataset: "INSERT INTO faucet_catalog_datasets \
545                (id, uri, kind, last_seen, body) VALUES ($1,$2,$3,$4,$5) \
546                ON CONFLICT (id) DO UPDATE SET uri=excluded.uri, kind=excluded.kind, \
547                last_seen=excluded.last_seen, body=excluded.body"
548                .into(),
549            catalog_select_datasets: "SELECT body FROM faucet_catalog_datasets".into(),
550            catalog_insert_schema_version: "INSERT INTO faucet_catalog_schema_versions \
551                (dataset_id, version, recorded_at, body) VALUES ($1,$2,$3,$4) \
552                ON CONFLICT (dataset_id, version) DO NOTHING"
553                .into(),
554            catalog_select_schema_versions: "SELECT body FROM faucet_catalog_schema_versions \
555                WHERE dataset_id=$1 ORDER BY CAST(version AS BIGINT) ASC"
556                .into(),
557            catalog_upsert_edge: "INSERT INTO faucet_catalog_edges \
558                (src_id, dst_id, last_seen, body) VALUES ($1,$2,$3,$4) \
559                ON CONFLICT (src_id, dst_id) DO UPDATE SET \
560                last_seen=excluded.last_seen, body=excluded.body"
561                .into(),
562            catalog_select_edges: "SELECT body FROM faucet_catalog_edges \
563                ORDER BY last_seen DESC, src_id, dst_id"
564                .into(),
565            catalog_insert_stat: "INSERT INTO faucet_catalog_stats \
566                (dataset_id, recorded_at, run_id, records) VALUES ($1,$2,$3,$4) \
567                ON CONFLICT (dataset_id, recorded_at) DO NOTHING"
568                .into(),
569            catalog_select_stats: "SELECT recorded_at, run_id, records \
570                FROM faucet_catalog_stats WHERE dataset_id=$1 \
571                ORDER BY recorded_at DESC LIMIT $2"
572                .into(),
573            catalog_prune_stats: "DELETE FROM faucet_catalog_stats \
574                WHERE dataset_id=$1 AND recorded_at NOT IN (\
575                    SELECT recorded_at FROM faucet_catalog_stats WHERE dataset_id=$2 \
576                    ORDER BY recorded_at DESC LIMIT $3)"
577                .into(),
578            catalog_upsert_config_snapshot: "INSERT INTO faucet_config_snapshots \
579                (pipeline, recorded_at, faucet_version, body) VALUES ($1,$2,$3,$4) \
580                ON CONFLICT (pipeline) DO UPDATE SET recorded_at=excluded.recorded_at, \
581                faucet_version=excluded.faucet_version, body=excluded.body"
582                .into(),
583            catalog_select_config_snapshot:
584                "SELECT body FROM faucet_config_snapshots WHERE pipeline=$1".into(),
585            template_max_version: "SELECT COALESCE(MAX(CAST(version AS BIGINT)), 0) AS v \
586                FROM faucet_templates WHERE id=$1"
587                .into(),
588            template_insert: "INSERT INTO faucet_templates \
589                (id, version, name, created_at, body) VALUES ($1,$2,$3,$4,$5)"
590                .into(),
591            template_select_version:
592                "SELECT body FROM faucet_templates WHERE id=$1 AND version=$2".into(),
593            template_select_latest: "SELECT body FROM faucet_templates WHERE id=$1 \
594                ORDER BY CAST(version AS BIGINT) DESC LIMIT 1"
595                .into(),
596            template_select_all: "SELECT body FROM faucet_templates".into(),
597            template_versions: "SELECT version FROM faucet_templates WHERE id=$1 \
598                ORDER BY CAST(version AS BIGINT) DESC"
599                .into(),
600            template_delete_version: "DELETE FROM faucet_templates WHERE id=$1 AND version=$2"
601                .into(),
602            template_delete_all: "DELETE FROM faucet_templates WHERE id=$1".into(),
603            template_upsert_tag: "INSERT INTO faucet_template_tags \
604                (id, tag, version, updated_at) VALUES ($1,$2,$3,$4) \
605                ON CONFLICT (id, tag) DO UPDATE SET version=excluded.version, \
606                updated_at=excluded.updated_at"
607                .into(),
608            template_select_tags: "SELECT tag, version FROM faucet_template_tags \
609                WHERE id=$1 ORDER BY tag"
610                .into(),
611            template_delete_tag: "DELETE FROM faucet_template_tags WHERE id=$1 AND tag=$2".into(),
612            template_delete_tags_all: "DELETE FROM faucet_template_tags WHERE id=$1".into(),
613            template_delete_tags_for_version:
614                "DELETE FROM faucet_template_tags WHERE id=$1 AND version=$2".into(),
615            template_max_launch_seq: "SELECT COALESCE(MAX(CAST(seq AS BIGINT)), 0) AS v \
616                FROM faucet_template_launches WHERE id=$1"
617                .into(),
618            template_insert_launch: "INSERT INTO faucet_template_launches \
619                (id, seq, version, launched_at, launched_by) VALUES ($1,$2,$3,$4,$5)"
620                .into(),
621            template_select_launches: "SELECT seq, version, launched_at, launched_by \
622                FROM faucet_template_launches WHERE id=$1 ORDER BY CAST(seq AS BIGINT) DESC"
623                .into(),
624            template_delete_launches_all: "DELETE FROM faucet_template_launches WHERE id=$1".into(),
625            template_delete_launches_for_version:
626                "DELETE FROM faucet_template_launches WHERE id=$1 AND version=$2".into(),
627            template_upsert_deprecation: "INSERT INTO faucet_template_deprecations \
628                (id, deprecated_at, deprecated_by, reason) VALUES ($1,$2,$3,$4) \
629                ON CONFLICT (id) DO UPDATE SET deprecated_at=excluded.deprecated_at, \
630                deprecated_by=excluded.deprecated_by, reason=excluded.reason"
631                .into(),
632            template_select_deprecation: "SELECT deprecated_at, deprecated_by, reason \
633                FROM faucet_template_deprecations WHERE id=$1"
634                .into(),
635            template_delete_deprecation: "DELETE FROM faucet_template_deprecations WHERE id=$1"
636                .into(),
637        }
638    }
639
640    fn sqlite() -> Self {
641        Self {
642            upsert: "INSERT INTO faucet_serve_runs \
643                (run_id,name,status,submitted_at,finished_at,idempotency_key,owner,lease_expires_at,body) \
644                VALUES (?,?,?,?,?,?,?,?,?) \
645                ON CONFLICT (run_id) DO UPDATE SET \
646                name=excluded.name,status=excluded.status,submitted_at=excluded.submitted_at,\
647                finished_at=excluded.finished_at,idempotency_key=excluded.idempotency_key,\
648                owner=excluded.owner,lease_expires_at=excluded.lease_expires_at,\
649                body=excluded.body"
650                .into(),
651            select_body: "SELECT body FROM faucet_serve_runs WHERE run_id=?".into(),
652            select_status: "SELECT status FROM faucet_serve_runs WHERE run_id=?".into(),
653            select_submitted: "SELECT submitted_at FROM faucet_serve_runs WHERE run_id=?".into(),
654            delete: "DELETE FROM faucet_serve_runs WHERE run_id=?".into(),
655            list: "SELECT body FROM faucet_serve_runs \
656                WHERE (? IS NULL OR status = ?) \
657                AND (? IS NULL OR name = ?) \
658                AND (? IS NULL OR submitted_at >= ?) \
659                AND (? IS NULL OR submitted_at <= ?) \
660                AND (? IS NULL OR (submitted_at < ? \
661                    OR (submitted_at = ? AND run_id < ?))) \
662                ORDER BY submitted_at DESC, run_id DESC LIMIT ?"
663                .into(),
664            purge_runs: "DELETE FROM faucet_serve_runs \
665                WHERE status IN ('completed','failed','cancelled') \
666                AND finished_at IS NOT NULL AND finished_at < ?"
667                .into(),
668            purge_idem: "DELETE FROM faucet_serve_idem WHERE claimed_at < ?".into(),
669            select_orphans: "SELECT body FROM faucet_serve_runs \
670                WHERE status IN ('queued','running') \
671                AND (lease_expires_at IS NULL OR lease_expires_at < ?)"
672                .into(),
673            renew_leases: "UPDATE faucet_serve_runs SET lease_expires_at = ? \
674                WHERE owner = ? AND status IN ('queued','running')"
675                .into(),
676            insert_idem: "INSERT INTO faucet_serve_idem (key,run_id,fingerprint,claimed_at) \
677                VALUES (?,?,?,?) ON CONFLICT (key) DO NOTHING"
678                .into(),
679            select_idem: "SELECT run_id,fingerprint,claimed_at FROM faucet_serve_idem WHERE key=?"
680                .into(),
681            takeover_idem: "UPDATE faucet_serve_idem \
682                SET run_id=?,fingerprint=?,claimed_at=? WHERE key=? AND claimed_at=?"
683                .into(),
684            delete_idem_by_run: "DELETE FROM faucet_serve_idem WHERE run_id=?".into(),
685            select_pending: "SELECT run_id, body FROM faucet_serve_runs \
686                WHERE status = 'pending' ORDER BY submitted_at ASC LIMIT ?"
687                .into(),
688            claim_one: "UPDATE faucet_serve_runs \
689                SET owner = ?, status = 'running', lease_expires_at = ?, body = ? \
690                WHERE run_id = ? AND status = 'pending'"
691                .into(),
692            reclaim_select: "SELECT body FROM faucet_serve_runs \
693                WHERE status = 'running' \
694                AND (lease_expires_at IS NULL OR lease_expires_at < ?)"
695                .into(),
696            // Preserve `cancel_requested` across a requeue (audit #321 M7).
697            reclaim_requeue: "UPDATE faucet_serve_runs \
698                SET status = 'pending', owner = NULL, lease_expires_at = NULL, \
699                    body = ? \
700                WHERE run_id = ? AND status = 'running' \
701                AND (lease_expires_at IS NULL OR lease_expires_at < ?)"
702                .into(),
703            reclaim_fail: "UPDATE faucet_serve_runs \
704                SET status = 'failed', finished_at = ?, body = ?, owner = NULL \
705                WHERE run_id = ? AND status = 'running' \
706                AND (lease_expires_at IS NULL OR lease_expires_at < ?)"
707                .into(),
708            // Status-fenced (audit #321 L5): first finalizer wins.
709            finalize_owned: "UPDATE faucet_serve_runs \
710                SET status = ?, finished_at = ?, lease_expires_at = ?, body = ? \
711                WHERE run_id = ? AND owner = ? \
712                AND status NOT IN ('completed','failed','cancelled')"
713                .into(),
714            cancel_pending: "UPDATE faucet_serve_runs \
715                SET status = 'cancelled', finished_at = ?, body = ? \
716                WHERE run_id = ? AND status = 'pending'"
717                .into(),
718            request_cancel: "UPDATE faucet_serve_runs \
719                SET cancel_requested = ? WHERE run_id = ? AND status IN ('running','sharded')"
720                .into(),
721            pending_cancellations: "SELECT run_id FROM faucet_serve_runs \
722                WHERE status = 'running' AND owner = ? AND cancel_requested IS NOT NULL"
723                .into(),
724            heartbeat_instance: "INSERT INTO faucet_serve_instances \
725                (instance_id, started_at, last_heartbeat, listen, max_concurrent, in_flight) \
726                VALUES (?,?,?,?,?,?) \
727                ON CONFLICT (instance_id) DO UPDATE SET \
728                last_heartbeat = excluded.last_heartbeat, listen = excluded.listen, \
729                max_concurrent = excluded.max_concurrent, in_flight = excluded.in_flight"
730                .into(),
731            live_instances: "SELECT instance_id, started_at, last_heartbeat, listen, \
732                max_concurrent, in_flight FROM faucet_serve_instances \
733                WHERE last_heartbeat >= ?"
734                .into(),
735            prune_instances: "DELETE FROM faucet_serve_instances WHERE last_heartbeat < ?".into(),
736            insert_shard: "INSERT INTO faucet_serve_shards \
737                (run_id, shard_id, descriptor, size_estimate, status, attempt) \
738                VALUES (?,?,?,?,'pending','0') \
739                ON CONFLICT (run_id, shard_id) DO NOTHING"
740                .into(),
741            claim_shards_select: "SELECT s.run_id, s.shard_id, s.descriptor, r.body \
742                FROM faucet_serve_shards s JOIN faucet_serve_runs r ON r.run_id = s.run_id \
743                WHERE s.status = 'pending' \
744                ORDER BY CAST(COALESCE(s.size_estimate, '0') AS INTEGER) DESC, s.run_id, s.shard_id \
745                LIMIT ?"
746                .into(),
747            claim_shard_one: "UPDATE faucet_serve_shards \
748                SET owner = ?, status = 'running', lease_expires_at = ? \
749                WHERE run_id = ? AND shard_id = ? AND status = 'pending'"
750                .into(),
751            renew_shard_leases: "UPDATE faucet_serve_shards SET lease_expires_at = ? \
752                WHERE owner = ? AND status = 'running'"
753                .into(),
754            reclaim_shards_select: "SELECT run_id, shard_id, attempt FROM faucet_serve_shards \
755                WHERE status = 'running' \
756                AND (lease_expires_at IS NULL OR lease_expires_at < ?)"
757                .into(),
758            reclaim_shard_requeue: "UPDATE faucet_serve_shards \
759                SET status = 'pending', owner = NULL, lease_expires_at = NULL, attempt = ? \
760                WHERE run_id = ? AND shard_id = ? AND status = 'running' \
761                AND (lease_expires_at IS NULL OR lease_expires_at < ?)"
762                .into(),
763            reclaim_shard_fail: "UPDATE faucet_serve_shards \
764                SET status = 'failed', finished_at = ?, owner = NULL \
765                WHERE run_id = ? AND shard_id = ? AND status = 'running' \
766                AND (lease_expires_at IS NULL OR lease_expires_at < ?)"
767                .into(),
768            finalize_shard: "UPDATE faucet_serve_shards \
769                SET status = ?, finished_at = ? \
770                WHERE run_id = ? AND shard_id = ? AND owner = ? AND status = 'running'"
771                .into(),
772            shard_progress: "SELECT status, COUNT(*) AS n FROM faucet_serve_shards \
773                WHERE run_id = ? GROUP BY status"
774                .into(),
775            pending_shard_cancellations: "SELECT DISTINCT s.run_id \
776                FROM faucet_serve_shards s \
777                JOIN faucet_serve_runs r ON r.run_id = s.run_id \
778                WHERE s.owner = ? AND s.status = 'running' \
779                AND r.cancel_requested IS NOT NULL"
780                .into(),
781            select_sharded_parents: "SELECT run_id FROM faucet_serve_runs \
782                WHERE status = 'sharded'"
783                .into(),
784            finalize_sharded_parent: "UPDATE faucet_serve_runs \
785                SET status = ?, finished_at = ?, body = ? \
786                WHERE run_id = ? AND status = 'sharded'"
787                .into(),
788            delete_shards_by_run: "DELETE FROM faucet_serve_shards WHERE run_id = ?".into(),
789            purge_orphan_shards: "DELETE FROM faucet_serve_shards \
790                WHERE run_id NOT IN (SELECT run_id FROM faucet_serve_runs)"
791                .into(),
792            insert_audit: "INSERT INTO faucet_serve_audit \
793                (id, ts, principal, role, action, run_id, config_fingerprint, source_ip, result) \
794                VALUES (?,?,?,?,?,?,?,?,?)"
795                .into(),
796            list_audit: "SELECT id, ts, principal, role, action, run_id, config_fingerprint, \
797                source_ip, result FROM faucet_serve_audit \
798                WHERE (? IS NULL OR principal = ?) \
799                AND (? IS NULL OR action = ?) \
800                AND (? IS NULL OR ts >= ?) \
801                AND (? IS NULL OR ts <= ?) \
802                ORDER BY ts DESC, id DESC LIMIT ?"
803                .into(),
804            purge_audit: "DELETE FROM faucet_serve_audit WHERE ts < ?".into(),
805            catalog_select_dataset: "SELECT body FROM faucet_catalog_datasets WHERE id=?".into(),
806            catalog_upsert_dataset: "INSERT INTO faucet_catalog_datasets \
807                (id, uri, kind, last_seen, body) VALUES (?,?,?,?,?) \
808                ON CONFLICT (id) DO UPDATE SET uri=excluded.uri, kind=excluded.kind, \
809                last_seen=excluded.last_seen, body=excluded.body"
810                .into(),
811            catalog_select_datasets: "SELECT body FROM faucet_catalog_datasets".into(),
812            catalog_insert_schema_version: "INSERT INTO faucet_catalog_schema_versions \
813                (dataset_id, version, recorded_at, body) VALUES (?,?,?,?) \
814                ON CONFLICT (dataset_id, version) DO NOTHING"
815                .into(),
816            catalog_select_schema_versions: "SELECT body FROM faucet_catalog_schema_versions \
817                WHERE dataset_id=? ORDER BY CAST(version AS INTEGER) ASC"
818                .into(),
819            catalog_upsert_edge: "INSERT INTO faucet_catalog_edges \
820                (src_id, dst_id, last_seen, body) VALUES (?,?,?,?) \
821                ON CONFLICT (src_id, dst_id) DO UPDATE SET \
822                last_seen=excluded.last_seen, body=excluded.body"
823                .into(),
824            catalog_select_edges: "SELECT body FROM faucet_catalog_edges \
825                ORDER BY last_seen DESC, src_id, dst_id"
826                .into(),
827            catalog_insert_stat: "INSERT INTO faucet_catalog_stats \
828                (dataset_id, recorded_at, run_id, records) VALUES (?,?,?,?) \
829                ON CONFLICT (dataset_id, recorded_at) DO NOTHING"
830                .into(),
831            catalog_select_stats: "SELECT recorded_at, run_id, records \
832                FROM faucet_catalog_stats WHERE dataset_id=? \
833                ORDER BY recorded_at DESC LIMIT ?"
834                .into(),
835            catalog_prune_stats: "DELETE FROM faucet_catalog_stats \
836                WHERE dataset_id=? AND recorded_at NOT IN (\
837                    SELECT recorded_at FROM faucet_catalog_stats WHERE dataset_id=? \
838                    ORDER BY recorded_at DESC LIMIT ?)"
839                .into(),
840            catalog_upsert_config_snapshot: "INSERT INTO faucet_config_snapshots \
841                (pipeline, recorded_at, faucet_version, body) VALUES (?,?,?,?) \
842                ON CONFLICT (pipeline) DO UPDATE SET recorded_at=excluded.recorded_at, \
843                faucet_version=excluded.faucet_version, body=excluded.body"
844                .into(),
845            catalog_select_config_snapshot:
846                "SELECT body FROM faucet_config_snapshots WHERE pipeline=?".into(),
847            template_max_version: "SELECT COALESCE(MAX(CAST(version AS INTEGER)), 0) AS v \
848                FROM faucet_templates WHERE id=?"
849                .into(),
850            template_insert: "INSERT INTO faucet_templates \
851                (id, version, name, created_at, body) VALUES (?,?,?,?,?)"
852                .into(),
853            template_select_version: "SELECT body FROM faucet_templates WHERE id=? AND version=?"
854                .into(),
855            template_select_latest: "SELECT body FROM faucet_templates WHERE id=? \
856                ORDER BY CAST(version AS INTEGER) DESC LIMIT 1"
857                .into(),
858            template_select_all: "SELECT body FROM faucet_templates".into(),
859            template_versions: "SELECT version FROM faucet_templates WHERE id=? \
860                ORDER BY CAST(version AS INTEGER) DESC"
861                .into(),
862            template_delete_version: "DELETE FROM faucet_templates WHERE id=? AND version=?".into(),
863            template_delete_all: "DELETE FROM faucet_templates WHERE id=?".into(),
864            template_upsert_tag: "INSERT INTO faucet_template_tags \
865                (id, tag, version, updated_at) VALUES (?,?,?,?) \
866                ON CONFLICT (id, tag) DO UPDATE SET version=excluded.version, \
867                updated_at=excluded.updated_at"
868                .into(),
869            template_select_tags: "SELECT tag, version FROM faucet_template_tags \
870                WHERE id=? ORDER BY tag"
871                .into(),
872            template_delete_tag: "DELETE FROM faucet_template_tags WHERE id=? AND tag=?".into(),
873            template_delete_tags_all: "DELETE FROM faucet_template_tags WHERE id=?".into(),
874            template_delete_tags_for_version:
875                "DELETE FROM faucet_template_tags WHERE id=? AND version=?".into(),
876            template_max_launch_seq: "SELECT COALESCE(MAX(CAST(seq AS INTEGER)), 0) AS v \
877                FROM faucet_template_launches WHERE id=?"
878                .into(),
879            template_insert_launch: "INSERT INTO faucet_template_launches \
880                (id, seq, version, launched_at, launched_by) VALUES (?,?,?,?,?)"
881                .into(),
882            template_select_launches: "SELECT seq, version, launched_at, launched_by \
883                FROM faucet_template_launches WHERE id=? ORDER BY CAST(seq AS INTEGER) DESC"
884                .into(),
885            template_delete_launches_all: "DELETE FROM faucet_template_launches WHERE id=?".into(),
886            template_delete_launches_for_version:
887                "DELETE FROM faucet_template_launches WHERE id=? AND version=?".into(),
888            template_upsert_deprecation: "INSERT INTO faucet_template_deprecations \
889                (id, deprecated_at, deprecated_by, reason) VALUES (?,?,?,?) \
890                ON CONFLICT (id) DO UPDATE SET deprecated_at=excluded.deprecated_at, \
891                deprecated_by=excluded.deprecated_by, reason=excluded.reason"
892                .into(),
893            template_select_deprecation: "SELECT deprecated_at, deprecated_by, reason \
894                FROM faucet_template_deprecations WHERE id=?"
895                .into(),
896            template_delete_deprecation: "DELETE FROM faucet_template_deprecations WHERE id=?"
897                .into(),
898        }
899    }
900}
901
902/// Bounded retry count for the atomic idempotency claim (handles a claim being
903/// purged concurrently between the insert attempt and the read-back) and for the
904/// read-max-then-insert paths (template versions, launch-log seqs).
905///
906/// Sized for *contention*, not just for a lost race. SQLite serializes writers
907/// and answers a write-write overlap on a deferred transaction with `database is
908/// locked` **immediately** (waiting would deadlock), so `busy_timeout` does not
909/// help there and the attempt budget is the only thing standing between a
910/// concurrent writer and a surfaced error. With N concurrent writers each needing
911/// its own turn, a budget of 4 is thinner than it looks: six writers on a loaded
912/// machine exhausted it and failed a register (#457 CI).
913pub const CLAIM_ATTEMPTS: usize = 8;
914
915/// Distinguishes concurrent retriers so their backoffs do not re-collide.
916///
917/// Every waiter sleeping the *same* duration just reproduces the same race one
918/// beat later. A per-call sequence number is a dependency-free way to stagger
919/// them (the alternative, a random jitter, would mean pulling in an RNG here).
920static RETRY_SEQ: std::sync::atomic::AtomicUsize = std::sync::atomic::AtomicUsize::new(0);
921
922/// Sleep before a read-max-then-insert retry (no-op on the first attempt).
923///
924/// Exponential (5ms, 10ms, 20ms, …, capped) plus a per-caller stagger, so a set
925/// of writers that collided on attempt 1 spreads out instead of colliding again.
926/// Worst case across the whole budget is a few hundred milliseconds — cheap next
927/// to failing a write the caller expected to succeed.
928pub async fn retry_backoff(attempt: usize) {
929    if attempt <= 1 {
930        return;
931    }
932    const BASE_MS: u64 = 5;
933    const CAP_MS: u64 = 160;
934    let exp = BASE_MS
935        .saturating_mul(1u64 << (attempt - 2).min(6))
936        .min(CAP_MS);
937    // 0..BASE_MS of per-caller offset, so equal-length sleeps do not re-align.
938    let stagger =
939        (RETRY_SEQ.fetch_add(1, std::sync::atomic::Ordering::Relaxed) as u64) % (BASE_MS + 1);
940    tokio::time::sleep(std::time::Duration::from_millis(exp + stagger)).await;
941}
942
943/// Fixed-width RFC3339 (nanoseconds + `Z`) — lexicographically sortable.
944pub fn fmt_ts(dt: DateTime<Utc>) -> String {
945    dt.to_rfc3339_opts(chrono::SecondsFormat::Nanos, true)
946}
947
948/// Inverse of [`fmt_ts`]. An unparseable value falls back to *now* rather than
949/// failing the read: a single corrupt timestamp must not make a whole template or
950/// audit row unusable, and every caller uses the value for display/ordering only.
951pub fn parse_ts(raw: &str) -> DateTime<Utc> {
952    DateTime::parse_from_rfc3339(raw)
953        .map(|d| d.to_utc())
954        .unwrap_or_else(|_| Utc::now())
955}
956
957/// True when a claim timestamped `claimed_at` (RFC3339) is older than `window`.
958/// An unparseable or future timestamp is treated as **not** expired (safe: it
959/// won't be silently re-claimed).
960pub fn is_expired(claimed_at: &str, now: DateTime<Utc>, window: Duration) -> bool {
961    match DateTime::parse_from_rfc3339(claimed_at) {
962        Ok(t) => now
963            .signed_duration_since(t.with_timezone(&Utc))
964            .to_std()
965            .map(|age| age >= window)
966            .unwrap_or(false),
967        Err(_) => false,
968    }
969}
970
971/// RFC3339 timestamp `window` before `now` (the purge / expiry threshold).
972pub fn threshold(now: DateTime<Utc>, window: Duration) -> String {
973    let delta =
974        chrono::Duration::from_std(window).unwrap_or_else(|_| chrono::Duration::days(36_500));
975    fmt_ts(now - delta)
976}
977
978pub fn encode_body(rec: &RunRecord) -> Result<String, HistoryError> {
979    serde_json::to_string(rec).map_err(|e| HistoryError::Backend(format!("encode run record: {e}")))
980}
981
982pub fn decode_body(body: &str) -> Result<RunRecord, HistoryError> {
983    serde_json::from_str(body).map_err(|e| HistoryError::Backend(format!("decode run record: {e}")))
984}
985
986/// Generic body (de)serialization for the catalog tables (#279).
987pub fn encode_json<T: serde::Serialize>(value: &T, what: &str) -> Result<String, HistoryError> {
988    serde_json::to_string(value).map_err(|e| HistoryError::Backend(format!("encode {what}: {e}")))
989}
990
991pub fn decode_json<T: serde::de::DeserializeOwned>(
992    body: &str,
993    what: &str,
994) -> Result<T, HistoryError> {
995    serde_json::from_str(body).map_err(|e| HistoryError::Backend(format!("decode {what}: {e}")))
996}
997
998pub fn parse_status(s: &str) -> RunStatus {
999    match s {
1000        "queued" => RunStatus::Queued,
1001        "pending" => RunStatus::Pending,
1002        "running" => RunStatus::Running,
1003        "sharded" => RunStatus::Sharded,
1004        "completed" => RunStatus::Completed,
1005        "cancelled" => RunStatus::Cancelled,
1006        _ => RunStatus::Failed,
1007    }
1008}
1009
1010/// Generate a concrete `RunHistory` implementation over a specific `sqlx` pool.
1011/// `$name` is the backend struct, `$pool` its `sqlx` pool type. The struct holds
1012/// the pool, the idempotency retention window, and the dialect's [`Stmts`].
1013macro_rules! impl_sql_history {
1014    ($name:ident, $pool:ty) => {
1015        /// SQL-backed [`RunHistory`](crate::serve::history::RunHistory). See
1016        /// [`crate::serve::history::sql`] for the shared schema + semantics.
1017        pub struct $name {
1018            pool: $pool,
1019            idem_retention: std::time::Duration,
1020            /// This serve instance's id, stamped as `owner` on every upsert.
1021            instance_id: String,
1022            /// How far ahead each upsert / heartbeat pushes a run's lease.
1023            lease_ttl: std::time::Duration,
1024            stmts: $crate::serve::history::sql::Stmts,
1025        }
1026
1027        impl $name {
1028            /// Assemble from an already-connected pool (used by `connect`).
1029            pub fn from_parts(
1030                pool: $pool,
1031                idem_retention: std::time::Duration,
1032                lease_ttl: std::time::Duration,
1033                instance_id: String,
1034                stmts: $crate::serve::history::sql::Stmts,
1035            ) -> Self {
1036                Self {
1037                    pool,
1038                    idem_retention,
1039                    instance_id,
1040                    lease_ttl,
1041                    stmts,
1042                }
1043            }
1044
1045            /// Borrow the underlying pool (tests close it to exercise fallback).
1046            pub fn pool(&self) -> &$pool {
1047                &self.pool
1048            }
1049        }
1050
1051        #[async_trait::async_trait]
1052        impl $crate::serve::history::RunHistory for $name {
1053            async fn claim_idempotency(
1054                &self,
1055                key: &str,
1056                fingerprint: &str,
1057                run_id: &str,
1058                window: std::time::Duration,
1059            ) -> Result<$crate::serve::history::Claim, $crate::serve::history::HistoryError> {
1060                use sqlx::Row as _;
1061                use $crate::serve::history::Claim;
1062                use $crate::serve::history::HistoryError;
1063                use $crate::serve::history::sql;
1064
1065                let now = chrono::Utc::now();
1066                let now_s = sql::fmt_ts(now);
1067                let backend = |e: sqlx::Error| HistoryError::Backend(e.to_string());
1068
1069                for _ in 0..sql::CLAIM_ATTEMPTS {
1070                    // 1) Atomic first-claim: the winner inserts exactly one row.
1071                    let inserted = sqlx::query(&self.stmts.insert_idem)
1072                        .bind(key)
1073                        .bind(run_id)
1074                        .bind(fingerprint)
1075                        .bind(&now_s)
1076                        .execute(&self.pool)
1077                        .await
1078                        .map_err(backend)?
1079                        .rows_affected();
1080                    if inserted == 1 {
1081                        return Ok(Claim::Fresh);
1082                    }
1083                    // 2) Conflict: inspect the existing claim.
1084                    let Some(row) = sqlx::query(&self.stmts.select_idem)
1085                        .bind(key)
1086                        .fetch_optional(&self.pool)
1087                        .await
1088                        .map_err(backend)?
1089                    else {
1090                        // Vanished between the insert and the read — retry.
1091                        continue;
1092                    };
1093                    let existing_run: String = row.try_get("run_id").map_err(backend)?;
1094                    let existing_fp: String = row.try_get("fingerprint").map_err(backend)?;
1095                    let claimed_at: String = row.try_get("claimed_at").map_err(backend)?;
1096
1097                    if sql::is_expired(&claimed_at, now, window) {
1098                        // 3) Optimistic, expiry-guarded takeover: only the request
1099                        // that still sees `claimed_at` succeeds.
1100                        let took = sqlx::query(&self.stmts.takeover_idem)
1101                            .bind(run_id)
1102                            .bind(fingerprint)
1103                            .bind(&now_s)
1104                            .bind(key)
1105                            .bind(&claimed_at)
1106                            .execute(&self.pool)
1107                            .await
1108                            .map_err(backend)?
1109                            .rows_affected();
1110                        if took == 1 {
1111                            return Ok(Claim::Fresh);
1112                        }
1113                        continue; // lost the race; re-evaluate
1114                    }
1115                    return Ok(if existing_fp == fingerprint {
1116                        Claim::Replay(existing_run)
1117                    } else {
1118                        Claim::Conflict
1119                    });
1120                }
1121                // Pathological contention only. Conservative: a 409 is safer than
1122                // risking a duplicate run.
1123                tracing::warn!(
1124                    key,
1125                    "idempotency claim exhausted retries; reporting conflict"
1126                );
1127                Ok(Claim::Conflict)
1128            }
1129
1130            async fn upsert(
1131                &self,
1132                rec: &$crate::serve::history::RunRecord,
1133            ) -> Result<(), $crate::serve::history::HistoryError> {
1134                use $crate::serve::history::HistoryError;
1135                use $crate::serve::history::sql;
1136                let body = sql::encode_body(rec)?;
1137                let submitted = sql::fmt_ts(rec.submitted_at);
1138                let finished = rec.finished_at.map(sql::fmt_ts);
1139                // Stamp this instance as the owner and start/renew the lease.
1140                // The owner/lease are SQL-column-only (never in the record body),
1141                // so the heartbeat can extend a lease without a body read-modify-
1142                // write race (#146 H7).
1143                let lease = sql::fmt_ts(chrono::Utc::now() + self.lease_ttl);
1144                sqlx::query(&self.stmts.upsert)
1145                    .bind(&rec.run_id)
1146                    .bind(rec.name.as_deref())
1147                    .bind(rec.status.as_str())
1148                    .bind(&submitted)
1149                    .bind(finished.as_deref())
1150                    .bind(rec.idempotency_key.as_deref())
1151                    .bind(&self.instance_id)
1152                    .bind(&lease)
1153                    .bind(&body)
1154                    .execute(&self.pool)
1155                    .await
1156                    .map_err(|e| HistoryError::Backend(e.to_string()))?;
1157                Ok(())
1158            }
1159
1160            async fn get(
1161                &self,
1162                id: &str,
1163            ) -> Result<
1164                Option<$crate::serve::history::RunRecord>,
1165                $crate::serve::history::HistoryError,
1166            > {
1167                use sqlx::Row as _;
1168                use $crate::serve::history::HistoryError;
1169                use $crate::serve::history::sql;
1170                let row = sqlx::query(&self.stmts.select_body)
1171                    .bind(id)
1172                    .fetch_optional(&self.pool)
1173                    .await
1174                    .map_err(|e| HistoryError::Backend(e.to_string()))?;
1175                match row {
1176                    None => Ok(None),
1177                    Some(r) => {
1178                        let body: String = r
1179                            .try_get("body")
1180                            .map_err(|e| HistoryError::Backend(e.to_string()))?;
1181                        Ok(Some(sql::decode_body(&body)?))
1182                    }
1183                }
1184            }
1185
1186            async fn list(
1187                &self,
1188                filter: &$crate::serve::history::ListFilter,
1189            ) -> Result<$crate::serve::history::ListPage, $crate::serve::history::HistoryError>
1190            {
1191                use sqlx::Row as _;
1192                use $crate::serve::history::HistoryError;
1193                use $crate::serve::history::ListPage;
1194                use $crate::serve::history::sql;
1195                let backend = |e: sqlx::Error| HistoryError::Backend(e.to_string());
1196
1197                // Resolve the cursor's submitted_at for keyset pagination. An
1198                // unknown cursor is ignored (page starts from the top), matching
1199                // the memory backend.
1200                let cursor_ts: Option<String> = match &filter.cursor {
1201                    None => None,
1202                    Some(c) => sqlx::query(&self.stmts.select_submitted)
1203                        .bind(c)
1204                        .fetch_optional(&self.pool)
1205                        .await
1206                        .map_err(backend)?
1207                        .map(|r| r.try_get::<String, _>("submitted_at"))
1208                        .transpose()
1209                        .map_err(backend)?,
1210                };
1211                let cur_id = if cursor_ts.is_some() {
1212                    filter.cursor.as_deref()
1213                } else {
1214                    None
1215                };
1216
1217                let status_s = filter.status.map(|s| s.as_str());
1218                let name_s = filter.name.as_deref();
1219                let since_s = filter.since.map(sql::fmt_ts);
1220                let until_s = filter.until.map(sql::fmt_ts);
1221                let limit = filter.limit.max(1);
1222                let fetch_n = limit as i64 + 1; // +1 to detect a next page
1223
1224                let rows = sqlx::query(&self.stmts.list)
1225                    .bind(status_s)
1226                    .bind(status_s)
1227                    .bind(name_s)
1228                    .bind(name_s)
1229                    .bind(since_s.as_deref())
1230                    .bind(since_s.as_deref())
1231                    .bind(until_s.as_deref())
1232                    .bind(until_s.as_deref())
1233                    .bind(cursor_ts.as_deref())
1234                    .bind(cursor_ts.as_deref())
1235                    .bind(cursor_ts.as_deref())
1236                    .bind(cur_id)
1237                    .bind(fetch_n)
1238                    .fetch_all(&self.pool)
1239                    .await
1240                    .map_err(backend)?;
1241
1242                let mut runs = Vec::with_capacity(rows.len());
1243                for r in &rows {
1244                    let body: String = r.try_get("body").map_err(backend)?;
1245                    runs.push(sql::decode_body(&body)?);
1246                }
1247                let next_cursor = if runs.len() > limit {
1248                    Some(runs[limit - 1].run_id.clone())
1249                } else {
1250                    None
1251                };
1252                runs.truncate(limit);
1253                Ok(ListPage { runs, next_cursor })
1254            }
1255
1256            async fn delete(
1257                &self,
1258                id: &str,
1259            ) -> Result<$crate::serve::history::DeleteOutcome, $crate::serve::history::HistoryError>
1260            {
1261                use sqlx::Row as _;
1262                use $crate::serve::history::DeleteOutcome;
1263                use $crate::serve::history::HistoryError;
1264                use $crate::serve::history::sql;
1265                let backend = |e: sqlx::Error| HistoryError::Backend(e.to_string());
1266                let status: Option<String> = sqlx::query(&self.stmts.select_status)
1267                    .bind(id)
1268                    .fetch_optional(&self.pool)
1269                    .await
1270                    .map_err(backend)?
1271                    .map(|r| r.try_get::<String, _>("status"))
1272                    .transpose()
1273                    .map_err(backend)?;
1274                match status {
1275                    None => Ok(DeleteOutcome::NotFound),
1276                    Some(s) if !sql::parse_status(&s).is_terminal() => {
1277                        Ok(DeleteOutcome::StillRunning)
1278                    }
1279                    Some(_) => {
1280                        sqlx::query(&self.stmts.delete)
1281                            .bind(id)
1282                            .execute(&self.pool)
1283                            .await
1284                            .map_err(backend)?;
1285                        // Drop the run's idempotency claim too, so a replay of
1286                        // the key starts fresh instead of 404-ing on the deleted
1287                        // record until the claim self-expires (#146 M8). Scoped
1288                        // by run_id, so a newer run that re-claimed the same key
1289                        // keeps its claim.
1290                        sqlx::query(&self.stmts.delete_idem_by_run)
1291                            .bind(id)
1292                            .execute(&self.pool)
1293                            .await
1294                            .map_err(backend)?;
1295                        // Drop the run's shard rows too (Mode B, #230), so a
1296                        // deleted run leaves no orphaned shard rows that would
1297                        // otherwise leak unboundedly (F25).
1298                        sqlx::query(&self.stmts.delete_shards_by_run)
1299                            .bind(id)
1300                            .execute(&self.pool)
1301                            .await
1302                            .map_err(backend)?;
1303                        Ok(DeleteOutcome::Deleted)
1304                    }
1305                }
1306            }
1307
1308            async fn release_idempotency(
1309                &self,
1310                run_id: &str,
1311            ) -> Result<(), $crate::serve::history::HistoryError> {
1312                use $crate::serve::history::HistoryError;
1313                sqlx::query(&self.stmts.delete_idem_by_run)
1314                    .bind(run_id)
1315                    .execute(&self.pool)
1316                    .await
1317                    .map_err(|e| HistoryError::Backend(e.to_string()))?;
1318                Ok(())
1319            }
1320
1321            async fn purge_expired(
1322                &self,
1323                retain_for: std::time::Duration,
1324            ) -> Result<usize, $crate::serve::history::HistoryError> {
1325                use $crate::serve::history::HistoryError;
1326                use $crate::serve::history::sql;
1327                let backend = |e: sqlx::Error| HistoryError::Backend(e.to_string());
1328                let now = chrono::Utc::now();
1329                let removed = sqlx::query(&self.stmts.purge_runs)
1330                    .bind(sql::threshold(now, retain_for))
1331                    .execute(&self.pool)
1332                    .await
1333                    .map_err(backend)?
1334                    .rows_affected() as usize;
1335                // Drop expired idempotency claims too (best-effort).
1336                let _ = sqlx::query(&self.stmts.purge_idem)
1337                    .bind(sql::threshold(now, self.idem_retention))
1338                    .execute(&self.pool)
1339                    .await;
1340                // Drop membership rows that have not heartbeated within the
1341                // run-retention window (far longer than the lease, so this never
1342                // prunes a live member — that's `live_instances(ttl)`'s job).
1343                let _ = sqlx::query(&self.stmts.prune_instances)
1344                    .bind(sql::threshold(now, retain_for))
1345                    .execute(&self.pool)
1346                    .await;
1347                // Reclaim shard rows whose parent run was just purged (F25):
1348                // `purge_runs` removed the expired terminal records above, so any
1349                // shard row no longer matching a run is orphaned. Best-effort.
1350                let _ = sqlx::query(&self.stmts.purge_orphan_shards)
1351                    .execute(&self.pool)
1352                    .await;
1353                // Drop audit records older than the run-retention window (#205).
1354                let _ = sqlx::query(&self.stmts.purge_audit)
1355                    .bind(sql::threshold(now, retain_for))
1356                    .execute(&self.pool)
1357                    .await;
1358                Ok(removed)
1359            }
1360
1361            async fn recover_orphans(&self) -> Result<usize, $crate::serve::history::HistoryError> {
1362                use sqlx::Row as _;
1363                use $crate::serve::history::HistoryError;
1364                use $crate::serve::history::RunStatus;
1365                use $crate::serve::history::sql;
1366                let backend = |e: sqlx::Error| HistoryError::Backend(e.to_string());
1367                let now = chrono::Utc::now();
1368                // Only non-terminal runs whose lease has expired (the owning
1369                // instance is presumed dead). A live instance heartbeats its
1370                // runs' leases into the future, so this never fails another
1371                // healthy instance's in-flight runs (#146 H7).
1372                let rows = sqlx::query(&self.stmts.select_orphans)
1373                    .bind(sql::fmt_ts(now))
1374                    .fetch_all(&self.pool)
1375                    .await
1376                    .map_err(backend)?;
1377                let mut count = 0usize;
1378                for r in &rows {
1379                    let body: String = r.try_get("body").map_err(backend)?;
1380                    let mut rec = sql::decode_body(&body)?;
1381                    rec.status = RunStatus::Failed;
1382                    rec.finished_at = Some(now);
1383                    rec.error = Some(
1384                        "owning serve instance's lease expired before the run finished".into(),
1385                    );
1386                    if rec.elapsed_secs.is_none()
1387                        && let Some(started) = rec.started_at
1388                    {
1389                        rec.elapsed_secs = (now - started).to_std().ok().map(|d| d.as_secs_f64());
1390                    }
1391                    self.upsert(&rec).await?;
1392                    count += 1;
1393                }
1394                Ok(count)
1395            }
1396
1397            async fn renew_leases(&self) -> Result<usize, $crate::serve::history::HistoryError> {
1398                use $crate::serve::history::HistoryError;
1399                use $crate::serve::history::sql;
1400                let backend = |e: sqlx::Error| HistoryError::Backend(e.to_string());
1401                let new_lease = sql::fmt_ts(chrono::Utc::now() + self.lease_ttl);
1402                let renewed = sqlx::query(&self.stmts.renew_leases)
1403                    .bind(&new_lease)
1404                    .bind(&self.instance_id)
1405                    .execute(&self.pool)
1406                    .await
1407                    .map_err(backend)?
1408                    .rows_affected() as usize;
1409                Ok(renewed)
1410            }
1411
1412            async fn claim_pending(
1413                &self,
1414                limit: usize,
1415            ) -> Result<Vec<$crate::serve::history::RunRecord>, $crate::serve::history::HistoryError>
1416            {
1417                use sqlx::Row as _;
1418                use $crate::serve::history::HistoryError;
1419                use $crate::serve::history::RunStatus;
1420                use $crate::serve::history::sql;
1421                let backend = |e: sqlx::Error| HistoryError::Backend(e.to_string());
1422                if limit == 0 {
1423                    return Ok(Vec::new());
1424                }
1425                let now = chrono::Utc::now();
1426                let lease = sql::fmt_ts(now + self.lease_ttl);
1427
1428                // 1. Candidate pending runs (oldest first), with their bodies.
1429                let rows = sqlx::query(&self.stmts.select_pending)
1430                    .bind(limit as i64)
1431                    .fetch_all(&self.pool)
1432                    .await
1433                    .map_err(backend)?;
1434
1435                // Per-row conditional claim (1 SELECT + N guarded UPDATEs). The
1436                // batch is bounded by the caller's free permits (small), and this
1437                // is portable across Postgres + SQLite — deliberately NOT a
1438                // Postgres-only `FOR UPDATE SKIP LOCKED`.
1439                let mut claimed = Vec::new();
1440                for row in &rows {
1441                    let run_id: String = row.try_get("run_id").map_err(backend)?;
1442                    let body: String = row.try_get("body").map_err(backend)?;
1443                    // Flip the record to Running and rewrite the body so the column
1444                    // and the (source-of-truth) body stay consistent — a GET right
1445                    // after the claim must not show a stale `pending`.
1446                    let mut r = sql::decode_body(&body)?;
1447                    r.status = RunStatus::Running;
1448                    let new_body = sql::encode_body(&r)?;
1449                    // 2. Conditional claim — only the first committer wins.
1450                    let won = sqlx::query(&self.stmts.claim_one)
1451                        .bind(&self.instance_id)
1452                        .bind(&lease)
1453                        .bind(&new_body)
1454                        .bind(&run_id)
1455                        .execute(&self.pool)
1456                        .await
1457                        .map_err(backend)?
1458                        .rows_affected();
1459                    if won == 1 {
1460                        claimed.push(r);
1461                    }
1462                }
1463                Ok(claimed)
1464            }
1465
1466            async fn reclaim_orphans(
1467                &self,
1468                max_attempts: u32,
1469            ) -> Result<$crate::serve::history::ReclaimReport, $crate::serve::history::HistoryError>
1470            {
1471                use sqlx::Row as _;
1472                use $crate::serve::history::HistoryError;
1473                use $crate::serve::history::ReclaimReport;
1474                use $crate::serve::history::RunStatus;
1475                use $crate::serve::history::sql;
1476                let backend = |e: sqlx::Error| HistoryError::Backend(e.to_string());
1477                let now = chrono::Utc::now();
1478                let now_s = sql::fmt_ts(now);
1479
1480                let rows = sqlx::query(&self.stmts.reclaim_select)
1481                    .bind(&now_s)
1482                    .fetch_all(&self.pool)
1483                    .await
1484                    .map_err(backend)?;
1485
1486                let mut report = ReclaimReport::default();
1487                for row in &rows {
1488                    let body: String = row.try_get("body").map_err(backend)?;
1489                    let mut rec = sql::decode_body(&body)?;
1490                    let next_attempt = rec.attempt + 1;
1491                    // Cap is on the attempts already made: a run that has been
1492                    // reclaimed fewer than `max_attempts` times gets another try;
1493                    // once it reaches the cap it is poisoned.
1494                    if rec.attempt < max_attempts {
1495                        // Re-queue for another instance to re-run.
1496                        rec.attempt = next_attempt;
1497                        rec.status = RunStatus::Pending;
1498                        let new_body = sql::encode_body(&rec)?;
1499                        let n = sqlx::query(&self.stmts.reclaim_requeue)
1500                            .bind(&new_body)
1501                            .bind(&rec.run_id)
1502                            .bind(&now_s)
1503                            .execute(&self.pool)
1504                            .await
1505                            .map_err(backend)?
1506                            .rows_affected();
1507                        if n == 1 {
1508                            report.requeued += 1;
1509                        }
1510                    } else {
1511                        // Poison: too many attempts.
1512                        rec.attempt = next_attempt;
1513                        rec.status = RunStatus::Failed;
1514                        rec.finished_at = Some(now);
1515                        rec.error = Some(format!(
1516                            "run reclaimed {next_attempt} times after its owning instance's \
1517                             lease expired; giving up (poison run)"
1518                        ));
1519                        if rec.elapsed_secs.is_none()
1520                            && let Some(started) = rec.started_at
1521                        {
1522                            rec.elapsed_secs =
1523                                (now - started).to_std().ok().map(|d| d.as_secs_f64());
1524                        }
1525                        let new_body = sql::encode_body(&rec)?;
1526                        let n = sqlx::query(&self.stmts.reclaim_fail)
1527                            .bind(&now_s)
1528                            .bind(&new_body)
1529                            .bind(&rec.run_id)
1530                            .bind(&now_s)
1531                            .execute(&self.pool)
1532                            .await
1533                            .map_err(backend)?
1534                            .rows_affected();
1535                        if n == 1 {
1536                            report.failed += 1;
1537                        }
1538                    }
1539                }
1540                Ok(report)
1541            }
1542
1543            async fn finalize_owned(
1544                &self,
1545                rec: &$crate::serve::history::RunRecord,
1546            ) -> Result<bool, $crate::serve::history::HistoryError> {
1547                use $crate::serve::history::HistoryError;
1548                use $crate::serve::history::sql;
1549                let backend = |e: sqlx::Error| HistoryError::Backend(e.to_string());
1550                // Defensive: a terminal record must carry finished_at, or
1551                // purge_runs (which requires finished_at IS NOT NULL) can never
1552                // reclaim it. Stamp it if a caller left it unset.
1553                let mut rec = rec.clone();
1554                if rec.status.is_terminal() && rec.finished_at.is_none() {
1555                    rec.finished_at = Some(chrono::Utc::now());
1556                }
1557                let body = sql::encode_body(&rec)?;
1558                let finished = rec.finished_at.map(sql::fmt_ts);
1559                let lease = sql::fmt_ts(chrono::Utc::now() + self.lease_ttl);
1560                let n = sqlx::query(&self.stmts.finalize_owned)
1561                    .bind(rec.status.as_str())
1562                    .bind(finished.as_deref())
1563                    .bind(&lease)
1564                    .bind(&body)
1565                    .bind(&rec.run_id)
1566                    .bind(&self.instance_id)
1567                    .execute(&self.pool)
1568                    .await
1569                    .map_err(backend)?
1570                    .rows_affected();
1571                Ok(n == 1)
1572            }
1573
1574            async fn finalize_sharded_parent(
1575                &self,
1576                run_id: &str,
1577                status: $crate::serve::history::RunStatus,
1578                finished_at: chrono::DateTime<chrono::Utc>,
1579                error: Option<String>,
1580            ) -> Result<bool, $crate::serve::history::HistoryError> {
1581                use sqlx::Row as _;
1582                use $crate::serve::history::HistoryError;
1583                use $crate::serve::history::RunStatus;
1584                use $crate::serve::history::sql;
1585                let backend = |e: sqlx::Error| HistoryError::Backend(e.to_string());
1586                // Read the parent body, apply the terminal status, and write back
1587                // conditional on it still being `sharded` — so a concurrent
1588                // double-finalize from two instances has exactly one winner and
1589                // neither re-stamps owner/lease on the terminal record (F45).
1590                let Some(row) = sqlx::query(&self.stmts.select_body)
1591                    .bind(run_id)
1592                    .fetch_optional(&self.pool)
1593                    .await
1594                    .map_err(backend)?
1595                else {
1596                    return Ok(false);
1597                };
1598                let body: String = row.try_get("body").map_err(backend)?;
1599                let mut rec = sql::decode_body(&body)?;
1600                if rec.status != RunStatus::Sharded {
1601                    return Ok(false);
1602                }
1603                rec.status = status;
1604                rec.finished_at = Some(finished_at);
1605                rec.error = error;
1606                let new_body = sql::encode_body(&rec)?;
1607                let n = sqlx::query(&self.stmts.finalize_sharded_parent)
1608                    .bind(status.as_str())
1609                    .bind(sql::fmt_ts(finished_at))
1610                    .bind(&new_body)
1611                    .bind(run_id)
1612                    .execute(&self.pool)
1613                    .await
1614                    .map_err(backend)?
1615                    .rows_affected();
1616                Ok(n == 1)
1617            }
1618
1619            async fn cancel_pending(
1620                &self,
1621                run_id: &str,
1622            ) -> Result<bool, $crate::serve::history::HistoryError> {
1623                use sqlx::Row as _;
1624                use $crate::serve::history::HistoryError;
1625                use $crate::serve::history::RunStatus;
1626                use $crate::serve::history::sql;
1627                let backend = |e: sqlx::Error| HistoryError::Backend(e.to_string());
1628                // Read the pending run's body, flip it to Cancelled, and write back
1629                // conditional on it still being pending (loses the race to a claim).
1630                let Some(row) = sqlx::query(&self.stmts.select_body)
1631                    .bind(run_id)
1632                    .fetch_optional(&self.pool)
1633                    .await
1634                    .map_err(backend)?
1635                else {
1636                    return Ok(false);
1637                };
1638                let body: String = row.try_get("body").map_err(backend)?;
1639                let mut rec = sql::decode_body(&body)?;
1640                if rec.status != RunStatus::Pending {
1641                    return Ok(false);
1642                }
1643                let now = chrono::Utc::now();
1644                rec.status = RunStatus::Cancelled;
1645                rec.finished_at = Some(now);
1646                let new_body = sql::encode_body(&rec)?;
1647                let n = sqlx::query(&self.stmts.cancel_pending)
1648                    .bind(sql::fmt_ts(now))
1649                    .bind(&new_body)
1650                    .bind(run_id)
1651                    .execute(&self.pool)
1652                    .await
1653                    .map_err(backend)?
1654                    .rows_affected();
1655                Ok(n == 1)
1656            }
1657
1658            async fn request_cancel(
1659                &self,
1660                run_id: &str,
1661            ) -> Result<(), $crate::serve::history::HistoryError> {
1662                use $crate::serve::history::HistoryError;
1663                use $crate::serve::history::sql;
1664                let backend = |e: sqlx::Error| HistoryError::Backend(e.to_string());
1665                sqlx::query(&self.stmts.request_cancel)
1666                    .bind(sql::fmt_ts(chrono::Utc::now()))
1667                    .bind(run_id)
1668                    .execute(&self.pool)
1669                    .await
1670                    .map_err(backend)?;
1671                Ok(())
1672            }
1673
1674            async fn pending_cancellations(
1675                &self,
1676            ) -> Result<Vec<String>, $crate::serve::history::HistoryError> {
1677                use sqlx::Row as _;
1678                use $crate::serve::history::HistoryError;
1679                let backend = |e: sqlx::Error| HistoryError::Backend(e.to_string());
1680                let rows = sqlx::query(&self.stmts.pending_cancellations)
1681                    .bind(&self.instance_id)
1682                    .fetch_all(&self.pool)
1683                    .await
1684                    .map_err(backend)?;
1685                let mut ids = Vec::with_capacity(rows.len());
1686                for r in &rows {
1687                    ids.push(r.try_get::<String, _>("run_id").map_err(backend)?);
1688                }
1689                Ok(ids)
1690            }
1691
1692            async fn heartbeat_instance(
1693                &self,
1694                beat: &$crate::serve::history::InstanceHeartbeat,
1695            ) -> Result<(), $crate::serve::history::HistoryError> {
1696                use $crate::serve::history::HistoryError;
1697                use $crate::serve::history::sql;
1698                let backend = |e: sqlx::Error| HistoryError::Backend(e.to_string());
1699                let now = sql::fmt_ts(chrono::Utc::now());
1700                sqlx::query(&self.stmts.heartbeat_instance)
1701                    .bind(&self.instance_id)
1702                    .bind(sql::fmt_ts(beat.started_at))
1703                    .bind(&now)
1704                    .bind(beat.listen.as_deref())
1705                    .bind(beat.max_concurrent.to_string())
1706                    .bind(beat.in_flight.to_string())
1707                    .execute(&self.pool)
1708                    .await
1709                    .map_err(backend)?;
1710                Ok(())
1711            }
1712
1713            async fn live_instances(
1714                &self,
1715                ttl: std::time::Duration,
1716            ) -> Result<Vec<$crate::serve::history::InstanceRecord>, $crate::serve::history::HistoryError>
1717            {
1718                use sqlx::Row as _;
1719                use $crate::serve::history::HistoryError;
1720                use $crate::serve::history::InstanceRecord;
1721                use $crate::serve::history::sql;
1722                let backend = |e: sqlx::Error| HistoryError::Backend(e.to_string());
1723                let now = chrono::Utc::now();
1724                let rows = sqlx::query(&self.stmts.live_instances)
1725                    .bind(sql::threshold(now, ttl))
1726                    .fetch_all(&self.pool)
1727                    .await
1728                    .map_err(backend)?;
1729                let parse_dt = |s: &str| {
1730                    chrono::DateTime::parse_from_rfc3339(s)
1731                        .map(|d| d.to_utc())
1732                        .unwrap_or(now)
1733                };
1734                let mut out = Vec::with_capacity(rows.len());
1735                for r in &rows {
1736                    let started: String = r.try_get("started_at").map_err(backend)?;
1737                    let hb: String = r.try_get("last_heartbeat").map_err(backend)?;
1738                    let mc: Option<String> = r.try_get("max_concurrent").map_err(backend)?;
1739                    let inf: Option<String> = r.try_get("in_flight").map_err(backend)?;
1740                    out.push(InstanceRecord {
1741                        instance_id: r.try_get("instance_id").map_err(backend)?,
1742                        started_at: parse_dt(&started),
1743                        last_heartbeat: parse_dt(&hb),
1744                        listen: r.try_get("listen").map_err(backend)?,
1745                        max_concurrent: mc.and_then(|s| s.parse().ok()).unwrap_or(0),
1746                        in_flight: inf.and_then(|s| s.parse().ok()).unwrap_or(0),
1747                    });
1748                }
1749                Ok(out)
1750            }
1751
1752            // ── Source shards (Mode B, #230) ─────────────────────────────────
1753
1754            async fn insert_shards(
1755                &self,
1756                run_id: &str,
1757                shards: &[$crate::serve::history::ShardInsert],
1758            ) -> Result<usize, $crate::serve::history::HistoryError> {
1759                use $crate::serve::history::HistoryError;
1760                let backend = |e: sqlx::Error| HistoryError::Backend(e.to_string());
1761                let mut inserted = 0usize;
1762                for s in shards {
1763                    let descriptor = serde_json::to_string(&s.descriptor).map_err(|e| {
1764                        HistoryError::Backend(format!("encode shard descriptor: {e}"))
1765                    })?;
1766                    let size = s.size_estimate.map(|n| n.to_string());
1767                    let n = sqlx::query(&self.stmts.insert_shard)
1768                        .bind(run_id)
1769                        .bind(&s.shard_id)
1770                        .bind(&descriptor)
1771                        .bind(size.as_deref())
1772                        .execute(&self.pool)
1773                        .await
1774                        .map_err(backend)?
1775                        .rows_affected();
1776                    inserted += n as usize;
1777                }
1778                Ok(inserted)
1779            }
1780
1781            async fn claim_shards(
1782                &self,
1783                limit: usize,
1784            ) -> Result<
1785                Vec<$crate::serve::history::ClaimedShard>,
1786                $crate::serve::history::HistoryError,
1787            > {
1788                use sqlx::Row as _;
1789                use $crate::serve::history::ClaimedShard;
1790                use $crate::serve::history::HistoryError;
1791                use $crate::serve::history::sql;
1792                let backend = |e: sqlx::Error| HistoryError::Backend(e.to_string());
1793                if limit == 0 {
1794                    return Ok(Vec::new());
1795                }
1796                let lease = sql::fmt_ts(chrono::Utc::now() + self.lease_ttl);
1797
1798                // 1. Candidate pending shards (largest estimated size first),
1799                //    joined to their parent run body.
1800                let rows = sqlx::query(&self.stmts.claim_shards_select)
1801                    .bind(limit as i64)
1802                    .fetch_all(&self.pool)
1803                    .await
1804                    .map_err(backend)?;
1805
1806                // 2. Per-row conditional claim (portable; not FOR UPDATE SKIP LOCKED).
1807                let mut claimed = Vec::new();
1808                for row in &rows {
1809                    let run_id: String = row.try_get("run_id").map_err(backend)?;
1810                    let shard_id: String = row.try_get("shard_id").map_err(backend)?;
1811                    let descriptor_s: String = row.try_get("descriptor").map_err(backend)?;
1812                    let body: String = row.try_get("body").map_err(backend)?;
1813                    let won = sqlx::query(&self.stmts.claim_shard_one)
1814                        .bind(&self.instance_id)
1815                        .bind(&lease)
1816                        .bind(&run_id)
1817                        .bind(&shard_id)
1818                        .execute(&self.pool)
1819                        .await
1820                        .map_err(backend)?
1821                        .rows_affected();
1822                    if won == 1 {
1823                        let descriptor: serde_json::Value = serde_json::from_str(&descriptor_s)
1824                            .map_err(|e| {
1825                                HistoryError::Backend(format!("decode shard descriptor: {e}"))
1826                            })?;
1827                        let run = sql::decode_body(&body)?;
1828                        claimed.push(ClaimedShard {
1829                            run_id,
1830                            shard_id,
1831                            descriptor,
1832                            run,
1833                        });
1834                    }
1835                }
1836                Ok(claimed)
1837            }
1838
1839            async fn renew_shard_leases(
1840                &self,
1841            ) -> Result<usize, $crate::serve::history::HistoryError> {
1842                use $crate::serve::history::HistoryError;
1843                use $crate::serve::history::sql;
1844                let backend = |e: sqlx::Error| HistoryError::Backend(e.to_string());
1845                let lease = sql::fmt_ts(chrono::Utc::now() + self.lease_ttl);
1846                let n = sqlx::query(&self.stmts.renew_shard_leases)
1847                    .bind(&lease)
1848                    .bind(&self.instance_id)
1849                    .execute(&self.pool)
1850                    .await
1851                    .map_err(backend)?
1852                    .rows_affected() as usize;
1853                Ok(n)
1854            }
1855
1856            async fn reclaim_shards(
1857                &self,
1858                max_attempts: u32,
1859            ) -> Result<$crate::serve::history::ReclaimReport, $crate::serve::history::HistoryError>
1860            {
1861                use sqlx::Row as _;
1862                use $crate::serve::history::HistoryError;
1863                use $crate::serve::history::ReclaimReport;
1864                use $crate::serve::history::sql;
1865                let backend = |e: sqlx::Error| HistoryError::Backend(e.to_string());
1866                let now_s = sql::fmt_ts(chrono::Utc::now());
1867
1868                let rows = sqlx::query(&self.stmts.reclaim_shards_select)
1869                    .bind(&now_s)
1870                    .fetch_all(&self.pool)
1871                    .await
1872                    .map_err(backend)?;
1873
1874                let mut report = ReclaimReport::default();
1875                for row in &rows {
1876                    let run_id: String = row.try_get("run_id").map_err(backend)?;
1877                    let shard_id: String = row.try_get("shard_id").map_err(backend)?;
1878                    let attempt_s: String = row.try_get("attempt").map_err(backend)?;
1879                    let attempt: u32 = attempt_s.parse().unwrap_or(0);
1880                    if attempt < max_attempts {
1881                        let next = (attempt + 1).to_string();
1882                        let n = sqlx::query(&self.stmts.reclaim_shard_requeue)
1883                            .bind(&next)
1884                            .bind(&run_id)
1885                            .bind(&shard_id)
1886                            .bind(&now_s)
1887                            .execute(&self.pool)
1888                            .await
1889                            .map_err(backend)?
1890                            .rows_affected();
1891                        if n == 1 {
1892                            report.requeued += 1;
1893                        }
1894                    } else {
1895                        let n = sqlx::query(&self.stmts.reclaim_shard_fail)
1896                            .bind(&now_s)
1897                            .bind(&run_id)
1898                            .bind(&shard_id)
1899                            .bind(&now_s)
1900                            .execute(&self.pool)
1901                            .await
1902                            .map_err(backend)?
1903                            .rows_affected();
1904                        if n == 1 {
1905                            report.failed += 1;
1906                        }
1907                    }
1908                }
1909                Ok(report)
1910            }
1911
1912            async fn finalize_shard(
1913                &self,
1914                run_id: &str,
1915                shard_id: &str,
1916                success: bool,
1917            ) -> Result<bool, $crate::serve::history::HistoryError> {
1918                use $crate::serve::history::HistoryError;
1919                use $crate::serve::history::sql;
1920                let backend = |e: sqlx::Error| HistoryError::Backend(e.to_string());
1921                let status = if success { "completed" } else { "failed" };
1922                let now_s = sql::fmt_ts(chrono::Utc::now());
1923                let n = sqlx::query(&self.stmts.finalize_shard)
1924                    .bind(status)
1925                    .bind(&now_s)
1926                    .bind(run_id)
1927                    .bind(shard_id)
1928                    .bind(&self.instance_id)
1929                    .execute(&self.pool)
1930                    .await
1931                    .map_err(backend)?
1932                    .rows_affected();
1933                Ok(n == 1)
1934            }
1935
1936            async fn shard_progress(
1937                &self,
1938                run_id: &str,
1939            ) -> Result<$crate::serve::history::ShardProgress, $crate::serve::history::HistoryError>
1940            {
1941                use sqlx::Row as _;
1942                use $crate::serve::history::HistoryError;
1943                use $crate::serve::history::ShardProgress;
1944                let backend = |e: sqlx::Error| HistoryError::Backend(e.to_string());
1945                let rows = sqlx::query(&self.stmts.shard_progress)
1946                    .bind(run_id)
1947                    .fetch_all(&self.pool)
1948                    .await
1949                    .map_err(backend)?;
1950                let mut p = ShardProgress::default();
1951                for row in &rows {
1952                    let status: String = row.try_get("status").map_err(backend)?;
1953                    let n: i64 = row.try_get("n").map_err(backend)?;
1954                    let n = n.max(0) as usize;
1955                    p.total += n;
1956                    match status.as_str() {
1957                        "completed" => p.completed += n,
1958                        "failed" => p.failed += n,
1959                        "running" => p.running += n,
1960                        _ => p.pending += n,
1961                    }
1962                }
1963                Ok(p)
1964            }
1965
1966            async fn pending_shard_cancellations(
1967                &self,
1968            ) -> Result<Vec<String>, $crate::serve::history::HistoryError> {
1969                use sqlx::Row as _;
1970                use $crate::serve::history::HistoryError;
1971                let backend = |e: sqlx::Error| HistoryError::Backend(e.to_string());
1972                let rows = sqlx::query(&self.stmts.pending_shard_cancellations)
1973                    .bind(&self.instance_id)
1974                    .fetch_all(&self.pool)
1975                    .await
1976                    .map_err(backend)?;
1977                let mut ids = Vec::with_capacity(rows.len());
1978                for r in &rows {
1979                    ids.push(r.try_get::<String, _>("run_id").map_err(backend)?);
1980                }
1981                Ok(ids)
1982            }
1983
1984            async fn finalize_completed_sharded_parents(
1985                &self,
1986            ) -> Result<usize, $crate::serve::history::HistoryError> {
1987                use sqlx::Row as _;
1988                use $crate::serve::history::HistoryError;
1989                use $crate::serve::history::RunStatus;
1990                use $crate::serve::history::sql;
1991                let backend = |e: sqlx::Error| HistoryError::Backend(e.to_string());
1992
1993                // Candidate `sharded` parents — finalize each whose shards are all
1994                // terminal. The status-fenced UPDATE makes a concurrent finalize
1995                // (here or in `maybe_finalize_parent`) a benign no-op.
1996                let rows = sqlx::query(&self.stmts.select_sharded_parents)
1997                    .fetch_all(&self.pool)
1998                    .await
1999                    .map_err(backend)?;
2000
2001                let mut finalized = 0usize;
2002                for row in &rows {
2003                    let run_id: String = row.try_get("run_id").map_err(backend)?;
2004                    let progress = self.shard_progress(&run_id).await?;
2005                    if !progress.all_terminal() {
2006                        continue;
2007                    }
2008                    let success = progress.failed == 0;
2009                    // Read-modify-write the body so the surfaced record stays
2010                    // consistent (status, finished_at, error) with the column.
2011                    let Some(body_row) = sqlx::query(&self.stmts.select_body)
2012                        .bind(&run_id)
2013                        .fetch_optional(&self.pool)
2014                        .await
2015                        .map_err(backend)?
2016                    else {
2017                        continue;
2018                    };
2019                    let body: String = body_row.try_get("body").map_err(backend)?;
2020                    let mut rec = sql::decode_body(&body)?;
2021                    // Skip if it raced to terminal already (column says sharded but
2022                    // the body was just updated). The fenced UPDATE is the real guard.
2023                    if rec.status != RunStatus::Sharded {
2024                        continue;
2025                    }
2026                    let now = chrono::Utc::now();
2027                    rec.status = if success {
2028                        RunStatus::Completed
2029                    } else {
2030                        RunStatus::Failed
2031                    };
2032                    rec.finished_at = Some(now);
2033                    if !success {
2034                        rec.error = Some(format!(
2035                            "{}/{} shard(s) failed",
2036                            progress.failed, progress.total
2037                        ));
2038                    }
2039                    let new_body = sql::encode_body(&rec)?;
2040                    let n = sqlx::query(&self.stmts.finalize_sharded_parent)
2041                        .bind(rec.status.as_str())
2042                        .bind(sql::fmt_ts(now))
2043                        .bind(&new_body)
2044                        .bind(&run_id)
2045                        .execute(&self.pool)
2046                        .await
2047                        .map_err(backend)?
2048                        .rows_affected();
2049                    if n == 1 {
2050                        finalized += 1;
2051                        $crate::serve::metrics::record_run_finished(
2052                            rec.status,
2053                            if success { "ok" } else { "error" },
2054                        );
2055                        tracing::info!(
2056                            run_id,
2057                            shards = progress.total,
2058                            failed = progress.failed,
2059                            "sharded run finalized by sweep (F11)"
2060                        );
2061                    }
2062                }
2063                Ok(finalized)
2064            }
2065
2066            // ── Audit log (RBAC, #205) ───────────────────────────────────────
2067
2068            async fn record_audit(
2069                &self,
2070                entry: &$crate::serve::history::AuditEntry,
2071            ) -> Result<(), $crate::serve::history::HistoryError> {
2072                use $crate::serve::history::HistoryError;
2073                use $crate::serve::history::sql;
2074                let backend = |e: sqlx::Error| HistoryError::Backend(e.to_string());
2075                sqlx::query(&self.stmts.insert_audit)
2076                    .bind(&entry.id)
2077                    .bind(sql::fmt_ts(entry.timestamp))
2078                    .bind(&entry.principal)
2079                    .bind(&entry.role)
2080                    .bind(&entry.action)
2081                    .bind(entry.run_id.as_deref())
2082                    .bind(entry.config_fingerprint.as_deref())
2083                    .bind(entry.source_ip.as_deref())
2084                    .bind(&entry.result)
2085                    .execute(&self.pool)
2086                    .await
2087                    .map_err(backend)?;
2088                Ok(())
2089            }
2090
2091            async fn list_audit(
2092                &self,
2093                filter: &$crate::serve::history::AuditFilter,
2094            ) -> Result<
2095                Vec<$crate::serve::history::AuditEntry>,
2096                $crate::serve::history::HistoryError,
2097            > {
2098                use sqlx::Row as _;
2099                use $crate::serve::history::AuditEntry;
2100                use $crate::serve::history::HistoryError;
2101                use $crate::serve::history::sql;
2102                let backend = |e: sqlx::Error| HistoryError::Backend(e.to_string());
2103                let principal = filter.principal.as_deref();
2104                let action = filter.action.as_deref();
2105                let since = filter.since.map(sql::fmt_ts);
2106                let until = filter.until.map(sql::fmt_ts);
2107                let limit = filter.limit.max(1) as i64;
2108                let rows = sqlx::query(&self.stmts.list_audit)
2109                    .bind(principal)
2110                    .bind(principal)
2111                    .bind(action)
2112                    .bind(action)
2113                    .bind(since.as_deref())
2114                    .bind(since.as_deref())
2115                    .bind(until.as_deref())
2116                    .bind(until.as_deref())
2117                    .bind(limit)
2118                    .fetch_all(&self.pool)
2119                    .await
2120                    .map_err(backend)?;
2121                let mut out = Vec::with_capacity(rows.len());
2122                for r in &rows {
2123                    let ts: String = r.try_get("ts").map_err(backend)?;
2124                    let timestamp = $crate::serve::history::sql::parse_ts(&ts);
2125                    out.push(AuditEntry {
2126                        id: r.try_get("id").map_err(backend)?,
2127                        timestamp,
2128                        principal: r.try_get("principal").map_err(backend)?,
2129                        role: r.try_get("role").map_err(backend)?,
2130                        action: r.try_get("action").map_err(backend)?,
2131                        run_id: r.try_get("run_id").map_err(backend)?,
2132                        config_fingerprint: r.try_get("config_fingerprint").map_err(backend)?,
2133                        source_ip: r.try_get("source_ip").map_err(backend)?,
2134                        result: r.try_get("result").map_err(backend)?,
2135                    });
2136                }
2137                Ok(out)
2138            }
2139
2140            // ── Data Movement Catalog (#279) ─────────────────────────────────
2141
2142            async fn catalog_record(
2143                &self,
2144                update: &$crate::serve::history::catalog::CatalogUpdate,
2145            ) -> Result<(), $crate::serve::history::HistoryError> {
2146                use sqlx::Row as _;
2147                use $crate::serve::history::HistoryError;
2148                use $crate::serve::history::catalog;
2149                use $crate::serve::history::sql;
2150                let backend = |e: sqlx::Error| HistoryError::Backend(e.to_string());
2151                let now_s = sql::fmt_ts(update.recorded_at);
2152
2153                for obs in update.sources.iter().chain(std::iter::once(&update.sink)) {
2154                    let id = catalog::dataset_id(&obs.uri);
2155                    // Read-merge-write; last-write-wins under cluster concurrency
2156                    // (counters may undercount on a race — acceptable for
2157                    // operational stats, never for correctness).
2158                    let existing = sqlx::query(&self.stmts.catalog_select_dataset)
2159                        .bind(&id)
2160                        .fetch_optional(&self.pool)
2161                        .await
2162                        .map_err(backend)?
2163                        .map(|r| r.try_get::<String, _>("body"))
2164                        .transpose()
2165                        .map_err(backend)?
2166                        .map(|b| {
2167                            sql::decode_json::<catalog::CatalogDataset>(&b, "catalog dataset")
2168                        })
2169                        .transpose()?;
2170                    let (ds, new_version) = catalog::apply_observation(
2171                        existing.as_ref(),
2172                        obs,
2173                        &update.run_id,
2174                        &update.pipeline,
2175                        &update.row,
2176                        update.recorded_at,
2177                    );
2178                    sqlx::query(&self.stmts.catalog_upsert_dataset)
2179                        .bind(&ds.id)
2180                        .bind(&ds.uri)
2181                        .bind(&ds.kind)
2182                        .bind(&now_s)
2183                        .bind(sql::encode_json(&ds, "catalog dataset")?)
2184                        .execute(&self.pool)
2185                        .await
2186                        .map_err(backend)?;
2187                    if let Some(v) = new_version {
2188                        sqlx::query(&self.stmts.catalog_insert_schema_version)
2189                            .bind(&v.dataset_id)
2190                            .bind(v.version.to_string())
2191                            .bind(sql::fmt_ts(v.recorded_at))
2192                            .bind(sql::encode_json(&v, "catalog schema version")?)
2193                            .execute(&self.pool)
2194                            .await
2195                            .map_err(backend)?;
2196                    }
2197                    sqlx::query(&self.stmts.catalog_insert_stat)
2198                        .bind(&id)
2199                        .bind(&now_s)
2200                        .bind(&update.run_id)
2201                        .bind(obs.records.to_string())
2202                        .execute(&self.pool)
2203                        .await
2204                        .map_err(backend)?;
2205                    sqlx::query(&self.stmts.catalog_prune_stats)
2206                        .bind(&id)
2207                        .bind(&id)
2208                        .bind(catalog::STATS_RETAIN as i64)
2209                        .execute(&self.pool)
2210                        .await
2211                        .map_err(backend)?;
2212                }
2213
2214                // One edge per input dataset — a merge/join sink has several
2215                // (#459). Fetched once and reused across the inputs.
2216                let dst_id = catalog::dataset_id(&update.sink.uri);
2217                let existing_edges = self.catalog_all_edges().await?;
2218                for source in &update.sources {
2219                    let src_id = catalog::dataset_id(&source.uri);
2220                    let existing = existing_edges
2221                        .iter()
2222                        .find(|e| e.src_id == src_id && e.dst_id == dst_id);
2223                    let edge = catalog::apply_edge(existing, update, source);
2224                    sqlx::query(&self.stmts.catalog_upsert_edge)
2225                        .bind(&edge.src_id)
2226                        .bind(&edge.dst_id)
2227                        .bind(&now_s)
2228                        .bind(sql::encode_json(&edge, "catalog edge")?)
2229                        .execute(&self.pool)
2230                        .await
2231                        .map_err(backend)?;
2232                }
2233                Ok(())
2234            }
2235
2236            async fn catalog_list_datasets(
2237                &self,
2238                filter: &$crate::serve::history::catalog::CatalogListFilter,
2239            ) -> Result<
2240                $crate::serve::history::catalog::CatalogDatasetPage,
2241                $crate::serve::history::HistoryError,
2242            > {
2243                use sqlx::Row as _;
2244                use $crate::serve::history::HistoryError;
2245                use $crate::serve::history::catalog;
2246                use $crate::serve::history::sql;
2247                let backend = |e: sqlx::Error| HistoryError::Backend(e.to_string());
2248                let rows = sqlx::query(&self.stmts.catalog_select_datasets)
2249                    .fetch_all(&self.pool)
2250                    .await
2251                    .map_err(backend)?;
2252                let mut all = Vec::with_capacity(rows.len());
2253                for r in &rows {
2254                    let body: String = r.try_get("body").map_err(backend)?;
2255                    all.push(sql::decode_json(&body, "catalog dataset")?);
2256                }
2257                Ok(catalog::filter_datasets(all, filter))
2258            }
2259
2260            async fn catalog_get_dataset(
2261                &self,
2262                id: &str,
2263            ) -> Result<
2264                Option<$crate::serve::history::catalog::CatalogDatasetDetail>,
2265                $crate::serve::history::HistoryError,
2266            > {
2267                use sqlx::Row as _;
2268                use $crate::serve::history::HistoryError;
2269                use $crate::serve::history::catalog;
2270                use $crate::serve::history::sql;
2271                let backend = |e: sqlx::Error| HistoryError::Backend(e.to_string());
2272                let Some(row) = sqlx::query(&self.stmts.catalog_select_dataset)
2273                    .bind(id)
2274                    .fetch_optional(&self.pool)
2275                    .await
2276                    .map_err(backend)?
2277                else {
2278                    return Ok(None);
2279                };
2280                let body: String = row.try_get("body").map_err(backend)?;
2281                let dataset: catalog::CatalogDataset =
2282                    sql::decode_json(&body, "catalog dataset")?;
2283
2284                let rows = sqlx::query(&self.stmts.catalog_select_schema_versions)
2285                    .bind(id)
2286                    .fetch_all(&self.pool)
2287                    .await
2288                    .map_err(backend)?;
2289                let mut schema_timeline = Vec::with_capacity(rows.len());
2290                for r in &rows {
2291                    let body: String = r.try_get("body").map_err(backend)?;
2292                    schema_timeline.push(sql::decode_json(&body, "catalog schema version")?);
2293                }
2294
2295                let rows = sqlx::query(&self.stmts.catalog_select_stats)
2296                    .bind(id)
2297                    .bind(catalog::STATS_DETAIL_LIMIT as i64)
2298                    .fetch_all(&self.pool)
2299                    .await
2300                    .map_err(backend)?;
2301                let mut stats = Vec::with_capacity(rows.len());
2302                for r in &rows {
2303                    let recorded: String = r.try_get("recorded_at").map_err(backend)?;
2304                    let run_id: String = r.try_get("run_id").map_err(backend)?;
2305                    let records: String = r.try_get("records").map_err(backend)?;
2306                    stats.push(catalog::CatalogStatsPoint {
2307                        recorded_at: chrono::DateTime::parse_from_rfc3339(&recorded)
2308                            .map(|d| d.to_utc())
2309                            .unwrap_or_else(|_| chrono::Utc::now()),
2310                        run_id,
2311                        records: records.parse().unwrap_or(0),
2312                    });
2313                }
2314
2315                let edges = self.catalog_all_edges().await?;
2316                let (downstream, rest): (Vec<_>, Vec<_>) =
2317                    edges.into_iter().partition(|e| e.src_id == id);
2318                let upstream = rest.into_iter().filter(|e| e.dst_id == id).collect();
2319                Ok(Some(catalog::CatalogDatasetDetail {
2320                    dataset,
2321                    schema_timeline,
2322                    stats,
2323                    upstream,
2324                    downstream,
2325                }))
2326            }
2327
2328            async fn catalog_lineage(
2329                &self,
2330                root: Option<&str>,
2331                depth: u32,
2332            ) -> Result<
2333                Vec<$crate::serve::history::catalog::CatalogLineageEdge>,
2334                $crate::serve::history::HistoryError,
2335            > {
2336                use $crate::serve::history::catalog;
2337                let edges = self.catalog_all_edges().await?;
2338                Ok(catalog::lineage_slice(edges, root, depth))
2339            }
2340
2341            async fn catalog_record_config_snapshot(
2342                &self,
2343                snapshot: &$crate::serve::history::catalog::ConfigSnapshot,
2344            ) -> Result<(), $crate::serve::history::HistoryError> {
2345                use $crate::serve::history::HistoryError;
2346                use $crate::serve::history::sql;
2347                let backend = |e: sqlx::Error| HistoryError::Backend(e.to_string());
2348                sqlx::query(&self.stmts.catalog_upsert_config_snapshot)
2349                    .bind(&snapshot.pipeline)
2350                    .bind(sql::fmt_ts(snapshot.recorded_at))
2351                    .bind(&snapshot.faucet_version)
2352                    .bind(sql::encode_json(snapshot, "config snapshot")?)
2353                    .execute(&self.pool)
2354                    .await
2355                    .map_err(backend)?;
2356                Ok(())
2357            }
2358
2359            async fn catalog_last_config_snapshot(
2360                &self,
2361                pipeline: &str,
2362            ) -> Result<
2363                Option<$crate::serve::history::catalog::ConfigSnapshot>,
2364                $crate::serve::history::HistoryError,
2365            > {
2366                use sqlx::Row as _;
2367                use $crate::serve::history::HistoryError;
2368                use $crate::serve::history::sql;
2369                let backend = |e: sqlx::Error| HistoryError::Backend(e.to_string());
2370                let Some(row) = sqlx::query(&self.stmts.catalog_select_config_snapshot)
2371                    .bind(pipeline)
2372                    .fetch_optional(&self.pool)
2373                    .await
2374                    .map_err(backend)?
2375                else {
2376                    return Ok(None);
2377                };
2378                let body: String = row.try_get("body").map_err(backend)?;
2379                Ok(Some(sql::decode_json(&body, "config snapshot")?))
2380            }
2381
2382            // ── Pipeline-template registry (#444) ────────────────────────────
2383
2384            async fn template_register(
2385                &self,
2386                draft: &$crate::serve::history::templates::TemplateDraft,
2387            ) -> Result<
2388                $crate::serve::history::templates::TemplateRecord,
2389                $crate::serve::history::HistoryError,
2390            > {
2391                use sqlx::Row as _;
2392                use $crate::serve::history::HistoryError;
2393                use $crate::serve::history::{sql, templates};
2394                let backend = |e: sqlx::Error| HistoryError::Backend(e.to_string());
2395                let id = draft.id.to_string();
2396
2397                // Read-max-then-insert inside a transaction. Two concurrent
2398                // registers can still both read the same max under READ
2399                // COMMITTED; the primary key then rejects one of them, and the
2400                // retry picks up the winner's version — so a register never
2401                // silently overwrites another (F: lost-update).
2402                for attempt in 1..=sql::CLAIM_ATTEMPTS {
2403                    sql::retry_backoff(attempt).await;
2404                    // Opening the transaction and reading the current max are as
2405                    // contention-prone as the insert itself (SQLite answers a
2406                    // write-write overlap with `database is locked` immediately,
2407                    // since waiting would deadlock). Feed those failures into the
2408                    // same retry rather than aborting the register.
2409                    let mut tx = match self.pool.begin().await {
2410                        Ok(tx) => tx,
2411                        Err(e) if attempt < sql::CLAIM_ATTEMPTS => {
2412                            tracing::debug!(
2413                                template = %id, attempt, error = %e,
2414                                "template version transaction lost a race; retrying"
2415                            );
2416                            continue;
2417                        }
2418                        Err(e) => return Err(backend(e)),
2419                    };
2420                    let row = match sqlx::query(&self.stmts.template_max_version)
2421                        .bind(&id)
2422                        .fetch_one(&mut *tx)
2423                        .await
2424                    {
2425                        Ok(row) => row,
2426                        Err(e) if attempt < sql::CLAIM_ATTEMPTS => {
2427                            let _ = tx.rollback().await;
2428                            tracing::debug!(
2429                                template = %id, attempt, error = %e,
2430                                "template version read lost a race; retrying"
2431                            );
2432                            continue;
2433                        }
2434                        Err(e) => {
2435                            let _ = tx.rollback().await;
2436                            return Err(backend(e));
2437                        }
2438                    };
2439                    let max: i64 = row.try_get("v").map_err(backend)?;
2440                    let next = (max as u32).saturating_add(1);
2441                    let record = templates::TemplateRecord {
2442                        id: id.clone(),
2443                        version: next,
2444                        name: draft.name.clone(),
2445                        description: draft.description.clone(),
2446                        body: draft.body.clone(),
2447                        format: draft.format,
2448                        params: draft.params.clone(),
2449                        created_at: chrono::Utc::now(),
2450                        created_by: draft.created_by.clone(),
2451                    };
2452                    let insert = sqlx::query(&self.stmts.template_insert)
2453                        .bind(&id)
2454                        .bind(next.to_string())
2455                        .bind(&record.name)
2456                        .bind(sql::fmt_ts(record.created_at))
2457                        .bind(sql::encode_json(&record, "pipeline template")?)
2458                        .execute(&mut *tx)
2459                        .await;
2460                    match insert {
2461                        Ok(_) => {
2462                            tx.commit().await.map_err(backend)?;
2463                            // Bound the version history so a template
2464                            // re-registered on every deploy can't grow forever.
2465                            let keep = self.template_versions(&id).await?;
2466                            for stale in templates::versions_to_prune(keep) {
2467                                let _ = sqlx::query(&self.stmts.template_delete_version)
2468                                    .bind(&id)
2469                                    .bind(stale.to_string())
2470                                    .execute(&self.pool)
2471                                    .await;
2472                            }
2473                            return Ok(record);
2474                        }
2475                        Err(e) if attempt < sql::CLAIM_ATTEMPTS => {
2476                            let _ = tx.rollback().await;
2477                            tracing::debug!(
2478                                template = %id, attempt, error = %e,
2479                                "template version insert lost a race; retrying with the next version"
2480                            );
2481                        }
2482                        Err(e) => {
2483                            let _ = tx.rollback().await;
2484                            return Err(backend(e));
2485                        }
2486                    }
2487                }
2488                Err(HistoryError::Backend(format!(
2489                    "could not assign a version for template '{id}' after {} attempts",
2490                    sql::CLAIM_ATTEMPTS
2491                )))
2492            }
2493
2494            async fn template_get(
2495                &self,
2496                id: &str,
2497                version: Option<u32>,
2498            ) -> Result<
2499                Option<$crate::serve::history::templates::TemplateRecord>,
2500                $crate::serve::history::HistoryError,
2501            > {
2502                use sqlx::Row as _;
2503                use $crate::serve::history::HistoryError;
2504                use $crate::serve::history::sql;
2505                let backend = |e: sqlx::Error| HistoryError::Backend(e.to_string());
2506                let row = match version {
2507                    Some(v) => sqlx::query(&self.stmts.template_select_version)
2508                        .bind(id)
2509                        .bind(v.to_string())
2510                        .fetch_optional(&self.pool)
2511                        .await,
2512                    None => sqlx::query(&self.stmts.template_select_latest)
2513                        .bind(id)
2514                        .fetch_optional(&self.pool)
2515                        .await,
2516                }
2517                .map_err(backend)?;
2518                let Some(row) = row else {
2519                    return Ok(None);
2520                };
2521                let body: String = row.try_get("body").map_err(backend)?;
2522                Ok(Some(sql::decode_json(&body, "pipeline template")?))
2523            }
2524
2525            async fn template_list(
2526                &self,
2527            ) -> Result<
2528                Vec<$crate::serve::history::templates::TemplateSummary>,
2529                $crate::serve::history::HistoryError,
2530            > {
2531                use sqlx::Row as _;
2532                use $crate::serve::history::HistoryError;
2533                use $crate::serve::history::{sql, templates};
2534                let backend = |e: sqlx::Error| HistoryError::Backend(e.to_string());
2535                let rows = sqlx::query(&self.stmts.template_select_all)
2536                    .fetch_all(&self.pool)
2537                    .await
2538                    .map_err(backend)?;
2539                let mut all = Vec::with_capacity(rows.len());
2540                for r in &rows {
2541                    let body: String = r.try_get("body").map_err(backend)?;
2542                    all.push(sql::decode_json(&body, "pipeline template")?);
2543                }
2544                Ok(templates::latest_per_id(all))
2545            }
2546
2547            async fn template_versions(
2548                &self,
2549                id: &str,
2550            ) -> Result<Vec<u32>, $crate::serve::history::HistoryError> {
2551                use sqlx::Row as _;
2552                use $crate::serve::history::HistoryError;
2553                let backend = |e: sqlx::Error| HistoryError::Backend(e.to_string());
2554                let rows = sqlx::query(&self.stmts.template_versions)
2555                    .bind(id)
2556                    .fetch_all(&self.pool)
2557                    .await
2558                    .map_err(backend)?;
2559                let mut out = Vec::with_capacity(rows.len());
2560                for r in &rows {
2561                    let v: String = r.try_get("version").map_err(backend)?;
2562                    if let Ok(n) = v.parse::<u32>() {
2563                        out.push(n);
2564                    }
2565                }
2566                Ok(out)
2567            }
2568
2569            async fn template_delete(
2570                &self,
2571                id: &str,
2572                version: Option<u32>,
2573            ) -> Result<usize, $crate::serve::history::HistoryError> {
2574                use $crate::serve::history::HistoryError;
2575                let backend = |e: sqlx::Error| HistoryError::Backend(e.to_string());
2576                let result = match version {
2577                    Some(v) => {
2578                        // Drop channels + launch entries aimed at this version
2579                        // first, so no pointer can outlive what it points at.
2580                        sqlx::query(&self.stmts.template_delete_tags_for_version)
2581                            .bind(id)
2582                            .bind(v.to_string())
2583                            .execute(&self.pool)
2584                            .await
2585                            .map_err(backend)?;
2586                        sqlx::query(&self.stmts.template_delete_launches_for_version)
2587                            .bind(id)
2588                            .bind(v.to_string())
2589                            .execute(&self.pool)
2590                            .await
2591                            .map_err(backend)?;
2592                        sqlx::query(&self.stmts.template_delete_version)
2593                            .bind(id)
2594                            .bind(v.to_string())
2595                            .execute(&self.pool)
2596                            .await
2597                    }
2598                    None => {
2599                        for stmt in [
2600                            &self.stmts.template_delete_tags_all,
2601                            &self.stmts.template_delete_launches_all,
2602                            &self.stmts.template_delete_deprecation,
2603                        ] {
2604                            sqlx::query(stmt)
2605                                .bind(id)
2606                                .execute(&self.pool)
2607                                .await
2608                                .map_err(backend)?;
2609                        }
2610                        sqlx::query(&self.stmts.template_delete_all)
2611                            .bind(id)
2612                            .execute(&self.pool)
2613                            .await
2614                    }
2615                }
2616                .map_err(backend)?;
2617                Ok(result.rows_affected() as usize)
2618            }
2619
2620            async fn template_set_tag(
2621                &self,
2622                id: &str,
2623                tag: &str,
2624                version: u32,
2625            ) -> Result<(), $crate::serve::history::HistoryError> {
2626                use $crate::serve::history::HistoryError;
2627                use $crate::serve::history::sql;
2628                let backend = |e: sqlx::Error| HistoryError::Backend(e.to_string());
2629                sqlx::query(&self.stmts.template_upsert_tag)
2630                    .bind(id)
2631                    .bind(tag)
2632                    .bind(version.to_string())
2633                    .bind(sql::fmt_ts(chrono::Utc::now()))
2634                    .execute(&self.pool)
2635                    .await
2636                    .map_err(backend)?;
2637                Ok(())
2638            }
2639
2640            async fn template_tags(
2641                &self,
2642                id: &str,
2643            ) -> Result<
2644                std::collections::BTreeMap<String, u32>,
2645                $crate::serve::history::HistoryError,
2646            > {
2647                use sqlx::Row as _;
2648                use $crate::serve::history::HistoryError;
2649                let backend = |e: sqlx::Error| HistoryError::Backend(e.to_string());
2650                let rows = sqlx::query(&self.stmts.template_select_tags)
2651                    .bind(id)
2652                    .fetch_all(&self.pool)
2653                    .await
2654                    .map_err(backend)?;
2655                let mut out = std::collections::BTreeMap::new();
2656                for r in &rows {
2657                    let tag: String = r.try_get("tag").map_err(backend)?;
2658                    let version: String = r.try_get("version").map_err(backend)?;
2659                    if let Ok(n) = version.parse::<u32>() {
2660                        out.insert(tag, n);
2661                    }
2662                }
2663                Ok(out)
2664            }
2665
2666            async fn template_delete_tag(
2667                &self,
2668                id: &str,
2669                tag: &str,
2670            ) -> Result<bool, $crate::serve::history::HistoryError> {
2671                use $crate::serve::history::HistoryError;
2672                let backend = |e: sqlx::Error| HistoryError::Backend(e.to_string());
2673                let result = sqlx::query(&self.stmts.template_delete_tag)
2674                    .bind(id)
2675                    .bind(tag)
2676                    .execute(&self.pool)
2677                    .await
2678                    .map_err(backend)?;
2679                Ok(result.rows_affected() > 0)
2680            }
2681
2682            async fn template_launch(
2683                &self,
2684                id: &str,
2685                version: u32,
2686                launched_by: Option<&str>,
2687            ) -> Result<Option<u32>, $crate::serve::history::HistoryError> {
2688                use sqlx::Row as _;
2689                use $crate::serve::history::HistoryError;
2690                use $crate::serve::history::{sql, templates};
2691                let backend = |e: sqlx::Error| HistoryError::Backend(e.to_string());
2692
2693                // Re-launching what is already stable is a no-op: appending would
2694                // make `previous` a duplicate of `stable` and destroy the rollback
2695                // target.
2696                let existing = self.template_launches(id).await?;
2697                if templates::stable_version(&existing) == Some(version) {
2698                    return Ok(None);
2699                }
2700                // Read-max-then-insert with a bounded PK-conflict retry, exactly
2701                // like version assignment: two concurrent launches must produce
2702                // two distinct entries, never a lost write.
2703                for attempt in 1..=sql::CLAIM_ATTEMPTS {
2704                    sql::retry_backoff(attempt).await;
2705                    // As in `template_register`: a contended *read* is transient,
2706                    // so let the loop retry instead of failing the launch.
2707                    let row = match sqlx::query(&self.stmts.template_max_launch_seq)
2708                        .bind(id)
2709                        .fetch_one(&self.pool)
2710                        .await
2711                    {
2712                        Ok(row) => row,
2713                        Err(e) if attempt < sql::CLAIM_ATTEMPTS => {
2714                            tracing::debug!(
2715                                template = %id, attempt, error = %e,
2716                                "launch-log seq read lost a race; retrying"
2717                            );
2718                            continue;
2719                        }
2720                        Err(e) => return Err(backend(e)),
2721                    };
2722                    let max: i64 = row.try_get("v").map_err(backend)?;
2723                    let seq = (max as u32).saturating_add(1);
2724                    let insert = sqlx::query(&self.stmts.template_insert_launch)
2725                        .bind(id)
2726                        .bind(seq.to_string())
2727                        .bind(version.to_string())
2728                        .bind(sql::fmt_ts(chrono::Utc::now()))
2729                        .bind(launched_by)
2730                        .execute(&self.pool)
2731                        .await;
2732                    match insert {
2733                        Ok(_) => return Ok(Some(seq)),
2734                        Err(e) if attempt < sql::CLAIM_ATTEMPTS => {
2735                            tracing::debug!(
2736                                template = %id, attempt, error = %e,
2737                                "launch-log insert lost a race; retrying with the next seq"
2738                            );
2739                        }
2740                        Err(e) => return Err(backend(e)),
2741                    }
2742                }
2743                Err(HistoryError::Backend(format!(
2744                    "could not append a launch for template '{id}' after {} attempts",
2745                    sql::CLAIM_ATTEMPTS
2746                )))
2747            }
2748
2749            async fn template_launches(
2750                &self,
2751                id: &str,
2752            ) -> Result<
2753                Vec<$crate::serve::history::templates::LaunchRecord>,
2754                $crate::serve::history::HistoryError,
2755            > {
2756                use sqlx::Row as _;
2757                use $crate::serve::history::HistoryError;
2758                use $crate::serve::history::{sql, templates};
2759                let backend = |e: sqlx::Error| HistoryError::Backend(e.to_string());
2760                let rows = sqlx::query(&self.stmts.template_select_launches)
2761                    .bind(id)
2762                    .fetch_all(&self.pool)
2763                    .await
2764                    .map_err(backend)?;
2765                let mut out = Vec::with_capacity(rows.len());
2766                for r in &rows {
2767                    let seq: String = r.try_get("seq").map_err(backend)?;
2768                    let version: String = r.try_get("version").map_err(backend)?;
2769                    let launched_at: String = r.try_get("launched_at").map_err(backend)?;
2770                    let launched_by: Option<String> = r.try_get("launched_by").map_err(backend)?;
2771                    // Skip an unparseable row rather than failing the whole read —
2772                    // a corrupt entry must not make a template unusable.
2773                    let (Ok(seq), Ok(version)) = (seq.parse::<u32>(), version.parse::<u32>()) else {
2774                        continue;
2775                    };
2776                    out.push(templates::LaunchRecord {
2777                        seq,
2778                        version,
2779                        launched_at: sql::parse_ts(&launched_at),
2780                        launched_by,
2781                    });
2782                }
2783                Ok(out)
2784            }
2785
2786            async fn template_set_deprecation(
2787                &self,
2788                id: &str,
2789                record: Option<&$crate::serve::history::templates::DeprecationRecord>,
2790            ) -> Result<(), $crate::serve::history::HistoryError> {
2791                use $crate::serve::history::HistoryError;
2792                use $crate::serve::history::sql;
2793                let backend = |e: sqlx::Error| HistoryError::Backend(e.to_string());
2794                match record {
2795                    Some(r) => {
2796                        sqlx::query(&self.stmts.template_upsert_deprecation)
2797                            .bind(id)
2798                            .bind(sql::fmt_ts(r.deprecated_at))
2799                            .bind(r.deprecated_by.as_deref())
2800                            .bind(r.reason.as_deref())
2801                            .execute(&self.pool)
2802                            .await
2803                            .map_err(backend)?;
2804                    }
2805                    None => {
2806                        sqlx::query(&self.stmts.template_delete_deprecation)
2807                            .bind(id)
2808                            .execute(&self.pool)
2809                            .await
2810                            .map_err(backend)?;
2811                    }
2812                }
2813                Ok(())
2814            }
2815
2816            async fn template_deprecation(
2817                &self,
2818                id: &str,
2819            ) -> Result<
2820                Option<$crate::serve::history::templates::DeprecationRecord>,
2821                $crate::serve::history::HistoryError,
2822            > {
2823                use sqlx::Row as _;
2824                use $crate::serve::history::HistoryError;
2825                use $crate::serve::history::{sql, templates};
2826                let backend = |e: sqlx::Error| HistoryError::Backend(e.to_string());
2827                let Some(row) = sqlx::query(&self.stmts.template_select_deprecation)
2828                    .bind(id)
2829                    .fetch_optional(&self.pool)
2830                    .await
2831                    .map_err(backend)?
2832                else {
2833                    return Ok(None);
2834                };
2835                let at: String = row.try_get("deprecated_at").map_err(backend)?;
2836                Ok(Some(templates::DeprecationRecord {
2837                    deprecated_at: sql::parse_ts(&at),
2838                    deprecated_by: row.try_get("deprecated_by").map_err(backend)?,
2839                    reason: row.try_get("reason").map_err(backend)?,
2840                }))
2841            }
2842
2843            fn degraded(&self) -> bool {
2844                // A live SQL backend is never self-degraded; the FallbackHistory
2845                // wrapper owns degradation when the backend becomes unreachable.
2846                false
2847            }
2848        }
2849
2850        impl $name {
2851            /// Every catalog lineage edge, newest activity first (#279).
2852            async fn catalog_all_edges(
2853                &self,
2854            ) -> Result<
2855                Vec<$crate::serve::history::catalog::CatalogLineageEdge>,
2856                $crate::serve::history::HistoryError,
2857            > {
2858                use sqlx::Row as _;
2859                use $crate::serve::history::HistoryError;
2860                use $crate::serve::history::sql;
2861                let backend = |e: sqlx::Error| HistoryError::Backend(e.to_string());
2862                let rows = sqlx::query(&self.stmts.catalog_select_edges)
2863                    .fetch_all(&self.pool)
2864                    .await
2865                    .map_err(backend)?;
2866                let mut edges = Vec::with_capacity(rows.len());
2867                for r in &rows {
2868                    let body: String = r.try_get("body").map_err(backend)?;
2869                    edges.push(sql::decode_json(&body, "catalog edge")?);
2870                }
2871                Ok(edges)
2872            }
2873        }
2874    };
2875}
2876
2877pub(crate) use impl_sql_history;
2878
2879#[cfg(test)]
2880mod tests {
2881    use super::*;
2882
2883    #[test]
2884    fn postgres_shard_statements_are_built() {
2885        // SQLite tests only build the Sqlite statement set; exercise the
2886        // Postgres shard-statement construction too (Mode B, #230).
2887        let s = Stmts::new(Dialect::Postgres);
2888        assert!(s.insert_shard.contains("faucet_serve_shards"));
2889        assert!(s.insert_shard.contains("ON CONFLICT"));
2890        assert!(s.claim_shards_select.contains("JOIN faucet_serve_runs"));
2891        assert!(s.claim_shard_one.contains("'running'"));
2892        assert!(s.renew_shard_leases.contains("lease_expires_at"));
2893        assert!(s.reclaim_shards_select.contains("'running'"));
2894        assert!(s.reclaim_shard_requeue.contains("'pending'"));
2895        assert!(s.reclaim_shard_fail.contains("'failed'"));
2896        assert!(s.finalize_shard.contains("owner"));
2897        assert!(s.shard_progress.contains("GROUP BY"));
2898    }
2899
2900    #[test]
2901    fn fmt_ts_is_fixed_width_and_sortable() {
2902        let a = fmt_ts(
2903            DateTime::parse_from_rfc3339("2026-01-01T00:00:00Z")
2904                .unwrap()
2905                .to_utc(),
2906        );
2907        let b = fmt_ts(
2908            DateTime::parse_from_rfc3339("2026-01-01T00:00:01Z")
2909                .unwrap()
2910                .to_utc(),
2911        );
2912        assert!(a.ends_with('Z'));
2913        assert_eq!(a.len(), b.len(), "fixed width");
2914        assert!(a < b, "lexicographic order matches chronological order");
2915    }
2916
2917    #[test]
2918    fn is_expired_respects_window() {
2919        let now = Utc::now();
2920        let old = fmt_ts(now - chrono::Duration::seconds(120));
2921        assert!(is_expired(&old, now, Duration::from_secs(60)));
2922        assert!(!is_expired(&old, now, Duration::from_secs(600)));
2923        // Unparseable → not expired (conservative).
2924        assert!(!is_expired("not-a-timestamp", now, Duration::ZERO));
2925    }
2926
2927    #[test]
2928    fn parse_status_round_trips_known_and_defaults_failed() {
2929        for s in [
2930            RunStatus::Queued,
2931            RunStatus::Pending,
2932            RunStatus::Running,
2933            RunStatus::Completed,
2934            RunStatus::Failed,
2935            RunStatus::Cancelled,
2936        ] {
2937            assert_eq!(parse_status(s.as_str()), s);
2938        }
2939        assert_eq!(parse_status("garbage"), RunStatus::Failed);
2940    }
2941
2942    #[test]
2943    fn body_round_trips() {
2944        let rec = RunRecord::queued(
2945            "r1".into(),
2946            Some("n".into()),
2947            Default::default(),
2948            Some("idem".into()),
2949            Utc::now(),
2950        );
2951        let encoded = encode_body(&rec).unwrap();
2952        let decoded = decode_body(&encoded).unwrap();
2953        assert_eq!(decoded.run_id, "r1");
2954        assert_eq!(decoded.idempotency_key.as_deref(), Some("idem"));
2955    }
2956
2957    #[test]
2958    fn postgres_and_sqlite_statements_differ_only_in_placeholders() {
2959        let pg = Stmts::new(Dialect::Postgres);
2960        let lite = Stmts::new(Dialect::Sqlite);
2961        assert!(pg.upsert.contains("$1") && lite.upsert.contains('?'));
2962        assert!(pg.list.contains("$13") && lite.list.contains('?'));
2963        // Both target the same tables / conflict targets.
2964        assert!(pg.insert_idem.contains("ON CONFLICT (key) DO NOTHING"));
2965        assert!(lite.insert_idem.contains("ON CONFLICT (key) DO NOTHING"));
2966        assert!(pg.claim_one.contains("$3") && lite.claim_one.contains('?'));
2967        assert!(pg.heartbeat_instance.contains("faucet_serve_instances"));
2968        assert!(lite.heartbeat_instance.contains("faucet_serve_instances"));
2969    }
2970}