Skip to main content

faucet_cli/serve/history/
sql.rs

1//! Shared SQL run-history machinery for the Postgres and SQLite backends
2//! (Phase 5, #127). Both backends are identical except for the connection setup
3//! and the placeholder dialect (`$n` vs `?`), so the schema, prepared-statement
4//! text, pure helpers, and the entire `RunHistory` impl live here once and are
5//! instantiated for each concrete `sqlx` pool via the `impl_sql_history!` macro.
6//!
7//! **Portability:** every column is `TEXT` (timestamps are stored as fixed-width
8//! RFC3339 with nanosecond precision + `Z`, which sorts lexicographically in
9//! chronological order, so keyset pagination and expiry comparisons work without
10//! any database date type — and thus without the `sqlx` `chrono` feature). The
11//! whole `RunRecord` is serialized into the `body` column (the source of truth on
12//! read); the dedicated columns exist only for filtering, ordering, and expiry.
13//!
14//! **Idempotency** lives in a separate `faucet_serve_idem` table whose `key`
15//! primary key is the required unique index (spec §10/§11). The claim is atomic
16//! via `INSERT … ON CONFLICT DO NOTHING` plus an optimistic, expiry-guarded
17//! takeover `UPDATE`, mirroring the memory backend's shard-locked semantics.
18
19use super::{HistoryError, RunRecord, RunStatus};
20use chrono::{DateTime, Utc};
21use std::time::Duration;
22
23/// DDL run at connect time. Valid verbatim on both Postgres and SQLite (only
24/// `TEXT` columns, `IF NOT EXISTS`, and standard indexes).
25pub const DDL: &[&str] = &[
26    // `owner` is the id of the serve instance that owns the run; `lease_expires_at`
27    // is the RFC3339 instant past which that ownership is presumed dead. Together
28    // they fence orphan recovery: an instance only fails a non-terminal run whose
29    // lease has expired, never another live instance's heartbeated runs (#146 H7).
30    "CREATE TABLE IF NOT EXISTS faucet_serve_runs (\
31        run_id TEXT PRIMARY KEY,\
32        name TEXT,\
33        status TEXT NOT NULL,\
34        submitted_at TEXT NOT NULL,\
35        finished_at TEXT,\
36        idempotency_key TEXT,\
37        owner TEXT,\
38        lease_expires_at TEXT,\
39        body TEXT NOT NULL)",
40    "CREATE INDEX IF NOT EXISTS faucet_serve_runs_submitted_idx \
41        ON faucet_serve_runs (submitted_at)",
42    // Speeds the per-tick orphan scan / lease renewal, which filter on
43    // (status, owner, lease_expires_at).
44    "CREATE INDEX IF NOT EXISTS faucet_serve_runs_status_lease_idx \
45        ON faucet_serve_runs (status, lease_expires_at)",
46    "CREATE TABLE IF NOT EXISTS faucet_serve_idem (\
47        key TEXT PRIMARY KEY,\
48        run_id TEXT NOT NULL,\
49        fingerprint TEXT NOT NULL,\
50        claimed_at TEXT NOT NULL)",
51];
52
53/// SQL placeholder dialect.
54#[derive(Clone, Copy, Debug)]
55pub enum Dialect {
56    Postgres,
57    Sqlite,
58}
59
60/// Prepared-statement text for a backend, built once per dialect at connect time.
61pub struct Stmts {
62    pub upsert: String,
63    pub select_body: String,
64    pub select_status: String,
65    pub select_submitted: String,
66    pub delete: String,
67    pub list: String,
68    pub purge_runs: String,
69    pub purge_idem: String,
70    /// Select non-terminal runs whose owning instance's lease has expired (or
71    /// is unset) — the orphans this instance may safely fail. Param: `now`.
72    pub select_orphans: String,
73    /// Extend the lease of this instance's own non-terminal runs (heartbeat).
74    /// Params: `new_lease_expiry`, `instance_id`.
75    pub renew_leases: String,
76    pub insert_idem: String,
77    pub select_idem: String,
78    pub takeover_idem: String,
79    /// Delete the idempotency claim(s) that point at a given run — used when a
80    /// run is deleted so a replay of the key starts fresh rather than 404-ing
81    /// on the missing record (#146 M8). Scoped by `run_id`, so a newer run that
82    /// re-claimed the same key keeps its claim.
83    pub delete_idem_by_run: String,
84}
85
86impl Stmts {
87    pub fn new(dialect: Dialect) -> Self {
88        match dialect {
89            Dialect::Postgres => Self::postgres(),
90            Dialect::Sqlite => Self::sqlite(),
91        }
92    }
93
94    fn postgres() -> Self {
95        Self {
96            upsert: "INSERT INTO faucet_serve_runs \
97                (run_id,name,status,submitted_at,finished_at,idempotency_key,owner,lease_expires_at,body) \
98                VALUES ($1,$2,$3,$4,$5,$6,$7,$8,$9) \
99                ON CONFLICT (run_id) DO UPDATE SET \
100                name=excluded.name,status=excluded.status,submitted_at=excluded.submitted_at,\
101                finished_at=excluded.finished_at,idempotency_key=excluded.idempotency_key,\
102                owner=excluded.owner,lease_expires_at=excluded.lease_expires_at,\
103                body=excluded.body"
104                .into(),
105            select_body: "SELECT body FROM faucet_serve_runs WHERE run_id=$1".into(),
106            select_status: "SELECT status FROM faucet_serve_runs WHERE run_id=$1".into(),
107            select_submitted: "SELECT submitted_at FROM faucet_serve_runs WHERE run_id=$1".into(),
108            delete: "DELETE FROM faucet_serve_runs WHERE run_id=$1".into(),
109            // Casts make the parameter types explicit so `$n IS NULL` cannot trip
110            // Postgres' "could not determine data type of parameter" check.
111            list: "SELECT body FROM faucet_serve_runs \
112                WHERE ($1::text IS NULL OR status = $2::text) \
113                AND ($3::text IS NULL OR name = $4::text) \
114                AND ($5::text IS NULL OR submitted_at >= $6::text) \
115                AND ($7::text IS NULL OR submitted_at <= $8::text) \
116                AND ($9::text IS NULL OR (submitted_at < $10::text \
117                    OR (submitted_at = $11::text AND run_id < $12::text))) \
118                ORDER BY submitted_at DESC, run_id DESC LIMIT $13"
119                .into(),
120            purge_runs: "DELETE FROM faucet_serve_runs \
121                WHERE status IN ('completed','failed','cancelled') \
122                AND finished_at IS NOT NULL AND finished_at < $1"
123                .into(),
124            purge_idem: "DELETE FROM faucet_serve_idem WHERE claimed_at < $1".into(),
125            select_orphans: "SELECT body FROM faucet_serve_runs \
126                WHERE status IN ('queued','running') \
127                AND (lease_expires_at IS NULL OR lease_expires_at < $1)"
128                .into(),
129            renew_leases: "UPDATE faucet_serve_runs SET lease_expires_at = $1 \
130                WHERE owner = $2 AND status IN ('queued','running')"
131                .into(),
132            insert_idem: "INSERT INTO faucet_serve_idem (key,run_id,fingerprint,claimed_at) \
133                VALUES ($1,$2,$3,$4) ON CONFLICT (key) DO NOTHING"
134                .into(),
135            select_idem: "SELECT run_id,fingerprint,claimed_at FROM faucet_serve_idem WHERE key=$1"
136                .into(),
137            takeover_idem: "UPDATE faucet_serve_idem \
138                SET run_id=$1,fingerprint=$2,claimed_at=$3 WHERE key=$4 AND claimed_at=$5"
139                .into(),
140            delete_idem_by_run: "DELETE FROM faucet_serve_idem WHERE run_id=$1".into(),
141        }
142    }
143
144    fn sqlite() -> Self {
145        Self {
146            upsert: "INSERT INTO faucet_serve_runs \
147                (run_id,name,status,submitted_at,finished_at,idempotency_key,owner,lease_expires_at,body) \
148                VALUES (?,?,?,?,?,?,?,?,?) \
149                ON CONFLICT (run_id) DO UPDATE SET \
150                name=excluded.name,status=excluded.status,submitted_at=excluded.submitted_at,\
151                finished_at=excluded.finished_at,idempotency_key=excluded.idempotency_key,\
152                owner=excluded.owner,lease_expires_at=excluded.lease_expires_at,\
153                body=excluded.body"
154                .into(),
155            select_body: "SELECT body FROM faucet_serve_runs WHERE run_id=?".into(),
156            select_status: "SELECT status FROM faucet_serve_runs WHERE run_id=?".into(),
157            select_submitted: "SELECT submitted_at FROM faucet_serve_runs WHERE run_id=?".into(),
158            delete: "DELETE FROM faucet_serve_runs WHERE run_id=?".into(),
159            list: "SELECT body FROM faucet_serve_runs \
160                WHERE (? IS NULL OR status = ?) \
161                AND (? IS NULL OR name = ?) \
162                AND (? IS NULL OR submitted_at >= ?) \
163                AND (? IS NULL OR submitted_at <= ?) \
164                AND (? IS NULL OR (submitted_at < ? \
165                    OR (submitted_at = ? AND run_id < ?))) \
166                ORDER BY submitted_at DESC, run_id DESC LIMIT ?"
167                .into(),
168            purge_runs: "DELETE FROM faucet_serve_runs \
169                WHERE status IN ('completed','failed','cancelled') \
170                AND finished_at IS NOT NULL AND finished_at < ?"
171                .into(),
172            purge_idem: "DELETE FROM faucet_serve_idem WHERE claimed_at < ?".into(),
173            select_orphans: "SELECT body FROM faucet_serve_runs \
174                WHERE status IN ('queued','running') \
175                AND (lease_expires_at IS NULL OR lease_expires_at < ?)"
176                .into(),
177            renew_leases: "UPDATE faucet_serve_runs SET lease_expires_at = ? \
178                WHERE owner = ? AND status IN ('queued','running')"
179                .into(),
180            insert_idem: "INSERT INTO faucet_serve_idem (key,run_id,fingerprint,claimed_at) \
181                VALUES (?,?,?,?) ON CONFLICT (key) DO NOTHING"
182                .into(),
183            select_idem: "SELECT run_id,fingerprint,claimed_at FROM faucet_serve_idem WHERE key=?"
184                .into(),
185            takeover_idem: "UPDATE faucet_serve_idem \
186                SET run_id=?,fingerprint=?,claimed_at=? WHERE key=? AND claimed_at=?"
187                .into(),
188            delete_idem_by_run: "DELETE FROM faucet_serve_idem WHERE run_id=?".into(),
189        }
190    }
191}
192
193/// Bounded retry count for the atomic idempotency claim (handles a claim being
194/// purged concurrently between the insert attempt and the read-back).
195pub const CLAIM_ATTEMPTS: usize = 4;
196
197/// Fixed-width RFC3339 (nanoseconds + `Z`) — lexicographically sortable.
198pub fn fmt_ts(dt: DateTime<Utc>) -> String {
199    dt.to_rfc3339_opts(chrono::SecondsFormat::Nanos, true)
200}
201
202/// True when a claim timestamped `claimed_at` (RFC3339) is older than `window`.
203/// An unparseable or future timestamp is treated as **not** expired (safe: it
204/// won't be silently re-claimed).
205pub fn is_expired(claimed_at: &str, now: DateTime<Utc>, window: Duration) -> bool {
206    match DateTime::parse_from_rfc3339(claimed_at) {
207        Ok(t) => now
208            .signed_duration_since(t.with_timezone(&Utc))
209            .to_std()
210            .map(|age| age >= window)
211            .unwrap_or(false),
212        Err(_) => false,
213    }
214}
215
216/// RFC3339 timestamp `window` before `now` (the purge / expiry threshold).
217pub fn threshold(now: DateTime<Utc>, window: Duration) -> String {
218    let delta =
219        chrono::Duration::from_std(window).unwrap_or_else(|_| chrono::Duration::days(36_500));
220    fmt_ts(now - delta)
221}
222
223pub fn encode_body(rec: &RunRecord) -> Result<String, HistoryError> {
224    serde_json::to_string(rec).map_err(|e| HistoryError::Backend(format!("encode run record: {e}")))
225}
226
227pub fn decode_body(body: &str) -> Result<RunRecord, HistoryError> {
228    serde_json::from_str(body).map_err(|e| HistoryError::Backend(format!("decode run record: {e}")))
229}
230
231pub fn parse_status(s: &str) -> RunStatus {
232    match s {
233        "queued" => RunStatus::Queued,
234        "running" => RunStatus::Running,
235        "completed" => RunStatus::Completed,
236        "cancelled" => RunStatus::Cancelled,
237        _ => RunStatus::Failed,
238    }
239}
240
241/// Generate a concrete `RunHistory` implementation over a specific `sqlx` pool.
242/// `$name` is the backend struct, `$pool` its `sqlx` pool type. The struct holds
243/// the pool, the idempotency retention window, and the dialect's [`Stmts`].
244macro_rules! impl_sql_history {
245    ($name:ident, $pool:ty) => {
246        /// SQL-backed [`RunHistory`](crate::serve::history::RunHistory). See
247        /// [`crate::serve::history::sql`] for the shared schema + semantics.
248        pub struct $name {
249            pool: $pool,
250            idem_retention: std::time::Duration,
251            /// This serve instance's id, stamped as `owner` on every upsert.
252            instance_id: String,
253            /// How far ahead each upsert / heartbeat pushes a run's lease.
254            lease_ttl: std::time::Duration,
255            stmts: $crate::serve::history::sql::Stmts,
256        }
257
258        impl $name {
259            /// Assemble from an already-connected pool (used by `connect`).
260            pub fn from_parts(
261                pool: $pool,
262                idem_retention: std::time::Duration,
263                lease_ttl: std::time::Duration,
264                instance_id: String,
265                stmts: $crate::serve::history::sql::Stmts,
266            ) -> Self {
267                Self {
268                    pool,
269                    idem_retention,
270                    instance_id,
271                    lease_ttl,
272                    stmts,
273                }
274            }
275
276            /// Borrow the underlying pool (tests close it to exercise fallback).
277            pub fn pool(&self) -> &$pool {
278                &self.pool
279            }
280        }
281
282        #[async_trait::async_trait]
283        impl $crate::serve::history::RunHistory for $name {
284            async fn claim_idempotency(
285                &self,
286                key: &str,
287                fingerprint: &str,
288                run_id: &str,
289                window: std::time::Duration,
290            ) -> Result<$crate::serve::history::Claim, $crate::serve::history::HistoryError> {
291                use sqlx::Row as _;
292                use $crate::serve::history::Claim;
293                use $crate::serve::history::HistoryError;
294                use $crate::serve::history::sql;
295
296                let now = chrono::Utc::now();
297                let now_s = sql::fmt_ts(now);
298                let backend = |e: sqlx::Error| HistoryError::Backend(e.to_string());
299
300                for _ in 0..sql::CLAIM_ATTEMPTS {
301                    // 1) Atomic first-claim: the winner inserts exactly one row.
302                    let inserted = sqlx::query(&self.stmts.insert_idem)
303                        .bind(key)
304                        .bind(run_id)
305                        .bind(fingerprint)
306                        .bind(&now_s)
307                        .execute(&self.pool)
308                        .await
309                        .map_err(backend)?
310                        .rows_affected();
311                    if inserted == 1 {
312                        return Ok(Claim::Fresh);
313                    }
314                    // 2) Conflict: inspect the existing claim.
315                    let Some(row) = sqlx::query(&self.stmts.select_idem)
316                        .bind(key)
317                        .fetch_optional(&self.pool)
318                        .await
319                        .map_err(backend)?
320                    else {
321                        // Vanished between the insert and the read — retry.
322                        continue;
323                    };
324                    let existing_run: String = row.try_get("run_id").map_err(backend)?;
325                    let existing_fp: String = row.try_get("fingerprint").map_err(backend)?;
326                    let claimed_at: String = row.try_get("claimed_at").map_err(backend)?;
327
328                    if sql::is_expired(&claimed_at, now, window) {
329                        // 3) Optimistic, expiry-guarded takeover: only the request
330                        // that still sees `claimed_at` succeeds.
331                        let took = sqlx::query(&self.stmts.takeover_idem)
332                            .bind(run_id)
333                            .bind(fingerprint)
334                            .bind(&now_s)
335                            .bind(key)
336                            .bind(&claimed_at)
337                            .execute(&self.pool)
338                            .await
339                            .map_err(backend)?
340                            .rows_affected();
341                        if took == 1 {
342                            return Ok(Claim::Fresh);
343                        }
344                        continue; // lost the race; re-evaluate
345                    }
346                    return Ok(if existing_fp == fingerprint {
347                        Claim::Replay(existing_run)
348                    } else {
349                        Claim::Conflict
350                    });
351                }
352                // Pathological contention only. Conservative: a 409 is safer than
353                // risking a duplicate run.
354                tracing::warn!(
355                    key,
356                    "idempotency claim exhausted retries; reporting conflict"
357                );
358                Ok(Claim::Conflict)
359            }
360
361            async fn upsert(
362                &self,
363                rec: &$crate::serve::history::RunRecord,
364            ) -> Result<(), $crate::serve::history::HistoryError> {
365                use $crate::serve::history::HistoryError;
366                use $crate::serve::history::sql;
367                let body = sql::encode_body(rec)?;
368                let submitted = sql::fmt_ts(rec.submitted_at);
369                let finished = rec.finished_at.map(sql::fmt_ts);
370                // Stamp this instance as the owner and start/renew the lease.
371                // The owner/lease are SQL-column-only (never in the record body),
372                // so the heartbeat can extend a lease without a body read-modify-
373                // write race (#146 H7).
374                let lease = sql::fmt_ts(chrono::Utc::now() + self.lease_ttl);
375                sqlx::query(&self.stmts.upsert)
376                    .bind(&rec.run_id)
377                    .bind(rec.name.as_deref())
378                    .bind(rec.status.as_str())
379                    .bind(&submitted)
380                    .bind(finished.as_deref())
381                    .bind(rec.idempotency_key.as_deref())
382                    .bind(&self.instance_id)
383                    .bind(&lease)
384                    .bind(&body)
385                    .execute(&self.pool)
386                    .await
387                    .map_err(|e| HistoryError::Backend(e.to_string()))?;
388                Ok(())
389            }
390
391            async fn get(
392                &self,
393                id: &str,
394            ) -> Result<
395                Option<$crate::serve::history::RunRecord>,
396                $crate::serve::history::HistoryError,
397            > {
398                use sqlx::Row as _;
399                use $crate::serve::history::HistoryError;
400                use $crate::serve::history::sql;
401                let row = sqlx::query(&self.stmts.select_body)
402                    .bind(id)
403                    .fetch_optional(&self.pool)
404                    .await
405                    .map_err(|e| HistoryError::Backend(e.to_string()))?;
406                match row {
407                    None => Ok(None),
408                    Some(r) => {
409                        let body: String = r
410                            .try_get("body")
411                            .map_err(|e| HistoryError::Backend(e.to_string()))?;
412                        Ok(Some(sql::decode_body(&body)?))
413                    }
414                }
415            }
416
417            async fn list(
418                &self,
419                filter: &$crate::serve::history::ListFilter,
420            ) -> Result<$crate::serve::history::ListPage, $crate::serve::history::HistoryError>
421            {
422                use sqlx::Row as _;
423                use $crate::serve::history::HistoryError;
424                use $crate::serve::history::ListPage;
425                use $crate::serve::history::sql;
426                let backend = |e: sqlx::Error| HistoryError::Backend(e.to_string());
427
428                // Resolve the cursor's submitted_at for keyset pagination. An
429                // unknown cursor is ignored (page starts from the top), matching
430                // the memory backend.
431                let cursor_ts: Option<String> = match &filter.cursor {
432                    None => None,
433                    Some(c) => sqlx::query(&self.stmts.select_submitted)
434                        .bind(c)
435                        .fetch_optional(&self.pool)
436                        .await
437                        .map_err(backend)?
438                        .map(|r| r.try_get::<String, _>("submitted_at"))
439                        .transpose()
440                        .map_err(backend)?,
441                };
442                let cur_id = if cursor_ts.is_some() {
443                    filter.cursor.as_deref()
444                } else {
445                    None
446                };
447
448                let status_s = filter.status.map(|s| s.as_str());
449                let name_s = filter.name.as_deref();
450                let since_s = filter.since.map(sql::fmt_ts);
451                let until_s = filter.until.map(sql::fmt_ts);
452                let limit = filter.limit.max(1);
453                let fetch_n = limit as i64 + 1; // +1 to detect a next page
454
455                let rows = sqlx::query(&self.stmts.list)
456                    .bind(status_s)
457                    .bind(status_s)
458                    .bind(name_s)
459                    .bind(name_s)
460                    .bind(since_s.as_deref())
461                    .bind(since_s.as_deref())
462                    .bind(until_s.as_deref())
463                    .bind(until_s.as_deref())
464                    .bind(cursor_ts.as_deref())
465                    .bind(cursor_ts.as_deref())
466                    .bind(cursor_ts.as_deref())
467                    .bind(cur_id)
468                    .bind(fetch_n)
469                    .fetch_all(&self.pool)
470                    .await
471                    .map_err(backend)?;
472
473                let mut runs = Vec::with_capacity(rows.len());
474                for r in &rows {
475                    let body: String = r.try_get("body").map_err(backend)?;
476                    runs.push(sql::decode_body(&body)?);
477                }
478                let next_cursor = if runs.len() > limit {
479                    Some(runs[limit - 1].run_id.clone())
480                } else {
481                    None
482                };
483                runs.truncate(limit);
484                Ok(ListPage { runs, next_cursor })
485            }
486
487            async fn delete(
488                &self,
489                id: &str,
490            ) -> Result<$crate::serve::history::DeleteOutcome, $crate::serve::history::HistoryError>
491            {
492                use sqlx::Row as _;
493                use $crate::serve::history::DeleteOutcome;
494                use $crate::serve::history::HistoryError;
495                use $crate::serve::history::sql;
496                let backend = |e: sqlx::Error| HistoryError::Backend(e.to_string());
497                let status: Option<String> = sqlx::query(&self.stmts.select_status)
498                    .bind(id)
499                    .fetch_optional(&self.pool)
500                    .await
501                    .map_err(backend)?
502                    .map(|r| r.try_get::<String, _>("status"))
503                    .transpose()
504                    .map_err(backend)?;
505                match status {
506                    None => Ok(DeleteOutcome::NotFound),
507                    Some(s) if !sql::parse_status(&s).is_terminal() => {
508                        Ok(DeleteOutcome::StillRunning)
509                    }
510                    Some(_) => {
511                        sqlx::query(&self.stmts.delete)
512                            .bind(id)
513                            .execute(&self.pool)
514                            .await
515                            .map_err(backend)?;
516                        // Drop the run's idempotency claim too, so a replay of
517                        // the key starts fresh instead of 404-ing on the deleted
518                        // record until the claim self-expires (#146 M8). Scoped
519                        // by run_id, so a newer run that re-claimed the same key
520                        // keeps its claim.
521                        sqlx::query(&self.stmts.delete_idem_by_run)
522                            .bind(id)
523                            .execute(&self.pool)
524                            .await
525                            .map_err(backend)?;
526                        Ok(DeleteOutcome::Deleted)
527                    }
528                }
529            }
530
531            async fn purge_expired(
532                &self,
533                retain_for: std::time::Duration,
534            ) -> Result<usize, $crate::serve::history::HistoryError> {
535                use $crate::serve::history::HistoryError;
536                use $crate::serve::history::sql;
537                let backend = |e: sqlx::Error| HistoryError::Backend(e.to_string());
538                let now = chrono::Utc::now();
539                let removed = sqlx::query(&self.stmts.purge_runs)
540                    .bind(sql::threshold(now, retain_for))
541                    .execute(&self.pool)
542                    .await
543                    .map_err(backend)?
544                    .rows_affected() as usize;
545                // Drop expired idempotency claims too (best-effort).
546                let _ = sqlx::query(&self.stmts.purge_idem)
547                    .bind(sql::threshold(now, self.idem_retention))
548                    .execute(&self.pool)
549                    .await;
550                Ok(removed)
551            }
552
553            async fn recover_orphans(&self) -> Result<usize, $crate::serve::history::HistoryError> {
554                use sqlx::Row as _;
555                use $crate::serve::history::HistoryError;
556                use $crate::serve::history::RunStatus;
557                use $crate::serve::history::sql;
558                let backend = |e: sqlx::Error| HistoryError::Backend(e.to_string());
559                let now = chrono::Utc::now();
560                // Only non-terminal runs whose lease has expired (the owning
561                // instance is presumed dead). A live instance heartbeats its
562                // runs' leases into the future, so this never fails another
563                // healthy instance's in-flight runs (#146 H7).
564                let rows = sqlx::query(&self.stmts.select_orphans)
565                    .bind(sql::fmt_ts(now))
566                    .fetch_all(&self.pool)
567                    .await
568                    .map_err(backend)?;
569                let mut count = 0usize;
570                for r in &rows {
571                    let body: String = r.try_get("body").map_err(backend)?;
572                    let mut rec = sql::decode_body(&body)?;
573                    rec.status = RunStatus::Failed;
574                    rec.finished_at = Some(now);
575                    rec.error = Some(
576                        "owning serve instance's lease expired before the run finished".into(),
577                    );
578                    if rec.elapsed_secs.is_none()
579                        && let Some(started) = rec.started_at
580                    {
581                        rec.elapsed_secs = (now - started).to_std().ok().map(|d| d.as_secs_f64());
582                    }
583                    self.upsert(&rec).await?;
584                    count += 1;
585                }
586                Ok(count)
587            }
588
589            async fn renew_leases(&self) -> Result<usize, $crate::serve::history::HistoryError> {
590                use $crate::serve::history::HistoryError;
591                use $crate::serve::history::sql;
592                let backend = |e: sqlx::Error| HistoryError::Backend(e.to_string());
593                let new_lease = sql::fmt_ts(chrono::Utc::now() + self.lease_ttl);
594                let renewed = sqlx::query(&self.stmts.renew_leases)
595                    .bind(&new_lease)
596                    .bind(&self.instance_id)
597                    .execute(&self.pool)
598                    .await
599                    .map_err(backend)?
600                    .rows_affected() as usize;
601                Ok(renewed)
602            }
603
604            fn degraded(&self) -> bool {
605                // A live SQL backend is never self-degraded; the FallbackHistory
606                // wrapper owns degradation when the backend becomes unreachable.
607                false
608            }
609        }
610    };
611}
612
613pub(crate) use impl_sql_history;
614
615#[cfg(test)]
616mod tests {
617    use super::*;
618
619    #[test]
620    fn fmt_ts_is_fixed_width_and_sortable() {
621        let a = fmt_ts(
622            DateTime::parse_from_rfc3339("2026-01-01T00:00:00Z")
623                .unwrap()
624                .to_utc(),
625        );
626        let b = fmt_ts(
627            DateTime::parse_from_rfc3339("2026-01-01T00:00:01Z")
628                .unwrap()
629                .to_utc(),
630        );
631        assert!(a.ends_with('Z'));
632        assert_eq!(a.len(), b.len(), "fixed width");
633        assert!(a < b, "lexicographic order matches chronological order");
634    }
635
636    #[test]
637    fn is_expired_respects_window() {
638        let now = Utc::now();
639        let old = fmt_ts(now - chrono::Duration::seconds(120));
640        assert!(is_expired(&old, now, Duration::from_secs(60)));
641        assert!(!is_expired(&old, now, Duration::from_secs(600)));
642        // Unparseable → not expired (conservative).
643        assert!(!is_expired("not-a-timestamp", now, Duration::ZERO));
644    }
645
646    #[test]
647    fn parse_status_round_trips_known_and_defaults_failed() {
648        for s in [
649            RunStatus::Queued,
650            RunStatus::Running,
651            RunStatus::Completed,
652            RunStatus::Failed,
653            RunStatus::Cancelled,
654        ] {
655            assert_eq!(parse_status(s.as_str()), s);
656        }
657        assert_eq!(parse_status("garbage"), RunStatus::Failed);
658    }
659
660    #[test]
661    fn body_round_trips() {
662        let rec = RunRecord::queued(
663            "r1".into(),
664            Some("n".into()),
665            Default::default(),
666            Some("idem".into()),
667            Utc::now(),
668        );
669        let encoded = encode_body(&rec).unwrap();
670        let decoded = decode_body(&encoded).unwrap();
671        assert_eq!(decoded.run_id, "r1");
672        assert_eq!(decoded.idempotency_key.as_deref(), Some("idem"));
673    }
674
675    #[test]
676    fn postgres_and_sqlite_statements_differ_only_in_placeholders() {
677        let pg = Stmts::new(Dialect::Postgres);
678        let lite = Stmts::new(Dialect::Sqlite);
679        assert!(pg.upsert.contains("$1") && lite.upsert.contains('?'));
680        assert!(pg.list.contains("$13") && lite.list.contains('?'));
681        // Both target the same tables / conflict targets.
682        assert!(pg.insert_idem.contains("ON CONFLICT (key) DO NOTHING"));
683        assert!(lite.insert_idem.contains("ON CONFLICT (key) DO NOTHING"));
684    }
685}