use super::{HistoryError, RunRecord, RunStatus};
use chrono::{DateTime, Utc};
use std::time::Duration;
pub const DDL: &[&str] = &[
"CREATE TABLE IF NOT EXISTS faucet_serve_runs (\
run_id TEXT PRIMARY KEY,\
name TEXT,\
status TEXT NOT NULL,\
submitted_at TEXT NOT NULL,\
finished_at TEXT,\
idempotency_key TEXT,\
owner TEXT,\
lease_expires_at TEXT,\
cancel_requested TEXT,\
body TEXT NOT NULL)",
"CREATE INDEX IF NOT EXISTS faucet_serve_runs_submitted_idx \
ON faucet_serve_runs (submitted_at)",
"CREATE INDEX IF NOT EXISTS faucet_serve_runs_status_lease_idx \
ON faucet_serve_runs (status, lease_expires_at)",
"CREATE INDEX IF NOT EXISTS faucet_serve_runs_pending_idx \
ON faucet_serve_runs (status, submitted_at)",
"CREATE TABLE IF NOT EXISTS faucet_serve_instances (\
instance_id TEXT PRIMARY KEY,\
started_at TEXT NOT NULL,\
last_heartbeat TEXT NOT NULL,\
listen TEXT,\
max_concurrent TEXT,\
in_flight TEXT)",
"CREATE INDEX IF NOT EXISTS faucet_serve_instances_hb_idx \
ON faucet_serve_instances (last_heartbeat)",
"CREATE TABLE IF NOT EXISTS faucet_serve_idem (\
key TEXT PRIMARY KEY,\
run_id TEXT NOT NULL,\
fingerprint TEXT NOT NULL,\
claimed_at TEXT NOT NULL)",
"CREATE TABLE IF NOT EXISTS faucet_serve_shards (\
run_id TEXT NOT NULL,\
shard_id TEXT NOT NULL,\
descriptor TEXT NOT NULL,\
size_estimate TEXT,\
status TEXT NOT NULL,\
owner TEXT,\
lease_expires_at TEXT,\
attempt TEXT NOT NULL,\
finished_at TEXT,\
PRIMARY KEY (run_id, shard_id))",
"CREATE INDEX IF NOT EXISTS faucet_serve_shards_claim_idx \
ON faucet_serve_shards (status, lease_expires_at)",
"CREATE TABLE IF NOT EXISTS faucet_serve_audit (\
id TEXT PRIMARY KEY,\
ts TEXT NOT NULL,\
principal TEXT NOT NULL,\
role TEXT NOT NULL,\
action TEXT NOT NULL,\
run_id TEXT,\
config_fingerprint TEXT,\
source_ip TEXT,\
result TEXT NOT NULL)",
"CREATE INDEX IF NOT EXISTS faucet_serve_audit_ts_idx \
ON faucet_serve_audit (ts)",
"CREATE TABLE IF NOT EXISTS faucet_catalog_datasets (\
id TEXT PRIMARY KEY,\
uri TEXT NOT NULL,\
kind TEXT NOT NULL,\
last_seen TEXT NOT NULL,\
body TEXT NOT NULL)",
"CREATE TABLE IF NOT EXISTS faucet_catalog_schema_versions (\
dataset_id TEXT NOT NULL,\
version TEXT NOT NULL,\
recorded_at TEXT NOT NULL,\
body TEXT NOT NULL,\
PRIMARY KEY (dataset_id, version))",
"CREATE TABLE IF NOT EXISTS faucet_catalog_edges (\
src_id TEXT NOT NULL,\
dst_id TEXT NOT NULL,\
last_seen TEXT NOT NULL,\
body TEXT NOT NULL,\
PRIMARY KEY (src_id, dst_id))",
"CREATE TABLE IF NOT EXISTS faucet_catalog_stats (\
dataset_id TEXT NOT NULL,\
recorded_at TEXT NOT NULL,\
run_id TEXT NOT NULL,\
records TEXT NOT NULL,\
PRIMARY KEY (dataset_id, recorded_at))",
"CREATE TABLE IF NOT EXISTS faucet_config_snapshots (\
pipeline TEXT PRIMARY KEY,\
recorded_at TEXT NOT NULL,\
faucet_version TEXT NOT NULL,\
body TEXT NOT NULL)",
];
#[derive(Clone, Copy, Debug)]
pub enum Dialect {
Postgres,
Sqlite,
}
pub struct Stmts {
pub upsert: String,
pub select_body: String,
pub select_status: String,
pub select_submitted: String,
pub delete: String,
pub list: String,
pub purge_runs: String,
pub purge_idem: String,
pub select_orphans: String,
pub renew_leases: String,
pub insert_idem: String,
pub select_idem: String,
pub takeover_idem: String,
pub delete_idem_by_run: String,
pub select_pending: String,
pub claim_one: String,
pub reclaim_select: String,
pub reclaim_requeue: String,
pub reclaim_fail: String,
pub finalize_owned: String,
pub cancel_pending: String,
pub request_cancel: String,
pub pending_cancellations: String,
pub heartbeat_instance: String,
pub live_instances: String,
pub prune_instances: String,
pub insert_shard: String,
pub claim_shards_select: String,
pub claim_shard_one: String,
pub renew_shard_leases: String,
pub reclaim_shards_select: String,
pub reclaim_shard_requeue: String,
pub reclaim_shard_fail: String,
pub finalize_shard: String,
pub shard_progress: String,
pub pending_shard_cancellations: String,
pub select_sharded_parents: String,
pub finalize_sharded_parent: String,
pub delete_shards_by_run: String,
pub purge_orphan_shards: String,
pub insert_audit: String,
pub list_audit: String,
pub purge_audit: String,
pub catalog_select_dataset: String,
pub catalog_upsert_dataset: String,
pub catalog_select_datasets: String,
pub catalog_insert_schema_version: String,
pub catalog_select_schema_versions: String,
pub catalog_upsert_edge: String,
pub catalog_select_edges: String,
pub catalog_insert_stat: String,
pub catalog_select_stats: String,
pub catalog_prune_stats: String,
pub catalog_upsert_config_snapshot: String,
pub catalog_select_config_snapshot: String,
}
impl Stmts {
pub fn new(dialect: Dialect) -> Self {
match dialect {
Dialect::Postgres => Self::postgres(),
Dialect::Sqlite => Self::sqlite(),
}
}
fn postgres() -> Self {
Self {
upsert: "INSERT INTO faucet_serve_runs \
(run_id,name,status,submitted_at,finished_at,idempotency_key,owner,lease_expires_at,body) \
VALUES ($1,$2,$3,$4,$5,$6,$7,$8,$9) \
ON CONFLICT (run_id) DO UPDATE SET \
name=excluded.name,status=excluded.status,submitted_at=excluded.submitted_at,\
finished_at=excluded.finished_at,idempotency_key=excluded.idempotency_key,\
owner=excluded.owner,lease_expires_at=excluded.lease_expires_at,\
body=excluded.body"
.into(),
select_body: "SELECT body FROM faucet_serve_runs WHERE run_id=$1".into(),
select_status: "SELECT status FROM faucet_serve_runs WHERE run_id=$1".into(),
select_submitted: "SELECT submitted_at FROM faucet_serve_runs WHERE run_id=$1".into(),
delete: "DELETE FROM faucet_serve_runs WHERE run_id=$1".into(),
list: "SELECT body FROM faucet_serve_runs \
WHERE ($1::text IS NULL OR status = $2::text) \
AND ($3::text IS NULL OR name = $4::text) \
AND ($5::text IS NULL OR submitted_at >= $6::text) \
AND ($7::text IS NULL OR submitted_at <= $8::text) \
AND ($9::text IS NULL OR (submitted_at < $10::text \
OR (submitted_at = $11::text AND run_id < $12::text))) \
ORDER BY submitted_at DESC, run_id DESC LIMIT $13"
.into(),
purge_runs: "DELETE FROM faucet_serve_runs \
WHERE status IN ('completed','failed','cancelled') \
AND finished_at IS NOT NULL AND finished_at < $1"
.into(),
purge_idem: "DELETE FROM faucet_serve_idem WHERE claimed_at < $1".into(),
select_orphans: "SELECT body FROM faucet_serve_runs \
WHERE status IN ('queued','running') \
AND (lease_expires_at IS NULL OR lease_expires_at < $1)"
.into(),
renew_leases: "UPDATE faucet_serve_runs SET lease_expires_at = $1 \
WHERE owner = $2 AND status IN ('queued','running')"
.into(),
insert_idem: "INSERT INTO faucet_serve_idem (key,run_id,fingerprint,claimed_at) \
VALUES ($1,$2,$3,$4) ON CONFLICT (key) DO NOTHING"
.into(),
select_idem: "SELECT run_id,fingerprint,claimed_at FROM faucet_serve_idem WHERE key=$1"
.into(),
takeover_idem: "UPDATE faucet_serve_idem \
SET run_id=$1,fingerprint=$2,claimed_at=$3 WHERE key=$4 AND claimed_at=$5"
.into(),
delete_idem_by_run: "DELETE FROM faucet_serve_idem WHERE run_id=$1".into(),
select_pending: "SELECT run_id, body FROM faucet_serve_runs \
WHERE status = 'pending' ORDER BY submitted_at ASC LIMIT $1"
.into(),
claim_one: "UPDATE faucet_serve_runs \
SET owner = $1, status = 'running', lease_expires_at = $2, body = $3 \
WHERE run_id = $4 AND status = 'pending'"
.into(),
reclaim_select: "SELECT body FROM faucet_serve_runs \
WHERE status = 'running' \
AND (lease_expires_at IS NULL OR lease_expires_at < $1)"
.into(),
reclaim_requeue: "UPDATE faucet_serve_runs \
SET status = 'pending', owner = NULL, lease_expires_at = NULL, \
body = $1 \
WHERE run_id = $2 AND status = 'running' \
AND (lease_expires_at IS NULL OR lease_expires_at < $3)"
.into(),
reclaim_fail: "UPDATE faucet_serve_runs \
SET status = 'failed', finished_at = $1, body = $2, owner = NULL \
WHERE run_id = $3 AND status = 'running' \
AND (lease_expires_at IS NULL OR lease_expires_at < $4)"
.into(),
finalize_owned: "UPDATE faucet_serve_runs \
SET status = $1, finished_at = $2, lease_expires_at = $3, body = $4 \
WHERE run_id = $5 AND owner = $6 \
AND status NOT IN ('completed','failed','cancelled')"
.into(),
cancel_pending: "UPDATE faucet_serve_runs \
SET status = 'cancelled', finished_at = $1, body = $2 \
WHERE run_id = $3 AND status = 'pending'"
.into(),
request_cancel: "UPDATE faucet_serve_runs \
SET cancel_requested = $1 WHERE run_id = $2 AND status IN ('running','sharded')"
.into(),
pending_cancellations: "SELECT run_id FROM faucet_serve_runs \
WHERE status = 'running' AND owner = $1 AND cancel_requested IS NOT NULL"
.into(),
heartbeat_instance: "INSERT INTO faucet_serve_instances \
(instance_id, started_at, last_heartbeat, listen, max_concurrent, in_flight) \
VALUES ($1,$2,$3,$4,$5,$6) \
ON CONFLICT (instance_id) DO UPDATE SET \
last_heartbeat = excluded.last_heartbeat, listen = excluded.listen, \
max_concurrent = excluded.max_concurrent, in_flight = excluded.in_flight"
.into(),
live_instances: "SELECT instance_id, started_at, last_heartbeat, listen, \
max_concurrent, in_flight FROM faucet_serve_instances \
WHERE last_heartbeat >= $1"
.into(),
prune_instances: "DELETE FROM faucet_serve_instances WHERE last_heartbeat < $1".into(),
insert_shard: "INSERT INTO faucet_serve_shards \
(run_id, shard_id, descriptor, size_estimate, status, attempt) \
VALUES ($1,$2,$3,$4,'pending','0') \
ON CONFLICT (run_id, shard_id) DO NOTHING"
.into(),
claim_shards_select: "SELECT s.run_id, s.shard_id, s.descriptor, r.body \
FROM faucet_serve_shards s JOIN faucet_serve_runs r ON r.run_id = s.run_id \
WHERE s.status = 'pending' \
ORDER BY CAST(COALESCE(s.size_estimate, '0') AS BIGINT) DESC, s.run_id, s.shard_id \
LIMIT $1"
.into(),
claim_shard_one: "UPDATE faucet_serve_shards \
SET owner = $1, status = 'running', lease_expires_at = $2 \
WHERE run_id = $3 AND shard_id = $4 AND status = 'pending'"
.into(),
renew_shard_leases: "UPDATE faucet_serve_shards SET lease_expires_at = $1 \
WHERE owner = $2 AND status = 'running'"
.into(),
reclaim_shards_select: "SELECT run_id, shard_id, attempt FROM faucet_serve_shards \
WHERE status = 'running' \
AND (lease_expires_at IS NULL OR lease_expires_at < $1)"
.into(),
reclaim_shard_requeue: "UPDATE faucet_serve_shards \
SET status = 'pending', owner = NULL, lease_expires_at = NULL, attempt = $1 \
WHERE run_id = $2 AND shard_id = $3 AND status = 'running' \
AND (lease_expires_at IS NULL OR lease_expires_at < $4)"
.into(),
reclaim_shard_fail: "UPDATE faucet_serve_shards \
SET status = 'failed', finished_at = $1, owner = NULL \
WHERE run_id = $2 AND shard_id = $3 AND status = 'running' \
AND (lease_expires_at IS NULL OR lease_expires_at < $4)"
.into(),
finalize_shard: "UPDATE faucet_serve_shards \
SET status = $1, finished_at = $2 \
WHERE run_id = $3 AND shard_id = $4 AND owner = $5 AND status = 'running'"
.into(),
shard_progress: "SELECT status, COUNT(*) AS n FROM faucet_serve_shards \
WHERE run_id = $1 GROUP BY status"
.into(),
pending_shard_cancellations: "SELECT DISTINCT s.run_id \
FROM faucet_serve_shards s \
JOIN faucet_serve_runs r ON r.run_id = s.run_id \
WHERE s.owner = $1 AND s.status = 'running' \
AND r.cancel_requested IS NOT NULL"
.into(),
select_sharded_parents: "SELECT run_id FROM faucet_serve_runs \
WHERE status = 'sharded'"
.into(),
finalize_sharded_parent: "UPDATE faucet_serve_runs \
SET status = $1, finished_at = $2, body = $3 \
WHERE run_id = $4 AND status = 'sharded'"
.into(),
delete_shards_by_run: "DELETE FROM faucet_serve_shards WHERE run_id = $1".into(),
purge_orphan_shards: "DELETE FROM faucet_serve_shards \
WHERE run_id NOT IN (SELECT run_id FROM faucet_serve_runs)"
.into(),
insert_audit: "INSERT INTO faucet_serve_audit \
(id, ts, principal, role, action, run_id, config_fingerprint, source_ip, result) \
VALUES ($1,$2,$3,$4,$5,$6,$7,$8,$9)"
.into(),
list_audit: "SELECT id, ts, principal, role, action, run_id, config_fingerprint, \
source_ip, result FROM faucet_serve_audit \
WHERE ($1::text IS NULL OR principal = $2::text) \
AND ($3::text IS NULL OR action = $4::text) \
AND ($5::text IS NULL OR ts >= $6::text) \
AND ($7::text IS NULL OR ts <= $8::text) \
ORDER BY ts DESC, id DESC LIMIT $9"
.into(),
purge_audit: "DELETE FROM faucet_serve_audit WHERE ts < $1".into(),
catalog_select_dataset: "SELECT body FROM faucet_catalog_datasets WHERE id=$1".into(),
catalog_upsert_dataset: "INSERT INTO faucet_catalog_datasets \
(id, uri, kind, last_seen, body) VALUES ($1,$2,$3,$4,$5) \
ON CONFLICT (id) DO UPDATE SET uri=excluded.uri, kind=excluded.kind, \
last_seen=excluded.last_seen, body=excluded.body"
.into(),
catalog_select_datasets: "SELECT body FROM faucet_catalog_datasets".into(),
catalog_insert_schema_version: "INSERT INTO faucet_catalog_schema_versions \
(dataset_id, version, recorded_at, body) VALUES ($1,$2,$3,$4) \
ON CONFLICT (dataset_id, version) DO NOTHING"
.into(),
catalog_select_schema_versions: "SELECT body FROM faucet_catalog_schema_versions \
WHERE dataset_id=$1 ORDER BY CAST(version AS BIGINT) ASC"
.into(),
catalog_upsert_edge: "INSERT INTO faucet_catalog_edges \
(src_id, dst_id, last_seen, body) VALUES ($1,$2,$3,$4) \
ON CONFLICT (src_id, dst_id) DO UPDATE SET \
last_seen=excluded.last_seen, body=excluded.body"
.into(),
catalog_select_edges: "SELECT body FROM faucet_catalog_edges \
ORDER BY last_seen DESC, src_id, dst_id"
.into(),
catalog_insert_stat: "INSERT INTO faucet_catalog_stats \
(dataset_id, recorded_at, run_id, records) VALUES ($1,$2,$3,$4) \
ON CONFLICT (dataset_id, recorded_at) DO NOTHING"
.into(),
catalog_select_stats: "SELECT recorded_at, run_id, records \
FROM faucet_catalog_stats WHERE dataset_id=$1 \
ORDER BY recorded_at DESC LIMIT $2"
.into(),
catalog_prune_stats: "DELETE FROM faucet_catalog_stats \
WHERE dataset_id=$1 AND recorded_at NOT IN (\
SELECT recorded_at FROM faucet_catalog_stats WHERE dataset_id=$2 \
ORDER BY recorded_at DESC LIMIT $3)"
.into(),
catalog_upsert_config_snapshot: "INSERT INTO faucet_config_snapshots \
(pipeline, recorded_at, faucet_version, body) VALUES ($1,$2,$3,$4) \
ON CONFLICT (pipeline) DO UPDATE SET recorded_at=excluded.recorded_at, \
faucet_version=excluded.faucet_version, body=excluded.body"
.into(),
catalog_select_config_snapshot:
"SELECT body FROM faucet_config_snapshots WHERE pipeline=$1".into(),
}
}
fn sqlite() -> Self {
Self {
upsert: "INSERT INTO faucet_serve_runs \
(run_id,name,status,submitted_at,finished_at,idempotency_key,owner,lease_expires_at,body) \
VALUES (?,?,?,?,?,?,?,?,?) \
ON CONFLICT (run_id) DO UPDATE SET \
name=excluded.name,status=excluded.status,submitted_at=excluded.submitted_at,\
finished_at=excluded.finished_at,idempotency_key=excluded.idempotency_key,\
owner=excluded.owner,lease_expires_at=excluded.lease_expires_at,\
body=excluded.body"
.into(),
select_body: "SELECT body FROM faucet_serve_runs WHERE run_id=?".into(),
select_status: "SELECT status FROM faucet_serve_runs WHERE run_id=?".into(),
select_submitted: "SELECT submitted_at FROM faucet_serve_runs WHERE run_id=?".into(),
delete: "DELETE FROM faucet_serve_runs WHERE run_id=?".into(),
list: "SELECT body FROM faucet_serve_runs \
WHERE (? IS NULL OR status = ?) \
AND (? IS NULL OR name = ?) \
AND (? IS NULL OR submitted_at >= ?) \
AND (? IS NULL OR submitted_at <= ?) \
AND (? IS NULL OR (submitted_at < ? \
OR (submitted_at = ? AND run_id < ?))) \
ORDER BY submitted_at DESC, run_id DESC LIMIT ?"
.into(),
purge_runs: "DELETE FROM faucet_serve_runs \
WHERE status IN ('completed','failed','cancelled') \
AND finished_at IS NOT NULL AND finished_at < ?"
.into(),
purge_idem: "DELETE FROM faucet_serve_idem WHERE claimed_at < ?".into(),
select_orphans: "SELECT body FROM faucet_serve_runs \
WHERE status IN ('queued','running') \
AND (lease_expires_at IS NULL OR lease_expires_at < ?)"
.into(),
renew_leases: "UPDATE faucet_serve_runs SET lease_expires_at = ? \
WHERE owner = ? AND status IN ('queued','running')"
.into(),
insert_idem: "INSERT INTO faucet_serve_idem (key,run_id,fingerprint,claimed_at) \
VALUES (?,?,?,?) ON CONFLICT (key) DO NOTHING"
.into(),
select_idem: "SELECT run_id,fingerprint,claimed_at FROM faucet_serve_idem WHERE key=?"
.into(),
takeover_idem: "UPDATE faucet_serve_idem \
SET run_id=?,fingerprint=?,claimed_at=? WHERE key=? AND claimed_at=?"
.into(),
delete_idem_by_run: "DELETE FROM faucet_serve_idem WHERE run_id=?".into(),
select_pending: "SELECT run_id, body FROM faucet_serve_runs \
WHERE status = 'pending' ORDER BY submitted_at ASC LIMIT ?"
.into(),
claim_one: "UPDATE faucet_serve_runs \
SET owner = ?, status = 'running', lease_expires_at = ?, body = ? \
WHERE run_id = ? AND status = 'pending'"
.into(),
reclaim_select: "SELECT body FROM faucet_serve_runs \
WHERE status = 'running' \
AND (lease_expires_at IS NULL OR lease_expires_at < ?)"
.into(),
reclaim_requeue: "UPDATE faucet_serve_runs \
SET status = 'pending', owner = NULL, lease_expires_at = NULL, \
body = ? \
WHERE run_id = ? AND status = 'running' \
AND (lease_expires_at IS NULL OR lease_expires_at < ?)"
.into(),
reclaim_fail: "UPDATE faucet_serve_runs \
SET status = 'failed', finished_at = ?, body = ?, owner = NULL \
WHERE run_id = ? AND status = 'running' \
AND (lease_expires_at IS NULL OR lease_expires_at < ?)"
.into(),
finalize_owned: "UPDATE faucet_serve_runs \
SET status = ?, finished_at = ?, lease_expires_at = ?, body = ? \
WHERE run_id = ? AND owner = ? \
AND status NOT IN ('completed','failed','cancelled')"
.into(),
cancel_pending: "UPDATE faucet_serve_runs \
SET status = 'cancelled', finished_at = ?, body = ? \
WHERE run_id = ? AND status = 'pending'"
.into(),
request_cancel: "UPDATE faucet_serve_runs \
SET cancel_requested = ? WHERE run_id = ? AND status IN ('running','sharded')"
.into(),
pending_cancellations: "SELECT run_id FROM faucet_serve_runs \
WHERE status = 'running' AND owner = ? AND cancel_requested IS NOT NULL"
.into(),
heartbeat_instance: "INSERT INTO faucet_serve_instances \
(instance_id, started_at, last_heartbeat, listen, max_concurrent, in_flight) \
VALUES (?,?,?,?,?,?) \
ON CONFLICT (instance_id) DO UPDATE SET \
last_heartbeat = excluded.last_heartbeat, listen = excluded.listen, \
max_concurrent = excluded.max_concurrent, in_flight = excluded.in_flight"
.into(),
live_instances: "SELECT instance_id, started_at, last_heartbeat, listen, \
max_concurrent, in_flight FROM faucet_serve_instances \
WHERE last_heartbeat >= ?"
.into(),
prune_instances: "DELETE FROM faucet_serve_instances WHERE last_heartbeat < ?".into(),
insert_shard: "INSERT INTO faucet_serve_shards \
(run_id, shard_id, descriptor, size_estimate, status, attempt) \
VALUES (?,?,?,?,'pending','0') \
ON CONFLICT (run_id, shard_id) DO NOTHING"
.into(),
claim_shards_select: "SELECT s.run_id, s.shard_id, s.descriptor, r.body \
FROM faucet_serve_shards s JOIN faucet_serve_runs r ON r.run_id = s.run_id \
WHERE s.status = 'pending' \
ORDER BY CAST(COALESCE(s.size_estimate, '0') AS INTEGER) DESC, s.run_id, s.shard_id \
LIMIT ?"
.into(),
claim_shard_one: "UPDATE faucet_serve_shards \
SET owner = ?, status = 'running', lease_expires_at = ? \
WHERE run_id = ? AND shard_id = ? AND status = 'pending'"
.into(),
renew_shard_leases: "UPDATE faucet_serve_shards SET lease_expires_at = ? \
WHERE owner = ? AND status = 'running'"
.into(),
reclaim_shards_select: "SELECT run_id, shard_id, attempt FROM faucet_serve_shards \
WHERE status = 'running' \
AND (lease_expires_at IS NULL OR lease_expires_at < ?)"
.into(),
reclaim_shard_requeue: "UPDATE faucet_serve_shards \
SET status = 'pending', owner = NULL, lease_expires_at = NULL, attempt = ? \
WHERE run_id = ? AND shard_id = ? AND status = 'running' \
AND (lease_expires_at IS NULL OR lease_expires_at < ?)"
.into(),
reclaim_shard_fail: "UPDATE faucet_serve_shards \
SET status = 'failed', finished_at = ?, owner = NULL \
WHERE run_id = ? AND shard_id = ? AND status = 'running' \
AND (lease_expires_at IS NULL OR lease_expires_at < ?)"
.into(),
finalize_shard: "UPDATE faucet_serve_shards \
SET status = ?, finished_at = ? \
WHERE run_id = ? AND shard_id = ? AND owner = ? AND status = 'running'"
.into(),
shard_progress: "SELECT status, COUNT(*) AS n FROM faucet_serve_shards \
WHERE run_id = ? GROUP BY status"
.into(),
pending_shard_cancellations: "SELECT DISTINCT s.run_id \
FROM faucet_serve_shards s \
JOIN faucet_serve_runs r ON r.run_id = s.run_id \
WHERE s.owner = ? AND s.status = 'running' \
AND r.cancel_requested IS NOT NULL"
.into(),
select_sharded_parents: "SELECT run_id FROM faucet_serve_runs \
WHERE status = 'sharded'"
.into(),
finalize_sharded_parent: "UPDATE faucet_serve_runs \
SET status = ?, finished_at = ?, body = ? \
WHERE run_id = ? AND status = 'sharded'"
.into(),
delete_shards_by_run: "DELETE FROM faucet_serve_shards WHERE run_id = ?".into(),
purge_orphan_shards: "DELETE FROM faucet_serve_shards \
WHERE run_id NOT IN (SELECT run_id FROM faucet_serve_runs)"
.into(),
insert_audit: "INSERT INTO faucet_serve_audit \
(id, ts, principal, role, action, run_id, config_fingerprint, source_ip, result) \
VALUES (?,?,?,?,?,?,?,?,?)"
.into(),
list_audit: "SELECT id, ts, principal, role, action, run_id, config_fingerprint, \
source_ip, result FROM faucet_serve_audit \
WHERE (? IS NULL OR principal = ?) \
AND (? IS NULL OR action = ?) \
AND (? IS NULL OR ts >= ?) \
AND (? IS NULL OR ts <= ?) \
ORDER BY ts DESC, id DESC LIMIT ?"
.into(),
purge_audit: "DELETE FROM faucet_serve_audit WHERE ts < ?".into(),
catalog_select_dataset: "SELECT body FROM faucet_catalog_datasets WHERE id=?".into(),
catalog_upsert_dataset: "INSERT INTO faucet_catalog_datasets \
(id, uri, kind, last_seen, body) VALUES (?,?,?,?,?) \
ON CONFLICT (id) DO UPDATE SET uri=excluded.uri, kind=excluded.kind, \
last_seen=excluded.last_seen, body=excluded.body"
.into(),
catalog_select_datasets: "SELECT body FROM faucet_catalog_datasets".into(),
catalog_insert_schema_version: "INSERT INTO faucet_catalog_schema_versions \
(dataset_id, version, recorded_at, body) VALUES (?,?,?,?) \
ON CONFLICT (dataset_id, version) DO NOTHING"
.into(),
catalog_select_schema_versions: "SELECT body FROM faucet_catalog_schema_versions \
WHERE dataset_id=? ORDER BY CAST(version AS INTEGER) ASC"
.into(),
catalog_upsert_edge: "INSERT INTO faucet_catalog_edges \
(src_id, dst_id, last_seen, body) VALUES (?,?,?,?) \
ON CONFLICT (src_id, dst_id) DO UPDATE SET \
last_seen=excluded.last_seen, body=excluded.body"
.into(),
catalog_select_edges: "SELECT body FROM faucet_catalog_edges \
ORDER BY last_seen DESC, src_id, dst_id"
.into(),
catalog_insert_stat: "INSERT INTO faucet_catalog_stats \
(dataset_id, recorded_at, run_id, records) VALUES (?,?,?,?) \
ON CONFLICT (dataset_id, recorded_at) DO NOTHING"
.into(),
catalog_select_stats: "SELECT recorded_at, run_id, records \
FROM faucet_catalog_stats WHERE dataset_id=? \
ORDER BY recorded_at DESC LIMIT ?"
.into(),
catalog_prune_stats: "DELETE FROM faucet_catalog_stats \
WHERE dataset_id=? AND recorded_at NOT IN (\
SELECT recorded_at FROM faucet_catalog_stats WHERE dataset_id=? \
ORDER BY recorded_at DESC LIMIT ?)"
.into(),
catalog_upsert_config_snapshot: "INSERT INTO faucet_config_snapshots \
(pipeline, recorded_at, faucet_version, body) VALUES (?,?,?,?) \
ON CONFLICT (pipeline) DO UPDATE SET recorded_at=excluded.recorded_at, \
faucet_version=excluded.faucet_version, body=excluded.body"
.into(),
catalog_select_config_snapshot:
"SELECT body FROM faucet_config_snapshots WHERE pipeline=?".into(),
}
}
}
pub const CLAIM_ATTEMPTS: usize = 4;
pub fn fmt_ts(dt: DateTime<Utc>) -> String {
dt.to_rfc3339_opts(chrono::SecondsFormat::Nanos, true)
}
pub fn is_expired(claimed_at: &str, now: DateTime<Utc>, window: Duration) -> bool {
match DateTime::parse_from_rfc3339(claimed_at) {
Ok(t) => now
.signed_duration_since(t.with_timezone(&Utc))
.to_std()
.map(|age| age >= window)
.unwrap_or(false),
Err(_) => false,
}
}
pub fn threshold(now: DateTime<Utc>, window: Duration) -> String {
let delta =
chrono::Duration::from_std(window).unwrap_or_else(|_| chrono::Duration::days(36_500));
fmt_ts(now - delta)
}
pub fn encode_body(rec: &RunRecord) -> Result<String, HistoryError> {
serde_json::to_string(rec).map_err(|e| HistoryError::Backend(format!("encode run record: {e}")))
}
pub fn decode_body(body: &str) -> Result<RunRecord, HistoryError> {
serde_json::from_str(body).map_err(|e| HistoryError::Backend(format!("decode run record: {e}")))
}
pub fn encode_json<T: serde::Serialize>(value: &T, what: &str) -> Result<String, HistoryError> {
serde_json::to_string(value).map_err(|e| HistoryError::Backend(format!("encode {what}: {e}")))
}
pub fn decode_json<T: serde::de::DeserializeOwned>(
body: &str,
what: &str,
) -> Result<T, HistoryError> {
serde_json::from_str(body).map_err(|e| HistoryError::Backend(format!("decode {what}: {e}")))
}
pub fn parse_status(s: &str) -> RunStatus {
match s {
"queued" => RunStatus::Queued,
"pending" => RunStatus::Pending,
"running" => RunStatus::Running,
"sharded" => RunStatus::Sharded,
"completed" => RunStatus::Completed,
"cancelled" => RunStatus::Cancelled,
_ => RunStatus::Failed,
}
}
macro_rules! impl_sql_history {
($name:ident, $pool:ty) => {
pub struct $name {
pool: $pool,
idem_retention: std::time::Duration,
instance_id: String,
lease_ttl: std::time::Duration,
stmts: $crate::serve::history::sql::Stmts,
}
impl $name {
pub fn from_parts(
pool: $pool,
idem_retention: std::time::Duration,
lease_ttl: std::time::Duration,
instance_id: String,
stmts: $crate::serve::history::sql::Stmts,
) -> Self {
Self {
pool,
idem_retention,
instance_id,
lease_ttl,
stmts,
}
}
pub fn pool(&self) -> &$pool {
&self.pool
}
}
#[async_trait::async_trait]
impl $crate::serve::history::RunHistory for $name {
async fn claim_idempotency(
&self,
key: &str,
fingerprint: &str,
run_id: &str,
window: std::time::Duration,
) -> Result<$crate::serve::history::Claim, $crate::serve::history::HistoryError> {
use sqlx::Row as _;
use $crate::serve::history::Claim;
use $crate::serve::history::HistoryError;
use $crate::serve::history::sql;
let now = chrono::Utc::now();
let now_s = sql::fmt_ts(now);
let backend = |e: sqlx::Error| HistoryError::Backend(e.to_string());
for _ in 0..sql::CLAIM_ATTEMPTS {
let inserted = sqlx::query(&self.stmts.insert_idem)
.bind(key)
.bind(run_id)
.bind(fingerprint)
.bind(&now_s)
.execute(&self.pool)
.await
.map_err(backend)?
.rows_affected();
if inserted == 1 {
return Ok(Claim::Fresh);
}
let Some(row) = sqlx::query(&self.stmts.select_idem)
.bind(key)
.fetch_optional(&self.pool)
.await
.map_err(backend)?
else {
continue;
};
let existing_run: String = row.try_get("run_id").map_err(backend)?;
let existing_fp: String = row.try_get("fingerprint").map_err(backend)?;
let claimed_at: String = row.try_get("claimed_at").map_err(backend)?;
if sql::is_expired(&claimed_at, now, window) {
let took = sqlx::query(&self.stmts.takeover_idem)
.bind(run_id)
.bind(fingerprint)
.bind(&now_s)
.bind(key)
.bind(&claimed_at)
.execute(&self.pool)
.await
.map_err(backend)?
.rows_affected();
if took == 1 {
return Ok(Claim::Fresh);
}
continue; }
return Ok(if existing_fp == fingerprint {
Claim::Replay(existing_run)
} else {
Claim::Conflict
});
}
tracing::warn!(
key,
"idempotency claim exhausted retries; reporting conflict"
);
Ok(Claim::Conflict)
}
async fn upsert(
&self,
rec: &$crate::serve::history::RunRecord,
) -> Result<(), $crate::serve::history::HistoryError> {
use $crate::serve::history::HistoryError;
use $crate::serve::history::sql;
let body = sql::encode_body(rec)?;
let submitted = sql::fmt_ts(rec.submitted_at);
let finished = rec.finished_at.map(sql::fmt_ts);
let lease = sql::fmt_ts(chrono::Utc::now() + self.lease_ttl);
sqlx::query(&self.stmts.upsert)
.bind(&rec.run_id)
.bind(rec.name.as_deref())
.bind(rec.status.as_str())
.bind(&submitted)
.bind(finished.as_deref())
.bind(rec.idempotency_key.as_deref())
.bind(&self.instance_id)
.bind(&lease)
.bind(&body)
.execute(&self.pool)
.await
.map_err(|e| HistoryError::Backend(e.to_string()))?;
Ok(())
}
async fn get(
&self,
id: &str,
) -> Result<
Option<$crate::serve::history::RunRecord>,
$crate::serve::history::HistoryError,
> {
use sqlx::Row as _;
use $crate::serve::history::HistoryError;
use $crate::serve::history::sql;
let row = sqlx::query(&self.stmts.select_body)
.bind(id)
.fetch_optional(&self.pool)
.await
.map_err(|e| HistoryError::Backend(e.to_string()))?;
match row {
None => Ok(None),
Some(r) => {
let body: String = r
.try_get("body")
.map_err(|e| HistoryError::Backend(e.to_string()))?;
Ok(Some(sql::decode_body(&body)?))
}
}
}
async fn list(
&self,
filter: &$crate::serve::history::ListFilter,
) -> Result<$crate::serve::history::ListPage, $crate::serve::history::HistoryError>
{
use sqlx::Row as _;
use $crate::serve::history::HistoryError;
use $crate::serve::history::ListPage;
use $crate::serve::history::sql;
let backend = |e: sqlx::Error| HistoryError::Backend(e.to_string());
let cursor_ts: Option<String> = match &filter.cursor {
None => None,
Some(c) => sqlx::query(&self.stmts.select_submitted)
.bind(c)
.fetch_optional(&self.pool)
.await
.map_err(backend)?
.map(|r| r.try_get::<String, _>("submitted_at"))
.transpose()
.map_err(backend)?,
};
let cur_id = if cursor_ts.is_some() {
filter.cursor.as_deref()
} else {
None
};
let status_s = filter.status.map(|s| s.as_str());
let name_s = filter.name.as_deref();
let since_s = filter.since.map(sql::fmt_ts);
let until_s = filter.until.map(sql::fmt_ts);
let limit = filter.limit.max(1);
let fetch_n = limit as i64 + 1;
let rows = sqlx::query(&self.stmts.list)
.bind(status_s)
.bind(status_s)
.bind(name_s)
.bind(name_s)
.bind(since_s.as_deref())
.bind(since_s.as_deref())
.bind(until_s.as_deref())
.bind(until_s.as_deref())
.bind(cursor_ts.as_deref())
.bind(cursor_ts.as_deref())
.bind(cursor_ts.as_deref())
.bind(cur_id)
.bind(fetch_n)
.fetch_all(&self.pool)
.await
.map_err(backend)?;
let mut runs = Vec::with_capacity(rows.len());
for r in &rows {
let body: String = r.try_get("body").map_err(backend)?;
runs.push(sql::decode_body(&body)?);
}
let next_cursor = if runs.len() > limit {
Some(runs[limit - 1].run_id.clone())
} else {
None
};
runs.truncate(limit);
Ok(ListPage { runs, next_cursor })
}
async fn delete(
&self,
id: &str,
) -> Result<$crate::serve::history::DeleteOutcome, $crate::serve::history::HistoryError>
{
use sqlx::Row as _;
use $crate::serve::history::DeleteOutcome;
use $crate::serve::history::HistoryError;
use $crate::serve::history::sql;
let backend = |e: sqlx::Error| HistoryError::Backend(e.to_string());
let status: Option<String> = sqlx::query(&self.stmts.select_status)
.bind(id)
.fetch_optional(&self.pool)
.await
.map_err(backend)?
.map(|r| r.try_get::<String, _>("status"))
.transpose()
.map_err(backend)?;
match status {
None => Ok(DeleteOutcome::NotFound),
Some(s) if !sql::parse_status(&s).is_terminal() => {
Ok(DeleteOutcome::StillRunning)
}
Some(_) => {
sqlx::query(&self.stmts.delete)
.bind(id)
.execute(&self.pool)
.await
.map_err(backend)?;
sqlx::query(&self.stmts.delete_idem_by_run)
.bind(id)
.execute(&self.pool)
.await
.map_err(backend)?;
sqlx::query(&self.stmts.delete_shards_by_run)
.bind(id)
.execute(&self.pool)
.await
.map_err(backend)?;
Ok(DeleteOutcome::Deleted)
}
}
}
async fn release_idempotency(
&self,
run_id: &str,
) -> Result<(), $crate::serve::history::HistoryError> {
use $crate::serve::history::HistoryError;
sqlx::query(&self.stmts.delete_idem_by_run)
.bind(run_id)
.execute(&self.pool)
.await
.map_err(|e| HistoryError::Backend(e.to_string()))?;
Ok(())
}
async fn purge_expired(
&self,
retain_for: std::time::Duration,
) -> Result<usize, $crate::serve::history::HistoryError> {
use $crate::serve::history::HistoryError;
use $crate::serve::history::sql;
let backend = |e: sqlx::Error| HistoryError::Backend(e.to_string());
let now = chrono::Utc::now();
let removed = sqlx::query(&self.stmts.purge_runs)
.bind(sql::threshold(now, retain_for))
.execute(&self.pool)
.await
.map_err(backend)?
.rows_affected() as usize;
let _ = sqlx::query(&self.stmts.purge_idem)
.bind(sql::threshold(now, self.idem_retention))
.execute(&self.pool)
.await;
let _ = sqlx::query(&self.stmts.prune_instances)
.bind(sql::threshold(now, retain_for))
.execute(&self.pool)
.await;
let _ = sqlx::query(&self.stmts.purge_orphan_shards)
.execute(&self.pool)
.await;
let _ = sqlx::query(&self.stmts.purge_audit)
.bind(sql::threshold(now, retain_for))
.execute(&self.pool)
.await;
Ok(removed)
}
async fn recover_orphans(&self) -> Result<usize, $crate::serve::history::HistoryError> {
use sqlx::Row as _;
use $crate::serve::history::HistoryError;
use $crate::serve::history::RunStatus;
use $crate::serve::history::sql;
let backend = |e: sqlx::Error| HistoryError::Backend(e.to_string());
let now = chrono::Utc::now();
let rows = sqlx::query(&self.stmts.select_orphans)
.bind(sql::fmt_ts(now))
.fetch_all(&self.pool)
.await
.map_err(backend)?;
let mut count = 0usize;
for r in &rows {
let body: String = r.try_get("body").map_err(backend)?;
let mut rec = sql::decode_body(&body)?;
rec.status = RunStatus::Failed;
rec.finished_at = Some(now);
rec.error = Some(
"owning serve instance's lease expired before the run finished".into(),
);
if rec.elapsed_secs.is_none()
&& let Some(started) = rec.started_at
{
rec.elapsed_secs = (now - started).to_std().ok().map(|d| d.as_secs_f64());
}
self.upsert(&rec).await?;
count += 1;
}
Ok(count)
}
async fn renew_leases(&self) -> Result<usize, $crate::serve::history::HistoryError> {
use $crate::serve::history::HistoryError;
use $crate::serve::history::sql;
let backend = |e: sqlx::Error| HistoryError::Backend(e.to_string());
let new_lease = sql::fmt_ts(chrono::Utc::now() + self.lease_ttl);
let renewed = sqlx::query(&self.stmts.renew_leases)
.bind(&new_lease)
.bind(&self.instance_id)
.execute(&self.pool)
.await
.map_err(backend)?
.rows_affected() as usize;
Ok(renewed)
}
async fn claim_pending(
&self,
limit: usize,
) -> Result<Vec<$crate::serve::history::RunRecord>, $crate::serve::history::HistoryError>
{
use sqlx::Row as _;
use $crate::serve::history::HistoryError;
use $crate::serve::history::RunStatus;
use $crate::serve::history::sql;
let backend = |e: sqlx::Error| HistoryError::Backend(e.to_string());
if limit == 0 {
return Ok(Vec::new());
}
let now = chrono::Utc::now();
let lease = sql::fmt_ts(now + self.lease_ttl);
let rows = sqlx::query(&self.stmts.select_pending)
.bind(limit as i64)
.fetch_all(&self.pool)
.await
.map_err(backend)?;
let mut claimed = Vec::new();
for row in &rows {
let run_id: String = row.try_get("run_id").map_err(backend)?;
let body: String = row.try_get("body").map_err(backend)?;
let mut r = sql::decode_body(&body)?;
r.status = RunStatus::Running;
let new_body = sql::encode_body(&r)?;
let won = sqlx::query(&self.stmts.claim_one)
.bind(&self.instance_id)
.bind(&lease)
.bind(&new_body)
.bind(&run_id)
.execute(&self.pool)
.await
.map_err(backend)?
.rows_affected();
if won == 1 {
claimed.push(r);
}
}
Ok(claimed)
}
async fn reclaim_orphans(
&self,
max_attempts: u32,
) -> Result<$crate::serve::history::ReclaimReport, $crate::serve::history::HistoryError>
{
use sqlx::Row as _;
use $crate::serve::history::HistoryError;
use $crate::serve::history::ReclaimReport;
use $crate::serve::history::RunStatus;
use $crate::serve::history::sql;
let backend = |e: sqlx::Error| HistoryError::Backend(e.to_string());
let now = chrono::Utc::now();
let now_s = sql::fmt_ts(now);
let rows = sqlx::query(&self.stmts.reclaim_select)
.bind(&now_s)
.fetch_all(&self.pool)
.await
.map_err(backend)?;
let mut report = ReclaimReport::default();
for row in &rows {
let body: String = row.try_get("body").map_err(backend)?;
let mut rec = sql::decode_body(&body)?;
let next_attempt = rec.attempt + 1;
if rec.attempt < max_attempts {
rec.attempt = next_attempt;
rec.status = RunStatus::Pending;
let new_body = sql::encode_body(&rec)?;
let n = sqlx::query(&self.stmts.reclaim_requeue)
.bind(&new_body)
.bind(&rec.run_id)
.bind(&now_s)
.execute(&self.pool)
.await
.map_err(backend)?
.rows_affected();
if n == 1 {
report.requeued += 1;
}
} else {
rec.attempt = next_attempt;
rec.status = RunStatus::Failed;
rec.finished_at = Some(now);
rec.error = Some(format!(
"run reclaimed {next_attempt} times after its owning instance's \
lease expired; giving up (poison run)"
));
if rec.elapsed_secs.is_none()
&& let Some(started) = rec.started_at
{
rec.elapsed_secs =
(now - started).to_std().ok().map(|d| d.as_secs_f64());
}
let new_body = sql::encode_body(&rec)?;
let n = sqlx::query(&self.stmts.reclaim_fail)
.bind(&now_s)
.bind(&new_body)
.bind(&rec.run_id)
.bind(&now_s)
.execute(&self.pool)
.await
.map_err(backend)?
.rows_affected();
if n == 1 {
report.failed += 1;
}
}
}
Ok(report)
}
async fn finalize_owned(
&self,
rec: &$crate::serve::history::RunRecord,
) -> Result<bool, $crate::serve::history::HistoryError> {
use $crate::serve::history::HistoryError;
use $crate::serve::history::sql;
let backend = |e: sqlx::Error| HistoryError::Backend(e.to_string());
let mut rec = rec.clone();
if rec.status.is_terminal() && rec.finished_at.is_none() {
rec.finished_at = Some(chrono::Utc::now());
}
let body = sql::encode_body(&rec)?;
let finished = rec.finished_at.map(sql::fmt_ts);
let lease = sql::fmt_ts(chrono::Utc::now() + self.lease_ttl);
let n = sqlx::query(&self.stmts.finalize_owned)
.bind(rec.status.as_str())
.bind(finished.as_deref())
.bind(&lease)
.bind(&body)
.bind(&rec.run_id)
.bind(&self.instance_id)
.execute(&self.pool)
.await
.map_err(backend)?
.rows_affected();
Ok(n == 1)
}
async fn finalize_sharded_parent(
&self,
run_id: &str,
status: $crate::serve::history::RunStatus,
finished_at: chrono::DateTime<chrono::Utc>,
error: Option<String>,
) -> Result<bool, $crate::serve::history::HistoryError> {
use sqlx::Row as _;
use $crate::serve::history::HistoryError;
use $crate::serve::history::RunStatus;
use $crate::serve::history::sql;
let backend = |e: sqlx::Error| HistoryError::Backend(e.to_string());
let Some(row) = sqlx::query(&self.stmts.select_body)
.bind(run_id)
.fetch_optional(&self.pool)
.await
.map_err(backend)?
else {
return Ok(false);
};
let body: String = row.try_get("body").map_err(backend)?;
let mut rec = sql::decode_body(&body)?;
if rec.status != RunStatus::Sharded {
return Ok(false);
}
rec.status = status;
rec.finished_at = Some(finished_at);
rec.error = error;
let new_body = sql::encode_body(&rec)?;
let n = sqlx::query(&self.stmts.finalize_sharded_parent)
.bind(status.as_str())
.bind(sql::fmt_ts(finished_at))
.bind(&new_body)
.bind(run_id)
.execute(&self.pool)
.await
.map_err(backend)?
.rows_affected();
Ok(n == 1)
}
async fn cancel_pending(
&self,
run_id: &str,
) -> Result<bool, $crate::serve::history::HistoryError> {
use sqlx::Row as _;
use $crate::serve::history::HistoryError;
use $crate::serve::history::RunStatus;
use $crate::serve::history::sql;
let backend = |e: sqlx::Error| HistoryError::Backend(e.to_string());
let Some(row) = sqlx::query(&self.stmts.select_body)
.bind(run_id)
.fetch_optional(&self.pool)
.await
.map_err(backend)?
else {
return Ok(false);
};
let body: String = row.try_get("body").map_err(backend)?;
let mut rec = sql::decode_body(&body)?;
if rec.status != RunStatus::Pending {
return Ok(false);
}
let now = chrono::Utc::now();
rec.status = RunStatus::Cancelled;
rec.finished_at = Some(now);
let new_body = sql::encode_body(&rec)?;
let n = sqlx::query(&self.stmts.cancel_pending)
.bind(sql::fmt_ts(now))
.bind(&new_body)
.bind(run_id)
.execute(&self.pool)
.await
.map_err(backend)?
.rows_affected();
Ok(n == 1)
}
async fn request_cancel(
&self,
run_id: &str,
) -> Result<(), $crate::serve::history::HistoryError> {
use $crate::serve::history::HistoryError;
use $crate::serve::history::sql;
let backend = |e: sqlx::Error| HistoryError::Backend(e.to_string());
sqlx::query(&self.stmts.request_cancel)
.bind(sql::fmt_ts(chrono::Utc::now()))
.bind(run_id)
.execute(&self.pool)
.await
.map_err(backend)?;
Ok(())
}
async fn pending_cancellations(
&self,
) -> Result<Vec<String>, $crate::serve::history::HistoryError> {
use sqlx::Row as _;
use $crate::serve::history::HistoryError;
let backend = |e: sqlx::Error| HistoryError::Backend(e.to_string());
let rows = sqlx::query(&self.stmts.pending_cancellations)
.bind(&self.instance_id)
.fetch_all(&self.pool)
.await
.map_err(backend)?;
let mut ids = Vec::with_capacity(rows.len());
for r in &rows {
ids.push(r.try_get::<String, _>("run_id").map_err(backend)?);
}
Ok(ids)
}
async fn heartbeat_instance(
&self,
beat: &$crate::serve::history::InstanceHeartbeat,
) -> Result<(), $crate::serve::history::HistoryError> {
use $crate::serve::history::HistoryError;
use $crate::serve::history::sql;
let backend = |e: sqlx::Error| HistoryError::Backend(e.to_string());
let now = sql::fmt_ts(chrono::Utc::now());
sqlx::query(&self.stmts.heartbeat_instance)
.bind(&self.instance_id)
.bind(sql::fmt_ts(beat.started_at))
.bind(&now)
.bind(beat.listen.as_deref())
.bind(beat.max_concurrent.to_string())
.bind(beat.in_flight.to_string())
.execute(&self.pool)
.await
.map_err(backend)?;
Ok(())
}
async fn live_instances(
&self,
ttl: std::time::Duration,
) -> Result<Vec<$crate::serve::history::InstanceRecord>, $crate::serve::history::HistoryError>
{
use sqlx::Row as _;
use $crate::serve::history::HistoryError;
use $crate::serve::history::InstanceRecord;
use $crate::serve::history::sql;
let backend = |e: sqlx::Error| HistoryError::Backend(e.to_string());
let now = chrono::Utc::now();
let rows = sqlx::query(&self.stmts.live_instances)
.bind(sql::threshold(now, ttl))
.fetch_all(&self.pool)
.await
.map_err(backend)?;
let parse_dt = |s: &str| {
chrono::DateTime::parse_from_rfc3339(s)
.map(|d| d.to_utc())
.unwrap_or(now)
};
let mut out = Vec::with_capacity(rows.len());
for r in &rows {
let started: String = r.try_get("started_at").map_err(backend)?;
let hb: String = r.try_get("last_heartbeat").map_err(backend)?;
let mc: Option<String> = r.try_get("max_concurrent").map_err(backend)?;
let inf: Option<String> = r.try_get("in_flight").map_err(backend)?;
out.push(InstanceRecord {
instance_id: r.try_get("instance_id").map_err(backend)?,
started_at: parse_dt(&started),
last_heartbeat: parse_dt(&hb),
listen: r.try_get("listen").map_err(backend)?,
max_concurrent: mc.and_then(|s| s.parse().ok()).unwrap_or(0),
in_flight: inf.and_then(|s| s.parse().ok()).unwrap_or(0),
});
}
Ok(out)
}
async fn insert_shards(
&self,
run_id: &str,
shards: &[$crate::serve::history::ShardInsert],
) -> Result<usize, $crate::serve::history::HistoryError> {
use $crate::serve::history::HistoryError;
let backend = |e: sqlx::Error| HistoryError::Backend(e.to_string());
let mut inserted = 0usize;
for s in shards {
let descriptor = serde_json::to_string(&s.descriptor).map_err(|e| {
HistoryError::Backend(format!("encode shard descriptor: {e}"))
})?;
let size = s.size_estimate.map(|n| n.to_string());
let n = sqlx::query(&self.stmts.insert_shard)
.bind(run_id)
.bind(&s.shard_id)
.bind(&descriptor)
.bind(size.as_deref())
.execute(&self.pool)
.await
.map_err(backend)?
.rows_affected();
inserted += n as usize;
}
Ok(inserted)
}
async fn claim_shards(
&self,
limit: usize,
) -> Result<
Vec<$crate::serve::history::ClaimedShard>,
$crate::serve::history::HistoryError,
> {
use sqlx::Row as _;
use $crate::serve::history::ClaimedShard;
use $crate::serve::history::HistoryError;
use $crate::serve::history::sql;
let backend = |e: sqlx::Error| HistoryError::Backend(e.to_string());
if limit == 0 {
return Ok(Vec::new());
}
let lease = sql::fmt_ts(chrono::Utc::now() + self.lease_ttl);
let rows = sqlx::query(&self.stmts.claim_shards_select)
.bind(limit as i64)
.fetch_all(&self.pool)
.await
.map_err(backend)?;
let mut claimed = Vec::new();
for row in &rows {
let run_id: String = row.try_get("run_id").map_err(backend)?;
let shard_id: String = row.try_get("shard_id").map_err(backend)?;
let descriptor_s: String = row.try_get("descriptor").map_err(backend)?;
let body: String = row.try_get("body").map_err(backend)?;
let won = sqlx::query(&self.stmts.claim_shard_one)
.bind(&self.instance_id)
.bind(&lease)
.bind(&run_id)
.bind(&shard_id)
.execute(&self.pool)
.await
.map_err(backend)?
.rows_affected();
if won == 1 {
let descriptor: serde_json::Value = serde_json::from_str(&descriptor_s)
.map_err(|e| {
HistoryError::Backend(format!("decode shard descriptor: {e}"))
})?;
let run = sql::decode_body(&body)?;
claimed.push(ClaimedShard {
run_id,
shard_id,
descriptor,
run,
});
}
}
Ok(claimed)
}
async fn renew_shard_leases(
&self,
) -> Result<usize, $crate::serve::history::HistoryError> {
use $crate::serve::history::HistoryError;
use $crate::serve::history::sql;
let backend = |e: sqlx::Error| HistoryError::Backend(e.to_string());
let lease = sql::fmt_ts(chrono::Utc::now() + self.lease_ttl);
let n = sqlx::query(&self.stmts.renew_shard_leases)
.bind(&lease)
.bind(&self.instance_id)
.execute(&self.pool)
.await
.map_err(backend)?
.rows_affected() as usize;
Ok(n)
}
async fn reclaim_shards(
&self,
max_attempts: u32,
) -> Result<$crate::serve::history::ReclaimReport, $crate::serve::history::HistoryError>
{
use sqlx::Row as _;
use $crate::serve::history::HistoryError;
use $crate::serve::history::ReclaimReport;
use $crate::serve::history::sql;
let backend = |e: sqlx::Error| HistoryError::Backend(e.to_string());
let now_s = sql::fmt_ts(chrono::Utc::now());
let rows = sqlx::query(&self.stmts.reclaim_shards_select)
.bind(&now_s)
.fetch_all(&self.pool)
.await
.map_err(backend)?;
let mut report = ReclaimReport::default();
for row in &rows {
let run_id: String = row.try_get("run_id").map_err(backend)?;
let shard_id: String = row.try_get("shard_id").map_err(backend)?;
let attempt_s: String = row.try_get("attempt").map_err(backend)?;
let attempt: u32 = attempt_s.parse().unwrap_or(0);
if attempt < max_attempts {
let next = (attempt + 1).to_string();
let n = sqlx::query(&self.stmts.reclaim_shard_requeue)
.bind(&next)
.bind(&run_id)
.bind(&shard_id)
.bind(&now_s)
.execute(&self.pool)
.await
.map_err(backend)?
.rows_affected();
if n == 1 {
report.requeued += 1;
}
} else {
let n = sqlx::query(&self.stmts.reclaim_shard_fail)
.bind(&now_s)
.bind(&run_id)
.bind(&shard_id)
.bind(&now_s)
.execute(&self.pool)
.await
.map_err(backend)?
.rows_affected();
if n == 1 {
report.failed += 1;
}
}
}
Ok(report)
}
async fn finalize_shard(
&self,
run_id: &str,
shard_id: &str,
success: bool,
) -> Result<bool, $crate::serve::history::HistoryError> {
use $crate::serve::history::HistoryError;
use $crate::serve::history::sql;
let backend = |e: sqlx::Error| HistoryError::Backend(e.to_string());
let status = if success { "completed" } else { "failed" };
let now_s = sql::fmt_ts(chrono::Utc::now());
let n = sqlx::query(&self.stmts.finalize_shard)
.bind(status)
.bind(&now_s)
.bind(run_id)
.bind(shard_id)
.bind(&self.instance_id)
.execute(&self.pool)
.await
.map_err(backend)?
.rows_affected();
Ok(n == 1)
}
async fn shard_progress(
&self,
run_id: &str,
) -> Result<$crate::serve::history::ShardProgress, $crate::serve::history::HistoryError>
{
use sqlx::Row as _;
use $crate::serve::history::HistoryError;
use $crate::serve::history::ShardProgress;
let backend = |e: sqlx::Error| HistoryError::Backend(e.to_string());
let rows = sqlx::query(&self.stmts.shard_progress)
.bind(run_id)
.fetch_all(&self.pool)
.await
.map_err(backend)?;
let mut p = ShardProgress::default();
for row in &rows {
let status: String = row.try_get("status").map_err(backend)?;
let n: i64 = row.try_get("n").map_err(backend)?;
let n = n.max(0) as usize;
p.total += n;
match status.as_str() {
"completed" => p.completed += n,
"failed" => p.failed += n,
"running" => p.running += n,
_ => p.pending += n,
}
}
Ok(p)
}
async fn pending_shard_cancellations(
&self,
) -> Result<Vec<String>, $crate::serve::history::HistoryError> {
use sqlx::Row as _;
use $crate::serve::history::HistoryError;
let backend = |e: sqlx::Error| HistoryError::Backend(e.to_string());
let rows = sqlx::query(&self.stmts.pending_shard_cancellations)
.bind(&self.instance_id)
.fetch_all(&self.pool)
.await
.map_err(backend)?;
let mut ids = Vec::with_capacity(rows.len());
for r in &rows {
ids.push(r.try_get::<String, _>("run_id").map_err(backend)?);
}
Ok(ids)
}
async fn finalize_completed_sharded_parents(
&self,
) -> Result<usize, $crate::serve::history::HistoryError> {
use sqlx::Row as _;
use $crate::serve::history::HistoryError;
use $crate::serve::history::RunStatus;
use $crate::serve::history::sql;
let backend = |e: sqlx::Error| HistoryError::Backend(e.to_string());
let rows = sqlx::query(&self.stmts.select_sharded_parents)
.fetch_all(&self.pool)
.await
.map_err(backend)?;
let mut finalized = 0usize;
for row in &rows {
let run_id: String = row.try_get("run_id").map_err(backend)?;
let progress = self.shard_progress(&run_id).await?;
if !progress.all_terminal() {
continue;
}
let success = progress.failed == 0;
let Some(body_row) = sqlx::query(&self.stmts.select_body)
.bind(&run_id)
.fetch_optional(&self.pool)
.await
.map_err(backend)?
else {
continue;
};
let body: String = body_row.try_get("body").map_err(backend)?;
let mut rec = sql::decode_body(&body)?;
if rec.status != RunStatus::Sharded {
continue;
}
let now = chrono::Utc::now();
rec.status = if success {
RunStatus::Completed
} else {
RunStatus::Failed
};
rec.finished_at = Some(now);
if !success {
rec.error = Some(format!(
"{}/{} shard(s) failed",
progress.failed, progress.total
));
}
let new_body = sql::encode_body(&rec)?;
let n = sqlx::query(&self.stmts.finalize_sharded_parent)
.bind(rec.status.as_str())
.bind(sql::fmt_ts(now))
.bind(&new_body)
.bind(&run_id)
.execute(&self.pool)
.await
.map_err(backend)?
.rows_affected();
if n == 1 {
finalized += 1;
$crate::serve::metrics::record_run_finished(
rec.status,
if success { "ok" } else { "error" },
);
tracing::info!(
run_id,
shards = progress.total,
failed = progress.failed,
"sharded run finalized by sweep (F11)"
);
}
}
Ok(finalized)
}
async fn record_audit(
&self,
entry: &$crate::serve::history::AuditEntry,
) -> Result<(), $crate::serve::history::HistoryError> {
use $crate::serve::history::HistoryError;
use $crate::serve::history::sql;
let backend = |e: sqlx::Error| HistoryError::Backend(e.to_string());
sqlx::query(&self.stmts.insert_audit)
.bind(&entry.id)
.bind(sql::fmt_ts(entry.timestamp))
.bind(&entry.principal)
.bind(&entry.role)
.bind(&entry.action)
.bind(entry.run_id.as_deref())
.bind(entry.config_fingerprint.as_deref())
.bind(entry.source_ip.as_deref())
.bind(&entry.result)
.execute(&self.pool)
.await
.map_err(backend)?;
Ok(())
}
async fn list_audit(
&self,
filter: &$crate::serve::history::AuditFilter,
) -> Result<
Vec<$crate::serve::history::AuditEntry>,
$crate::serve::history::HistoryError,
> {
use sqlx::Row as _;
use $crate::serve::history::AuditEntry;
use $crate::serve::history::HistoryError;
use $crate::serve::history::sql;
let backend = |e: sqlx::Error| HistoryError::Backend(e.to_string());
let principal = filter.principal.as_deref();
let action = filter.action.as_deref();
let since = filter.since.map(sql::fmt_ts);
let until = filter.until.map(sql::fmt_ts);
let limit = filter.limit.max(1) as i64;
let rows = sqlx::query(&self.stmts.list_audit)
.bind(principal)
.bind(principal)
.bind(action)
.bind(action)
.bind(since.as_deref())
.bind(since.as_deref())
.bind(until.as_deref())
.bind(until.as_deref())
.bind(limit)
.fetch_all(&self.pool)
.await
.map_err(backend)?;
let mut out = Vec::with_capacity(rows.len());
for r in &rows {
let ts: String = r.try_get("ts").map_err(backend)?;
let timestamp = chrono::DateTime::parse_from_rfc3339(&ts)
.map(|d| d.to_utc())
.unwrap_or_else(|_| chrono::Utc::now());
out.push(AuditEntry {
id: r.try_get("id").map_err(backend)?,
timestamp,
principal: r.try_get("principal").map_err(backend)?,
role: r.try_get("role").map_err(backend)?,
action: r.try_get("action").map_err(backend)?,
run_id: r.try_get("run_id").map_err(backend)?,
config_fingerprint: r.try_get("config_fingerprint").map_err(backend)?,
source_ip: r.try_get("source_ip").map_err(backend)?,
result: r.try_get("result").map_err(backend)?,
});
}
Ok(out)
}
async fn catalog_record(
&self,
update: &$crate::serve::history::catalog::CatalogUpdate,
) -> Result<(), $crate::serve::history::HistoryError> {
use sqlx::Row as _;
use $crate::serve::history::HistoryError;
use $crate::serve::history::catalog;
use $crate::serve::history::sql;
let backend = |e: sqlx::Error| HistoryError::Backend(e.to_string());
let now_s = sql::fmt_ts(update.recorded_at);
for obs in [&update.source, &update.sink] {
let id = catalog::dataset_id(&obs.uri);
let existing = sqlx::query(&self.stmts.catalog_select_dataset)
.bind(&id)
.fetch_optional(&self.pool)
.await
.map_err(backend)?
.map(|r| r.try_get::<String, _>("body"))
.transpose()
.map_err(backend)?
.map(|b| {
sql::decode_json::<catalog::CatalogDataset>(&b, "catalog dataset")
})
.transpose()?;
let (ds, new_version) = catalog::apply_observation(
existing.as_ref(),
obs,
&update.run_id,
&update.pipeline,
&update.row,
update.recorded_at,
);
sqlx::query(&self.stmts.catalog_upsert_dataset)
.bind(&ds.id)
.bind(&ds.uri)
.bind(&ds.kind)
.bind(&now_s)
.bind(sql::encode_json(&ds, "catalog dataset")?)
.execute(&self.pool)
.await
.map_err(backend)?;
if let Some(v) = new_version {
sqlx::query(&self.stmts.catalog_insert_schema_version)
.bind(&v.dataset_id)
.bind(v.version.to_string())
.bind(sql::fmt_ts(v.recorded_at))
.bind(sql::encode_json(&v, "catalog schema version")?)
.execute(&self.pool)
.await
.map_err(backend)?;
}
sqlx::query(&self.stmts.catalog_insert_stat)
.bind(&id)
.bind(&now_s)
.bind(&update.run_id)
.bind(obs.records.to_string())
.execute(&self.pool)
.await
.map_err(backend)?;
sqlx::query(&self.stmts.catalog_prune_stats)
.bind(&id)
.bind(&id)
.bind(catalog::STATS_RETAIN as i64)
.execute(&self.pool)
.await
.map_err(backend)?;
}
let src_id = catalog::dataset_id(&update.source.uri);
let dst_id = catalog::dataset_id(&update.sink.uri);
let existing_edges = self.catalog_all_edges().await?;
let existing = existing_edges
.iter()
.find(|e| e.src_id == src_id && e.dst_id == dst_id);
let edge = catalog::apply_edge(existing, update);
sqlx::query(&self.stmts.catalog_upsert_edge)
.bind(&edge.src_id)
.bind(&edge.dst_id)
.bind(&now_s)
.bind(sql::encode_json(&edge, "catalog edge")?)
.execute(&self.pool)
.await
.map_err(backend)?;
Ok(())
}
async fn catalog_list_datasets(
&self,
filter: &$crate::serve::history::catalog::CatalogListFilter,
) -> Result<
$crate::serve::history::catalog::CatalogDatasetPage,
$crate::serve::history::HistoryError,
> {
use sqlx::Row as _;
use $crate::serve::history::HistoryError;
use $crate::serve::history::catalog;
use $crate::serve::history::sql;
let backend = |e: sqlx::Error| HistoryError::Backend(e.to_string());
let rows = sqlx::query(&self.stmts.catalog_select_datasets)
.fetch_all(&self.pool)
.await
.map_err(backend)?;
let mut all = Vec::with_capacity(rows.len());
for r in &rows {
let body: String = r.try_get("body").map_err(backend)?;
all.push(sql::decode_json(&body, "catalog dataset")?);
}
Ok(catalog::filter_datasets(all, filter))
}
async fn catalog_get_dataset(
&self,
id: &str,
) -> Result<
Option<$crate::serve::history::catalog::CatalogDatasetDetail>,
$crate::serve::history::HistoryError,
> {
use sqlx::Row as _;
use $crate::serve::history::HistoryError;
use $crate::serve::history::catalog;
use $crate::serve::history::sql;
let backend = |e: sqlx::Error| HistoryError::Backend(e.to_string());
let Some(row) = sqlx::query(&self.stmts.catalog_select_dataset)
.bind(id)
.fetch_optional(&self.pool)
.await
.map_err(backend)?
else {
return Ok(None);
};
let body: String = row.try_get("body").map_err(backend)?;
let dataset: catalog::CatalogDataset =
sql::decode_json(&body, "catalog dataset")?;
let rows = sqlx::query(&self.stmts.catalog_select_schema_versions)
.bind(id)
.fetch_all(&self.pool)
.await
.map_err(backend)?;
let mut schema_timeline = Vec::with_capacity(rows.len());
for r in &rows {
let body: String = r.try_get("body").map_err(backend)?;
schema_timeline.push(sql::decode_json(&body, "catalog schema version")?);
}
let rows = sqlx::query(&self.stmts.catalog_select_stats)
.bind(id)
.bind(catalog::STATS_DETAIL_LIMIT as i64)
.fetch_all(&self.pool)
.await
.map_err(backend)?;
let mut stats = Vec::with_capacity(rows.len());
for r in &rows {
let recorded: String = r.try_get("recorded_at").map_err(backend)?;
let run_id: String = r.try_get("run_id").map_err(backend)?;
let records: String = r.try_get("records").map_err(backend)?;
stats.push(catalog::CatalogStatsPoint {
recorded_at: chrono::DateTime::parse_from_rfc3339(&recorded)
.map(|d| d.to_utc())
.unwrap_or_else(|_| chrono::Utc::now()),
run_id,
records: records.parse().unwrap_or(0),
});
}
let edges = self.catalog_all_edges().await?;
let (downstream, rest): (Vec<_>, Vec<_>) =
edges.into_iter().partition(|e| e.src_id == id);
let upstream = rest.into_iter().filter(|e| e.dst_id == id).collect();
Ok(Some(catalog::CatalogDatasetDetail {
dataset,
schema_timeline,
stats,
upstream,
downstream,
}))
}
async fn catalog_lineage(
&self,
root: Option<&str>,
depth: u32,
) -> Result<
Vec<$crate::serve::history::catalog::CatalogLineageEdge>,
$crate::serve::history::HistoryError,
> {
use $crate::serve::history::catalog;
let edges = self.catalog_all_edges().await?;
Ok(catalog::lineage_slice(edges, root, depth))
}
async fn catalog_record_config_snapshot(
&self,
snapshot: &$crate::serve::history::catalog::ConfigSnapshot,
) -> Result<(), $crate::serve::history::HistoryError> {
use $crate::serve::history::HistoryError;
use $crate::serve::history::sql;
let backend = |e: sqlx::Error| HistoryError::Backend(e.to_string());
sqlx::query(&self.stmts.catalog_upsert_config_snapshot)
.bind(&snapshot.pipeline)
.bind(sql::fmt_ts(snapshot.recorded_at))
.bind(&snapshot.faucet_version)
.bind(sql::encode_json(snapshot, "config snapshot")?)
.execute(&self.pool)
.await
.map_err(backend)?;
Ok(())
}
async fn catalog_last_config_snapshot(
&self,
pipeline: &str,
) -> Result<
Option<$crate::serve::history::catalog::ConfigSnapshot>,
$crate::serve::history::HistoryError,
> {
use sqlx::Row as _;
use $crate::serve::history::HistoryError;
use $crate::serve::history::sql;
let backend = |e: sqlx::Error| HistoryError::Backend(e.to_string());
let Some(row) = sqlx::query(&self.stmts.catalog_select_config_snapshot)
.bind(pipeline)
.fetch_optional(&self.pool)
.await
.map_err(backend)?
else {
return Ok(None);
};
let body: String = row.try_get("body").map_err(backend)?;
Ok(Some(sql::decode_json(&body, "config snapshot")?))
}
fn degraded(&self) -> bool {
false
}
}
impl $name {
async fn catalog_all_edges(
&self,
) -> Result<
Vec<$crate::serve::history::catalog::CatalogLineageEdge>,
$crate::serve::history::HistoryError,
> {
use sqlx::Row as _;
use $crate::serve::history::HistoryError;
use $crate::serve::history::sql;
let backend = |e: sqlx::Error| HistoryError::Backend(e.to_string());
let rows = sqlx::query(&self.stmts.catalog_select_edges)
.fetch_all(&self.pool)
.await
.map_err(backend)?;
let mut edges = Vec::with_capacity(rows.len());
for r in &rows {
let body: String = r.try_get("body").map_err(backend)?;
edges.push(sql::decode_json(&body, "catalog edge")?);
}
Ok(edges)
}
}
};
}
pub(crate) use impl_sql_history;
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn postgres_shard_statements_are_built() {
let s = Stmts::new(Dialect::Postgres);
assert!(s.insert_shard.contains("faucet_serve_shards"));
assert!(s.insert_shard.contains("ON CONFLICT"));
assert!(s.claim_shards_select.contains("JOIN faucet_serve_runs"));
assert!(s.claim_shard_one.contains("'running'"));
assert!(s.renew_shard_leases.contains("lease_expires_at"));
assert!(s.reclaim_shards_select.contains("'running'"));
assert!(s.reclaim_shard_requeue.contains("'pending'"));
assert!(s.reclaim_shard_fail.contains("'failed'"));
assert!(s.finalize_shard.contains("owner"));
assert!(s.shard_progress.contains("GROUP BY"));
}
#[test]
fn fmt_ts_is_fixed_width_and_sortable() {
let a = fmt_ts(
DateTime::parse_from_rfc3339("2026-01-01T00:00:00Z")
.unwrap()
.to_utc(),
);
let b = fmt_ts(
DateTime::parse_from_rfc3339("2026-01-01T00:00:01Z")
.unwrap()
.to_utc(),
);
assert!(a.ends_with('Z'));
assert_eq!(a.len(), b.len(), "fixed width");
assert!(a < b, "lexicographic order matches chronological order");
}
#[test]
fn is_expired_respects_window() {
let now = Utc::now();
let old = fmt_ts(now - chrono::Duration::seconds(120));
assert!(is_expired(&old, now, Duration::from_secs(60)));
assert!(!is_expired(&old, now, Duration::from_secs(600)));
assert!(!is_expired("not-a-timestamp", now, Duration::ZERO));
}
#[test]
fn parse_status_round_trips_known_and_defaults_failed() {
for s in [
RunStatus::Queued,
RunStatus::Pending,
RunStatus::Running,
RunStatus::Completed,
RunStatus::Failed,
RunStatus::Cancelled,
] {
assert_eq!(parse_status(s.as_str()), s);
}
assert_eq!(parse_status("garbage"), RunStatus::Failed);
}
#[test]
fn body_round_trips() {
let rec = RunRecord::queued(
"r1".into(),
Some("n".into()),
Default::default(),
Some("idem".into()),
Utc::now(),
);
let encoded = encode_body(&rec).unwrap();
let decoded = decode_body(&encoded).unwrap();
assert_eq!(decoded.run_id, "r1");
assert_eq!(decoded.idempotency_key.as_deref(), Some("idem"));
}
#[test]
fn postgres_and_sqlite_statements_differ_only_in_placeholders() {
let pg = Stmts::new(Dialect::Postgres);
let lite = Stmts::new(Dialect::Sqlite);
assert!(pg.upsert.contains("$1") && lite.upsert.contains('?'));
assert!(pg.list.contains("$13") && lite.list.contains('?'));
assert!(pg.insert_idem.contains("ON CONFLICT (key) DO NOTHING"));
assert!(lite.insert_idem.contains("ON CONFLICT (key) DO NOTHING"));
assert!(pg.claim_one.contains("$3") && lite.claim_one.contains('?'));
assert!(pg.heartbeat_instance.contains("faucet_serve_instances"));
assert!(lite.heartbeat_instance.contains("faucet_serve_instances"));
}
}