1use super::{HistoryError, RunRecord, RunStatus};
20use chrono::{DateTime, Utc};
21use std::time::Duration;
22
23pub const DDL: &[&str] = &[
26 "CREATE TABLE IF NOT EXISTS faucet_serve_runs (\
31 run_id TEXT PRIMARY KEY,\
32 name TEXT,\
33 status TEXT NOT NULL,\
34 submitted_at TEXT NOT NULL,\
35 finished_at TEXT,\
36 idempotency_key TEXT,\
37 owner TEXT,\
38 lease_expires_at TEXT,\
39 cancel_requested TEXT,\
40 body TEXT NOT NULL)",
41 "CREATE INDEX IF NOT EXISTS faucet_serve_runs_submitted_idx \
42 ON faucet_serve_runs (submitted_at)",
43 "CREATE INDEX IF NOT EXISTS faucet_serve_runs_status_lease_idx \
46 ON faucet_serve_runs (status, lease_expires_at)",
47 "CREATE INDEX IF NOT EXISTS faucet_serve_runs_pending_idx \
49 ON faucet_serve_runs (status, submitted_at)",
50 "CREATE TABLE IF NOT EXISTS faucet_serve_instances (\
51 instance_id TEXT PRIMARY KEY,\
52 started_at TEXT NOT NULL,\
53 last_heartbeat TEXT NOT NULL,\
54 listen TEXT,\
55 max_concurrent TEXT,\
56 in_flight TEXT)",
57 "CREATE INDEX IF NOT EXISTS faucet_serve_instances_hb_idx \
58 ON faucet_serve_instances (last_heartbeat)",
59 "CREATE TABLE IF NOT EXISTS faucet_serve_idem (\
60 key TEXT PRIMARY KEY,\
61 run_id TEXT NOT NULL,\
62 fingerprint TEXT NOT NULL,\
63 claimed_at TEXT NOT NULL)",
64 "CREATE TABLE IF NOT EXISTS faucet_serve_shards (\
70 run_id TEXT NOT NULL,\
71 shard_id TEXT NOT NULL,\
72 descriptor TEXT NOT NULL,\
73 size_estimate TEXT,\
74 status TEXT NOT NULL,\
75 owner TEXT,\
76 lease_expires_at TEXT,\
77 attempt TEXT NOT NULL,\
78 finished_at TEXT,\
79 PRIMARY KEY (run_id, shard_id))",
80 "CREATE INDEX IF NOT EXISTS faucet_serve_shards_claim_idx \
81 ON faucet_serve_shards (status, lease_expires_at)",
82 "CREATE TABLE IF NOT EXISTS faucet_serve_audit (\
86 id TEXT PRIMARY KEY,\
87 ts TEXT NOT NULL,\
88 principal TEXT NOT NULL,\
89 role TEXT NOT NULL,\
90 action TEXT NOT NULL,\
91 run_id TEXT,\
92 config_fingerprint TEXT,\
93 source_ip TEXT,\
94 result TEXT NOT NULL)",
95 "CREATE INDEX IF NOT EXISTS faucet_serve_audit_ts_idx \
96 ON faucet_serve_audit (ts)",
97 "CREATE TABLE IF NOT EXISTS faucet_serve_run_logs (\
100 run_id TEXT NOT NULL,\
101 seq TEXT NOT NULL,\
102 ts TEXT NOT NULL,\
103 level TEXT NOT NULL,\
104 line TEXT NOT NULL,\
105 PRIMARY KEY (run_id, seq))",
106 "CREATE INDEX IF NOT EXISTS faucet_serve_run_logs_ts_idx \
107 ON faucet_serve_run_logs (ts)",
108 "CREATE TABLE IF NOT EXISTS faucet_catalog_datasets (\
113 id TEXT PRIMARY KEY,\
114 uri TEXT NOT NULL,\
115 kind TEXT NOT NULL,\
116 last_seen TEXT NOT NULL,\
117 body TEXT NOT NULL)",
118 "CREATE TABLE IF NOT EXISTS faucet_catalog_schema_versions (\
122 dataset_id TEXT NOT NULL,\
123 version TEXT NOT NULL,\
124 recorded_at TEXT NOT NULL,\
125 body TEXT NOT NULL,\
126 PRIMARY KEY (dataset_id, version))",
127 "CREATE TABLE IF NOT EXISTS faucet_catalog_edges (\
129 src_id TEXT NOT NULL,\
130 dst_id TEXT NOT NULL,\
131 last_seen TEXT NOT NULL,\
132 body TEXT NOT NULL,\
133 PRIMARY KEY (src_id, dst_id))",
134 "CREATE TABLE IF NOT EXISTS faucet_catalog_stats (\
136 dataset_id TEXT NOT NULL,\
137 recorded_at TEXT NOT NULL,\
138 run_id TEXT NOT NULL,\
139 records TEXT NOT NULL,\
140 PRIMARY KEY (dataset_id, recorded_at))",
141 "CREATE TABLE IF NOT EXISTS faucet_config_snapshots (\
145 pipeline TEXT PRIMARY KEY,\
146 recorded_at TEXT NOT NULL,\
147 faucet_version TEXT NOT NULL,\
148 body TEXT NOT NULL)",
149 "CREATE TABLE IF NOT EXISTS faucet_templates (\
156 id TEXT NOT NULL,\
157 version TEXT NOT NULL,\
158 name TEXT,\
159 created_at TEXT NOT NULL,\
160 body TEXT NOT NULL,\
161 PRIMARY KEY (id, version))",
162 "CREATE TABLE IF NOT EXISTS faucet_template_tags (\
167 id TEXT NOT NULL,\
168 tag TEXT NOT NULL,\
169 version TEXT NOT NULL,\
170 updated_at TEXT NOT NULL,\
171 PRIMARY KEY (id, tag))",
172 "CREATE TABLE IF NOT EXISTS faucet_template_launches (\
177 id TEXT NOT NULL,\
178 seq TEXT NOT NULL,\
179 version TEXT NOT NULL,\
180 launched_at TEXT NOT NULL,\
181 launched_by TEXT,\
182 PRIMARY KEY (id, seq))",
183 "CREATE TABLE IF NOT EXISTS faucet_template_deprecations (\
186 id TEXT PRIMARY KEY,\
187 deprecated_at TEXT NOT NULL,\
188 deprecated_by TEXT,\
189 reason TEXT)",
190];
191
192#[derive(Clone, Copy, Debug)]
194pub enum Dialect {
195 Postgres,
196 Sqlite,
197}
198
199pub struct Stmts {
201 pub upsert: String,
205 pub select_body: String,
206 pub select_status: String,
207 pub select_submitted: String,
208 pub delete: String,
209 pub list: String,
210 pub purge_runs: String,
211 pub purge_idem: String,
212 pub select_orphans: String,
215 pub renew_leases: String,
218 pub insert_idem: String,
219 pub select_idem: String,
220 pub takeover_idem: String,
221 pub delete_idem_by_run: String,
226 pub select_pending: String,
228 pub claim_one: String,
230 pub reclaim_select: String,
234 pub reclaim_requeue: String,
236 pub reclaim_fail: String,
238 pub finalize_owned: String,
240 pub cancel_pending: String,
242 pub request_cancel: String,
244 pub pending_cancellations: String,
246 pub heartbeat_instance: String,
248 pub live_instances: String,
250 pub prune_instances: String,
252 pub insert_shard: String,
255 pub claim_shards_select: String,
257 pub claim_shard_one: String,
259 pub renew_shard_leases: String,
261 pub reclaim_shards_select: String,
263 pub reclaim_shard_requeue: String,
265 pub reclaim_shard_fail: String,
267 pub finalize_shard: String,
269 pub shard_progress: String,
271 pub pending_shard_cancellations: String,
275 pub select_sharded_parents: String,
278 pub finalize_sharded_parent: String,
282 pub delete_shards_by_run: String,
285 pub purge_orphan_shards: String,
288 pub insert_audit: String,
291 pub list_audit: String,
294 pub purge_audit: String,
296 pub insert_run_log: String,
299 pub list_run_logs: String,
302 pub run_log_truncated: String,
304 pub purge_run_logs: String,
306 pub catalog_select_dataset: String,
309 pub catalog_upsert_dataset: String,
312 pub catalog_select_datasets: String,
316 pub catalog_insert_schema_version: String,
319 pub catalog_select_schema_versions: String,
321 pub catalog_upsert_edge: String,
323 pub catalog_select_edges: String,
325 pub catalog_insert_stat: String,
327 pub catalog_select_stats: String,
329 pub catalog_prune_stats: String,
332 pub catalog_upsert_config_snapshot: String,
335 pub catalog_select_config_snapshot: String,
337 pub template_max_version: String,
340 pub template_insert: String,
342 pub template_select_version: String,
344 pub template_select_latest: String,
346 pub template_select_all: String,
349 pub template_versions: String,
351 pub template_delete_version: String,
353 pub template_delete_all: String,
355 pub template_upsert_tag: String,
357 pub template_select_tags: String,
359 pub template_delete_tag: String,
361 pub template_delete_tags_all: String,
363 pub template_delete_tags_for_version: String,
365 pub template_max_launch_seq: String,
367 pub template_insert_launch: String,
369 pub template_select_launches: String,
371 pub template_delete_launches_all: String,
373 pub template_delete_launches_for_version: String,
375 pub template_upsert_deprecation: String,
377 pub template_select_deprecation: String,
379 pub template_delete_deprecation: String,
381}
382
383impl Stmts {
384 pub fn new(dialect: Dialect) -> Self {
385 match dialect {
386 Dialect::Postgres => Self::postgres(),
387 Dialect::Sqlite => Self::sqlite(),
388 }
389 }
390
391 fn postgres() -> Self {
392 Self {
393 upsert: "INSERT INTO faucet_serve_runs \
394 (run_id,name,status,submitted_at,finished_at,idempotency_key,owner,lease_expires_at,body) \
395 VALUES ($1,$2,$3,$4,$5,$6,$7,$8,$9) \
396 ON CONFLICT (run_id) DO UPDATE SET \
397 name=excluded.name,status=excluded.status,submitted_at=excluded.submitted_at,\
398 finished_at=excluded.finished_at,idempotency_key=excluded.idempotency_key,\
399 owner=excluded.owner,lease_expires_at=excluded.lease_expires_at,\
400 body=excluded.body"
401 .into(),
402 select_body: "SELECT body FROM faucet_serve_runs WHERE run_id=$1".into(),
403 select_status: "SELECT status FROM faucet_serve_runs WHERE run_id=$1".into(),
404 select_submitted: "SELECT submitted_at FROM faucet_serve_runs WHERE run_id=$1".into(),
405 delete: "DELETE FROM faucet_serve_runs WHERE run_id=$1".into(),
406 list: "SELECT body FROM faucet_serve_runs \
409 WHERE ($1::text IS NULL OR status = $2::text) \
410 AND ($3::text IS NULL OR name = $4::text) \
411 AND ($5::text IS NULL OR submitted_at >= $6::text) \
412 AND ($7::text IS NULL OR submitted_at <= $8::text) \
413 AND ($9::text IS NULL OR (submitted_at < $10::text \
414 OR (submitted_at = $11::text AND run_id < $12::text))) \
415 ORDER BY submitted_at DESC, run_id DESC LIMIT $13"
416 .into(),
417 purge_runs: "DELETE FROM faucet_serve_runs \
418 WHERE status IN ('completed','failed','cancelled') \
419 AND finished_at IS NOT NULL AND finished_at < $1"
420 .into(),
421 purge_idem: "DELETE FROM faucet_serve_idem WHERE claimed_at < $1".into(),
422 select_orphans: "SELECT body FROM faucet_serve_runs \
423 WHERE status IN ('queued','running') \
424 AND (lease_expires_at IS NULL OR lease_expires_at < $1)"
425 .into(),
426 renew_leases: "UPDATE faucet_serve_runs SET lease_expires_at = $1 \
427 WHERE owner = $2 AND status IN ('queued','running')"
428 .into(),
429 insert_idem: "INSERT INTO faucet_serve_idem (key,run_id,fingerprint,claimed_at) \
430 VALUES ($1,$2,$3,$4) ON CONFLICT (key) DO NOTHING"
431 .into(),
432 select_idem: "SELECT run_id,fingerprint,claimed_at FROM faucet_serve_idem WHERE key=$1"
433 .into(),
434 takeover_idem: "UPDATE faucet_serve_idem \
435 SET run_id=$1,fingerprint=$2,claimed_at=$3 WHERE key=$4 AND claimed_at=$5"
436 .into(),
437 delete_idem_by_run: "DELETE FROM faucet_serve_idem WHERE run_id=$1".into(),
438 select_pending: "SELECT run_id, body FROM faucet_serve_runs \
439 WHERE status = 'pending' ORDER BY submitted_at ASC LIMIT $1"
440 .into(),
441 claim_one: "UPDATE faucet_serve_runs \
442 SET owner = $1, status = 'running', lease_expires_at = $2, body = $3 \
443 WHERE run_id = $4 AND status = 'pending'"
444 .into(),
445 reclaim_select: "SELECT body FROM faucet_serve_runs \
446 WHERE status = 'running' \
447 AND (lease_expires_at IS NULL OR lease_expires_at < $1)"
448 .into(),
449 reclaim_requeue: "UPDATE faucet_serve_runs \
454 SET status = 'pending', owner = NULL, lease_expires_at = NULL, \
455 body = $1 \
456 WHERE run_id = $2 AND status = 'running' \
457 AND (lease_expires_at IS NULL OR lease_expires_at < $3)"
458 .into(),
459 reclaim_fail: "UPDATE faucet_serve_runs \
460 SET status = 'failed', finished_at = $1, body = $2, owner = NULL \
461 WHERE run_id = $3 AND status = 'running' \
462 AND (lease_expires_at IS NULL OR lease_expires_at < $4)"
463 .into(),
464 finalize_owned: "UPDATE faucet_serve_runs \
469 SET status = $1, finished_at = $2, lease_expires_at = $3, body = $4 \
470 WHERE run_id = $5 AND owner = $6 \
471 AND status NOT IN ('completed','failed','cancelled')"
472 .into(),
473 cancel_pending: "UPDATE faucet_serve_runs \
474 SET status = 'cancelled', finished_at = $1, body = $2 \
475 WHERE run_id = $3 AND status = 'pending'"
476 .into(),
477 request_cancel: "UPDATE faucet_serve_runs \
478 SET cancel_requested = $1 WHERE run_id = $2 AND status IN ('running','sharded')"
479 .into(),
480 pending_cancellations: "SELECT run_id FROM faucet_serve_runs \
481 WHERE status = 'running' AND owner = $1 AND cancel_requested IS NOT NULL"
482 .into(),
483 heartbeat_instance: "INSERT INTO faucet_serve_instances \
484 (instance_id, started_at, last_heartbeat, listen, max_concurrent, in_flight) \
485 VALUES ($1,$2,$3,$4,$5,$6) \
486 ON CONFLICT (instance_id) DO UPDATE SET \
487 last_heartbeat = excluded.last_heartbeat, listen = excluded.listen, \
488 max_concurrent = excluded.max_concurrent, in_flight = excluded.in_flight"
489 .into(),
490 live_instances: "SELECT instance_id, started_at, last_heartbeat, listen, \
491 max_concurrent, in_flight FROM faucet_serve_instances \
492 WHERE last_heartbeat >= $1"
493 .into(),
494 prune_instances: "DELETE FROM faucet_serve_instances WHERE last_heartbeat < $1".into(),
495 insert_shard: "INSERT INTO faucet_serve_shards \
496 (run_id, shard_id, descriptor, size_estimate, status, attempt) \
497 VALUES ($1,$2,$3,$4,'pending','0') \
498 ON CONFLICT (run_id, shard_id) DO NOTHING"
499 .into(),
500 claim_shards_select: "SELECT s.run_id, s.shard_id, s.descriptor, r.body \
501 FROM faucet_serve_shards s JOIN faucet_serve_runs r ON r.run_id = s.run_id \
502 WHERE s.status = 'pending' \
503 ORDER BY CAST(COALESCE(s.size_estimate, '0') AS BIGINT) DESC, s.run_id, s.shard_id \
504 LIMIT $1"
505 .into(),
506 claim_shard_one: "UPDATE faucet_serve_shards \
507 SET owner = $1, status = 'running', lease_expires_at = $2 \
508 WHERE run_id = $3 AND shard_id = $4 AND status = 'pending'"
509 .into(),
510 renew_shard_leases: "UPDATE faucet_serve_shards SET lease_expires_at = $1 \
511 WHERE owner = $2 AND status = 'running'"
512 .into(),
513 reclaim_shards_select: "SELECT run_id, shard_id, attempt FROM faucet_serve_shards \
514 WHERE status = 'running' \
515 AND (lease_expires_at IS NULL OR lease_expires_at < $1)"
516 .into(),
517 reclaim_shard_requeue: "UPDATE faucet_serve_shards \
518 SET status = 'pending', owner = NULL, lease_expires_at = NULL, attempt = $1 \
519 WHERE run_id = $2 AND shard_id = $3 AND status = 'running' \
520 AND (lease_expires_at IS NULL OR lease_expires_at < $4)"
521 .into(),
522 reclaim_shard_fail: "UPDATE faucet_serve_shards \
523 SET status = 'failed', finished_at = $1, owner = NULL \
524 WHERE run_id = $2 AND shard_id = $3 AND status = 'running' \
525 AND (lease_expires_at IS NULL OR lease_expires_at < $4)"
526 .into(),
527 finalize_shard: "UPDATE faucet_serve_shards \
528 SET status = $1, finished_at = $2 \
529 WHERE run_id = $3 AND shard_id = $4 AND owner = $5 AND status = 'running'"
530 .into(),
531 shard_progress: "SELECT status, COUNT(*) AS n FROM faucet_serve_shards \
532 WHERE run_id = $1 GROUP BY status"
533 .into(),
534 pending_shard_cancellations: "SELECT DISTINCT s.run_id \
535 FROM faucet_serve_shards s \
536 JOIN faucet_serve_runs r ON r.run_id = s.run_id \
537 WHERE s.owner = $1 AND s.status = 'running' \
538 AND r.cancel_requested IS NOT NULL"
539 .into(),
540 select_sharded_parents: "SELECT run_id FROM faucet_serve_runs \
541 WHERE status = 'sharded'"
542 .into(),
543 finalize_sharded_parent: "UPDATE faucet_serve_runs \
544 SET status = $1, finished_at = $2, body = $3 \
545 WHERE run_id = $4 AND status = 'sharded'"
546 .into(),
547 delete_shards_by_run: "DELETE FROM faucet_serve_shards WHERE run_id = $1".into(),
548 purge_orphan_shards: "DELETE FROM faucet_serve_shards \
549 WHERE run_id NOT IN (SELECT run_id FROM faucet_serve_runs)"
550 .into(),
551 insert_audit: "INSERT INTO faucet_serve_audit \
552 (id, ts, principal, role, action, run_id, config_fingerprint, source_ip, result) \
553 VALUES ($1,$2,$3,$4,$5,$6,$7,$8,$9)"
554 .into(),
555 list_audit: "SELECT id, ts, principal, role, action, run_id, config_fingerprint, \
556 source_ip, result FROM faucet_serve_audit \
557 WHERE ($1::text IS NULL OR principal = $2::text) \
558 AND ($3::text IS NULL OR action = $4::text) \
559 AND ($5::text IS NULL OR ts >= $6::text) \
560 AND ($7::text IS NULL OR ts <= $8::text) \
561 ORDER BY ts DESC, id DESC LIMIT $9"
562 .into(),
563 purge_audit: "DELETE FROM faucet_serve_audit WHERE ts < $1".into(),
564 insert_run_log: "INSERT INTO faucet_serve_run_logs \
565 (run_id, seq, ts, level, line) VALUES ($1,$2,$3,$4,$5) \
566 ON CONFLICT (run_id, seq) DO NOTHING"
567 .into(),
568 list_run_logs: "SELECT seq, ts, level, line FROM faucet_serve_run_logs \
569 WHERE run_id = $1 AND seq <> $2 AND ($3::text IS NULL OR seq > $4::text) \
570 ORDER BY seq ASC LIMIT $5"
571 .into(),
572 run_log_truncated: "SELECT 1 FROM faucet_serve_run_logs \
573 WHERE run_id = $1 AND seq = $2 LIMIT 1"
574 .into(),
575 purge_run_logs: "DELETE FROM faucet_serve_run_logs WHERE ts < $1".into(),
576 catalog_select_dataset: "SELECT body FROM faucet_catalog_datasets WHERE id=$1".into(),
577 catalog_upsert_dataset: "INSERT INTO faucet_catalog_datasets \
578 (id, uri, kind, last_seen, body) VALUES ($1,$2,$3,$4,$5) \
579 ON CONFLICT (id) DO UPDATE SET uri=excluded.uri, kind=excluded.kind, \
580 last_seen=excluded.last_seen, body=excluded.body"
581 .into(),
582 catalog_select_datasets: "SELECT body FROM faucet_catalog_datasets".into(),
583 catalog_insert_schema_version: "INSERT INTO faucet_catalog_schema_versions \
584 (dataset_id, version, recorded_at, body) VALUES ($1,$2,$3,$4) \
585 ON CONFLICT (dataset_id, version) DO NOTHING"
586 .into(),
587 catalog_select_schema_versions: "SELECT body FROM faucet_catalog_schema_versions \
588 WHERE dataset_id=$1 ORDER BY CAST(version AS BIGINT) ASC"
589 .into(),
590 catalog_upsert_edge: "INSERT INTO faucet_catalog_edges \
591 (src_id, dst_id, last_seen, body) VALUES ($1,$2,$3,$4) \
592 ON CONFLICT (src_id, dst_id) DO UPDATE SET \
593 last_seen=excluded.last_seen, body=excluded.body"
594 .into(),
595 catalog_select_edges: "SELECT body FROM faucet_catalog_edges \
596 ORDER BY last_seen DESC, src_id, dst_id"
597 .into(),
598 catalog_insert_stat: "INSERT INTO faucet_catalog_stats \
599 (dataset_id, recorded_at, run_id, records) VALUES ($1,$2,$3,$4) \
600 ON CONFLICT (dataset_id, recorded_at) DO NOTHING"
601 .into(),
602 catalog_select_stats: "SELECT recorded_at, run_id, records \
603 FROM faucet_catalog_stats WHERE dataset_id=$1 \
604 ORDER BY recorded_at DESC LIMIT $2"
605 .into(),
606 catalog_prune_stats: "DELETE FROM faucet_catalog_stats \
607 WHERE dataset_id=$1 AND recorded_at NOT IN (\
608 SELECT recorded_at FROM faucet_catalog_stats WHERE dataset_id=$2 \
609 ORDER BY recorded_at DESC LIMIT $3)"
610 .into(),
611 catalog_upsert_config_snapshot: "INSERT INTO faucet_config_snapshots \
612 (pipeline, recorded_at, faucet_version, body) VALUES ($1,$2,$3,$4) \
613 ON CONFLICT (pipeline) DO UPDATE SET recorded_at=excluded.recorded_at, \
614 faucet_version=excluded.faucet_version, body=excluded.body"
615 .into(),
616 catalog_select_config_snapshot:
617 "SELECT body FROM faucet_config_snapshots WHERE pipeline=$1".into(),
618 template_max_version: "SELECT COALESCE(MAX(CAST(version AS BIGINT)), 0) AS v \
619 FROM faucet_templates WHERE id=$1"
620 .into(),
621 template_insert: "INSERT INTO faucet_templates \
622 (id, version, name, created_at, body) VALUES ($1,$2,$3,$4,$5)"
623 .into(),
624 template_select_version:
625 "SELECT body FROM faucet_templates WHERE id=$1 AND version=$2".into(),
626 template_select_latest: "SELECT body FROM faucet_templates WHERE id=$1 \
627 ORDER BY CAST(version AS BIGINT) DESC LIMIT 1"
628 .into(),
629 template_select_all: "SELECT body FROM faucet_templates".into(),
630 template_versions: "SELECT version FROM faucet_templates WHERE id=$1 \
631 ORDER BY CAST(version AS BIGINT) DESC"
632 .into(),
633 template_delete_version: "DELETE FROM faucet_templates WHERE id=$1 AND version=$2"
634 .into(),
635 template_delete_all: "DELETE FROM faucet_templates WHERE id=$1".into(),
636 template_upsert_tag: "INSERT INTO faucet_template_tags \
637 (id, tag, version, updated_at) VALUES ($1,$2,$3,$4) \
638 ON CONFLICT (id, tag) DO UPDATE SET version=excluded.version, \
639 updated_at=excluded.updated_at"
640 .into(),
641 template_select_tags: "SELECT tag, version FROM faucet_template_tags \
642 WHERE id=$1 ORDER BY tag"
643 .into(),
644 template_delete_tag: "DELETE FROM faucet_template_tags WHERE id=$1 AND tag=$2".into(),
645 template_delete_tags_all: "DELETE FROM faucet_template_tags WHERE id=$1".into(),
646 template_delete_tags_for_version:
647 "DELETE FROM faucet_template_tags WHERE id=$1 AND version=$2".into(),
648 template_max_launch_seq: "SELECT COALESCE(MAX(CAST(seq AS BIGINT)), 0) AS v \
649 FROM faucet_template_launches WHERE id=$1"
650 .into(),
651 template_insert_launch: "INSERT INTO faucet_template_launches \
652 (id, seq, version, launched_at, launched_by) VALUES ($1,$2,$3,$4,$5)"
653 .into(),
654 template_select_launches: "SELECT seq, version, launched_at, launched_by \
655 FROM faucet_template_launches WHERE id=$1 ORDER BY CAST(seq AS BIGINT) DESC"
656 .into(),
657 template_delete_launches_all: "DELETE FROM faucet_template_launches WHERE id=$1".into(),
658 template_delete_launches_for_version:
659 "DELETE FROM faucet_template_launches WHERE id=$1 AND version=$2".into(),
660 template_upsert_deprecation: "INSERT INTO faucet_template_deprecations \
661 (id, deprecated_at, deprecated_by, reason) VALUES ($1,$2,$3,$4) \
662 ON CONFLICT (id) DO UPDATE SET deprecated_at=excluded.deprecated_at, \
663 deprecated_by=excluded.deprecated_by, reason=excluded.reason"
664 .into(),
665 template_select_deprecation: "SELECT deprecated_at, deprecated_by, reason \
666 FROM faucet_template_deprecations WHERE id=$1"
667 .into(),
668 template_delete_deprecation: "DELETE FROM faucet_template_deprecations WHERE id=$1"
669 .into(),
670 }
671 }
672
673 fn sqlite() -> Self {
674 Self {
675 upsert: "INSERT INTO faucet_serve_runs \
676 (run_id,name,status,submitted_at,finished_at,idempotency_key,owner,lease_expires_at,body) \
677 VALUES (?,?,?,?,?,?,?,?,?) \
678 ON CONFLICT (run_id) DO UPDATE SET \
679 name=excluded.name,status=excluded.status,submitted_at=excluded.submitted_at,\
680 finished_at=excluded.finished_at,idempotency_key=excluded.idempotency_key,\
681 owner=excluded.owner,lease_expires_at=excluded.lease_expires_at,\
682 body=excluded.body"
683 .into(),
684 select_body: "SELECT body FROM faucet_serve_runs WHERE run_id=?".into(),
685 select_status: "SELECT status FROM faucet_serve_runs WHERE run_id=?".into(),
686 select_submitted: "SELECT submitted_at FROM faucet_serve_runs WHERE run_id=?".into(),
687 delete: "DELETE FROM faucet_serve_runs WHERE run_id=?".into(),
688 list: "SELECT body FROM faucet_serve_runs \
689 WHERE (? IS NULL OR status = ?) \
690 AND (? IS NULL OR name = ?) \
691 AND (? IS NULL OR submitted_at >= ?) \
692 AND (? IS NULL OR submitted_at <= ?) \
693 AND (? IS NULL OR (submitted_at < ? \
694 OR (submitted_at = ? AND run_id < ?))) \
695 ORDER BY submitted_at DESC, run_id DESC LIMIT ?"
696 .into(),
697 purge_runs: "DELETE FROM faucet_serve_runs \
698 WHERE status IN ('completed','failed','cancelled') \
699 AND finished_at IS NOT NULL AND finished_at < ?"
700 .into(),
701 purge_idem: "DELETE FROM faucet_serve_idem WHERE claimed_at < ?".into(),
702 select_orphans: "SELECT body FROM faucet_serve_runs \
703 WHERE status IN ('queued','running') \
704 AND (lease_expires_at IS NULL OR lease_expires_at < ?)"
705 .into(),
706 renew_leases: "UPDATE faucet_serve_runs SET lease_expires_at = ? \
707 WHERE owner = ? AND status IN ('queued','running')"
708 .into(),
709 insert_idem: "INSERT INTO faucet_serve_idem (key,run_id,fingerprint,claimed_at) \
710 VALUES (?,?,?,?) ON CONFLICT (key) DO NOTHING"
711 .into(),
712 select_idem: "SELECT run_id,fingerprint,claimed_at FROM faucet_serve_idem WHERE key=?"
713 .into(),
714 takeover_idem: "UPDATE faucet_serve_idem \
715 SET run_id=?,fingerprint=?,claimed_at=? WHERE key=? AND claimed_at=?"
716 .into(),
717 delete_idem_by_run: "DELETE FROM faucet_serve_idem WHERE run_id=?".into(),
718 select_pending: "SELECT run_id, body FROM faucet_serve_runs \
719 WHERE status = 'pending' ORDER BY submitted_at ASC LIMIT ?"
720 .into(),
721 claim_one: "UPDATE faucet_serve_runs \
722 SET owner = ?, status = 'running', lease_expires_at = ?, body = ? \
723 WHERE run_id = ? AND status = 'pending'"
724 .into(),
725 reclaim_select: "SELECT body FROM faucet_serve_runs \
726 WHERE status = 'running' \
727 AND (lease_expires_at IS NULL OR lease_expires_at < ?)"
728 .into(),
729 reclaim_requeue: "UPDATE faucet_serve_runs \
731 SET status = 'pending', owner = NULL, lease_expires_at = NULL, \
732 body = ? \
733 WHERE run_id = ? AND status = 'running' \
734 AND (lease_expires_at IS NULL OR lease_expires_at < ?)"
735 .into(),
736 reclaim_fail: "UPDATE faucet_serve_runs \
737 SET status = 'failed', finished_at = ?, body = ?, owner = NULL \
738 WHERE run_id = ? AND status = 'running' \
739 AND (lease_expires_at IS NULL OR lease_expires_at < ?)"
740 .into(),
741 finalize_owned: "UPDATE faucet_serve_runs \
743 SET status = ?, finished_at = ?, lease_expires_at = ?, body = ? \
744 WHERE run_id = ? AND owner = ? \
745 AND status NOT IN ('completed','failed','cancelled')"
746 .into(),
747 cancel_pending: "UPDATE faucet_serve_runs \
748 SET status = 'cancelled', finished_at = ?, body = ? \
749 WHERE run_id = ? AND status = 'pending'"
750 .into(),
751 request_cancel: "UPDATE faucet_serve_runs \
752 SET cancel_requested = ? WHERE run_id = ? AND status IN ('running','sharded')"
753 .into(),
754 pending_cancellations: "SELECT run_id FROM faucet_serve_runs \
755 WHERE status = 'running' AND owner = ? AND cancel_requested IS NOT NULL"
756 .into(),
757 heartbeat_instance: "INSERT INTO faucet_serve_instances \
758 (instance_id, started_at, last_heartbeat, listen, max_concurrent, in_flight) \
759 VALUES (?,?,?,?,?,?) \
760 ON CONFLICT (instance_id) DO UPDATE SET \
761 last_heartbeat = excluded.last_heartbeat, listen = excluded.listen, \
762 max_concurrent = excluded.max_concurrent, in_flight = excluded.in_flight"
763 .into(),
764 live_instances: "SELECT instance_id, started_at, last_heartbeat, listen, \
765 max_concurrent, in_flight FROM faucet_serve_instances \
766 WHERE last_heartbeat >= ?"
767 .into(),
768 prune_instances: "DELETE FROM faucet_serve_instances WHERE last_heartbeat < ?".into(),
769 insert_shard: "INSERT INTO faucet_serve_shards \
770 (run_id, shard_id, descriptor, size_estimate, status, attempt) \
771 VALUES (?,?,?,?,'pending','0') \
772 ON CONFLICT (run_id, shard_id) DO NOTHING"
773 .into(),
774 claim_shards_select: "SELECT s.run_id, s.shard_id, s.descriptor, r.body \
775 FROM faucet_serve_shards s JOIN faucet_serve_runs r ON r.run_id = s.run_id \
776 WHERE s.status = 'pending' \
777 ORDER BY CAST(COALESCE(s.size_estimate, '0') AS INTEGER) DESC, s.run_id, s.shard_id \
778 LIMIT ?"
779 .into(),
780 claim_shard_one: "UPDATE faucet_serve_shards \
781 SET owner = ?, status = 'running', lease_expires_at = ? \
782 WHERE run_id = ? AND shard_id = ? AND status = 'pending'"
783 .into(),
784 renew_shard_leases: "UPDATE faucet_serve_shards SET lease_expires_at = ? \
785 WHERE owner = ? AND status = 'running'"
786 .into(),
787 reclaim_shards_select: "SELECT run_id, shard_id, attempt FROM faucet_serve_shards \
788 WHERE status = 'running' \
789 AND (lease_expires_at IS NULL OR lease_expires_at < ?)"
790 .into(),
791 reclaim_shard_requeue: "UPDATE faucet_serve_shards \
792 SET status = 'pending', owner = NULL, lease_expires_at = NULL, attempt = ? \
793 WHERE run_id = ? AND shard_id = ? AND status = 'running' \
794 AND (lease_expires_at IS NULL OR lease_expires_at < ?)"
795 .into(),
796 reclaim_shard_fail: "UPDATE faucet_serve_shards \
797 SET status = 'failed', finished_at = ?, owner = NULL \
798 WHERE run_id = ? AND shard_id = ? AND status = 'running' \
799 AND (lease_expires_at IS NULL OR lease_expires_at < ?)"
800 .into(),
801 finalize_shard: "UPDATE faucet_serve_shards \
802 SET status = ?, finished_at = ? \
803 WHERE run_id = ? AND shard_id = ? AND owner = ? AND status = 'running'"
804 .into(),
805 shard_progress: "SELECT status, COUNT(*) AS n FROM faucet_serve_shards \
806 WHERE run_id = ? GROUP BY status"
807 .into(),
808 pending_shard_cancellations: "SELECT DISTINCT s.run_id \
809 FROM faucet_serve_shards s \
810 JOIN faucet_serve_runs r ON r.run_id = s.run_id \
811 WHERE s.owner = ? AND s.status = 'running' \
812 AND r.cancel_requested IS NOT NULL"
813 .into(),
814 select_sharded_parents: "SELECT run_id FROM faucet_serve_runs \
815 WHERE status = 'sharded'"
816 .into(),
817 finalize_sharded_parent: "UPDATE faucet_serve_runs \
818 SET status = ?, finished_at = ?, body = ? \
819 WHERE run_id = ? AND status = 'sharded'"
820 .into(),
821 delete_shards_by_run: "DELETE FROM faucet_serve_shards WHERE run_id = ?".into(),
822 purge_orphan_shards: "DELETE FROM faucet_serve_shards \
823 WHERE run_id NOT IN (SELECT run_id FROM faucet_serve_runs)"
824 .into(),
825 insert_audit: "INSERT INTO faucet_serve_audit \
826 (id, ts, principal, role, action, run_id, config_fingerprint, source_ip, result) \
827 VALUES (?,?,?,?,?,?,?,?,?)"
828 .into(),
829 list_audit: "SELECT id, ts, principal, role, action, run_id, config_fingerprint, \
830 source_ip, result FROM faucet_serve_audit \
831 WHERE (? IS NULL OR principal = ?) \
832 AND (? IS NULL OR action = ?) \
833 AND (? IS NULL OR ts >= ?) \
834 AND (? IS NULL OR ts <= ?) \
835 ORDER BY ts DESC, id DESC LIMIT ?"
836 .into(),
837 purge_audit: "DELETE FROM faucet_serve_audit WHERE ts < ?".into(),
838 insert_run_log: "INSERT INTO faucet_serve_run_logs \
839 (run_id, seq, ts, level, line) VALUES (?,?,?,?,?) \
840 ON CONFLICT (run_id, seq) DO NOTHING"
841 .into(),
842 list_run_logs: "SELECT seq, ts, level, line FROM faucet_serve_run_logs \
843 WHERE run_id = ? AND seq <> ? AND (? IS NULL OR seq > ?) \
844 ORDER BY seq ASC LIMIT ?"
845 .into(),
846 run_log_truncated: "SELECT 1 FROM faucet_serve_run_logs \
847 WHERE run_id = ? AND seq = ? LIMIT 1"
848 .into(),
849 purge_run_logs: "DELETE FROM faucet_serve_run_logs WHERE ts < ?".into(),
850 catalog_select_dataset: "SELECT body FROM faucet_catalog_datasets WHERE id=?".into(),
851 catalog_upsert_dataset: "INSERT INTO faucet_catalog_datasets \
852 (id, uri, kind, last_seen, body) VALUES (?,?,?,?,?) \
853 ON CONFLICT (id) DO UPDATE SET uri=excluded.uri, kind=excluded.kind, \
854 last_seen=excluded.last_seen, body=excluded.body"
855 .into(),
856 catalog_select_datasets: "SELECT body FROM faucet_catalog_datasets".into(),
857 catalog_insert_schema_version: "INSERT INTO faucet_catalog_schema_versions \
858 (dataset_id, version, recorded_at, body) VALUES (?,?,?,?) \
859 ON CONFLICT (dataset_id, version) DO NOTHING"
860 .into(),
861 catalog_select_schema_versions: "SELECT body FROM faucet_catalog_schema_versions \
862 WHERE dataset_id=? ORDER BY CAST(version AS INTEGER) ASC"
863 .into(),
864 catalog_upsert_edge: "INSERT INTO faucet_catalog_edges \
865 (src_id, dst_id, last_seen, body) VALUES (?,?,?,?) \
866 ON CONFLICT (src_id, dst_id) DO UPDATE SET \
867 last_seen=excluded.last_seen, body=excluded.body"
868 .into(),
869 catalog_select_edges: "SELECT body FROM faucet_catalog_edges \
870 ORDER BY last_seen DESC, src_id, dst_id"
871 .into(),
872 catalog_insert_stat: "INSERT INTO faucet_catalog_stats \
873 (dataset_id, recorded_at, run_id, records) VALUES (?,?,?,?) \
874 ON CONFLICT (dataset_id, recorded_at) DO NOTHING"
875 .into(),
876 catalog_select_stats: "SELECT recorded_at, run_id, records \
877 FROM faucet_catalog_stats WHERE dataset_id=? \
878 ORDER BY recorded_at DESC LIMIT ?"
879 .into(),
880 catalog_prune_stats: "DELETE FROM faucet_catalog_stats \
881 WHERE dataset_id=? AND recorded_at NOT IN (\
882 SELECT recorded_at FROM faucet_catalog_stats WHERE dataset_id=? \
883 ORDER BY recorded_at DESC LIMIT ?)"
884 .into(),
885 catalog_upsert_config_snapshot: "INSERT INTO faucet_config_snapshots \
886 (pipeline, recorded_at, faucet_version, body) VALUES (?,?,?,?) \
887 ON CONFLICT (pipeline) DO UPDATE SET recorded_at=excluded.recorded_at, \
888 faucet_version=excluded.faucet_version, body=excluded.body"
889 .into(),
890 catalog_select_config_snapshot:
891 "SELECT body FROM faucet_config_snapshots WHERE pipeline=?".into(),
892 template_max_version: "SELECT COALESCE(MAX(CAST(version AS INTEGER)), 0) AS v \
893 FROM faucet_templates WHERE id=?"
894 .into(),
895 template_insert: "INSERT INTO faucet_templates \
896 (id, version, name, created_at, body) VALUES (?,?,?,?,?)"
897 .into(),
898 template_select_version: "SELECT body FROM faucet_templates WHERE id=? AND version=?"
899 .into(),
900 template_select_latest: "SELECT body FROM faucet_templates WHERE id=? \
901 ORDER BY CAST(version AS INTEGER) DESC LIMIT 1"
902 .into(),
903 template_select_all: "SELECT body FROM faucet_templates".into(),
904 template_versions: "SELECT version FROM faucet_templates WHERE id=? \
905 ORDER BY CAST(version AS INTEGER) DESC"
906 .into(),
907 template_delete_version: "DELETE FROM faucet_templates WHERE id=? AND version=?".into(),
908 template_delete_all: "DELETE FROM faucet_templates WHERE id=?".into(),
909 template_upsert_tag: "INSERT INTO faucet_template_tags \
910 (id, tag, version, updated_at) VALUES (?,?,?,?) \
911 ON CONFLICT (id, tag) DO UPDATE SET version=excluded.version, \
912 updated_at=excluded.updated_at"
913 .into(),
914 template_select_tags: "SELECT tag, version FROM faucet_template_tags \
915 WHERE id=? ORDER BY tag"
916 .into(),
917 template_delete_tag: "DELETE FROM faucet_template_tags WHERE id=? AND tag=?".into(),
918 template_delete_tags_all: "DELETE FROM faucet_template_tags WHERE id=?".into(),
919 template_delete_tags_for_version:
920 "DELETE FROM faucet_template_tags WHERE id=? AND version=?".into(),
921 template_max_launch_seq: "SELECT COALESCE(MAX(CAST(seq AS INTEGER)), 0) AS v \
922 FROM faucet_template_launches WHERE id=?"
923 .into(),
924 template_insert_launch: "INSERT INTO faucet_template_launches \
925 (id, seq, version, launched_at, launched_by) VALUES (?,?,?,?,?)"
926 .into(),
927 template_select_launches: "SELECT seq, version, launched_at, launched_by \
928 FROM faucet_template_launches WHERE id=? ORDER BY CAST(seq AS INTEGER) DESC"
929 .into(),
930 template_delete_launches_all: "DELETE FROM faucet_template_launches WHERE id=?".into(),
931 template_delete_launches_for_version:
932 "DELETE FROM faucet_template_launches WHERE id=? AND version=?".into(),
933 template_upsert_deprecation: "INSERT INTO faucet_template_deprecations \
934 (id, deprecated_at, deprecated_by, reason) VALUES (?,?,?,?) \
935 ON CONFLICT (id) DO UPDATE SET deprecated_at=excluded.deprecated_at, \
936 deprecated_by=excluded.deprecated_by, reason=excluded.reason"
937 .into(),
938 template_select_deprecation: "SELECT deprecated_at, deprecated_by, reason \
939 FROM faucet_template_deprecations WHERE id=?"
940 .into(),
941 template_delete_deprecation: "DELETE FROM faucet_template_deprecations WHERE id=?"
942 .into(),
943 }
944 }
945}
946
947pub const CLAIM_ATTEMPTS: usize = 8;
959
960static RETRY_SEQ: std::sync::atomic::AtomicUsize = std::sync::atomic::AtomicUsize::new(0);
966
967pub async fn retry_backoff(attempt: usize) {
974 if attempt <= 1 {
975 return;
976 }
977 const BASE_MS: u64 = 5;
978 const CAP_MS: u64 = 160;
979 let exp = BASE_MS
980 .saturating_mul(1u64 << (attempt - 2).min(6))
981 .min(CAP_MS);
982 let stagger =
984 (RETRY_SEQ.fetch_add(1, std::sync::atomic::Ordering::Relaxed) as u64) % (BASE_MS + 1);
985 tokio::time::sleep(std::time::Duration::from_millis(exp + stagger)).await;
986}
987
988pub fn fmt_ts(dt: DateTime<Utc>) -> String {
990 dt.to_rfc3339_opts(chrono::SecondsFormat::Nanos, true)
991}
992
993pub fn pad_seq(seq: u64) -> String {
996 format!("{seq:020}")
997}
998
999pub fn unpad_seq(raw: &str) -> u64 {
1001 raw.trim_start_matches('0').parse().unwrap_or(0)
1002}
1003
1004pub fn parse_ts(raw: &str) -> DateTime<Utc> {
1008 DateTime::parse_from_rfc3339(raw)
1009 .map(|d| d.to_utc())
1010 .unwrap_or_else(|_| Utc::now())
1011}
1012
1013pub fn is_expired(claimed_at: &str, now: DateTime<Utc>, window: Duration) -> bool {
1017 match DateTime::parse_from_rfc3339(claimed_at) {
1018 Ok(t) => now
1019 .signed_duration_since(t.with_timezone(&Utc))
1020 .to_std()
1021 .map(|age| age >= window)
1022 .unwrap_or(false),
1023 Err(_) => false,
1024 }
1025}
1026
1027pub fn threshold(now: DateTime<Utc>, window: Duration) -> String {
1029 let delta =
1030 chrono::Duration::from_std(window).unwrap_or_else(|_| chrono::Duration::days(36_500));
1031 fmt_ts(now - delta)
1032}
1033
1034pub fn encode_body(rec: &RunRecord) -> Result<String, HistoryError> {
1035 serde_json::to_string(rec).map_err(|e| HistoryError::Backend(format!("encode run record: {e}")))
1036}
1037
1038pub fn decode_body(body: &str) -> Result<RunRecord, HistoryError> {
1039 serde_json::from_str(body).map_err(|e| HistoryError::Backend(format!("decode run record: {e}")))
1040}
1041
1042pub fn encode_json<T: serde::Serialize>(value: &T, what: &str) -> Result<String, HistoryError> {
1044 serde_json::to_string(value).map_err(|e| HistoryError::Backend(format!("encode {what}: {e}")))
1045}
1046
1047pub fn decode_json<T: serde::de::DeserializeOwned>(
1048 body: &str,
1049 what: &str,
1050) -> Result<T, HistoryError> {
1051 serde_json::from_str(body).map_err(|e| HistoryError::Backend(format!("decode {what}: {e}")))
1052}
1053
1054pub fn parse_status(s: &str) -> RunStatus {
1055 match s {
1056 "queued" => RunStatus::Queued,
1057 "pending" => RunStatus::Pending,
1058 "running" => RunStatus::Running,
1059 "sharded" => RunStatus::Sharded,
1060 "completed" => RunStatus::Completed,
1061 "cancelled" => RunStatus::Cancelled,
1062 _ => RunStatus::Failed,
1063 }
1064}
1065
1066macro_rules! impl_sql_history {
1070 ($name:ident, $pool:ty) => {
1071 pub struct $name {
1074 pool: $pool,
1075 idem_retention: std::time::Duration,
1076 instance_id: String,
1078 lease_ttl: std::time::Duration,
1080 stmts: $crate::serve::history::sql::Stmts,
1081 }
1082
1083 impl $name {
1084 pub fn from_parts(
1086 pool: $pool,
1087 idem_retention: std::time::Duration,
1088 lease_ttl: std::time::Duration,
1089 instance_id: String,
1090 stmts: $crate::serve::history::sql::Stmts,
1091 ) -> Self {
1092 Self {
1093 pool,
1094 idem_retention,
1095 instance_id,
1096 lease_ttl,
1097 stmts,
1098 }
1099 }
1100
1101 pub fn pool(&self) -> &$pool {
1103 &self.pool
1104 }
1105 }
1106
1107 #[async_trait::async_trait]
1108 impl $crate::serve::history::RunHistory for $name {
1109 async fn claim_idempotency(
1110 &self,
1111 key: &str,
1112 fingerprint: &str,
1113 run_id: &str,
1114 window: std::time::Duration,
1115 ) -> Result<$crate::serve::history::Claim, $crate::serve::history::HistoryError> {
1116 use sqlx::Row as _;
1117 use $crate::serve::history::Claim;
1118 use $crate::serve::history::HistoryError;
1119 use $crate::serve::history::sql;
1120
1121 let now = chrono::Utc::now();
1122 let now_s = sql::fmt_ts(now);
1123 let backend = |e: sqlx::Error| HistoryError::Backend(e.to_string());
1124
1125 for _ in 0..sql::CLAIM_ATTEMPTS {
1126 let inserted = sqlx::query(&self.stmts.insert_idem)
1128 .bind(key)
1129 .bind(run_id)
1130 .bind(fingerprint)
1131 .bind(&now_s)
1132 .execute(&self.pool)
1133 .await
1134 .map_err(backend)?
1135 .rows_affected();
1136 if inserted == 1 {
1137 return Ok(Claim::Fresh);
1138 }
1139 let Some(row) = sqlx::query(&self.stmts.select_idem)
1141 .bind(key)
1142 .fetch_optional(&self.pool)
1143 .await
1144 .map_err(backend)?
1145 else {
1146 continue;
1148 };
1149 let existing_run: String = row.try_get("run_id").map_err(backend)?;
1150 let existing_fp: String = row.try_get("fingerprint").map_err(backend)?;
1151 let claimed_at: String = row.try_get("claimed_at").map_err(backend)?;
1152
1153 if sql::is_expired(&claimed_at, now, window) {
1154 let took = sqlx::query(&self.stmts.takeover_idem)
1157 .bind(run_id)
1158 .bind(fingerprint)
1159 .bind(&now_s)
1160 .bind(key)
1161 .bind(&claimed_at)
1162 .execute(&self.pool)
1163 .await
1164 .map_err(backend)?
1165 .rows_affected();
1166 if took == 1 {
1167 return Ok(Claim::Fresh);
1168 }
1169 continue; }
1171 return Ok(if existing_fp == fingerprint {
1172 Claim::Replay(existing_run)
1173 } else {
1174 Claim::Conflict
1175 });
1176 }
1177 tracing::warn!(
1180 key,
1181 "idempotency claim exhausted retries; reporting conflict"
1182 );
1183 Ok(Claim::Conflict)
1184 }
1185
1186 async fn upsert(
1187 &self,
1188 rec: &$crate::serve::history::RunRecord,
1189 ) -> Result<(), $crate::serve::history::HistoryError> {
1190 use $crate::serve::history::HistoryError;
1191 use $crate::serve::history::sql;
1192 let body = sql::encode_body(rec)?;
1193 let submitted = sql::fmt_ts(rec.submitted_at);
1194 let finished = rec.finished_at.map(sql::fmt_ts);
1195 let lease = sql::fmt_ts(chrono::Utc::now() + self.lease_ttl);
1200 sqlx::query(&self.stmts.upsert)
1201 .bind(&rec.run_id)
1202 .bind(rec.name.as_deref())
1203 .bind(rec.status.as_str())
1204 .bind(&submitted)
1205 .bind(finished.as_deref())
1206 .bind(rec.idempotency_key.as_deref())
1207 .bind(&self.instance_id)
1208 .bind(&lease)
1209 .bind(&body)
1210 .execute(&self.pool)
1211 .await
1212 .map_err(|e| HistoryError::Backend(e.to_string()))?;
1213 Ok(())
1214 }
1215
1216 async fn get(
1217 &self,
1218 id: &str,
1219 ) -> Result<
1220 Option<$crate::serve::history::RunRecord>,
1221 $crate::serve::history::HistoryError,
1222 > {
1223 use sqlx::Row as _;
1224 use $crate::serve::history::HistoryError;
1225 use $crate::serve::history::sql;
1226 let row = sqlx::query(&self.stmts.select_body)
1227 .bind(id)
1228 .fetch_optional(&self.pool)
1229 .await
1230 .map_err(|e| HistoryError::Backend(e.to_string()))?;
1231 match row {
1232 None => Ok(None),
1233 Some(r) => {
1234 let body: String = r
1235 .try_get("body")
1236 .map_err(|e| HistoryError::Backend(e.to_string()))?;
1237 Ok(Some(sql::decode_body(&body)?))
1238 }
1239 }
1240 }
1241
1242 async fn list(
1243 &self,
1244 filter: &$crate::serve::history::ListFilter,
1245 ) -> Result<$crate::serve::history::ListPage, $crate::serve::history::HistoryError>
1246 {
1247 use sqlx::Row as _;
1248 use $crate::serve::history::HistoryError;
1249 use $crate::serve::history::ListPage;
1250 use $crate::serve::history::sql;
1251 let backend = |e: sqlx::Error| HistoryError::Backend(e.to_string());
1252
1253 let cursor_ts: Option<String> = match &filter.cursor {
1257 None => None,
1258 Some(c) => sqlx::query(&self.stmts.select_submitted)
1259 .bind(c)
1260 .fetch_optional(&self.pool)
1261 .await
1262 .map_err(backend)?
1263 .map(|r| r.try_get::<String, _>("submitted_at"))
1264 .transpose()
1265 .map_err(backend)?,
1266 };
1267 let cur_id = if cursor_ts.is_some() {
1268 filter.cursor.as_deref()
1269 } else {
1270 None
1271 };
1272
1273 let status_s = filter.status.map(|s| s.as_str());
1274 let name_s = filter.name.as_deref();
1275 let since_s = filter.since.map(sql::fmt_ts);
1276 let until_s = filter.until.map(sql::fmt_ts);
1277 let limit = filter.limit.max(1);
1278 let fetch_n = limit as i64 + 1; let rows = sqlx::query(&self.stmts.list)
1281 .bind(status_s)
1282 .bind(status_s)
1283 .bind(name_s)
1284 .bind(name_s)
1285 .bind(since_s.as_deref())
1286 .bind(since_s.as_deref())
1287 .bind(until_s.as_deref())
1288 .bind(until_s.as_deref())
1289 .bind(cursor_ts.as_deref())
1290 .bind(cursor_ts.as_deref())
1291 .bind(cursor_ts.as_deref())
1292 .bind(cur_id)
1293 .bind(fetch_n)
1294 .fetch_all(&self.pool)
1295 .await
1296 .map_err(backend)?;
1297
1298 let mut runs = Vec::with_capacity(rows.len());
1299 for r in &rows {
1300 let body: String = r.try_get("body").map_err(backend)?;
1301 runs.push(sql::decode_body(&body)?);
1302 }
1303 let next_cursor = if runs.len() > limit {
1304 Some(runs[limit - 1].run_id.clone())
1305 } else {
1306 None
1307 };
1308 runs.truncate(limit);
1309 Ok(ListPage { runs, next_cursor })
1310 }
1311
1312 async fn delete(
1313 &self,
1314 id: &str,
1315 ) -> Result<$crate::serve::history::DeleteOutcome, $crate::serve::history::HistoryError>
1316 {
1317 use sqlx::Row as _;
1318 use $crate::serve::history::DeleteOutcome;
1319 use $crate::serve::history::HistoryError;
1320 use $crate::serve::history::sql;
1321 let backend = |e: sqlx::Error| HistoryError::Backend(e.to_string());
1322 let status: Option<String> = sqlx::query(&self.stmts.select_status)
1323 .bind(id)
1324 .fetch_optional(&self.pool)
1325 .await
1326 .map_err(backend)?
1327 .map(|r| r.try_get::<String, _>("status"))
1328 .transpose()
1329 .map_err(backend)?;
1330 match status {
1331 None => Ok(DeleteOutcome::NotFound),
1332 Some(s) if !sql::parse_status(&s).is_terminal() => {
1333 Ok(DeleteOutcome::StillRunning)
1334 }
1335 Some(_) => {
1336 sqlx::query(&self.stmts.delete)
1337 .bind(id)
1338 .execute(&self.pool)
1339 .await
1340 .map_err(backend)?;
1341 sqlx::query(&self.stmts.delete_idem_by_run)
1347 .bind(id)
1348 .execute(&self.pool)
1349 .await
1350 .map_err(backend)?;
1351 sqlx::query(&self.stmts.delete_shards_by_run)
1355 .bind(id)
1356 .execute(&self.pool)
1357 .await
1358 .map_err(backend)?;
1359 Ok(DeleteOutcome::Deleted)
1360 }
1361 }
1362 }
1363
1364 async fn release_idempotency(
1365 &self,
1366 run_id: &str,
1367 ) -> Result<(), $crate::serve::history::HistoryError> {
1368 use $crate::serve::history::HistoryError;
1369 sqlx::query(&self.stmts.delete_idem_by_run)
1370 .bind(run_id)
1371 .execute(&self.pool)
1372 .await
1373 .map_err(|e| HistoryError::Backend(e.to_string()))?;
1374 Ok(())
1375 }
1376
1377 async fn purge_expired(
1378 &self,
1379 retain_for: std::time::Duration,
1380 ) -> Result<usize, $crate::serve::history::HistoryError> {
1381 use $crate::serve::history::HistoryError;
1382 use $crate::serve::history::sql;
1383 let backend = |e: sqlx::Error| HistoryError::Backend(e.to_string());
1384 let now = chrono::Utc::now();
1385 let removed = sqlx::query(&self.stmts.purge_runs)
1386 .bind(sql::threshold(now, retain_for))
1387 .execute(&self.pool)
1388 .await
1389 .map_err(backend)?
1390 .rows_affected() as usize;
1391 let _ = sqlx::query(&self.stmts.purge_idem)
1393 .bind(sql::threshold(now, self.idem_retention))
1394 .execute(&self.pool)
1395 .await;
1396 let _ = sqlx::query(&self.stmts.prune_instances)
1400 .bind(sql::threshold(now, retain_for))
1401 .execute(&self.pool)
1402 .await;
1403 let _ = sqlx::query(&self.stmts.purge_orphan_shards)
1407 .execute(&self.pool)
1408 .await;
1409 let _ = sqlx::query(&self.stmts.purge_audit)
1411 .bind(sql::threshold(now, retain_for))
1412 .execute(&self.pool)
1413 .await;
1414 Ok(removed)
1415 }
1416
1417 async fn recover_orphans(&self) -> Result<usize, $crate::serve::history::HistoryError> {
1418 use sqlx::Row as _;
1419 use $crate::serve::history::HistoryError;
1420 use $crate::serve::history::RunStatus;
1421 use $crate::serve::history::sql;
1422 let backend = |e: sqlx::Error| HistoryError::Backend(e.to_string());
1423 let now = chrono::Utc::now();
1424 let rows = sqlx::query(&self.stmts.select_orphans)
1429 .bind(sql::fmt_ts(now))
1430 .fetch_all(&self.pool)
1431 .await
1432 .map_err(backend)?;
1433 let mut count = 0usize;
1434 for r in &rows {
1435 let body: String = r.try_get("body").map_err(backend)?;
1436 let mut rec = sql::decode_body(&body)?;
1437 rec.status = RunStatus::Failed;
1438 rec.finished_at = Some(now);
1439 rec.error = Some(
1440 "owning serve instance's lease expired before the run finished".into(),
1441 );
1442 if rec.elapsed_secs.is_none()
1443 && let Some(started) = rec.started_at
1444 {
1445 rec.elapsed_secs = (now - started).to_std().ok().map(|d| d.as_secs_f64());
1446 }
1447 self.upsert(&rec).await?;
1448 count += 1;
1449 }
1450 Ok(count)
1451 }
1452
1453 async fn renew_leases(&self) -> Result<usize, $crate::serve::history::HistoryError> {
1454 use $crate::serve::history::HistoryError;
1455 use $crate::serve::history::sql;
1456 let backend = |e: sqlx::Error| HistoryError::Backend(e.to_string());
1457 let new_lease = sql::fmt_ts(chrono::Utc::now() + self.lease_ttl);
1458 let renewed = sqlx::query(&self.stmts.renew_leases)
1459 .bind(&new_lease)
1460 .bind(&self.instance_id)
1461 .execute(&self.pool)
1462 .await
1463 .map_err(backend)?
1464 .rows_affected() as usize;
1465 Ok(renewed)
1466 }
1467
1468 async fn claim_pending(
1469 &self,
1470 limit: usize,
1471 ) -> Result<Vec<$crate::serve::history::RunRecord>, $crate::serve::history::HistoryError>
1472 {
1473 use sqlx::Row as _;
1474 use $crate::serve::history::HistoryError;
1475 use $crate::serve::history::RunStatus;
1476 use $crate::serve::history::sql;
1477 let backend = |e: sqlx::Error| HistoryError::Backend(e.to_string());
1478 if limit == 0 {
1479 return Ok(Vec::new());
1480 }
1481 let now = chrono::Utc::now();
1482 let lease = sql::fmt_ts(now + self.lease_ttl);
1483
1484 let rows = sqlx::query(&self.stmts.select_pending)
1486 .bind(limit as i64)
1487 .fetch_all(&self.pool)
1488 .await
1489 .map_err(backend)?;
1490
1491 let mut claimed = Vec::new();
1496 for row in &rows {
1497 let run_id: String = row.try_get("run_id").map_err(backend)?;
1498 let body: String = row.try_get("body").map_err(backend)?;
1499 let mut r = sql::decode_body(&body)?;
1503 r.status = RunStatus::Running;
1504 let new_body = sql::encode_body(&r)?;
1505 let won = sqlx::query(&self.stmts.claim_one)
1507 .bind(&self.instance_id)
1508 .bind(&lease)
1509 .bind(&new_body)
1510 .bind(&run_id)
1511 .execute(&self.pool)
1512 .await
1513 .map_err(backend)?
1514 .rows_affected();
1515 if won == 1 {
1516 claimed.push(r);
1517 }
1518 }
1519 Ok(claimed)
1520 }
1521
1522 async fn reclaim_orphans(
1523 &self,
1524 max_attempts: u32,
1525 ) -> Result<$crate::serve::history::ReclaimReport, $crate::serve::history::HistoryError>
1526 {
1527 use sqlx::Row as _;
1528 use $crate::serve::history::HistoryError;
1529 use $crate::serve::history::ReclaimReport;
1530 use $crate::serve::history::RunStatus;
1531 use $crate::serve::history::sql;
1532 let backend = |e: sqlx::Error| HistoryError::Backend(e.to_string());
1533 let now = chrono::Utc::now();
1534 let now_s = sql::fmt_ts(now);
1535
1536 let rows = sqlx::query(&self.stmts.reclaim_select)
1537 .bind(&now_s)
1538 .fetch_all(&self.pool)
1539 .await
1540 .map_err(backend)?;
1541
1542 let mut report = ReclaimReport::default();
1543 for row in &rows {
1544 let body: String = row.try_get("body").map_err(backend)?;
1545 let mut rec = sql::decode_body(&body)?;
1546 let next_attempt = rec.attempt + 1;
1547 if rec.attempt < max_attempts {
1551 rec.attempt = next_attempt;
1553 rec.status = RunStatus::Pending;
1554 let new_body = sql::encode_body(&rec)?;
1555 let n = sqlx::query(&self.stmts.reclaim_requeue)
1556 .bind(&new_body)
1557 .bind(&rec.run_id)
1558 .bind(&now_s)
1559 .execute(&self.pool)
1560 .await
1561 .map_err(backend)?
1562 .rows_affected();
1563 if n == 1 {
1564 report.requeued += 1;
1565 }
1566 } else {
1567 rec.attempt = next_attempt;
1569 rec.status = RunStatus::Failed;
1570 rec.finished_at = Some(now);
1571 rec.error = Some(format!(
1572 "run reclaimed {next_attempt} times after its owning instance's \
1573 lease expired; giving up (poison run)"
1574 ));
1575 if rec.elapsed_secs.is_none()
1576 && let Some(started) = rec.started_at
1577 {
1578 rec.elapsed_secs =
1579 (now - started).to_std().ok().map(|d| d.as_secs_f64());
1580 }
1581 let new_body = sql::encode_body(&rec)?;
1582 let n = sqlx::query(&self.stmts.reclaim_fail)
1583 .bind(&now_s)
1584 .bind(&new_body)
1585 .bind(&rec.run_id)
1586 .bind(&now_s)
1587 .execute(&self.pool)
1588 .await
1589 .map_err(backend)?
1590 .rows_affected();
1591 if n == 1 {
1592 report.failed += 1;
1593 }
1594 }
1595 }
1596 Ok(report)
1597 }
1598
1599 async fn finalize_owned(
1600 &self,
1601 rec: &$crate::serve::history::RunRecord,
1602 ) -> Result<bool, $crate::serve::history::HistoryError> {
1603 use $crate::serve::history::HistoryError;
1604 use $crate::serve::history::sql;
1605 let backend = |e: sqlx::Error| HistoryError::Backend(e.to_string());
1606 let mut rec = rec.clone();
1610 if rec.status.is_terminal() && rec.finished_at.is_none() {
1611 rec.finished_at = Some(chrono::Utc::now());
1612 }
1613 let body = sql::encode_body(&rec)?;
1614 let finished = rec.finished_at.map(sql::fmt_ts);
1615 let lease = sql::fmt_ts(chrono::Utc::now() + self.lease_ttl);
1616 let n = sqlx::query(&self.stmts.finalize_owned)
1617 .bind(rec.status.as_str())
1618 .bind(finished.as_deref())
1619 .bind(&lease)
1620 .bind(&body)
1621 .bind(&rec.run_id)
1622 .bind(&self.instance_id)
1623 .execute(&self.pool)
1624 .await
1625 .map_err(backend)?
1626 .rows_affected();
1627 Ok(n == 1)
1628 }
1629
1630 async fn finalize_sharded_parent(
1631 &self,
1632 run_id: &str,
1633 status: $crate::serve::history::RunStatus,
1634 finished_at: chrono::DateTime<chrono::Utc>,
1635 error: Option<String>,
1636 ) -> Result<bool, $crate::serve::history::HistoryError> {
1637 use sqlx::Row as _;
1638 use $crate::serve::history::HistoryError;
1639 use $crate::serve::history::RunStatus;
1640 use $crate::serve::history::sql;
1641 let backend = |e: sqlx::Error| HistoryError::Backend(e.to_string());
1642 let Some(row) = sqlx::query(&self.stmts.select_body)
1647 .bind(run_id)
1648 .fetch_optional(&self.pool)
1649 .await
1650 .map_err(backend)?
1651 else {
1652 return Ok(false);
1653 };
1654 let body: String = row.try_get("body").map_err(backend)?;
1655 let mut rec = sql::decode_body(&body)?;
1656 if rec.status != RunStatus::Sharded {
1657 return Ok(false);
1658 }
1659 rec.status = status;
1660 rec.finished_at = Some(finished_at);
1661 rec.error = error;
1662 let new_body = sql::encode_body(&rec)?;
1663 let n = sqlx::query(&self.stmts.finalize_sharded_parent)
1664 .bind(status.as_str())
1665 .bind(sql::fmt_ts(finished_at))
1666 .bind(&new_body)
1667 .bind(run_id)
1668 .execute(&self.pool)
1669 .await
1670 .map_err(backend)?
1671 .rows_affected();
1672 Ok(n == 1)
1673 }
1674
1675 async fn cancel_pending(
1676 &self,
1677 run_id: &str,
1678 ) -> Result<bool, $crate::serve::history::HistoryError> {
1679 use sqlx::Row as _;
1680 use $crate::serve::history::HistoryError;
1681 use $crate::serve::history::RunStatus;
1682 use $crate::serve::history::sql;
1683 let backend = |e: sqlx::Error| HistoryError::Backend(e.to_string());
1684 let Some(row) = sqlx::query(&self.stmts.select_body)
1687 .bind(run_id)
1688 .fetch_optional(&self.pool)
1689 .await
1690 .map_err(backend)?
1691 else {
1692 return Ok(false);
1693 };
1694 let body: String = row.try_get("body").map_err(backend)?;
1695 let mut rec = sql::decode_body(&body)?;
1696 if rec.status != RunStatus::Pending {
1697 return Ok(false);
1698 }
1699 let now = chrono::Utc::now();
1700 rec.status = RunStatus::Cancelled;
1701 rec.finished_at = Some(now);
1702 let new_body = sql::encode_body(&rec)?;
1703 let n = sqlx::query(&self.stmts.cancel_pending)
1704 .bind(sql::fmt_ts(now))
1705 .bind(&new_body)
1706 .bind(run_id)
1707 .execute(&self.pool)
1708 .await
1709 .map_err(backend)?
1710 .rows_affected();
1711 Ok(n == 1)
1712 }
1713
1714 async fn request_cancel(
1715 &self,
1716 run_id: &str,
1717 ) -> Result<(), $crate::serve::history::HistoryError> {
1718 use $crate::serve::history::HistoryError;
1719 use $crate::serve::history::sql;
1720 let backend = |e: sqlx::Error| HistoryError::Backend(e.to_string());
1721 sqlx::query(&self.stmts.request_cancel)
1722 .bind(sql::fmt_ts(chrono::Utc::now()))
1723 .bind(run_id)
1724 .execute(&self.pool)
1725 .await
1726 .map_err(backend)?;
1727 Ok(())
1728 }
1729
1730 async fn pending_cancellations(
1731 &self,
1732 ) -> Result<Vec<String>, $crate::serve::history::HistoryError> {
1733 use sqlx::Row as _;
1734 use $crate::serve::history::HistoryError;
1735 let backend = |e: sqlx::Error| HistoryError::Backend(e.to_string());
1736 let rows = sqlx::query(&self.stmts.pending_cancellations)
1737 .bind(&self.instance_id)
1738 .fetch_all(&self.pool)
1739 .await
1740 .map_err(backend)?;
1741 let mut ids = Vec::with_capacity(rows.len());
1742 for r in &rows {
1743 ids.push(r.try_get::<String, _>("run_id").map_err(backend)?);
1744 }
1745 Ok(ids)
1746 }
1747
1748 async fn heartbeat_instance(
1749 &self,
1750 beat: &$crate::serve::history::InstanceHeartbeat,
1751 ) -> Result<(), $crate::serve::history::HistoryError> {
1752 use $crate::serve::history::HistoryError;
1753 use $crate::serve::history::sql;
1754 let backend = |e: sqlx::Error| HistoryError::Backend(e.to_string());
1755 let now = sql::fmt_ts(chrono::Utc::now());
1756 sqlx::query(&self.stmts.heartbeat_instance)
1757 .bind(&self.instance_id)
1758 .bind(sql::fmt_ts(beat.started_at))
1759 .bind(&now)
1760 .bind(beat.listen.as_deref())
1761 .bind(beat.max_concurrent.to_string())
1762 .bind(beat.in_flight.to_string())
1763 .execute(&self.pool)
1764 .await
1765 .map_err(backend)?;
1766 Ok(())
1767 }
1768
1769 async fn live_instances(
1770 &self,
1771 ttl: std::time::Duration,
1772 ) -> Result<Vec<$crate::serve::history::InstanceRecord>, $crate::serve::history::HistoryError>
1773 {
1774 use sqlx::Row as _;
1775 use $crate::serve::history::HistoryError;
1776 use $crate::serve::history::InstanceRecord;
1777 use $crate::serve::history::sql;
1778 let backend = |e: sqlx::Error| HistoryError::Backend(e.to_string());
1779 let now = chrono::Utc::now();
1780 let rows = sqlx::query(&self.stmts.live_instances)
1781 .bind(sql::threshold(now, ttl))
1782 .fetch_all(&self.pool)
1783 .await
1784 .map_err(backend)?;
1785 let parse_dt = |s: &str| {
1786 chrono::DateTime::parse_from_rfc3339(s)
1787 .map(|d| d.to_utc())
1788 .unwrap_or(now)
1789 };
1790 let mut out = Vec::with_capacity(rows.len());
1791 for r in &rows {
1792 let started: String = r.try_get("started_at").map_err(backend)?;
1793 let hb: String = r.try_get("last_heartbeat").map_err(backend)?;
1794 let mc: Option<String> = r.try_get("max_concurrent").map_err(backend)?;
1795 let inf: Option<String> = r.try_get("in_flight").map_err(backend)?;
1796 out.push(InstanceRecord {
1797 instance_id: r.try_get("instance_id").map_err(backend)?,
1798 started_at: parse_dt(&started),
1799 last_heartbeat: parse_dt(&hb),
1800 listen: r.try_get("listen").map_err(backend)?,
1801 max_concurrent: mc.and_then(|s| s.parse().ok()).unwrap_or(0),
1802 in_flight: inf.and_then(|s| s.parse().ok()).unwrap_or(0),
1803 });
1804 }
1805 Ok(out)
1806 }
1807
1808 async fn insert_shards(
1811 &self,
1812 run_id: &str,
1813 shards: &[$crate::serve::history::ShardInsert],
1814 ) -> Result<usize, $crate::serve::history::HistoryError> {
1815 use $crate::serve::history::HistoryError;
1816 let backend = |e: sqlx::Error| HistoryError::Backend(e.to_string());
1817 let mut inserted = 0usize;
1818 for s in shards {
1819 let descriptor = serde_json::to_string(&s.descriptor).map_err(|e| {
1820 HistoryError::Backend(format!("encode shard descriptor: {e}"))
1821 })?;
1822 let size = s.size_estimate.map(|n| n.to_string());
1823 let n = sqlx::query(&self.stmts.insert_shard)
1824 .bind(run_id)
1825 .bind(&s.shard_id)
1826 .bind(&descriptor)
1827 .bind(size.as_deref())
1828 .execute(&self.pool)
1829 .await
1830 .map_err(backend)?
1831 .rows_affected();
1832 inserted += n as usize;
1833 }
1834 Ok(inserted)
1835 }
1836
1837 async fn claim_shards(
1838 &self,
1839 limit: usize,
1840 ) -> Result<
1841 Vec<$crate::serve::history::ClaimedShard>,
1842 $crate::serve::history::HistoryError,
1843 > {
1844 use sqlx::Row as _;
1845 use $crate::serve::history::ClaimedShard;
1846 use $crate::serve::history::HistoryError;
1847 use $crate::serve::history::sql;
1848 let backend = |e: sqlx::Error| HistoryError::Backend(e.to_string());
1849 if limit == 0 {
1850 return Ok(Vec::new());
1851 }
1852 let lease = sql::fmt_ts(chrono::Utc::now() + self.lease_ttl);
1853
1854 let rows = sqlx::query(&self.stmts.claim_shards_select)
1857 .bind(limit as i64)
1858 .fetch_all(&self.pool)
1859 .await
1860 .map_err(backend)?;
1861
1862 let mut claimed = Vec::new();
1864 for row in &rows {
1865 let run_id: String = row.try_get("run_id").map_err(backend)?;
1866 let shard_id: String = row.try_get("shard_id").map_err(backend)?;
1867 let descriptor_s: String = row.try_get("descriptor").map_err(backend)?;
1868 let body: String = row.try_get("body").map_err(backend)?;
1869 let won = sqlx::query(&self.stmts.claim_shard_one)
1870 .bind(&self.instance_id)
1871 .bind(&lease)
1872 .bind(&run_id)
1873 .bind(&shard_id)
1874 .execute(&self.pool)
1875 .await
1876 .map_err(backend)?
1877 .rows_affected();
1878 if won == 1 {
1879 let descriptor: serde_json::Value = serde_json::from_str(&descriptor_s)
1880 .map_err(|e| {
1881 HistoryError::Backend(format!("decode shard descriptor: {e}"))
1882 })?;
1883 let run = sql::decode_body(&body)?;
1884 claimed.push(ClaimedShard {
1885 run_id,
1886 shard_id,
1887 descriptor,
1888 run,
1889 });
1890 }
1891 }
1892 Ok(claimed)
1893 }
1894
1895 async fn renew_shard_leases(
1896 &self,
1897 ) -> Result<usize, $crate::serve::history::HistoryError> {
1898 use $crate::serve::history::HistoryError;
1899 use $crate::serve::history::sql;
1900 let backend = |e: sqlx::Error| HistoryError::Backend(e.to_string());
1901 let lease = sql::fmt_ts(chrono::Utc::now() + self.lease_ttl);
1902 let n = sqlx::query(&self.stmts.renew_shard_leases)
1903 .bind(&lease)
1904 .bind(&self.instance_id)
1905 .execute(&self.pool)
1906 .await
1907 .map_err(backend)?
1908 .rows_affected() as usize;
1909 Ok(n)
1910 }
1911
1912 async fn reclaim_shards(
1913 &self,
1914 max_attempts: u32,
1915 ) -> Result<$crate::serve::history::ReclaimReport, $crate::serve::history::HistoryError>
1916 {
1917 use sqlx::Row as _;
1918 use $crate::serve::history::HistoryError;
1919 use $crate::serve::history::ReclaimReport;
1920 use $crate::serve::history::sql;
1921 let backend = |e: sqlx::Error| HistoryError::Backend(e.to_string());
1922 let now_s = sql::fmt_ts(chrono::Utc::now());
1923
1924 let rows = sqlx::query(&self.stmts.reclaim_shards_select)
1925 .bind(&now_s)
1926 .fetch_all(&self.pool)
1927 .await
1928 .map_err(backend)?;
1929
1930 let mut report = ReclaimReport::default();
1931 for row in &rows {
1932 let run_id: String = row.try_get("run_id").map_err(backend)?;
1933 let shard_id: String = row.try_get("shard_id").map_err(backend)?;
1934 let attempt_s: String = row.try_get("attempt").map_err(backend)?;
1935 let attempt: u32 = attempt_s.parse().unwrap_or(0);
1936 if attempt < max_attempts {
1937 let next = (attempt + 1).to_string();
1938 let n = sqlx::query(&self.stmts.reclaim_shard_requeue)
1939 .bind(&next)
1940 .bind(&run_id)
1941 .bind(&shard_id)
1942 .bind(&now_s)
1943 .execute(&self.pool)
1944 .await
1945 .map_err(backend)?
1946 .rows_affected();
1947 if n == 1 {
1948 report.requeued += 1;
1949 }
1950 } else {
1951 let n = sqlx::query(&self.stmts.reclaim_shard_fail)
1952 .bind(&now_s)
1953 .bind(&run_id)
1954 .bind(&shard_id)
1955 .bind(&now_s)
1956 .execute(&self.pool)
1957 .await
1958 .map_err(backend)?
1959 .rows_affected();
1960 if n == 1 {
1961 report.failed += 1;
1962 }
1963 }
1964 }
1965 Ok(report)
1966 }
1967
1968 async fn finalize_shard(
1969 &self,
1970 run_id: &str,
1971 shard_id: &str,
1972 success: bool,
1973 ) -> Result<bool, $crate::serve::history::HistoryError> {
1974 use $crate::serve::history::HistoryError;
1975 use $crate::serve::history::sql;
1976 let backend = |e: sqlx::Error| HistoryError::Backend(e.to_string());
1977 let status = if success { "completed" } else { "failed" };
1978 let now_s = sql::fmt_ts(chrono::Utc::now());
1979 let n = sqlx::query(&self.stmts.finalize_shard)
1980 .bind(status)
1981 .bind(&now_s)
1982 .bind(run_id)
1983 .bind(shard_id)
1984 .bind(&self.instance_id)
1985 .execute(&self.pool)
1986 .await
1987 .map_err(backend)?
1988 .rows_affected();
1989 Ok(n == 1)
1990 }
1991
1992 async fn shard_progress(
1993 &self,
1994 run_id: &str,
1995 ) -> Result<$crate::serve::history::ShardProgress, $crate::serve::history::HistoryError>
1996 {
1997 use sqlx::Row as _;
1998 use $crate::serve::history::HistoryError;
1999 use $crate::serve::history::ShardProgress;
2000 let backend = |e: sqlx::Error| HistoryError::Backend(e.to_string());
2001 let rows = sqlx::query(&self.stmts.shard_progress)
2002 .bind(run_id)
2003 .fetch_all(&self.pool)
2004 .await
2005 .map_err(backend)?;
2006 let mut p = ShardProgress::default();
2007 for row in &rows {
2008 let status: String = row.try_get("status").map_err(backend)?;
2009 let n: i64 = row.try_get("n").map_err(backend)?;
2010 let n = n.max(0) as usize;
2011 p.total += n;
2012 match status.as_str() {
2013 "completed" => p.completed += n,
2014 "failed" => p.failed += n,
2015 "running" => p.running += n,
2016 _ => p.pending += n,
2017 }
2018 }
2019 Ok(p)
2020 }
2021
2022 async fn pending_shard_cancellations(
2023 &self,
2024 ) -> Result<Vec<String>, $crate::serve::history::HistoryError> {
2025 use sqlx::Row as _;
2026 use $crate::serve::history::HistoryError;
2027 let backend = |e: sqlx::Error| HistoryError::Backend(e.to_string());
2028 let rows = sqlx::query(&self.stmts.pending_shard_cancellations)
2029 .bind(&self.instance_id)
2030 .fetch_all(&self.pool)
2031 .await
2032 .map_err(backend)?;
2033 let mut ids = Vec::with_capacity(rows.len());
2034 for r in &rows {
2035 ids.push(r.try_get::<String, _>("run_id").map_err(backend)?);
2036 }
2037 Ok(ids)
2038 }
2039
2040 async fn finalize_completed_sharded_parents(
2041 &self,
2042 ) -> Result<usize, $crate::serve::history::HistoryError> {
2043 use sqlx::Row as _;
2044 use $crate::serve::history::HistoryError;
2045 use $crate::serve::history::RunStatus;
2046 use $crate::serve::history::sql;
2047 let backend = |e: sqlx::Error| HistoryError::Backend(e.to_string());
2048
2049 let rows = sqlx::query(&self.stmts.select_sharded_parents)
2053 .fetch_all(&self.pool)
2054 .await
2055 .map_err(backend)?;
2056
2057 let mut finalized = 0usize;
2058 for row in &rows {
2059 let run_id: String = row.try_get("run_id").map_err(backend)?;
2060 let progress = self.shard_progress(&run_id).await?;
2061 if !progress.all_terminal() {
2062 continue;
2063 }
2064 let success = progress.failed == 0;
2065 let Some(body_row) = sqlx::query(&self.stmts.select_body)
2068 .bind(&run_id)
2069 .fetch_optional(&self.pool)
2070 .await
2071 .map_err(backend)?
2072 else {
2073 continue;
2074 };
2075 let body: String = body_row.try_get("body").map_err(backend)?;
2076 let mut rec = sql::decode_body(&body)?;
2077 if rec.status != RunStatus::Sharded {
2080 continue;
2081 }
2082 let now = chrono::Utc::now();
2083 rec.status = if success {
2084 RunStatus::Completed
2085 } else {
2086 RunStatus::Failed
2087 };
2088 rec.finished_at = Some(now);
2089 if !success {
2090 rec.error = Some(format!(
2091 "{}/{} shard(s) failed",
2092 progress.failed, progress.total
2093 ));
2094 }
2095 let new_body = sql::encode_body(&rec)?;
2096 let n = sqlx::query(&self.stmts.finalize_sharded_parent)
2097 .bind(rec.status.as_str())
2098 .bind(sql::fmt_ts(now))
2099 .bind(&new_body)
2100 .bind(&run_id)
2101 .execute(&self.pool)
2102 .await
2103 .map_err(backend)?
2104 .rows_affected();
2105 if n == 1 {
2106 finalized += 1;
2107 $crate::serve::metrics::record_run_finished(
2108 rec.status,
2109 if success { "ok" } else { "error" },
2110 );
2111 tracing::info!(
2112 run_id,
2113 shards = progress.total,
2114 failed = progress.failed,
2115 "sharded run finalized by sweep (F11)"
2116 );
2117 }
2118 }
2119 Ok(finalized)
2120 }
2121
2122 async fn record_audit(
2125 &self,
2126 entry: &$crate::serve::history::AuditEntry,
2127 ) -> Result<(), $crate::serve::history::HistoryError> {
2128 use $crate::serve::history::HistoryError;
2129 use $crate::serve::history::sql;
2130 let backend = |e: sqlx::Error| HistoryError::Backend(e.to_string());
2131 sqlx::query(&self.stmts.insert_audit)
2132 .bind(&entry.id)
2133 .bind(sql::fmt_ts(entry.timestamp))
2134 .bind(&entry.principal)
2135 .bind(&entry.role)
2136 .bind(&entry.action)
2137 .bind(entry.run_id.as_deref())
2138 .bind(entry.config_fingerprint.as_deref())
2139 .bind(entry.source_ip.as_deref())
2140 .bind(&entry.result)
2141 .execute(&self.pool)
2142 .await
2143 .map_err(backend)?;
2144 Ok(())
2145 }
2146
2147 async fn list_audit(
2148 &self,
2149 filter: &$crate::serve::history::AuditFilter,
2150 ) -> Result<
2151 Vec<$crate::serve::history::AuditEntry>,
2152 $crate::serve::history::HistoryError,
2153 > {
2154 use sqlx::Row as _;
2155 use $crate::serve::history::AuditEntry;
2156 use $crate::serve::history::HistoryError;
2157 use $crate::serve::history::sql;
2158 let backend = |e: sqlx::Error| HistoryError::Backend(e.to_string());
2159 let principal = filter.principal.as_deref();
2160 let action = filter.action.as_deref();
2161 let since = filter.since.map(sql::fmt_ts);
2162 let until = filter.until.map(sql::fmt_ts);
2163 let limit = filter.limit.max(1) as i64;
2164 let rows = sqlx::query(&self.stmts.list_audit)
2165 .bind(principal)
2166 .bind(principal)
2167 .bind(action)
2168 .bind(action)
2169 .bind(since.as_deref())
2170 .bind(since.as_deref())
2171 .bind(until.as_deref())
2172 .bind(until.as_deref())
2173 .bind(limit)
2174 .fetch_all(&self.pool)
2175 .await
2176 .map_err(backend)?;
2177 let mut out = Vec::with_capacity(rows.len());
2178 for r in &rows {
2179 let ts: String = r.try_get("ts").map_err(backend)?;
2180 let timestamp = $crate::serve::history::sql::parse_ts(&ts);
2181 out.push(AuditEntry {
2182 id: r.try_get("id").map_err(backend)?,
2183 timestamp,
2184 principal: r.try_get("principal").map_err(backend)?,
2185 role: r.try_get("role").map_err(backend)?,
2186 action: r.try_get("action").map_err(backend)?,
2187 run_id: r.try_get("run_id").map_err(backend)?,
2188 config_fingerprint: r.try_get("config_fingerprint").map_err(backend)?,
2189 source_ip: r.try_get("source_ip").map_err(backend)?,
2190 result: r.try_get("result").map_err(backend)?,
2191 });
2192 }
2193 Ok(out)
2194 }
2195
2196 async fn record_run_logs(
2199 &self,
2200 run_id: &str,
2201 lines: &[$crate::serve::history::RunLogLine],
2202 ) -> Result<(), $crate::serve::history::HistoryError> {
2203 use $crate::serve::history::HistoryError;
2204 use $crate::serve::history::sql;
2205 if lines.is_empty() {
2206 return Ok(());
2207 }
2208 let backend = |e: sqlx::Error| HistoryError::Backend(e.to_string());
2209 let mut tx = self.pool.begin().await.map_err(backend)?;
2210 for l in lines {
2211 sqlx::query(&self.stmts.insert_run_log)
2212 .bind(run_id)
2213 .bind(sql::pad_seq(l.seq))
2214 .bind(&l.ts)
2215 .bind(&l.level)
2216 .bind(&l.line)
2217 .execute(&mut *tx)
2218 .await
2219 .map_err(backend)?;
2220 }
2221 tx.commit().await.map_err(backend)?;
2222 Ok(())
2223 }
2224
2225 async fn list_run_logs(
2226 &self,
2227 run_id: &str,
2228 after_seq: Option<u64>,
2229 limit: usize,
2230 ) -> Result<
2231 $crate::serve::history::RunLogPage,
2232 $crate::serve::history::HistoryError,
2233 > {
2234 use sqlx::Row as _;
2235 use $crate::serve::history::HistoryError;
2236 use $crate::serve::history::sql;
2237 use $crate::serve::history::{RunLogLine, RunLogPage, RUN_LOG_TRUNCATED_SEQ};
2238 let backend = |e: sqlx::Error| HistoryError::Backend(e.to_string());
2239 let sentinel = sql::pad_seq(RUN_LOG_TRUNCATED_SEQ);
2240 let after = after_seq.map(sql::pad_seq);
2241 let limit = limit.max(1) as i64;
2242 let rows = sqlx::query(&self.stmts.list_run_logs)
2243 .bind(run_id)
2244 .bind(&sentinel)
2245 .bind(after.as_deref())
2246 .bind(after.as_deref())
2247 .bind(limit)
2248 .fetch_all(&self.pool)
2249 .await
2250 .map_err(backend)?;
2251 let mut out = Vec::with_capacity(rows.len());
2252 for r in &rows {
2253 let seq: String = r.try_get("seq").map_err(backend)?;
2254 out.push(RunLogLine {
2255 seq: sql::unpad_seq(&seq),
2256 ts: r.try_get("ts").map_err(backend)?,
2257 level: r.try_get("level").map_err(backend)?,
2258 line: r.try_get("line").map_err(backend)?,
2259 });
2260 }
2261 let truncated = sqlx::query(&self.stmts.run_log_truncated)
2262 .bind(run_id)
2263 .bind(&sentinel)
2264 .fetch_optional(&self.pool)
2265 .await
2266 .map_err(backend)?
2267 .is_some();
2268 Ok(RunLogPage { lines: out, truncated })
2269 }
2270
2271 async fn purge_run_logs(
2272 &self,
2273 older_than: std::time::Duration,
2274 ) -> Result<usize, $crate::serve::history::HistoryError> {
2275 use $crate::serve::history::HistoryError;
2276 use $crate::serve::history::sql;
2277 let backend = |e: sqlx::Error| HistoryError::Backend(e.to_string());
2278 let cutoff = sql::threshold(chrono::Utc::now(), older_than);
2279 let res = sqlx::query(&self.stmts.purge_run_logs)
2280 .bind(cutoff)
2281 .execute(&self.pool)
2282 .await
2283 .map_err(backend)?;
2284 Ok(res.rows_affected() as usize)
2285 }
2286
2287 async fn catalog_record(
2290 &self,
2291 update: &$crate::serve::history::catalog::CatalogUpdate,
2292 ) -> Result<(), $crate::serve::history::HistoryError> {
2293 use sqlx::Row as _;
2294 use $crate::serve::history::HistoryError;
2295 use $crate::serve::history::catalog;
2296 use $crate::serve::history::sql;
2297 let backend = |e: sqlx::Error| HistoryError::Backend(e.to_string());
2298 let now_s = sql::fmt_ts(update.recorded_at);
2299
2300 for obs in update.sources.iter().chain(std::iter::once(&update.sink)) {
2301 let id = catalog::dataset_id(&obs.uri);
2302 let existing = sqlx::query(&self.stmts.catalog_select_dataset)
2306 .bind(&id)
2307 .fetch_optional(&self.pool)
2308 .await
2309 .map_err(backend)?
2310 .map(|r| r.try_get::<String, _>("body"))
2311 .transpose()
2312 .map_err(backend)?
2313 .map(|b| {
2314 sql::decode_json::<catalog::CatalogDataset>(&b, "catalog dataset")
2315 })
2316 .transpose()?;
2317 let (ds, new_version) = catalog::apply_observation(
2318 existing.as_ref(),
2319 obs,
2320 &update.run_id,
2321 &update.pipeline,
2322 &update.row,
2323 update.recorded_at,
2324 );
2325 sqlx::query(&self.stmts.catalog_upsert_dataset)
2326 .bind(&ds.id)
2327 .bind(&ds.uri)
2328 .bind(&ds.kind)
2329 .bind(&now_s)
2330 .bind(sql::encode_json(&ds, "catalog dataset")?)
2331 .execute(&self.pool)
2332 .await
2333 .map_err(backend)?;
2334 if let Some(v) = new_version {
2335 sqlx::query(&self.stmts.catalog_insert_schema_version)
2336 .bind(&v.dataset_id)
2337 .bind(v.version.to_string())
2338 .bind(sql::fmt_ts(v.recorded_at))
2339 .bind(sql::encode_json(&v, "catalog schema version")?)
2340 .execute(&self.pool)
2341 .await
2342 .map_err(backend)?;
2343 }
2344 sqlx::query(&self.stmts.catalog_insert_stat)
2345 .bind(&id)
2346 .bind(&now_s)
2347 .bind(&update.run_id)
2348 .bind(obs.records.to_string())
2349 .execute(&self.pool)
2350 .await
2351 .map_err(backend)?;
2352 sqlx::query(&self.stmts.catalog_prune_stats)
2353 .bind(&id)
2354 .bind(&id)
2355 .bind(catalog::STATS_RETAIN as i64)
2356 .execute(&self.pool)
2357 .await
2358 .map_err(backend)?;
2359 }
2360
2361 let dst_id = catalog::dataset_id(&update.sink.uri);
2364 let existing_edges = self.catalog_all_edges().await?;
2365 for source in &update.sources {
2366 let src_id = catalog::dataset_id(&source.uri);
2367 let existing = existing_edges
2368 .iter()
2369 .find(|e| e.src_id == src_id && e.dst_id == dst_id);
2370 let edge = catalog::apply_edge(existing, update, source);
2371 sqlx::query(&self.stmts.catalog_upsert_edge)
2372 .bind(&edge.src_id)
2373 .bind(&edge.dst_id)
2374 .bind(&now_s)
2375 .bind(sql::encode_json(&edge, "catalog edge")?)
2376 .execute(&self.pool)
2377 .await
2378 .map_err(backend)?;
2379 }
2380 Ok(())
2381 }
2382
2383 async fn catalog_list_datasets(
2384 &self,
2385 filter: &$crate::serve::history::catalog::CatalogListFilter,
2386 ) -> Result<
2387 $crate::serve::history::catalog::CatalogDatasetPage,
2388 $crate::serve::history::HistoryError,
2389 > {
2390 use sqlx::Row as _;
2391 use $crate::serve::history::HistoryError;
2392 use $crate::serve::history::catalog;
2393 use $crate::serve::history::sql;
2394 let backend = |e: sqlx::Error| HistoryError::Backend(e.to_string());
2395 let rows = sqlx::query(&self.stmts.catalog_select_datasets)
2396 .fetch_all(&self.pool)
2397 .await
2398 .map_err(backend)?;
2399 let mut all = Vec::with_capacity(rows.len());
2400 for r in &rows {
2401 let body: String = r.try_get("body").map_err(backend)?;
2402 all.push(sql::decode_json(&body, "catalog dataset")?);
2403 }
2404 Ok(catalog::filter_datasets(all, filter))
2405 }
2406
2407 async fn catalog_get_dataset(
2408 &self,
2409 id: &str,
2410 ) -> Result<
2411 Option<$crate::serve::history::catalog::CatalogDatasetDetail>,
2412 $crate::serve::history::HistoryError,
2413 > {
2414 use sqlx::Row as _;
2415 use $crate::serve::history::HistoryError;
2416 use $crate::serve::history::catalog;
2417 use $crate::serve::history::sql;
2418 let backend = |e: sqlx::Error| HistoryError::Backend(e.to_string());
2419 let Some(row) = sqlx::query(&self.stmts.catalog_select_dataset)
2420 .bind(id)
2421 .fetch_optional(&self.pool)
2422 .await
2423 .map_err(backend)?
2424 else {
2425 return Ok(None);
2426 };
2427 let body: String = row.try_get("body").map_err(backend)?;
2428 let dataset: catalog::CatalogDataset =
2429 sql::decode_json(&body, "catalog dataset")?;
2430
2431 let rows = sqlx::query(&self.stmts.catalog_select_schema_versions)
2432 .bind(id)
2433 .fetch_all(&self.pool)
2434 .await
2435 .map_err(backend)?;
2436 let mut schema_timeline = Vec::with_capacity(rows.len());
2437 for r in &rows {
2438 let body: String = r.try_get("body").map_err(backend)?;
2439 schema_timeline.push(sql::decode_json(&body, "catalog schema version")?);
2440 }
2441
2442 let rows = sqlx::query(&self.stmts.catalog_select_stats)
2443 .bind(id)
2444 .bind(catalog::STATS_DETAIL_LIMIT as i64)
2445 .fetch_all(&self.pool)
2446 .await
2447 .map_err(backend)?;
2448 let mut stats = Vec::with_capacity(rows.len());
2449 for r in &rows {
2450 let recorded: String = r.try_get("recorded_at").map_err(backend)?;
2451 let run_id: String = r.try_get("run_id").map_err(backend)?;
2452 let records: String = r.try_get("records").map_err(backend)?;
2453 stats.push(catalog::CatalogStatsPoint {
2454 recorded_at: chrono::DateTime::parse_from_rfc3339(&recorded)
2455 .map(|d| d.to_utc())
2456 .unwrap_or_else(|_| chrono::Utc::now()),
2457 run_id,
2458 records: records.parse().unwrap_or(0),
2459 });
2460 }
2461
2462 let edges = self.catalog_all_edges().await?;
2463 let (downstream, rest): (Vec<_>, Vec<_>) =
2464 edges.into_iter().partition(|e| e.src_id == id);
2465 let upstream = rest.into_iter().filter(|e| e.dst_id == id).collect();
2466 Ok(Some(catalog::CatalogDatasetDetail {
2467 dataset,
2468 schema_timeline,
2469 stats,
2470 upstream,
2471 downstream,
2472 }))
2473 }
2474
2475 async fn catalog_lineage(
2476 &self,
2477 root: Option<&str>,
2478 depth: u32,
2479 ) -> Result<
2480 Vec<$crate::serve::history::catalog::CatalogLineageEdge>,
2481 $crate::serve::history::HistoryError,
2482 > {
2483 use $crate::serve::history::catalog;
2484 let edges = self.catalog_all_edges().await?;
2485 Ok(catalog::lineage_slice(edges, root, depth))
2486 }
2487
2488 async fn catalog_record_config_snapshot(
2489 &self,
2490 snapshot: &$crate::serve::history::catalog::ConfigSnapshot,
2491 ) -> Result<(), $crate::serve::history::HistoryError> {
2492 use $crate::serve::history::HistoryError;
2493 use $crate::serve::history::sql;
2494 let backend = |e: sqlx::Error| HistoryError::Backend(e.to_string());
2495 sqlx::query(&self.stmts.catalog_upsert_config_snapshot)
2496 .bind(&snapshot.pipeline)
2497 .bind(sql::fmt_ts(snapshot.recorded_at))
2498 .bind(&snapshot.faucet_version)
2499 .bind(sql::encode_json(snapshot, "config snapshot")?)
2500 .execute(&self.pool)
2501 .await
2502 .map_err(backend)?;
2503 Ok(())
2504 }
2505
2506 async fn catalog_last_config_snapshot(
2507 &self,
2508 pipeline: &str,
2509 ) -> Result<
2510 Option<$crate::serve::history::catalog::ConfigSnapshot>,
2511 $crate::serve::history::HistoryError,
2512 > {
2513 use sqlx::Row as _;
2514 use $crate::serve::history::HistoryError;
2515 use $crate::serve::history::sql;
2516 let backend = |e: sqlx::Error| HistoryError::Backend(e.to_string());
2517 let Some(row) = sqlx::query(&self.stmts.catalog_select_config_snapshot)
2518 .bind(pipeline)
2519 .fetch_optional(&self.pool)
2520 .await
2521 .map_err(backend)?
2522 else {
2523 return Ok(None);
2524 };
2525 let body: String = row.try_get("body").map_err(backend)?;
2526 Ok(Some(sql::decode_json(&body, "config snapshot")?))
2527 }
2528
2529 async fn template_register(
2532 &self,
2533 draft: &$crate::serve::history::templates::TemplateDraft,
2534 ) -> Result<
2535 $crate::serve::history::templates::TemplateRecord,
2536 $crate::serve::history::HistoryError,
2537 > {
2538 use sqlx::Row as _;
2539 use $crate::serve::history::HistoryError;
2540 use $crate::serve::history::{sql, templates};
2541 let backend = |e: sqlx::Error| HistoryError::Backend(e.to_string());
2542 let id = draft.id.to_string();
2543
2544 for attempt in 1..=sql::CLAIM_ATTEMPTS {
2550 sql::retry_backoff(attempt).await;
2551 let mut tx = match self.pool.begin().await {
2557 Ok(tx) => tx,
2558 Err(e) if attempt < sql::CLAIM_ATTEMPTS => {
2559 tracing::debug!(
2560 template = %id, attempt, error = %e,
2561 "template version transaction lost a race; retrying"
2562 );
2563 continue;
2564 }
2565 Err(e) => return Err(backend(e)),
2566 };
2567 let row = match sqlx::query(&self.stmts.template_max_version)
2568 .bind(&id)
2569 .fetch_one(&mut *tx)
2570 .await
2571 {
2572 Ok(row) => row,
2573 Err(e) if attempt < sql::CLAIM_ATTEMPTS => {
2574 let _ = tx.rollback().await;
2575 tracing::debug!(
2576 template = %id, attempt, error = %e,
2577 "template version read lost a race; retrying"
2578 );
2579 continue;
2580 }
2581 Err(e) => {
2582 let _ = tx.rollback().await;
2583 return Err(backend(e));
2584 }
2585 };
2586 let max: i64 = row.try_get("v").map_err(backend)?;
2587 let next = (max as u32).saturating_add(1);
2588 let record = templates::TemplateRecord {
2589 id: id.clone(),
2590 version: next,
2591 name: draft.name.clone(),
2592 description: draft.description.clone(),
2593 body: draft.body.clone(),
2594 format: draft.format,
2595 params: draft.params.clone(),
2596 created_at: chrono::Utc::now(),
2597 created_by: draft.created_by.clone(),
2598 };
2599 let insert = sqlx::query(&self.stmts.template_insert)
2600 .bind(&id)
2601 .bind(next.to_string())
2602 .bind(&record.name)
2603 .bind(sql::fmt_ts(record.created_at))
2604 .bind(sql::encode_json(&record, "pipeline template")?)
2605 .execute(&mut *tx)
2606 .await;
2607 match insert {
2608 Ok(_) => {
2609 tx.commit().await.map_err(backend)?;
2610 let keep = self.template_versions(&id).await?;
2613 for stale in templates::versions_to_prune(keep) {
2614 let _ = sqlx::query(&self.stmts.template_delete_version)
2615 .bind(&id)
2616 .bind(stale.to_string())
2617 .execute(&self.pool)
2618 .await;
2619 }
2620 return Ok(record);
2621 }
2622 Err(e) if attempt < sql::CLAIM_ATTEMPTS => {
2623 let _ = tx.rollback().await;
2624 tracing::debug!(
2625 template = %id, attempt, error = %e,
2626 "template version insert lost a race; retrying with the next version"
2627 );
2628 }
2629 Err(e) => {
2630 let _ = tx.rollback().await;
2631 return Err(backend(e));
2632 }
2633 }
2634 }
2635 Err(HistoryError::Backend(format!(
2636 "could not assign a version for template '{id}' after {} attempts",
2637 sql::CLAIM_ATTEMPTS
2638 )))
2639 }
2640
2641 async fn template_get(
2642 &self,
2643 id: &str,
2644 version: Option<u32>,
2645 ) -> Result<
2646 Option<$crate::serve::history::templates::TemplateRecord>,
2647 $crate::serve::history::HistoryError,
2648 > {
2649 use sqlx::Row as _;
2650 use $crate::serve::history::HistoryError;
2651 use $crate::serve::history::sql;
2652 let backend = |e: sqlx::Error| HistoryError::Backend(e.to_string());
2653 let row = match version {
2654 Some(v) => sqlx::query(&self.stmts.template_select_version)
2655 .bind(id)
2656 .bind(v.to_string())
2657 .fetch_optional(&self.pool)
2658 .await,
2659 None => sqlx::query(&self.stmts.template_select_latest)
2660 .bind(id)
2661 .fetch_optional(&self.pool)
2662 .await,
2663 }
2664 .map_err(backend)?;
2665 let Some(row) = row else {
2666 return Ok(None);
2667 };
2668 let body: String = row.try_get("body").map_err(backend)?;
2669 Ok(Some(sql::decode_json(&body, "pipeline template")?))
2670 }
2671
2672 async fn template_list(
2673 &self,
2674 ) -> Result<
2675 Vec<$crate::serve::history::templates::TemplateSummary>,
2676 $crate::serve::history::HistoryError,
2677 > {
2678 use sqlx::Row as _;
2679 use $crate::serve::history::HistoryError;
2680 use $crate::serve::history::{sql, templates};
2681 let backend = |e: sqlx::Error| HistoryError::Backend(e.to_string());
2682 let rows = sqlx::query(&self.stmts.template_select_all)
2683 .fetch_all(&self.pool)
2684 .await
2685 .map_err(backend)?;
2686 let mut all = Vec::with_capacity(rows.len());
2687 for r in &rows {
2688 let body: String = r.try_get("body").map_err(backend)?;
2689 all.push(sql::decode_json(&body, "pipeline template")?);
2690 }
2691 Ok(templates::latest_per_id(all))
2692 }
2693
2694 async fn template_versions(
2695 &self,
2696 id: &str,
2697 ) -> Result<Vec<u32>, $crate::serve::history::HistoryError> {
2698 use sqlx::Row as _;
2699 use $crate::serve::history::HistoryError;
2700 let backend = |e: sqlx::Error| HistoryError::Backend(e.to_string());
2701 let rows = sqlx::query(&self.stmts.template_versions)
2702 .bind(id)
2703 .fetch_all(&self.pool)
2704 .await
2705 .map_err(backend)?;
2706 let mut out = Vec::with_capacity(rows.len());
2707 for r in &rows {
2708 let v: String = r.try_get("version").map_err(backend)?;
2709 if let Ok(n) = v.parse::<u32>() {
2710 out.push(n);
2711 }
2712 }
2713 Ok(out)
2714 }
2715
2716 async fn template_delete(
2717 &self,
2718 id: &str,
2719 version: Option<u32>,
2720 ) -> Result<usize, $crate::serve::history::HistoryError> {
2721 use $crate::serve::history::HistoryError;
2722 let backend = |e: sqlx::Error| HistoryError::Backend(e.to_string());
2723 let result = match version {
2724 Some(v) => {
2725 sqlx::query(&self.stmts.template_delete_tags_for_version)
2728 .bind(id)
2729 .bind(v.to_string())
2730 .execute(&self.pool)
2731 .await
2732 .map_err(backend)?;
2733 sqlx::query(&self.stmts.template_delete_launches_for_version)
2734 .bind(id)
2735 .bind(v.to_string())
2736 .execute(&self.pool)
2737 .await
2738 .map_err(backend)?;
2739 sqlx::query(&self.stmts.template_delete_version)
2740 .bind(id)
2741 .bind(v.to_string())
2742 .execute(&self.pool)
2743 .await
2744 }
2745 None => {
2746 for stmt in [
2747 &self.stmts.template_delete_tags_all,
2748 &self.stmts.template_delete_launches_all,
2749 &self.stmts.template_delete_deprecation,
2750 ] {
2751 sqlx::query(stmt)
2752 .bind(id)
2753 .execute(&self.pool)
2754 .await
2755 .map_err(backend)?;
2756 }
2757 sqlx::query(&self.stmts.template_delete_all)
2758 .bind(id)
2759 .execute(&self.pool)
2760 .await
2761 }
2762 }
2763 .map_err(backend)?;
2764 Ok(result.rows_affected() as usize)
2765 }
2766
2767 async fn template_set_tag(
2768 &self,
2769 id: &str,
2770 tag: &str,
2771 version: u32,
2772 ) -> Result<(), $crate::serve::history::HistoryError> {
2773 use $crate::serve::history::HistoryError;
2774 use $crate::serve::history::sql;
2775 let backend = |e: sqlx::Error| HistoryError::Backend(e.to_string());
2776 sqlx::query(&self.stmts.template_upsert_tag)
2777 .bind(id)
2778 .bind(tag)
2779 .bind(version.to_string())
2780 .bind(sql::fmt_ts(chrono::Utc::now()))
2781 .execute(&self.pool)
2782 .await
2783 .map_err(backend)?;
2784 Ok(())
2785 }
2786
2787 async fn template_tags(
2788 &self,
2789 id: &str,
2790 ) -> Result<
2791 std::collections::BTreeMap<String, u32>,
2792 $crate::serve::history::HistoryError,
2793 > {
2794 use sqlx::Row as _;
2795 use $crate::serve::history::HistoryError;
2796 let backend = |e: sqlx::Error| HistoryError::Backend(e.to_string());
2797 let rows = sqlx::query(&self.stmts.template_select_tags)
2798 .bind(id)
2799 .fetch_all(&self.pool)
2800 .await
2801 .map_err(backend)?;
2802 let mut out = std::collections::BTreeMap::new();
2803 for r in &rows {
2804 let tag: String = r.try_get("tag").map_err(backend)?;
2805 let version: String = r.try_get("version").map_err(backend)?;
2806 if let Ok(n) = version.parse::<u32>() {
2807 out.insert(tag, n);
2808 }
2809 }
2810 Ok(out)
2811 }
2812
2813 async fn template_delete_tag(
2814 &self,
2815 id: &str,
2816 tag: &str,
2817 ) -> Result<bool, $crate::serve::history::HistoryError> {
2818 use $crate::serve::history::HistoryError;
2819 let backend = |e: sqlx::Error| HistoryError::Backend(e.to_string());
2820 let result = sqlx::query(&self.stmts.template_delete_tag)
2821 .bind(id)
2822 .bind(tag)
2823 .execute(&self.pool)
2824 .await
2825 .map_err(backend)?;
2826 Ok(result.rows_affected() > 0)
2827 }
2828
2829 async fn template_launch(
2830 &self,
2831 id: &str,
2832 version: u32,
2833 launched_by: Option<&str>,
2834 ) -> Result<Option<u32>, $crate::serve::history::HistoryError> {
2835 use sqlx::Row as _;
2836 use $crate::serve::history::HistoryError;
2837 use $crate::serve::history::{sql, templates};
2838 let backend = |e: sqlx::Error| HistoryError::Backend(e.to_string());
2839
2840 let existing = self.template_launches(id).await?;
2844 if templates::stable_version(&existing) == Some(version) {
2845 return Ok(None);
2846 }
2847 for attempt in 1..=sql::CLAIM_ATTEMPTS {
2851 sql::retry_backoff(attempt).await;
2852 let row = match sqlx::query(&self.stmts.template_max_launch_seq)
2855 .bind(id)
2856 .fetch_one(&self.pool)
2857 .await
2858 {
2859 Ok(row) => row,
2860 Err(e) if attempt < sql::CLAIM_ATTEMPTS => {
2861 tracing::debug!(
2862 template = %id, attempt, error = %e,
2863 "launch-log seq read lost a race; retrying"
2864 );
2865 continue;
2866 }
2867 Err(e) => return Err(backend(e)),
2868 };
2869 let max: i64 = row.try_get("v").map_err(backend)?;
2870 let seq = (max as u32).saturating_add(1);
2871 let insert = sqlx::query(&self.stmts.template_insert_launch)
2872 .bind(id)
2873 .bind(seq.to_string())
2874 .bind(version.to_string())
2875 .bind(sql::fmt_ts(chrono::Utc::now()))
2876 .bind(launched_by)
2877 .execute(&self.pool)
2878 .await;
2879 match insert {
2880 Ok(_) => return Ok(Some(seq)),
2881 Err(e) if attempt < sql::CLAIM_ATTEMPTS => {
2882 tracing::debug!(
2883 template = %id, attempt, error = %e,
2884 "launch-log insert lost a race; retrying with the next seq"
2885 );
2886 }
2887 Err(e) => return Err(backend(e)),
2888 }
2889 }
2890 Err(HistoryError::Backend(format!(
2891 "could not append a launch for template '{id}' after {} attempts",
2892 sql::CLAIM_ATTEMPTS
2893 )))
2894 }
2895
2896 async fn template_launches(
2897 &self,
2898 id: &str,
2899 ) -> Result<
2900 Vec<$crate::serve::history::templates::LaunchRecord>,
2901 $crate::serve::history::HistoryError,
2902 > {
2903 use sqlx::Row as _;
2904 use $crate::serve::history::HistoryError;
2905 use $crate::serve::history::{sql, templates};
2906 let backend = |e: sqlx::Error| HistoryError::Backend(e.to_string());
2907 let rows = sqlx::query(&self.stmts.template_select_launches)
2908 .bind(id)
2909 .fetch_all(&self.pool)
2910 .await
2911 .map_err(backend)?;
2912 let mut out = Vec::with_capacity(rows.len());
2913 for r in &rows {
2914 let seq: String = r.try_get("seq").map_err(backend)?;
2915 let version: String = r.try_get("version").map_err(backend)?;
2916 let launched_at: String = r.try_get("launched_at").map_err(backend)?;
2917 let launched_by: Option<String> = r.try_get("launched_by").map_err(backend)?;
2918 let (Ok(seq), Ok(version)) = (seq.parse::<u32>(), version.parse::<u32>()) else {
2921 continue;
2922 };
2923 out.push(templates::LaunchRecord {
2924 seq,
2925 version,
2926 launched_at: sql::parse_ts(&launched_at),
2927 launched_by,
2928 });
2929 }
2930 Ok(out)
2931 }
2932
2933 async fn template_set_deprecation(
2934 &self,
2935 id: &str,
2936 record: Option<&$crate::serve::history::templates::DeprecationRecord>,
2937 ) -> Result<(), $crate::serve::history::HistoryError> {
2938 use $crate::serve::history::HistoryError;
2939 use $crate::serve::history::sql;
2940 let backend = |e: sqlx::Error| HistoryError::Backend(e.to_string());
2941 match record {
2942 Some(r) => {
2943 sqlx::query(&self.stmts.template_upsert_deprecation)
2944 .bind(id)
2945 .bind(sql::fmt_ts(r.deprecated_at))
2946 .bind(r.deprecated_by.as_deref())
2947 .bind(r.reason.as_deref())
2948 .execute(&self.pool)
2949 .await
2950 .map_err(backend)?;
2951 }
2952 None => {
2953 sqlx::query(&self.stmts.template_delete_deprecation)
2954 .bind(id)
2955 .execute(&self.pool)
2956 .await
2957 .map_err(backend)?;
2958 }
2959 }
2960 Ok(())
2961 }
2962
2963 async fn template_deprecation(
2964 &self,
2965 id: &str,
2966 ) -> Result<
2967 Option<$crate::serve::history::templates::DeprecationRecord>,
2968 $crate::serve::history::HistoryError,
2969 > {
2970 use sqlx::Row as _;
2971 use $crate::serve::history::HistoryError;
2972 use $crate::serve::history::{sql, templates};
2973 let backend = |e: sqlx::Error| HistoryError::Backend(e.to_string());
2974 let Some(row) = sqlx::query(&self.stmts.template_select_deprecation)
2975 .bind(id)
2976 .fetch_optional(&self.pool)
2977 .await
2978 .map_err(backend)?
2979 else {
2980 return Ok(None);
2981 };
2982 let at: String = row.try_get("deprecated_at").map_err(backend)?;
2983 Ok(Some(templates::DeprecationRecord {
2984 deprecated_at: sql::parse_ts(&at),
2985 deprecated_by: row.try_get("deprecated_by").map_err(backend)?,
2986 reason: row.try_get("reason").map_err(backend)?,
2987 }))
2988 }
2989
2990 fn degraded(&self) -> bool {
2991 false
2994 }
2995 }
2996
2997 impl $name {
2998 async fn catalog_all_edges(
3000 &self,
3001 ) -> Result<
3002 Vec<$crate::serve::history::catalog::CatalogLineageEdge>,
3003 $crate::serve::history::HistoryError,
3004 > {
3005 use sqlx::Row as _;
3006 use $crate::serve::history::HistoryError;
3007 use $crate::serve::history::sql;
3008 let backend = |e: sqlx::Error| HistoryError::Backend(e.to_string());
3009 let rows = sqlx::query(&self.stmts.catalog_select_edges)
3010 .fetch_all(&self.pool)
3011 .await
3012 .map_err(backend)?;
3013 let mut edges = Vec::with_capacity(rows.len());
3014 for r in &rows {
3015 let body: String = r.try_get("body").map_err(backend)?;
3016 edges.push(sql::decode_json(&body, "catalog edge")?);
3017 }
3018 Ok(edges)
3019 }
3020 }
3021 };
3022}
3023
3024pub(crate) use impl_sql_history;
3025
3026#[cfg(test)]
3027mod tests {
3028 use super::*;
3029
3030 #[test]
3031 fn postgres_shard_statements_are_built() {
3032 let s = Stmts::new(Dialect::Postgres);
3035 assert!(s.insert_shard.contains("faucet_serve_shards"));
3036 assert!(s.insert_shard.contains("ON CONFLICT"));
3037 assert!(s.claim_shards_select.contains("JOIN faucet_serve_runs"));
3038 assert!(s.claim_shard_one.contains("'running'"));
3039 assert!(s.renew_shard_leases.contains("lease_expires_at"));
3040 assert!(s.reclaim_shards_select.contains("'running'"));
3041 assert!(s.reclaim_shard_requeue.contains("'pending'"));
3042 assert!(s.reclaim_shard_fail.contains("'failed'"));
3043 assert!(s.finalize_shard.contains("owner"));
3044 assert!(s.shard_progress.contains("GROUP BY"));
3045 }
3046
3047 #[test]
3048 fn fmt_ts_is_fixed_width_and_sortable() {
3049 let a = fmt_ts(
3050 DateTime::parse_from_rfc3339("2026-01-01T00:00:00Z")
3051 .unwrap()
3052 .to_utc(),
3053 );
3054 let b = fmt_ts(
3055 DateTime::parse_from_rfc3339("2026-01-01T00:00:01Z")
3056 .unwrap()
3057 .to_utc(),
3058 );
3059 assert!(a.ends_with('Z'));
3060 assert_eq!(a.len(), b.len(), "fixed width");
3061 assert!(a < b, "lexicographic order matches chronological order");
3062 }
3063
3064 #[test]
3065 fn is_expired_respects_window() {
3066 let now = Utc::now();
3067 let old = fmt_ts(now - chrono::Duration::seconds(120));
3068 assert!(is_expired(&old, now, Duration::from_secs(60)));
3069 assert!(!is_expired(&old, now, Duration::from_secs(600)));
3070 assert!(!is_expired("not-a-timestamp", now, Duration::ZERO));
3072 }
3073
3074 #[test]
3075 fn parse_status_round_trips_known_and_defaults_failed() {
3076 for s in [
3077 RunStatus::Queued,
3078 RunStatus::Pending,
3079 RunStatus::Running,
3080 RunStatus::Completed,
3081 RunStatus::Failed,
3082 RunStatus::Cancelled,
3083 ] {
3084 assert_eq!(parse_status(s.as_str()), s);
3085 }
3086 assert_eq!(parse_status("garbage"), RunStatus::Failed);
3087 }
3088
3089 #[test]
3090 fn body_round_trips() {
3091 let rec = RunRecord::queued(
3092 "r1".into(),
3093 Some("n".into()),
3094 Default::default(),
3095 Some("idem".into()),
3096 Utc::now(),
3097 );
3098 let encoded = encode_body(&rec).unwrap();
3099 let decoded = decode_body(&encoded).unwrap();
3100 assert_eq!(decoded.run_id, "r1");
3101 assert_eq!(decoded.idempotency_key.as_deref(), Some("idem"));
3102 }
3103
3104 #[test]
3105 fn postgres_and_sqlite_statements_differ_only_in_placeholders() {
3106 let pg = Stmts::new(Dialect::Postgres);
3107 let lite = Stmts::new(Dialect::Sqlite);
3108 assert!(pg.upsert.contains("$1") && lite.upsert.contains('?'));
3109 assert!(pg.list.contains("$13") && lite.list.contains('?'));
3110 assert!(pg.insert_idem.contains("ON CONFLICT (key) DO NOTHING"));
3112 assert!(lite.insert_idem.contains("ON CONFLICT (key) DO NOTHING"));
3113 assert!(pg.claim_one.contains("$3") && lite.claim_one.contains('?'));
3114 assert!(pg.heartbeat_instance.contains("faucet_serve_instances"));
3115 assert!(lite.heartbeat_instance.contains("faucet_serve_instances"));
3116 }
3117}