1use super::{HistoryError, RunRecord, RunStatus};
20use chrono::{DateTime, Utc};
21use std::time::Duration;
22
23pub const DDL: &[&str] = &[
26 "CREATE TABLE IF NOT EXISTS faucet_serve_runs (\
31 run_id TEXT PRIMARY KEY,\
32 name TEXT,\
33 status TEXT NOT NULL,\
34 submitted_at TEXT NOT NULL,\
35 finished_at TEXT,\
36 idempotency_key TEXT,\
37 owner TEXT,\
38 lease_expires_at TEXT,\
39 cancel_requested TEXT,\
40 body TEXT NOT NULL)",
41 "CREATE INDEX IF NOT EXISTS faucet_serve_runs_submitted_idx \
42 ON faucet_serve_runs (submitted_at)",
43 "CREATE INDEX IF NOT EXISTS faucet_serve_runs_status_lease_idx \
46 ON faucet_serve_runs (status, lease_expires_at)",
47 "CREATE INDEX IF NOT EXISTS faucet_serve_runs_pending_idx \
49 ON faucet_serve_runs (status, submitted_at)",
50 "CREATE TABLE IF NOT EXISTS faucet_serve_instances (\
51 instance_id TEXT PRIMARY KEY,\
52 started_at TEXT NOT NULL,\
53 last_heartbeat TEXT NOT NULL,\
54 listen TEXT,\
55 max_concurrent TEXT,\
56 in_flight TEXT)",
57 "CREATE INDEX IF NOT EXISTS faucet_serve_instances_hb_idx \
58 ON faucet_serve_instances (last_heartbeat)",
59 "CREATE TABLE IF NOT EXISTS faucet_serve_idem (\
60 key TEXT PRIMARY KEY,\
61 run_id TEXT NOT NULL,\
62 fingerprint TEXT NOT NULL,\
63 claimed_at TEXT NOT NULL)",
64 "CREATE TABLE IF NOT EXISTS faucet_serve_shards (\
70 run_id TEXT NOT NULL,\
71 shard_id TEXT NOT NULL,\
72 descriptor TEXT NOT NULL,\
73 size_estimate TEXT,\
74 status TEXT NOT NULL,\
75 owner TEXT,\
76 lease_expires_at TEXT,\
77 attempt TEXT NOT NULL,\
78 finished_at TEXT,\
79 PRIMARY KEY (run_id, shard_id))",
80 "CREATE INDEX IF NOT EXISTS faucet_serve_shards_claim_idx \
81 ON faucet_serve_shards (status, lease_expires_at)",
82 "CREATE TABLE IF NOT EXISTS faucet_serve_audit (\
86 id TEXT PRIMARY KEY,\
87 ts TEXT NOT NULL,\
88 principal TEXT NOT NULL,\
89 role TEXT NOT NULL,\
90 action TEXT NOT NULL,\
91 run_id TEXT,\
92 config_fingerprint TEXT,\
93 source_ip TEXT,\
94 result TEXT NOT NULL)",
95 "CREATE INDEX IF NOT EXISTS faucet_serve_audit_ts_idx \
96 ON faucet_serve_audit (ts)",
97 "CREATE TABLE IF NOT EXISTS faucet_catalog_datasets (\
102 id TEXT PRIMARY KEY,\
103 uri TEXT NOT NULL,\
104 kind TEXT NOT NULL,\
105 last_seen TEXT NOT NULL,\
106 body TEXT NOT NULL)",
107 "CREATE TABLE IF NOT EXISTS faucet_catalog_schema_versions (\
111 dataset_id TEXT NOT NULL,\
112 version TEXT NOT NULL,\
113 recorded_at TEXT NOT NULL,\
114 body TEXT NOT NULL,\
115 PRIMARY KEY (dataset_id, version))",
116 "CREATE TABLE IF NOT EXISTS faucet_catalog_edges (\
118 src_id TEXT NOT NULL,\
119 dst_id TEXT NOT NULL,\
120 last_seen TEXT NOT NULL,\
121 body TEXT NOT NULL,\
122 PRIMARY KEY (src_id, dst_id))",
123 "CREATE TABLE IF NOT EXISTS faucet_catalog_stats (\
125 dataset_id TEXT NOT NULL,\
126 recorded_at TEXT NOT NULL,\
127 run_id TEXT NOT NULL,\
128 records TEXT NOT NULL,\
129 PRIMARY KEY (dataset_id, recorded_at))",
130 "CREATE TABLE IF NOT EXISTS faucet_config_snapshots (\
134 pipeline TEXT PRIMARY KEY,\
135 recorded_at TEXT NOT NULL,\
136 faucet_version TEXT NOT NULL,\
137 body TEXT NOT NULL)",
138 "CREATE TABLE IF NOT EXISTS faucet_templates (\
145 id TEXT NOT NULL,\
146 version TEXT NOT NULL,\
147 name TEXT,\
148 created_at TEXT NOT NULL,\
149 body TEXT NOT NULL,\
150 PRIMARY KEY (id, version))",
151 "CREATE TABLE IF NOT EXISTS faucet_template_tags (\
156 id TEXT NOT NULL,\
157 tag TEXT NOT NULL,\
158 version TEXT NOT NULL,\
159 updated_at TEXT NOT NULL,\
160 PRIMARY KEY (id, tag))",
161 "CREATE TABLE IF NOT EXISTS faucet_template_launches (\
166 id TEXT NOT NULL,\
167 seq TEXT NOT NULL,\
168 version TEXT NOT NULL,\
169 launched_at TEXT NOT NULL,\
170 launched_by TEXT,\
171 PRIMARY KEY (id, seq))",
172 "CREATE TABLE IF NOT EXISTS faucet_template_deprecations (\
175 id TEXT PRIMARY KEY,\
176 deprecated_at TEXT NOT NULL,\
177 deprecated_by TEXT,\
178 reason TEXT)",
179];
180
181#[derive(Clone, Copy, Debug)]
183pub enum Dialect {
184 Postgres,
185 Sqlite,
186}
187
188pub struct Stmts {
190 pub upsert: String,
194 pub select_body: String,
195 pub select_status: String,
196 pub select_submitted: String,
197 pub delete: String,
198 pub list: String,
199 pub purge_runs: String,
200 pub purge_idem: String,
201 pub select_orphans: String,
204 pub renew_leases: String,
207 pub insert_idem: String,
208 pub select_idem: String,
209 pub takeover_idem: String,
210 pub delete_idem_by_run: String,
215 pub select_pending: String,
217 pub claim_one: String,
219 pub reclaim_select: String,
223 pub reclaim_requeue: String,
225 pub reclaim_fail: String,
227 pub finalize_owned: String,
229 pub cancel_pending: String,
231 pub request_cancel: String,
233 pub pending_cancellations: String,
235 pub heartbeat_instance: String,
237 pub live_instances: String,
239 pub prune_instances: String,
241 pub insert_shard: String,
244 pub claim_shards_select: String,
246 pub claim_shard_one: String,
248 pub renew_shard_leases: String,
250 pub reclaim_shards_select: String,
252 pub reclaim_shard_requeue: String,
254 pub reclaim_shard_fail: String,
256 pub finalize_shard: String,
258 pub shard_progress: String,
260 pub pending_shard_cancellations: String,
264 pub select_sharded_parents: String,
267 pub finalize_sharded_parent: String,
271 pub delete_shards_by_run: String,
274 pub purge_orphan_shards: String,
277 pub insert_audit: String,
280 pub list_audit: String,
283 pub purge_audit: String,
285 pub catalog_select_dataset: String,
288 pub catalog_upsert_dataset: String,
291 pub catalog_select_datasets: String,
295 pub catalog_insert_schema_version: String,
298 pub catalog_select_schema_versions: String,
300 pub catalog_upsert_edge: String,
302 pub catalog_select_edges: String,
304 pub catalog_insert_stat: String,
306 pub catalog_select_stats: String,
308 pub catalog_prune_stats: String,
311 pub catalog_upsert_config_snapshot: String,
314 pub catalog_select_config_snapshot: String,
316 pub template_max_version: String,
319 pub template_insert: String,
321 pub template_select_version: String,
323 pub template_select_latest: String,
325 pub template_select_all: String,
328 pub template_versions: String,
330 pub template_delete_version: String,
332 pub template_delete_all: String,
334 pub template_upsert_tag: String,
336 pub template_select_tags: String,
338 pub template_delete_tag: String,
340 pub template_delete_tags_all: String,
342 pub template_delete_tags_for_version: String,
344 pub template_max_launch_seq: String,
346 pub template_insert_launch: String,
348 pub template_select_launches: String,
350 pub template_delete_launches_all: String,
352 pub template_delete_launches_for_version: String,
354 pub template_upsert_deprecation: String,
356 pub template_select_deprecation: String,
358 pub template_delete_deprecation: String,
360}
361
362impl Stmts {
363 pub fn new(dialect: Dialect) -> Self {
364 match dialect {
365 Dialect::Postgres => Self::postgres(),
366 Dialect::Sqlite => Self::sqlite(),
367 }
368 }
369
370 fn postgres() -> Self {
371 Self {
372 upsert: "INSERT INTO faucet_serve_runs \
373 (run_id,name,status,submitted_at,finished_at,idempotency_key,owner,lease_expires_at,body) \
374 VALUES ($1,$2,$3,$4,$5,$6,$7,$8,$9) \
375 ON CONFLICT (run_id) DO UPDATE SET \
376 name=excluded.name,status=excluded.status,submitted_at=excluded.submitted_at,\
377 finished_at=excluded.finished_at,idempotency_key=excluded.idempotency_key,\
378 owner=excluded.owner,lease_expires_at=excluded.lease_expires_at,\
379 body=excluded.body"
380 .into(),
381 select_body: "SELECT body FROM faucet_serve_runs WHERE run_id=$1".into(),
382 select_status: "SELECT status FROM faucet_serve_runs WHERE run_id=$1".into(),
383 select_submitted: "SELECT submitted_at FROM faucet_serve_runs WHERE run_id=$1".into(),
384 delete: "DELETE FROM faucet_serve_runs WHERE run_id=$1".into(),
385 list: "SELECT body FROM faucet_serve_runs \
388 WHERE ($1::text IS NULL OR status = $2::text) \
389 AND ($3::text IS NULL OR name = $4::text) \
390 AND ($5::text IS NULL OR submitted_at >= $6::text) \
391 AND ($7::text IS NULL OR submitted_at <= $8::text) \
392 AND ($9::text IS NULL OR (submitted_at < $10::text \
393 OR (submitted_at = $11::text AND run_id < $12::text))) \
394 ORDER BY submitted_at DESC, run_id DESC LIMIT $13"
395 .into(),
396 purge_runs: "DELETE FROM faucet_serve_runs \
397 WHERE status IN ('completed','failed','cancelled') \
398 AND finished_at IS NOT NULL AND finished_at < $1"
399 .into(),
400 purge_idem: "DELETE FROM faucet_serve_idem WHERE claimed_at < $1".into(),
401 select_orphans: "SELECT body FROM faucet_serve_runs \
402 WHERE status IN ('queued','running') \
403 AND (lease_expires_at IS NULL OR lease_expires_at < $1)"
404 .into(),
405 renew_leases: "UPDATE faucet_serve_runs SET lease_expires_at = $1 \
406 WHERE owner = $2 AND status IN ('queued','running')"
407 .into(),
408 insert_idem: "INSERT INTO faucet_serve_idem (key,run_id,fingerprint,claimed_at) \
409 VALUES ($1,$2,$3,$4) ON CONFLICT (key) DO NOTHING"
410 .into(),
411 select_idem: "SELECT run_id,fingerprint,claimed_at FROM faucet_serve_idem WHERE key=$1"
412 .into(),
413 takeover_idem: "UPDATE faucet_serve_idem \
414 SET run_id=$1,fingerprint=$2,claimed_at=$3 WHERE key=$4 AND claimed_at=$5"
415 .into(),
416 delete_idem_by_run: "DELETE FROM faucet_serve_idem WHERE run_id=$1".into(),
417 select_pending: "SELECT run_id, body FROM faucet_serve_runs \
418 WHERE status = 'pending' ORDER BY submitted_at ASC LIMIT $1"
419 .into(),
420 claim_one: "UPDATE faucet_serve_runs \
421 SET owner = $1, status = 'running', lease_expires_at = $2, body = $3 \
422 WHERE run_id = $4 AND status = 'pending'"
423 .into(),
424 reclaim_select: "SELECT body FROM faucet_serve_runs \
425 WHERE status = 'running' \
426 AND (lease_expires_at IS NULL OR lease_expires_at < $1)"
427 .into(),
428 reclaim_requeue: "UPDATE faucet_serve_runs \
433 SET status = 'pending', owner = NULL, lease_expires_at = NULL, \
434 body = $1 \
435 WHERE run_id = $2 AND status = 'running' \
436 AND (lease_expires_at IS NULL OR lease_expires_at < $3)"
437 .into(),
438 reclaim_fail: "UPDATE faucet_serve_runs \
439 SET status = 'failed', finished_at = $1, body = $2, owner = NULL \
440 WHERE run_id = $3 AND status = 'running' \
441 AND (lease_expires_at IS NULL OR lease_expires_at < $4)"
442 .into(),
443 finalize_owned: "UPDATE faucet_serve_runs \
448 SET status = $1, finished_at = $2, lease_expires_at = $3, body = $4 \
449 WHERE run_id = $5 AND owner = $6 \
450 AND status NOT IN ('completed','failed','cancelled')"
451 .into(),
452 cancel_pending: "UPDATE faucet_serve_runs \
453 SET status = 'cancelled', finished_at = $1, body = $2 \
454 WHERE run_id = $3 AND status = 'pending'"
455 .into(),
456 request_cancel: "UPDATE faucet_serve_runs \
457 SET cancel_requested = $1 WHERE run_id = $2 AND status IN ('running','sharded')"
458 .into(),
459 pending_cancellations: "SELECT run_id FROM faucet_serve_runs \
460 WHERE status = 'running' AND owner = $1 AND cancel_requested IS NOT NULL"
461 .into(),
462 heartbeat_instance: "INSERT INTO faucet_serve_instances \
463 (instance_id, started_at, last_heartbeat, listen, max_concurrent, in_flight) \
464 VALUES ($1,$2,$3,$4,$5,$6) \
465 ON CONFLICT (instance_id) DO UPDATE SET \
466 last_heartbeat = excluded.last_heartbeat, listen = excluded.listen, \
467 max_concurrent = excluded.max_concurrent, in_flight = excluded.in_flight"
468 .into(),
469 live_instances: "SELECT instance_id, started_at, last_heartbeat, listen, \
470 max_concurrent, in_flight FROM faucet_serve_instances \
471 WHERE last_heartbeat >= $1"
472 .into(),
473 prune_instances: "DELETE FROM faucet_serve_instances WHERE last_heartbeat < $1".into(),
474 insert_shard: "INSERT INTO faucet_serve_shards \
475 (run_id, shard_id, descriptor, size_estimate, status, attempt) \
476 VALUES ($1,$2,$3,$4,'pending','0') \
477 ON CONFLICT (run_id, shard_id) DO NOTHING"
478 .into(),
479 claim_shards_select: "SELECT s.run_id, s.shard_id, s.descriptor, r.body \
480 FROM faucet_serve_shards s JOIN faucet_serve_runs r ON r.run_id = s.run_id \
481 WHERE s.status = 'pending' \
482 ORDER BY CAST(COALESCE(s.size_estimate, '0') AS BIGINT) DESC, s.run_id, s.shard_id \
483 LIMIT $1"
484 .into(),
485 claim_shard_one: "UPDATE faucet_serve_shards \
486 SET owner = $1, status = 'running', lease_expires_at = $2 \
487 WHERE run_id = $3 AND shard_id = $4 AND status = 'pending'"
488 .into(),
489 renew_shard_leases: "UPDATE faucet_serve_shards SET lease_expires_at = $1 \
490 WHERE owner = $2 AND status = 'running'"
491 .into(),
492 reclaim_shards_select: "SELECT run_id, shard_id, attempt FROM faucet_serve_shards \
493 WHERE status = 'running' \
494 AND (lease_expires_at IS NULL OR lease_expires_at < $1)"
495 .into(),
496 reclaim_shard_requeue: "UPDATE faucet_serve_shards \
497 SET status = 'pending', owner = NULL, lease_expires_at = NULL, attempt = $1 \
498 WHERE run_id = $2 AND shard_id = $3 AND status = 'running' \
499 AND (lease_expires_at IS NULL OR lease_expires_at < $4)"
500 .into(),
501 reclaim_shard_fail: "UPDATE faucet_serve_shards \
502 SET status = 'failed', finished_at = $1, owner = NULL \
503 WHERE run_id = $2 AND shard_id = $3 AND status = 'running' \
504 AND (lease_expires_at IS NULL OR lease_expires_at < $4)"
505 .into(),
506 finalize_shard: "UPDATE faucet_serve_shards \
507 SET status = $1, finished_at = $2 \
508 WHERE run_id = $3 AND shard_id = $4 AND owner = $5 AND status = 'running'"
509 .into(),
510 shard_progress: "SELECT status, COUNT(*) AS n FROM faucet_serve_shards \
511 WHERE run_id = $1 GROUP BY status"
512 .into(),
513 pending_shard_cancellations: "SELECT DISTINCT s.run_id \
514 FROM faucet_serve_shards s \
515 JOIN faucet_serve_runs r ON r.run_id = s.run_id \
516 WHERE s.owner = $1 AND s.status = 'running' \
517 AND r.cancel_requested IS NOT NULL"
518 .into(),
519 select_sharded_parents: "SELECT run_id FROM faucet_serve_runs \
520 WHERE status = 'sharded'"
521 .into(),
522 finalize_sharded_parent: "UPDATE faucet_serve_runs \
523 SET status = $1, finished_at = $2, body = $3 \
524 WHERE run_id = $4 AND status = 'sharded'"
525 .into(),
526 delete_shards_by_run: "DELETE FROM faucet_serve_shards WHERE run_id = $1".into(),
527 purge_orphan_shards: "DELETE FROM faucet_serve_shards \
528 WHERE run_id NOT IN (SELECT run_id FROM faucet_serve_runs)"
529 .into(),
530 insert_audit: "INSERT INTO faucet_serve_audit \
531 (id, ts, principal, role, action, run_id, config_fingerprint, source_ip, result) \
532 VALUES ($1,$2,$3,$4,$5,$6,$7,$8,$9)"
533 .into(),
534 list_audit: "SELECT id, ts, principal, role, action, run_id, config_fingerprint, \
535 source_ip, result FROM faucet_serve_audit \
536 WHERE ($1::text IS NULL OR principal = $2::text) \
537 AND ($3::text IS NULL OR action = $4::text) \
538 AND ($5::text IS NULL OR ts >= $6::text) \
539 AND ($7::text IS NULL OR ts <= $8::text) \
540 ORDER BY ts DESC, id DESC LIMIT $9"
541 .into(),
542 purge_audit: "DELETE FROM faucet_serve_audit WHERE ts < $1".into(),
543 catalog_select_dataset: "SELECT body FROM faucet_catalog_datasets WHERE id=$1".into(),
544 catalog_upsert_dataset: "INSERT INTO faucet_catalog_datasets \
545 (id, uri, kind, last_seen, body) VALUES ($1,$2,$3,$4,$5) \
546 ON CONFLICT (id) DO UPDATE SET uri=excluded.uri, kind=excluded.kind, \
547 last_seen=excluded.last_seen, body=excluded.body"
548 .into(),
549 catalog_select_datasets: "SELECT body FROM faucet_catalog_datasets".into(),
550 catalog_insert_schema_version: "INSERT INTO faucet_catalog_schema_versions \
551 (dataset_id, version, recorded_at, body) VALUES ($1,$2,$3,$4) \
552 ON CONFLICT (dataset_id, version) DO NOTHING"
553 .into(),
554 catalog_select_schema_versions: "SELECT body FROM faucet_catalog_schema_versions \
555 WHERE dataset_id=$1 ORDER BY CAST(version AS BIGINT) ASC"
556 .into(),
557 catalog_upsert_edge: "INSERT INTO faucet_catalog_edges \
558 (src_id, dst_id, last_seen, body) VALUES ($1,$2,$3,$4) \
559 ON CONFLICT (src_id, dst_id) DO UPDATE SET \
560 last_seen=excluded.last_seen, body=excluded.body"
561 .into(),
562 catalog_select_edges: "SELECT body FROM faucet_catalog_edges \
563 ORDER BY last_seen DESC, src_id, dst_id"
564 .into(),
565 catalog_insert_stat: "INSERT INTO faucet_catalog_stats \
566 (dataset_id, recorded_at, run_id, records) VALUES ($1,$2,$3,$4) \
567 ON CONFLICT (dataset_id, recorded_at) DO NOTHING"
568 .into(),
569 catalog_select_stats: "SELECT recorded_at, run_id, records \
570 FROM faucet_catalog_stats WHERE dataset_id=$1 \
571 ORDER BY recorded_at DESC LIMIT $2"
572 .into(),
573 catalog_prune_stats: "DELETE FROM faucet_catalog_stats \
574 WHERE dataset_id=$1 AND recorded_at NOT IN (\
575 SELECT recorded_at FROM faucet_catalog_stats WHERE dataset_id=$2 \
576 ORDER BY recorded_at DESC LIMIT $3)"
577 .into(),
578 catalog_upsert_config_snapshot: "INSERT INTO faucet_config_snapshots \
579 (pipeline, recorded_at, faucet_version, body) VALUES ($1,$2,$3,$4) \
580 ON CONFLICT (pipeline) DO UPDATE SET recorded_at=excluded.recorded_at, \
581 faucet_version=excluded.faucet_version, body=excluded.body"
582 .into(),
583 catalog_select_config_snapshot:
584 "SELECT body FROM faucet_config_snapshots WHERE pipeline=$1".into(),
585 template_max_version: "SELECT COALESCE(MAX(CAST(version AS BIGINT)), 0) AS v \
586 FROM faucet_templates WHERE id=$1"
587 .into(),
588 template_insert: "INSERT INTO faucet_templates \
589 (id, version, name, created_at, body) VALUES ($1,$2,$3,$4,$5)"
590 .into(),
591 template_select_version:
592 "SELECT body FROM faucet_templates WHERE id=$1 AND version=$2".into(),
593 template_select_latest: "SELECT body FROM faucet_templates WHERE id=$1 \
594 ORDER BY CAST(version AS BIGINT) DESC LIMIT 1"
595 .into(),
596 template_select_all: "SELECT body FROM faucet_templates".into(),
597 template_versions: "SELECT version FROM faucet_templates WHERE id=$1 \
598 ORDER BY CAST(version AS BIGINT) DESC"
599 .into(),
600 template_delete_version: "DELETE FROM faucet_templates WHERE id=$1 AND version=$2"
601 .into(),
602 template_delete_all: "DELETE FROM faucet_templates WHERE id=$1".into(),
603 template_upsert_tag: "INSERT INTO faucet_template_tags \
604 (id, tag, version, updated_at) VALUES ($1,$2,$3,$4) \
605 ON CONFLICT (id, tag) DO UPDATE SET version=excluded.version, \
606 updated_at=excluded.updated_at"
607 .into(),
608 template_select_tags: "SELECT tag, version FROM faucet_template_tags \
609 WHERE id=$1 ORDER BY tag"
610 .into(),
611 template_delete_tag: "DELETE FROM faucet_template_tags WHERE id=$1 AND tag=$2".into(),
612 template_delete_tags_all: "DELETE FROM faucet_template_tags WHERE id=$1".into(),
613 template_delete_tags_for_version:
614 "DELETE FROM faucet_template_tags WHERE id=$1 AND version=$2".into(),
615 template_max_launch_seq: "SELECT COALESCE(MAX(CAST(seq AS BIGINT)), 0) AS v \
616 FROM faucet_template_launches WHERE id=$1"
617 .into(),
618 template_insert_launch: "INSERT INTO faucet_template_launches \
619 (id, seq, version, launched_at, launched_by) VALUES ($1,$2,$3,$4,$5)"
620 .into(),
621 template_select_launches: "SELECT seq, version, launched_at, launched_by \
622 FROM faucet_template_launches WHERE id=$1 ORDER BY CAST(seq AS BIGINT) DESC"
623 .into(),
624 template_delete_launches_all: "DELETE FROM faucet_template_launches WHERE id=$1".into(),
625 template_delete_launches_for_version:
626 "DELETE FROM faucet_template_launches WHERE id=$1 AND version=$2".into(),
627 template_upsert_deprecation: "INSERT INTO faucet_template_deprecations \
628 (id, deprecated_at, deprecated_by, reason) VALUES ($1,$2,$3,$4) \
629 ON CONFLICT (id) DO UPDATE SET deprecated_at=excluded.deprecated_at, \
630 deprecated_by=excluded.deprecated_by, reason=excluded.reason"
631 .into(),
632 template_select_deprecation: "SELECT deprecated_at, deprecated_by, reason \
633 FROM faucet_template_deprecations WHERE id=$1"
634 .into(),
635 template_delete_deprecation: "DELETE FROM faucet_template_deprecations WHERE id=$1"
636 .into(),
637 }
638 }
639
640 fn sqlite() -> Self {
641 Self {
642 upsert: "INSERT INTO faucet_serve_runs \
643 (run_id,name,status,submitted_at,finished_at,idempotency_key,owner,lease_expires_at,body) \
644 VALUES (?,?,?,?,?,?,?,?,?) \
645 ON CONFLICT (run_id) DO UPDATE SET \
646 name=excluded.name,status=excluded.status,submitted_at=excluded.submitted_at,\
647 finished_at=excluded.finished_at,idempotency_key=excluded.idempotency_key,\
648 owner=excluded.owner,lease_expires_at=excluded.lease_expires_at,\
649 body=excluded.body"
650 .into(),
651 select_body: "SELECT body FROM faucet_serve_runs WHERE run_id=?".into(),
652 select_status: "SELECT status FROM faucet_serve_runs WHERE run_id=?".into(),
653 select_submitted: "SELECT submitted_at FROM faucet_serve_runs WHERE run_id=?".into(),
654 delete: "DELETE FROM faucet_serve_runs WHERE run_id=?".into(),
655 list: "SELECT body FROM faucet_serve_runs \
656 WHERE (? IS NULL OR status = ?) \
657 AND (? IS NULL OR name = ?) \
658 AND (? IS NULL OR submitted_at >= ?) \
659 AND (? IS NULL OR submitted_at <= ?) \
660 AND (? IS NULL OR (submitted_at < ? \
661 OR (submitted_at = ? AND run_id < ?))) \
662 ORDER BY submitted_at DESC, run_id DESC LIMIT ?"
663 .into(),
664 purge_runs: "DELETE FROM faucet_serve_runs \
665 WHERE status IN ('completed','failed','cancelled') \
666 AND finished_at IS NOT NULL AND finished_at < ?"
667 .into(),
668 purge_idem: "DELETE FROM faucet_serve_idem WHERE claimed_at < ?".into(),
669 select_orphans: "SELECT body FROM faucet_serve_runs \
670 WHERE status IN ('queued','running') \
671 AND (lease_expires_at IS NULL OR lease_expires_at < ?)"
672 .into(),
673 renew_leases: "UPDATE faucet_serve_runs SET lease_expires_at = ? \
674 WHERE owner = ? AND status IN ('queued','running')"
675 .into(),
676 insert_idem: "INSERT INTO faucet_serve_idem (key,run_id,fingerprint,claimed_at) \
677 VALUES (?,?,?,?) ON CONFLICT (key) DO NOTHING"
678 .into(),
679 select_idem: "SELECT run_id,fingerprint,claimed_at FROM faucet_serve_idem WHERE key=?"
680 .into(),
681 takeover_idem: "UPDATE faucet_serve_idem \
682 SET run_id=?,fingerprint=?,claimed_at=? WHERE key=? AND claimed_at=?"
683 .into(),
684 delete_idem_by_run: "DELETE FROM faucet_serve_idem WHERE run_id=?".into(),
685 select_pending: "SELECT run_id, body FROM faucet_serve_runs \
686 WHERE status = 'pending' ORDER BY submitted_at ASC LIMIT ?"
687 .into(),
688 claim_one: "UPDATE faucet_serve_runs \
689 SET owner = ?, status = 'running', lease_expires_at = ?, body = ? \
690 WHERE run_id = ? AND status = 'pending'"
691 .into(),
692 reclaim_select: "SELECT body FROM faucet_serve_runs \
693 WHERE status = 'running' \
694 AND (lease_expires_at IS NULL OR lease_expires_at < ?)"
695 .into(),
696 reclaim_requeue: "UPDATE faucet_serve_runs \
698 SET status = 'pending', owner = NULL, lease_expires_at = NULL, \
699 body = ? \
700 WHERE run_id = ? AND status = 'running' \
701 AND (lease_expires_at IS NULL OR lease_expires_at < ?)"
702 .into(),
703 reclaim_fail: "UPDATE faucet_serve_runs \
704 SET status = 'failed', finished_at = ?, body = ?, owner = NULL \
705 WHERE run_id = ? AND status = 'running' \
706 AND (lease_expires_at IS NULL OR lease_expires_at < ?)"
707 .into(),
708 finalize_owned: "UPDATE faucet_serve_runs \
710 SET status = ?, finished_at = ?, lease_expires_at = ?, body = ? \
711 WHERE run_id = ? AND owner = ? \
712 AND status NOT IN ('completed','failed','cancelled')"
713 .into(),
714 cancel_pending: "UPDATE faucet_serve_runs \
715 SET status = 'cancelled', finished_at = ?, body = ? \
716 WHERE run_id = ? AND status = 'pending'"
717 .into(),
718 request_cancel: "UPDATE faucet_serve_runs \
719 SET cancel_requested = ? WHERE run_id = ? AND status IN ('running','sharded')"
720 .into(),
721 pending_cancellations: "SELECT run_id FROM faucet_serve_runs \
722 WHERE status = 'running' AND owner = ? AND cancel_requested IS NOT NULL"
723 .into(),
724 heartbeat_instance: "INSERT INTO faucet_serve_instances \
725 (instance_id, started_at, last_heartbeat, listen, max_concurrent, in_flight) \
726 VALUES (?,?,?,?,?,?) \
727 ON CONFLICT (instance_id) DO UPDATE SET \
728 last_heartbeat = excluded.last_heartbeat, listen = excluded.listen, \
729 max_concurrent = excluded.max_concurrent, in_flight = excluded.in_flight"
730 .into(),
731 live_instances: "SELECT instance_id, started_at, last_heartbeat, listen, \
732 max_concurrent, in_flight FROM faucet_serve_instances \
733 WHERE last_heartbeat >= ?"
734 .into(),
735 prune_instances: "DELETE FROM faucet_serve_instances WHERE last_heartbeat < ?".into(),
736 insert_shard: "INSERT INTO faucet_serve_shards \
737 (run_id, shard_id, descriptor, size_estimate, status, attempt) \
738 VALUES (?,?,?,?,'pending','0') \
739 ON CONFLICT (run_id, shard_id) DO NOTHING"
740 .into(),
741 claim_shards_select: "SELECT s.run_id, s.shard_id, s.descriptor, r.body \
742 FROM faucet_serve_shards s JOIN faucet_serve_runs r ON r.run_id = s.run_id \
743 WHERE s.status = 'pending' \
744 ORDER BY CAST(COALESCE(s.size_estimate, '0') AS INTEGER) DESC, s.run_id, s.shard_id \
745 LIMIT ?"
746 .into(),
747 claim_shard_one: "UPDATE faucet_serve_shards \
748 SET owner = ?, status = 'running', lease_expires_at = ? \
749 WHERE run_id = ? AND shard_id = ? AND status = 'pending'"
750 .into(),
751 renew_shard_leases: "UPDATE faucet_serve_shards SET lease_expires_at = ? \
752 WHERE owner = ? AND status = 'running'"
753 .into(),
754 reclaim_shards_select: "SELECT run_id, shard_id, attempt FROM faucet_serve_shards \
755 WHERE status = 'running' \
756 AND (lease_expires_at IS NULL OR lease_expires_at < ?)"
757 .into(),
758 reclaim_shard_requeue: "UPDATE faucet_serve_shards \
759 SET status = 'pending', owner = NULL, lease_expires_at = NULL, attempt = ? \
760 WHERE run_id = ? AND shard_id = ? AND status = 'running' \
761 AND (lease_expires_at IS NULL OR lease_expires_at < ?)"
762 .into(),
763 reclaim_shard_fail: "UPDATE faucet_serve_shards \
764 SET status = 'failed', finished_at = ?, owner = NULL \
765 WHERE run_id = ? AND shard_id = ? AND status = 'running' \
766 AND (lease_expires_at IS NULL OR lease_expires_at < ?)"
767 .into(),
768 finalize_shard: "UPDATE faucet_serve_shards \
769 SET status = ?, finished_at = ? \
770 WHERE run_id = ? AND shard_id = ? AND owner = ? AND status = 'running'"
771 .into(),
772 shard_progress: "SELECT status, COUNT(*) AS n FROM faucet_serve_shards \
773 WHERE run_id = ? GROUP BY status"
774 .into(),
775 pending_shard_cancellations: "SELECT DISTINCT s.run_id \
776 FROM faucet_serve_shards s \
777 JOIN faucet_serve_runs r ON r.run_id = s.run_id \
778 WHERE s.owner = ? AND s.status = 'running' \
779 AND r.cancel_requested IS NOT NULL"
780 .into(),
781 select_sharded_parents: "SELECT run_id FROM faucet_serve_runs \
782 WHERE status = 'sharded'"
783 .into(),
784 finalize_sharded_parent: "UPDATE faucet_serve_runs \
785 SET status = ?, finished_at = ?, body = ? \
786 WHERE run_id = ? AND status = 'sharded'"
787 .into(),
788 delete_shards_by_run: "DELETE FROM faucet_serve_shards WHERE run_id = ?".into(),
789 purge_orphan_shards: "DELETE FROM faucet_serve_shards \
790 WHERE run_id NOT IN (SELECT run_id FROM faucet_serve_runs)"
791 .into(),
792 insert_audit: "INSERT INTO faucet_serve_audit \
793 (id, ts, principal, role, action, run_id, config_fingerprint, source_ip, result) \
794 VALUES (?,?,?,?,?,?,?,?,?)"
795 .into(),
796 list_audit: "SELECT id, ts, principal, role, action, run_id, config_fingerprint, \
797 source_ip, result FROM faucet_serve_audit \
798 WHERE (? IS NULL OR principal = ?) \
799 AND (? IS NULL OR action = ?) \
800 AND (? IS NULL OR ts >= ?) \
801 AND (? IS NULL OR ts <= ?) \
802 ORDER BY ts DESC, id DESC LIMIT ?"
803 .into(),
804 purge_audit: "DELETE FROM faucet_serve_audit WHERE ts < ?".into(),
805 catalog_select_dataset: "SELECT body FROM faucet_catalog_datasets WHERE id=?".into(),
806 catalog_upsert_dataset: "INSERT INTO faucet_catalog_datasets \
807 (id, uri, kind, last_seen, body) VALUES (?,?,?,?,?) \
808 ON CONFLICT (id) DO UPDATE SET uri=excluded.uri, kind=excluded.kind, \
809 last_seen=excluded.last_seen, body=excluded.body"
810 .into(),
811 catalog_select_datasets: "SELECT body FROM faucet_catalog_datasets".into(),
812 catalog_insert_schema_version: "INSERT INTO faucet_catalog_schema_versions \
813 (dataset_id, version, recorded_at, body) VALUES (?,?,?,?) \
814 ON CONFLICT (dataset_id, version) DO NOTHING"
815 .into(),
816 catalog_select_schema_versions: "SELECT body FROM faucet_catalog_schema_versions \
817 WHERE dataset_id=? ORDER BY CAST(version AS INTEGER) ASC"
818 .into(),
819 catalog_upsert_edge: "INSERT INTO faucet_catalog_edges \
820 (src_id, dst_id, last_seen, body) VALUES (?,?,?,?) \
821 ON CONFLICT (src_id, dst_id) DO UPDATE SET \
822 last_seen=excluded.last_seen, body=excluded.body"
823 .into(),
824 catalog_select_edges: "SELECT body FROM faucet_catalog_edges \
825 ORDER BY last_seen DESC, src_id, dst_id"
826 .into(),
827 catalog_insert_stat: "INSERT INTO faucet_catalog_stats \
828 (dataset_id, recorded_at, run_id, records) VALUES (?,?,?,?) \
829 ON CONFLICT (dataset_id, recorded_at) DO NOTHING"
830 .into(),
831 catalog_select_stats: "SELECT recorded_at, run_id, records \
832 FROM faucet_catalog_stats WHERE dataset_id=? \
833 ORDER BY recorded_at DESC LIMIT ?"
834 .into(),
835 catalog_prune_stats: "DELETE FROM faucet_catalog_stats \
836 WHERE dataset_id=? AND recorded_at NOT IN (\
837 SELECT recorded_at FROM faucet_catalog_stats WHERE dataset_id=? \
838 ORDER BY recorded_at DESC LIMIT ?)"
839 .into(),
840 catalog_upsert_config_snapshot: "INSERT INTO faucet_config_snapshots \
841 (pipeline, recorded_at, faucet_version, body) VALUES (?,?,?,?) \
842 ON CONFLICT (pipeline) DO UPDATE SET recorded_at=excluded.recorded_at, \
843 faucet_version=excluded.faucet_version, body=excluded.body"
844 .into(),
845 catalog_select_config_snapshot:
846 "SELECT body FROM faucet_config_snapshots WHERE pipeline=?".into(),
847 template_max_version: "SELECT COALESCE(MAX(CAST(version AS INTEGER)), 0) AS v \
848 FROM faucet_templates WHERE id=?"
849 .into(),
850 template_insert: "INSERT INTO faucet_templates \
851 (id, version, name, created_at, body) VALUES (?,?,?,?,?)"
852 .into(),
853 template_select_version: "SELECT body FROM faucet_templates WHERE id=? AND version=?"
854 .into(),
855 template_select_latest: "SELECT body FROM faucet_templates WHERE id=? \
856 ORDER BY CAST(version AS INTEGER) DESC LIMIT 1"
857 .into(),
858 template_select_all: "SELECT body FROM faucet_templates".into(),
859 template_versions: "SELECT version FROM faucet_templates WHERE id=? \
860 ORDER BY CAST(version AS INTEGER) DESC"
861 .into(),
862 template_delete_version: "DELETE FROM faucet_templates WHERE id=? AND version=?".into(),
863 template_delete_all: "DELETE FROM faucet_templates WHERE id=?".into(),
864 template_upsert_tag: "INSERT INTO faucet_template_tags \
865 (id, tag, version, updated_at) VALUES (?,?,?,?) \
866 ON CONFLICT (id, tag) DO UPDATE SET version=excluded.version, \
867 updated_at=excluded.updated_at"
868 .into(),
869 template_select_tags: "SELECT tag, version FROM faucet_template_tags \
870 WHERE id=? ORDER BY tag"
871 .into(),
872 template_delete_tag: "DELETE FROM faucet_template_tags WHERE id=? AND tag=?".into(),
873 template_delete_tags_all: "DELETE FROM faucet_template_tags WHERE id=?".into(),
874 template_delete_tags_for_version:
875 "DELETE FROM faucet_template_tags WHERE id=? AND version=?".into(),
876 template_max_launch_seq: "SELECT COALESCE(MAX(CAST(seq AS INTEGER)), 0) AS v \
877 FROM faucet_template_launches WHERE id=?"
878 .into(),
879 template_insert_launch: "INSERT INTO faucet_template_launches \
880 (id, seq, version, launched_at, launched_by) VALUES (?,?,?,?,?)"
881 .into(),
882 template_select_launches: "SELECT seq, version, launched_at, launched_by \
883 FROM faucet_template_launches WHERE id=? ORDER BY CAST(seq AS INTEGER) DESC"
884 .into(),
885 template_delete_launches_all: "DELETE FROM faucet_template_launches WHERE id=?".into(),
886 template_delete_launches_for_version:
887 "DELETE FROM faucet_template_launches WHERE id=? AND version=?".into(),
888 template_upsert_deprecation: "INSERT INTO faucet_template_deprecations \
889 (id, deprecated_at, deprecated_by, reason) VALUES (?,?,?,?) \
890 ON CONFLICT (id) DO UPDATE SET deprecated_at=excluded.deprecated_at, \
891 deprecated_by=excluded.deprecated_by, reason=excluded.reason"
892 .into(),
893 template_select_deprecation: "SELECT deprecated_at, deprecated_by, reason \
894 FROM faucet_template_deprecations WHERE id=?"
895 .into(),
896 template_delete_deprecation: "DELETE FROM faucet_template_deprecations WHERE id=?"
897 .into(),
898 }
899 }
900}
901
902pub const CLAIM_ATTEMPTS: usize = 8;
914
915static RETRY_SEQ: std::sync::atomic::AtomicUsize = std::sync::atomic::AtomicUsize::new(0);
921
922pub async fn retry_backoff(attempt: usize) {
929 if attempt <= 1 {
930 return;
931 }
932 const BASE_MS: u64 = 5;
933 const CAP_MS: u64 = 160;
934 let exp = BASE_MS
935 .saturating_mul(1u64 << (attempt - 2).min(6))
936 .min(CAP_MS);
937 let stagger =
939 (RETRY_SEQ.fetch_add(1, std::sync::atomic::Ordering::Relaxed) as u64) % (BASE_MS + 1);
940 tokio::time::sleep(std::time::Duration::from_millis(exp + stagger)).await;
941}
942
943pub fn fmt_ts(dt: DateTime<Utc>) -> String {
945 dt.to_rfc3339_opts(chrono::SecondsFormat::Nanos, true)
946}
947
948pub fn parse_ts(raw: &str) -> DateTime<Utc> {
952 DateTime::parse_from_rfc3339(raw)
953 .map(|d| d.to_utc())
954 .unwrap_or_else(|_| Utc::now())
955}
956
957pub fn is_expired(claimed_at: &str, now: DateTime<Utc>, window: Duration) -> bool {
961 match DateTime::parse_from_rfc3339(claimed_at) {
962 Ok(t) => now
963 .signed_duration_since(t.with_timezone(&Utc))
964 .to_std()
965 .map(|age| age >= window)
966 .unwrap_or(false),
967 Err(_) => false,
968 }
969}
970
971pub fn threshold(now: DateTime<Utc>, window: Duration) -> String {
973 let delta =
974 chrono::Duration::from_std(window).unwrap_or_else(|_| chrono::Duration::days(36_500));
975 fmt_ts(now - delta)
976}
977
978pub fn encode_body(rec: &RunRecord) -> Result<String, HistoryError> {
979 serde_json::to_string(rec).map_err(|e| HistoryError::Backend(format!("encode run record: {e}")))
980}
981
982pub fn decode_body(body: &str) -> Result<RunRecord, HistoryError> {
983 serde_json::from_str(body).map_err(|e| HistoryError::Backend(format!("decode run record: {e}")))
984}
985
986pub fn encode_json<T: serde::Serialize>(value: &T, what: &str) -> Result<String, HistoryError> {
988 serde_json::to_string(value).map_err(|e| HistoryError::Backend(format!("encode {what}: {e}")))
989}
990
991pub fn decode_json<T: serde::de::DeserializeOwned>(
992 body: &str,
993 what: &str,
994) -> Result<T, HistoryError> {
995 serde_json::from_str(body).map_err(|e| HistoryError::Backend(format!("decode {what}: {e}")))
996}
997
998pub fn parse_status(s: &str) -> RunStatus {
999 match s {
1000 "queued" => RunStatus::Queued,
1001 "pending" => RunStatus::Pending,
1002 "running" => RunStatus::Running,
1003 "sharded" => RunStatus::Sharded,
1004 "completed" => RunStatus::Completed,
1005 "cancelled" => RunStatus::Cancelled,
1006 _ => RunStatus::Failed,
1007 }
1008}
1009
1010macro_rules! impl_sql_history {
1014 ($name:ident, $pool:ty) => {
1015 pub struct $name {
1018 pool: $pool,
1019 idem_retention: std::time::Duration,
1020 instance_id: String,
1022 lease_ttl: std::time::Duration,
1024 stmts: $crate::serve::history::sql::Stmts,
1025 }
1026
1027 impl $name {
1028 pub fn from_parts(
1030 pool: $pool,
1031 idem_retention: std::time::Duration,
1032 lease_ttl: std::time::Duration,
1033 instance_id: String,
1034 stmts: $crate::serve::history::sql::Stmts,
1035 ) -> Self {
1036 Self {
1037 pool,
1038 idem_retention,
1039 instance_id,
1040 lease_ttl,
1041 stmts,
1042 }
1043 }
1044
1045 pub fn pool(&self) -> &$pool {
1047 &self.pool
1048 }
1049 }
1050
1051 #[async_trait::async_trait]
1052 impl $crate::serve::history::RunHistory for $name {
1053 async fn claim_idempotency(
1054 &self,
1055 key: &str,
1056 fingerprint: &str,
1057 run_id: &str,
1058 window: std::time::Duration,
1059 ) -> Result<$crate::serve::history::Claim, $crate::serve::history::HistoryError> {
1060 use sqlx::Row as _;
1061 use $crate::serve::history::Claim;
1062 use $crate::serve::history::HistoryError;
1063 use $crate::serve::history::sql;
1064
1065 let now = chrono::Utc::now();
1066 let now_s = sql::fmt_ts(now);
1067 let backend = |e: sqlx::Error| HistoryError::Backend(e.to_string());
1068
1069 for _ in 0..sql::CLAIM_ATTEMPTS {
1070 let inserted = sqlx::query(&self.stmts.insert_idem)
1072 .bind(key)
1073 .bind(run_id)
1074 .bind(fingerprint)
1075 .bind(&now_s)
1076 .execute(&self.pool)
1077 .await
1078 .map_err(backend)?
1079 .rows_affected();
1080 if inserted == 1 {
1081 return Ok(Claim::Fresh);
1082 }
1083 let Some(row) = sqlx::query(&self.stmts.select_idem)
1085 .bind(key)
1086 .fetch_optional(&self.pool)
1087 .await
1088 .map_err(backend)?
1089 else {
1090 continue;
1092 };
1093 let existing_run: String = row.try_get("run_id").map_err(backend)?;
1094 let existing_fp: String = row.try_get("fingerprint").map_err(backend)?;
1095 let claimed_at: String = row.try_get("claimed_at").map_err(backend)?;
1096
1097 if sql::is_expired(&claimed_at, now, window) {
1098 let took = sqlx::query(&self.stmts.takeover_idem)
1101 .bind(run_id)
1102 .bind(fingerprint)
1103 .bind(&now_s)
1104 .bind(key)
1105 .bind(&claimed_at)
1106 .execute(&self.pool)
1107 .await
1108 .map_err(backend)?
1109 .rows_affected();
1110 if took == 1 {
1111 return Ok(Claim::Fresh);
1112 }
1113 continue; }
1115 return Ok(if existing_fp == fingerprint {
1116 Claim::Replay(existing_run)
1117 } else {
1118 Claim::Conflict
1119 });
1120 }
1121 tracing::warn!(
1124 key,
1125 "idempotency claim exhausted retries; reporting conflict"
1126 );
1127 Ok(Claim::Conflict)
1128 }
1129
1130 async fn upsert(
1131 &self,
1132 rec: &$crate::serve::history::RunRecord,
1133 ) -> Result<(), $crate::serve::history::HistoryError> {
1134 use $crate::serve::history::HistoryError;
1135 use $crate::serve::history::sql;
1136 let body = sql::encode_body(rec)?;
1137 let submitted = sql::fmt_ts(rec.submitted_at);
1138 let finished = rec.finished_at.map(sql::fmt_ts);
1139 let lease = sql::fmt_ts(chrono::Utc::now() + self.lease_ttl);
1144 sqlx::query(&self.stmts.upsert)
1145 .bind(&rec.run_id)
1146 .bind(rec.name.as_deref())
1147 .bind(rec.status.as_str())
1148 .bind(&submitted)
1149 .bind(finished.as_deref())
1150 .bind(rec.idempotency_key.as_deref())
1151 .bind(&self.instance_id)
1152 .bind(&lease)
1153 .bind(&body)
1154 .execute(&self.pool)
1155 .await
1156 .map_err(|e| HistoryError::Backend(e.to_string()))?;
1157 Ok(())
1158 }
1159
1160 async fn get(
1161 &self,
1162 id: &str,
1163 ) -> Result<
1164 Option<$crate::serve::history::RunRecord>,
1165 $crate::serve::history::HistoryError,
1166 > {
1167 use sqlx::Row as _;
1168 use $crate::serve::history::HistoryError;
1169 use $crate::serve::history::sql;
1170 let row = sqlx::query(&self.stmts.select_body)
1171 .bind(id)
1172 .fetch_optional(&self.pool)
1173 .await
1174 .map_err(|e| HistoryError::Backend(e.to_string()))?;
1175 match row {
1176 None => Ok(None),
1177 Some(r) => {
1178 let body: String = r
1179 .try_get("body")
1180 .map_err(|e| HistoryError::Backend(e.to_string()))?;
1181 Ok(Some(sql::decode_body(&body)?))
1182 }
1183 }
1184 }
1185
1186 async fn list(
1187 &self,
1188 filter: &$crate::serve::history::ListFilter,
1189 ) -> Result<$crate::serve::history::ListPage, $crate::serve::history::HistoryError>
1190 {
1191 use sqlx::Row as _;
1192 use $crate::serve::history::HistoryError;
1193 use $crate::serve::history::ListPage;
1194 use $crate::serve::history::sql;
1195 let backend = |e: sqlx::Error| HistoryError::Backend(e.to_string());
1196
1197 let cursor_ts: Option<String> = match &filter.cursor {
1201 None => None,
1202 Some(c) => sqlx::query(&self.stmts.select_submitted)
1203 .bind(c)
1204 .fetch_optional(&self.pool)
1205 .await
1206 .map_err(backend)?
1207 .map(|r| r.try_get::<String, _>("submitted_at"))
1208 .transpose()
1209 .map_err(backend)?,
1210 };
1211 let cur_id = if cursor_ts.is_some() {
1212 filter.cursor.as_deref()
1213 } else {
1214 None
1215 };
1216
1217 let status_s = filter.status.map(|s| s.as_str());
1218 let name_s = filter.name.as_deref();
1219 let since_s = filter.since.map(sql::fmt_ts);
1220 let until_s = filter.until.map(sql::fmt_ts);
1221 let limit = filter.limit.max(1);
1222 let fetch_n = limit as i64 + 1; let rows = sqlx::query(&self.stmts.list)
1225 .bind(status_s)
1226 .bind(status_s)
1227 .bind(name_s)
1228 .bind(name_s)
1229 .bind(since_s.as_deref())
1230 .bind(since_s.as_deref())
1231 .bind(until_s.as_deref())
1232 .bind(until_s.as_deref())
1233 .bind(cursor_ts.as_deref())
1234 .bind(cursor_ts.as_deref())
1235 .bind(cursor_ts.as_deref())
1236 .bind(cur_id)
1237 .bind(fetch_n)
1238 .fetch_all(&self.pool)
1239 .await
1240 .map_err(backend)?;
1241
1242 let mut runs = Vec::with_capacity(rows.len());
1243 for r in &rows {
1244 let body: String = r.try_get("body").map_err(backend)?;
1245 runs.push(sql::decode_body(&body)?);
1246 }
1247 let next_cursor = if runs.len() > limit {
1248 Some(runs[limit - 1].run_id.clone())
1249 } else {
1250 None
1251 };
1252 runs.truncate(limit);
1253 Ok(ListPage { runs, next_cursor })
1254 }
1255
1256 async fn delete(
1257 &self,
1258 id: &str,
1259 ) -> Result<$crate::serve::history::DeleteOutcome, $crate::serve::history::HistoryError>
1260 {
1261 use sqlx::Row as _;
1262 use $crate::serve::history::DeleteOutcome;
1263 use $crate::serve::history::HistoryError;
1264 use $crate::serve::history::sql;
1265 let backend = |e: sqlx::Error| HistoryError::Backend(e.to_string());
1266 let status: Option<String> = sqlx::query(&self.stmts.select_status)
1267 .bind(id)
1268 .fetch_optional(&self.pool)
1269 .await
1270 .map_err(backend)?
1271 .map(|r| r.try_get::<String, _>("status"))
1272 .transpose()
1273 .map_err(backend)?;
1274 match status {
1275 None => Ok(DeleteOutcome::NotFound),
1276 Some(s) if !sql::parse_status(&s).is_terminal() => {
1277 Ok(DeleteOutcome::StillRunning)
1278 }
1279 Some(_) => {
1280 sqlx::query(&self.stmts.delete)
1281 .bind(id)
1282 .execute(&self.pool)
1283 .await
1284 .map_err(backend)?;
1285 sqlx::query(&self.stmts.delete_idem_by_run)
1291 .bind(id)
1292 .execute(&self.pool)
1293 .await
1294 .map_err(backend)?;
1295 sqlx::query(&self.stmts.delete_shards_by_run)
1299 .bind(id)
1300 .execute(&self.pool)
1301 .await
1302 .map_err(backend)?;
1303 Ok(DeleteOutcome::Deleted)
1304 }
1305 }
1306 }
1307
1308 async fn release_idempotency(
1309 &self,
1310 run_id: &str,
1311 ) -> Result<(), $crate::serve::history::HistoryError> {
1312 use $crate::serve::history::HistoryError;
1313 sqlx::query(&self.stmts.delete_idem_by_run)
1314 .bind(run_id)
1315 .execute(&self.pool)
1316 .await
1317 .map_err(|e| HistoryError::Backend(e.to_string()))?;
1318 Ok(())
1319 }
1320
1321 async fn purge_expired(
1322 &self,
1323 retain_for: std::time::Duration,
1324 ) -> Result<usize, $crate::serve::history::HistoryError> {
1325 use $crate::serve::history::HistoryError;
1326 use $crate::serve::history::sql;
1327 let backend = |e: sqlx::Error| HistoryError::Backend(e.to_string());
1328 let now = chrono::Utc::now();
1329 let removed = sqlx::query(&self.stmts.purge_runs)
1330 .bind(sql::threshold(now, retain_for))
1331 .execute(&self.pool)
1332 .await
1333 .map_err(backend)?
1334 .rows_affected() as usize;
1335 let _ = sqlx::query(&self.stmts.purge_idem)
1337 .bind(sql::threshold(now, self.idem_retention))
1338 .execute(&self.pool)
1339 .await;
1340 let _ = sqlx::query(&self.stmts.prune_instances)
1344 .bind(sql::threshold(now, retain_for))
1345 .execute(&self.pool)
1346 .await;
1347 let _ = sqlx::query(&self.stmts.purge_orphan_shards)
1351 .execute(&self.pool)
1352 .await;
1353 let _ = sqlx::query(&self.stmts.purge_audit)
1355 .bind(sql::threshold(now, retain_for))
1356 .execute(&self.pool)
1357 .await;
1358 Ok(removed)
1359 }
1360
1361 async fn recover_orphans(&self) -> Result<usize, $crate::serve::history::HistoryError> {
1362 use sqlx::Row as _;
1363 use $crate::serve::history::HistoryError;
1364 use $crate::serve::history::RunStatus;
1365 use $crate::serve::history::sql;
1366 let backend = |e: sqlx::Error| HistoryError::Backend(e.to_string());
1367 let now = chrono::Utc::now();
1368 let rows = sqlx::query(&self.stmts.select_orphans)
1373 .bind(sql::fmt_ts(now))
1374 .fetch_all(&self.pool)
1375 .await
1376 .map_err(backend)?;
1377 let mut count = 0usize;
1378 for r in &rows {
1379 let body: String = r.try_get("body").map_err(backend)?;
1380 let mut rec = sql::decode_body(&body)?;
1381 rec.status = RunStatus::Failed;
1382 rec.finished_at = Some(now);
1383 rec.error = Some(
1384 "owning serve instance's lease expired before the run finished".into(),
1385 );
1386 if rec.elapsed_secs.is_none()
1387 && let Some(started) = rec.started_at
1388 {
1389 rec.elapsed_secs = (now - started).to_std().ok().map(|d| d.as_secs_f64());
1390 }
1391 self.upsert(&rec).await?;
1392 count += 1;
1393 }
1394 Ok(count)
1395 }
1396
1397 async fn renew_leases(&self) -> Result<usize, $crate::serve::history::HistoryError> {
1398 use $crate::serve::history::HistoryError;
1399 use $crate::serve::history::sql;
1400 let backend = |e: sqlx::Error| HistoryError::Backend(e.to_string());
1401 let new_lease = sql::fmt_ts(chrono::Utc::now() + self.lease_ttl);
1402 let renewed = sqlx::query(&self.stmts.renew_leases)
1403 .bind(&new_lease)
1404 .bind(&self.instance_id)
1405 .execute(&self.pool)
1406 .await
1407 .map_err(backend)?
1408 .rows_affected() as usize;
1409 Ok(renewed)
1410 }
1411
1412 async fn claim_pending(
1413 &self,
1414 limit: usize,
1415 ) -> Result<Vec<$crate::serve::history::RunRecord>, $crate::serve::history::HistoryError>
1416 {
1417 use sqlx::Row as _;
1418 use $crate::serve::history::HistoryError;
1419 use $crate::serve::history::RunStatus;
1420 use $crate::serve::history::sql;
1421 let backend = |e: sqlx::Error| HistoryError::Backend(e.to_string());
1422 if limit == 0 {
1423 return Ok(Vec::new());
1424 }
1425 let now = chrono::Utc::now();
1426 let lease = sql::fmt_ts(now + self.lease_ttl);
1427
1428 let rows = sqlx::query(&self.stmts.select_pending)
1430 .bind(limit as i64)
1431 .fetch_all(&self.pool)
1432 .await
1433 .map_err(backend)?;
1434
1435 let mut claimed = Vec::new();
1440 for row in &rows {
1441 let run_id: String = row.try_get("run_id").map_err(backend)?;
1442 let body: String = row.try_get("body").map_err(backend)?;
1443 let mut r = sql::decode_body(&body)?;
1447 r.status = RunStatus::Running;
1448 let new_body = sql::encode_body(&r)?;
1449 let won = sqlx::query(&self.stmts.claim_one)
1451 .bind(&self.instance_id)
1452 .bind(&lease)
1453 .bind(&new_body)
1454 .bind(&run_id)
1455 .execute(&self.pool)
1456 .await
1457 .map_err(backend)?
1458 .rows_affected();
1459 if won == 1 {
1460 claimed.push(r);
1461 }
1462 }
1463 Ok(claimed)
1464 }
1465
1466 async fn reclaim_orphans(
1467 &self,
1468 max_attempts: u32,
1469 ) -> Result<$crate::serve::history::ReclaimReport, $crate::serve::history::HistoryError>
1470 {
1471 use sqlx::Row as _;
1472 use $crate::serve::history::HistoryError;
1473 use $crate::serve::history::ReclaimReport;
1474 use $crate::serve::history::RunStatus;
1475 use $crate::serve::history::sql;
1476 let backend = |e: sqlx::Error| HistoryError::Backend(e.to_string());
1477 let now = chrono::Utc::now();
1478 let now_s = sql::fmt_ts(now);
1479
1480 let rows = sqlx::query(&self.stmts.reclaim_select)
1481 .bind(&now_s)
1482 .fetch_all(&self.pool)
1483 .await
1484 .map_err(backend)?;
1485
1486 let mut report = ReclaimReport::default();
1487 for row in &rows {
1488 let body: String = row.try_get("body").map_err(backend)?;
1489 let mut rec = sql::decode_body(&body)?;
1490 let next_attempt = rec.attempt + 1;
1491 if rec.attempt < max_attempts {
1495 rec.attempt = next_attempt;
1497 rec.status = RunStatus::Pending;
1498 let new_body = sql::encode_body(&rec)?;
1499 let n = sqlx::query(&self.stmts.reclaim_requeue)
1500 .bind(&new_body)
1501 .bind(&rec.run_id)
1502 .bind(&now_s)
1503 .execute(&self.pool)
1504 .await
1505 .map_err(backend)?
1506 .rows_affected();
1507 if n == 1 {
1508 report.requeued += 1;
1509 }
1510 } else {
1511 rec.attempt = next_attempt;
1513 rec.status = RunStatus::Failed;
1514 rec.finished_at = Some(now);
1515 rec.error = Some(format!(
1516 "run reclaimed {next_attempt} times after its owning instance's \
1517 lease expired; giving up (poison run)"
1518 ));
1519 if rec.elapsed_secs.is_none()
1520 && let Some(started) = rec.started_at
1521 {
1522 rec.elapsed_secs =
1523 (now - started).to_std().ok().map(|d| d.as_secs_f64());
1524 }
1525 let new_body = sql::encode_body(&rec)?;
1526 let n = sqlx::query(&self.stmts.reclaim_fail)
1527 .bind(&now_s)
1528 .bind(&new_body)
1529 .bind(&rec.run_id)
1530 .bind(&now_s)
1531 .execute(&self.pool)
1532 .await
1533 .map_err(backend)?
1534 .rows_affected();
1535 if n == 1 {
1536 report.failed += 1;
1537 }
1538 }
1539 }
1540 Ok(report)
1541 }
1542
1543 async fn finalize_owned(
1544 &self,
1545 rec: &$crate::serve::history::RunRecord,
1546 ) -> Result<bool, $crate::serve::history::HistoryError> {
1547 use $crate::serve::history::HistoryError;
1548 use $crate::serve::history::sql;
1549 let backend = |e: sqlx::Error| HistoryError::Backend(e.to_string());
1550 let mut rec = rec.clone();
1554 if rec.status.is_terminal() && rec.finished_at.is_none() {
1555 rec.finished_at = Some(chrono::Utc::now());
1556 }
1557 let body = sql::encode_body(&rec)?;
1558 let finished = rec.finished_at.map(sql::fmt_ts);
1559 let lease = sql::fmt_ts(chrono::Utc::now() + self.lease_ttl);
1560 let n = sqlx::query(&self.stmts.finalize_owned)
1561 .bind(rec.status.as_str())
1562 .bind(finished.as_deref())
1563 .bind(&lease)
1564 .bind(&body)
1565 .bind(&rec.run_id)
1566 .bind(&self.instance_id)
1567 .execute(&self.pool)
1568 .await
1569 .map_err(backend)?
1570 .rows_affected();
1571 Ok(n == 1)
1572 }
1573
1574 async fn finalize_sharded_parent(
1575 &self,
1576 run_id: &str,
1577 status: $crate::serve::history::RunStatus,
1578 finished_at: chrono::DateTime<chrono::Utc>,
1579 error: Option<String>,
1580 ) -> Result<bool, $crate::serve::history::HistoryError> {
1581 use sqlx::Row as _;
1582 use $crate::serve::history::HistoryError;
1583 use $crate::serve::history::RunStatus;
1584 use $crate::serve::history::sql;
1585 let backend = |e: sqlx::Error| HistoryError::Backend(e.to_string());
1586 let Some(row) = sqlx::query(&self.stmts.select_body)
1591 .bind(run_id)
1592 .fetch_optional(&self.pool)
1593 .await
1594 .map_err(backend)?
1595 else {
1596 return Ok(false);
1597 };
1598 let body: String = row.try_get("body").map_err(backend)?;
1599 let mut rec = sql::decode_body(&body)?;
1600 if rec.status != RunStatus::Sharded {
1601 return Ok(false);
1602 }
1603 rec.status = status;
1604 rec.finished_at = Some(finished_at);
1605 rec.error = error;
1606 let new_body = sql::encode_body(&rec)?;
1607 let n = sqlx::query(&self.stmts.finalize_sharded_parent)
1608 .bind(status.as_str())
1609 .bind(sql::fmt_ts(finished_at))
1610 .bind(&new_body)
1611 .bind(run_id)
1612 .execute(&self.pool)
1613 .await
1614 .map_err(backend)?
1615 .rows_affected();
1616 Ok(n == 1)
1617 }
1618
1619 async fn cancel_pending(
1620 &self,
1621 run_id: &str,
1622 ) -> Result<bool, $crate::serve::history::HistoryError> {
1623 use sqlx::Row as _;
1624 use $crate::serve::history::HistoryError;
1625 use $crate::serve::history::RunStatus;
1626 use $crate::serve::history::sql;
1627 let backend = |e: sqlx::Error| HistoryError::Backend(e.to_string());
1628 let Some(row) = sqlx::query(&self.stmts.select_body)
1631 .bind(run_id)
1632 .fetch_optional(&self.pool)
1633 .await
1634 .map_err(backend)?
1635 else {
1636 return Ok(false);
1637 };
1638 let body: String = row.try_get("body").map_err(backend)?;
1639 let mut rec = sql::decode_body(&body)?;
1640 if rec.status != RunStatus::Pending {
1641 return Ok(false);
1642 }
1643 let now = chrono::Utc::now();
1644 rec.status = RunStatus::Cancelled;
1645 rec.finished_at = Some(now);
1646 let new_body = sql::encode_body(&rec)?;
1647 let n = sqlx::query(&self.stmts.cancel_pending)
1648 .bind(sql::fmt_ts(now))
1649 .bind(&new_body)
1650 .bind(run_id)
1651 .execute(&self.pool)
1652 .await
1653 .map_err(backend)?
1654 .rows_affected();
1655 Ok(n == 1)
1656 }
1657
1658 async fn request_cancel(
1659 &self,
1660 run_id: &str,
1661 ) -> Result<(), $crate::serve::history::HistoryError> {
1662 use $crate::serve::history::HistoryError;
1663 use $crate::serve::history::sql;
1664 let backend = |e: sqlx::Error| HistoryError::Backend(e.to_string());
1665 sqlx::query(&self.stmts.request_cancel)
1666 .bind(sql::fmt_ts(chrono::Utc::now()))
1667 .bind(run_id)
1668 .execute(&self.pool)
1669 .await
1670 .map_err(backend)?;
1671 Ok(())
1672 }
1673
1674 async fn pending_cancellations(
1675 &self,
1676 ) -> Result<Vec<String>, $crate::serve::history::HistoryError> {
1677 use sqlx::Row as _;
1678 use $crate::serve::history::HistoryError;
1679 let backend = |e: sqlx::Error| HistoryError::Backend(e.to_string());
1680 let rows = sqlx::query(&self.stmts.pending_cancellations)
1681 .bind(&self.instance_id)
1682 .fetch_all(&self.pool)
1683 .await
1684 .map_err(backend)?;
1685 let mut ids = Vec::with_capacity(rows.len());
1686 for r in &rows {
1687 ids.push(r.try_get::<String, _>("run_id").map_err(backend)?);
1688 }
1689 Ok(ids)
1690 }
1691
1692 async fn heartbeat_instance(
1693 &self,
1694 beat: &$crate::serve::history::InstanceHeartbeat,
1695 ) -> Result<(), $crate::serve::history::HistoryError> {
1696 use $crate::serve::history::HistoryError;
1697 use $crate::serve::history::sql;
1698 let backend = |e: sqlx::Error| HistoryError::Backend(e.to_string());
1699 let now = sql::fmt_ts(chrono::Utc::now());
1700 sqlx::query(&self.stmts.heartbeat_instance)
1701 .bind(&self.instance_id)
1702 .bind(sql::fmt_ts(beat.started_at))
1703 .bind(&now)
1704 .bind(beat.listen.as_deref())
1705 .bind(beat.max_concurrent.to_string())
1706 .bind(beat.in_flight.to_string())
1707 .execute(&self.pool)
1708 .await
1709 .map_err(backend)?;
1710 Ok(())
1711 }
1712
1713 async fn live_instances(
1714 &self,
1715 ttl: std::time::Duration,
1716 ) -> Result<Vec<$crate::serve::history::InstanceRecord>, $crate::serve::history::HistoryError>
1717 {
1718 use sqlx::Row as _;
1719 use $crate::serve::history::HistoryError;
1720 use $crate::serve::history::InstanceRecord;
1721 use $crate::serve::history::sql;
1722 let backend = |e: sqlx::Error| HistoryError::Backend(e.to_string());
1723 let now = chrono::Utc::now();
1724 let rows = sqlx::query(&self.stmts.live_instances)
1725 .bind(sql::threshold(now, ttl))
1726 .fetch_all(&self.pool)
1727 .await
1728 .map_err(backend)?;
1729 let parse_dt = |s: &str| {
1730 chrono::DateTime::parse_from_rfc3339(s)
1731 .map(|d| d.to_utc())
1732 .unwrap_or(now)
1733 };
1734 let mut out = Vec::with_capacity(rows.len());
1735 for r in &rows {
1736 let started: String = r.try_get("started_at").map_err(backend)?;
1737 let hb: String = r.try_get("last_heartbeat").map_err(backend)?;
1738 let mc: Option<String> = r.try_get("max_concurrent").map_err(backend)?;
1739 let inf: Option<String> = r.try_get("in_flight").map_err(backend)?;
1740 out.push(InstanceRecord {
1741 instance_id: r.try_get("instance_id").map_err(backend)?,
1742 started_at: parse_dt(&started),
1743 last_heartbeat: parse_dt(&hb),
1744 listen: r.try_get("listen").map_err(backend)?,
1745 max_concurrent: mc.and_then(|s| s.parse().ok()).unwrap_or(0),
1746 in_flight: inf.and_then(|s| s.parse().ok()).unwrap_or(0),
1747 });
1748 }
1749 Ok(out)
1750 }
1751
1752 async fn insert_shards(
1755 &self,
1756 run_id: &str,
1757 shards: &[$crate::serve::history::ShardInsert],
1758 ) -> Result<usize, $crate::serve::history::HistoryError> {
1759 use $crate::serve::history::HistoryError;
1760 let backend = |e: sqlx::Error| HistoryError::Backend(e.to_string());
1761 let mut inserted = 0usize;
1762 for s in shards {
1763 let descriptor = serde_json::to_string(&s.descriptor).map_err(|e| {
1764 HistoryError::Backend(format!("encode shard descriptor: {e}"))
1765 })?;
1766 let size = s.size_estimate.map(|n| n.to_string());
1767 let n = sqlx::query(&self.stmts.insert_shard)
1768 .bind(run_id)
1769 .bind(&s.shard_id)
1770 .bind(&descriptor)
1771 .bind(size.as_deref())
1772 .execute(&self.pool)
1773 .await
1774 .map_err(backend)?
1775 .rows_affected();
1776 inserted += n as usize;
1777 }
1778 Ok(inserted)
1779 }
1780
1781 async fn claim_shards(
1782 &self,
1783 limit: usize,
1784 ) -> Result<
1785 Vec<$crate::serve::history::ClaimedShard>,
1786 $crate::serve::history::HistoryError,
1787 > {
1788 use sqlx::Row as _;
1789 use $crate::serve::history::ClaimedShard;
1790 use $crate::serve::history::HistoryError;
1791 use $crate::serve::history::sql;
1792 let backend = |e: sqlx::Error| HistoryError::Backend(e.to_string());
1793 if limit == 0 {
1794 return Ok(Vec::new());
1795 }
1796 let lease = sql::fmt_ts(chrono::Utc::now() + self.lease_ttl);
1797
1798 let rows = sqlx::query(&self.stmts.claim_shards_select)
1801 .bind(limit as i64)
1802 .fetch_all(&self.pool)
1803 .await
1804 .map_err(backend)?;
1805
1806 let mut claimed = Vec::new();
1808 for row in &rows {
1809 let run_id: String = row.try_get("run_id").map_err(backend)?;
1810 let shard_id: String = row.try_get("shard_id").map_err(backend)?;
1811 let descriptor_s: String = row.try_get("descriptor").map_err(backend)?;
1812 let body: String = row.try_get("body").map_err(backend)?;
1813 let won = sqlx::query(&self.stmts.claim_shard_one)
1814 .bind(&self.instance_id)
1815 .bind(&lease)
1816 .bind(&run_id)
1817 .bind(&shard_id)
1818 .execute(&self.pool)
1819 .await
1820 .map_err(backend)?
1821 .rows_affected();
1822 if won == 1 {
1823 let descriptor: serde_json::Value = serde_json::from_str(&descriptor_s)
1824 .map_err(|e| {
1825 HistoryError::Backend(format!("decode shard descriptor: {e}"))
1826 })?;
1827 let run = sql::decode_body(&body)?;
1828 claimed.push(ClaimedShard {
1829 run_id,
1830 shard_id,
1831 descriptor,
1832 run,
1833 });
1834 }
1835 }
1836 Ok(claimed)
1837 }
1838
1839 async fn renew_shard_leases(
1840 &self,
1841 ) -> Result<usize, $crate::serve::history::HistoryError> {
1842 use $crate::serve::history::HistoryError;
1843 use $crate::serve::history::sql;
1844 let backend = |e: sqlx::Error| HistoryError::Backend(e.to_string());
1845 let lease = sql::fmt_ts(chrono::Utc::now() + self.lease_ttl);
1846 let n = sqlx::query(&self.stmts.renew_shard_leases)
1847 .bind(&lease)
1848 .bind(&self.instance_id)
1849 .execute(&self.pool)
1850 .await
1851 .map_err(backend)?
1852 .rows_affected() as usize;
1853 Ok(n)
1854 }
1855
1856 async fn reclaim_shards(
1857 &self,
1858 max_attempts: u32,
1859 ) -> Result<$crate::serve::history::ReclaimReport, $crate::serve::history::HistoryError>
1860 {
1861 use sqlx::Row as _;
1862 use $crate::serve::history::HistoryError;
1863 use $crate::serve::history::ReclaimReport;
1864 use $crate::serve::history::sql;
1865 let backend = |e: sqlx::Error| HistoryError::Backend(e.to_string());
1866 let now_s = sql::fmt_ts(chrono::Utc::now());
1867
1868 let rows = sqlx::query(&self.stmts.reclaim_shards_select)
1869 .bind(&now_s)
1870 .fetch_all(&self.pool)
1871 .await
1872 .map_err(backend)?;
1873
1874 let mut report = ReclaimReport::default();
1875 for row in &rows {
1876 let run_id: String = row.try_get("run_id").map_err(backend)?;
1877 let shard_id: String = row.try_get("shard_id").map_err(backend)?;
1878 let attempt_s: String = row.try_get("attempt").map_err(backend)?;
1879 let attempt: u32 = attempt_s.parse().unwrap_or(0);
1880 if attempt < max_attempts {
1881 let next = (attempt + 1).to_string();
1882 let n = sqlx::query(&self.stmts.reclaim_shard_requeue)
1883 .bind(&next)
1884 .bind(&run_id)
1885 .bind(&shard_id)
1886 .bind(&now_s)
1887 .execute(&self.pool)
1888 .await
1889 .map_err(backend)?
1890 .rows_affected();
1891 if n == 1 {
1892 report.requeued += 1;
1893 }
1894 } else {
1895 let n = sqlx::query(&self.stmts.reclaim_shard_fail)
1896 .bind(&now_s)
1897 .bind(&run_id)
1898 .bind(&shard_id)
1899 .bind(&now_s)
1900 .execute(&self.pool)
1901 .await
1902 .map_err(backend)?
1903 .rows_affected();
1904 if n == 1 {
1905 report.failed += 1;
1906 }
1907 }
1908 }
1909 Ok(report)
1910 }
1911
1912 async fn finalize_shard(
1913 &self,
1914 run_id: &str,
1915 shard_id: &str,
1916 success: bool,
1917 ) -> Result<bool, $crate::serve::history::HistoryError> {
1918 use $crate::serve::history::HistoryError;
1919 use $crate::serve::history::sql;
1920 let backend = |e: sqlx::Error| HistoryError::Backend(e.to_string());
1921 let status = if success { "completed" } else { "failed" };
1922 let now_s = sql::fmt_ts(chrono::Utc::now());
1923 let n = sqlx::query(&self.stmts.finalize_shard)
1924 .bind(status)
1925 .bind(&now_s)
1926 .bind(run_id)
1927 .bind(shard_id)
1928 .bind(&self.instance_id)
1929 .execute(&self.pool)
1930 .await
1931 .map_err(backend)?
1932 .rows_affected();
1933 Ok(n == 1)
1934 }
1935
1936 async fn shard_progress(
1937 &self,
1938 run_id: &str,
1939 ) -> Result<$crate::serve::history::ShardProgress, $crate::serve::history::HistoryError>
1940 {
1941 use sqlx::Row as _;
1942 use $crate::serve::history::HistoryError;
1943 use $crate::serve::history::ShardProgress;
1944 let backend = |e: sqlx::Error| HistoryError::Backend(e.to_string());
1945 let rows = sqlx::query(&self.stmts.shard_progress)
1946 .bind(run_id)
1947 .fetch_all(&self.pool)
1948 .await
1949 .map_err(backend)?;
1950 let mut p = ShardProgress::default();
1951 for row in &rows {
1952 let status: String = row.try_get("status").map_err(backend)?;
1953 let n: i64 = row.try_get("n").map_err(backend)?;
1954 let n = n.max(0) as usize;
1955 p.total += n;
1956 match status.as_str() {
1957 "completed" => p.completed += n,
1958 "failed" => p.failed += n,
1959 "running" => p.running += n,
1960 _ => p.pending += n,
1961 }
1962 }
1963 Ok(p)
1964 }
1965
1966 async fn pending_shard_cancellations(
1967 &self,
1968 ) -> Result<Vec<String>, $crate::serve::history::HistoryError> {
1969 use sqlx::Row as _;
1970 use $crate::serve::history::HistoryError;
1971 let backend = |e: sqlx::Error| HistoryError::Backend(e.to_string());
1972 let rows = sqlx::query(&self.stmts.pending_shard_cancellations)
1973 .bind(&self.instance_id)
1974 .fetch_all(&self.pool)
1975 .await
1976 .map_err(backend)?;
1977 let mut ids = Vec::with_capacity(rows.len());
1978 for r in &rows {
1979 ids.push(r.try_get::<String, _>("run_id").map_err(backend)?);
1980 }
1981 Ok(ids)
1982 }
1983
1984 async fn finalize_completed_sharded_parents(
1985 &self,
1986 ) -> Result<usize, $crate::serve::history::HistoryError> {
1987 use sqlx::Row as _;
1988 use $crate::serve::history::HistoryError;
1989 use $crate::serve::history::RunStatus;
1990 use $crate::serve::history::sql;
1991 let backend = |e: sqlx::Error| HistoryError::Backend(e.to_string());
1992
1993 let rows = sqlx::query(&self.stmts.select_sharded_parents)
1997 .fetch_all(&self.pool)
1998 .await
1999 .map_err(backend)?;
2000
2001 let mut finalized = 0usize;
2002 for row in &rows {
2003 let run_id: String = row.try_get("run_id").map_err(backend)?;
2004 let progress = self.shard_progress(&run_id).await?;
2005 if !progress.all_terminal() {
2006 continue;
2007 }
2008 let success = progress.failed == 0;
2009 let Some(body_row) = sqlx::query(&self.stmts.select_body)
2012 .bind(&run_id)
2013 .fetch_optional(&self.pool)
2014 .await
2015 .map_err(backend)?
2016 else {
2017 continue;
2018 };
2019 let body: String = body_row.try_get("body").map_err(backend)?;
2020 let mut rec = sql::decode_body(&body)?;
2021 if rec.status != RunStatus::Sharded {
2024 continue;
2025 }
2026 let now = chrono::Utc::now();
2027 rec.status = if success {
2028 RunStatus::Completed
2029 } else {
2030 RunStatus::Failed
2031 };
2032 rec.finished_at = Some(now);
2033 if !success {
2034 rec.error = Some(format!(
2035 "{}/{} shard(s) failed",
2036 progress.failed, progress.total
2037 ));
2038 }
2039 let new_body = sql::encode_body(&rec)?;
2040 let n = sqlx::query(&self.stmts.finalize_sharded_parent)
2041 .bind(rec.status.as_str())
2042 .bind(sql::fmt_ts(now))
2043 .bind(&new_body)
2044 .bind(&run_id)
2045 .execute(&self.pool)
2046 .await
2047 .map_err(backend)?
2048 .rows_affected();
2049 if n == 1 {
2050 finalized += 1;
2051 $crate::serve::metrics::record_run_finished(
2052 rec.status,
2053 if success { "ok" } else { "error" },
2054 );
2055 tracing::info!(
2056 run_id,
2057 shards = progress.total,
2058 failed = progress.failed,
2059 "sharded run finalized by sweep (F11)"
2060 );
2061 }
2062 }
2063 Ok(finalized)
2064 }
2065
2066 async fn record_audit(
2069 &self,
2070 entry: &$crate::serve::history::AuditEntry,
2071 ) -> Result<(), $crate::serve::history::HistoryError> {
2072 use $crate::serve::history::HistoryError;
2073 use $crate::serve::history::sql;
2074 let backend = |e: sqlx::Error| HistoryError::Backend(e.to_string());
2075 sqlx::query(&self.stmts.insert_audit)
2076 .bind(&entry.id)
2077 .bind(sql::fmt_ts(entry.timestamp))
2078 .bind(&entry.principal)
2079 .bind(&entry.role)
2080 .bind(&entry.action)
2081 .bind(entry.run_id.as_deref())
2082 .bind(entry.config_fingerprint.as_deref())
2083 .bind(entry.source_ip.as_deref())
2084 .bind(&entry.result)
2085 .execute(&self.pool)
2086 .await
2087 .map_err(backend)?;
2088 Ok(())
2089 }
2090
2091 async fn list_audit(
2092 &self,
2093 filter: &$crate::serve::history::AuditFilter,
2094 ) -> Result<
2095 Vec<$crate::serve::history::AuditEntry>,
2096 $crate::serve::history::HistoryError,
2097 > {
2098 use sqlx::Row as _;
2099 use $crate::serve::history::AuditEntry;
2100 use $crate::serve::history::HistoryError;
2101 use $crate::serve::history::sql;
2102 let backend = |e: sqlx::Error| HistoryError::Backend(e.to_string());
2103 let principal = filter.principal.as_deref();
2104 let action = filter.action.as_deref();
2105 let since = filter.since.map(sql::fmt_ts);
2106 let until = filter.until.map(sql::fmt_ts);
2107 let limit = filter.limit.max(1) as i64;
2108 let rows = sqlx::query(&self.stmts.list_audit)
2109 .bind(principal)
2110 .bind(principal)
2111 .bind(action)
2112 .bind(action)
2113 .bind(since.as_deref())
2114 .bind(since.as_deref())
2115 .bind(until.as_deref())
2116 .bind(until.as_deref())
2117 .bind(limit)
2118 .fetch_all(&self.pool)
2119 .await
2120 .map_err(backend)?;
2121 let mut out = Vec::with_capacity(rows.len());
2122 for r in &rows {
2123 let ts: String = r.try_get("ts").map_err(backend)?;
2124 let timestamp = $crate::serve::history::sql::parse_ts(&ts);
2125 out.push(AuditEntry {
2126 id: r.try_get("id").map_err(backend)?,
2127 timestamp,
2128 principal: r.try_get("principal").map_err(backend)?,
2129 role: r.try_get("role").map_err(backend)?,
2130 action: r.try_get("action").map_err(backend)?,
2131 run_id: r.try_get("run_id").map_err(backend)?,
2132 config_fingerprint: r.try_get("config_fingerprint").map_err(backend)?,
2133 source_ip: r.try_get("source_ip").map_err(backend)?,
2134 result: r.try_get("result").map_err(backend)?,
2135 });
2136 }
2137 Ok(out)
2138 }
2139
2140 async fn catalog_record(
2143 &self,
2144 update: &$crate::serve::history::catalog::CatalogUpdate,
2145 ) -> Result<(), $crate::serve::history::HistoryError> {
2146 use sqlx::Row as _;
2147 use $crate::serve::history::HistoryError;
2148 use $crate::serve::history::catalog;
2149 use $crate::serve::history::sql;
2150 let backend = |e: sqlx::Error| HistoryError::Backend(e.to_string());
2151 let now_s = sql::fmt_ts(update.recorded_at);
2152
2153 for obs in update.sources.iter().chain(std::iter::once(&update.sink)) {
2154 let id = catalog::dataset_id(&obs.uri);
2155 let existing = sqlx::query(&self.stmts.catalog_select_dataset)
2159 .bind(&id)
2160 .fetch_optional(&self.pool)
2161 .await
2162 .map_err(backend)?
2163 .map(|r| r.try_get::<String, _>("body"))
2164 .transpose()
2165 .map_err(backend)?
2166 .map(|b| {
2167 sql::decode_json::<catalog::CatalogDataset>(&b, "catalog dataset")
2168 })
2169 .transpose()?;
2170 let (ds, new_version) = catalog::apply_observation(
2171 existing.as_ref(),
2172 obs,
2173 &update.run_id,
2174 &update.pipeline,
2175 &update.row,
2176 update.recorded_at,
2177 );
2178 sqlx::query(&self.stmts.catalog_upsert_dataset)
2179 .bind(&ds.id)
2180 .bind(&ds.uri)
2181 .bind(&ds.kind)
2182 .bind(&now_s)
2183 .bind(sql::encode_json(&ds, "catalog dataset")?)
2184 .execute(&self.pool)
2185 .await
2186 .map_err(backend)?;
2187 if let Some(v) = new_version {
2188 sqlx::query(&self.stmts.catalog_insert_schema_version)
2189 .bind(&v.dataset_id)
2190 .bind(v.version.to_string())
2191 .bind(sql::fmt_ts(v.recorded_at))
2192 .bind(sql::encode_json(&v, "catalog schema version")?)
2193 .execute(&self.pool)
2194 .await
2195 .map_err(backend)?;
2196 }
2197 sqlx::query(&self.stmts.catalog_insert_stat)
2198 .bind(&id)
2199 .bind(&now_s)
2200 .bind(&update.run_id)
2201 .bind(obs.records.to_string())
2202 .execute(&self.pool)
2203 .await
2204 .map_err(backend)?;
2205 sqlx::query(&self.stmts.catalog_prune_stats)
2206 .bind(&id)
2207 .bind(&id)
2208 .bind(catalog::STATS_RETAIN as i64)
2209 .execute(&self.pool)
2210 .await
2211 .map_err(backend)?;
2212 }
2213
2214 let dst_id = catalog::dataset_id(&update.sink.uri);
2217 let existing_edges = self.catalog_all_edges().await?;
2218 for source in &update.sources {
2219 let src_id = catalog::dataset_id(&source.uri);
2220 let existing = existing_edges
2221 .iter()
2222 .find(|e| e.src_id == src_id && e.dst_id == dst_id);
2223 let edge = catalog::apply_edge(existing, update, source);
2224 sqlx::query(&self.stmts.catalog_upsert_edge)
2225 .bind(&edge.src_id)
2226 .bind(&edge.dst_id)
2227 .bind(&now_s)
2228 .bind(sql::encode_json(&edge, "catalog edge")?)
2229 .execute(&self.pool)
2230 .await
2231 .map_err(backend)?;
2232 }
2233 Ok(())
2234 }
2235
2236 async fn catalog_list_datasets(
2237 &self,
2238 filter: &$crate::serve::history::catalog::CatalogListFilter,
2239 ) -> Result<
2240 $crate::serve::history::catalog::CatalogDatasetPage,
2241 $crate::serve::history::HistoryError,
2242 > {
2243 use sqlx::Row as _;
2244 use $crate::serve::history::HistoryError;
2245 use $crate::serve::history::catalog;
2246 use $crate::serve::history::sql;
2247 let backend = |e: sqlx::Error| HistoryError::Backend(e.to_string());
2248 let rows = sqlx::query(&self.stmts.catalog_select_datasets)
2249 .fetch_all(&self.pool)
2250 .await
2251 .map_err(backend)?;
2252 let mut all = Vec::with_capacity(rows.len());
2253 for r in &rows {
2254 let body: String = r.try_get("body").map_err(backend)?;
2255 all.push(sql::decode_json(&body, "catalog dataset")?);
2256 }
2257 Ok(catalog::filter_datasets(all, filter))
2258 }
2259
2260 async fn catalog_get_dataset(
2261 &self,
2262 id: &str,
2263 ) -> Result<
2264 Option<$crate::serve::history::catalog::CatalogDatasetDetail>,
2265 $crate::serve::history::HistoryError,
2266 > {
2267 use sqlx::Row as _;
2268 use $crate::serve::history::HistoryError;
2269 use $crate::serve::history::catalog;
2270 use $crate::serve::history::sql;
2271 let backend = |e: sqlx::Error| HistoryError::Backend(e.to_string());
2272 let Some(row) = sqlx::query(&self.stmts.catalog_select_dataset)
2273 .bind(id)
2274 .fetch_optional(&self.pool)
2275 .await
2276 .map_err(backend)?
2277 else {
2278 return Ok(None);
2279 };
2280 let body: String = row.try_get("body").map_err(backend)?;
2281 let dataset: catalog::CatalogDataset =
2282 sql::decode_json(&body, "catalog dataset")?;
2283
2284 let rows = sqlx::query(&self.stmts.catalog_select_schema_versions)
2285 .bind(id)
2286 .fetch_all(&self.pool)
2287 .await
2288 .map_err(backend)?;
2289 let mut schema_timeline = Vec::with_capacity(rows.len());
2290 for r in &rows {
2291 let body: String = r.try_get("body").map_err(backend)?;
2292 schema_timeline.push(sql::decode_json(&body, "catalog schema version")?);
2293 }
2294
2295 let rows = sqlx::query(&self.stmts.catalog_select_stats)
2296 .bind(id)
2297 .bind(catalog::STATS_DETAIL_LIMIT as i64)
2298 .fetch_all(&self.pool)
2299 .await
2300 .map_err(backend)?;
2301 let mut stats = Vec::with_capacity(rows.len());
2302 for r in &rows {
2303 let recorded: String = r.try_get("recorded_at").map_err(backend)?;
2304 let run_id: String = r.try_get("run_id").map_err(backend)?;
2305 let records: String = r.try_get("records").map_err(backend)?;
2306 stats.push(catalog::CatalogStatsPoint {
2307 recorded_at: chrono::DateTime::parse_from_rfc3339(&recorded)
2308 .map(|d| d.to_utc())
2309 .unwrap_or_else(|_| chrono::Utc::now()),
2310 run_id,
2311 records: records.parse().unwrap_or(0),
2312 });
2313 }
2314
2315 let edges = self.catalog_all_edges().await?;
2316 let (downstream, rest): (Vec<_>, Vec<_>) =
2317 edges.into_iter().partition(|e| e.src_id == id);
2318 let upstream = rest.into_iter().filter(|e| e.dst_id == id).collect();
2319 Ok(Some(catalog::CatalogDatasetDetail {
2320 dataset,
2321 schema_timeline,
2322 stats,
2323 upstream,
2324 downstream,
2325 }))
2326 }
2327
2328 async fn catalog_lineage(
2329 &self,
2330 root: Option<&str>,
2331 depth: u32,
2332 ) -> Result<
2333 Vec<$crate::serve::history::catalog::CatalogLineageEdge>,
2334 $crate::serve::history::HistoryError,
2335 > {
2336 use $crate::serve::history::catalog;
2337 let edges = self.catalog_all_edges().await?;
2338 Ok(catalog::lineage_slice(edges, root, depth))
2339 }
2340
2341 async fn catalog_record_config_snapshot(
2342 &self,
2343 snapshot: &$crate::serve::history::catalog::ConfigSnapshot,
2344 ) -> Result<(), $crate::serve::history::HistoryError> {
2345 use $crate::serve::history::HistoryError;
2346 use $crate::serve::history::sql;
2347 let backend = |e: sqlx::Error| HistoryError::Backend(e.to_string());
2348 sqlx::query(&self.stmts.catalog_upsert_config_snapshot)
2349 .bind(&snapshot.pipeline)
2350 .bind(sql::fmt_ts(snapshot.recorded_at))
2351 .bind(&snapshot.faucet_version)
2352 .bind(sql::encode_json(snapshot, "config snapshot")?)
2353 .execute(&self.pool)
2354 .await
2355 .map_err(backend)?;
2356 Ok(())
2357 }
2358
2359 async fn catalog_last_config_snapshot(
2360 &self,
2361 pipeline: &str,
2362 ) -> Result<
2363 Option<$crate::serve::history::catalog::ConfigSnapshot>,
2364 $crate::serve::history::HistoryError,
2365 > {
2366 use sqlx::Row as _;
2367 use $crate::serve::history::HistoryError;
2368 use $crate::serve::history::sql;
2369 let backend = |e: sqlx::Error| HistoryError::Backend(e.to_string());
2370 let Some(row) = sqlx::query(&self.stmts.catalog_select_config_snapshot)
2371 .bind(pipeline)
2372 .fetch_optional(&self.pool)
2373 .await
2374 .map_err(backend)?
2375 else {
2376 return Ok(None);
2377 };
2378 let body: String = row.try_get("body").map_err(backend)?;
2379 Ok(Some(sql::decode_json(&body, "config snapshot")?))
2380 }
2381
2382 async fn template_register(
2385 &self,
2386 draft: &$crate::serve::history::templates::TemplateDraft,
2387 ) -> Result<
2388 $crate::serve::history::templates::TemplateRecord,
2389 $crate::serve::history::HistoryError,
2390 > {
2391 use sqlx::Row as _;
2392 use $crate::serve::history::HistoryError;
2393 use $crate::serve::history::{sql, templates};
2394 let backend = |e: sqlx::Error| HistoryError::Backend(e.to_string());
2395 let id = draft.id.to_string();
2396
2397 for attempt in 1..=sql::CLAIM_ATTEMPTS {
2403 sql::retry_backoff(attempt).await;
2404 let mut tx = match self.pool.begin().await {
2410 Ok(tx) => tx,
2411 Err(e) if attempt < sql::CLAIM_ATTEMPTS => {
2412 tracing::debug!(
2413 template = %id, attempt, error = %e,
2414 "template version transaction lost a race; retrying"
2415 );
2416 continue;
2417 }
2418 Err(e) => return Err(backend(e)),
2419 };
2420 let row = match sqlx::query(&self.stmts.template_max_version)
2421 .bind(&id)
2422 .fetch_one(&mut *tx)
2423 .await
2424 {
2425 Ok(row) => row,
2426 Err(e) if attempt < sql::CLAIM_ATTEMPTS => {
2427 let _ = tx.rollback().await;
2428 tracing::debug!(
2429 template = %id, attempt, error = %e,
2430 "template version read lost a race; retrying"
2431 );
2432 continue;
2433 }
2434 Err(e) => {
2435 let _ = tx.rollback().await;
2436 return Err(backend(e));
2437 }
2438 };
2439 let max: i64 = row.try_get("v").map_err(backend)?;
2440 let next = (max as u32).saturating_add(1);
2441 let record = templates::TemplateRecord {
2442 id: id.clone(),
2443 version: next,
2444 name: draft.name.clone(),
2445 description: draft.description.clone(),
2446 body: draft.body.clone(),
2447 format: draft.format,
2448 params: draft.params.clone(),
2449 created_at: chrono::Utc::now(),
2450 created_by: draft.created_by.clone(),
2451 };
2452 let insert = sqlx::query(&self.stmts.template_insert)
2453 .bind(&id)
2454 .bind(next.to_string())
2455 .bind(&record.name)
2456 .bind(sql::fmt_ts(record.created_at))
2457 .bind(sql::encode_json(&record, "pipeline template")?)
2458 .execute(&mut *tx)
2459 .await;
2460 match insert {
2461 Ok(_) => {
2462 tx.commit().await.map_err(backend)?;
2463 let keep = self.template_versions(&id).await?;
2466 for stale in templates::versions_to_prune(keep) {
2467 let _ = sqlx::query(&self.stmts.template_delete_version)
2468 .bind(&id)
2469 .bind(stale.to_string())
2470 .execute(&self.pool)
2471 .await;
2472 }
2473 return Ok(record);
2474 }
2475 Err(e) if attempt < sql::CLAIM_ATTEMPTS => {
2476 let _ = tx.rollback().await;
2477 tracing::debug!(
2478 template = %id, attempt, error = %e,
2479 "template version insert lost a race; retrying with the next version"
2480 );
2481 }
2482 Err(e) => {
2483 let _ = tx.rollback().await;
2484 return Err(backend(e));
2485 }
2486 }
2487 }
2488 Err(HistoryError::Backend(format!(
2489 "could not assign a version for template '{id}' after {} attempts",
2490 sql::CLAIM_ATTEMPTS
2491 )))
2492 }
2493
2494 async fn template_get(
2495 &self,
2496 id: &str,
2497 version: Option<u32>,
2498 ) -> Result<
2499 Option<$crate::serve::history::templates::TemplateRecord>,
2500 $crate::serve::history::HistoryError,
2501 > {
2502 use sqlx::Row as _;
2503 use $crate::serve::history::HistoryError;
2504 use $crate::serve::history::sql;
2505 let backend = |e: sqlx::Error| HistoryError::Backend(e.to_string());
2506 let row = match version {
2507 Some(v) => sqlx::query(&self.stmts.template_select_version)
2508 .bind(id)
2509 .bind(v.to_string())
2510 .fetch_optional(&self.pool)
2511 .await,
2512 None => sqlx::query(&self.stmts.template_select_latest)
2513 .bind(id)
2514 .fetch_optional(&self.pool)
2515 .await,
2516 }
2517 .map_err(backend)?;
2518 let Some(row) = row else {
2519 return Ok(None);
2520 };
2521 let body: String = row.try_get("body").map_err(backend)?;
2522 Ok(Some(sql::decode_json(&body, "pipeline template")?))
2523 }
2524
2525 async fn template_list(
2526 &self,
2527 ) -> Result<
2528 Vec<$crate::serve::history::templates::TemplateSummary>,
2529 $crate::serve::history::HistoryError,
2530 > {
2531 use sqlx::Row as _;
2532 use $crate::serve::history::HistoryError;
2533 use $crate::serve::history::{sql, templates};
2534 let backend = |e: sqlx::Error| HistoryError::Backend(e.to_string());
2535 let rows = sqlx::query(&self.stmts.template_select_all)
2536 .fetch_all(&self.pool)
2537 .await
2538 .map_err(backend)?;
2539 let mut all = Vec::with_capacity(rows.len());
2540 for r in &rows {
2541 let body: String = r.try_get("body").map_err(backend)?;
2542 all.push(sql::decode_json(&body, "pipeline template")?);
2543 }
2544 Ok(templates::latest_per_id(all))
2545 }
2546
2547 async fn template_versions(
2548 &self,
2549 id: &str,
2550 ) -> Result<Vec<u32>, $crate::serve::history::HistoryError> {
2551 use sqlx::Row as _;
2552 use $crate::serve::history::HistoryError;
2553 let backend = |e: sqlx::Error| HistoryError::Backend(e.to_string());
2554 let rows = sqlx::query(&self.stmts.template_versions)
2555 .bind(id)
2556 .fetch_all(&self.pool)
2557 .await
2558 .map_err(backend)?;
2559 let mut out = Vec::with_capacity(rows.len());
2560 for r in &rows {
2561 let v: String = r.try_get("version").map_err(backend)?;
2562 if let Ok(n) = v.parse::<u32>() {
2563 out.push(n);
2564 }
2565 }
2566 Ok(out)
2567 }
2568
2569 async fn template_delete(
2570 &self,
2571 id: &str,
2572 version: Option<u32>,
2573 ) -> Result<usize, $crate::serve::history::HistoryError> {
2574 use $crate::serve::history::HistoryError;
2575 let backend = |e: sqlx::Error| HistoryError::Backend(e.to_string());
2576 let result = match version {
2577 Some(v) => {
2578 sqlx::query(&self.stmts.template_delete_tags_for_version)
2581 .bind(id)
2582 .bind(v.to_string())
2583 .execute(&self.pool)
2584 .await
2585 .map_err(backend)?;
2586 sqlx::query(&self.stmts.template_delete_launches_for_version)
2587 .bind(id)
2588 .bind(v.to_string())
2589 .execute(&self.pool)
2590 .await
2591 .map_err(backend)?;
2592 sqlx::query(&self.stmts.template_delete_version)
2593 .bind(id)
2594 .bind(v.to_string())
2595 .execute(&self.pool)
2596 .await
2597 }
2598 None => {
2599 for stmt in [
2600 &self.stmts.template_delete_tags_all,
2601 &self.stmts.template_delete_launches_all,
2602 &self.stmts.template_delete_deprecation,
2603 ] {
2604 sqlx::query(stmt)
2605 .bind(id)
2606 .execute(&self.pool)
2607 .await
2608 .map_err(backend)?;
2609 }
2610 sqlx::query(&self.stmts.template_delete_all)
2611 .bind(id)
2612 .execute(&self.pool)
2613 .await
2614 }
2615 }
2616 .map_err(backend)?;
2617 Ok(result.rows_affected() as usize)
2618 }
2619
2620 async fn template_set_tag(
2621 &self,
2622 id: &str,
2623 tag: &str,
2624 version: u32,
2625 ) -> Result<(), $crate::serve::history::HistoryError> {
2626 use $crate::serve::history::HistoryError;
2627 use $crate::serve::history::sql;
2628 let backend = |e: sqlx::Error| HistoryError::Backend(e.to_string());
2629 sqlx::query(&self.stmts.template_upsert_tag)
2630 .bind(id)
2631 .bind(tag)
2632 .bind(version.to_string())
2633 .bind(sql::fmt_ts(chrono::Utc::now()))
2634 .execute(&self.pool)
2635 .await
2636 .map_err(backend)?;
2637 Ok(())
2638 }
2639
2640 async fn template_tags(
2641 &self,
2642 id: &str,
2643 ) -> Result<
2644 std::collections::BTreeMap<String, u32>,
2645 $crate::serve::history::HistoryError,
2646 > {
2647 use sqlx::Row as _;
2648 use $crate::serve::history::HistoryError;
2649 let backend = |e: sqlx::Error| HistoryError::Backend(e.to_string());
2650 let rows = sqlx::query(&self.stmts.template_select_tags)
2651 .bind(id)
2652 .fetch_all(&self.pool)
2653 .await
2654 .map_err(backend)?;
2655 let mut out = std::collections::BTreeMap::new();
2656 for r in &rows {
2657 let tag: String = r.try_get("tag").map_err(backend)?;
2658 let version: String = r.try_get("version").map_err(backend)?;
2659 if let Ok(n) = version.parse::<u32>() {
2660 out.insert(tag, n);
2661 }
2662 }
2663 Ok(out)
2664 }
2665
2666 async fn template_delete_tag(
2667 &self,
2668 id: &str,
2669 tag: &str,
2670 ) -> Result<bool, $crate::serve::history::HistoryError> {
2671 use $crate::serve::history::HistoryError;
2672 let backend = |e: sqlx::Error| HistoryError::Backend(e.to_string());
2673 let result = sqlx::query(&self.stmts.template_delete_tag)
2674 .bind(id)
2675 .bind(tag)
2676 .execute(&self.pool)
2677 .await
2678 .map_err(backend)?;
2679 Ok(result.rows_affected() > 0)
2680 }
2681
2682 async fn template_launch(
2683 &self,
2684 id: &str,
2685 version: u32,
2686 launched_by: Option<&str>,
2687 ) -> Result<Option<u32>, $crate::serve::history::HistoryError> {
2688 use sqlx::Row as _;
2689 use $crate::serve::history::HistoryError;
2690 use $crate::serve::history::{sql, templates};
2691 let backend = |e: sqlx::Error| HistoryError::Backend(e.to_string());
2692
2693 let existing = self.template_launches(id).await?;
2697 if templates::stable_version(&existing) == Some(version) {
2698 return Ok(None);
2699 }
2700 for attempt in 1..=sql::CLAIM_ATTEMPTS {
2704 sql::retry_backoff(attempt).await;
2705 let row = match sqlx::query(&self.stmts.template_max_launch_seq)
2708 .bind(id)
2709 .fetch_one(&self.pool)
2710 .await
2711 {
2712 Ok(row) => row,
2713 Err(e) if attempt < sql::CLAIM_ATTEMPTS => {
2714 tracing::debug!(
2715 template = %id, attempt, error = %e,
2716 "launch-log seq read lost a race; retrying"
2717 );
2718 continue;
2719 }
2720 Err(e) => return Err(backend(e)),
2721 };
2722 let max: i64 = row.try_get("v").map_err(backend)?;
2723 let seq = (max as u32).saturating_add(1);
2724 let insert = sqlx::query(&self.stmts.template_insert_launch)
2725 .bind(id)
2726 .bind(seq.to_string())
2727 .bind(version.to_string())
2728 .bind(sql::fmt_ts(chrono::Utc::now()))
2729 .bind(launched_by)
2730 .execute(&self.pool)
2731 .await;
2732 match insert {
2733 Ok(_) => return Ok(Some(seq)),
2734 Err(e) if attempt < sql::CLAIM_ATTEMPTS => {
2735 tracing::debug!(
2736 template = %id, attempt, error = %e,
2737 "launch-log insert lost a race; retrying with the next seq"
2738 );
2739 }
2740 Err(e) => return Err(backend(e)),
2741 }
2742 }
2743 Err(HistoryError::Backend(format!(
2744 "could not append a launch for template '{id}' after {} attempts",
2745 sql::CLAIM_ATTEMPTS
2746 )))
2747 }
2748
2749 async fn template_launches(
2750 &self,
2751 id: &str,
2752 ) -> Result<
2753 Vec<$crate::serve::history::templates::LaunchRecord>,
2754 $crate::serve::history::HistoryError,
2755 > {
2756 use sqlx::Row as _;
2757 use $crate::serve::history::HistoryError;
2758 use $crate::serve::history::{sql, templates};
2759 let backend = |e: sqlx::Error| HistoryError::Backend(e.to_string());
2760 let rows = sqlx::query(&self.stmts.template_select_launches)
2761 .bind(id)
2762 .fetch_all(&self.pool)
2763 .await
2764 .map_err(backend)?;
2765 let mut out = Vec::with_capacity(rows.len());
2766 for r in &rows {
2767 let seq: String = r.try_get("seq").map_err(backend)?;
2768 let version: String = r.try_get("version").map_err(backend)?;
2769 let launched_at: String = r.try_get("launched_at").map_err(backend)?;
2770 let launched_by: Option<String> = r.try_get("launched_by").map_err(backend)?;
2771 let (Ok(seq), Ok(version)) = (seq.parse::<u32>(), version.parse::<u32>()) else {
2774 continue;
2775 };
2776 out.push(templates::LaunchRecord {
2777 seq,
2778 version,
2779 launched_at: sql::parse_ts(&launched_at),
2780 launched_by,
2781 });
2782 }
2783 Ok(out)
2784 }
2785
2786 async fn template_set_deprecation(
2787 &self,
2788 id: &str,
2789 record: Option<&$crate::serve::history::templates::DeprecationRecord>,
2790 ) -> Result<(), $crate::serve::history::HistoryError> {
2791 use $crate::serve::history::HistoryError;
2792 use $crate::serve::history::sql;
2793 let backend = |e: sqlx::Error| HistoryError::Backend(e.to_string());
2794 match record {
2795 Some(r) => {
2796 sqlx::query(&self.stmts.template_upsert_deprecation)
2797 .bind(id)
2798 .bind(sql::fmt_ts(r.deprecated_at))
2799 .bind(r.deprecated_by.as_deref())
2800 .bind(r.reason.as_deref())
2801 .execute(&self.pool)
2802 .await
2803 .map_err(backend)?;
2804 }
2805 None => {
2806 sqlx::query(&self.stmts.template_delete_deprecation)
2807 .bind(id)
2808 .execute(&self.pool)
2809 .await
2810 .map_err(backend)?;
2811 }
2812 }
2813 Ok(())
2814 }
2815
2816 async fn template_deprecation(
2817 &self,
2818 id: &str,
2819 ) -> Result<
2820 Option<$crate::serve::history::templates::DeprecationRecord>,
2821 $crate::serve::history::HistoryError,
2822 > {
2823 use sqlx::Row as _;
2824 use $crate::serve::history::HistoryError;
2825 use $crate::serve::history::{sql, templates};
2826 let backend = |e: sqlx::Error| HistoryError::Backend(e.to_string());
2827 let Some(row) = sqlx::query(&self.stmts.template_select_deprecation)
2828 .bind(id)
2829 .fetch_optional(&self.pool)
2830 .await
2831 .map_err(backend)?
2832 else {
2833 return Ok(None);
2834 };
2835 let at: String = row.try_get("deprecated_at").map_err(backend)?;
2836 Ok(Some(templates::DeprecationRecord {
2837 deprecated_at: sql::parse_ts(&at),
2838 deprecated_by: row.try_get("deprecated_by").map_err(backend)?,
2839 reason: row.try_get("reason").map_err(backend)?,
2840 }))
2841 }
2842
2843 fn degraded(&self) -> bool {
2844 false
2847 }
2848 }
2849
2850 impl $name {
2851 async fn catalog_all_edges(
2853 &self,
2854 ) -> Result<
2855 Vec<$crate::serve::history::catalog::CatalogLineageEdge>,
2856 $crate::serve::history::HistoryError,
2857 > {
2858 use sqlx::Row as _;
2859 use $crate::serve::history::HistoryError;
2860 use $crate::serve::history::sql;
2861 let backend = |e: sqlx::Error| HistoryError::Backend(e.to_string());
2862 let rows = sqlx::query(&self.stmts.catalog_select_edges)
2863 .fetch_all(&self.pool)
2864 .await
2865 .map_err(backend)?;
2866 let mut edges = Vec::with_capacity(rows.len());
2867 for r in &rows {
2868 let body: String = r.try_get("body").map_err(backend)?;
2869 edges.push(sql::decode_json(&body, "catalog edge")?);
2870 }
2871 Ok(edges)
2872 }
2873 }
2874 };
2875}
2876
2877pub(crate) use impl_sql_history;
2878
2879#[cfg(test)]
2880mod tests {
2881 use super::*;
2882
2883 #[test]
2884 fn postgres_shard_statements_are_built() {
2885 let s = Stmts::new(Dialect::Postgres);
2888 assert!(s.insert_shard.contains("faucet_serve_shards"));
2889 assert!(s.insert_shard.contains("ON CONFLICT"));
2890 assert!(s.claim_shards_select.contains("JOIN faucet_serve_runs"));
2891 assert!(s.claim_shard_one.contains("'running'"));
2892 assert!(s.renew_shard_leases.contains("lease_expires_at"));
2893 assert!(s.reclaim_shards_select.contains("'running'"));
2894 assert!(s.reclaim_shard_requeue.contains("'pending'"));
2895 assert!(s.reclaim_shard_fail.contains("'failed'"));
2896 assert!(s.finalize_shard.contains("owner"));
2897 assert!(s.shard_progress.contains("GROUP BY"));
2898 }
2899
2900 #[test]
2901 fn fmt_ts_is_fixed_width_and_sortable() {
2902 let a = fmt_ts(
2903 DateTime::parse_from_rfc3339("2026-01-01T00:00:00Z")
2904 .unwrap()
2905 .to_utc(),
2906 );
2907 let b = fmt_ts(
2908 DateTime::parse_from_rfc3339("2026-01-01T00:00:01Z")
2909 .unwrap()
2910 .to_utc(),
2911 );
2912 assert!(a.ends_with('Z'));
2913 assert_eq!(a.len(), b.len(), "fixed width");
2914 assert!(a < b, "lexicographic order matches chronological order");
2915 }
2916
2917 #[test]
2918 fn is_expired_respects_window() {
2919 let now = Utc::now();
2920 let old = fmt_ts(now - chrono::Duration::seconds(120));
2921 assert!(is_expired(&old, now, Duration::from_secs(60)));
2922 assert!(!is_expired(&old, now, Duration::from_secs(600)));
2923 assert!(!is_expired("not-a-timestamp", now, Duration::ZERO));
2925 }
2926
2927 #[test]
2928 fn parse_status_round_trips_known_and_defaults_failed() {
2929 for s in [
2930 RunStatus::Queued,
2931 RunStatus::Pending,
2932 RunStatus::Running,
2933 RunStatus::Completed,
2934 RunStatus::Failed,
2935 RunStatus::Cancelled,
2936 ] {
2937 assert_eq!(parse_status(s.as_str()), s);
2938 }
2939 assert_eq!(parse_status("garbage"), RunStatus::Failed);
2940 }
2941
2942 #[test]
2943 fn body_round_trips() {
2944 let rec = RunRecord::queued(
2945 "r1".into(),
2946 Some("n".into()),
2947 Default::default(),
2948 Some("idem".into()),
2949 Utc::now(),
2950 );
2951 let encoded = encode_body(&rec).unwrap();
2952 let decoded = decode_body(&encoded).unwrap();
2953 assert_eq!(decoded.run_id, "r1");
2954 assert_eq!(decoded.idempotency_key.as_deref(), Some("idem"));
2955 }
2956
2957 #[test]
2958 fn postgres_and_sqlite_statements_differ_only_in_placeholders() {
2959 let pg = Stmts::new(Dialect::Postgres);
2960 let lite = Stmts::new(Dialect::Sqlite);
2961 assert!(pg.upsert.contains("$1") && lite.upsert.contains('?'));
2962 assert!(pg.list.contains("$13") && lite.list.contains('?'));
2963 assert!(pg.insert_idem.contains("ON CONFLICT (key) DO NOTHING"));
2965 assert!(lite.insert_idem.contains("ON CONFLICT (key) DO NOTHING"));
2966 assert!(pg.claim_one.contains("$3") && lite.claim_one.contains('?'));
2967 assert!(pg.heartbeat_instance.contains("faucet_serve_instances"));
2968 assert!(lite.heartbeat_instance.contains("faucet_serve_instances"));
2969 }
2970}