1use super::{HistoryError, RunRecord, RunStatus};
20use chrono::{DateTime, Utc};
21use std::time::Duration;
22
23pub const DDL: &[&str] = &[
26 "CREATE TABLE IF NOT EXISTS faucet_serve_runs (\
31 run_id TEXT PRIMARY KEY,\
32 name TEXT,\
33 status TEXT NOT NULL,\
34 submitted_at TEXT NOT NULL,\
35 finished_at TEXT,\
36 idempotency_key TEXT,\
37 owner TEXT,\
38 lease_expires_at TEXT,\
39 body TEXT NOT NULL)",
40 "CREATE INDEX IF NOT EXISTS faucet_serve_runs_submitted_idx \
41 ON faucet_serve_runs (submitted_at)",
42 "CREATE INDEX IF NOT EXISTS faucet_serve_runs_status_lease_idx \
45 ON faucet_serve_runs (status, lease_expires_at)",
46 "CREATE TABLE IF NOT EXISTS faucet_serve_idem (\
47 key TEXT PRIMARY KEY,\
48 run_id TEXT NOT NULL,\
49 fingerprint TEXT NOT NULL,\
50 claimed_at TEXT NOT NULL)",
51];
52
53#[derive(Clone, Copy, Debug)]
55pub enum Dialect {
56 Postgres,
57 Sqlite,
58}
59
60pub struct Stmts {
62 pub upsert: String,
63 pub select_body: String,
64 pub select_status: String,
65 pub select_submitted: String,
66 pub delete: String,
67 pub list: String,
68 pub purge_runs: String,
69 pub purge_idem: String,
70 pub select_orphans: String,
73 pub renew_leases: String,
76 pub insert_idem: String,
77 pub select_idem: String,
78 pub takeover_idem: String,
79 pub delete_idem_by_run: String,
84}
85
86impl Stmts {
87 pub fn new(dialect: Dialect) -> Self {
88 match dialect {
89 Dialect::Postgres => Self::postgres(),
90 Dialect::Sqlite => Self::sqlite(),
91 }
92 }
93
94 fn postgres() -> Self {
95 Self {
96 upsert: "INSERT INTO faucet_serve_runs \
97 (run_id,name,status,submitted_at,finished_at,idempotency_key,owner,lease_expires_at,body) \
98 VALUES ($1,$2,$3,$4,$5,$6,$7,$8,$9) \
99 ON CONFLICT (run_id) DO UPDATE SET \
100 name=excluded.name,status=excluded.status,submitted_at=excluded.submitted_at,\
101 finished_at=excluded.finished_at,idempotency_key=excluded.idempotency_key,\
102 owner=excluded.owner,lease_expires_at=excluded.lease_expires_at,\
103 body=excluded.body"
104 .into(),
105 select_body: "SELECT body FROM faucet_serve_runs WHERE run_id=$1".into(),
106 select_status: "SELECT status FROM faucet_serve_runs WHERE run_id=$1".into(),
107 select_submitted: "SELECT submitted_at FROM faucet_serve_runs WHERE run_id=$1".into(),
108 delete: "DELETE FROM faucet_serve_runs WHERE run_id=$1".into(),
109 list: "SELECT body FROM faucet_serve_runs \
112 WHERE ($1::text IS NULL OR status = $2::text) \
113 AND ($3::text IS NULL OR name = $4::text) \
114 AND ($5::text IS NULL OR submitted_at >= $6::text) \
115 AND ($7::text IS NULL OR submitted_at <= $8::text) \
116 AND ($9::text IS NULL OR (submitted_at < $10::text \
117 OR (submitted_at = $11::text AND run_id < $12::text))) \
118 ORDER BY submitted_at DESC, run_id DESC LIMIT $13"
119 .into(),
120 purge_runs: "DELETE FROM faucet_serve_runs \
121 WHERE status IN ('completed','failed','cancelled') \
122 AND finished_at IS NOT NULL AND finished_at < $1"
123 .into(),
124 purge_idem: "DELETE FROM faucet_serve_idem WHERE claimed_at < $1".into(),
125 select_orphans: "SELECT body FROM faucet_serve_runs \
126 WHERE status IN ('queued','running') \
127 AND (lease_expires_at IS NULL OR lease_expires_at < $1)"
128 .into(),
129 renew_leases: "UPDATE faucet_serve_runs SET lease_expires_at = $1 \
130 WHERE owner = $2 AND status IN ('queued','running')"
131 .into(),
132 insert_idem: "INSERT INTO faucet_serve_idem (key,run_id,fingerprint,claimed_at) \
133 VALUES ($1,$2,$3,$4) ON CONFLICT (key) DO NOTHING"
134 .into(),
135 select_idem: "SELECT run_id,fingerprint,claimed_at FROM faucet_serve_idem WHERE key=$1"
136 .into(),
137 takeover_idem: "UPDATE faucet_serve_idem \
138 SET run_id=$1,fingerprint=$2,claimed_at=$3 WHERE key=$4 AND claimed_at=$5"
139 .into(),
140 delete_idem_by_run: "DELETE FROM faucet_serve_idem WHERE run_id=$1".into(),
141 }
142 }
143
144 fn sqlite() -> Self {
145 Self {
146 upsert: "INSERT INTO faucet_serve_runs \
147 (run_id,name,status,submitted_at,finished_at,idempotency_key,owner,lease_expires_at,body) \
148 VALUES (?,?,?,?,?,?,?,?,?) \
149 ON CONFLICT (run_id) DO UPDATE SET \
150 name=excluded.name,status=excluded.status,submitted_at=excluded.submitted_at,\
151 finished_at=excluded.finished_at,idempotency_key=excluded.idempotency_key,\
152 owner=excluded.owner,lease_expires_at=excluded.lease_expires_at,\
153 body=excluded.body"
154 .into(),
155 select_body: "SELECT body FROM faucet_serve_runs WHERE run_id=?".into(),
156 select_status: "SELECT status FROM faucet_serve_runs WHERE run_id=?".into(),
157 select_submitted: "SELECT submitted_at FROM faucet_serve_runs WHERE run_id=?".into(),
158 delete: "DELETE FROM faucet_serve_runs WHERE run_id=?".into(),
159 list: "SELECT body FROM faucet_serve_runs \
160 WHERE (? IS NULL OR status = ?) \
161 AND (? IS NULL OR name = ?) \
162 AND (? IS NULL OR submitted_at >= ?) \
163 AND (? IS NULL OR submitted_at <= ?) \
164 AND (? IS NULL OR (submitted_at < ? \
165 OR (submitted_at = ? AND run_id < ?))) \
166 ORDER BY submitted_at DESC, run_id DESC LIMIT ?"
167 .into(),
168 purge_runs: "DELETE FROM faucet_serve_runs \
169 WHERE status IN ('completed','failed','cancelled') \
170 AND finished_at IS NOT NULL AND finished_at < ?"
171 .into(),
172 purge_idem: "DELETE FROM faucet_serve_idem WHERE claimed_at < ?".into(),
173 select_orphans: "SELECT body FROM faucet_serve_runs \
174 WHERE status IN ('queued','running') \
175 AND (lease_expires_at IS NULL OR lease_expires_at < ?)"
176 .into(),
177 renew_leases: "UPDATE faucet_serve_runs SET lease_expires_at = ? \
178 WHERE owner = ? AND status IN ('queued','running')"
179 .into(),
180 insert_idem: "INSERT INTO faucet_serve_idem (key,run_id,fingerprint,claimed_at) \
181 VALUES (?,?,?,?) ON CONFLICT (key) DO NOTHING"
182 .into(),
183 select_idem: "SELECT run_id,fingerprint,claimed_at FROM faucet_serve_idem WHERE key=?"
184 .into(),
185 takeover_idem: "UPDATE faucet_serve_idem \
186 SET run_id=?,fingerprint=?,claimed_at=? WHERE key=? AND claimed_at=?"
187 .into(),
188 delete_idem_by_run: "DELETE FROM faucet_serve_idem WHERE run_id=?".into(),
189 }
190 }
191}
192
193pub const CLAIM_ATTEMPTS: usize = 4;
196
197pub fn fmt_ts(dt: DateTime<Utc>) -> String {
199 dt.to_rfc3339_opts(chrono::SecondsFormat::Nanos, true)
200}
201
202pub fn is_expired(claimed_at: &str, now: DateTime<Utc>, window: Duration) -> bool {
206 match DateTime::parse_from_rfc3339(claimed_at) {
207 Ok(t) => now
208 .signed_duration_since(t.with_timezone(&Utc))
209 .to_std()
210 .map(|age| age >= window)
211 .unwrap_or(false),
212 Err(_) => false,
213 }
214}
215
216pub fn threshold(now: DateTime<Utc>, window: Duration) -> String {
218 let delta =
219 chrono::Duration::from_std(window).unwrap_or_else(|_| chrono::Duration::days(36_500));
220 fmt_ts(now - delta)
221}
222
223pub fn encode_body(rec: &RunRecord) -> Result<String, HistoryError> {
224 serde_json::to_string(rec).map_err(|e| HistoryError::Backend(format!("encode run record: {e}")))
225}
226
227pub fn decode_body(body: &str) -> Result<RunRecord, HistoryError> {
228 serde_json::from_str(body).map_err(|e| HistoryError::Backend(format!("decode run record: {e}")))
229}
230
231pub fn parse_status(s: &str) -> RunStatus {
232 match s {
233 "queued" => RunStatus::Queued,
234 "running" => RunStatus::Running,
235 "completed" => RunStatus::Completed,
236 "cancelled" => RunStatus::Cancelled,
237 _ => RunStatus::Failed,
238 }
239}
240
241macro_rules! impl_sql_history {
245 ($name:ident, $pool:ty) => {
246 pub struct $name {
249 pool: $pool,
250 idem_retention: std::time::Duration,
251 instance_id: String,
253 lease_ttl: std::time::Duration,
255 stmts: $crate::serve::history::sql::Stmts,
256 }
257
258 impl $name {
259 pub fn from_parts(
261 pool: $pool,
262 idem_retention: std::time::Duration,
263 lease_ttl: std::time::Duration,
264 instance_id: String,
265 stmts: $crate::serve::history::sql::Stmts,
266 ) -> Self {
267 Self {
268 pool,
269 idem_retention,
270 instance_id,
271 lease_ttl,
272 stmts,
273 }
274 }
275
276 pub fn pool(&self) -> &$pool {
278 &self.pool
279 }
280 }
281
282 #[async_trait::async_trait]
283 impl $crate::serve::history::RunHistory for $name {
284 async fn claim_idempotency(
285 &self,
286 key: &str,
287 fingerprint: &str,
288 run_id: &str,
289 window: std::time::Duration,
290 ) -> Result<$crate::serve::history::Claim, $crate::serve::history::HistoryError> {
291 use sqlx::Row as _;
292 use $crate::serve::history::Claim;
293 use $crate::serve::history::HistoryError;
294 use $crate::serve::history::sql;
295
296 let now = chrono::Utc::now();
297 let now_s = sql::fmt_ts(now);
298 let backend = |e: sqlx::Error| HistoryError::Backend(e.to_string());
299
300 for _ in 0..sql::CLAIM_ATTEMPTS {
301 let inserted = sqlx::query(&self.stmts.insert_idem)
303 .bind(key)
304 .bind(run_id)
305 .bind(fingerprint)
306 .bind(&now_s)
307 .execute(&self.pool)
308 .await
309 .map_err(backend)?
310 .rows_affected();
311 if inserted == 1 {
312 return Ok(Claim::Fresh);
313 }
314 let Some(row) = sqlx::query(&self.stmts.select_idem)
316 .bind(key)
317 .fetch_optional(&self.pool)
318 .await
319 .map_err(backend)?
320 else {
321 continue;
323 };
324 let existing_run: String = row.try_get("run_id").map_err(backend)?;
325 let existing_fp: String = row.try_get("fingerprint").map_err(backend)?;
326 let claimed_at: String = row.try_get("claimed_at").map_err(backend)?;
327
328 if sql::is_expired(&claimed_at, now, window) {
329 let took = sqlx::query(&self.stmts.takeover_idem)
332 .bind(run_id)
333 .bind(fingerprint)
334 .bind(&now_s)
335 .bind(key)
336 .bind(&claimed_at)
337 .execute(&self.pool)
338 .await
339 .map_err(backend)?
340 .rows_affected();
341 if took == 1 {
342 return Ok(Claim::Fresh);
343 }
344 continue; }
346 return Ok(if existing_fp == fingerprint {
347 Claim::Replay(existing_run)
348 } else {
349 Claim::Conflict
350 });
351 }
352 tracing::warn!(
355 key,
356 "idempotency claim exhausted retries; reporting conflict"
357 );
358 Ok(Claim::Conflict)
359 }
360
361 async fn upsert(
362 &self,
363 rec: &$crate::serve::history::RunRecord,
364 ) -> Result<(), $crate::serve::history::HistoryError> {
365 use $crate::serve::history::HistoryError;
366 use $crate::serve::history::sql;
367 let body = sql::encode_body(rec)?;
368 let submitted = sql::fmt_ts(rec.submitted_at);
369 let finished = rec.finished_at.map(sql::fmt_ts);
370 let lease = sql::fmt_ts(chrono::Utc::now() + self.lease_ttl);
375 sqlx::query(&self.stmts.upsert)
376 .bind(&rec.run_id)
377 .bind(rec.name.as_deref())
378 .bind(rec.status.as_str())
379 .bind(&submitted)
380 .bind(finished.as_deref())
381 .bind(rec.idempotency_key.as_deref())
382 .bind(&self.instance_id)
383 .bind(&lease)
384 .bind(&body)
385 .execute(&self.pool)
386 .await
387 .map_err(|e| HistoryError::Backend(e.to_string()))?;
388 Ok(())
389 }
390
391 async fn get(
392 &self,
393 id: &str,
394 ) -> Result<
395 Option<$crate::serve::history::RunRecord>,
396 $crate::serve::history::HistoryError,
397 > {
398 use sqlx::Row as _;
399 use $crate::serve::history::HistoryError;
400 use $crate::serve::history::sql;
401 let row = sqlx::query(&self.stmts.select_body)
402 .bind(id)
403 .fetch_optional(&self.pool)
404 .await
405 .map_err(|e| HistoryError::Backend(e.to_string()))?;
406 match row {
407 None => Ok(None),
408 Some(r) => {
409 let body: String = r
410 .try_get("body")
411 .map_err(|e| HistoryError::Backend(e.to_string()))?;
412 Ok(Some(sql::decode_body(&body)?))
413 }
414 }
415 }
416
417 async fn list(
418 &self,
419 filter: &$crate::serve::history::ListFilter,
420 ) -> Result<$crate::serve::history::ListPage, $crate::serve::history::HistoryError>
421 {
422 use sqlx::Row as _;
423 use $crate::serve::history::HistoryError;
424 use $crate::serve::history::ListPage;
425 use $crate::serve::history::sql;
426 let backend = |e: sqlx::Error| HistoryError::Backend(e.to_string());
427
428 let cursor_ts: Option<String> = match &filter.cursor {
432 None => None,
433 Some(c) => sqlx::query(&self.stmts.select_submitted)
434 .bind(c)
435 .fetch_optional(&self.pool)
436 .await
437 .map_err(backend)?
438 .map(|r| r.try_get::<String, _>("submitted_at"))
439 .transpose()
440 .map_err(backend)?,
441 };
442 let cur_id = if cursor_ts.is_some() {
443 filter.cursor.as_deref()
444 } else {
445 None
446 };
447
448 let status_s = filter.status.map(|s| s.as_str());
449 let name_s = filter.name.as_deref();
450 let since_s = filter.since.map(sql::fmt_ts);
451 let until_s = filter.until.map(sql::fmt_ts);
452 let limit = filter.limit.max(1);
453 let fetch_n = limit as i64 + 1; let rows = sqlx::query(&self.stmts.list)
456 .bind(status_s)
457 .bind(status_s)
458 .bind(name_s)
459 .bind(name_s)
460 .bind(since_s.as_deref())
461 .bind(since_s.as_deref())
462 .bind(until_s.as_deref())
463 .bind(until_s.as_deref())
464 .bind(cursor_ts.as_deref())
465 .bind(cursor_ts.as_deref())
466 .bind(cursor_ts.as_deref())
467 .bind(cur_id)
468 .bind(fetch_n)
469 .fetch_all(&self.pool)
470 .await
471 .map_err(backend)?;
472
473 let mut runs = Vec::with_capacity(rows.len());
474 for r in &rows {
475 let body: String = r.try_get("body").map_err(backend)?;
476 runs.push(sql::decode_body(&body)?);
477 }
478 let next_cursor = if runs.len() > limit {
479 Some(runs[limit - 1].run_id.clone())
480 } else {
481 None
482 };
483 runs.truncate(limit);
484 Ok(ListPage { runs, next_cursor })
485 }
486
487 async fn delete(
488 &self,
489 id: &str,
490 ) -> Result<$crate::serve::history::DeleteOutcome, $crate::serve::history::HistoryError>
491 {
492 use sqlx::Row as _;
493 use $crate::serve::history::DeleteOutcome;
494 use $crate::serve::history::HistoryError;
495 use $crate::serve::history::sql;
496 let backend = |e: sqlx::Error| HistoryError::Backend(e.to_string());
497 let status: Option<String> = sqlx::query(&self.stmts.select_status)
498 .bind(id)
499 .fetch_optional(&self.pool)
500 .await
501 .map_err(backend)?
502 .map(|r| r.try_get::<String, _>("status"))
503 .transpose()
504 .map_err(backend)?;
505 match status {
506 None => Ok(DeleteOutcome::NotFound),
507 Some(s) if !sql::parse_status(&s).is_terminal() => {
508 Ok(DeleteOutcome::StillRunning)
509 }
510 Some(_) => {
511 sqlx::query(&self.stmts.delete)
512 .bind(id)
513 .execute(&self.pool)
514 .await
515 .map_err(backend)?;
516 sqlx::query(&self.stmts.delete_idem_by_run)
522 .bind(id)
523 .execute(&self.pool)
524 .await
525 .map_err(backend)?;
526 Ok(DeleteOutcome::Deleted)
527 }
528 }
529 }
530
531 async fn purge_expired(
532 &self,
533 retain_for: std::time::Duration,
534 ) -> Result<usize, $crate::serve::history::HistoryError> {
535 use $crate::serve::history::HistoryError;
536 use $crate::serve::history::sql;
537 let backend = |e: sqlx::Error| HistoryError::Backend(e.to_string());
538 let now = chrono::Utc::now();
539 let removed = sqlx::query(&self.stmts.purge_runs)
540 .bind(sql::threshold(now, retain_for))
541 .execute(&self.pool)
542 .await
543 .map_err(backend)?
544 .rows_affected() as usize;
545 let _ = sqlx::query(&self.stmts.purge_idem)
547 .bind(sql::threshold(now, self.idem_retention))
548 .execute(&self.pool)
549 .await;
550 Ok(removed)
551 }
552
553 async fn recover_orphans(&self) -> Result<usize, $crate::serve::history::HistoryError> {
554 use sqlx::Row as _;
555 use $crate::serve::history::HistoryError;
556 use $crate::serve::history::RunStatus;
557 use $crate::serve::history::sql;
558 let backend = |e: sqlx::Error| HistoryError::Backend(e.to_string());
559 let now = chrono::Utc::now();
560 let rows = sqlx::query(&self.stmts.select_orphans)
565 .bind(sql::fmt_ts(now))
566 .fetch_all(&self.pool)
567 .await
568 .map_err(backend)?;
569 let mut count = 0usize;
570 for r in &rows {
571 let body: String = r.try_get("body").map_err(backend)?;
572 let mut rec = sql::decode_body(&body)?;
573 rec.status = RunStatus::Failed;
574 rec.finished_at = Some(now);
575 rec.error = Some(
576 "owning serve instance's lease expired before the run finished".into(),
577 );
578 if rec.elapsed_secs.is_none()
579 && let Some(started) = rec.started_at
580 {
581 rec.elapsed_secs = (now - started).to_std().ok().map(|d| d.as_secs_f64());
582 }
583 self.upsert(&rec).await?;
584 count += 1;
585 }
586 Ok(count)
587 }
588
589 async fn renew_leases(&self) -> Result<usize, $crate::serve::history::HistoryError> {
590 use $crate::serve::history::HistoryError;
591 use $crate::serve::history::sql;
592 let backend = |e: sqlx::Error| HistoryError::Backend(e.to_string());
593 let new_lease = sql::fmt_ts(chrono::Utc::now() + self.lease_ttl);
594 let renewed = sqlx::query(&self.stmts.renew_leases)
595 .bind(&new_lease)
596 .bind(&self.instance_id)
597 .execute(&self.pool)
598 .await
599 .map_err(backend)?
600 .rows_affected() as usize;
601 Ok(renewed)
602 }
603
604 fn degraded(&self) -> bool {
605 false
608 }
609 }
610 };
611}
612
613pub(crate) use impl_sql_history;
614
615#[cfg(test)]
616mod tests {
617 use super::*;
618
619 #[test]
620 fn fmt_ts_is_fixed_width_and_sortable() {
621 let a = fmt_ts(
622 DateTime::parse_from_rfc3339("2026-01-01T00:00:00Z")
623 .unwrap()
624 .to_utc(),
625 );
626 let b = fmt_ts(
627 DateTime::parse_from_rfc3339("2026-01-01T00:00:01Z")
628 .unwrap()
629 .to_utc(),
630 );
631 assert!(a.ends_with('Z'));
632 assert_eq!(a.len(), b.len(), "fixed width");
633 assert!(a < b, "lexicographic order matches chronological order");
634 }
635
636 #[test]
637 fn is_expired_respects_window() {
638 let now = Utc::now();
639 let old = fmt_ts(now - chrono::Duration::seconds(120));
640 assert!(is_expired(&old, now, Duration::from_secs(60)));
641 assert!(!is_expired(&old, now, Duration::from_secs(600)));
642 assert!(!is_expired("not-a-timestamp", now, Duration::ZERO));
644 }
645
646 #[test]
647 fn parse_status_round_trips_known_and_defaults_failed() {
648 for s in [
649 RunStatus::Queued,
650 RunStatus::Running,
651 RunStatus::Completed,
652 RunStatus::Failed,
653 RunStatus::Cancelled,
654 ] {
655 assert_eq!(parse_status(s.as_str()), s);
656 }
657 assert_eq!(parse_status("garbage"), RunStatus::Failed);
658 }
659
660 #[test]
661 fn body_round_trips() {
662 let rec = RunRecord::queued(
663 "r1".into(),
664 Some("n".into()),
665 Default::default(),
666 Some("idem".into()),
667 Utc::now(),
668 );
669 let encoded = encode_body(&rec).unwrap();
670 let decoded = decode_body(&encoded).unwrap();
671 assert_eq!(decoded.run_id, "r1");
672 assert_eq!(decoded.idempotency_key.as_deref(), Some("idem"));
673 }
674
675 #[test]
676 fn postgres_and_sqlite_statements_differ_only_in_placeholders() {
677 let pg = Stmts::new(Dialect::Postgres);
678 let lite = Stmts::new(Dialect::Sqlite);
679 assert!(pg.upsert.contains("$1") && lite.upsert.contains('?'));
680 assert!(pg.list.contains("$13") && lite.list.contains('?'));
681 assert!(pg.insert_idem.contains("ON CONFLICT (key) DO NOTHING"));
683 assert!(lite.insert_idem.contains("ON CONFLICT (key) DO NOTHING"));
684 }
685}