1use anyhow::{Context, Result};
10use rusqlite::{params, Connection, OptionalExtension};
11use serde::Serialize;
12
13pub const TRACKED_COMMANDS: [&str; 8] = [
31 "scan",
32 "faces",
33 "embed",
34 "classify",
35 "dedupe",
36 "fix-dates",
37 "prune",
38 "locations",
39];
40
41pub fn ensure_pipeline_runs_table(conn: &Connection) -> rusqlite::Result<()> {
42 conn.execute_batch(
43 "CREATE TABLE IF NOT EXISTS pipeline_runs (
44 command TEXT PRIMARY KEY,
45 started_at TEXT NOT NULL,
46 finished_at TEXT,
47 status TEXT NOT NULL,
48 duration_ms INTEGER,
49 summary TEXT
50 );",
51 )
52}
53
54pub fn start_run(conn: &Connection, command: &str) -> rusqlite::Result<()> {
55 conn.execute(
56 "INSERT INTO pipeline_runs (command, started_at, status)
57 VALUES (?1, datetime('now'), 'running')
58 ON CONFLICT(command) DO UPDATE SET
59 started_at = excluded.started_at,
60 status = 'running',
61 finished_at = NULL,
62 duration_ms = NULL,
63 summary = NULL",
64 params![command],
65 )?;
66 Ok(())
67}
68
69pub fn finish_run(
70 conn: &Connection,
71 command: &str,
72 status: &str,
73 duration_ms: i64,
74 summary: Option<&str>,
75) -> rusqlite::Result<()> {
76 conn.execute(
77 "UPDATE pipeline_runs SET
78 finished_at = datetime('now'),
79 status = ?2,
80 duration_ms = ?3,
81 summary = ?4
82 WHERE command = ?1",
83 params![command, status, duration_ms, summary],
84 )?;
85 Ok(())
86}
87
88pub fn track_in<T, F>(
100 conn: &Connection,
101 ctx: &crate::library::LibraryContext,
102 guard: &crate::library_locks::CommandGuard,
103 command: &str,
104 f: F,
105) -> Result<T>
106where
107 F: FnOnce() -> Result<T>,
108{
109 track_in_as(conn, ctx, guard, command, command, f)
110}
111
112pub fn track_in_as<T, F>(
120 conn: &Connection,
121 ctx: &crate::library::LibraryContext,
122 guard: &crate::library_locks::CommandGuard,
123 lock_command: &str,
124 label: &str,
125 f: F,
126) -> Result<T>
127where
128 F: FnOnce() -> Result<T>,
129{
130 guard.ensure_matches(ctx, lock_command)?;
131 ctx.ensure_root_identity()?;
136 ensure_pipeline_runs_table(conn)?;
137 start_run(conn, label)?;
138 let started = std::time::Instant::now();
139 let result = f();
140 let duration_ms = started.elapsed().as_millis().min(i64::MAX as u128) as i64;
141 match result {
142 Ok(value) => {
143 finish_run(conn, label, "success", duration_ms, None)?;
144 Ok(value)
145 }
146 Err(error) => {
147 if let Err(record_error) =
150 finish_run(conn, label, "failed", duration_ms, Some(&error.to_string()))
151 {
152 return Err(error.context(format!(
153 "also could not record the failed run: {record_error}"
154 )));
155 }
156 Err(error)
157 }
158 }
159}
160
161pub fn record_heartbeat_in(
167 conn: &Connection,
168 ctx: &crate::library::LibraryContext,
169 command: &str,
170) -> Result<()> {
171 ctx.ensure_root_identity()?;
174 ensure_pipeline_runs_table(conn)?;
175 conn.execute(
176 "INSERT INTO pipeline_runs (command, started_at, status, summary)
177 VALUES (?1, datetime('now'), 'success', 'last successful cycle')
178 ON CONFLICT(command) DO UPDATE SET
179 started_at = excluded.started_at,
180 status = 'success',
181 finished_at = NULL,
182 duration_ms = NULL,
183 summary = excluded.summary",
184 params![command],
185 )?;
186 Ok(())
187}
188
189pub fn read_all_in(
195 conn: &Connection,
196 ctx: &crate::library::LibraryContext,
197) -> Result<Vec<PipelineRunStatus>> {
198 ensure_pipeline_runs_table(conn)?;
199 let mut out = Vec::with_capacity(TRACKED_COMMANDS.len());
200 for command in TRACKED_COMMANDS {
201 out.push(read_one_in(conn, ctx, command)?);
202 }
203 if conn
209 .query_row(
210 "SELECT 1 FROM pipeline_runs WHERE command = 'watch'",
211 [],
212 |r| r.get::<_, i64>(0),
213 )
214 .optional()?
215 .is_some()
216 {
217 out.push(read_one_in(conn, ctx, "watch")?);
218 }
219 if conn
226 .query_row(
227 "SELECT 1 FROM pipeline_runs WHERE command = 'location-names'",
228 [],
229 |r| r.get::<_, i64>(0),
230 )
231 .optional()?
232 .is_some()
233 {
234 out.push(read_one_in(conn, ctx, "location-names")?);
235 }
236 Ok(out)
237}
238
239fn read_one_in(
247 conn: &Connection,
248 ctx: &crate::library::LibraryContext,
249 command: &str,
250) -> Result<PipelineRunStatus> {
251 let row: Option<(String, Option<i64>, String)> = conn
252 .query_row(
253 "SELECT started_at, duration_ms, status FROM pipeline_runs WHERE command = ?1",
254 params![command],
255 |r| Ok((r.get(0)?, r.get(1)?, r.get(2)?)),
256 )
257 .optional()?;
258 let currently_running = match command {
259 "locations" => {
260 crate::library_locks::command_locked(ctx, "locations")?
261 && !crate::library_locks::command_locked(ctx, "location-names")?
262 }
263 other => crate::library_locks::command_locked(ctx, other)?,
264 };
265 let (last_run_at, status, duration_ms) = match row {
266 None => (None, None, None),
267 Some((started_at, duration_ms, stored_status)) => {
268 let status = if stored_status == "running" && !currently_running {
269 "crashed".to_string()
270 } else {
271 stored_status
272 };
273 (Some(started_at), Some(status), duration_ms)
274 }
275 };
276 Ok(PipelineRunStatus {
277 command: command.to_string(),
278 last_run_at,
279 status,
280 duration_ms,
281 currently_running,
282 })
283}
284
285static SIGINT_INSTALLED: std::sync::atomic::AtomicBool = std::sync::atomic::AtomicBool::new(false);
303static SIGINT_COMMAND: std::sync::Mutex<Option<&'static str>> = std::sync::Mutex::new(None);
304
305pub fn install_sigint_handler_in(
306 ctx: std::sync::Arc<crate::library::LibraryContext>,
307 command: &'static str,
308) -> Result<()> {
309 ctx.ensure_root_identity()
310 .context("validating the library before installing the SIGINT handler")?;
311 if let Ok(mut current) = SIGINT_COMMAND.lock() {
313 *current = Some(command);
314 }
315 if SIGINT_INSTALLED.load(std::sync::atomic::Ordering::SeqCst) {
318 return Ok(());
319 }
320 ctrlc::set_handler(move || {
321 let command = SIGINT_COMMAND
322 .lock()
323 .ok()
324 .and_then(|c| *c)
325 .unwrap_or(command);
326 if ctx.ensure_root_identity().is_ok() {
327 if let Ok(conn) = crate::library_db::open_without_create(&ctx.paths.db) {
328 let _ = conn.busy_timeout(std::time::Duration::from_secs(5));
329 let started_at: Option<String> = conn
330 .query_row(
331 "SELECT started_at FROM pipeline_runs WHERE command = ?1",
332 params![command],
333 |r| r.get(0),
334 )
335 .optional()
336 .ok()
337 .flatten();
338 let duration_ms = started_at
339 .and_then(|s| {
340 chrono::NaiveDateTime::parse_from_str(&s, "%Y-%m-%d %H:%M:%S").ok()
341 })
342 .map(|started| {
343 (chrono::Utc::now().naive_utc() - started)
344 .num_milliseconds()
345 .max(0)
346 })
347 .unwrap_or(0);
348 let _ = finish_run(&conn, command, "interrupted", duration_ms, None);
349 }
350 }
351 std::process::exit(130);
352 })
353 .context("installing SIGINT handler")?;
354 SIGINT_INSTALLED.store(true, std::sync::atomic::Ordering::SeqCst);
355 Ok(())
356}
357
358#[derive(Debug, Clone, PartialEq, Serialize)]
359pub struct PipelineRunStatus {
360 pub command: String,
361 pub last_run_at: Option<String>,
362 pub status: Option<String>,
365 pub duration_ms: Option<i64>,
366 pub currently_running: bool,
367}
368
369#[cfg(test)]
370mod tests {
371 use super::*;
372
373 fn test_db() -> Connection {
374 let conn = Connection::open_in_memory().unwrap();
375 ensure_pipeline_runs_table(&conn).unwrap();
376 conn
377 }
378
379 #[test]
380 fn heartbeat_records_last_cycle_and_reads_back() {
381 let (_t, ctx, conn) = in_library();
382 assert!(read_all_in(&conn, &ctx)
384 .unwrap()
385 .iter()
386 .all(|r| r.command != "watch"));
387 record_heartbeat_in(&conn, &ctx, "watch").unwrap();
388 let w = read_all_in(&conn, &ctx)
389 .unwrap()
390 .into_iter()
391 .find(|r| r.command == "watch")
392 .expect("the heartbeat row must surface in the run read");
393 assert!(w.last_run_at.is_some(), "started_at is the last-cycle time");
394 assert_eq!(w.status.as_deref(), Some("success"));
395 assert!(!w.currently_running, "no watch process holds the lock");
396
397 record_heartbeat_in(&conn, &ctx, "watch").unwrap();
400 assert_eq!(
401 read_all_in(&conn, &ctx)
402 .unwrap()
403 .iter()
404 .filter(|r| r.command == "watch")
405 .count(),
406 1
407 );
408 }
409
410 #[test]
411 fn ensure_pipeline_runs_table_is_idempotent() {
412 let conn = test_db();
413 ensure_pipeline_runs_table(&conn).unwrap();
414 }
415
416 #[test]
417 fn start_run_then_finish_run_records_success() {
418 let conn = test_db();
419 start_run(&conn, "embed").unwrap();
420
421 let status: String = conn
422 .query_row(
423 "SELECT status FROM pipeline_runs WHERE command = 'embed'",
424 [],
425 |r| r.get(0),
426 )
427 .unwrap();
428 assert_eq!(status, "running");
429
430 finish_run(&conn, "embed", "success", 1234, None).unwrap();
431
432 let (status, duration_ms, summary): (String, i64, Option<String>) = conn
433 .query_row(
434 "SELECT status, duration_ms, summary FROM pipeline_runs WHERE command = 'embed'",
435 [],
436 |r| Ok((r.get(0)?, r.get(1)?, r.get(2)?)),
437 )
438 .unwrap();
439 assert_eq!(status, "success");
440 assert_eq!(duration_ms, 1234);
441 assert_eq!(summary, None);
442 }
443
444 #[test]
445 fn start_run_upserts_resetting_prior_finish_fields() {
446 let conn = test_db();
447 start_run(&conn, "embed").unwrap();
448 finish_run(&conn, "embed", "failed", 500, Some("boom")).unwrap();
449
450 start_run(&conn, "embed").unwrap(); let (status, duration_ms, summary): (String, Option<i64>, Option<String>) = conn
453 .query_row(
454 "SELECT status, duration_ms, summary FROM pipeline_runs WHERE command = 'embed'",
455 [],
456 |r| Ok((r.get(0)?, r.get(1)?, r.get(2)?)),
457 )
458 .unwrap();
459 assert_eq!(status, "running");
460 assert_eq!(duration_ms, None);
461 assert_eq!(summary, None);
462
463 let count: i64 = conn
464 .query_row("SELECT COUNT(*) FROM pipeline_runs", [], |r| r.get(0))
465 .unwrap();
466 assert_eq!(count, 1, "upsert, not a second row");
467 }
468
469 fn in_library() -> (
474 tempfile::TempDir,
475 crate::library::LibraryContext,
476 Connection,
477 ) {
478 let temp = tempfile::tempdir().unwrap();
479 let root = temp.path().join("photos");
480 std::fs::create_dir(&root).unwrap();
481 let ctx = crate::library::LibraryContext::new(&root, &temp.path().join("cache")).unwrap();
482 std::fs::create_dir_all(&ctx.paths.locks).unwrap();
483 let conn = Connection::open_in_memory().unwrap();
484 ensure_pipeline_runs_table(&conn).unwrap();
485 (temp, ctx, conn)
486 }
487
488 #[test]
489 fn track_in_records_runs_under_an_already_held_command_guard() {
490 let (_t, ctx, conn) = in_library();
491 let guard = crate::library_locks::try_command(&ctx, "scan").unwrap();
492 let result = track_in(&conn, &ctx, &guard, "scan", || Ok(7)).unwrap();
493 assert_eq!(result, 7);
494 let (status, summary): (String, Option<String>) = conn
495 .query_row(
496 "SELECT status, summary FROM pipeline_runs WHERE command = 'scan'",
497 [],
498 |r| Ok((r.get(0)?, r.get(1)?)),
499 )
500 .unwrap();
501 assert_eq!(status, "success");
502 assert_eq!(summary, None);
503 let count: i64 = conn
506 .query_row("SELECT COUNT(*) FROM pipeline_runs", [], |r| r.get(0))
507 .unwrap();
508 assert_eq!(count, 1);
509 let failed: Result<()> =
510 track_in(&conn, &ctx, &guard, "scan", || Err(anyhow::anyhow!("boom")));
511 failed.unwrap_err();
512 let (status, summary): (String, Option<String>) = conn
513 .query_row(
514 "SELECT status, summary FROM pipeline_runs WHERE command = 'scan'",
515 [],
516 |r| Ok((r.get(0)?, r.get(1)?)),
517 )
518 .unwrap();
519 assert_eq!(status, "failed");
520 assert_eq!(summary.as_deref(), Some("boom"));
521 }
522
523 #[test]
524 fn track_in_refuses_a_guard_from_another_command_or_library() {
525 let (_t, ctx, conn) = in_library();
526 let guard = crate::library_locks::try_command(&ctx, "scan").unwrap();
527 let result: Result<()> = track_in(&conn, &ctx, &guard, "embed", || Ok(()));
529 let err = result.unwrap_err();
530 assert!(format!("{err:#}").contains("scan"), "{err:#}");
531 let count: i64 = conn
532 .query_row("SELECT COUNT(*) FROM pipeline_runs", [], |r| r.get(0))
533 .unwrap();
534 assert_eq!(count, 0, "a refused guard must write no row");
535
536 let temp = tempfile::tempdir().unwrap();
539 let other_root = temp.path().join("other");
540 std::fs::create_dir(&other_root).unwrap();
541 let other =
542 crate::library::LibraryContext::new(&other_root, &temp.path().join("cache")).unwrap();
543 let result: Result<()> = track_in(&conn, &other, &guard, "scan", || Ok(()));
544 let err = result.unwrap_err();
545 assert!(
546 format!("{err:#}").contains(other_root.file_name().unwrap().to_string_lossy().as_ref()),
547 "{err:#}"
548 );
549 let count: i64 = conn
550 .query_row("SELECT COUNT(*) FROM pipeline_runs", [], |r| r.get(0))
551 .unwrap();
552 assert_eq!(count, 0);
553 }
554
555 #[test]
556 fn read_all_in_answers_liveness_from_the_library_locks() {
557 let (_t, ctx, conn) = in_library();
558 let guard = crate::library_locks::try_command(&ctx, "scan").unwrap();
559 track_in(&conn, &ctx, &guard, "scan", || Ok(())).unwrap();
560 drop(guard);
562 let _faces = crate::library_locks::try_command(&ctx, "faces").unwrap();
566 let statuses = read_all_in(&conn, &ctx).unwrap();
567 let scan = statuses.iter().find(|s| s.command == "scan").unwrap();
568 assert_eq!(scan.status.as_deref(), Some("success"));
569 assert!(!scan.currently_running);
570 let faces = statuses.iter().find(|s| s.command == "faces").unwrap();
571 assert_eq!(faces.status, None);
572 assert!(faces.currently_running);
573 }
574
575 #[test]
576 fn location_names_stage_surfaces_only_once_a_row_exists() {
577 let (_t, ctx, conn) = in_library();
578 assert!(read_all_in(&conn, &ctx)
582 .unwrap()
583 .iter()
584 .all(|r| r.command != "location-names"));
585 start_run(&conn, "location-names").unwrap();
588 finish_run(&conn, "location-names", "success", 5, None).unwrap();
589 let names = read_all_in(&conn, &ctx)
590 .unwrap()
591 .into_iter()
592 .find(|r| r.command == "location-names")
593 .expect("a written location-names row must surface in the run read");
594 assert_eq!(names.status.as_deref(), Some("success"));
595 assert!(!names.currently_running);
596 }
597
598 #[test]
599 fn a_running_location_names_stage_reads_running_not_crashed() {
600 let (_t, ctx, conn) = in_library();
606 let guard = crate::library_locks::try_command(&ctx, "locations").unwrap();
607 let names_guard = crate::library_locks::try_command(&ctx, "location-names").unwrap();
608 start_run(&conn, "location-names").unwrap();
609
610 let names = read_all_in(&conn, &ctx)
611 .unwrap()
612 .into_iter()
613 .find(|r| r.command == "location-names")
614 .expect("a started location-names row must surface");
615 assert_eq!(
616 names.status.as_deref(),
617 Some("running"),
618 "an actively running stage is running, not crashed"
619 );
620 assert!(names.currently_running);
621 let locations = read_all_in(&conn, &ctx)
625 .unwrap()
626 .into_iter()
627 .find(|r| r.command == "locations")
628 .unwrap();
629 assert!(!locations.currently_running);
630
631 drop(guard);
634 drop(names_guard);
635 let names = read_all_in(&conn, &ctx)
636 .unwrap()
637 .into_iter()
638 .find(|r| r.command == "location-names")
639 .unwrap();
640 assert_eq!(names.status.as_deref(), Some("crashed"));
641 }
642
643 #[test]
644 fn a_stale_names_row_does_not_mask_a_running_recompute() {
645 let (_t, ctx, conn) = in_library();
652 start_run(&conn, "location-names").unwrap();
653
654 let guard = crate::library_locks::try_command(&ctx, "locations").unwrap();
655 start_run(&conn, "locations").unwrap();
656
657 let statuses = read_all_in(&conn, &ctx).unwrap();
658 let names = statuses
659 .iter()
660 .find(|r| r.command == "location-names")
661 .unwrap();
662 assert_eq!(
663 names.status.as_deref(),
664 Some("crashed"),
665 "a stale row must not borrow the recompute's lock liveness"
666 );
667 let locations = statuses.iter().find(|r| r.command == "locations").unwrap();
668 assert_eq!(locations.status.as_deref(), Some("running"));
669 assert!(
670 locations.currently_running,
671 "the recompute is the real live holder and must be reported as such"
672 );
673 drop(guard);
674 }
675
676 #[test]
677 fn track_in_as_records_a_label_under_another_commands_guard() {
678 let (_t, ctx, conn) = in_library();
683 let guard = crate::library_locks::try_command(&ctx, "locations").unwrap();
684 track_in_as(
685 &conn,
686 &ctx,
687 &guard,
688 "locations",
689 "location-names",
690 || Ok(()),
691 )
692 .unwrap();
693 let (command, status): (String, String) = conn
694 .query_row("SELECT command, status FROM pipeline_runs", [], |r| {
695 Ok((r.get(0)?, r.get(1)?))
696 })
697 .unwrap();
698 assert_eq!(command, "location-names");
699 assert_eq!(status, "success");
700
701 let guard = crate::library_locks::try_command(&ctx, "scan").unwrap();
704 let result: Result<()> = track_in_as(
705 &conn,
706 &ctx,
707 &guard,
708 "locations",
709 "location-names",
710 || Ok(()),
711 );
712 assert!(
713 result.is_err(),
714 "a scan guard must not bookkeep under the locations lock"
715 );
716 }
717
718 #[test]
719 fn install_sigint_handler_in_validates_the_library_before_installing() {
720 let temp = tempfile::tempdir().unwrap();
725 let root = temp.path().join("photos");
726 std::fs::create_dir(&root).unwrap();
727 let ctx = std::sync::Arc::new(
728 crate::library::LibraryContext::new(&root, &temp.path().join("cache")).unwrap(),
729 );
730 std::fs::rename(&root, temp.path().join("moved")).unwrap();
731 std::fs::create_dir(&root).unwrap();
732 let err = install_sigint_handler_in(ctx, "scan").unwrap_err();
733 assert!(format!("{err:#}").contains("no longer names"), "{err:#}");
734 }
735
736 #[test]
737 fn install_sigint_handler_in_is_idempotent_within_a_process() {
738 let temp = tempfile::tempdir().unwrap();
743 let root = temp.path().join("photos");
744 std::fs::create_dir(&root).unwrap();
745 let ctx = std::sync::Arc::new(
746 crate::library::LibraryContext::new(&root, &temp.path().join("cache")).unwrap(),
747 );
748 install_sigint_handler_in(ctx.clone(), "scan").expect("first install");
749 install_sigint_handler_in(ctx, "faces")
750 .expect("a second install in the same process must be a no-op, not an error");
751 }
752}