1use std::path::Path;
38
39use keel_journal::admin::JournalAdmin;
40use serde::Serialize;
41
42use crate::render::to_json;
43use crate::{EXIT_FAILURE, EXIT_USAGE, Rendered, evidence};
44
45#[derive(Debug, Clone, Default)]
48pub struct FsckOptions {
49 pub fix: bool,
51 pub prune: Option<String>,
53}
54
55#[derive(Debug, Serialize)]
57struct IntegrityReport {
58 detail: Vec<String>,
59 ok: bool,
60}
61
62#[derive(Debug, Serialize)]
64struct FlowFinding {
65 count: u64,
66 flow_ids: Vec<String>,
67}
68
69#[derive(Debug, Serialize)]
71struct StaleStep {
72 flow_id: String,
73 seq: i64,
74}
75
76#[derive(Debug, Serialize)]
78struct ChecksReport {
79 dangling_leases: FlowFinding,
80 dead_flows: FlowFinding,
81 expired_cache_rows: u64,
82 orphan_steps: OrphanReport,
83 stale_running_steps: Vec<StaleStep>,
84}
85
86#[derive(Debug, Serialize)]
88struct OrphanReport {
89 missing_flow_ids: Vec<String>,
90 rows: u64,
91}
92
93#[derive(Debug, Serialize)]
95struct RepairsReport {
96 dangling_leases_cleared: u64,
97 expired_cache_swept: u64,
98 orphan_steps_deleted: u64,
99 stale_running_steps_deleted: u64,
100 wal_checkpointed: bool,
101}
102
103#[derive(Debug, Serialize)]
105struct PruneReport {
106 age: String,
107 flow_ids: Vec<String>,
108 flows_deleted: u64,
109 steps_deleted: u64,
110}
111
112#[derive(Debug, Serialize)]
115struct FsckReport {
116 checks: Option<ChecksReport>,
117 fix: bool,
118 integrity: Option<IntegrityReport>,
119 journal: String,
120 journal_present: bool,
121 ok: bool,
122 prune: Option<PruneReport>,
123 repairs: Option<RepairsReport>,
124 schema_present: Option<bool>,
125}
126
127pub fn run(project: &Path, options: &FsckOptions, now_ms: i64) -> Rendered {
130 let prune_age = match &options.prune {
132 None => None,
133 Some(raw) => match parse_age_ms(raw) {
134 Ok(ms) => Some((raw.clone(), ms)),
135 Err(why) => return usage_error(raw, &why),
136 },
137 };
138
139 let resolved = evidence::resolved_journal(project);
140 if !resolved.path.exists() {
141 let report = FsckReport {
142 checks: None,
143 fix: options.fix,
144 integrity: None,
145 journal: resolved.display,
146 journal_present: false,
147 ok: true,
148 prune: None,
149 repairs: None,
150 schema_present: None,
151 };
152 return Rendered::ok(
153 format!(
154 "keel \u{25b8} fsck {}: no journal yet \u{2014} nothing to check.",
155 report.journal
156 ),
157 to_json(&report),
158 );
159 }
160
161 let writes = options.fix || prune_age.is_some();
162 let admin = match open(&resolved.path, writes) {
163 Ok(a) => a,
164 Err(e) => return soft_error(&e),
165 };
166
167 build_report(&admin, resolved.display, options.fix, prune_age, now_ms)
168}
169
170fn build_report(
172 admin: &JournalAdmin,
173 journal: String,
174 fix: bool,
175 prune_age: Option<(String, i64)>,
176 now_ms: i64,
177) -> Rendered {
178 let integrity = admin.integrity_check();
179 if !integrity.ok {
180 let report = FsckReport {
181 checks: None,
182 fix,
183 integrity: Some(IntegrityReport {
184 detail: integrity.detail,
185 ok: false,
186 }),
187 journal,
188 journal_present: true,
189 ok: false,
190 prune: None,
191 repairs: None,
192 schema_present: None,
193 };
194 let human = format!(
195 "keel \u{25b8} fsck {}: FAILED SQLite integrity_check.\n why: the file is corrupt \
196 beyond what WAL recovery repairs (detail in --json).\n next: restore the journal \
197 from a backup, or delete it \u{2014} flows lose replay, Tier 1 is unaffected.",
198 report.journal
199 );
200 return Rendered::ok(human, to_json(&report)).with_exit(EXIT_FAILURE);
201 }
202
203 let schema_present = admin.schema_present().unwrap_or(false);
204 if !schema_present {
205 let report = FsckReport {
206 checks: None,
207 fix,
208 integrity: Some(IntegrityReport {
209 detail: Vec::new(),
210 ok: true,
211 }),
212 journal,
213 journal_present: true,
214 ok: false,
215 prune: None,
216 repairs: None,
217 schema_present: Some(false),
218 };
219 let human = format!(
220 "keel \u{25b8} fsck {}: not a keel journal.\n why: the file is valid SQLite but \
221 lacks the frozen journal schema (flows/steps/cache).\n next: check the `journal` \
222 key in keel.toml \u{2014} it points at some other database.",
223 report.journal
224 );
225 return Rendered::ok(human, to_json(&report)).with_exit(EXIT_FAILURE);
226 }
227
228 let checks = match read_checks(admin, now_ms) {
230 Ok(c) => c,
231 Err(e) => return soft_error(&e),
232 };
233
234 let repairs = if fix {
235 match apply_repairs(admin, now_ms) {
236 Ok(r) => Some(r),
237 Err(e) => return soft_error(&e),
238 }
239 } else {
240 None
241 };
242
243 let prune = match prune_age {
244 None => None,
245 Some((age, age_ms)) => {
246 let cutoff = now_ms.saturating_sub(age_ms);
247 match apply_prune(admin, age, cutoff) {
248 Ok(p) => Some(p),
249 Err(e) => return soft_error(&e),
250 }
251 }
252 };
253
254 let findings = checks.orphan_steps.rows
257 + checks.dangling_leases.count
258 + checks.stale_running_steps.len() as u64;
259 let ok = fix || findings == 0;
260
261 let report = FsckReport {
262 checks: Some(checks),
263 fix,
264 integrity: Some(IntegrityReport {
265 detail: Vec::new(),
266 ok: true,
267 }),
268 journal,
269 journal_present: true,
270 ok,
271 prune,
272 repairs,
273 schema_present: Some(true),
274 };
275 let human = human(&report);
276 let exit = if ok { crate::EXIT_OK } else { EXIT_FAILURE };
277 Rendered::ok(human, to_json(&report)).with_exit(exit)
278}
279
280fn open(path: &Path, writes: bool) -> Result<JournalAdmin, String> {
281 let open = if writes {
282 JournalAdmin::open_readwrite(path)
283 } else {
284 JournalAdmin::open_readonly(path)
285 };
286 open.map_err(|e| format!("could not open {}: {e}", path.display()))
287}
288
289fn read_checks(admin: &JournalAdmin, now_ms: i64) -> Result<ChecksReport, String> {
290 let q = |e: keel_journal::Error| format!("journal query failed: {e}");
291 let dead = admin.dead_flow_ids().map_err(q)?;
292 let leases = admin.dangling_lease_flow_ids().map_err(q)?;
293 Ok(ChecksReport {
294 dangling_leases: FlowFinding {
295 count: leases.len() as u64,
296 flow_ids: leases,
297 },
298 dead_flows: FlowFinding {
299 count: dead.len() as u64,
300 flow_ids: dead,
301 },
302 expired_cache_rows: admin.expired_cache_count(now_ms).map_err(q)?,
303 orphan_steps: OrphanReport {
304 missing_flow_ids: admin.orphan_step_flow_ids().map_err(q)?,
305 rows: admin.orphan_step_count().map_err(q)?,
306 },
307 stale_running_steps: admin
308 .stale_running_steps()
309 .map_err(q)?
310 .into_iter()
311 .map(|s| StaleStep {
312 flow_id: s.flow_id,
313 seq: s.seq,
314 })
315 .collect(),
316 })
317}
318
319fn apply_repairs(admin: &JournalAdmin, now_ms: i64) -> Result<RepairsReport, String> {
320 let q = |e: keel_journal::Error| format!("journal repair failed: {e}");
321 let orphan_steps_deleted = admin.delete_orphan_steps().map_err(q)?;
322 let dangling_leases_cleared = admin.clear_dangling_leases().map_err(q)?;
323 let stale_running_steps_deleted = admin.sweep_stale_running_steps().map_err(q)?;
324 let expired_cache_swept = admin.sweep_expired_cache(now_ms).map_err(q)?;
325 admin.wal_checkpoint().map_err(q)?;
326 Ok(RepairsReport {
327 dangling_leases_cleared,
328 expired_cache_swept,
329 orphan_steps_deleted,
330 stale_running_steps_deleted,
331 wal_checkpointed: true,
332 })
333}
334
335fn apply_prune(admin: &JournalAdmin, age: String, cutoff_ms: i64) -> Result<PruneReport, String> {
336 let q = |e: keel_journal::Error| format!("journal prune failed: {e}");
337 let flow_ids = admin.prunable_completed_flow_ids(cutoff_ms).map_err(q)?;
338 let outcome = admin.prune_completed_flows(cutoff_ms).map_err(q)?;
339 Ok(PruneReport {
340 age,
341 flow_ids,
342 flows_deleted: outcome.flows_deleted,
343 steps_deleted: outcome.steps_deleted,
344 })
345}
346
347fn human(report: &FsckReport) -> String {
349 let checks = report
350 .checks
351 .as_ref()
352 .expect("human() runs on full reports");
353 let verdict = if report.ok { "clean" } else { "FINDINGS" };
354 let mut lines = vec![format!(
355 "keel \u{25b8} fsck {} \u{2014} {verdict}\n",
356 report.journal
357 )];
358 lines.push(" integrity: ok\n".to_owned());
359 lines.push(format!(
360 " orphan steps: {} row(s){}\n",
361 checks.orphan_steps.rows,
362 if checks.orphan_steps.missing_flow_ids.is_empty() {
363 String::new()
364 } else {
365 format!(
366 " pointing at missing flow(s) {}",
367 checks.orphan_steps.missing_flow_ids.join(", ")
368 )
369 }
370 ));
371 lines.push(format!(
372 " dangling leases: {}\n",
373 checks.dangling_leases.count
374 ));
375 lines.push(format!(
376 " stale running steps: {}\n",
377 checks.stale_running_steps.len()
378 ));
379 lines.push(format!(
380 " expired cache rows: {}\n",
381 checks.expired_cache_rows
382 ));
383 lines.push(format!(
384 " dead flows: {}{}\n",
385 checks.dead_flows.count,
386 if checks.dead_flows.count == 0 {
387 ""
388 } else {
389 " (inspect with `keel flows --dead`)"
390 }
391 ));
392 if let Some(r) = &report.repairs {
393 lines.push(format!(
394 " repaired: {} orphan step(s), {} lease(s), {} stale step(s); swept {} expired \
395 cache row(s); WAL checkpointed.\n",
396 r.orphan_steps_deleted,
397 r.dangling_leases_cleared,
398 r.stale_running_steps_deleted,
399 r.expired_cache_swept
400 ));
401 }
402 if let Some(p) = &report.prune {
403 lines.push(format!(
404 " pruned: {} completed flow(s) older than {} ({} step row(s)).\n note: a pruned \
405 flow cannot replay \u{2014} rerunning the same entrypoint+args starts fresh.\n",
406 p.flows_deleted, p.age, p.steps_deleted
407 ));
408 }
409 if !report.ok {
410 lines.push(
411 " next: run `keel fsck --fix` to repair the findings above (safe: repairs never \
412 touch resumable flows).\n"
413 .to_owned(),
414 );
415 }
416 lines.concat()
417}
418
419fn parse_age_ms(raw: &str) -> Result<i64, String> {
421 let (digits, unit) = raw.split_at(raw.len().saturating_sub(1));
422 let per_unit: i64 = match unit {
423 "s" => 1_000,
424 "m" => 60_000,
425 "h" => 3_600_000,
426 "d" => 86_400_000,
427 _ => return Err("age needs a unit suffix: s, m, h, or d".to_owned()),
428 };
429 let n: i64 = digits
430 .parse()
431 .map_err(|_| "age needs a whole number before the unit".to_owned())?;
432 if n < 0 {
433 return Err("age cannot be negative".to_owned());
434 }
435 n.checked_mul(per_unit)
436 .ok_or_else(|| "age is out of range".to_owned())
437}
438
439fn usage_error(raw: &str, why: &str) -> Rendered {
440 #[derive(Serialize)]
441 struct UsageReport<'a> {
442 error: &'static str,
443 next: &'static str,
444 what: String,
445 why: &'a str,
446 }
447 let what = format!("Cannot parse --prune {raw:?} as an age.");
448 let next = "Use <number><unit> with unit s/m/h/d, e.g. `keel fsck --prune 30d`.";
449 let human = format!("keel \u{25b8} {what}\n why: {why}\n next: {next}");
450 Rendered {
451 human,
452 json: to_json(&UsageReport {
453 error: "bad-age",
454 next,
455 what,
456 why,
457 }),
458 exit: EXIT_USAGE,
459 to_stderr: true,
460 }
461}
462
463fn soft_error(message: &str) -> Rendered {
465 #[derive(Serialize)]
466 struct ErrReport<'a> {
467 error: &'a str,
468 }
469 Rendered {
470 human: format!("keel \u{25b8} {message}"),
471 json: to_json(&ErrReport { error: message }),
472 exit: EXIT_FAILURE,
473 to_stderr: true,
474 }
475}
476
477#[cfg(test)]
478mod tests {
479 use super::*;
480 use rusqlite::{Connection, params};
481 use std::path::PathBuf;
482
483 const T0: i64 = 1_783_728_000_000;
484
485 fn project_with_fixtures(fixtures: &[&str]) -> (tempfile::TempDir, PathBuf) {
488 let root = Path::new(env!("CARGO_MANIFEST_DIR")).join("../..");
489 let dir = tempfile::TempDir::new().unwrap();
490 let keel = dir.path().join(".keel");
491 std::fs::create_dir_all(&keel).unwrap();
492 let conn = Connection::open(keel.join("journal.db")).unwrap();
493 let schema = std::fs::read_to_string(root.join("contracts/journal.sql")).unwrap();
494 conn.execute_batch(&schema).unwrap();
495 for f in fixtures {
496 let sql =
497 std::fs::read_to_string(root.join("conformance/fixtures/journal").join(f)).unwrap();
498 conn.execute_batch(&sql).unwrap();
499 }
500 let project = dir.path().to_path_buf();
501 (dir, project)
502 }
503
504 fn damage(project: &Path) {
505 let conn = Connection::open(project.join(".keel/journal.db")).unwrap();
506 conn.execute_batch("PRAGMA foreign_keys = OFF;").unwrap();
507 conn.execute(
510 "INSERT INTO steps VALUES ('09GHOST', 1, 'x#1', 'effect', 1, 'ok', NULL, NULL, ?1, ?1)",
511 params![T0],
512 )
513 .unwrap();
514 conn.execute(
515 "UPDATE flows SET lease_holder = 'host-z:pid-9', lease_expires = ?1 \
516 WHERE flow_id = '01JZWY0A0000000000000003'",
517 params![T0],
518 )
519 .unwrap();
520 conn.execute(
521 "INSERT INTO steps VALUES ('01JZWY0A0000000000000001', 6, 'y#1', 'effect', 1, \
522 'running', NULL, NULL, ?1, NULL)",
523 params![T0],
524 )
525 .unwrap();
526 conn.execute(
527 "INSERT INTO cache (key, value, expires_at) VALUES ('old', X'C0', ?1)",
528 params![T0 - 1],
529 )
530 .unwrap();
531 conn.execute(
532 "INSERT INTO cache (key, value, expires_at) VALUES ('new', X'C0', ?1)",
533 params![T0 + 3_600_000],
534 )
535 .unwrap();
536 }
537
538 #[test]
539 fn clean_fixtures_pass() {
540 let (_d, project) = project_with_fixtures(&[
541 "completed-flow.sql",
542 "interrupted-flow.sql",
543 "dead-flow.sql",
544 ]);
545 let r = run(&project, &FsckOptions::default(), T0);
546 assert_eq!(r.exit, crate::EXIT_OK);
547 assert_eq!(r.json["ok"], true);
548 assert_eq!(r.json["integrity"]["ok"], true);
549 assert_eq!(r.json["checks"]["orphan_steps"]["rows"], 0);
550 assert_eq!(r.json["checks"]["dangling_leases"]["count"], 0);
551 assert_eq!(
553 r.json["checks"]["stale_running_steps"]
554 .as_array()
555 .unwrap()
556 .len(),
557 0
558 );
559 assert_eq!(r.json["checks"]["dead_flows"]["count"], 1);
561 assert!(r.human.contains("clean"));
562 }
563
564 #[test]
565 fn findings_fail_the_check_and_fix_repairs_them() {
566 let (_d, project) = project_with_fixtures(&[
567 "completed-flow.sql",
568 "interrupted-flow.sql",
569 "dead-flow.sql",
570 ]);
571 damage(&project);
572
573 let r = run(&project, &FsckOptions::default(), T0);
574 assert_eq!(r.exit, EXIT_FAILURE);
575 assert_eq!(r.json["ok"], false);
576 assert_eq!(r.json["checks"]["orphan_steps"]["rows"], 1);
577 assert_eq!(
578 r.json["checks"]["orphan_steps"]["missing_flow_ids"][0],
579 "09GHOST"
580 );
581 assert_eq!(r.json["checks"]["dangling_leases"]["count"], 1);
582 assert_eq!(r.json["checks"]["stale_running_steps"][0]["seq"], 6);
583 assert_eq!(r.json["checks"]["expired_cache_rows"], 1);
584 assert!(r.human.contains("next: run `keel fsck --fix`"));
585
586 let fixed = run(
587 &project,
588 &FsckOptions {
589 fix: true,
590 prune: None,
591 },
592 T0,
593 );
594 assert_eq!(fixed.exit, crate::EXIT_OK);
595 assert_eq!(fixed.json["ok"], true);
596 assert_eq!(fixed.json["repairs"]["orphan_steps_deleted"], 1);
597 assert_eq!(fixed.json["repairs"]["dangling_leases_cleared"], 1);
598 assert_eq!(fixed.json["repairs"]["stale_running_steps_deleted"], 1);
599 assert_eq!(fixed.json["repairs"]["expired_cache_swept"], 1);
600
601 let again = run(&project, &FsckOptions::default(), T0);
603 assert_eq!(again.exit, crate::EXIT_OK);
604 assert_eq!(again.json["checks"]["orphan_steps"]["rows"], 0);
605 assert_eq!(again.json["checks"]["expired_cache_rows"], 0);
606 }
607
608 #[test]
609 fn prune_removes_old_completed_flows_only() {
610 let (_d, project) = project_with_fixtures(&[
611 "completed-flow.sql",
612 "interrupted-flow.sql",
613 "dead-flow.sql",
614 ]);
615 let now = T0 + 40 * 86_400_000;
618 let r = run(
619 &project,
620 &FsckOptions {
621 fix: false,
622 prune: Some("30d".to_owned()),
623 },
624 now,
625 );
626 assert_eq!(r.exit, crate::EXIT_OK);
627 assert_eq!(r.json["prune"]["flows_deleted"], 1);
628 assert_eq!(r.json["prune"]["flow_ids"][0], "01JZWY0A0000000000000001");
629 assert_eq!(r.json["prune"]["steps_deleted"], 5);
630 assert!(r.human.contains("cannot replay"));
631
632 let conn = Connection::open(project.join(".keel/journal.db")).unwrap();
634 let left: i64 = conn
635 .query_row("SELECT COUNT(*) FROM flows", [], |row| row.get(0))
636 .unwrap();
637 assert_eq!(left, 2);
638 }
639
640 #[test]
641 fn prune_age_too_young_removes_nothing() {
642 let (_d, project) = project_with_fixtures(&["completed-flow.sql"]);
643 let r = run(
644 &project,
645 &FsckOptions {
646 fix: false,
647 prune: Some("30d".to_owned()),
648 },
649 T0 + 86_400_000, );
651 assert_eq!(r.json["prune"]["flows_deleted"], 0);
652 assert_eq!(r.json["prune"]["flow_ids"].as_array().unwrap().len(), 0);
653 }
654
655 #[test]
656 fn bad_age_is_a_usage_error() {
657 let dir = tempfile::TempDir::new().unwrap();
658 for bad in ["30", "d", "", "-3d", "30w", "3.5h"] {
659 let r = run(
660 dir.path(),
661 &FsckOptions {
662 fix: false,
663 prune: Some(bad.to_owned()),
664 },
665 T0,
666 );
667 assert_eq!(r.exit, EXIT_USAGE, "{bad:?} must be rejected");
668 assert!(r.to_stderr);
669 assert_eq!(r.json["error"], "bad-age");
670 assert!(r.human.contains("next:"));
671 }
672 }
673
674 #[test]
675 fn age_parses_all_units() {
676 assert_eq!(parse_age_ms("90s").unwrap(), 90_000);
677 assert_eq!(parse_age_ms("45m").unwrap(), 2_700_000);
678 assert_eq!(parse_age_ms("12h").unwrap(), 43_200_000);
679 assert_eq!(parse_age_ms("30d").unwrap(), 2_592_000_000);
680 assert_eq!(parse_age_ms("0d").unwrap(), 0);
681 assert!(parse_age_ms("99999999999999999999d").is_err());
682 }
683
684 #[test]
685 fn missing_journal_is_a_clean_no_op() {
686 let dir = tempfile::TempDir::new().unwrap();
687 let r = run(dir.path(), &FsckOptions::default(), T0);
688 assert_eq!(r.exit, crate::EXIT_OK);
689 assert_eq!(r.json["journal_present"], false);
690 assert_eq!(r.json["ok"], true);
691 assert!(r.human.contains("nothing to check"));
692 }
693
694 #[test]
695 fn garbage_file_fails_integrity_with_what_why_next() {
696 let dir = tempfile::TempDir::new().unwrap();
697 let keel = dir.path().join(".keel");
698 std::fs::create_dir_all(&keel).unwrap();
699 std::fs::write(keel.join("journal.db"), b"not a database").unwrap();
700 let r = run(dir.path(), &FsckOptions::default(), T0);
701 assert_eq!(r.exit, EXIT_FAILURE);
702 assert_eq!(r.json["ok"], false);
703 assert_eq!(r.json["integrity"]["ok"], false);
704 assert!(r.human.contains("why:"));
705 assert!(r.human.contains("next:"));
706 }
707
708 #[test]
709 fn foreign_sqlite_file_is_not_a_keel_journal() {
710 let dir = tempfile::TempDir::new().unwrap();
711 let keel = dir.path().join(".keel");
712 std::fs::create_dir_all(&keel).unwrap();
713 let conn = Connection::open(keel.join("journal.db")).unwrap();
714 conn.execute_batch("CREATE TABLE t (x INTEGER);").unwrap();
715 drop(conn);
716 let r = run(dir.path(), &FsckOptions::default(), T0);
717 assert_eq!(r.exit, EXIT_FAILURE);
718 assert_eq!(r.json["schema_present"], false);
719 assert!(r.human.contains("not a keel journal"));
720 }
721
722 #[test]
723 fn check_pass_never_writes() {
724 let (_d, project) = project_with_fixtures(&["completed-flow.sql"]);
728 damage(&project);
729 let first = run(&project, &FsckOptions::default(), T0);
730 let second = run(&project, &FsckOptions::default(), T0);
731 assert_eq!(first.json, second.json, "a check pass must not mutate");
732 }
733}