1use chrono::{DateTime, Utc};
32use serde::{Deserialize, Serialize};
33use std::collections::BTreeMap;
34use std::path::Path;
35
36#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Serialize, Deserialize)]
39#[serde(rename_all = "lowercase")]
40pub enum Severity {
41 Broken,
43 Attention,
46}
47
48impl Severity {
49 pub fn as_str(&self) -> &'static str {
50 match self {
51 Severity::Broken => "broken",
52 Severity::Attention => "attention",
53 }
54 }
55}
56
57#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
59pub struct Remedy {
60 pub description: String,
63 pub argv: Vec<String>,
65 pub needs_terminal: bool,
69}
70
71#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
74pub struct Finding {
75 pub component: String,
76 pub severity: Severity,
77 pub summary: String,
78 pub detail: String,
79 #[serde(default, skip_serializing_if = "Option::is_none")]
80 pub remedy: Option<Remedy>,
81}
82
83impl Finding {
84 fn unreadable(component: &str, what: &str, why: impl std::fmt::Display) -> Finding {
87 Finding {
88 component: component.to_string(),
89 severity: Severity::Attention,
90 summary: format!("store unreadable: {what}"),
91 detail: why.to_string(),
92 remedy: None,
93 }
94 }
95}
96
97const STUCK_DRAFT_AFTER: chrono::Duration = chrono::Duration::hours(48);
100
101const STALE_REQUEST_AFTER: chrono::Duration = chrono::Duration::hours(72);
104
105pub fn examine(home: &Path, now: DateTime<Utc>) -> Vec<Finding> {
111 let mut findings = Vec::new();
112 findings.extend(check_mail(&home.join("mail")));
113 findings.extend(check_legacy_mail(home));
114 findings.extend(check_outbox(&home.join("outbox"), now));
115 findings.extend(check_frontdoor(&home.join("requests"), now));
116 findings.extend(check_triggers(&home.join("triggers"), now));
117 findings.extend(check_runs(&home.join("sessions")));
118 if let Some(parent) = home.parent() {
122 findings.extend(check_graph_nightly(&parent.join(".mecha-graph"), now));
123 }
124 sort(&mut findings);
125 findings
126}
127
128pub fn sort(findings: &mut [Finding]) {
131 findings.sort_by(|a, b| {
132 a.severity
133 .cmp(&b.severity)
134 .then_with(|| a.component.cmp(&b.component))
135 });
136}
137
138#[derive(Debug, Deserialize)]
145struct AuthMarker {
146 at: String,
147 message: String,
148}
149
150#[derive(Debug, Default, Deserialize)]
153struct MailAccounts {
154 #[serde(default, rename = "account")]
155 accounts: Vec<MailAccount>,
156}
157
158#[derive(Debug, Deserialize)]
159struct MailAccount {
160 name: String,
161 provider: String,
162 #[serde(default)]
166 grant_lifetime_days: Option<u32>,
167}
168
169fn check_mail(mail: &Path) -> Vec<Finding> {
173 let mut out = Vec::new();
174 if !mail.is_dir() {
175 return out;
176 }
177
178 let declared: Vec<MailAccount> = std::fs::read_to_string(mail.join("accounts.toml"))
181 .ok()
182 .and_then(|text| toml::from_str::<MailAccounts>(&text).ok())
183 .map(|file| file.accounts)
184 .unwrap_or_default();
185 let providers: BTreeMap<String, String> = declared
186 .iter()
187 .map(|a| (a.name.clone(), a.provider.clone()))
188 .collect();
189 let lifetimes: BTreeMap<String, u32> = declared
190 .iter()
191 .filter_map(|a| a.grant_lifetime_days.map(|d| (a.name.clone(), d)))
192 .collect();
193
194 let entries = match std::fs::read_dir(mail) {
195 Ok(entries) => entries,
196 Err(e) => {
197 out.push(Finding::unreadable(
198 "mail",
199 "the mail directory",
200 format!("{}: {e}", mail.display()),
201 ));
202 return out;
203 }
204 };
205
206 for entry in entries.flatten() {
207 let dir = entry.path();
208 if !dir.is_dir() {
209 continue;
210 }
211 let Some(account) = dir.file_name().and_then(|n| n.to_str()).map(String::from) else {
212 continue;
213 };
214 out.extend(check_triage_scope(&dir, &account, providers.get(&account)));
219 out.extend(check_grant_age(
220 &dir,
221 &account,
222 providers.get(&account),
223 lifetimes.get(&account).copied(),
224 ));
225
226 let marker_path = dir.join("auth_error.json");
227 if !marker_path.is_file() {
228 continue;
229 }
230 let text = match std::fs::read_to_string(&marker_path) {
231 Ok(text) => text,
232 Err(e) => {
233 out.push(Finding::unreadable(
234 "mail",
235 &format!("auth_error.json for `{account}`"),
236 format!("{}: {e}", marker_path.display()),
237 ));
238 continue;
239 }
240 };
241 match serde_json::from_str::<AuthMarker>(&text) {
242 Ok(marker) => {
243 let provider = providers.get(&account);
244 let mut argv = vec![
245 "mecha-mail".to_string(),
246 "auth".to_string(),
247 account.clone(),
248 ];
249 if let Some(provider) = provider {
250 argv.push("--provider".to_string());
251 argv.push(provider.clone());
252 }
253 out.push(Finding {
254 component: "mail".to_string(),
255 severity: Severity::Broken,
256 summary: format!("mail auth for `{account}` is dead"),
257 detail: format!(
262 "permanent refresh failure since {}: {}",
263 marker.at, marker.message
264 ),
265 remedy: Some(Remedy {
266 description: format!(
267 "re-authenticate the `{account}` account (opens an OAuth flow)"
268 ),
269 argv,
270 needs_terminal: true,
271 }),
272 });
273 }
274 Err(e) => out.push(Finding::unreadable(
275 "mail",
276 &format!("auth_error.json for `{account}` did not parse"),
277 format!("{}: {e}", marker_path.display()),
278 )),
279 }
280 }
281 out
282}
283
284#[derive(Debug, serde::Deserialize)]
291struct StoredGrant {
292 #[serde(default)]
293 granted_scopes: Option<String>,
294 #[serde(default)]
295 granted_at: Option<String>,
296}
297
298const GRANT_WARN_WITHIN_DAYS: i64 = 2;
305
306fn check_grant_age(
319 dir: &Path,
320 account: &str,
321 provider: Option<&String>,
322 lifetime_days: Option<u32>,
323) -> Vec<Finding> {
324 let Some(lifetime) = lifetime_days.filter(|d| *d > 0) else {
325 return Vec::new();
326 };
327 let Ok(text) = std::fs::read_to_string(dir.join("oauth.json")) else {
328 return Vec::new();
329 };
330 let Ok(grant) = serde_json::from_str::<StoredGrant>(&text) else {
331 return Vec::new(); };
333 let Some(granted_at) = grant.granted_at.as_deref() else {
337 return Vec::new();
338 };
339 let Ok(granted) = chrono::DateTime::parse_from_rfc3339(granted_at) else {
340 return Vec::new();
341 };
342 let expires = granted.with_timezone(&chrono::Utc) + chrono::Duration::days(lifetime as i64);
343 let hours_left = (expires - chrono::Utc::now()).num_hours();
348 let left = (hours_left as f64 / 24.0).ceil() as i64;
349 if left > GRANT_WARN_WITHIN_DAYS {
350 return Vec::new();
351 }
352 let when = if hours_left < 0 {
353 "has expired".to_string()
354 } else if hours_left < 24 {
355 "expires within a day".to_string()
356 } else {
357 format!("expires in {left} days")
358 };
359 let mut argv = vec![
360 "mecha-mail".to_string(),
361 "auth".to_string(),
362 account.to_string(),
363 ];
364 if let Some(p) = provider {
365 argv.push("--provider".to_string());
366 argv.push(p.clone());
367 }
368 vec![Finding {
369 component: "mail".to_string(),
370 severity: Severity::Attention,
371 summary: format!("`{account}` sign-in {when}"),
372 detail: format!(
373 "this grant lasts {lifetime} days from consent ({granted_at}) and refreshing does \
374 not extend it. Re-authenticate before it lapses — once it does, the failure looks \
375 like a revoked token and every scheduled run using this account stops."
376 ),
377 remedy: Some(Remedy {
378 description: format!("re-authenticate `{account}` now (opens an OAuth flow)"),
379 argv,
380 needs_terminal: true,
381 }),
382 }]
383}
384
385fn triage_scope_for(provider: &str) -> Option<&'static str> {
389 match provider {
390 "google" => Some("gmail.modify"),
391 "outlook" | "microsoft" => Some("Mail.ReadWrite"),
392 _ => None,
393 }
394}
395
396fn check_triage_scope(dir: &Path, account: &str, provider: Option<&String>) -> Vec<Finding> {
412 let Some(provider) = provider else {
413 return Vec::new();
414 };
415 let Some(needed) = triage_scope_for(provider) else {
416 return Vec::new();
417 };
418 let path = dir.join("oauth.json");
419 let Ok(text) = std::fs::read_to_string(&path) else {
420 return Vec::new();
423 };
424 let Ok(grant) = serde_json::from_str::<StoredGrant>(&text) else {
425 return vec![Finding::unreadable(
426 "mail",
427 &format!("oauth.json for `{account}` did not parse"),
428 format!("{}", path.display()),
429 )];
430 };
431 if grant
432 .granted_scopes
433 .as_deref()
434 .is_some_and(|g| g.contains(needed))
435 {
436 return Vec::new();
437 }
438 let admin_note = if provider == "outlook" || provider == "microsoft" {
439 " Microsoft blocks `Mail.ReadWrite` from end-user consent under its \
440 recommended policy, so on a managed tenant an administrator has to \
441 grant it to the app registration before this can succeed."
442 } else {
443 ""
444 };
445 vec![Finding {
446 component: "mail".to_string(),
447 severity: Severity::Attention,
448 summary: format!("`{account}` cannot archive, spam or mark mail read"),
449 detail: format!(
450 "the stored grant does not include `{needed}`, so mail_triage will fail on this \
451 account. Reading, sending and calendar work are unaffected.{admin_note}"
452 ),
453 remedy: Some(Remedy {
454 description: format!(
455 "re-authenticate `{account}` to add the triage scope (opens an OAuth flow)"
456 ),
457 argv: vec![
458 "mecha-mail".to_string(),
459 "auth".to_string(),
460 account.to_string(),
461 "--provider".to_string(),
462 provider.clone(),
463 ],
464 needs_terminal: true,
465 }),
466 }]
467}
468
469#[cfg(test)]
470mod grant_age_tests {
471 use super::*;
472
473 fn store(dir: &Path, granted_at: Option<&str>) {
474 std::fs::create_dir_all(dir).unwrap();
475 let stamp = granted_at
476 .map(|g| format!(r#","granted_at":"{g}""#))
477 .unwrap_or_default();
478 std::fs::write(
479 dir.join("oauth.json"),
480 format!(r#"{{"client_id":"i","access_token":"a","refresh_token":"r","expires_at":1{stamp}}}"#),
481 )
482 .unwrap();
483 }
484
485 fn days_ago(n: i64) -> String {
486 (chrono::Utc::now() - chrono::Duration::days(n)).to_rfc3339()
487 }
488
489 #[test]
491 fn a_grant_nearing_its_declared_lifetime_is_reported_early() {
492 let tmp = std::env::temp_dir().join(format!("mecha-grant-{}", std::process::id()));
493 let g = "google".to_string();
494
495 store(&tmp, Some(&days_ago(1)));
497 assert!(check_grant_age(&tmp, "personal", Some(&g), Some(7)).is_empty());
498
499 store(&tmp, Some(&days_ago(5)));
501 let f = check_grant_age(&tmp, "personal", Some(&g), Some(7));
502 assert_eq!(f.len(), 1, "should warn with 2 days left");
503 assert!(
504 f[0].summary.contains("expires in 2 days"),
505 "{}",
506 f[0].summary
507 );
508 assert!(f[0].remedy.as_ref().unwrap().needs_terminal);
509
510 store(&tmp, Some(&days_ago(7)));
512 let f = check_grant_age(&tmp, "personal", Some(&g), Some(7));
513 assert!(f[0].summary.contains("within a day"), "{}", f[0].summary);
514
515 store(&tmp, Some(&days_ago(9)));
517 let f = check_grant_age(&tmp, "personal", Some(&g), Some(7));
518 assert!(f[0].summary.contains("has expired"), "{}", f[0].summary);
519
520 assert!(check_grant_age(&tmp, "personal", Some(&g), None).is_empty());
522
523 store(&tmp, None);
525 assert!(check_grant_age(&tmp, "personal", Some(&g), Some(7)).is_empty());
526
527 std::fs::remove_dir_all(&tmp).ok();
528 }
529}
530
531fn check_legacy_mail(home: &Path) -> Vec<Finding> {
538 let mut out = Vec::new();
539 for provider in ["google", "outlook"] {
540 let marker_path = home.join(provider).join("auth_error.json");
541 if !marker_path.is_file() {
542 continue;
543 }
544 let text = match std::fs::read_to_string(&marker_path) {
545 Ok(text) => text,
546 Err(e) => {
547 out.push(Finding::unreadable(
548 "mail",
549 &format!("auth_error.json for the legacy {provider} store"),
550 format!("{}: {e}", marker_path.display()),
551 ));
552 continue;
553 }
554 };
555 match serde_json::from_str::<AuthMarker>(&text) {
556 Ok(marker) => out.push(Finding {
557 component: "mail".to_string(),
558 severity: Severity::Broken,
559 summary: format!("legacy {provider} mail auth is dead"),
560 detail: format!(
564 "permanent refresh failure since {}: {}",
565 marker.at, marker.message
566 ),
567 remedy: Some(Remedy {
568 description: format!(
569 "bring the legacy {provider} login into the unified registry — \
570 and re-authenticate it per the detail, which no import fixes"
571 ),
572 argv: vec![
573 "mecha-mail".to_string(),
574 "import".to_string(),
575 provider.to_string(),
576 "--provider".to_string(),
577 provider.to_string(),
578 ],
579 needs_terminal: false,
580 }),
581 }),
582 Err(e) => out.push(Finding::unreadable(
583 "mail",
584 &format!("auth_error.json for the legacy {provider} store did not parse"),
585 format!("{}: {e}", marker_path.display()),
586 )),
587 }
588 }
589 out
590}
591
592fn check_outbox(root: &Path, now: DateTime<Utc>) -> Vec<Finding> {
598 let mut out = Vec::new();
599 if !root.is_dir() {
600 return out;
601 }
602 let entries = match std::fs::read_dir(root) {
603 Ok(entries) => entries,
604 Err(e) => {
605 out.push(Finding::unreadable(
606 "outbox",
607 "the outbox directory",
608 format!("{}: {e}", root.display()),
609 ));
610 return out;
611 }
612 };
613
614 let review = Remedy {
615 description: "open the outbox review surface — doctor never releases a draft".to_string(),
616 argv: vec!["mecha".into(), "outbox".into(), "review".into()],
617 needs_terminal: true,
618 };
619
620 let mut stale: Vec<String> = Vec::new();
621 for entry in entries.flatten() {
622 let path = entry.path();
623 if path.extension().and_then(|e| e.to_str()) != Some("json") {
624 continue;
625 }
626 let item: crate::outbox::OutboxItem =
627 match std::fs::read_to_string(&path).map(|t| serde_json::from_str(&t)) {
628 Ok(Ok(item)) => item,
629 Ok(Err(e)) => {
630 out.push(Finding::unreadable(
631 "outbox",
632 &format!(
633 "item {} did not parse",
634 path.file_name().unwrap_or_default().to_string_lossy()
635 ),
636 format!("{}: {e}", path.display()),
637 ));
638 continue;
639 }
640 Err(e) => {
641 out.push(Finding::unreadable(
642 "outbox",
643 &format!(
644 "item {} could not be read",
645 path.file_name().unwrap_or_default().to_string_lossy()
646 ),
647 format!("{}: {e}", path.display()),
648 ));
649 continue;
650 }
651 };
652 if item.status != "pending" {
653 continue;
654 }
655 if let Some(error) = &item.error {
656 out.push(Finding {
657 component: "outbox".to_string(),
658 severity: Severity::Broken,
659 summary: format!("release failed: {error}"),
660 detail: format!(
661 "{} · {} — still pending; the draft is good, the delivery was not",
662 item.id, item.summary
663 ),
664 remedy: Some(review.clone()),
665 });
666 } else if age_of(&item.created_at, now).is_some_and(|age| age > STUCK_DRAFT_AFTER) {
667 stale.push(format!(
668 "{} · {} — staged {}",
669 item.id,
670 item.summary,
671 render_age(now, &item.created_at)
672 ));
673 }
674 }
675
676 if !stale.is_empty() {
677 stale.sort();
679 out.push(Finding {
680 component: "outbox".to_string(),
681 severity: Severity::Attention,
682 summary: format!(
683 "{} draft{} pending for more than 48h",
684 stale.len(),
685 if stale.len() == 1 { "" } else { "s" }
686 ),
687 detail: stale.join("\n"),
688 remedy: Some(review),
689 });
690 }
691 out
692}
693
694const WAITING_ON_ME: [&str; 3] = [
702 crate::frontdoor::EXTRACTED,
703 crate::frontdoor::AWAITING_ME,
704 crate::frontdoor::TRIAGED,
705];
706
707fn check_frontdoor(root: &Path, now: DateTime<Utc>) -> Vec<Finding> {
710 let mut out = Vec::new();
711 if !root.is_dir() {
712 return out;
713 }
714 let entries = match std::fs::read_dir(root) {
715 Ok(entries) => entries,
716 Err(e) => {
717 out.push(Finding::unreadable(
718 "frontdoor",
719 "the request store",
720 format!("{}: {e}", root.display()),
721 ));
722 return out;
723 }
724 };
725
726 let list = Remedy {
727 description: "list the frontdoor queue".to_string(),
728 argv: vec!["mecha".into(), "frontdoor".into(), "list".into()],
729 needs_terminal: false,
730 };
731
732 let mut stale: Vec<(i64, String)> = Vec::new();
733 for entry in entries.flatten() {
734 let path = entry.path();
735 if path.extension().and_then(|e| e.to_str()) != Some("json") {
736 continue;
737 }
738 let Ok(Ok(record)) = std::fs::read_to_string(&path)
739 .map(|t| serde_json::from_str::<crate::frontdoor::Record>(&t))
740 else {
741 out.push(Finding::unreadable(
744 "frontdoor",
745 &format!(
746 "request {} did not parse",
747 path.file_name().unwrap_or_default().to_string_lossy()
748 ),
749 path.display().to_string(),
750 ));
751 continue;
752 };
753 if record.state == crate::frontdoor::EXTRACTION_FAILED {
754 out.push(Finding {
755 component: "frontdoor".to_string(),
756 severity: Severity::Broken,
757 summary: format!(
758 "request {} failed extraction and waits for a human",
759 record.seq
760 ),
761 detail: format!(
762 "{} ({}) — {}",
763 record.seq,
764 record.type_id,
765 record
766 .extraction_error
767 .as_deref()
768 .unwrap_or("no error recorded")
769 ),
770 remedy: Some(list.clone()),
771 });
772 } else if WAITING_ON_ME.contains(&record.state.as_str())
773 && request_age(&record, now).is_some_and(|age| age > STALE_REQUEST_AFTER)
774 {
775 stale.push((
776 record.seq,
777 format!(
778 "{} ({}) — {}, received {}",
779 record.seq,
780 record.type_id,
781 record.state,
782 render_age(now, &record.created_at)
783 ),
784 ));
785 }
786 }
787
788 if !stale.is_empty() {
789 stale.sort_by_key(|(seq, _)| *seq);
791 out.push(Finding {
792 component: "frontdoor".to_string(),
793 severity: Severity::Attention,
794 summary: format!(
795 "{} request{} waiting on you for more than 72h",
796 stale.len(),
797 if stale.len() == 1 { "" } else { "s" }
798 ),
799 detail: stale
800 .into_iter()
801 .map(|(_, line)| line)
802 .collect::<Vec<_>>()
803 .join("\n"),
804 remedy: Some(list),
805 });
806 }
807 out
808}
809
810fn request_age(record: &crate::frontdoor::Record, now: DateTime<Utc>) -> Option<chrono::Duration> {
815 age_of(&record.drained_at, now).or_else(|| age_of(&record.created_at, now))
816}
817
818const HEALTH_WINDOW: usize = 5;
825
826const HEALTH_MIN_CALLS: u32 = 10;
831
832const HEALTH_ERROR_RATE: f64 = 1.0 / 3.0;
839
840fn check_triggers(root: &Path, now: DateTime<Utc>) -> Vec<Finding> {
843 let mut out = Vec::new();
844 if !root.is_dir() {
845 return out;
846 }
847 let entries = match std::fs::read_dir(root) {
848 Ok(entries) => entries,
849 Err(e) => {
850 out.push(Finding::unreadable(
851 "triggers",
852 "the trigger store",
853 format!("{}: {e}", root.display()),
854 ));
855 return out;
856 }
857 };
858
859 let mut triggers: Vec<crate::trigger::Trigger> = Vec::new();
860 for entry in entries.flatten() {
861 let path = entry.path();
862 if path.extension().and_then(|e| e.to_str()) != Some("toml") {
863 continue;
864 }
865 let name = path
866 .file_stem()
867 .and_then(|s| s.to_str())
868 .unwrap_or_default()
869 .to_string();
870 match std::fs::read_to_string(&path).map(|t| toml::from_str::<crate::trigger::Trigger>(&t))
871 {
872 Ok(Ok(mut trigger)) => {
873 trigger.name = name;
874 triggers.push(trigger);
875 }
876 _ => out.push(Finding::unreadable(
877 "triggers",
878 &format!("trigger file `{name}.toml` did not parse"),
879 path.display().to_string(),
880 )),
881 }
882 }
883
884 let mut recent: BTreeMap<String, Vec<crate::trigger::RunRecord>> = BTreeMap::new();
888 let mut last_slot: BTreeMap<String, DateTime<Utc>> = BTreeMap::new();
889 let ledger = root.join("runs.jsonl");
890 if ledger.is_file() {
891 match std::fs::read_to_string(&ledger) {
892 Ok(text) => {
893 for line in text.lines().filter(|l| !l.trim().is_empty()) {
894 let Ok(row) = serde_json::from_str::<crate::trigger::RunRecord>(line) else {
897 continue;
898 };
899 if let Some(slot) = row.slot {
900 let newest = last_slot.entry(row.trigger.clone()).or_insert(slot);
901 if slot > *newest {
902 *newest = slot;
903 }
904 }
905 if matches!(
910 row.status,
911 crate::trigger::RunStatus::Ok | crate::trigger::RunStatus::Error
912 ) {
913 let window = recent.entry(row.trigger.clone()).or_default();
914 window.push(row);
915 if window.len() > HEALTH_WINDOW {
916 window.remove(0);
917 }
918 }
919 }
920 }
921 Err(e) => out.push(Finding::unreadable(
922 "triggers",
923 "the run ledger",
924 format!("{}: {e}", ledger.display()),
925 )),
926 }
927 }
928
929 for trigger in &triggers {
930 if !trigger.enabled {
931 continue;
932 }
933
934 let window = recent.get(&trigger.name);
937 if let Some(row) = window.and_then(|w| w.last()) {
938 if row.status == crate::trigger::RunStatus::Error {
939 out.push(Finding {
940 component: "triggers".to_string(),
941 severity: Severity::Attention,
942 summary: format!("trigger `{}`'s most recent run failed", trigger.name),
943 detail: format!(
944 "started {}: {}",
945 row.started_at.to_rfc3339(),
946 row.error.as_deref().unwrap_or("no error recorded")
947 ),
948 remedy: Some(Remedy {
949 description: format!(
950 "run `{}` by hand — a manual run is evidence, not a fire; it never advances the schedule",
951 trigger.name
952 ),
953 argv: vec![
954 "mecha".into(),
955 "trigger".into(),
956 "run".into(),
957 trigger.name.clone(),
958 ],
959 needs_terminal: false,
960 }),
961 });
962 }
963 }
964
965 let (calls, errors) = window
971 .map(|w| {
972 w.iter().fold((0u32, 0u32), |(c, e), r| {
973 (c + r.tool_calls, e + r.tool_errors)
974 })
975 })
976 .unwrap_or((0, 0));
977 if calls >= HEALTH_MIN_CALLS && f64::from(errors) / f64::from(calls) >= HEALTH_ERROR_RATE {
978 let runs = window.map(Vec::len).unwrap_or(0);
979 out.push(Finding {
980 component: "triggers".to_string(),
981 severity: Severity::Attention,
982 summary: format!(
983 "trigger `{}` failed {errors} of {calls} tool calls",
984 trigger.name
985 ),
986 detail: format!(
987 "across its last {runs} run(s){}. A run's answer arrives either way, so this is invisible in the ledger's status — and per-step reliability is what decides how long a task the run can finish, so a third of the calls failing is not a third of the work lost.",
988 if window.is_some_and(|w| w.last().is_some_and(|r| r.ended_on_failed_call)) {
989 ", and the most recent run answered with its last call failed"
990 } else {
991 ""
992 }
993 ),
994 remedy: Some(Remedy {
997 description: format!("read `{}`'s recent runs", trigger.name),
998 argv: vec![
999 "mecha".into(),
1000 "trigger".into(),
1001 "show".into(),
1002 trigger.name.clone(),
1003 ],
1004 needs_terminal: false,
1005 }),
1006 });
1007 }
1008
1009 if let Some(window) = window {
1023 let newest = window.last();
1024 let before: u32 = window[..window.len().saturating_sub(1)]
1025 .iter()
1026 .map(|r| r.tool_calls)
1027 .sum();
1028 let stopped = newest
1031 .is_some_and(|r| r.tool_calls == 0 && r.status == crate::trigger::RunStatus::Ok)
1032 && before >= HEALTH_MIN_CALLS;
1033 if stopped {
1034 out.push(Finding {
1035 component: "triggers".to_string(),
1036 severity: Severity::Attention,
1037 summary: format!(
1038 "trigger `{}`'s most recent run did no work",
1039 trigger.name
1040 ),
1041 detail: format!(
1042 "it succeeded having made no tool calls, where its previous {} run(s) made {before}. A run that does nothing and reports success is indistinguishable from a healthy one in every other signal — the status is `ok`, the schedule advanced, and the answer arrived.",
1043 window.len() - 1
1044 ),
1045 remedy: Some(Remedy {
1046 description: format!("read `{}`'s recent runs", trigger.name),
1047 argv: vec![
1048 "mecha".into(),
1049 "trigger".into(),
1050 "show".into(),
1051 trigger.name.clone(),
1052 ],
1053 needs_terminal: false,
1054 }),
1055 });
1056 }
1057 }
1058
1059 if trigger.catch_up != crate::trigger::CatchUp::Always {
1065 continue;
1066 }
1067 let Some(anchor) = last_slot.get(&trigger.name).copied().or(trigger.created_at) else {
1068 continue;
1071 };
1072 let tz = trigger.tz(None);
1073 let step = chrono::Duration::seconds(1);
1074 let missed_more_than_two = trigger
1075 .schedule
1076 .prev_at_or_before(now, tz)
1077 .and_then(|s0| trigger.schedule.prev_at_or_before(s0 - step, tz))
1078 .and_then(|s1| trigger.schedule.prev_at_or_before(s1 - step, tz))
1079 .is_some_and(|s2| s2 > anchor);
1080 if missed_more_than_two {
1081 out.push(Finding {
1082 component: "triggers".to_string(),
1083 severity: Severity::Attention,
1084 summary: format!("trigger `{}` has missed more than two slots", trigger.name),
1085 detail: format!(
1086 "last accounted slot {}; with catch_up=always a healthy scheduler fires \
1087 the most recent slot every tick, so the daemon or its timer may be down \
1088 (systemctl --user status mecha-triggers)",
1089 anchor.to_rfc3339()
1090 ),
1091 remedy: None,
1094 });
1095 }
1096 }
1097 out
1098}
1099
1100const RUNS_WINDOW: usize = 200;
1109
1110const RUNS_MIN: usize = 20;
1116
1117const ENDED_ON_FAILURE_RATE: f64 = 0.20;
1123
1124const TOOL_ERROR_RATE: f64 = 0.25;
1126
1127const RUNS_MIN_CALLS: u64 = 20;
1130
1131const CUT_SHORT_RATE: f64 = 0.25;
1135
1136fn cut_short(stats: &crate::session::RunStats) -> bool {
1139 stats.stop_cause.is_some_and(|c| c.cut_short())
1140}
1141
1142fn check_runs(sessions: &Path) -> Vec<Finding> {
1151 use crate::runlog::{Corpus, Scan};
1152
1153 let mut out = Vec::new();
1154 if !sessions.is_dir() {
1155 return out;
1156 }
1157 let corpus = match Corpus::scan(
1158 sessions,
1159 &Scan {
1160 max_sessions: Some(RUNS_WINDOW),
1161 since: None,
1162 },
1163 ) {
1164 Ok(c) => c,
1165 Err(e) => {
1166 out.push(Finding::unreadable(
1167 "runs",
1168 "the session store",
1169 format!("{}: {e}", sessions.display()),
1170 ));
1171 return out;
1172 }
1173 };
1174
1175 let remedy = |what: &str| {
1176 Some(Remedy {
1177 description: format!("read the run-quality summary ({what})"),
1178 argv: vec![
1179 "mecha".into(),
1180 "sessions".into(),
1181 "health".into(),
1182 "--days".into(),
1183 "30".into(),
1184 ],
1185 needs_terminal: false,
1186 })
1187 };
1188
1189 for (model, runs) in corpus.by_model() {
1190 if runs.len() < RUNS_MIN {
1191 continue;
1192 }
1193 let n = runs.len();
1194
1195 if let Some(rate) = runs.rate_of(|r| r.stats.ended_on_failed_call) {
1196 if rate >= ENDED_ON_FAILURE_RATE {
1197 out.push(Finding {
1198 component: "runs".to_string(),
1199 severity: Severity::Attention,
1200 summary: format!(
1201 "{:.0}% of `{model}` runs finished on a failed tool call",
1202 rate * 100.0
1203 ),
1204 detail: format!(
1205 "{} of {n} recent run(s). The model stopped of its own accord with its last call failed, and the answer it wrote may report success over it — which nothing in the text or the stop reason can show.",
1206 runs.ended_on_failed_call()
1207 ),
1208 remedy: remedy("which runs, and what failed"),
1209 });
1210 }
1211 }
1212
1213 if let Some(rate) = runs.tool_error_rate() {
1214 if rate >= TOOL_ERROR_RATE && runs.tool_calls() >= RUNS_MIN_CALLS {
1219 out.push(Finding {
1220 component: "runs".to_string(),
1221 severity: Severity::Attention,
1222 summary: format!(
1223 "`{model}` runs fail {:.0}% of their tool calls",
1224 rate * 100.0
1225 ),
1226 detail: format!(
1227 "{} of {} call(s) across {n} run(s) were refused by the environment. Errors are how a run learns where it is, so some are healthy — a quarter of them says something moved: a renamed path, a revoked grant, a tool whose schema the model keeps mis-filling.",
1228 runs.tool_errors(),
1229 runs.tool_calls()
1230 ),
1231 remedy: remedy("which tool, and how it failed"),
1232 });
1233 }
1234 }
1235
1236 if let Some(rate) = runs.rate_of(|r| cut_short(&r.stats)) {
1237 if rate >= CUT_SHORT_RATE {
1238 let cut = runs.rows.iter().filter(|r| cut_short(&r.stats)).count();
1239 out.push(Finding {
1240 component: "runs".to_string(),
1241 severity: Severity::Attention,
1242 summary: format!(
1243 "the harness cut {:.0}% of `{model}` runs short",
1244 rate * 100.0
1245 ),
1246 detail: format!(
1247 "{cut} of {n} recent run(s) hit a turn, token or cost ceiling, or tripped the loop guard. A budget that stops a quarter of runs is measuring the budget rather than the work — the answers are truncated and say so only in `stop_cause`. Cancellations are not counted here.",
1248 ),
1249 remedy: remedy("which ceiling, and how often"),
1250 });
1251 }
1252 }
1253 }
1254 out
1255}
1256
1257const GRAPH_NIGHTLIES: &[(&str, &str)] = &[
1267 ("nightly-", "the graph's own sweep (ingest, extract, decay)"),
1268 ("mecha-nightly-", "the mecha half (vet, precheck, gossip)"),
1269];
1270
1271fn check_graph_nightly(store: &Path, now: DateTime<Utc>) -> Vec<Finding> {
1279 let mut out = Vec::new();
1280 let logs = store.join("logs");
1281 if !logs.is_dir() {
1282 return out;
1283 }
1284 let names: Vec<String> = match std::fs::read_dir(&logs) {
1285 Ok(entries) => entries
1286 .flatten()
1287 .filter_map(|e| e.file_name().to_str().map(String::from))
1288 .collect(),
1289 Err(e) => {
1290 out.push(Finding::unreadable(
1291 "graph",
1292 "the graph nightly logs",
1293 format!("{}: {e}", logs.display()),
1294 ));
1295 return out;
1296 }
1297 };
1298
1299 for (prefix, what) in GRAPH_NIGHTLIES {
1300 let newest = names
1301 .iter()
1302 .filter_map(|n| {
1303 n.strip_prefix(prefix)?
1304 .strip_suffix(".log")
1305 .and_then(|d| chrono::NaiveDate::parse_from_str(d, "%Y%m%d").ok())
1306 })
1307 .max();
1308 let Some(newest) = newest else { continue };
1311 let days_quiet = (now.date_naive() - newest).num_days();
1312 if days_quiet > 1 {
1313 out.push(Finding {
1314 component: "graph".to_string(),
1315 severity: Severity::Attention,
1316 summary: format!(
1317 "the graph nightly ({}) has not run for {days_quiet} days",
1318 prefix.trim_end_matches('-'),
1319 ),
1320 detail: format!(
1321 "{what} last wrote {}{}.log under {}; it logs every \
1322 run including deferred ones, so a missing day means the \
1323 script never started — cron reports that nowhere",
1324 prefix,
1325 newest.format("%Y%m%d"),
1326 logs.display(),
1327 ),
1328 remedy: Some(Remedy {
1329 description: "list the cron entries that fire the graph nightlies, \
1330 then run the silent one by hand and read its error"
1331 .to_string(),
1332 argv: vec!["crontab".into(), "-l".into()],
1333 needs_terminal: false,
1334 }),
1335 });
1336 }
1337 }
1338 out
1339}
1340
1341fn age_of(stamp: &str, now: DateTime<Utc>) -> Option<chrono::Duration> {
1346 DateTime::parse_from_rfc3339(stamp)
1347 .ok()
1348 .map(|at| now - at.with_timezone(&Utc))
1349}
1350
1351fn render_age(now: DateTime<Utc>, stamp: &str) -> String {
1353 match age_of(stamp, now) {
1354 Some(age) if age >= chrono::Duration::days(2) => format!("{}d ago", age.num_days()),
1355 Some(age) if age >= chrono::Duration::hours(1) => format!("{}h ago", age.num_hours()),
1356 Some(age) => format!("{}m ago", age.num_minutes().max(0)),
1357 None => stamp.to_string(),
1358 }
1359}
1360
1361#[cfg(test)]
1362mod tests {
1363 use super::*;
1364 use crate::agent::Taint;
1365 use crate::outbox::{OutboxItem, OutboxKind};
1366 use serde_json::json;
1367 use std::path::PathBuf;
1368
1369 fn utc(s: &str) -> DateTime<Utc> {
1370 DateTime::parse_from_rfc3339(s).unwrap().with_timezone(&Utc)
1371 }
1372
1373 const NOW: &str = "2026-08-14T12:00:00Z";
1374
1375 fn home(name: &str) -> PathBuf {
1377 let dir = std::env::temp_dir().join(format!(
1378 "mecha-doctor-test-{name}-{}-{:?}",
1379 std::process::id(),
1380 std::thread::current().id()
1381 ));
1382 let _ = std::fs::remove_dir_all(&dir);
1383 std::fs::create_dir_all(&dir).unwrap();
1384 dir
1385 }
1386
1387 fn write_marker(home: &Path, account: &str, body: &str) {
1388 let dir = home.join("mail").join(account);
1389 std::fs::create_dir_all(&dir).unwrap();
1390 std::fs::write(dir.join("auth_error.json"), body).unwrap();
1391 }
1392
1393 fn valid_marker() -> String {
1394 json!({
1395 "at": "2026-08-11T09:00:00Z",
1396 "message": "the refresh token was revoked — run `mecha-mail auth personal --provider google` to sign in again",
1397 })
1398 .to_string()
1399 }
1400
1401 fn pending_item(home: &Path, id: &str, created_at: &str, error: Option<&str>) {
1402 let item = OutboxItem {
1403 id: id.to_string(),
1404 status: "pending".into(),
1405 tool: "mail__send".into(),
1406 kind: OutboxKind::Message,
1407 args_before: json!({"to": "a@x.org"}),
1408 args: json!({"to": "a@x.org"}),
1409 summary: "mail__send to a@x.org".into(),
1410 session_id: None,
1411 workspace: None,
1412 taint: Taint::default(),
1413 created_at: created_at.to_string(),
1414 resolved_at: None,
1415 reason: None,
1416 error: error.map(String::from),
1417 };
1418 let dir = home.join("outbox");
1419 std::fs::create_dir_all(&dir).unwrap();
1420 std::fs::write(
1421 dir.join(format!("{id}.json")),
1422 serde_json::to_string_pretty(&item).unwrap(),
1423 )
1424 .unwrap();
1425 }
1426
1427 fn request(home: &Path, seq: i64, state: &str, drained_at: &str) {
1428 let dir = home.join("requests");
1429 std::fs::create_dir_all(&dir).unwrap();
1430 let record = json!({
1431 "seq": seq,
1432 "type_id": "meeting",
1433 "state": state,
1434 "created_at": drained_at,
1435 "drained_at": drained_at,
1436 "valid": true,
1437 "values": {},
1438 "free_text": [],
1439 });
1440 std::fs::write(
1441 dir.join(format!("{seq:010}-meeting.json")),
1442 record.to_string(),
1443 )
1444 .unwrap();
1445 }
1446
1447 fn trigger_file(home: &Path, name: &str, extra: &str) {
1448 let dir = home.join("triggers");
1449 std::fs::create_dir_all(&dir).unwrap();
1450 std::fs::write(
1451 dir.join(format!("{name}.toml")),
1452 format!(
1453 "schedule = \"0 7 * * *\"\nprompt = \"brief me\"\ntimezone = \"UTC\"\n\
1454 created_at = \"2026-08-01T00:00:00Z\"\n{extra}"
1455 ),
1456 )
1457 .unwrap();
1458 }
1459
1460 fn ledger_row(home: &Path, row: &serde_json::Value) {
1461 use std::io::Write;
1462 let dir = home.join("triggers");
1463 std::fs::create_dir_all(&dir).unwrap();
1464 let mut file = std::fs::OpenOptions::new()
1465 .create(true)
1466 .append(true)
1467 .open(dir.join("runs.jsonl"))
1468 .unwrap();
1469 writeln!(file, "{row}").unwrap();
1470 }
1471
1472 fn of<'a>(findings: &'a [Finding], component: &str) -> Vec<&'a Finding> {
1473 findings
1474 .iter()
1475 .filter(|f| f.component == component)
1476 .collect()
1477 }
1478
1479 #[test]
1480 fn a_dead_auth_marker_is_found_and_an_absent_one_is_not() {
1481 let home = home("dead-auth");
1482 write_marker(&home, "personal", &valid_marker());
1483 std::fs::create_dir_all(home.join("mail").join("dartmouth")).unwrap();
1485 std::fs::write(
1486 home.join("mail").join("accounts.toml"),
1487 "[[account]]\nname = \"personal\"\nprovider = \"google\"\n\
1488 [[account]]\nname = \"dartmouth\"\nprovider = \"outlook\"\n",
1489 )
1490 .unwrap();
1491
1492 let findings = examine(&home, utc(NOW));
1493 let mail = of(&findings, "mail");
1494 assert_eq!(mail.len(), 1, "{findings:#?}");
1495 assert_eq!(mail[0].severity, Severity::Broken);
1496 assert!(mail[0].summary.contains("personal"), "{}", mail[0].summary);
1497 let remedy = mail[0].remedy.as_ref().expect("a dead login has a way out");
1498 assert_eq!(
1499 remedy.argv,
1500 vec!["mecha-mail", "auth", "personal", "--provider", "google"]
1501 );
1502 assert!(
1503 remedy.needs_terminal,
1504 "an OAuth flow needs the real terminal"
1505 );
1506
1507 let _ = std::fs::remove_dir_all(&home);
1508 }
1509
1510 #[test]
1511 fn a_provider_the_registry_cannot_name_is_omitted_from_the_remedy_not_guessed() {
1512 let home = home("no-registry");
1513 write_marker(&home, "personal", &valid_marker());
1515
1516 let findings = examine(&home, utc(NOW));
1517 let mail = of(&findings, "mail");
1518 assert_eq!(mail.len(), 1);
1519 let remedy = mail[0].remedy.as_ref().unwrap();
1520 assert_eq!(remedy.argv, vec!["mecha-mail", "auth", "personal"]);
1521 assert!(
1524 mail[0].detail.contains("--provider google"),
1525 "{}",
1526 mail[0].detail
1527 );
1528
1529 let _ = std::fs::remove_dir_all(&home);
1530 }
1531
1532 #[test]
1537 fn a_marker_in_a_legacy_per_provider_store_is_found_and_proposes_import() {
1538 let home = home("legacy-auth");
1539 let dir = home.join("google");
1540 std::fs::create_dir_all(&dir).unwrap();
1541 std::fs::write(
1542 dir.join("auth_error.json"),
1543 json!({
1544 "at": "2026-08-11T09:00:00Z",
1545 "message": "account `google`: refresh token expired or revoked — run `mecha-mail auth google --provider google` (invalid_grant)",
1546 })
1547 .to_string(),
1548 )
1549 .unwrap();
1550
1551 let findings = examine(&home, utc(NOW));
1552 let mail = of(&findings, "mail");
1553 assert_eq!(mail.len(), 1, "{findings:#?}");
1554 assert_eq!(mail[0].severity, Severity::Broken);
1555 assert!(
1556 mail[0].summary.contains("legacy google"),
1557 "{}",
1558 mail[0].summary
1559 );
1560 assert!(
1563 mail[0]
1564 .detail
1565 .contains("run `mecha-mail auth google --provider google`"),
1566 "{}",
1567 mail[0].detail
1568 );
1569 let remedy = mail[0].remedy.as_ref().expect("a way out");
1570 assert_eq!(
1571 remedy.argv,
1572 vec!["mecha-mail", "import", "google", "--provider", "google"]
1573 );
1574
1575 let _ = std::fs::remove_dir_all(&home);
1576 }
1577
1578 #[test]
1579 fn an_unparseable_marker_is_a_store_unreadable_finding_not_a_crash() {
1580 let home = home("bad-marker");
1581 write_marker(&home, "personal", "{ this is not json");
1582
1583 let findings = examine(&home, utc(NOW));
1584 let mail = of(&findings, "mail");
1585 assert_eq!(mail.len(), 1, "{findings:#?}");
1586 assert!(
1587 mail[0].summary.starts_with("store unreadable:"),
1588 "{}",
1589 mail[0].summary
1590 );
1591 assert!(mail[0].summary.contains("personal"), "{}", mail[0].summary);
1592
1593 let _ = std::fs::remove_dir_all(&home);
1594 }
1595
1596 #[test]
1597 fn a_pending_item_with_an_error_is_broken_and_a_resolved_one_is_not() {
1598 let home = home("outbox-error");
1599 pending_item(
1600 &home,
1601 "20260814-000001-aaa",
1602 NOW,
1603 Some("server unreachable"),
1604 );
1605 let mut sent = json!({
1607 "id": "20260810-000001-bbb",
1608 "status": "sent",
1609 "tool": "mail__send",
1610 "args_before": {},
1611 "args": {},
1612 "summary": "mail__send",
1613 "created_at": "2026-08-01T00:00:00Z",
1614 });
1615 sent["error"] = json!(null);
1616 std::fs::write(
1617 home.join("outbox").join("20260810-000001-bbb.json"),
1618 sent.to_string(),
1619 )
1620 .unwrap();
1621
1622 let findings = examine(&home, utc(NOW));
1623 let outbox = of(&findings, "outbox");
1624 assert_eq!(outbox.len(), 1, "{findings:#?}");
1625 assert_eq!(outbox[0].severity, Severity::Broken);
1626 assert!(
1627 outbox[0]
1628 .summary
1629 .contains("release failed: server unreachable"),
1630 "{}",
1631 outbox[0].summary
1632 );
1633 let remedy = outbox[0].remedy.as_ref().unwrap();
1634 assert_eq!(remedy.argv, vec!["mecha", "outbox", "review"]);
1635
1636 let _ = std::fs::remove_dir_all(&home);
1637 }
1638
1639 #[test]
1640 fn a_pending_draft_is_stale_at_49_hours_and_not_at_47() {
1641 let home = home("outbox-stale");
1642 pending_item(&home, "20260812-110000-old", "2026-08-12T11:00:00Z", None);
1644 let findings = examine(&home, utc(NOW));
1645 let outbox = of(&findings, "outbox");
1646 assert_eq!(outbox.len(), 1, "{findings:#?}");
1647 assert_eq!(outbox[0].severity, Severity::Attention);
1648 assert!(outbox[0].summary.contains("pending for more than 48h"));
1649 assert_eq!(
1650 outbox[0].remedy.as_ref().unwrap().argv,
1651 vec!["mecha", "outbox", "review"],
1652 "the remedy is the review surface, never send"
1653 );
1654
1655 let fresh = home;
1657 let _ = std::fs::remove_dir_all(fresh.join("outbox"));
1658 pending_item(&fresh, "20260812-130000-new", "2026-08-12T13:00:00Z", None);
1659 let findings = examine(&fresh, utc(NOW));
1660 assert!(of(&findings, "outbox").is_empty(), "{findings:#?}");
1661
1662 let _ = std::fs::remove_dir_all(&fresh);
1663 }
1664
1665 #[test]
1666 fn a_failed_extraction_is_broken_at_any_age() {
1667 let home = home("frontdoor-failed");
1668 request(&home, 12, crate::frontdoor::EXTRACTION_FAILED, NOW);
1669
1670 let findings = examine(&home, utc(NOW));
1671 let front = of(&findings, "frontdoor");
1672 assert_eq!(front.len(), 1, "{findings:#?}");
1673 assert_eq!(front[0].severity, Severity::Broken);
1674 assert!(front[0].summary.contains("12"), "{}", front[0].summary);
1675 assert_eq!(
1676 front[0].remedy.as_ref().unwrap().argv,
1677 vec!["mecha", "frontdoor", "list"]
1678 );
1679
1680 let _ = std::fs::remove_dir_all(&home);
1681 }
1682
1683 #[test]
1684 fn a_request_waiting_on_me_is_stale_at_73_hours_and_not_at_71() {
1685 let home = home("frontdoor-stale");
1686 request(
1688 &home,
1689 1,
1690 crate::frontdoor::AWAITING_ME,
1691 "2026-08-11T11:00:00Z",
1692 );
1693 let findings = examine(&home, utc(NOW));
1694 let front = of(&findings, "frontdoor");
1695 assert_eq!(front.len(), 1, "{findings:#?}");
1696 assert_eq!(front[0].severity, Severity::Attention);
1697 assert!(front[0].summary.contains("waiting on you"));
1698
1699 let _ = std::fs::remove_dir_all(home.join("requests"));
1701 request(
1702 &home,
1703 2,
1704 crate::frontdoor::AWAITING_ME,
1705 "2026-08-11T13:00:00Z",
1706 );
1707 let findings = examine(&home, utc(NOW));
1708 assert!(of(&findings, "frontdoor").is_empty(), "{findings:#?}");
1709
1710 let _ = std::fs::remove_dir_all(home.join("requests"));
1712 request(
1713 &home,
1714 3,
1715 crate::frontdoor::NEEDS_INFO,
1716 "2026-08-01T00:00:00Z",
1717 );
1718 let findings = examine(&home, utc(NOW));
1719 assert!(of(&findings, "frontdoor").is_empty(), "{findings:#?}");
1720
1721 let _ = std::fs::remove_dir_all(&home);
1722 }
1723
1724 #[test]
1728 fn a_triaged_request_nothing_will_revisit_goes_stale() {
1729 let home = home("frontdoor-triaged");
1730 request(&home, 4, crate::frontdoor::TRIAGED, "2026-08-11T11:00:00Z");
1732 request(
1734 &home,
1735 5,
1736 crate::frontdoor::NEEDS_INFO,
1737 "2026-08-01T00:00:00Z",
1738 );
1739
1740 let findings = examine(&home, utc(NOW));
1741 let front = of(&findings, "frontdoor");
1742 assert_eq!(front.len(), 1, "{findings:#?}");
1743 assert_eq!(front[0].severity, Severity::Attention);
1744 assert!(front[0].detail.contains("triaged"), "{}", front[0].detail);
1745 assert!(
1746 !front[0].detail.contains("needs_info"),
1747 "needs_info waits on the requester: {}",
1748 front[0].detail
1749 );
1750
1751 let _ = std::fs::remove_dir_all(&home);
1752 }
1753
1754 #[test]
1755 fn a_trigger_whose_last_run_failed_is_flagged_with_the_manual_probe() {
1756 let home = home("trigger-failed");
1757 trigger_file(&home, "morning", "");
1758 ledger_row(
1759 &home,
1760 &json!({
1761 "trigger": "morning",
1762 "slot": "2026-08-13T07:00:00Z",
1763 "started_at": "2026-08-13T07:00:01Z",
1764 "status": "ok",
1765 "summary": "fine",
1766 }),
1767 );
1768 ledger_row(
1769 &home,
1770 &json!({
1771 "trigger": "morning",
1772 "slot": "2026-08-14T07:00:00Z",
1773 "started_at": "2026-08-14T07:00:01Z",
1774 "status": "error",
1775 "error": "provider unreachable",
1776 }),
1777 );
1778
1779 let findings = examine(&home, utc(NOW));
1780 let triggers = of(&findings, "triggers");
1781 assert_eq!(triggers.len(), 1, "{findings:#?}");
1782 assert_eq!(triggers[0].severity, Severity::Attention);
1783 assert!(triggers[0].summary.contains("morning"));
1784 assert!(triggers[0].detail.contains("provider unreachable"));
1785 assert_eq!(
1786 triggers[0].remedy.as_ref().unwrap().argv,
1787 vec!["mecha", "trigger", "run", "morning"],
1788 "a manual run is the safe probe: it never advances the schedule"
1789 );
1790
1791 let _ = std::fs::remove_dir_all(&home);
1792 }
1793
1794 #[test]
1798 fn a_skip_row_after_a_failed_run_does_not_hide_the_failure() {
1799 let home = home("trigger-skip-hides-error");
1800 trigger_file(&home, "morning", "");
1801 ledger_row(
1802 &home,
1803 &json!({
1804 "trigger": "morning",
1805 "slot": "2026-08-13T07:00:00Z",
1806 "started_at": "2026-08-13T07:00:01Z",
1807 "status": "error",
1808 "error": "provider unreachable",
1809 }),
1810 );
1811 ledger_row(
1812 &home,
1813 &json!({
1814 "trigger": "morning",
1815 "slot": "2026-08-14T07:00:00Z",
1816 "started_at": "2026-08-14T07:00:01Z",
1817 "status": "skipped-stale",
1818 }),
1819 );
1820
1821 let findings = examine(&home, utc(NOW));
1822 let triggers = of(&findings, "triggers");
1823 assert_eq!(triggers.len(), 1, "{findings:#?}");
1824 assert!(
1825 triggers[0].summary.contains("most recent run failed"),
1826 "{}",
1827 triggers[0].summary
1828 );
1829 assert!(triggers[0].detail.contains("provider unreachable"));
1830
1831 let _ = std::fs::remove_dir_all(&home);
1832 }
1833
1834 #[test]
1835 fn an_ok_run_followed_by_a_skip_is_healthy() {
1836 let home = home("trigger-ok-then-skip");
1837 trigger_file(&home, "morning", "");
1838 ledger_row(
1839 &home,
1840 &json!({
1841 "trigger": "morning",
1842 "slot": "2026-08-13T07:00:00Z",
1843 "started_at": "2026-08-13T07:00:01Z",
1844 "status": "ok",
1845 }),
1846 );
1847 ledger_row(
1848 &home,
1849 &json!({
1850 "trigger": "morning",
1851 "slot": "2026-08-14T07:00:00Z",
1852 "started_at": "2026-08-14T07:00:01Z",
1853 "status": "skipped-overlap",
1854 }),
1855 );
1856
1857 let findings = examine(&home, utc(NOW));
1858 assert!(of(&findings, "triggers").is_empty(), "{findings:#?}");
1859
1860 let _ = std::fs::remove_dir_all(&home);
1861 }
1862
1863 #[test]
1864 fn a_trigger_quietly_failing_a_third_of_its_calls_is_reported() {
1865 let home = home("trigger-tool-errors");
1868 trigger_file(&home, "morning", "");
1869 for day in 10..15 {
1870 ledger_row(
1871 &home,
1872 &json!({
1873 "trigger": "morning",
1874 "slot": format!("2026-08-{day}T07:00:00Z"),
1875 "started_at": format!("2026-08-{day}T07:00:01Z"),
1876 "status": "ok",
1877 "summary": "briefed",
1878 "tool_calls": 6,
1879 "tool_errors": 3,
1880 }),
1881 );
1882 }
1883
1884 let findings = examine(&home, utc(NOW));
1885 let triggers = of(&findings, "triggers");
1886 assert_eq!(triggers.len(), 1, "{findings:#?}");
1887 assert_eq!(triggers[0].severity, Severity::Attention);
1888 assert!(
1889 triggers[0].summary.contains("15 of 30"),
1890 "{}",
1891 triggers[0].summary
1892 );
1893 assert_eq!(
1894 triggers[0].remedy.as_ref().unwrap().argv,
1895 vec!["mecha", "trigger", "show", "morning"],
1896 "reading is the remedy — what to change is in the transcript"
1897 );
1898
1899 let _ = std::fs::remove_dir_all(&home);
1900 }
1901
1902 #[test]
1903 fn a_handful_of_failed_calls_is_not_a_trend() {
1904 let home = home("trigger-tool-errors-quiet");
1909 trigger_file(&home, "morning", "");
1910 ledger_row(
1912 &home,
1913 &json!({
1914 "trigger": "morning",
1915 "slot": "2026-08-14T07:00:00Z",
1916 "started_at": "2026-08-14T07:00:01Z",
1917 "status": "ok",
1918 "tool_calls": 3,
1919 "tool_errors": 3,
1920 }),
1921 );
1922 assert!(of(&examine(&home, utc(NOW)), "triggers").is_empty());
1923
1924 ledger_row(
1926 &home,
1927 &json!({
1928 "trigger": "morning",
1929 "slot": "2026-08-15T07:00:00Z",
1930 "started_at": "2026-08-15T07:00:01Z",
1931 "status": "ok",
1932 "tool_calls": 40,
1933 "tool_errors": 4,
1934 }),
1935 );
1936 assert!(of(&examine(&home, utc(NOW)), "triggers").is_empty());
1937
1938 let _ = std::fs::remove_dir_all(&home);
1939 }
1940
1941 #[test]
1942 fn a_trigger_that_stopped_doing_anything_is_reported() {
1943 let home = home("trigger-stopped-working");
1946 trigger_file(&home, "morning", "");
1947 for day in 10..14 {
1948 ledger_row(
1949 &home,
1950 &json!({
1951 "trigger": "morning",
1952 "slot": format!("2026-08-{day}T07:00:00Z"),
1953 "started_at": format!("2026-08-{day}T07:00:01Z"),
1954 "status": "ok",
1955 "tool_calls": 8,
1956 "tool_errors": 0,
1957 }),
1958 );
1959 }
1960 ledger_row(
1961 &home,
1962 &json!({
1963 "trigger": "morning",
1964 "slot": "2026-08-14T07:00:00Z",
1965 "started_at": "2026-08-14T07:00:01Z",
1966 "status": "ok",
1967 "summary": "nothing to report",
1968 "tool_calls": 0,
1969 "tool_errors": 0,
1970 }),
1971 );
1972
1973 let findings = examine(&home, utc(NOW));
1974 let triggers = of(&findings, "triggers");
1975 assert_eq!(triggers.len(), 1, "{findings:#?}");
1976 assert!(
1977 triggers[0].summary.contains("did no work"),
1978 "{}",
1979 triggers[0].summary
1980 );
1981 assert!(triggers[0].detail.contains("made 32"));
1982
1983 let _ = std::fs::remove_dir_all(&home);
1984 }
1985
1986 #[test]
1987 fn a_trigger_that_never_needed_tools_is_not_broken_for_not_using_them() {
1988 let home = home("trigger-never-used-tools");
1993 trigger_file(&home, "haiku", "");
1994 for day in 10..15 {
1995 ledger_row(
1996 &home,
1997 &json!({
1998 "trigger": "haiku",
1999 "slot": format!("2026-08-{day}T07:00:00Z"),
2000 "started_at": format!("2026-08-{day}T07:00:01Z"),
2001 "status": "ok",
2002 "tool_calls": 0,
2003 "tool_errors": 0,
2004 }),
2005 );
2006 }
2007 assert!(of(&examine(&home, utc(NOW)), "triggers").is_empty());
2008
2009 let _ = std::fs::remove_dir_all(&home);
2010 }
2011
2012 #[test]
2013 fn a_failed_run_that_did_no_work_is_reported_once_not_twice() {
2014 let home = home("trigger-failed-no-work");
2019 trigger_file(&home, "morning", "");
2020 for day in 10..14 {
2021 ledger_row(
2022 &home,
2023 &json!({
2024 "trigger": "morning",
2025 "slot": format!("2026-08-{day}T07:00:00Z"),
2026 "started_at": format!("2026-08-{day}T07:00:01Z"),
2027 "status": "ok",
2028 "tool_calls": 8,
2029 "tool_errors": 0,
2030 }),
2031 );
2032 }
2033 ledger_row(
2034 &home,
2035 &json!({
2036 "trigger": "morning",
2037 "slot": "2026-08-14T07:00:00Z",
2038 "started_at": "2026-08-14T07:00:01Z",
2039 "status": "error",
2040 "error": "provider unreachable",
2041 "tool_calls": 0,
2042 "tool_errors": 0,
2043 }),
2044 );
2045
2046 let triggers = of(&examine(&home, utc(NOW)), "triggers")
2047 .into_iter()
2048 .cloned()
2049 .collect::<Vec<_>>();
2050 assert_eq!(triggers.len(), 1, "{triggers:#?}");
2051 assert!(triggers[0].detail.contains("provider unreachable"));
2052
2053 let _ = std::fs::remove_dir_all(&home);
2054 }
2055
2056 fn runs_in(
2059 home: &Path,
2060 model: &str,
2061 n: usize,
2062 stats: impl Fn(usize) -> crate::session::RunStats,
2063 ) {
2064 let dir = home.join("sessions");
2065 std::fs::create_dir_all(&dir).unwrap();
2066 for i in 0..n {
2067 let session = crate::session::Session::create(
2068 &dir,
2069 crate::session::SessionMeta {
2070 id: format!("2026080{}T00000{i:03}-{model}", 1 + i % 9),
2075 created_at: utc(NOW),
2076 provider: "local".into(),
2077 model: model.to_string(),
2078 workspace: std::path::PathBuf::from("/tmp"),
2079 title: None,
2080 },
2081 )
2082 .unwrap();
2083 session
2084 .append(&crate::session::Record::Outcome(stats(i)))
2085 .unwrap();
2086 }
2087 }
2088
2089 fn run_stats(
2090 calls: u32,
2091 errors: u32,
2092 ended_failed: bool,
2093 cause: crate::agent::StopCause,
2094 ) -> crate::session::RunStats {
2095 crate::session::RunStats {
2096 tool_calls: calls,
2097 tool_errors: errors,
2098 ended_on_failed_call: ended_failed,
2099 stop_cause: Some(cause),
2100 ..Default::default()
2101 }
2102 }
2103
2104 #[test]
2105 fn a_model_that_keeps_finishing_over_failures_is_reported() {
2106 use crate::agent::StopCause;
2107 let home = home("runs-ended-on-failure");
2108 runs_in(&home, "tiny-local", 30, |i| {
2110 run_stats(6, 0, i % 3 == 0, StopCause::Completed)
2111 });
2112
2113 let all = examine(&home, utc(NOW));
2114 let findings = of(&all, "runs");
2115 assert_eq!(findings.len(), 1, "{findings:#?}");
2116 assert!(
2117 findings[0].summary.contains("tiny-local"),
2118 "{}",
2119 findings[0].summary
2120 );
2121 assert!(
2122 findings[0].summary.contains("33%"),
2123 "{}",
2124 findings[0].summary
2125 );
2126 assert_eq!(
2127 findings[0].remedy.as_ref().unwrap().argv,
2128 vec!["mecha", "sessions", "health", "--days", "30"],
2129 "reading is the remedy; doctor never decides what to change"
2130 );
2131
2132 let _ = std::fs::remove_dir_all(&home);
2133 }
2134
2135 #[test]
2136 fn a_cancelled_run_is_not_the_harness_cutting_it_short() {
2137 use crate::agent::StopCause;
2138 let home = home("runs-interrupted");
2141 runs_in(&home, "tiny-local", 30, |_| {
2142 run_stats(6, 0, false, StopCause::Interrupted)
2143 });
2144 let findings = examine(&home, utc(NOW));
2145 assert!(of(&findings, "runs").is_empty());
2146 let _ = std::fs::remove_dir_all(&home);
2147 }
2148
2149 #[test]
2150 fn a_turn_ceiling_stopping_a_quarter_of_runs_is_a_finding() {
2151 use crate::agent::StopCause;
2152 let home = home("runs-max-turns");
2153 runs_in(&home, "tiny-local", 30, |_| {
2154 run_stats(6, 0, false, StopCause::MaxTurns)
2155 });
2156 let all = examine(&home, utc(NOW));
2157 let findings = of(&all, "runs");
2158 assert_eq!(findings.len(), 1, "{findings:#?}");
2159 assert!(
2160 findings[0].summary.contains("cut"),
2161 "{}",
2162 findings[0].summary
2163 );
2164 let _ = std::fs::remove_dir_all(&home);
2165 }
2166
2167 #[test]
2168 fn a_thin_sample_of_one_model_says_nothing_about_it() {
2169 use crate::agent::StopCause;
2170 let home = home("runs-thin");
2173 runs_in(&home, "tiny-local", 19, |_| {
2174 run_stats(6, 6, true, StopCause::MaxTurns)
2175 });
2176 let all = examine(&home, utc(NOW));
2177 assert!(of(&all, "runs").is_empty());
2178 let _ = std::fs::remove_dir_all(&home);
2179 }
2180
2181 #[test]
2182 fn a_bad_model_does_not_drag_a_good_one_into_a_finding() {
2183 use crate::agent::StopCause;
2184 let home = home("runs-two-models");
2187 runs_in(&home, "steady", 25, |_| {
2188 run_stats(10, 0, false, StopCause::Completed)
2189 });
2190 runs_in(&home, "flaky", 25, |_| {
2191 run_stats(10, 9, false, StopCause::Completed)
2192 });
2193
2194 let all = examine(&home, utc(NOW));
2195 let findings = of(&all, "runs");
2196 assert_eq!(findings.len(), 1, "{findings:#?}");
2197 assert!(
2198 findings[0].summary.contains("flaky"),
2199 "{}",
2200 findings[0].summary
2201 );
2202 assert!(
2203 !findings[0].summary.contains("steady"),
2204 "the healthy model was named in a finding about the other one"
2205 );
2206 let _ = std::fs::remove_dir_all(&home);
2207 }
2208
2209 #[test]
2210 fn a_ledger_written_before_the_counts_existed_reports_nothing() {
2211 let home = home("trigger-tool-errors-bare");
2214 trigger_file(&home, "morning", "");
2215 ledger_row(
2216 &home,
2217 &json!({
2218 "trigger": "morning",
2219 "slot": "2026-08-14T07:00:00Z",
2220 "started_at": "2026-08-14T07:00:01Z",
2221 "status": "ok",
2222 }),
2223 );
2224 assert!(of(&examine(&home, utc(NOW)), "triggers").is_empty());
2225
2226 let _ = std::fs::remove_dir_all(&home);
2227 }
2228
2229 #[test]
2230 fn a_disabled_trigger_is_nobody_s_emergency() {
2231 let home = home("trigger-disabled");
2232 trigger_file(&home, "morning", "enabled = false\n");
2233 ledger_row(
2234 &home,
2235 &json!({
2236 "trigger": "morning",
2237 "started_at": "2026-08-14T07:00:01Z",
2238 "status": "error",
2239 "error": "boom",
2240 }),
2241 );
2242 let findings = examine(&home, utc(NOW));
2243 assert!(of(&findings, "triggers").is_empty(), "{findings:#?}");
2244 let _ = std::fs::remove_dir_all(&home);
2245 }
2246
2247 #[test]
2248 fn a_catch_up_trigger_whose_slots_stopped_advancing_names_the_daemon() {
2249 let home = home("trigger-stale");
2250 trigger_file(&home, "morning", "");
2251 ledger_row(
2254 &home,
2255 &json!({
2256 "trigger": "morning",
2257 "slot": "2026-08-09T07:00:00Z",
2258 "started_at": "2026-08-09T07:00:01Z",
2259 "status": "ok",
2260 }),
2261 );
2262
2263 let findings = examine(&home, utc(NOW));
2264 let triggers = of(&findings, "triggers");
2265 assert_eq!(triggers.len(), 1, "{findings:#?}");
2266 assert_eq!(triggers[0].severity, Severity::Attention);
2267 assert!(triggers[0].summary.contains("missed more than two slots"));
2268 assert!(
2269 triggers[0].detail.contains("daemon"),
2270 "{}",
2271 triggers[0].detail
2272 );
2273 assert!(
2274 triggers[0].remedy.is_none(),
2275 "running the trigger would not restart the scheduler"
2276 );
2277
2278 ledger_row(
2280 &home,
2281 &json!({
2282 "trigger": "morning",
2283 "slot": "2026-08-14T07:00:00Z",
2284 "started_at": "2026-08-14T07:00:01Z",
2285 "status": "ok",
2286 }),
2287 );
2288 let findings = examine(&home, utc(NOW));
2289 assert!(of(&findings, "triggers").is_empty(), "{findings:#?}");
2290
2291 let _ = std::fs::remove_dir_all(&home);
2292 }
2293
2294 #[cfg(unix)]
2297 #[test]
2298 fn one_poisoned_store_does_not_suppress_the_others() {
2299 use std::os::unix::fs::PermissionsExt;
2300 if unsafe { libc::geteuid() } == 0 {
2303 return;
2304 }
2305
2306 let home = home("poisoned");
2307 write_marker(&home, "personal", &valid_marker());
2308 let outbox = home.join("outbox");
2309 std::fs::create_dir_all(&outbox).unwrap();
2310 std::fs::set_permissions(&outbox, std::fs::Permissions::from_mode(0o000)).unwrap();
2311
2312 let findings = examine(&home, utc(NOW));
2313
2314 std::fs::set_permissions(&outbox, std::fs::Permissions::from_mode(0o700)).unwrap();
2316
2317 let mail = of(&findings, "mail");
2318 assert_eq!(mail.len(), 1, "the mail finding survived: {findings:#?}");
2319 assert_eq!(mail[0].severity, Severity::Broken);
2320 let broken_store = of(&findings, "outbox");
2321 assert_eq!(broken_store.len(), 1, "{findings:#?}");
2322 assert!(
2323 broken_store[0].summary.starts_with("store unreadable:"),
2324 "{}",
2325 broken_store[0].summary
2326 );
2327
2328 let _ = std::fs::remove_dir_all(&home);
2329 }
2330
2331 #[test]
2338 fn the_golden_marker_literal_parses_into_the_dead_auth_finding() {
2339 const GOLDEN: &str = r#"{
2340 "at": "2026-08-11T09:00:00Z",
2341 "message": "account `personal`: refresh token expired or revoked — run `mecha-mail auth personal --provider google` (invalid_grant: Token has been revoked.)"
2342}"#;
2343 let home = home("golden-marker");
2344 write_marker(&home, "personal", GOLDEN);
2345
2346 let findings = examine(&home, utc(NOW));
2347 let mail = of(&findings, "mail");
2348 assert_eq!(mail.len(), 1, "{findings:#?}");
2349 assert_eq!(mail[0].severity, Severity::Broken);
2350 assert!(
2351 mail[0].detail.contains("since 2026-08-11T09:00:00Z"),
2352 "the marker's `at` must reach the detail: {}",
2353 mail[0].detail
2354 );
2355 assert!(
2356 mail[0]
2357 .detail
2358 .contains("run `mecha-mail auth personal --provider google`"),
2359 "the marker's `message` must reach the detail: {}",
2360 mail[0].detail
2361 );
2362
2363 let _ = std::fs::remove_dir_all(&home);
2364 }
2365
2366 #[test]
2367 fn findings_sort_broken_first() {
2368 let mut findings = vec![
2369 Finding {
2370 component: "outbox".into(),
2371 severity: Severity::Attention,
2372 summary: "stale".into(),
2373 detail: String::new(),
2374 remedy: None,
2375 },
2376 Finding {
2377 component: "mail".into(),
2378 severity: Severity::Broken,
2379 summary: "dead".into(),
2380 detail: String::new(),
2381 remedy: None,
2382 },
2383 ];
2384 sort(&mut findings);
2385 assert_eq!(findings[0].severity, Severity::Broken);
2386 }
2387
2388 #[test]
2389 fn an_empty_home_is_healthy() {
2390 let home = home("empty");
2391 assert!(examine(&home, utc(NOW)).is_empty());
2392 let _ = std::fs::remove_dir_all(&home);
2393 }
2394
2395 fn graph_store(name: &str) -> PathBuf {
2400 let store = home(name).join(".mecha-graph");
2401 std::fs::create_dir_all(store.join("logs")).unwrap();
2402 store
2403 }
2404
2405 fn nightly_log(store: &Path, file: &str) {
2406 std::fs::write(store.join("logs").join(file), "ran\n").unwrap();
2407 }
2408
2409 #[test]
2413 fn a_graph_nightly_that_stopped_writing_logs_is_a_finding() {
2414 let store = graph_store("graph-stale");
2415 nightly_log(&store, "nightly-20260812.log");
2416 let findings = check_graph_nightly(&store, utc(NOW));
2417 assert_eq!(findings.len(), 1);
2418 assert_eq!(findings[0].component, "graph");
2419 assert_eq!(findings[0].severity, Severity::Attention);
2420 assert!(
2421 findings[0].summary.contains("2 days"),
2422 "{}",
2423 findings[0].summary
2424 );
2425 assert!(
2426 findings[0].detail.contains("nightly-20260812.log"),
2427 "{}",
2428 findings[0].detail
2429 );
2430 }
2431
2432 #[test]
2433 fn yesterdays_log_is_healthy_because_todays_slot_may_not_have_fired() {
2434 let store = graph_store("graph-yesterday");
2435 nightly_log(&store, "nightly-20260813.log");
2436 nightly_log(&store, "mecha-nightly-20260813.log");
2437 assert!(check_graph_nightly(&store, utc(NOW)).is_empty());
2438 }
2439
2440 #[test]
2444 fn each_nightly_family_is_judged_alone() {
2445 let store = graph_store("graph-split");
2446 nightly_log(&store, "nightly-20260814.log");
2447 nightly_log(&store, "mecha-nightly-20260811.log");
2448 let findings = check_graph_nightly(&store, utc(NOW));
2449 assert_eq!(findings.len(), 1);
2450 assert!(
2451 findings[0].summary.contains("mecha-nightly"),
2452 "{}",
2453 findings[0].summary
2454 );
2455 }
2456
2457 #[test]
2460 fn the_shorter_prefix_does_not_claim_the_longer_familys_logs() {
2461 let store = graph_store("graph-prefix");
2462 nightly_log(&store, "mecha-nightly-20260814.log");
2463 nightly_log(&store, "nightly-20260810.log");
2464 let findings = check_graph_nightly(&store, utc(NOW));
2465 assert_eq!(findings.len(), 1);
2466 assert!(
2467 findings[0].detail.contains("nightly-20260810.log"),
2468 "{}",
2469 findings[0].detail
2470 );
2471 }
2472
2473 #[test]
2476 fn a_graph_that_never_ran_is_not_a_finding() {
2477 let missing = home("graph-missing").join(".mecha-graph");
2478 assert!(check_graph_nightly(&missing, utc(NOW)).is_empty());
2479
2480 let empty = graph_store("graph-empty");
2481 assert!(check_graph_nightly(&empty, utc(NOW)).is_empty());
2482
2483 let odd = graph_store("graph-odd-names");
2484 nightly_log(&odd, "nightly-garbage.log");
2485 nightly_log(&odd, "gossip-20260812.jsonl");
2486 assert!(check_graph_nightly(&odd, utc(NOW)).is_empty());
2487 }
2488
2489 #[test]
2491 fn examine_reads_the_graph_store_beside_the_home() {
2492 let scratch = home("graph-sibling");
2493 let mecha_home = scratch.join(".mecha");
2494 std::fs::create_dir_all(&mecha_home).unwrap();
2495 let store = scratch.join(".mecha-graph");
2496 std::fs::create_dir_all(store.join("logs")).unwrap();
2497 nightly_log(&store, "nightly-20260810.log");
2498 let findings = examine(&mecha_home, utc(NOW));
2499 assert_eq!(findings.len(), 1);
2500 assert_eq!(findings[0].component, "graph");
2501 let _ = std::fs::remove_dir_all(&scratch);
2502 }
2503}