1use chrono::{DateTime, Utc};
32use serde::{Deserialize, Serialize};
33use std::collections::BTreeMap;
34use std::path::Path;
35
36#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Serialize, Deserialize)]
39#[serde(rename_all = "lowercase")]
40pub enum Severity {
41 Broken,
43 Attention,
46}
47
48impl Severity {
49 pub fn as_str(&self) -> &'static str {
50 match self {
51 Severity::Broken => "broken",
52 Severity::Attention => "attention",
53 }
54 }
55}
56
57#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
59pub struct Remedy {
60 pub description: String,
63 pub argv: Vec<String>,
65 pub needs_terminal: bool,
69}
70
71#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
74pub struct Finding {
75 pub component: String,
76 pub severity: Severity,
77 pub summary: String,
78 pub detail: String,
79 #[serde(default, skip_serializing_if = "Option::is_none")]
80 pub remedy: Option<Remedy>,
81}
82
83impl Finding {
84 fn unreadable(component: &str, what: &str, why: impl std::fmt::Display) -> Finding {
87 Finding {
88 component: component.to_string(),
89 severity: Severity::Attention,
90 summary: format!("store unreadable: {what}"),
91 detail: why.to_string(),
92 remedy: None,
93 }
94 }
95}
96
97const STUCK_DRAFT_AFTER: chrono::Duration = chrono::Duration::hours(48);
100
101const STALE_REQUEST_AFTER: chrono::Duration = chrono::Duration::hours(72);
104
105pub fn examine(home: &Path, now: DateTime<Utc>) -> Vec<Finding> {
111 let mut findings = Vec::new();
112 findings.extend(check_mail(&home.join("mail")));
113 findings.extend(check_legacy_mail(home));
114 findings.extend(check_outbox(&home.join("outbox"), now));
115 findings.extend(check_frontdoor(&home.join("requests"), now));
116 findings.extend(check_triggers(&home.join("triggers"), now));
117 findings.extend(check_runs(&home.join("sessions")));
118 findings.extend(check_harness(&home.join("learning").join("harness"), now));
119 findings.extend(check_learning(&home.join("learning"), now));
120 if let Some(parent) = home.parent() {
124 findings.extend(check_graph_nightly(&parent.join(".mecha-graph"), now));
125 }
126 sort(&mut findings);
127 findings
128}
129
130pub fn sort(findings: &mut [Finding]) {
133 findings.sort_by(|a, b| {
134 a.severity
135 .cmp(&b.severity)
136 .then_with(|| a.component.cmp(&b.component))
137 });
138}
139
140#[derive(Debug, Deserialize)]
147struct AuthMarker {
148 at: String,
149 message: String,
150}
151
152#[derive(Debug, Default, Deserialize)]
155struct MailAccounts {
156 #[serde(default, rename = "account")]
157 accounts: Vec<MailAccount>,
158}
159
160#[derive(Debug, Deserialize)]
161struct MailAccount {
162 name: String,
163 provider: String,
164 #[serde(default)]
168 grant_lifetime_days: Option<u32>,
169}
170
171fn check_mail(mail: &Path) -> Vec<Finding> {
175 let mut out = Vec::new();
176 if !mail.is_dir() {
177 return out;
178 }
179
180 let declared: Vec<MailAccount> = std::fs::read_to_string(mail.join("accounts.toml"))
183 .ok()
184 .and_then(|text| toml::from_str::<MailAccounts>(&text).ok())
185 .map(|file| file.accounts)
186 .unwrap_or_default();
187 let providers: BTreeMap<String, String> = declared
188 .iter()
189 .map(|a| (a.name.clone(), a.provider.clone()))
190 .collect();
191 let lifetimes: BTreeMap<String, u32> = declared
192 .iter()
193 .filter_map(|a| a.grant_lifetime_days.map(|d| (a.name.clone(), d)))
194 .collect();
195
196 let entries = match std::fs::read_dir(mail) {
197 Ok(entries) => entries,
198 Err(e) => {
199 out.push(Finding::unreadable(
200 "mail",
201 "the mail directory",
202 format!("{}: {e}", mail.display()),
203 ));
204 return out;
205 }
206 };
207
208 for entry in entries.flatten() {
209 let dir = entry.path();
210 if !dir.is_dir() {
211 continue;
212 }
213 let Some(account) = dir.file_name().and_then(|n| n.to_str()).map(String::from) else {
214 continue;
215 };
216 out.extend(check_triage_scope(&dir, &account, providers.get(&account)));
221 out.extend(check_grant_age(
222 &dir,
223 &account,
224 providers.get(&account),
225 lifetimes.get(&account).copied(),
226 ));
227
228 let marker_path = dir.join("auth_error.json");
229 if !marker_path.is_file() {
230 continue;
231 }
232 let text = match std::fs::read_to_string(&marker_path) {
233 Ok(text) => text,
234 Err(e) => {
235 out.push(Finding::unreadable(
236 "mail",
237 &format!("auth_error.json for `{account}`"),
238 format!("{}: {e}", marker_path.display()),
239 ));
240 continue;
241 }
242 };
243 match serde_json::from_str::<AuthMarker>(&text) {
244 Ok(marker) => {
245 let provider = providers.get(&account);
246 let mut argv = vec![
247 "mecha-mail".to_string(),
248 "auth".to_string(),
249 account.clone(),
250 ];
251 if let Some(provider) = provider {
252 argv.push("--provider".to_string());
253 argv.push(provider.clone());
254 }
255 out.push(Finding {
256 component: "mail".to_string(),
257 severity: Severity::Broken,
258 summary: format!("mail auth for `{account}` is dead"),
259 detail: format!(
264 "permanent refresh failure since {}: {}",
265 marker.at, marker.message
266 ),
267 remedy: Some(Remedy {
268 description: format!(
269 "re-authenticate the `{account}` account (opens an OAuth flow)"
270 ),
271 argv,
272 needs_terminal: true,
273 }),
274 });
275 }
276 Err(e) => out.push(Finding::unreadable(
277 "mail",
278 &format!("auth_error.json for `{account}` did not parse"),
279 format!("{}: {e}", marker_path.display()),
280 )),
281 }
282 }
283 out
284}
285
286#[derive(Debug, serde::Deserialize)]
293struct StoredGrant {
294 #[serde(default)]
295 granted_scopes: Option<String>,
296 #[serde(default)]
297 granted_at: Option<String>,
298}
299
300const GRANT_WARN_WITHIN_DAYS: i64 = 2;
307
308fn check_grant_age(
321 dir: &Path,
322 account: &str,
323 provider: Option<&String>,
324 lifetime_days: Option<u32>,
325) -> Vec<Finding> {
326 let Some(lifetime) = lifetime_days.filter(|d| *d > 0) else {
327 return Vec::new();
328 };
329 let Ok(text) = std::fs::read_to_string(dir.join("oauth.json")) else {
330 return Vec::new();
331 };
332 let Ok(grant) = serde_json::from_str::<StoredGrant>(&text) else {
333 return Vec::new(); };
335 let Some(granted_at) = grant.granted_at.as_deref() else {
339 return Vec::new();
340 };
341 let Ok(granted) = chrono::DateTime::parse_from_rfc3339(granted_at) else {
342 return Vec::new();
343 };
344 let expires = granted.with_timezone(&chrono::Utc) + chrono::Duration::days(lifetime as i64);
345 let hours_left = (expires - chrono::Utc::now()).num_hours();
350 let left = (hours_left as f64 / 24.0).ceil() as i64;
351 if left > GRANT_WARN_WITHIN_DAYS {
352 return Vec::new();
353 }
354 let when = if hours_left < 0 {
355 "has expired".to_string()
356 } else if hours_left < 24 {
357 "expires within a day".to_string()
358 } else {
359 format!("expires in {left} days")
360 };
361 let mut argv = vec![
362 "mecha-mail".to_string(),
363 "auth".to_string(),
364 account.to_string(),
365 ];
366 if let Some(p) = provider {
367 argv.push("--provider".to_string());
368 argv.push(p.clone());
369 }
370 vec![Finding {
371 component: "mail".to_string(),
372 severity: Severity::Attention,
373 summary: format!("`{account}` sign-in {when}"),
374 detail: format!(
375 "this grant lasts {lifetime} days from consent ({granted_at}) and refreshing does \
376 not extend it. Re-authenticate before it lapses — once it does, the failure looks \
377 like a revoked token and every scheduled run using this account stops."
378 ),
379 remedy: Some(Remedy {
380 description: format!("re-authenticate `{account}` now (opens an OAuth flow)"),
381 argv,
382 needs_terminal: true,
383 }),
384 }]
385}
386
387fn triage_scope_for(provider: &str) -> Option<&'static str> {
391 match provider {
392 "google" => Some("gmail.modify"),
393 "outlook" | "microsoft" => Some("Mail.ReadWrite"),
394 _ => None,
395 }
396}
397
398fn check_triage_scope(dir: &Path, account: &str, provider: Option<&String>) -> Vec<Finding> {
414 let Some(provider) = provider else {
415 return Vec::new();
416 };
417 let Some(needed) = triage_scope_for(provider) else {
418 return Vec::new();
419 };
420 let path = dir.join("oauth.json");
421 let Ok(text) = std::fs::read_to_string(&path) else {
422 return Vec::new();
425 };
426 let Ok(grant) = serde_json::from_str::<StoredGrant>(&text) else {
427 return vec![Finding::unreadable(
428 "mail",
429 &format!("oauth.json for `{account}` did not parse"),
430 format!("{}", path.display()),
431 )];
432 };
433 if grant
434 .granted_scopes
435 .as_deref()
436 .is_some_and(|g| g.contains(needed))
437 {
438 return Vec::new();
439 }
440 let admin_note = if provider == "outlook" || provider == "microsoft" {
441 " Microsoft blocks `Mail.ReadWrite` from end-user consent under its \
442 recommended policy, so on a managed tenant an administrator has to \
443 grant it to the app registration before this can succeed."
444 } else {
445 ""
446 };
447 vec![Finding {
448 component: "mail".to_string(),
449 severity: Severity::Attention,
450 summary: format!("`{account}` cannot archive, spam or mark mail read"),
451 detail: format!(
452 "the stored grant does not include `{needed}`, so mail_triage will fail on this \
453 account. Reading, sending and calendar work are unaffected.{admin_note}"
454 ),
455 remedy: Some(Remedy {
456 description: format!(
457 "re-authenticate `{account}` to add the triage scope (opens an OAuth flow)"
458 ),
459 argv: vec![
460 "mecha-mail".to_string(),
461 "auth".to_string(),
462 account.to_string(),
463 "--provider".to_string(),
464 provider.clone(),
465 ],
466 needs_terminal: true,
467 }),
468 }]
469}
470
471#[cfg(test)]
472mod grant_age_tests {
473 use super::*;
474
475 fn store(dir: &Path, granted_at: Option<&str>) {
476 std::fs::create_dir_all(dir).unwrap();
477 let stamp = granted_at
478 .map(|g| format!(r#","granted_at":"{g}""#))
479 .unwrap_or_default();
480 std::fs::write(
481 dir.join("oauth.json"),
482 format!(r#"{{"client_id":"i","access_token":"a","refresh_token":"r","expires_at":1{stamp}}}"#),
483 )
484 .unwrap();
485 }
486
487 fn days_ago(n: i64) -> String {
488 (chrono::Utc::now() - chrono::Duration::days(n)).to_rfc3339()
489 }
490
491 #[test]
493 fn a_grant_nearing_its_declared_lifetime_is_reported_early() {
494 let tmp = std::env::temp_dir().join(format!("mecha-grant-{}", std::process::id()));
495 let g = "google".to_string();
496
497 store(&tmp, Some(&days_ago(1)));
499 assert!(check_grant_age(&tmp, "personal", Some(&g), Some(7)).is_empty());
500
501 store(&tmp, Some(&days_ago(5)));
503 let f = check_grant_age(&tmp, "personal", Some(&g), Some(7));
504 assert_eq!(f.len(), 1, "should warn with 2 days left");
505 assert!(
506 f[0].summary.contains("expires in 2 days"),
507 "{}",
508 f[0].summary
509 );
510 assert!(f[0].remedy.as_ref().unwrap().needs_terminal);
511
512 store(&tmp, Some(&days_ago(7)));
514 let f = check_grant_age(&tmp, "personal", Some(&g), Some(7));
515 assert!(f[0].summary.contains("within a day"), "{}", f[0].summary);
516
517 store(&tmp, Some(&days_ago(9)));
519 let f = check_grant_age(&tmp, "personal", Some(&g), Some(7));
520 assert!(f[0].summary.contains("has expired"), "{}", f[0].summary);
521
522 assert!(check_grant_age(&tmp, "personal", Some(&g), None).is_empty());
524
525 store(&tmp, None);
527 assert!(check_grant_age(&tmp, "personal", Some(&g), Some(7)).is_empty());
528
529 std::fs::remove_dir_all(&tmp).ok();
530 }
531}
532
533fn check_legacy_mail(home: &Path) -> Vec<Finding> {
540 let mut out = Vec::new();
541 for provider in ["google", "outlook"] {
542 let marker_path = home.join(provider).join("auth_error.json");
543 if !marker_path.is_file() {
544 continue;
545 }
546 let text = match std::fs::read_to_string(&marker_path) {
547 Ok(text) => text,
548 Err(e) => {
549 out.push(Finding::unreadable(
550 "mail",
551 &format!("auth_error.json for the legacy {provider} store"),
552 format!("{}: {e}", marker_path.display()),
553 ));
554 continue;
555 }
556 };
557 match serde_json::from_str::<AuthMarker>(&text) {
558 Ok(marker) => out.push(Finding {
559 component: "mail".to_string(),
560 severity: Severity::Broken,
561 summary: format!("legacy {provider} mail auth is dead"),
562 detail: format!(
566 "permanent refresh failure since {}: {}",
567 marker.at, marker.message
568 ),
569 remedy: Some(Remedy {
570 description: format!(
571 "bring the legacy {provider} login into the unified registry — \
572 and re-authenticate it per the detail, which no import fixes"
573 ),
574 argv: vec![
575 "mecha-mail".to_string(),
576 "import".to_string(),
577 provider.to_string(),
578 "--provider".to_string(),
579 provider.to_string(),
580 ],
581 needs_terminal: false,
582 }),
583 }),
584 Err(e) => out.push(Finding::unreadable(
585 "mail",
586 &format!("auth_error.json for the legacy {provider} store did not parse"),
587 format!("{}: {e}", marker_path.display()),
588 )),
589 }
590 }
591 out
592}
593
594fn check_outbox(root: &Path, now: DateTime<Utc>) -> Vec<Finding> {
600 let mut out = Vec::new();
601 if !root.is_dir() {
602 return out;
603 }
604 let entries = match std::fs::read_dir(root) {
605 Ok(entries) => entries,
606 Err(e) => {
607 out.push(Finding::unreadable(
608 "outbox",
609 "the outbox directory",
610 format!("{}: {e}", root.display()),
611 ));
612 return out;
613 }
614 };
615
616 let review = Remedy {
617 description: "open the outbox review surface — doctor never releases a draft".to_string(),
618 argv: vec!["mecha".into(), "outbox".into(), "review".into()],
619 needs_terminal: true,
620 };
621
622 let mut stale: Vec<String> = Vec::new();
623 for entry in entries.flatten() {
624 let path = entry.path();
625 if path.extension().and_then(|e| e.to_str()) != Some("json") {
626 continue;
627 }
628 let item: crate::outbox::OutboxItem =
629 match std::fs::read_to_string(&path).map(|t| serde_json::from_str(&t)) {
630 Ok(Ok(item)) => item,
631 Ok(Err(e)) => {
632 out.push(Finding::unreadable(
633 "outbox",
634 &format!(
635 "item {} did not parse",
636 path.file_name().unwrap_or_default().to_string_lossy()
637 ),
638 format!("{}: {e}", path.display()),
639 ));
640 continue;
641 }
642 Err(e) => {
643 out.push(Finding::unreadable(
644 "outbox",
645 &format!(
646 "item {} could not be read",
647 path.file_name().unwrap_or_default().to_string_lossy()
648 ),
649 format!("{}: {e}", path.display()),
650 ));
651 continue;
652 }
653 };
654 if item.status != "pending" {
655 continue;
656 }
657 if let Some(error) = &item.error {
658 out.push(Finding {
659 component: "outbox".to_string(),
660 severity: Severity::Broken,
661 summary: format!("release failed: {error}"),
662 detail: format!(
663 "{} · {} — still pending; the draft is good, the delivery was not",
664 item.id, item.summary
665 ),
666 remedy: Some(review.clone()),
667 });
668 } else if age_of(&item.created_at, now).is_some_and(|age| age > STUCK_DRAFT_AFTER) {
669 stale.push(format!(
670 "{} · {} — staged {}",
671 item.id,
672 item.summary,
673 render_age(now, &item.created_at)
674 ));
675 }
676 }
677
678 if !stale.is_empty() {
679 stale.sort();
681 out.push(Finding {
682 component: "outbox".to_string(),
683 severity: Severity::Attention,
684 summary: format!(
685 "{} draft{} pending for more than 48h",
686 stale.len(),
687 if stale.len() == 1 { "" } else { "s" }
688 ),
689 detail: stale.join("\n"),
690 remedy: Some(review),
691 });
692 }
693 out
694}
695
696const WAITING_ON_ME: [&str; 3] = [
704 crate::frontdoor::EXTRACTED,
705 crate::frontdoor::AWAITING_ME,
706 crate::frontdoor::TRIAGED,
707];
708
709fn check_frontdoor(root: &Path, now: DateTime<Utc>) -> Vec<Finding> {
712 let mut out = Vec::new();
713 if !root.is_dir() {
714 return out;
715 }
716 let entries = match std::fs::read_dir(root) {
717 Ok(entries) => entries,
718 Err(e) => {
719 out.push(Finding::unreadable(
720 "frontdoor",
721 "the request store",
722 format!("{}: {e}", root.display()),
723 ));
724 return out;
725 }
726 };
727
728 let list = Remedy {
729 description: "list the frontdoor queue".to_string(),
730 argv: vec!["mecha".into(), "frontdoor".into(), "list".into()],
731 needs_terminal: false,
732 };
733
734 let mut stale: Vec<(i64, String)> = Vec::new();
735 for entry in entries.flatten() {
736 let path = entry.path();
737 if path.extension().and_then(|e| e.to_str()) != Some("json") {
738 continue;
739 }
740 let Ok(Ok(record)) = std::fs::read_to_string(&path)
741 .map(|t| serde_json::from_str::<crate::frontdoor::Record>(&t))
742 else {
743 out.push(Finding::unreadable(
746 "frontdoor",
747 &format!(
748 "request {} did not parse",
749 path.file_name().unwrap_or_default().to_string_lossy()
750 ),
751 path.display().to_string(),
752 ));
753 continue;
754 };
755 if record.state == crate::frontdoor::EXTRACTION_FAILED {
756 out.push(Finding {
757 component: "frontdoor".to_string(),
758 severity: Severity::Broken,
759 summary: format!(
760 "request {} failed extraction and waits for a human",
761 record.seq
762 ),
763 detail: format!(
764 "{} ({}) — {}",
765 record.seq,
766 record.type_id,
767 record
768 .extraction_error
769 .as_deref()
770 .unwrap_or("no error recorded")
771 ),
772 remedy: Some(list.clone()),
773 });
774 } else if WAITING_ON_ME.contains(&record.state.as_str())
775 && request_age(&record, now).is_some_and(|age| age > STALE_REQUEST_AFTER)
776 {
777 stale.push((
778 record.seq,
779 format!(
780 "{} ({}) — {}, received {}",
781 record.seq,
782 record.type_id,
783 record.state,
784 render_age(now, &record.created_at)
785 ),
786 ));
787 }
788 }
789
790 if !stale.is_empty() {
791 stale.sort_by_key(|(seq, _)| *seq);
793 out.push(Finding {
794 component: "frontdoor".to_string(),
795 severity: Severity::Attention,
796 summary: format!(
797 "{} request{} waiting on you for more than 72h",
798 stale.len(),
799 if stale.len() == 1 { "" } else { "s" }
800 ),
801 detail: stale
802 .into_iter()
803 .map(|(_, line)| line)
804 .collect::<Vec<_>>()
805 .join("\n"),
806 remedy: Some(list),
807 });
808 }
809 out
810}
811
812fn request_age(record: &crate::frontdoor::Record, now: DateTime<Utc>) -> Option<chrono::Duration> {
817 age_of(&record.drained_at, now).or_else(|| age_of(&record.created_at, now))
818}
819
820const HEALTH_WINDOW: usize = 5;
827
828const HEALTH_MIN_CALLS: u32 = 10;
833
834const HEALTH_ERROR_RATE: f64 = 1.0 / 3.0;
841
842fn check_triggers(root: &Path, now: DateTime<Utc>) -> Vec<Finding> {
845 let mut out = Vec::new();
846 if !root.is_dir() {
847 return out;
848 }
849 let entries = match std::fs::read_dir(root) {
850 Ok(entries) => entries,
851 Err(e) => {
852 out.push(Finding::unreadable(
853 "triggers",
854 "the trigger store",
855 format!("{}: {e}", root.display()),
856 ));
857 return out;
858 }
859 };
860
861 let mut triggers: Vec<crate::trigger::Trigger> = Vec::new();
862 for entry in entries.flatten() {
863 let path = entry.path();
864 if path.extension().and_then(|e| e.to_str()) != Some("toml") {
865 continue;
866 }
867 let name = path
868 .file_stem()
869 .and_then(|s| s.to_str())
870 .unwrap_or_default()
871 .to_string();
872 match std::fs::read_to_string(&path).map(|t| toml::from_str::<crate::trigger::Trigger>(&t))
873 {
874 Ok(Ok(mut trigger)) => {
875 trigger.name = name;
876 triggers.push(trigger);
877 }
878 _ => out.push(Finding::unreadable(
879 "triggers",
880 &format!("trigger file `{name}.toml` did not parse"),
881 path.display().to_string(),
882 )),
883 }
884 }
885
886 let mut recent: BTreeMap<String, Vec<crate::trigger::RunRecord>> = BTreeMap::new();
890 let mut last_slot: BTreeMap<String, DateTime<Utc>> = BTreeMap::new();
891 let ledger = root.join("runs.jsonl");
892 if ledger.is_file() {
893 match std::fs::read_to_string(&ledger) {
894 Ok(text) => {
895 for line in text.lines().filter(|l| !l.trim().is_empty()) {
896 let Ok(row) = serde_json::from_str::<crate::trigger::RunRecord>(line) else {
899 continue;
900 };
901 if let Some(slot) = row.slot {
902 let newest = last_slot.entry(row.trigger.clone()).or_insert(slot);
903 if slot > *newest {
904 *newest = slot;
905 }
906 }
907 if matches!(
912 row.status,
913 crate::trigger::RunStatus::Ok | crate::trigger::RunStatus::Error
914 ) {
915 let window = recent.entry(row.trigger.clone()).or_default();
916 window.push(row);
917 if window.len() > HEALTH_WINDOW {
918 window.remove(0);
919 }
920 }
921 }
922 }
923 Err(e) => out.push(Finding::unreadable(
924 "triggers",
925 "the run ledger",
926 format!("{}: {e}", ledger.display()),
927 )),
928 }
929 }
930
931 for trigger in &triggers {
932 if !trigger.enabled {
933 continue;
934 }
935
936 let window = recent.get(&trigger.name);
939 if let Some(row) = window.and_then(|w| w.last()) {
940 if row.status == crate::trigger::RunStatus::Error {
941 out.push(Finding {
942 component: "triggers".to_string(),
943 severity: Severity::Attention,
944 summary: format!("trigger `{}`'s most recent run failed", trigger.name),
945 detail: format!(
946 "started {}: {}",
947 row.started_at.to_rfc3339(),
948 row.error.as_deref().unwrap_or("no error recorded")
949 ),
950 remedy: Some(Remedy {
951 description: format!(
952 "run `{}` by hand — a manual run is evidence, not a fire; it never advances the schedule",
953 trigger.name
954 ),
955 argv: vec![
956 "mecha".into(),
957 "trigger".into(),
958 "run".into(),
959 trigger.name.clone(),
960 ],
961 needs_terminal: false,
962 }),
963 });
964 }
965 }
966
967 let (calls, errors) = window
973 .map(|w| {
974 w.iter().fold((0u32, 0u32), |(c, e), r| {
975 (c + r.tool_calls, e + r.tool_errors)
976 })
977 })
978 .unwrap_or((0, 0));
979 if calls >= HEALTH_MIN_CALLS && f64::from(errors) / f64::from(calls) >= HEALTH_ERROR_RATE {
980 let runs = window.map(Vec::len).unwrap_or(0);
981 out.push(Finding {
982 component: "triggers".to_string(),
983 severity: Severity::Attention,
984 summary: format!(
985 "trigger `{}` failed {errors} of {calls} tool calls",
986 trigger.name
987 ),
988 detail: format!(
989 "across its last {runs} run(s){}. A run's answer arrives either way, so this is invisible in the ledger's status — and per-step reliability is what decides how long a task the run can finish, so a third of the calls failing is not a third of the work lost.",
990 if window.is_some_and(|w| w.last().is_some_and(|r| r.ended_on_failed_call)) {
991 ", and the most recent run answered with its last call failed"
992 } else {
993 ""
994 }
995 ),
996 remedy: Some(Remedy {
999 description: format!("read `{}`'s recent runs", trigger.name),
1000 argv: vec![
1001 "mecha".into(),
1002 "trigger".into(),
1003 "show".into(),
1004 trigger.name.clone(),
1005 ],
1006 needs_terminal: false,
1007 }),
1008 });
1009 }
1010
1011 if let Some(window) = window {
1025 let newest = window.last();
1026 let before: u32 = window[..window.len().saturating_sub(1)]
1027 .iter()
1028 .map(|r| r.tool_calls)
1029 .sum();
1030 let stopped = newest
1033 .is_some_and(|r| r.tool_calls == 0 && r.status == crate::trigger::RunStatus::Ok)
1034 && before >= HEALTH_MIN_CALLS;
1035 if stopped {
1036 out.push(Finding {
1037 component: "triggers".to_string(),
1038 severity: Severity::Attention,
1039 summary: format!(
1040 "trigger `{}`'s most recent run did no work",
1041 trigger.name
1042 ),
1043 detail: format!(
1044 "it succeeded having made no tool calls, where its previous {} run(s) made {before}. A run that does nothing and reports success is indistinguishable from a healthy one in every other signal — the status is `ok`, the schedule advanced, and the answer arrived.",
1045 window.len() - 1
1046 ),
1047 remedy: Some(Remedy {
1048 description: format!("read `{}`'s recent runs", trigger.name),
1049 argv: vec![
1050 "mecha".into(),
1051 "trigger".into(),
1052 "show".into(),
1053 trigger.name.clone(),
1054 ],
1055 needs_terminal: false,
1056 }),
1057 });
1058 }
1059 }
1060
1061 if trigger.catch_up != crate::trigger::CatchUp::Always {
1067 continue;
1068 }
1069 let Some(anchor) = last_slot.get(&trigger.name).copied().or(trigger.created_at) else {
1070 continue;
1073 };
1074 let tz = trigger.tz(None);
1075 let step = chrono::Duration::seconds(1);
1076 let missed_more_than_two = trigger
1077 .schedule
1078 .prev_at_or_before(now, tz)
1079 .and_then(|s0| trigger.schedule.prev_at_or_before(s0 - step, tz))
1080 .and_then(|s1| trigger.schedule.prev_at_or_before(s1 - step, tz))
1081 .is_some_and(|s2| s2 > anchor);
1082 if missed_more_than_two {
1083 out.push(Finding {
1084 component: "triggers".to_string(),
1085 severity: Severity::Attention,
1086 summary: format!("trigger `{}` has missed more than two slots", trigger.name),
1087 detail: format!(
1088 "last accounted slot {}; with catch_up=always a healthy scheduler fires \
1089 the most recent slot every tick, so the daemon or its timer may be down \
1090 (systemctl --user status mecha-triggers)",
1091 anchor.to_rfc3339()
1092 ),
1093 remedy: None,
1096 });
1097 }
1098 }
1099 out
1100}
1101
1102const RUNS_WINDOW: usize = 200;
1111
1112const RUNS_MIN: usize = 20;
1118
1119const ENDED_ON_FAILURE_RATE: f64 = 0.20;
1125
1126const TOOL_ERROR_RATE: f64 = 0.25;
1128
1129const RUNS_MIN_CALLS: u64 = 20;
1132
1133const CUT_SHORT_RATE: f64 = 0.25;
1137
1138fn cut_short(stats: &crate::session::RunStats) -> bool {
1141 stats.stop_cause.is_some_and(|c| c.cut_short())
1142}
1143
1144fn check_runs(sessions: &Path) -> Vec<Finding> {
1153 use crate::runlog::{Corpus, Scan};
1154
1155 let mut out = Vec::new();
1156 if !sessions.is_dir() {
1157 return out;
1158 }
1159 let corpus = match Corpus::scan(
1160 sessions,
1161 &Scan {
1162 max_sessions: Some(RUNS_WINDOW),
1163 since: None,
1164 },
1165 ) {
1166 Ok(c) => c,
1167 Err(e) => {
1168 out.push(Finding::unreadable(
1169 "runs",
1170 "the session store",
1171 format!("{}: {e}", sessions.display()),
1172 ));
1173 return out;
1174 }
1175 };
1176
1177 let remedy = |what: &str| {
1178 Some(Remedy {
1179 description: format!("read the run-quality summary ({what})"),
1180 argv: vec![
1181 "mecha".into(),
1182 "sessions".into(),
1183 "health".into(),
1184 "--days".into(),
1185 "30".into(),
1186 ],
1187 needs_terminal: false,
1188 })
1189 };
1190
1191 for (model, runs) in corpus.by_model() {
1192 if runs.len() < RUNS_MIN {
1193 continue;
1194 }
1195 let n = runs.len();
1196
1197 if let Some(rate) = runs.rate_of(|r| r.stats.ended_on_failed_call) {
1198 if rate >= ENDED_ON_FAILURE_RATE {
1199 out.push(Finding {
1200 component: "runs".to_string(),
1201 severity: Severity::Attention,
1202 summary: format!(
1203 "{:.0}% of `{model}` runs finished on a failed tool call",
1204 rate * 100.0
1205 ),
1206 detail: format!(
1207 "{} of {n} recent run(s). The model stopped of its own accord with its last call failed, and the answer it wrote may report success over it — which nothing in the text or the stop reason can show.",
1208 runs.ended_on_failed_call()
1209 ),
1210 remedy: remedy("which runs, and what failed"),
1211 });
1212 }
1213 }
1214
1215 if let Some(rate) = runs.tool_error_rate() {
1216 if rate >= TOOL_ERROR_RATE && runs.tool_calls() >= RUNS_MIN_CALLS {
1221 out.push(Finding {
1222 component: "runs".to_string(),
1223 severity: Severity::Attention,
1224 summary: format!(
1225 "`{model}` runs fail {:.0}% of their tool calls",
1226 rate * 100.0
1227 ),
1228 detail: format!(
1229 "{} of {} call(s) across {n} run(s) were refused by the environment. Errors are how a run learns where it is, so some are healthy — a quarter of them says something moved: a renamed path, a revoked grant, a tool whose schema the model keeps mis-filling.",
1230 runs.tool_errors(),
1231 runs.tool_calls()
1232 ),
1233 remedy: remedy("which tool, and how it failed"),
1234 });
1235 }
1236 }
1237
1238 if let Some(rate) = runs.rate_of(|r| cut_short(&r.stats)) {
1239 if rate >= CUT_SHORT_RATE {
1240 let cut = runs.rows.iter().filter(|r| cut_short(&r.stats)).count();
1241 out.push(Finding {
1242 component: "runs".to_string(),
1243 severity: Severity::Attention,
1244 summary: format!(
1245 "the harness cut {:.0}% of `{model}` runs short",
1246 rate * 100.0
1247 ),
1248 detail: format!(
1249 "{cut} of {n} recent run(s) hit a turn, token or cost ceiling, or tripped the loop guard. A budget that stops a quarter of runs is measuring the budget rather than the work — the answers are truncated and say so only in `stop_cause`. Cancellations are not counted here.",
1250 ),
1251 remedy: remedy("which ceiling, and how often"),
1252 });
1253 }
1254 }
1255 }
1256 out
1257}
1258
1259const GRAPH_NIGHTLIES: &[(&str, &str)] = &[
1269 ("nightly-", "the graph's own sweep (ingest, extract, decay)"),
1270 ("mecha-nightly-", "the mecha half (vet, precheck, gossip)"),
1271];
1272
1273const STALE_CANDIDATE_AFTER: chrono::Duration = chrono::Duration::hours(72);
1277
1278const STARVED_LEARNER_MIN_EXCLUDED: usize = 10;
1283
1284fn check_learning(root: &Path, now: DateTime<Utc>) -> Vec<Finding> {
1297 let mut out = Vec::new();
1298 let path = root.join("reflections.jsonl");
1299 if !path.is_file() {
1300 return out;
1301 }
1302 let text = match std::fs::read_to_string(&path) {
1303 Ok(t) => t,
1304 Err(e) => {
1305 out.push(Finding::unreadable(
1306 "learning",
1307 "the reflections archive",
1308 format!("{}: {e}", path.display()),
1309 ));
1310 return out;
1311 }
1312 };
1313
1314 let mut total = 0usize;
1315 let mut excluded = 0usize;
1316 let mut newest_excluded: Option<DateTime<Utc>> = None;
1317 let mut waiting: std::collections::BTreeMap<String, usize> = Default::default();
1319 for line in text.lines().filter(|l| !l.trim().is_empty()) {
1320 let Ok(r) = serde_json::from_str::<crate::learning::Reflexion>(line) else {
1321 continue;
1325 };
1326 total += 1;
1327 if r.origin == crate::learning::Origin::Clean {
1328 if !r.is_processed {
1329 *waiting.entry(r.domain.clone()).or_default() += 1;
1330 }
1331 } else {
1332 excluded += 1;
1333 if let Ok(t) = DateTime::parse_from_rfc3339(&r.created_at) {
1334 let t = t.with_timezone(&Utc);
1335 if newest_excluded.is_none_or(|n| t > n) {
1336 newest_excluded = Some(t);
1337 }
1338 }
1339 }
1340 }
1341
1342 let floor = crate::learning::LEARN_MIN_REFLECTIONS;
1343 if waiting.values().any(|&n| n >= floor) {
1346 return out;
1347 }
1348 if excluded < STARVED_LEARNER_MIN_EXCLUDED {
1349 return out;
1350 }
1351 let alive = newest_excluded.is_some_and(|t| now.signed_duration_since(t).num_days() <= 30);
1355 if !alive {
1356 return out;
1357 }
1358
1359 let pool = if waiting.is_empty() {
1360 "none clean and unprocessed".to_string()
1361 } else {
1362 waiting
1363 .iter()
1364 .map(|(d, n)| format!("{d} {n}/{floor}"))
1365 .collect::<Vec<_>>()
1366 .join(", ")
1367 };
1368 out.push(Finding {
1369 component: "learning".to_string(),
1370 severity: Severity::Attention,
1371 summary: format!(
1372 "the rule learner is starved: {excluded} of {total} reflections excluded by \
1373 origin, and no domain reaches the learn floor of {floor}"
1374 ),
1375 detail: format!(
1376 "reflect keeps mining and the provenance gate keeps excluding — the gate working \
1377 as designed, every night, with nothing downstream to show for it. Clean pool: \
1378 {pool}. The excluded evidence stays readable in {}; the decision this proposes \
1379 is yours, not a command's: accept the rate, or change what evidence the loop \
1380 may consolidate.",
1381 path.display()
1382 ),
1383 remedy: Some(Remedy {
1384 description: "see how new interventions classify — doctor never loosens the gate"
1385 .to_string(),
1386 argv: vec!["mecha".into(), "reflect".into(), "--dry-run".into()],
1387 needs_terminal: false,
1388 }),
1389 });
1390 out
1391}
1392
1393fn check_harness(root: &Path, now: DateTime<Utc>) -> Vec<Finding> {
1398 let mut out = Vec::new();
1399 let dir = root.join("candidates");
1400 if !dir.is_dir() {
1401 return out;
1402 }
1403 let entries = match std::fs::read_dir(&dir) {
1404 Ok(entries) => entries,
1405 Err(e) => {
1406 out.push(Finding::unreadable(
1407 "harness",
1408 "the harness candidate directory",
1409 format!("{}: {e}", dir.display()),
1410 ));
1411 return out;
1412 }
1413 };
1414 let mut stale: Vec<String> = Vec::new();
1415 for entry in entries.flatten() {
1416 let path = entry.path();
1417 if path.extension().and_then(|e| e.to_str()) != Some("json") {
1418 continue;
1419 }
1420 let cand: crate::harness::HarnessCandidate =
1421 match std::fs::read_to_string(&path).map(|t| serde_json::from_str(&t)) {
1422 Ok(Ok(cand)) => cand,
1423 Ok(Err(e)) => {
1424 out.push(Finding::unreadable(
1425 "harness",
1426 &format!(
1427 "candidate {} did not parse",
1428 path.file_name().unwrap_or_default().to_string_lossy()
1429 ),
1430 format!("{}: {e}", path.display()),
1431 ));
1432 continue;
1433 }
1434 Err(e) => {
1435 out.push(Finding::unreadable(
1436 "harness",
1437 &format!(
1438 "candidate {} could not be read",
1439 path.file_name().unwrap_or_default().to_string_lossy()
1440 ),
1441 format!("{}: {e}", path.display()),
1442 ));
1443 continue;
1444 }
1445 };
1446 if !cand.pending() {
1447 continue;
1448 }
1449 let old_enough = chrono::DateTime::parse_from_rfc3339(&cand.created_at)
1450 .map(|t| {
1451 now.signed_duration_since(t.with_timezone(&chrono::Utc)) > STALE_CANDIDATE_AFTER
1452 })
1453 .unwrap_or(true);
1455 if old_enough {
1456 stale.push(format!("{} · {:?} {}", cand.id, cand.class, cand.change));
1457 }
1458 }
1459 if !stale.is_empty() {
1460 stale.sort();
1461 out.push(Finding {
1462 component: "harness".to_string(),
1463 severity: Severity::Attention,
1464 summary: format!(
1465 "{} harness candidate(s) staged for more than {}h",
1466 stale.len(),
1467 STALE_CANDIDATE_AFTER.num_hours()
1468 ),
1469 detail: stale.join("\n"),
1470 remedy: Some(Remedy {
1471 description: "review the staged candidates — doctor never accepts one".to_string(),
1472 argv: vec!["mecha".into(), "harness".into(), "list".into()],
1473 needs_terminal: false,
1474 }),
1475 });
1476 }
1477 out
1478}
1479
1480fn check_graph_nightly(store: &Path, now: DateTime<Utc>) -> Vec<Finding> {
1488 let mut out = Vec::new();
1489 let logs = store.join("logs");
1490 if !logs.is_dir() {
1491 return out;
1492 }
1493 let names: Vec<String> = match std::fs::read_dir(&logs) {
1494 Ok(entries) => entries
1495 .flatten()
1496 .filter_map(|e| e.file_name().to_str().map(String::from))
1497 .collect(),
1498 Err(e) => {
1499 out.push(Finding::unreadable(
1500 "graph",
1501 "the graph nightly logs",
1502 format!("{}: {e}", logs.display()),
1503 ));
1504 return out;
1505 }
1506 };
1507
1508 for (prefix, what) in GRAPH_NIGHTLIES {
1509 let newest = names
1510 .iter()
1511 .filter_map(|n| {
1512 n.strip_prefix(prefix)?
1513 .strip_suffix(".log")
1514 .and_then(|d| chrono::NaiveDate::parse_from_str(d, "%Y%m%d").ok())
1515 })
1516 .max();
1517 let Some(newest) = newest else { continue };
1520 let days_quiet = (now.date_naive() - newest).num_days();
1521 if days_quiet > 1 {
1522 out.push(Finding {
1523 component: "graph".to_string(),
1524 severity: Severity::Attention,
1525 summary: format!(
1526 "the graph nightly ({}) has not run for {days_quiet} days",
1527 prefix.trim_end_matches('-'),
1528 ),
1529 detail: format!(
1530 "{what} last wrote {}{}.log under {}; it logs every \
1531 run including deferred ones, so a missing day means the \
1532 script never started — cron reports that nowhere",
1533 prefix,
1534 newest.format("%Y%m%d"),
1535 logs.display(),
1536 ),
1537 remedy: Some(Remedy {
1538 description: "list the cron entries that fire the graph nightlies, \
1539 then run the silent one by hand and read its error"
1540 .to_string(),
1541 argv: vec!["crontab".into(), "-l".into()],
1542 needs_terminal: false,
1543 }),
1544 });
1545 }
1546 }
1547 out
1548}
1549
1550fn age_of(stamp: &str, now: DateTime<Utc>) -> Option<chrono::Duration> {
1555 DateTime::parse_from_rfc3339(stamp)
1556 .ok()
1557 .map(|at| now - at.with_timezone(&Utc))
1558}
1559
1560fn render_age(now: DateTime<Utc>, stamp: &str) -> String {
1562 match age_of(stamp, now) {
1563 Some(age) if age >= chrono::Duration::days(2) => format!("{}d ago", age.num_days()),
1564 Some(age) if age >= chrono::Duration::hours(1) => format!("{}h ago", age.num_hours()),
1565 Some(age) => format!("{}m ago", age.num_minutes().max(0)),
1566 None => stamp.to_string(),
1567 }
1568}
1569
1570#[cfg(test)]
1571mod tests {
1572 use super::*;
1573 use crate::agent::Taint;
1574 use crate::outbox::{OutboxItem, OutboxKind};
1575 use serde_json::json;
1576 use std::path::PathBuf;
1577
1578 fn utc(s: &str) -> DateTime<Utc> {
1579 DateTime::parse_from_rfc3339(s).unwrap().with_timezone(&Utc)
1580 }
1581
1582 const NOW: &str = "2026-08-14T12:00:00Z";
1583
1584 fn home(name: &str) -> PathBuf {
1586 let dir = std::env::temp_dir().join(format!(
1587 "mecha-doctor-test-{name}-{}-{:?}",
1588 std::process::id(),
1589 std::thread::current().id()
1590 ));
1591 let _ = std::fs::remove_dir_all(&dir);
1592 std::fs::create_dir_all(&dir).unwrap();
1593 dir
1594 }
1595
1596 fn write_marker(home: &Path, account: &str, body: &str) {
1597 let dir = home.join("mail").join(account);
1598 std::fs::create_dir_all(&dir).unwrap();
1599 std::fs::write(dir.join("auth_error.json"), body).unwrap();
1600 }
1601
1602 fn valid_marker() -> String {
1603 json!({
1604 "at": "2026-08-11T09:00:00Z",
1605 "message": "the refresh token was revoked — run `mecha-mail auth personal --provider google` to sign in again",
1606 })
1607 .to_string()
1608 }
1609
1610 fn pending_item(home: &Path, id: &str, created_at: &str, error: Option<&str>) {
1611 let item = OutboxItem {
1612 id: id.to_string(),
1613 status: "pending".into(),
1614 tool: "mail__send".into(),
1615 kind: OutboxKind::Message,
1616 args_before: json!({"to": "a@x.org"}),
1617 args: json!({"to": "a@x.org"}),
1618 summary: "mail__send to a@x.org".into(),
1619 session_id: None,
1620 workspace: None,
1621 taint: Taint::default(),
1622 created_at: created_at.to_string(),
1623 resolved_at: None,
1624 reason: None,
1625 error: error.map(String::from),
1626 };
1627 let dir = home.join("outbox");
1628 std::fs::create_dir_all(&dir).unwrap();
1629 std::fs::write(
1630 dir.join(format!("{id}.json")),
1631 serde_json::to_string_pretty(&item).unwrap(),
1632 )
1633 .unwrap();
1634 }
1635
1636 fn request(home: &Path, seq: i64, state: &str, drained_at: &str) {
1637 let dir = home.join("requests");
1638 std::fs::create_dir_all(&dir).unwrap();
1639 let record = json!({
1640 "seq": seq,
1641 "type_id": "meeting",
1642 "state": state,
1643 "created_at": drained_at,
1644 "drained_at": drained_at,
1645 "valid": true,
1646 "values": {},
1647 "free_text": [],
1648 });
1649 std::fs::write(
1650 dir.join(format!("{seq:010}-meeting.json")),
1651 record.to_string(),
1652 )
1653 .unwrap();
1654 }
1655
1656 fn trigger_file(home: &Path, name: &str, extra: &str) {
1657 let dir = home.join("triggers");
1658 std::fs::create_dir_all(&dir).unwrap();
1659 std::fs::write(
1660 dir.join(format!("{name}.toml")),
1661 format!(
1662 "schedule = \"0 7 * * *\"\nprompt = \"brief me\"\ntimezone = \"UTC\"\n\
1663 created_at = \"2026-08-01T00:00:00Z\"\n{extra}"
1664 ),
1665 )
1666 .unwrap();
1667 }
1668
1669 fn ledger_row(home: &Path, row: &serde_json::Value) {
1670 use std::io::Write;
1671 let dir = home.join("triggers");
1672 std::fs::create_dir_all(&dir).unwrap();
1673 let mut file = std::fs::OpenOptions::new()
1674 .create(true)
1675 .append(true)
1676 .open(dir.join("runs.jsonl"))
1677 .unwrap();
1678 writeln!(file, "{row}").unwrap();
1679 }
1680
1681 fn of<'a>(findings: &'a [Finding], component: &str) -> Vec<&'a Finding> {
1682 findings
1683 .iter()
1684 .filter(|f| f.component == component)
1685 .collect()
1686 }
1687
1688 #[test]
1689 fn a_dead_auth_marker_is_found_and_an_absent_one_is_not() {
1690 let home = home("dead-auth");
1691 write_marker(&home, "personal", &valid_marker());
1692 std::fs::create_dir_all(home.join("mail").join("dartmouth")).unwrap();
1694 std::fs::write(
1695 home.join("mail").join("accounts.toml"),
1696 "[[account]]\nname = \"personal\"\nprovider = \"google\"\n\
1697 [[account]]\nname = \"dartmouth\"\nprovider = \"outlook\"\n",
1698 )
1699 .unwrap();
1700
1701 let findings = examine(&home, utc(NOW));
1702 let mail = of(&findings, "mail");
1703 assert_eq!(mail.len(), 1, "{findings:#?}");
1704 assert_eq!(mail[0].severity, Severity::Broken);
1705 assert!(mail[0].summary.contains("personal"), "{}", mail[0].summary);
1706 let remedy = mail[0].remedy.as_ref().expect("a dead login has a way out");
1707 assert_eq!(
1708 remedy.argv,
1709 vec!["mecha-mail", "auth", "personal", "--provider", "google"]
1710 );
1711 assert!(
1712 remedy.needs_terminal,
1713 "an OAuth flow needs the real terminal"
1714 );
1715
1716 let _ = std::fs::remove_dir_all(&home);
1717 }
1718
1719 #[test]
1720 fn a_provider_the_registry_cannot_name_is_omitted_from_the_remedy_not_guessed() {
1721 let home = home("no-registry");
1722 write_marker(&home, "personal", &valid_marker());
1724
1725 let findings = examine(&home, utc(NOW));
1726 let mail = of(&findings, "mail");
1727 assert_eq!(mail.len(), 1);
1728 let remedy = mail[0].remedy.as_ref().unwrap();
1729 assert_eq!(remedy.argv, vec!["mecha-mail", "auth", "personal"]);
1730 assert!(
1733 mail[0].detail.contains("--provider google"),
1734 "{}",
1735 mail[0].detail
1736 );
1737
1738 let _ = std::fs::remove_dir_all(&home);
1739 }
1740
1741 #[test]
1746 fn a_marker_in_a_legacy_per_provider_store_is_found_and_proposes_import() {
1747 let home = home("legacy-auth");
1748 let dir = home.join("google");
1749 std::fs::create_dir_all(&dir).unwrap();
1750 std::fs::write(
1751 dir.join("auth_error.json"),
1752 json!({
1753 "at": "2026-08-11T09:00:00Z",
1754 "message": "account `google`: refresh token expired or revoked — run `mecha-mail auth google --provider google` (invalid_grant)",
1755 })
1756 .to_string(),
1757 )
1758 .unwrap();
1759
1760 let findings = examine(&home, utc(NOW));
1761 let mail = of(&findings, "mail");
1762 assert_eq!(mail.len(), 1, "{findings:#?}");
1763 assert_eq!(mail[0].severity, Severity::Broken);
1764 assert!(
1765 mail[0].summary.contains("legacy google"),
1766 "{}",
1767 mail[0].summary
1768 );
1769 assert!(
1772 mail[0]
1773 .detail
1774 .contains("run `mecha-mail auth google --provider google`"),
1775 "{}",
1776 mail[0].detail
1777 );
1778 let remedy = mail[0].remedy.as_ref().expect("a way out");
1779 assert_eq!(
1780 remedy.argv,
1781 vec!["mecha-mail", "import", "google", "--provider", "google"]
1782 );
1783
1784 let _ = std::fs::remove_dir_all(&home);
1785 }
1786
1787 #[test]
1788 fn an_unparseable_marker_is_a_store_unreadable_finding_not_a_crash() {
1789 let home = home("bad-marker");
1790 write_marker(&home, "personal", "{ this is not json");
1791
1792 let findings = examine(&home, utc(NOW));
1793 let mail = of(&findings, "mail");
1794 assert_eq!(mail.len(), 1, "{findings:#?}");
1795 assert!(
1796 mail[0].summary.starts_with("store unreadable:"),
1797 "{}",
1798 mail[0].summary
1799 );
1800 assert!(mail[0].summary.contains("personal"), "{}", mail[0].summary);
1801
1802 let _ = std::fs::remove_dir_all(&home);
1803 }
1804
1805 #[test]
1806 fn a_pending_item_with_an_error_is_broken_and_a_resolved_one_is_not() {
1807 let home = home("outbox-error");
1808 pending_item(
1809 &home,
1810 "20260814-000001-aaa",
1811 NOW,
1812 Some("server unreachable"),
1813 );
1814 let mut sent = json!({
1816 "id": "20260810-000001-bbb",
1817 "status": "sent",
1818 "tool": "mail__send",
1819 "args_before": {},
1820 "args": {},
1821 "summary": "mail__send",
1822 "created_at": "2026-08-01T00:00:00Z",
1823 });
1824 sent["error"] = json!(null);
1825 std::fs::write(
1826 home.join("outbox").join("20260810-000001-bbb.json"),
1827 sent.to_string(),
1828 )
1829 .unwrap();
1830
1831 let findings = examine(&home, utc(NOW));
1832 let outbox = of(&findings, "outbox");
1833 assert_eq!(outbox.len(), 1, "{findings:#?}");
1834 assert_eq!(outbox[0].severity, Severity::Broken);
1835 assert!(
1836 outbox[0]
1837 .summary
1838 .contains("release failed: server unreachable"),
1839 "{}",
1840 outbox[0].summary
1841 );
1842 let remedy = outbox[0].remedy.as_ref().unwrap();
1843 assert_eq!(remedy.argv, vec!["mecha", "outbox", "review"]);
1844
1845 let _ = std::fs::remove_dir_all(&home);
1846 }
1847
1848 #[test]
1849 fn a_pending_draft_is_stale_at_49_hours_and_not_at_47() {
1850 let home = home("outbox-stale");
1851 pending_item(&home, "20260812-110000-old", "2026-08-12T11:00:00Z", None);
1853 let findings = examine(&home, utc(NOW));
1854 let outbox = of(&findings, "outbox");
1855 assert_eq!(outbox.len(), 1, "{findings:#?}");
1856 assert_eq!(outbox[0].severity, Severity::Attention);
1857 assert!(outbox[0].summary.contains("pending for more than 48h"));
1858 assert_eq!(
1859 outbox[0].remedy.as_ref().unwrap().argv,
1860 vec!["mecha", "outbox", "review"],
1861 "the remedy is the review surface, never send"
1862 );
1863
1864 let fresh = home;
1866 let _ = std::fs::remove_dir_all(fresh.join("outbox"));
1867 pending_item(&fresh, "20260812-130000-new", "2026-08-12T13:00:00Z", None);
1868 let findings = examine(&fresh, utc(NOW));
1869 assert!(of(&findings, "outbox").is_empty(), "{findings:#?}");
1870
1871 let _ = std::fs::remove_dir_all(&fresh);
1872 }
1873
1874 fn harness_candidate(home: &Path, id: &str, created_at: &str, status: &str) {
1875 let dir = home.join("learning").join("harness").join("candidates");
1876 std::fs::create_dir_all(&dir).unwrap();
1877 let cand = crate::harness::HarnessCandidate {
1878 id: id.into(),
1879 created_at: created_at.into(),
1880 class: crate::candidate::ChangeClass::Config,
1881 change: "compact_at_tokens=24000".into(),
1882 metric: crate::candidate::Metric::CutShort,
1883 rationale: "test".into(),
1884 evidence: String::new(),
1885 model: None,
1886 status: status.into(),
1887 measurement: None,
1888 resolved_at: None,
1889 reason: None,
1890 };
1891 std::fs::write(
1892 dir.join(format!("{id}.json")),
1893 serde_json::to_string_pretty(&cand).unwrap(),
1894 )
1895 .unwrap();
1896 }
1897
1898 fn reflection_line(id: &str, origin: &str, processed: bool, created_at: &str) -> String {
1899 serde_json::json!({
1900 "id": id,
1901 "domain": "behavior",
1902 "session_id": "s",
1903 "trigger": "steer",
1904 "context": "",
1905 "intervention": "",
1906 "reflexion_text": "test",
1907 "is_processed": processed,
1908 "created_at": created_at,
1909 "origin": origin,
1910 })
1911 .to_string()
1912 }
1913
1914 fn write_reflections(home: &Path, lines: &[String]) {
1915 let dir = home.join("learning");
1916 std::fs::create_dir_all(&dir).unwrap();
1917 std::fs::write(dir.join("reflections.jsonl"), lines.join("\n")).unwrap();
1918 }
1919
1920 #[test]
1921 fn a_learner_fed_only_excluded_evidence_is_starved_and_a_met_floor_is_not() {
1922 let home = home("learning-starved");
1923 let mut lines: Vec<String> = (0..12)
1925 .map(|i| reflection_line(&format!("u{i}"), "untrusted", false, "2026-08-13T12:00:00Z"))
1926 .collect();
1927 lines.push(reflection_line(
1928 "c1",
1929 "clean",
1930 false,
1931 "2026-08-05T00:00:00Z",
1932 ));
1933 write_reflections(&home, &lines);
1934
1935 let findings = examine(&home, utc(NOW));
1936 let learning = of(&findings, "learning");
1937 assert_eq!(learning.len(), 1, "{findings:#?}");
1938 assert_eq!(learning[0].severity, Severity::Attention);
1939 assert!(
1940 learning[0].summary.contains("starved"),
1941 "{}",
1942 learning[0].summary
1943 );
1944 assert!(
1945 learning[0].summary.contains("12 of 13"),
1946 "{}",
1947 learning[0].summary
1948 );
1949 assert_eq!(
1950 learning[0].remedy.as_ref().unwrap().argv,
1951 vec!["mecha", "reflect", "--dry-run"],
1952 "the remedy shows classifications; nothing may loosen the gate"
1953 );
1954
1955 lines.push(reflection_line(
1957 "c2",
1958 "clean",
1959 false,
1960 "2026-08-06T00:00:00Z",
1961 ));
1962 lines.push(reflection_line(
1963 "c3",
1964 "clean",
1965 false,
1966 "2026-08-07T00:00:00Z",
1967 ));
1968 write_reflections(&home, &lines);
1969 let findings = examine(&home, utc(NOW));
1970 assert!(of(&findings, "learning").is_empty(), "{findings:#?}");
1971
1972 let _ = std::fs::remove_dir_all(&home);
1973 }
1974
1975 #[test]
1976 fn thin_or_dormant_exclusion_is_not_starvation() {
1977 let home = home("learning-thin");
1978 let lines: Vec<String> = (0..9)
1980 .map(|i| reflection_line(&format!("u{i}"), "untrusted", false, "2026-08-13T12:00:00Z"))
1981 .collect();
1982 write_reflections(&home, &lines);
1983 assert!(of(&examine(&home, utc(NOW)), "learning").is_empty());
1984
1985 let lines: Vec<String> = (0..12)
1988 .map(|i| reflection_line(&format!("u{i}"), "untrusted", false, "2026-05-01T12:00:00Z"))
1989 .collect();
1990 write_reflections(&home, &lines);
1991 assert!(of(&examine(&home, utc(NOW)), "learning").is_empty());
1992
1993 let mut lines: Vec<String> = (0..12)
1996 .map(|i| reflection_line(&format!("u{i}"), "untrusted", false, "2026-08-13T12:00:00Z"))
1997 .collect();
1998 for i in 0..3 {
1999 lines.push(reflection_line(
2000 &format!("p{i}"),
2001 "clean",
2002 true,
2003 "2026-08-05T00:00:00Z",
2004 ));
2005 }
2006 write_reflections(&home, &lines);
2007 let findings = examine(&home, utc(NOW));
2008 assert_eq!(of(&findings, "learning").len(), 1, "{findings:#?}");
2009
2010 let _ = std::fs::remove_dir_all(&home);
2011 }
2012
2013 #[test]
2014 fn a_staged_harness_candidate_is_stale_at_73_hours_and_not_at_71() {
2015 let home = home("harness-stale");
2016 harness_candidate(&home, "hc-old", "2026-08-11T11:00:00Z", "staged");
2018 harness_candidate(&home, "hc-done", "2026-08-01T00:00:00Z", "rejected");
2020 let findings = examine(&home, utc(NOW));
2021 let harness = of(&findings, "harness");
2022 assert_eq!(harness.len(), 1, "{findings:#?}");
2023 assert_eq!(harness[0].severity, Severity::Attention);
2024 assert!(harness[0].summary.contains("staged for more than 72h"));
2025 assert!(
2026 harness[0].detail.contains("hc-old"),
2027 "{}",
2028 harness[0].detail
2029 );
2030 assert_eq!(
2031 harness[0].remedy.as_ref().unwrap().argv,
2032 vec!["mecha", "harness", "list"],
2033 "the remedy is the review surface, never accept"
2034 );
2035
2036 let _ = std::fs::remove_dir_all(home.join("learning"));
2038 harness_candidate(&home, "hc-new", "2026-08-11T13:00:00Z", "staged");
2039 let findings = examine(&home, utc(NOW));
2040 assert!(of(&findings, "harness").is_empty(), "{findings:#?}");
2041
2042 let _ = std::fs::remove_dir_all(&home);
2043 }
2044
2045 #[test]
2046 fn a_failed_extraction_is_broken_at_any_age() {
2047 let home = home("frontdoor-failed");
2048 request(&home, 12, crate::frontdoor::EXTRACTION_FAILED, NOW);
2049
2050 let findings = examine(&home, utc(NOW));
2051 let front = of(&findings, "frontdoor");
2052 assert_eq!(front.len(), 1, "{findings:#?}");
2053 assert_eq!(front[0].severity, Severity::Broken);
2054 assert!(front[0].summary.contains("12"), "{}", front[0].summary);
2055 assert_eq!(
2056 front[0].remedy.as_ref().unwrap().argv,
2057 vec!["mecha", "frontdoor", "list"]
2058 );
2059
2060 let _ = std::fs::remove_dir_all(&home);
2061 }
2062
2063 #[test]
2064 fn a_request_waiting_on_me_is_stale_at_73_hours_and_not_at_71() {
2065 let home = home("frontdoor-stale");
2066 request(
2068 &home,
2069 1,
2070 crate::frontdoor::AWAITING_ME,
2071 "2026-08-11T11:00:00Z",
2072 );
2073 let findings = examine(&home, utc(NOW));
2074 let front = of(&findings, "frontdoor");
2075 assert_eq!(front.len(), 1, "{findings:#?}");
2076 assert_eq!(front[0].severity, Severity::Attention);
2077 assert!(front[0].summary.contains("waiting on you"));
2078
2079 let _ = std::fs::remove_dir_all(home.join("requests"));
2081 request(
2082 &home,
2083 2,
2084 crate::frontdoor::AWAITING_ME,
2085 "2026-08-11T13:00:00Z",
2086 );
2087 let findings = examine(&home, utc(NOW));
2088 assert!(of(&findings, "frontdoor").is_empty(), "{findings:#?}");
2089
2090 let _ = std::fs::remove_dir_all(home.join("requests"));
2092 request(
2093 &home,
2094 3,
2095 crate::frontdoor::NEEDS_INFO,
2096 "2026-08-01T00:00:00Z",
2097 );
2098 let findings = examine(&home, utc(NOW));
2099 assert!(of(&findings, "frontdoor").is_empty(), "{findings:#?}");
2100
2101 let _ = std::fs::remove_dir_all(&home);
2102 }
2103
2104 #[test]
2108 fn a_triaged_request_nothing_will_revisit_goes_stale() {
2109 let home = home("frontdoor-triaged");
2110 request(&home, 4, crate::frontdoor::TRIAGED, "2026-08-11T11:00:00Z");
2112 request(
2114 &home,
2115 5,
2116 crate::frontdoor::NEEDS_INFO,
2117 "2026-08-01T00:00:00Z",
2118 );
2119
2120 let findings = examine(&home, utc(NOW));
2121 let front = of(&findings, "frontdoor");
2122 assert_eq!(front.len(), 1, "{findings:#?}");
2123 assert_eq!(front[0].severity, Severity::Attention);
2124 assert!(front[0].detail.contains("triaged"), "{}", front[0].detail);
2125 assert!(
2126 !front[0].detail.contains("needs_info"),
2127 "needs_info waits on the requester: {}",
2128 front[0].detail
2129 );
2130
2131 let _ = std::fs::remove_dir_all(&home);
2132 }
2133
2134 #[test]
2135 fn a_trigger_whose_last_run_failed_is_flagged_with_the_manual_probe() {
2136 let home = home("trigger-failed");
2137 trigger_file(&home, "morning", "");
2138 ledger_row(
2139 &home,
2140 &json!({
2141 "trigger": "morning",
2142 "slot": "2026-08-13T07:00:00Z",
2143 "started_at": "2026-08-13T07:00:01Z",
2144 "status": "ok",
2145 "summary": "fine",
2146 }),
2147 );
2148 ledger_row(
2149 &home,
2150 &json!({
2151 "trigger": "morning",
2152 "slot": "2026-08-14T07:00:00Z",
2153 "started_at": "2026-08-14T07:00:01Z",
2154 "status": "error",
2155 "error": "provider unreachable",
2156 }),
2157 );
2158
2159 let findings = examine(&home, utc(NOW));
2160 let triggers = of(&findings, "triggers");
2161 assert_eq!(triggers.len(), 1, "{findings:#?}");
2162 assert_eq!(triggers[0].severity, Severity::Attention);
2163 assert!(triggers[0].summary.contains("morning"));
2164 assert!(triggers[0].detail.contains("provider unreachable"));
2165 assert_eq!(
2166 triggers[0].remedy.as_ref().unwrap().argv,
2167 vec!["mecha", "trigger", "run", "morning"],
2168 "a manual run is the safe probe: it never advances the schedule"
2169 );
2170
2171 let _ = std::fs::remove_dir_all(&home);
2172 }
2173
2174 #[test]
2178 fn a_skip_row_after_a_failed_run_does_not_hide_the_failure() {
2179 let home = home("trigger-skip-hides-error");
2180 trigger_file(&home, "morning", "");
2181 ledger_row(
2182 &home,
2183 &json!({
2184 "trigger": "morning",
2185 "slot": "2026-08-13T07:00:00Z",
2186 "started_at": "2026-08-13T07:00:01Z",
2187 "status": "error",
2188 "error": "provider unreachable",
2189 }),
2190 );
2191 ledger_row(
2192 &home,
2193 &json!({
2194 "trigger": "morning",
2195 "slot": "2026-08-14T07:00:00Z",
2196 "started_at": "2026-08-14T07:00:01Z",
2197 "status": "skipped-stale",
2198 }),
2199 );
2200
2201 let findings = examine(&home, utc(NOW));
2202 let triggers = of(&findings, "triggers");
2203 assert_eq!(triggers.len(), 1, "{findings:#?}");
2204 assert!(
2205 triggers[0].summary.contains("most recent run failed"),
2206 "{}",
2207 triggers[0].summary
2208 );
2209 assert!(triggers[0].detail.contains("provider unreachable"));
2210
2211 let _ = std::fs::remove_dir_all(&home);
2212 }
2213
2214 #[test]
2215 fn an_ok_run_followed_by_a_skip_is_healthy() {
2216 let home = home("trigger-ok-then-skip");
2217 trigger_file(&home, "morning", "");
2218 ledger_row(
2219 &home,
2220 &json!({
2221 "trigger": "morning",
2222 "slot": "2026-08-13T07:00:00Z",
2223 "started_at": "2026-08-13T07:00:01Z",
2224 "status": "ok",
2225 }),
2226 );
2227 ledger_row(
2228 &home,
2229 &json!({
2230 "trigger": "morning",
2231 "slot": "2026-08-14T07:00:00Z",
2232 "started_at": "2026-08-14T07:00:01Z",
2233 "status": "skipped-overlap",
2234 }),
2235 );
2236
2237 let findings = examine(&home, utc(NOW));
2238 assert!(of(&findings, "triggers").is_empty(), "{findings:#?}");
2239
2240 let _ = std::fs::remove_dir_all(&home);
2241 }
2242
2243 #[test]
2244 fn a_trigger_quietly_failing_a_third_of_its_calls_is_reported() {
2245 let home = home("trigger-tool-errors");
2248 trigger_file(&home, "morning", "");
2249 for day in 10..15 {
2250 ledger_row(
2251 &home,
2252 &json!({
2253 "trigger": "morning",
2254 "slot": format!("2026-08-{day}T07:00:00Z"),
2255 "started_at": format!("2026-08-{day}T07:00:01Z"),
2256 "status": "ok",
2257 "summary": "briefed",
2258 "tool_calls": 6,
2259 "tool_errors": 3,
2260 }),
2261 );
2262 }
2263
2264 let findings = examine(&home, utc(NOW));
2265 let triggers = of(&findings, "triggers");
2266 assert_eq!(triggers.len(), 1, "{findings:#?}");
2267 assert_eq!(triggers[0].severity, Severity::Attention);
2268 assert!(
2269 triggers[0].summary.contains("15 of 30"),
2270 "{}",
2271 triggers[0].summary
2272 );
2273 assert_eq!(
2274 triggers[0].remedy.as_ref().unwrap().argv,
2275 vec!["mecha", "trigger", "show", "morning"],
2276 "reading is the remedy — what to change is in the transcript"
2277 );
2278
2279 let _ = std::fs::remove_dir_all(&home);
2280 }
2281
2282 #[test]
2283 fn a_handful_of_failed_calls_is_not_a_trend() {
2284 let home = home("trigger-tool-errors-quiet");
2289 trigger_file(&home, "morning", "");
2290 ledger_row(
2292 &home,
2293 &json!({
2294 "trigger": "morning",
2295 "slot": "2026-08-14T07:00:00Z",
2296 "started_at": "2026-08-14T07:00:01Z",
2297 "status": "ok",
2298 "tool_calls": 3,
2299 "tool_errors": 3,
2300 }),
2301 );
2302 assert!(of(&examine(&home, utc(NOW)), "triggers").is_empty());
2303
2304 ledger_row(
2306 &home,
2307 &json!({
2308 "trigger": "morning",
2309 "slot": "2026-08-15T07:00:00Z",
2310 "started_at": "2026-08-15T07:00:01Z",
2311 "status": "ok",
2312 "tool_calls": 40,
2313 "tool_errors": 4,
2314 }),
2315 );
2316 assert!(of(&examine(&home, utc(NOW)), "triggers").is_empty());
2317
2318 let _ = std::fs::remove_dir_all(&home);
2319 }
2320
2321 #[test]
2322 fn a_trigger_that_stopped_doing_anything_is_reported() {
2323 let home = home("trigger-stopped-working");
2326 trigger_file(&home, "morning", "");
2327 for day in 10..14 {
2328 ledger_row(
2329 &home,
2330 &json!({
2331 "trigger": "morning",
2332 "slot": format!("2026-08-{day}T07:00:00Z"),
2333 "started_at": format!("2026-08-{day}T07:00:01Z"),
2334 "status": "ok",
2335 "tool_calls": 8,
2336 "tool_errors": 0,
2337 }),
2338 );
2339 }
2340 ledger_row(
2341 &home,
2342 &json!({
2343 "trigger": "morning",
2344 "slot": "2026-08-14T07:00:00Z",
2345 "started_at": "2026-08-14T07:00:01Z",
2346 "status": "ok",
2347 "summary": "nothing to report",
2348 "tool_calls": 0,
2349 "tool_errors": 0,
2350 }),
2351 );
2352
2353 let findings = examine(&home, utc(NOW));
2354 let triggers = of(&findings, "triggers");
2355 assert_eq!(triggers.len(), 1, "{findings:#?}");
2356 assert!(
2357 triggers[0].summary.contains("did no work"),
2358 "{}",
2359 triggers[0].summary
2360 );
2361 assert!(triggers[0].detail.contains("made 32"));
2362
2363 let _ = std::fs::remove_dir_all(&home);
2364 }
2365
2366 #[test]
2367 fn a_trigger_that_never_needed_tools_is_not_broken_for_not_using_them() {
2368 let home = home("trigger-never-used-tools");
2373 trigger_file(&home, "haiku", "");
2374 for day in 10..15 {
2375 ledger_row(
2376 &home,
2377 &json!({
2378 "trigger": "haiku",
2379 "slot": format!("2026-08-{day}T07:00:00Z"),
2380 "started_at": format!("2026-08-{day}T07:00:01Z"),
2381 "status": "ok",
2382 "tool_calls": 0,
2383 "tool_errors": 0,
2384 }),
2385 );
2386 }
2387 assert!(of(&examine(&home, utc(NOW)), "triggers").is_empty());
2388
2389 let _ = std::fs::remove_dir_all(&home);
2390 }
2391
2392 #[test]
2393 fn a_failed_run_that_did_no_work_is_reported_once_not_twice() {
2394 let home = home("trigger-failed-no-work");
2399 trigger_file(&home, "morning", "");
2400 for day in 10..14 {
2401 ledger_row(
2402 &home,
2403 &json!({
2404 "trigger": "morning",
2405 "slot": format!("2026-08-{day}T07:00:00Z"),
2406 "started_at": format!("2026-08-{day}T07:00:01Z"),
2407 "status": "ok",
2408 "tool_calls": 8,
2409 "tool_errors": 0,
2410 }),
2411 );
2412 }
2413 ledger_row(
2414 &home,
2415 &json!({
2416 "trigger": "morning",
2417 "slot": "2026-08-14T07:00:00Z",
2418 "started_at": "2026-08-14T07:00:01Z",
2419 "status": "error",
2420 "error": "provider unreachable",
2421 "tool_calls": 0,
2422 "tool_errors": 0,
2423 }),
2424 );
2425
2426 let triggers = of(&examine(&home, utc(NOW)), "triggers")
2427 .into_iter()
2428 .cloned()
2429 .collect::<Vec<_>>();
2430 assert_eq!(triggers.len(), 1, "{triggers:#?}");
2431 assert!(triggers[0].detail.contains("provider unreachable"));
2432
2433 let _ = std::fs::remove_dir_all(&home);
2434 }
2435
2436 fn runs_in(
2439 home: &Path,
2440 model: &str,
2441 n: usize,
2442 stats: impl Fn(usize) -> crate::session::RunStats,
2443 ) {
2444 let dir = home.join("sessions");
2445 std::fs::create_dir_all(&dir).unwrap();
2446 for i in 0..n {
2447 let session = crate::session::Session::create(
2448 &dir,
2449 crate::session::SessionMeta {
2450 id: format!("2026080{}T00000{i:03}-{model}", 1 + i % 9),
2455 created_at: utc(NOW),
2456 provider: "local".into(),
2457 model: model.to_string(),
2458 workspace: std::path::PathBuf::from("/tmp"),
2459 title: None,
2460 },
2461 )
2462 .unwrap();
2463 session
2464 .append(&crate::session::Record::Outcome(stats(i)))
2465 .unwrap();
2466 }
2467 }
2468
2469 fn run_stats(
2470 calls: u32,
2471 errors: u32,
2472 ended_failed: bool,
2473 cause: crate::agent::StopCause,
2474 ) -> crate::session::RunStats {
2475 crate::session::RunStats {
2476 tool_calls: calls,
2477 tool_errors: errors,
2478 ended_on_failed_call: ended_failed,
2479 stop_cause: Some(cause),
2480 ..Default::default()
2481 }
2482 }
2483
2484 #[test]
2485 fn a_model_that_keeps_finishing_over_failures_is_reported() {
2486 use crate::agent::StopCause;
2487 let home = home("runs-ended-on-failure");
2488 runs_in(&home, "tiny-local", 30, |i| {
2490 run_stats(6, 0, i % 3 == 0, StopCause::Completed)
2491 });
2492
2493 let all = examine(&home, utc(NOW));
2494 let findings = of(&all, "runs");
2495 assert_eq!(findings.len(), 1, "{findings:#?}");
2496 assert!(
2497 findings[0].summary.contains("tiny-local"),
2498 "{}",
2499 findings[0].summary
2500 );
2501 assert!(
2502 findings[0].summary.contains("33%"),
2503 "{}",
2504 findings[0].summary
2505 );
2506 assert_eq!(
2507 findings[0].remedy.as_ref().unwrap().argv,
2508 vec!["mecha", "sessions", "health", "--days", "30"],
2509 "reading is the remedy; doctor never decides what to change"
2510 );
2511
2512 let _ = std::fs::remove_dir_all(&home);
2513 }
2514
2515 #[test]
2516 fn a_cancelled_run_is_not_the_harness_cutting_it_short() {
2517 use crate::agent::StopCause;
2518 let home = home("runs-interrupted");
2521 runs_in(&home, "tiny-local", 30, |_| {
2522 run_stats(6, 0, false, StopCause::Interrupted)
2523 });
2524 let findings = examine(&home, utc(NOW));
2525 assert!(of(&findings, "runs").is_empty());
2526 let _ = std::fs::remove_dir_all(&home);
2527 }
2528
2529 #[test]
2530 fn a_turn_ceiling_stopping_a_quarter_of_runs_is_a_finding() {
2531 use crate::agent::StopCause;
2532 let home = home("runs-max-turns");
2533 runs_in(&home, "tiny-local", 30, |_| {
2534 run_stats(6, 0, false, StopCause::MaxTurns)
2535 });
2536 let all = examine(&home, utc(NOW));
2537 let findings = of(&all, "runs");
2538 assert_eq!(findings.len(), 1, "{findings:#?}");
2539 assert!(
2540 findings[0].summary.contains("cut"),
2541 "{}",
2542 findings[0].summary
2543 );
2544 let _ = std::fs::remove_dir_all(&home);
2545 }
2546
2547 #[test]
2548 fn a_thin_sample_of_one_model_says_nothing_about_it() {
2549 use crate::agent::StopCause;
2550 let home = home("runs-thin");
2553 runs_in(&home, "tiny-local", 19, |_| {
2554 run_stats(6, 6, true, StopCause::MaxTurns)
2555 });
2556 let all = examine(&home, utc(NOW));
2557 assert!(of(&all, "runs").is_empty());
2558 let _ = std::fs::remove_dir_all(&home);
2559 }
2560
2561 #[test]
2562 fn a_bad_model_does_not_drag_a_good_one_into_a_finding() {
2563 use crate::agent::StopCause;
2564 let home = home("runs-two-models");
2567 runs_in(&home, "steady", 25, |_| {
2568 run_stats(10, 0, false, StopCause::Completed)
2569 });
2570 runs_in(&home, "flaky", 25, |_| {
2571 run_stats(10, 9, false, StopCause::Completed)
2572 });
2573
2574 let all = examine(&home, utc(NOW));
2575 let findings = of(&all, "runs");
2576 assert_eq!(findings.len(), 1, "{findings:#?}");
2577 assert!(
2578 findings[0].summary.contains("flaky"),
2579 "{}",
2580 findings[0].summary
2581 );
2582 assert!(
2583 !findings[0].summary.contains("steady"),
2584 "the healthy model was named in a finding about the other one"
2585 );
2586 let _ = std::fs::remove_dir_all(&home);
2587 }
2588
2589 #[test]
2590 fn a_ledger_written_before_the_counts_existed_reports_nothing() {
2591 let home = home("trigger-tool-errors-bare");
2594 trigger_file(&home, "morning", "");
2595 ledger_row(
2596 &home,
2597 &json!({
2598 "trigger": "morning",
2599 "slot": "2026-08-14T07:00:00Z",
2600 "started_at": "2026-08-14T07:00:01Z",
2601 "status": "ok",
2602 }),
2603 );
2604 assert!(of(&examine(&home, utc(NOW)), "triggers").is_empty());
2605
2606 let _ = std::fs::remove_dir_all(&home);
2607 }
2608
2609 #[test]
2610 fn a_disabled_trigger_is_nobody_s_emergency() {
2611 let home = home("trigger-disabled");
2612 trigger_file(&home, "morning", "enabled = false\n");
2613 ledger_row(
2614 &home,
2615 &json!({
2616 "trigger": "morning",
2617 "started_at": "2026-08-14T07:00:01Z",
2618 "status": "error",
2619 "error": "boom",
2620 }),
2621 );
2622 let findings = examine(&home, utc(NOW));
2623 assert!(of(&findings, "triggers").is_empty(), "{findings:#?}");
2624 let _ = std::fs::remove_dir_all(&home);
2625 }
2626
2627 #[test]
2628 fn a_catch_up_trigger_whose_slots_stopped_advancing_names_the_daemon() {
2629 let home = home("trigger-stale");
2630 trigger_file(&home, "morning", "");
2631 ledger_row(
2634 &home,
2635 &json!({
2636 "trigger": "morning",
2637 "slot": "2026-08-09T07:00:00Z",
2638 "started_at": "2026-08-09T07:00:01Z",
2639 "status": "ok",
2640 }),
2641 );
2642
2643 let findings = examine(&home, utc(NOW));
2644 let triggers = of(&findings, "triggers");
2645 assert_eq!(triggers.len(), 1, "{findings:#?}");
2646 assert_eq!(triggers[0].severity, Severity::Attention);
2647 assert!(triggers[0].summary.contains("missed more than two slots"));
2648 assert!(
2649 triggers[0].detail.contains("daemon"),
2650 "{}",
2651 triggers[0].detail
2652 );
2653 assert!(
2654 triggers[0].remedy.is_none(),
2655 "running the trigger would not restart the scheduler"
2656 );
2657
2658 ledger_row(
2660 &home,
2661 &json!({
2662 "trigger": "morning",
2663 "slot": "2026-08-14T07:00:00Z",
2664 "started_at": "2026-08-14T07:00:01Z",
2665 "status": "ok",
2666 }),
2667 );
2668 let findings = examine(&home, utc(NOW));
2669 assert!(of(&findings, "triggers").is_empty(), "{findings:#?}");
2670
2671 let _ = std::fs::remove_dir_all(&home);
2672 }
2673
2674 #[cfg(unix)]
2677 #[test]
2678 fn one_poisoned_store_does_not_suppress_the_others() {
2679 use std::os::unix::fs::PermissionsExt;
2680 if unsafe { libc::geteuid() } == 0 {
2683 return;
2684 }
2685
2686 let home = home("poisoned");
2687 write_marker(&home, "personal", &valid_marker());
2688 let outbox = home.join("outbox");
2689 std::fs::create_dir_all(&outbox).unwrap();
2690 std::fs::set_permissions(&outbox, std::fs::Permissions::from_mode(0o000)).unwrap();
2691
2692 let findings = examine(&home, utc(NOW));
2693
2694 std::fs::set_permissions(&outbox, std::fs::Permissions::from_mode(0o700)).unwrap();
2696
2697 let mail = of(&findings, "mail");
2698 assert_eq!(mail.len(), 1, "the mail finding survived: {findings:#?}");
2699 assert_eq!(mail[0].severity, Severity::Broken);
2700 let broken_store = of(&findings, "outbox");
2701 assert_eq!(broken_store.len(), 1, "{findings:#?}");
2702 assert!(
2703 broken_store[0].summary.starts_with("store unreadable:"),
2704 "{}",
2705 broken_store[0].summary
2706 );
2707
2708 let _ = std::fs::remove_dir_all(&home);
2709 }
2710
2711 #[test]
2718 fn the_golden_marker_literal_parses_into_the_dead_auth_finding() {
2719 const GOLDEN: &str = r#"{
2720 "at": "2026-08-11T09:00:00Z",
2721 "message": "account `personal`: refresh token expired or revoked — run `mecha-mail auth personal --provider google` (invalid_grant: Token has been revoked.)"
2722}"#;
2723 let home = home("golden-marker");
2724 write_marker(&home, "personal", GOLDEN);
2725
2726 let findings = examine(&home, utc(NOW));
2727 let mail = of(&findings, "mail");
2728 assert_eq!(mail.len(), 1, "{findings:#?}");
2729 assert_eq!(mail[0].severity, Severity::Broken);
2730 assert!(
2731 mail[0].detail.contains("since 2026-08-11T09:00:00Z"),
2732 "the marker's `at` must reach the detail: {}",
2733 mail[0].detail
2734 );
2735 assert!(
2736 mail[0]
2737 .detail
2738 .contains("run `mecha-mail auth personal --provider google`"),
2739 "the marker's `message` must reach the detail: {}",
2740 mail[0].detail
2741 );
2742
2743 let _ = std::fs::remove_dir_all(&home);
2744 }
2745
2746 #[test]
2747 fn findings_sort_broken_first() {
2748 let mut findings = vec![
2749 Finding {
2750 component: "outbox".into(),
2751 severity: Severity::Attention,
2752 summary: "stale".into(),
2753 detail: String::new(),
2754 remedy: None,
2755 },
2756 Finding {
2757 component: "mail".into(),
2758 severity: Severity::Broken,
2759 summary: "dead".into(),
2760 detail: String::new(),
2761 remedy: None,
2762 },
2763 ];
2764 sort(&mut findings);
2765 assert_eq!(findings[0].severity, Severity::Broken);
2766 }
2767
2768 #[test]
2769 fn an_empty_home_is_healthy() {
2770 let home = home("empty");
2771 assert!(examine(&home, utc(NOW)).is_empty());
2772 let _ = std::fs::remove_dir_all(&home);
2773 }
2774
2775 fn graph_store(name: &str) -> PathBuf {
2780 let store = home(name).join(".mecha-graph");
2781 std::fs::create_dir_all(store.join("logs")).unwrap();
2782 store
2783 }
2784
2785 fn nightly_log(store: &Path, file: &str) {
2786 std::fs::write(store.join("logs").join(file), "ran\n").unwrap();
2787 }
2788
2789 #[test]
2793 fn a_graph_nightly_that_stopped_writing_logs_is_a_finding() {
2794 let store = graph_store("graph-stale");
2795 nightly_log(&store, "nightly-20260812.log");
2796 let findings = check_graph_nightly(&store, utc(NOW));
2797 assert_eq!(findings.len(), 1);
2798 assert_eq!(findings[0].component, "graph");
2799 assert_eq!(findings[0].severity, Severity::Attention);
2800 assert!(
2801 findings[0].summary.contains("2 days"),
2802 "{}",
2803 findings[0].summary
2804 );
2805 assert!(
2806 findings[0].detail.contains("nightly-20260812.log"),
2807 "{}",
2808 findings[0].detail
2809 );
2810 }
2811
2812 #[test]
2813 fn yesterdays_log_is_healthy_because_todays_slot_may_not_have_fired() {
2814 let store = graph_store("graph-yesterday");
2815 nightly_log(&store, "nightly-20260813.log");
2816 nightly_log(&store, "mecha-nightly-20260813.log");
2817 assert!(check_graph_nightly(&store, utc(NOW)).is_empty());
2818 }
2819
2820 #[test]
2824 fn each_nightly_family_is_judged_alone() {
2825 let store = graph_store("graph-split");
2826 nightly_log(&store, "nightly-20260814.log");
2827 nightly_log(&store, "mecha-nightly-20260811.log");
2828 let findings = check_graph_nightly(&store, utc(NOW));
2829 assert_eq!(findings.len(), 1);
2830 assert!(
2831 findings[0].summary.contains("mecha-nightly"),
2832 "{}",
2833 findings[0].summary
2834 );
2835 }
2836
2837 #[test]
2840 fn the_shorter_prefix_does_not_claim_the_longer_familys_logs() {
2841 let store = graph_store("graph-prefix");
2842 nightly_log(&store, "mecha-nightly-20260814.log");
2843 nightly_log(&store, "nightly-20260810.log");
2844 let findings = check_graph_nightly(&store, utc(NOW));
2845 assert_eq!(findings.len(), 1);
2846 assert!(
2847 findings[0].detail.contains("nightly-20260810.log"),
2848 "{}",
2849 findings[0].detail
2850 );
2851 }
2852
2853 #[test]
2856 fn a_graph_that_never_ran_is_not_a_finding() {
2857 let missing = home("graph-missing").join(".mecha-graph");
2858 assert!(check_graph_nightly(&missing, utc(NOW)).is_empty());
2859
2860 let empty = graph_store("graph-empty");
2861 assert!(check_graph_nightly(&empty, utc(NOW)).is_empty());
2862
2863 let odd = graph_store("graph-odd-names");
2864 nightly_log(&odd, "nightly-garbage.log");
2865 nightly_log(&odd, "gossip-20260812.jsonl");
2866 assert!(check_graph_nightly(&odd, utc(NOW)).is_empty());
2867 }
2868
2869 #[test]
2871 fn examine_reads_the_graph_store_beside_the_home() {
2872 let scratch = home("graph-sibling");
2873 let mecha_home = scratch.join(".mecha");
2874 std::fs::create_dir_all(&mecha_home).unwrap();
2875 let store = scratch.join(".mecha-graph");
2876 std::fs::create_dir_all(store.join("logs")).unwrap();
2877 nightly_log(&store, "nightly-20260810.log");
2878 let findings = examine(&mecha_home, utc(NOW));
2879 assert_eq!(findings.len(), 1);
2880 assert_eq!(findings[0].component, "graph");
2881 let _ = std::fs::remove_dir_all(&scratch);
2882 }
2883}