Skip to main content

mecha_core/
doctor.rs

1//! `mecha doctor` — every store's distress, read in one pass.
2//!
3//! The incident this exists to end: a revoked OAuth token killed the
4//! scheduling pipeline for three days, and every component recorded its
5//! trouble correctly *in its own store* — an `auth_error.json` marker beside
6//! the credentials, outbox items pending with a release error, frontdoor
7//! requests parked in `awaiting_me`, trigger-ledger rows — while the operator
8//! learned nothing, because nothing reads **across** the stores. Doctor is
9//! that read.
10//!
11//! Two rules carry the design:
12//!
13//! - **Doctor is an observer, never load-bearing.** No network, no model, no
14//!   tokens — and no writes: the stores are read directly rather than through
15//!   the store constructors, because those create and re-chmod their
16//!   directories on open, and an examination that heals the permissions it
17//!   was about to report is measuring itself. Every check is individually
18//!   best-effort: an unreadable or unparseable store is itself a finding
19//!   ("store unreadable: <why>"), never a crash, and one check's failure
20//!   never stops the others.
21//! - **Fixes go through existing commands only.** A [`Remedy`] is an argv —
22//!   `mecha-mail auth personal --provider google`, `mecha outbox review` —
23//!   never a direct mutation of a store. In particular doctor never releases
24//!   an outbox draft: the remedy for stuck drafts is opening the review
25//!   surface, full stop.
26//!
27//! The checks are pure functions over injected store roots and an injected
28//! `now`, which is what makes "a 49-hour-old pending draft" a unit test
29//! instead of a two-day wait.
30
31use chrono::{DateTime, Utc};
32use serde::{Deserialize, Serialize};
33use std::collections::BTreeMap;
34use std::path::Path;
35
36/// How bad a finding is. Declared broken-first so the derived order is the
37/// display order: what is broken outranks what merely wants attention.
38#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Serialize, Deserialize)]
39#[serde(rename_all = "lowercase")]
40pub enum Severity {
41    /// Something is failing right now — a dead login, a release that errored.
42    Broken,
43    /// Nothing is failing, but something has sat unresolved long enough that
44    /// silence is the more likely explanation than intent.
45    Attention,
46}
47
48impl Severity {
49    pub fn as_str(&self) -> &'static str {
50        match self {
51            Severity::Broken => "broken",
52            Severity::Attention => "attention",
53        }
54    }
55}
56
57/// A proposed fix: an existing command, never a store mutation.
58#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
59pub struct Remedy {
60    /// One line saying what running it does — and, where ordering matters,
61    /// what to do first.
62    pub description: String,
63    /// The command as an argv, ready to spawn. Never empty.
64    pub argv: Vec<String>,
65    /// Whether the command needs the real terminal — an OAuth flow, an
66    /// `$EDITOR` — and must therefore inherit stdin and the screen rather
67    /// than being run with its output captured.
68    pub needs_terminal: bool,
69}
70
71/// One observation: which component, how bad, what, and the way out if one
72/// is known.
73#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
74pub struct Finding {
75    pub component: String,
76    pub severity: Severity,
77    pub summary: String,
78    pub detail: String,
79    #[serde(default, skip_serializing_if = "Option::is_none")]
80    pub remedy: Option<Remedy>,
81}
82
83impl Finding {
84    /// The observer rule made a constructor: a store doctor cannot read is a
85    /// finding about that store, never an error that stops the other checks.
86    fn unreadable(component: &str, what: &str, why: impl std::fmt::Display) -> Finding {
87        Finding {
88            component: component.to_string(),
89            severity: Severity::Attention,
90            summary: format!("store unreadable: {what}"),
91            detail: why.to_string(),
92            remedy: None,
93        }
94    }
95}
96
97/// A pending draft older than this with no error has most likely been
98/// forgotten rather than deliberately parked.
99const STUCK_DRAFT_AFTER: chrono::Duration = chrono::Duration::hours(48);
100
101/// A frontdoor request waiting on the user for longer than this is the
102/// stranger-facing silence the front door exists to prevent.
103const STALE_REQUEST_AFTER: chrono::Duration = chrono::Duration::hours(72);
104
105/// Examine every store under `home` and report what is wrong.
106///
107/// `now` is injected for testability; nothing here consults the clock.
108/// Best-effort throughout: each check appends what it found, a failed check
109/// appends a finding about the failure, and no check can stop another.
110pub fn examine(home: &Path, now: DateTime<Utc>) -> Vec<Finding> {
111    let mut findings = Vec::new();
112    findings.extend(check_mail(&home.join("mail")));
113    findings.extend(check_legacy_mail(home));
114    findings.extend(check_outbox(&home.join("outbox"), now));
115    findings.extend(check_frontdoor(&home.join("requests"), now));
116    findings.extend(check_triggers(&home.join("triggers"), now));
117    findings.extend(check_runs(&home.join("sessions")));
118    // The graph store is `~/.mecha-graph`, a hidden sibling of the mecha home
119    // by that store's own convention — resolved relative to `home` so a test
120    // (or a relocated home) carries its sibling with it.
121    if let Some(parent) = home.parent() {
122        findings.extend(check_graph_nightly(&parent.join(".mecha-graph"), now));
123    }
124    sort(&mut findings);
125    findings
126}
127
128/// Severity first, then component, then insertion order — the shape both the
129/// renderer and the JSON output present.
130pub fn sort(findings: &mut [Finding]) {
131    findings.sort_by(|a, b| {
132        a.severity
133            .cmp(&b.severity)
134            .then_with(|| a.component.cmp(&b.component))
135    });
136}
137
138// --- dead mail auth ---------------------------------------------------------
139
140/// `auth_error.json`, structurally. The writer is `mecha-mail`'s token
141/// lifecycle; the seam is a file of JSON exactly like the frontdoor's
142/// directory-of-JSON, which is why core takes no `mecha-mail` dependency to
143/// read it.
144#[derive(Debug, Deserialize)]
145struct AuthMarker {
146    at: String,
147    message: String,
148}
149
150/// `accounts.toml`, structurally, for the same reason. Only the fields doctor
151/// needs; unknown ones are ignored.
152#[derive(Debug, Default, Deserialize)]
153struct MailAccounts {
154    #[serde(default, rename = "account")]
155    accounts: Vec<MailAccount>,
156}
157
158#[derive(Debug, Deserialize)]
159struct MailAccount {
160    name: String,
161    provider: String,
162    /// Declared lifetime of the refresh credential, in days. See
163    /// `mecha_mail::accounts::AccountEntry::grant_lifetime_days` — absent
164    /// means no known expiry, and no warning.
165    #[serde(default)]
166    grant_lifetime_days: Option<u32>,
167}
168
169/// Scan `<mail>/*/auth_error.json`. Presence means a *permanent* refresh
170/// failure — the marker is written on `invalid_grant` and cleared by the next
171/// successful credential save — so a marker is Broken, not a maybe.
172fn check_mail(mail: &Path) -> Vec<Finding> {
173    let mut out = Vec::new();
174    if !mail.is_dir() {
175        return out;
176    }
177
178    // Provider per account, best-effort: an unparseable registry costs the
179    // `--provider` flag on the remedy, never the finding itself.
180    let declared: Vec<MailAccount> = std::fs::read_to_string(mail.join("accounts.toml"))
181        .ok()
182        .and_then(|text| toml::from_str::<MailAccounts>(&text).ok())
183        .map(|file| file.accounts)
184        .unwrap_or_default();
185    let providers: BTreeMap<String, String> = declared
186        .iter()
187        .map(|a| (a.name.clone(), a.provider.clone()))
188        .collect();
189    let lifetimes: BTreeMap<String, u32> = declared
190        .iter()
191        .filter_map(|a| a.grant_lifetime_days.map(|d| (a.name.clone(), d)))
192        .collect();
193
194    let entries = match std::fs::read_dir(mail) {
195        Ok(entries) => entries,
196        Err(e) => {
197            out.push(Finding::unreadable(
198                "mail",
199                "the mail directory",
200                format!("{}: {e}", mail.display()),
201            ));
202            return out;
203        }
204    };
205
206    for entry in entries.flatten() {
207        let dir = entry.path();
208        if !dir.is_dir() {
209            continue;
210        }
211        let Some(account) = dir.file_name().and_then(|n| n.to_str()).map(String::from) else {
212            continue;
213        };
214        // A grant that predates the triage scopes refreshes cleanly forever
215        // and then 403s the first time something archives. That failure has
216        // no marker — nothing has gone wrong yet — so it is read off the
217        // credential file directly, structurally, like everything else here.
218        out.extend(check_triage_scope(&dir, &account, providers.get(&account)));
219        out.extend(check_grant_age(
220            &dir,
221            &account,
222            providers.get(&account),
223            lifetimes.get(&account).copied(),
224        ));
225
226        let marker_path = dir.join("auth_error.json");
227        if !marker_path.is_file() {
228            continue;
229        }
230        let text = match std::fs::read_to_string(&marker_path) {
231            Ok(text) => text,
232            Err(e) => {
233                out.push(Finding::unreadable(
234                    "mail",
235                    &format!("auth_error.json for `{account}`"),
236                    format!("{}: {e}", marker_path.display()),
237                ));
238                continue;
239            }
240        };
241        match serde_json::from_str::<AuthMarker>(&text) {
242            Ok(marker) => {
243                let provider = providers.get(&account);
244                let mut argv = vec![
245                    "mecha-mail".to_string(),
246                    "auth".to_string(),
247                    account.clone(),
248                ];
249                if let Some(provider) = provider {
250                    argv.push("--provider".to_string());
251                    argv.push(provider.clone());
252                }
253                out.push(Finding {
254                    component: "mail".to_string(),
255                    severity: Severity::Broken,
256                    summary: format!("mail auth for `{account}` is dead"),
257                    // The marker's message already names the exact re-auth
258                    // command, so it rides in the detail — which also covers
259                    // the case where accounts.toml could not say which
260                    // provider the remedy needs.
261                    detail: format!(
262                        "permanent refresh failure since {}: {}",
263                        marker.at, marker.message
264                    ),
265                    remedy: Some(Remedy {
266                        description: format!(
267                            "re-authenticate the `{account}` account (opens an OAuth flow)"
268                        ),
269                        argv,
270                        needs_terminal: true,
271                    }),
272                });
273            }
274            Err(e) => out.push(Finding::unreadable(
275                "mail",
276                &format!("auth_error.json for `{account}` did not parse"),
277                format!("{}: {e}", marker_path.display()),
278            )),
279        }
280    }
281    out
282}
283
284/// The scope a grant was minted with, as `mecha-mail` records it.
285///
286/// Read structurally rather than through `mecha-mail`'s type, for the reason
287/// the whole module gives: doctor takes no dependency on the crates it
288/// examines, and a field it does not know about must not stop it reading the
289/// one it does.
290#[derive(Debug, serde::Deserialize)]
291struct StoredGrant {
292    #[serde(default)]
293    granted_scopes: Option<String>,
294    #[serde(default)]
295    granted_at: Option<String>,
296}
297
298/// How many days before a grant expires to start saying so.
299///
300/// Two, because the remedy is a two-minute re-auth that needs a human at a
301/// terminal — long enough to survive a weekend-adjacent lapse, short enough
302/// that it is not background noise on a 7-day cycle. A warning that fires
303/// for most of the grant's life is a warning nobody reads.
304const GRANT_WARN_WITHIN_DAYS: i64 = 2;
305
306/// Warn before a grant with a known, fixed lifetime expires.
307///
308/// This exists because of the 2026-08-11 outage: Google expires the refresh
309/// token of an app in *Testing* publishing status exactly 7 days after
310/// consent, returns `invalid_grant` when it does — indistinguishable from a
311/// revocation — and scheduling went down for three days. Doctor reported it
312/// correctly *after* the fact. A recurring, dated failure deserves to be
313/// reported before it happens, which is the one thing a marker written on
314/// failure can never do.
315///
316/// Silent unless the lifetime was declared: see
317/// `AccountEntry::grant_lifetime_days` for why this is not inferred.
318fn check_grant_age(
319    dir: &Path,
320    account: &str,
321    provider: Option<&String>,
322    lifetime_days: Option<u32>,
323) -> Vec<Finding> {
324    let Some(lifetime) = lifetime_days.filter(|d| *d > 0) else {
325        return Vec::new();
326    };
327    let Ok(text) = std::fs::read_to_string(dir.join("oauth.json")) else {
328        return Vec::new();
329    };
330    let Ok(grant) = serde_json::from_str::<StoredGrant>(&text) else {
331        return Vec::new(); // already reported by the scope check
332    };
333    // An un-stamped grant predates the field. Its age is genuinely unknown,
334    // and inventing one would either cry wolf or promise safety — so say
335    // nothing and let the next re-auth start the clock honestly.
336    let Some(granted_at) = grant.granted_at.as_deref() else {
337        return Vec::new();
338    };
339    let Ok(granted) = chrono::DateTime::parse_from_rfc3339(granted_at) else {
340        return Vec::new();
341    };
342    let expires = granted.with_timezone(&chrono::Utc) + chrono::Duration::days(lifetime as i64);
343    // Hours, then round *up* to whole days. `num_days()` truncates toward
344    // zero, so a grant with 47 hours left reports "1 day" — which is both
345    // wrong and the wrong direction, since it makes the warning look more
346    // urgent than it is and then says "1 day" again tomorrow.
347    let hours_left = (expires - chrono::Utc::now()).num_hours();
348    let left = (hours_left as f64 / 24.0).ceil() as i64;
349    if left > GRANT_WARN_WITHIN_DAYS {
350        return Vec::new();
351    }
352    let when = if hours_left < 0 {
353        "has expired".to_string()
354    } else if hours_left < 24 {
355        "expires within a day".to_string()
356    } else {
357        format!("expires in {left} days")
358    };
359    let mut argv = vec![
360        "mecha-mail".to_string(),
361        "auth".to_string(),
362        account.to_string(),
363    ];
364    if let Some(p) = provider {
365        argv.push("--provider".to_string());
366        argv.push(p.clone());
367    }
368    vec![Finding {
369        component: "mail".to_string(),
370        severity: Severity::Attention,
371        summary: format!("`{account}` sign-in {when}"),
372        detail: format!(
373            "this grant lasts {lifetime} days from consent ({granted_at}) and refreshing does \
374             not extend it. Re-authenticate before it lapses — once it does, the failure looks \
375             like a revoked token and every scheduled run using this account stops."
376        ),
377        remedy: Some(Remedy {
378            description: format!("re-authenticate `{account}` now (opens an OAuth flow)"),
379            argv,
380            needs_terminal: true,
381        }),
382    }]
383}
384
385/// Which scope each provider needs before the triage verbs work. Mirrors
386/// `mecha_mail::token::triage_scope_for`; duplicated rather than imported
387/// because the seam here is a directory of JSON, not a crate dependency.
388fn triage_scope_for(provider: &str) -> Option<&'static str> {
389    match provider {
390        "google" => Some("gmail.modify"),
391        "outlook" | "microsoft" => Some("Mail.ReadWrite"),
392        _ => None,
393    }
394}
395
396/// Report an account whose OAuth grant does not cover archive/spam/read-state.
397///
398/// Only reported when the provider is known: guessing which scope a grant
399/// should carry would turn an unrecognised provider into a permanent false
400/// finding, and a doctor that cries wolf stops being read. An **absent**
401/// `granted_scopes` counts as not covered, which is correct rather than
402/// harsh — every grant written before the field existed predates the scopes
403/// too.
404///
405/// `Attention`, not `Broken`: nothing is failing right now — mail reads,
406/// sends and stages drafts exactly as before — but the first archive will
407/// fail, and that is precisely the "silence is the likely explanation"
408/// shape this severity is for. On a managed Microsoft tenant the remedy may
409/// also need an administrator rather than the user, so the detail says so
410/// instead of implying a re-auth alone will fix it.
411fn check_triage_scope(dir: &Path, account: &str, provider: Option<&String>) -> Vec<Finding> {
412    let Some(provider) = provider else {
413        return Vec::new();
414    };
415    let Some(needed) = triage_scope_for(provider) else {
416        return Vec::new();
417    };
418    let path = dir.join("oauth.json");
419    let Ok(text) = std::fs::read_to_string(&path) else {
420        // No credentials is not a scope problem; the account simply is not
421        // signed in, which other checks and the first real call will say.
422        return Vec::new();
423    };
424    let Ok(grant) = serde_json::from_str::<StoredGrant>(&text) else {
425        return vec![Finding::unreadable(
426            "mail",
427            &format!("oauth.json for `{account}` did not parse"),
428            format!("{}", path.display()),
429        )];
430    };
431    if grant
432        .granted_scopes
433        .as_deref()
434        .is_some_and(|g| g.contains(needed))
435    {
436        return Vec::new();
437    }
438    let admin_note = if provider == "outlook" || provider == "microsoft" {
439        " Microsoft blocks `Mail.ReadWrite` from end-user consent under its \
440         recommended policy, so on a managed tenant an administrator has to \
441         grant it to the app registration before this can succeed."
442    } else {
443        ""
444    };
445    vec![Finding {
446        component: "mail".to_string(),
447        severity: Severity::Attention,
448        summary: format!("`{account}` cannot archive, spam or mark mail read"),
449        detail: format!(
450            "the stored grant does not include `{needed}`, so mail_triage will fail on this \
451             account. Reading, sending and calendar work are unaffected.{admin_note}"
452        ),
453        remedy: Some(Remedy {
454            description: format!(
455                "re-authenticate `{account}` to add the triage scope (opens an OAuth flow)"
456            ),
457            argv: vec![
458                "mecha-mail".to_string(),
459                "auth".to_string(),
460                account.to_string(),
461                "--provider".to_string(),
462                provider.clone(),
463            ],
464            needs_terminal: true,
465        }),
466    }]
467}
468
469#[cfg(test)]
470mod grant_age_tests {
471    use super::*;
472
473    fn store(dir: &Path, granted_at: Option<&str>) {
474        std::fs::create_dir_all(dir).unwrap();
475        let stamp = granted_at
476            .map(|g| format!(r#","granted_at":"{g}""#))
477            .unwrap_or_default();
478        std::fs::write(
479            dir.join("oauth.json"),
480            format!(r#"{{"client_id":"i","access_token":"a","refresh_token":"r","expires_at":1{stamp}}}"#),
481        )
482        .unwrap();
483    }
484
485    fn days_ago(n: i64) -> String {
486        (chrono::Utc::now() - chrono::Duration::days(n)).to_rfc3339()
487    }
488
489    /// The 7-day Testing clock, reported before it fires rather than after.
490    #[test]
491    fn a_grant_nearing_its_declared_lifetime_is_reported_early() {
492        let tmp = std::env::temp_dir().join(format!("mecha-grant-{}", std::process::id()));
493        let g = "google".to_string();
494
495        // Fresh: silent. A warning that fires all week is not a warning.
496        store(&tmp, Some(&days_ago(1)));
497        assert!(check_grant_age(&tmp, "personal", Some(&g), Some(7)).is_empty());
498
499        // Day 5 of 7 — two days left, inside the window.
500        store(&tmp, Some(&days_ago(5)));
501        let f = check_grant_age(&tmp, "personal", Some(&g), Some(7));
502        assert_eq!(f.len(), 1, "should warn with 2 days left");
503        assert!(
504            f[0].summary.contains("expires in 2 days"),
505            "{}",
506            f[0].summary
507        );
508        assert!(f[0].remedy.as_ref().unwrap().needs_terminal);
509
510        // Under 24h: worded without a misleading whole-day count.
511        store(&tmp, Some(&days_ago(7)));
512        let f = check_grant_age(&tmp, "personal", Some(&g), Some(7));
513        assert!(f[0].summary.contains("within a day"), "{}", f[0].summary);
514
515        // Past it: still a finding, worded as past.
516        store(&tmp, Some(&days_ago(9)));
517        let f = check_grant_age(&tmp, "personal", Some(&g), Some(7));
518        assert!(f[0].summary.contains("has expired"), "{}", f[0].summary);
519
520        // No declared lifetime: silent however old. Not inferred, ever.
521        assert!(check_grant_age(&tmp, "personal", Some(&g), None).is_empty());
522
523        // Un-stamped grant: age unknown, so no claim either way.
524        store(&tmp, None);
525        assert!(check_grant_age(&tmp, "personal", Some(&g), Some(7)).is_empty());
526
527        std::fs::remove_dir_all(&tmp).ok();
528    }
529}
530
531/// The legacy per-provider stores — `<home>/google/oauth.json` and
532/// `<home>/outlook/oauth.json`, still served by the shipped `mecha-google`
533/// and `mecha-outlook` binaries and what `mecha-mail import` exists to
534/// migrate — get the same marker written beside their credentials by the
535/// same token lifecycle. A doctor that reads only the registry layout
536/// reports "all clear" over a dead legacy login.
537fn check_legacy_mail(home: &Path) -> Vec<Finding> {
538    let mut out = Vec::new();
539    for provider in ["google", "outlook"] {
540        let marker_path = home.join(provider).join("auth_error.json");
541        if !marker_path.is_file() {
542            continue;
543        }
544        let text = match std::fs::read_to_string(&marker_path) {
545            Ok(text) => text,
546            Err(e) => {
547                out.push(Finding::unreadable(
548                    "mail",
549                    &format!("auth_error.json for the legacy {provider} store"),
550                    format!("{}: {e}", marker_path.display()),
551                ));
552                continue;
553            }
554        };
555        match serde_json::from_str::<AuthMarker>(&text) {
556            Ok(marker) => out.push(Finding {
557                component: "mail".to_string(),
558                severity: Severity::Broken,
559                summary: format!("legacy {provider} mail auth is dead"),
560                // The marker's message names the exact re-auth command (the
561                // writer derives it from the store's directory), so it rides
562                // in the detail.
563                detail: format!(
564                    "permanent refresh failure since {}: {}",
565                    marker.at, marker.message
566                ),
567                remedy: Some(Remedy {
568                    description: format!(
569                        "bring the legacy {provider} login into the unified registry — \
570                         and re-authenticate it per the detail, which no import fixes"
571                    ),
572                    argv: vec![
573                        "mecha-mail".to_string(),
574                        "import".to_string(),
575                        provider.to_string(),
576                        "--provider".to_string(),
577                        provider.to_string(),
578                    ],
579                    needs_terminal: false,
580                }),
581            }),
582            Err(e) => out.push(Finding::unreadable(
583                "mail",
584                &format!("auth_error.json for the legacy {provider} store did not parse"),
585                format!("{}: {e}", marker_path.display()),
586            )),
587        }
588    }
589    out
590}
591
592// --- stuck outbox items -----------------------------------------------------
593
594/// Read the outbox items directly — one JSON file per item, the store's own
595/// on-disk contract — so that examining the store never creates or re-chmods
596/// it the way [`crate::outbox::OutboxStore::open`] deliberately does.
597fn check_outbox(root: &Path, now: DateTime<Utc>) -> Vec<Finding> {
598    let mut out = Vec::new();
599    if !root.is_dir() {
600        return out;
601    }
602    let entries = match std::fs::read_dir(root) {
603        Ok(entries) => entries,
604        Err(e) => {
605            out.push(Finding::unreadable(
606                "outbox",
607                "the outbox directory",
608                format!("{}: {e}", root.display()),
609            ));
610            return out;
611        }
612    };
613
614    let review = Remedy {
615        description: "open the outbox review surface — doctor never releases a draft".to_string(),
616        argv: vec!["mecha".into(), "outbox".into(), "review".into()],
617        needs_terminal: true,
618    };
619
620    let mut stale: Vec<String> = Vec::new();
621    for entry in entries.flatten() {
622        let path = entry.path();
623        if path.extension().and_then(|e| e.to_str()) != Some("json") {
624            continue;
625        }
626        let item: crate::outbox::OutboxItem =
627            match std::fs::read_to_string(&path).map(|t| serde_json::from_str(&t)) {
628                Ok(Ok(item)) => item,
629                Ok(Err(e)) => {
630                    out.push(Finding::unreadable(
631                        "outbox",
632                        &format!(
633                            "item {} did not parse",
634                            path.file_name().unwrap_or_default().to_string_lossy()
635                        ),
636                        format!("{}: {e}", path.display()),
637                    ));
638                    continue;
639                }
640                Err(e) => {
641                    out.push(Finding::unreadable(
642                        "outbox",
643                        &format!(
644                            "item {} could not be read",
645                            path.file_name().unwrap_or_default().to_string_lossy()
646                        ),
647                        format!("{}: {e}", path.display()),
648                    ));
649                    continue;
650                }
651            };
652        if item.status != "pending" {
653            continue;
654        }
655        if let Some(error) = &item.error {
656            out.push(Finding {
657                component: "outbox".to_string(),
658                severity: Severity::Broken,
659                summary: format!("release failed: {error}"),
660                detail: format!(
661                    "{} · {} — still pending; the draft is good, the delivery was not",
662                    item.id, item.summary
663                ),
664                remedy: Some(review.clone()),
665            });
666        } else if age_of(&item.created_at, now).is_some_and(|age| age > STUCK_DRAFT_AFTER) {
667            stale.push(format!(
668                "{} · {} — staged {}",
669                item.id,
670                item.summary,
671                render_age(now, &item.created_at)
672            ));
673        }
674    }
675
676    if !stale.is_empty() {
677        // read_dir order is arbitrary; ids sort by staging time.
678        stale.sort();
679        out.push(Finding {
680            component: "outbox".to_string(),
681            severity: Severity::Attention,
682            summary: format!(
683                "{} draft{} pending for more than 48h",
684                stale.len(),
685                if stale.len() == 1 { "" } else { "s" }
686            ),
687            detail: stale.join("\n"),
688            remedy: Some(review),
689        });
690    }
691    out
692}
693
694// --- frontdoor --------------------------------------------------------------
695
696/// The states that mean a request is waiting on the user rather than on the
697/// requester: `extracted` awaits triage, `awaiting_me` awaits a draft review,
698/// and `triaged` is triage's "I drafted nothing — this needs a person":
699/// nothing ever re-triages it, so left alone it waits forever, invisibly.
700/// (`needs_info` waits on the stranger, and `drained` on the extraction pass.)
701const WAITING_ON_ME: [&str; 3] = [
702    crate::frontdoor::EXTRACTED,
703    crate::frontdoor::AWAITING_ME,
704    crate::frontdoor::TRIAGED,
705];
706
707/// Read the request records directly, for the same no-side-effects reason as
708/// the outbox — [`crate::frontdoor::Frontdoor::open`] creates the directory.
709fn check_frontdoor(root: &Path, now: DateTime<Utc>) -> Vec<Finding> {
710    let mut out = Vec::new();
711    if !root.is_dir() {
712        return out;
713    }
714    let entries = match std::fs::read_dir(root) {
715        Ok(entries) => entries,
716        Err(e) => {
717            out.push(Finding::unreadable(
718                "frontdoor",
719                "the request store",
720                format!("{}: {e}", root.display()),
721            ));
722            return out;
723        }
724    };
725
726    let list = Remedy {
727        description: "list the frontdoor queue".to_string(),
728        argv: vec!["mecha".into(), "frontdoor".into(), "list".into()],
729        needs_terminal: false,
730    };
731
732    let mut stale: Vec<(i64, String)> = Vec::new();
733    for entry in entries.flatten() {
734        let path = entry.path();
735        if path.extension().and_then(|e| e.to_str()) != Some("json") {
736            continue;
737        }
738        let Ok(Ok(record)) = std::fs::read_to_string(&path)
739            .map(|t| serde_json::from_str::<crate::frontdoor::Record>(&t))
740        else {
741            // The frontdoor store itself skips unreadable records; doctor
742            // says so instead, because silent skipping is the disease here.
743            out.push(Finding::unreadable(
744                "frontdoor",
745                &format!(
746                    "request {} did not parse",
747                    path.file_name().unwrap_or_default().to_string_lossy()
748                ),
749                path.display().to_string(),
750            ));
751            continue;
752        };
753        if record.state == crate::frontdoor::EXTRACTION_FAILED {
754            out.push(Finding {
755                component: "frontdoor".to_string(),
756                severity: Severity::Broken,
757                summary: format!(
758                    "request {} failed extraction and waits for a human",
759                    record.seq
760                ),
761                detail: format!(
762                    "{} ({}) — {}",
763                    record.seq,
764                    record.type_id,
765                    record
766                        .extraction_error
767                        .as_deref()
768                        .unwrap_or("no error recorded")
769                ),
770                remedy: Some(list.clone()),
771            });
772        } else if WAITING_ON_ME.contains(&record.state.as_str())
773            && request_age(&record, now).is_some_and(|age| age > STALE_REQUEST_AFTER)
774        {
775            stale.push((
776                record.seq,
777                format!(
778                    "{} ({}) — {}, received {}",
779                    record.seq,
780                    record.type_id,
781                    record.state,
782                    render_age(now, &record.created_at)
783                ),
784            ));
785        }
786    }
787
788    if !stale.is_empty() {
789        // read_dir order is arbitrary; the queue reads oldest-first by seq.
790        stale.sort_by_key(|(seq, _)| *seq);
791        out.push(Finding {
792            component: "frontdoor".to_string(),
793            severity: Severity::Attention,
794            summary: format!(
795                "{} request{} waiting on you for more than 72h",
796                stale.len(),
797                if stale.len() == 1 { "" } else { "s" }
798            ),
799            detail: stale
800                .into_iter()
801                .map(|(_, line)| line)
802                .collect::<Vec<_>>()
803                .join("\n"),
804            remedy: Some(list),
805        });
806    }
807    out
808}
809
810/// How long a request has waited: from when it arrived here (`drained_at`),
811/// falling back to when the stranger sent it. Unparseable stamps mean the age
812/// is unknown, and unknown never counts as stale — a doctor that guesses is
813/// worse than one that says nothing.
814fn request_age(record: &crate::frontdoor::Record, now: DateTime<Utc>) -> Option<chrono::Duration> {
815    age_of(&record.drained_at, now).or_else(|| age_of(&record.created_at, now))
816}
817
818// --- trigger health ---------------------------------------------------------
819
820/// How many recent runs the reliability check averages over.
821///
822/// Five, because one bad morning is not a trend and a long window would hide a
823/// trigger that broke this week behind a month of health.
824const HEALTH_WINDOW: usize = 5;
825
826/// Below this many calls in the window, no rate is reported.
827///
828/// A rate over three calls is noise, and a doctor that cries wolf stops being
829/// read — the same reasoning as the scope check declining to guess.
830const HEALTH_MIN_CALLS: u32 = 10;
831
832/// The share of failed calls that is worth a human's attention.
833///
834/// A third. Deliberately not near-zero: a model that tries a path, is told it
835/// does not exist, and tries the right one has done nothing wrong, and errors
836/// are how a run learns about its environment. What this is looking for is a
837/// trigger whose environment has moved out from under it.
838const HEALTH_ERROR_RATE: f64 = 1.0 / 3.0;
839
840/// Read the trigger files and the ledger directly — same reason as above:
841/// [`crate::trigger::TriggerStore::open`] creates and re-chmods the root.
842fn check_triggers(root: &Path, now: DateTime<Utc>) -> Vec<Finding> {
843    let mut out = Vec::new();
844    if !root.is_dir() {
845        return out;
846    }
847    let entries = match std::fs::read_dir(root) {
848        Ok(entries) => entries,
849        Err(e) => {
850            out.push(Finding::unreadable(
851                "triggers",
852                "the trigger store",
853                format!("{}: {e}", root.display()),
854            ));
855            return out;
856        }
857    };
858
859    let mut triggers: Vec<crate::trigger::Trigger> = Vec::new();
860    for entry in entries.flatten() {
861        let path = entry.path();
862        if path.extension().and_then(|e| e.to_str()) != Some("toml") {
863            continue;
864        }
865        let name = path
866            .file_stem()
867            .and_then(|s| s.to_str())
868            .unwrap_or_default()
869            .to_string();
870        match std::fs::read_to_string(&path).map(|t| toml::from_str::<crate::trigger::Trigger>(&t))
871        {
872            Ok(Ok(mut trigger)) => {
873                trigger.name = name;
874                triggers.push(trigger);
875            }
876            _ => out.push(Finding::unreadable(
877                "triggers",
878                &format!("trigger file `{name}.toml` did not parse"),
879                path.display().to_string(),
880            )),
881        }
882    }
883
884    // One ledger scan for both questions: the newest row that actually *ran*
885    // per trigger, and the newest *accounted slot* per trigger (manual runs
886    // carry no slot, so they are invisible to the schedule on purpose).
887    let mut recent: BTreeMap<String, Vec<crate::trigger::RunRecord>> = BTreeMap::new();
888    let mut last_slot: BTreeMap<String, DateTime<Utc>> = BTreeMap::new();
889    let ledger = root.join("runs.jsonl");
890    if ledger.is_file() {
891        match std::fs::read_to_string(&ledger) {
892            Ok(text) => {
893                for line in text.lines().filter(|l| !l.trim().is_empty()) {
894                    // A torn line is the store's problem, not this row's
895                    // neighbours': skip it the way the ledger reader does.
896                    let Ok(row) = serde_json::from_str::<crate::trigger::RunRecord>(line) else {
897                        continue;
898                    };
899                    if let Some(slot) = row.slot {
900                        let newest = last_slot.entry(row.trigger.clone()).or_insert(slot);
901                        if slot > *newest {
902                            *newest = slot;
903                        }
904                    }
905                    // A skip is a row, not a run: a skipped-stale or
906                    // skipped-overlap appended after an error is bookkeeping,
907                    // not a recovery, and keying on the literal last row let
908                    // it hide the failure the operator needed to see.
909                    if matches!(
910                        row.status,
911                        crate::trigger::RunStatus::Ok | crate::trigger::RunStatus::Error
912                    ) {
913                        let window = recent.entry(row.trigger.clone()).or_default();
914                        window.push(row);
915                        if window.len() > HEALTH_WINDOW {
916                            window.remove(0);
917                        }
918                    }
919                }
920            }
921            Err(e) => out.push(Finding::unreadable(
922                "triggers",
923                "the run ledger",
924                format!("{}: {e}", ledger.display()),
925            )),
926        }
927    }
928
929    for trigger in &triggers {
930        if !trigger.enabled {
931            continue;
932        }
933
934        // The most recent run failed: a manual run is the safe probe, because
935        // it records a row with no slot and so never advances the schedule.
936        let window = recent.get(&trigger.name);
937        if let Some(row) = window.and_then(|w| w.last()) {
938            if row.status == crate::trigger::RunStatus::Error {
939                out.push(Finding {
940                    component: "triggers".to_string(),
941                    severity: Severity::Attention,
942                    summary: format!("trigger `{}`'s most recent run failed", trigger.name),
943                    detail: format!(
944                        "started {}: {}",
945                        row.started_at.to_rfc3339(),
946                        row.error.as_deref().unwrap_or("no error recorded")
947                    ),
948                    remedy: Some(Remedy {
949                        description: format!(
950                            "run `{}` by hand — a manual run is evidence, not a fire; it never advances the schedule",
951                            trigger.name
952                        ),
953                        argv: vec![
954                            "mecha".into(),
955                            "trigger".into(),
956                            "run".into(),
957                            trigger.name.clone(),
958                        ],
959                        needs_terminal: false,
960                    }),
961                });
962            }
963        }
964
965        // Reliability across the window. An unattended run has nobody
966        // watching it fail: the briefing still arrives, the ledger still says
967        // `ok`, and a trigger failing a third of its calls looks exactly like
968        // one that works. Silent below a floor of calls, because a rate over
969        // three of them is noise, and unknown is never a finding.
970        let (calls, errors) = window
971            .map(|w| {
972                w.iter().fold((0u32, 0u32), |(c, e), r| {
973                    (c + r.tool_calls, e + r.tool_errors)
974                })
975            })
976            .unwrap_or((0, 0));
977        if calls >= HEALTH_MIN_CALLS && f64::from(errors) / f64::from(calls) >= HEALTH_ERROR_RATE {
978            let runs = window.map(Vec::len).unwrap_or(0);
979            out.push(Finding {
980                component: "triggers".to_string(),
981                severity: Severity::Attention,
982                summary: format!(
983                    "trigger `{}` failed {errors} of {calls} tool calls",
984                    trigger.name
985                ),
986                detail: format!(
987                    "across its last {runs} run(s){}. A run's answer arrives either way, so this is invisible in the ledger's status — and per-step reliability is what decides how long a task the run can finish, so a third of the calls failing is not a third of the work lost.",
988                    if window.is_some_and(|w| w.last().is_some_and(|r| r.ended_on_failed_call)) {
989                        ", and the most recent run answered with its last call failed"
990                    } else {
991                        ""
992                    }
993                ),
994                // Reading is the remedy: what to change is in the transcript,
995                // and doctor never decides that.
996                remedy: Some(Remedy {
997                    description: format!("read `{}`'s recent runs", trigger.name),
998                    argv: vec![
999                        "mecha".into(),
1000                        "trigger".into(),
1001                        "show".into(),
1002                        trigger.name.clone(),
1003                    ],
1004                    needs_terminal: false,
1005                }),
1006            });
1007        }
1008
1009        // The null run: it fired, it succeeded, and it did nothing. The rate
1010        // check above cannot see this one — a rate over zero calls is
1011        // undefined rather than bad — so a trigger that made thirty calls a
1012        // morning and now makes none is silent in every signal the ledger
1013        // carries. Found by a sibling arc hitting the same shape one layer
1014        // down, where `mecha mail classify` returned success having classified
1015        // 0 of 16.
1016        //
1017        // Measured against the trigger's *own* history, never an absolute
1018        // floor: a prompt that legitimately needs no tools makes zero calls
1019        // every morning, and a check that called that broken would be wrong
1020        // about the healthiest trigger on the machine. So the earlier runs in
1021        // the window have to show the work that stopped.
1022        if let Some(window) = window {
1023            let newest = window.last();
1024            let before: u32 = window[..window.len().saturating_sub(1)]
1025                .iter()
1026                .map(|r| r.tool_calls)
1027                .sum();
1028            // Only an `ok` run: an errored one already has a finding above,
1029            // and two findings for one fact leave neither meaning anything.
1030            let stopped = newest
1031                .is_some_and(|r| r.tool_calls == 0 && r.status == crate::trigger::RunStatus::Ok)
1032                && before >= HEALTH_MIN_CALLS;
1033            if stopped {
1034                out.push(Finding {
1035                    component: "triggers".to_string(),
1036                    severity: Severity::Attention,
1037                    summary: format!(
1038                        "trigger `{}`'s most recent run did no work",
1039                        trigger.name
1040                    ),
1041                    detail: format!(
1042                        "it succeeded having made no tool calls, where its previous {} run(s) made {before}. A run that does nothing and reports success is indistinguishable from a healthy one in every other signal — the status is `ok`, the schedule advanced, and the answer arrived.",
1043                        window.len() - 1
1044                    ),
1045                    remedy: Some(Remedy {
1046                        description: format!("read `{}`'s recent runs", trigger.name),
1047                        argv: vec![
1048                            "mecha".into(),
1049                            "trigger".into(),
1050                            "show".into(),
1051                            trigger.name.clone(),
1052                        ],
1053                        needs_terminal: false,
1054                    }),
1055                });
1056            }
1057        }
1058
1059        // A catch-up-always trigger whose accounted slots stopped advancing:
1060        // a healthy daemon fires the most recent slot every tick, so more
1061        // than two slots newer than the last accounted one means nothing is
1062        // ticking at all. Cheap by construction — three `prev_at_or_before`
1063        // calls, no schedule re-derivation.
1064        if trigger.catch_up != crate::trigger::CatchUp::Always {
1065            continue;
1066        }
1067        let Some(anchor) = last_slot.get(&trigger.name).copied().or(trigger.created_at) else {
1068            // No ledger row and no creation stamp: there is no baseline to
1069            // measure staleness against, and unknown is not stale.
1070            continue;
1071        };
1072        let tz = trigger.tz(None);
1073        let step = chrono::Duration::seconds(1);
1074        let missed_more_than_two = trigger
1075            .schedule
1076            .prev_at_or_before(now, tz)
1077            .and_then(|s0| trigger.schedule.prev_at_or_before(s0 - step, tz))
1078            .and_then(|s1| trigger.schedule.prev_at_or_before(s1 - step, tz))
1079            .is_some_and(|s2| s2 > anchor);
1080        if missed_more_than_two {
1081            out.push(Finding {
1082                component: "triggers".to_string(),
1083                severity: Severity::Attention,
1084                summary: format!("trigger `{}` has missed more than two slots", trigger.name),
1085                detail: format!(
1086                    "last accounted slot {}; with catch_up=always a healthy scheduler fires \
1087                     the most recent slot every tick, so the daemon or its timer may be down \
1088                     (systemctl --user status mecha-triggers)",
1089                    anchor.to_rfc3339()
1090                ),
1091                // No argv on purpose: running the trigger by hand would not
1092                // restart whatever stopped ticking.
1093                remedy: None,
1094            });
1095        }
1096    }
1097    out
1098}
1099
1100// --- run quality ------------------------------------------------------------
1101
1102/// How many sessions back a run-quality check reads.
1103///
1104/// Doctor runs in one pass with no network and no model, and each session is
1105/// a file read — so this is a budget, not a claim about relevance. Two hundred
1106/// covers weeks of ordinary use and stays well inside "fast enough to run
1107/// whenever you wonder".
1108const RUNS_WINDOW: usize = 200;
1109
1110/// Below this many runs *for one model*, no rate is reported.
1111///
1112/// Twenty rather than the trigger check's ten, because these rates are
1113/// population statistics across mixed work rather than one job doing the same
1114/// thing every morning, and the noise is correspondingly higher.
1115const RUNS_MIN: usize = 20;
1116
1117/// The share of runs finishing over a failed call that is worth saying out
1118/// loud. Deliberately high: rule-based evaluators are measured to *under*
1119/// report success — they mark good trajectories as failures more often than
1120/// humans do (AgentRewardBench) — so a low bar here would fire constantly on
1121/// runs that were fine, and a doctor that cries wolf stops being read.
1122const ENDED_ON_FAILURE_RATE: f64 = 0.20;
1123
1124/// The share of attempted tool calls the environment refuses.
1125const TOOL_ERROR_RATE: f64 = 0.25;
1126
1127/// And below this many *calls* across the window, no rate at all. Runs and
1128/// calls are different denominators: twenty runs can hold four calls.
1129const RUNS_MIN_CALLS: u64 = 20;
1130
1131/// The share of runs the *harness* cut short. `Interrupted` is excluded from
1132/// the numerator by [`cut_short`]: a person pressing Ctrl-C is the system
1133/// working, and counting it would make an attentive user look like a problem.
1134const CUT_SHORT_RATE: f64 = 0.25;
1135
1136/// Did the harness end this run? One definition, on [`crate::agent::StopCause`],
1137/// shared with the candidate gate's metric — see its doc for why there were two.
1138fn cut_short(stats: &crate::session::RunStats) -> bool {
1139    stats.stop_cause.is_some_and(|c| c.cut_short())
1140}
1141
1142/// Report population-level run quality: the signals that are invisible in any
1143/// single run and obvious across a few hundred.
1144///
1145/// Split by model, because a corpus spanning two has no single rate worth
1146/// quoting — the blend is true and useless, and a threshold on it fires for
1147/// the wrong model. Silent until there is enough of one model to say
1148/// anything, which is the same rule as everywhere else here: unknown is not a
1149/// finding.
1150fn check_runs(sessions: &Path) -> Vec<Finding> {
1151    use crate::runlog::{Corpus, Scan};
1152
1153    let mut out = Vec::new();
1154    if !sessions.is_dir() {
1155        return out;
1156    }
1157    let corpus = match Corpus::scan(
1158        sessions,
1159        &Scan {
1160            max_sessions: Some(RUNS_WINDOW),
1161            since: None,
1162        },
1163    ) {
1164        Ok(c) => c,
1165        Err(e) => {
1166            out.push(Finding::unreadable(
1167                "runs",
1168                "the session store",
1169                format!("{}: {e}", sessions.display()),
1170            ));
1171            return out;
1172        }
1173    };
1174
1175    let remedy = |what: &str| {
1176        Some(Remedy {
1177            description: format!("read the run-quality summary ({what})"),
1178            argv: vec![
1179                "mecha".into(),
1180                "sessions".into(),
1181                "health".into(),
1182                "--days".into(),
1183                "30".into(),
1184            ],
1185            needs_terminal: false,
1186        })
1187    };
1188
1189    for (model, runs) in corpus.by_model() {
1190        if runs.len() < RUNS_MIN {
1191            continue;
1192        }
1193        let n = runs.len();
1194
1195        if let Some(rate) = runs.rate_of(|r| r.stats.ended_on_failed_call) {
1196            if rate >= ENDED_ON_FAILURE_RATE {
1197                out.push(Finding {
1198                    component: "runs".to_string(),
1199                    severity: Severity::Attention,
1200                    summary: format!(
1201                        "{:.0}% of `{model}` runs finished on a failed tool call",
1202                        rate * 100.0
1203                    ),
1204                    detail: format!(
1205                        "{} of {n} recent run(s). The model stopped of its own accord with its last call failed, and the answer it wrote may report success over it — which nothing in the text or the stop reason can show.",
1206                        runs.ended_on_failed_call()
1207                    ),
1208                    remedy: remedy("which runs, and what failed"),
1209                });
1210            }
1211        }
1212
1213        if let Some(rate) = runs.tool_error_rate() {
1214            // The sibling trigger check states the rule this one omitted: a
1215            // rate over three calls is noise. Twenty conversational runs that
1216            // made four calls between them must not raise a finding because
1217            // one of them errored.
1218            if rate >= TOOL_ERROR_RATE && runs.tool_calls() >= RUNS_MIN_CALLS {
1219                out.push(Finding {
1220                    component: "runs".to_string(),
1221                    severity: Severity::Attention,
1222                    summary: format!(
1223                        "`{model}` runs fail {:.0}% of their tool calls",
1224                        rate * 100.0
1225                    ),
1226                    detail: format!(
1227                        "{} of {} call(s) across {n} run(s) were refused by the environment. Errors are how a run learns where it is, so some are healthy — a quarter of them says something moved: a renamed path, a revoked grant, a tool whose schema the model keeps mis-filling.",
1228                        runs.tool_errors(),
1229                        runs.tool_calls()
1230                    ),
1231                    remedy: remedy("which tool, and how it failed"),
1232                });
1233            }
1234        }
1235
1236        if let Some(rate) = runs.rate_of(|r| cut_short(&r.stats)) {
1237            if rate >= CUT_SHORT_RATE {
1238                let cut = runs.rows.iter().filter(|r| cut_short(&r.stats)).count();
1239                out.push(Finding {
1240                    component: "runs".to_string(),
1241                    severity: Severity::Attention,
1242                    summary: format!(
1243                        "the harness cut {:.0}% of `{model}` runs short",
1244                        rate * 100.0
1245                    ),
1246                    detail: format!(
1247                        "{cut} of {n} recent run(s) hit a turn, token or cost ceiling, or tripped the loop guard. A budget that stops a quarter of runs is measuring the budget rather than the work — the answers are truncated and say so only in `stop_cause`. Cancellations are not counted here.",
1248                    ),
1249                    remedy: remedy("which ceiling, and how often"),
1250                });
1251            }
1252        }
1253    }
1254    out
1255}
1256
1257// --- graph nightly silence --------------------------------------------------
1258
1259/// The two daily jobs that keep the knowledge graph current, each of which
1260/// writes `<prefix>YYYYMMDD.log` on *every* run — a deferred night says so in
1261/// the log — so a day with no file means the script never started. That is
1262/// exactly the failure cron cannot report: no MTA, and the script's own
1263/// logging begins after the point where an exec failure kills it (measured
1264/// 2026-08-17, when a missing execute bit cost a night of vet and gossip and
1265/// nothing anywhere said so).
1266const GRAPH_NIGHTLIES: &[(&str, &str)] = &[
1267    ("nightly-", "the graph's own sweep (ingest, extract, decay)"),
1268    ("mecha-nightly-", "the mecha half (vet, precheck, gossip)"),
1269];
1270
1271/// Scan `<graph store>/logs` for each nightly family's newest dated log.
1272///
1273/// Quiet when the store, the logs directory, or a family has never existed —
1274/// absence is "not installed", which is not a finding. The bar is "newer than
1275/// the day before yesterday": today's file legitimately does not exist before
1276/// that job's cron slot, so yesterday's is the newest a healthy quiet morning
1277/// can show.
1278fn check_graph_nightly(store: &Path, now: DateTime<Utc>) -> Vec<Finding> {
1279    let mut out = Vec::new();
1280    let logs = store.join("logs");
1281    if !logs.is_dir() {
1282        return out;
1283    }
1284    let names: Vec<String> = match std::fs::read_dir(&logs) {
1285        Ok(entries) => entries
1286            .flatten()
1287            .filter_map(|e| e.file_name().to_str().map(String::from))
1288            .collect(),
1289        Err(e) => {
1290            out.push(Finding::unreadable(
1291                "graph",
1292                "the graph nightly logs",
1293                format!("{}: {e}", logs.display()),
1294            ));
1295            return out;
1296        }
1297    };
1298
1299    for (prefix, what) in GRAPH_NIGHTLIES {
1300        let newest = names
1301            .iter()
1302            .filter_map(|n| {
1303                n.strip_prefix(prefix)?
1304                    .strip_suffix(".log")
1305                    .and_then(|d| chrono::NaiveDate::parse_from_str(d, "%Y%m%d").ok())
1306            })
1307            .max();
1308        // Never ran at all: indistinguishable from "this half is not set up",
1309        // and a doctor that guesses teaches people to ignore it.
1310        let Some(newest) = newest else { continue };
1311        let days_quiet = (now.date_naive() - newest).num_days();
1312        if days_quiet > 1 {
1313            out.push(Finding {
1314                component: "graph".to_string(),
1315                severity: Severity::Attention,
1316                summary: format!(
1317                    "the graph nightly ({}) has not run for {days_quiet} days",
1318                    prefix.trim_end_matches('-'),
1319                ),
1320                detail: format!(
1321                    "{what} last wrote {}{}.log under {}; it logs every \
1322                     run including deferred ones, so a missing day means the \
1323                     script never started — cron reports that nowhere",
1324                    prefix,
1325                    newest.format("%Y%m%d"),
1326                    logs.display(),
1327                ),
1328                remedy: Some(Remedy {
1329                    description: "list the cron entries that fire the graph nightlies, \
1330                                  then run the silent one by hand and read its error"
1331                        .to_string(),
1332                    argv: vec!["crontab".into(), "-l".into()],
1333                    needs_terminal: false,
1334                }),
1335            });
1336        }
1337    }
1338    out
1339}
1340
1341// --- shared helpers ---------------------------------------------------------
1342
1343/// The age of an RFC 3339 stamp, or `None` when it does not parse — unknown
1344/// must never masquerade as old (or as fresh).
1345fn age_of(stamp: &str, now: DateTime<Utc>) -> Option<chrono::Duration> {
1346    DateTime::parse_from_rfc3339(stamp)
1347        .ok()
1348        .map(|at| now - at.with_timezone(&Utc))
1349}
1350
1351/// "49h ago", "3d ago", or the raw stamp when it does not parse.
1352fn render_age(now: DateTime<Utc>, stamp: &str) -> String {
1353    match age_of(stamp, now) {
1354        Some(age) if age >= chrono::Duration::days(2) => format!("{}d ago", age.num_days()),
1355        Some(age) if age >= chrono::Duration::hours(1) => format!("{}h ago", age.num_hours()),
1356        Some(age) => format!("{}m ago", age.num_minutes().max(0)),
1357        None => stamp.to_string(),
1358    }
1359}
1360
1361#[cfg(test)]
1362mod tests {
1363    use super::*;
1364    use crate::agent::Taint;
1365    use crate::outbox::{OutboxItem, OutboxKind};
1366    use serde_json::json;
1367    use std::path::PathBuf;
1368
1369    fn utc(s: &str) -> DateTime<Utc> {
1370        DateTime::parse_from_rfc3339(s).unwrap().with_timezone(&Utc)
1371    }
1372
1373    const NOW: &str = "2026-08-14T12:00:00Z";
1374
1375    /// A scratch mecha home, unique per test and thread.
1376    fn home(name: &str) -> PathBuf {
1377        let dir = std::env::temp_dir().join(format!(
1378            "mecha-doctor-test-{name}-{}-{:?}",
1379            std::process::id(),
1380            std::thread::current().id()
1381        ));
1382        let _ = std::fs::remove_dir_all(&dir);
1383        std::fs::create_dir_all(&dir).unwrap();
1384        dir
1385    }
1386
1387    fn write_marker(home: &Path, account: &str, body: &str) {
1388        let dir = home.join("mail").join(account);
1389        std::fs::create_dir_all(&dir).unwrap();
1390        std::fs::write(dir.join("auth_error.json"), body).unwrap();
1391    }
1392
1393    fn valid_marker() -> String {
1394        json!({
1395            "at": "2026-08-11T09:00:00Z",
1396            "message": "the refresh token was revoked — run `mecha-mail auth personal --provider google` to sign in again",
1397        })
1398        .to_string()
1399    }
1400
1401    fn pending_item(home: &Path, id: &str, created_at: &str, error: Option<&str>) {
1402        let item = OutboxItem {
1403            id: id.to_string(),
1404            status: "pending".into(),
1405            tool: "mail__send".into(),
1406            kind: OutboxKind::Message,
1407            args_before: json!({"to": "a@x.org"}),
1408            args: json!({"to": "a@x.org"}),
1409            summary: "mail__send to a@x.org".into(),
1410            session_id: None,
1411            workspace: None,
1412            taint: Taint::default(),
1413            created_at: created_at.to_string(),
1414            resolved_at: None,
1415            reason: None,
1416            error: error.map(String::from),
1417        };
1418        let dir = home.join("outbox");
1419        std::fs::create_dir_all(&dir).unwrap();
1420        std::fs::write(
1421            dir.join(format!("{id}.json")),
1422            serde_json::to_string_pretty(&item).unwrap(),
1423        )
1424        .unwrap();
1425    }
1426
1427    fn request(home: &Path, seq: i64, state: &str, drained_at: &str) {
1428        let dir = home.join("requests");
1429        std::fs::create_dir_all(&dir).unwrap();
1430        let record = json!({
1431            "seq": seq,
1432            "type_id": "meeting",
1433            "state": state,
1434            "created_at": drained_at,
1435            "drained_at": drained_at,
1436            "valid": true,
1437            "values": {},
1438            "free_text": [],
1439        });
1440        std::fs::write(
1441            dir.join(format!("{seq:010}-meeting.json")),
1442            record.to_string(),
1443        )
1444        .unwrap();
1445    }
1446
1447    fn trigger_file(home: &Path, name: &str, extra: &str) {
1448        let dir = home.join("triggers");
1449        std::fs::create_dir_all(&dir).unwrap();
1450        std::fs::write(
1451            dir.join(format!("{name}.toml")),
1452            format!(
1453                "schedule = \"0 7 * * *\"\nprompt = \"brief me\"\ntimezone = \"UTC\"\n\
1454                 created_at = \"2026-08-01T00:00:00Z\"\n{extra}"
1455            ),
1456        )
1457        .unwrap();
1458    }
1459
1460    fn ledger_row(home: &Path, row: &serde_json::Value) {
1461        use std::io::Write;
1462        let dir = home.join("triggers");
1463        std::fs::create_dir_all(&dir).unwrap();
1464        let mut file = std::fs::OpenOptions::new()
1465            .create(true)
1466            .append(true)
1467            .open(dir.join("runs.jsonl"))
1468            .unwrap();
1469        writeln!(file, "{row}").unwrap();
1470    }
1471
1472    fn of<'a>(findings: &'a [Finding], component: &str) -> Vec<&'a Finding> {
1473        findings
1474            .iter()
1475            .filter(|f| f.component == component)
1476            .collect()
1477    }
1478
1479    #[test]
1480    fn a_dead_auth_marker_is_found_and_an_absent_one_is_not() {
1481        let home = home("dead-auth");
1482        write_marker(&home, "personal", &valid_marker());
1483        // A healthy account: a directory with credentials and no marker.
1484        std::fs::create_dir_all(home.join("mail").join("dartmouth")).unwrap();
1485        std::fs::write(
1486            home.join("mail").join("accounts.toml"),
1487            "[[account]]\nname = \"personal\"\nprovider = \"google\"\n\
1488             [[account]]\nname = \"dartmouth\"\nprovider = \"outlook\"\n",
1489        )
1490        .unwrap();
1491
1492        let findings = examine(&home, utc(NOW));
1493        let mail = of(&findings, "mail");
1494        assert_eq!(mail.len(), 1, "{findings:#?}");
1495        assert_eq!(mail[0].severity, Severity::Broken);
1496        assert!(mail[0].summary.contains("personal"), "{}", mail[0].summary);
1497        let remedy = mail[0].remedy.as_ref().expect("a dead login has a way out");
1498        assert_eq!(
1499            remedy.argv,
1500            vec!["mecha-mail", "auth", "personal", "--provider", "google"]
1501        );
1502        assert!(
1503            remedy.needs_terminal,
1504            "an OAuth flow needs the real terminal"
1505        );
1506
1507        let _ = std::fs::remove_dir_all(&home);
1508    }
1509
1510    #[test]
1511    fn a_provider_the_registry_cannot_name_is_omitted_from_the_remedy_not_guessed() {
1512        let home = home("no-registry");
1513        // No accounts.toml at all.
1514        write_marker(&home, "personal", &valid_marker());
1515
1516        let findings = examine(&home, utc(NOW));
1517        let mail = of(&findings, "mail");
1518        assert_eq!(mail.len(), 1);
1519        let remedy = mail[0].remedy.as_ref().unwrap();
1520        assert_eq!(remedy.argv, vec!["mecha-mail", "auth", "personal"]);
1521        // The marker's message names the full command, and it rides in the
1522        // detail so the operator still sees the provider.
1523        assert!(
1524            mail[0].detail.contains("--provider google"),
1525            "{}",
1526            mail[0].detail
1527        );
1528
1529        let _ = std::fs::remove_dir_all(&home);
1530    }
1531
1532    /// Legacy per-provider stores (`<home>/google/oauth.json`, still served
1533    /// by the shipped `mecha-google` binary) get the same marker beside
1534    /// their credentials — and the old scan, which read only
1535    /// `<home>/mail/*/`, walked straight past it.
1536    #[test]
1537    fn a_marker_in_a_legacy_per_provider_store_is_found_and_proposes_import() {
1538        let home = home("legacy-auth");
1539        let dir = home.join("google");
1540        std::fs::create_dir_all(&dir).unwrap();
1541        std::fs::write(
1542            dir.join("auth_error.json"),
1543            json!({
1544                "at": "2026-08-11T09:00:00Z",
1545                "message": "account `google`: refresh token expired or revoked — run `mecha-mail auth google --provider google` (invalid_grant)",
1546            })
1547            .to_string(),
1548        )
1549        .unwrap();
1550
1551        let findings = examine(&home, utc(NOW));
1552        let mail = of(&findings, "mail");
1553        assert_eq!(mail.len(), 1, "{findings:#?}");
1554        assert_eq!(mail[0].severity, Severity::Broken);
1555        assert!(
1556            mail[0].summary.contains("legacy google"),
1557            "{}",
1558            mail[0].summary
1559        );
1560        // The marker's message names the exact re-auth command; it must ride
1561        // in the detail.
1562        assert!(
1563            mail[0]
1564                .detail
1565                .contains("run `mecha-mail auth google --provider google`"),
1566            "{}",
1567            mail[0].detail
1568        );
1569        let remedy = mail[0].remedy.as_ref().expect("a way out");
1570        assert_eq!(
1571            remedy.argv,
1572            vec!["mecha-mail", "import", "google", "--provider", "google"]
1573        );
1574
1575        let _ = std::fs::remove_dir_all(&home);
1576    }
1577
1578    #[test]
1579    fn an_unparseable_marker_is_a_store_unreadable_finding_not_a_crash() {
1580        let home = home("bad-marker");
1581        write_marker(&home, "personal", "{ this is not json");
1582
1583        let findings = examine(&home, utc(NOW));
1584        let mail = of(&findings, "mail");
1585        assert_eq!(mail.len(), 1, "{findings:#?}");
1586        assert!(
1587            mail[0].summary.starts_with("store unreadable:"),
1588            "{}",
1589            mail[0].summary
1590        );
1591        assert!(mail[0].summary.contains("personal"), "{}", mail[0].summary);
1592
1593        let _ = std::fs::remove_dir_all(&home);
1594    }
1595
1596    #[test]
1597    fn a_pending_item_with_an_error_is_broken_and_a_resolved_one_is_not() {
1598        let home = home("outbox-error");
1599        pending_item(
1600            &home,
1601            "20260814-000001-aaa",
1602            NOW,
1603            Some("server unreachable"),
1604        );
1605        // A sent item with an old date and even an error field: never flagged.
1606        let mut sent = json!({
1607            "id": "20260810-000001-bbb",
1608            "status": "sent",
1609            "tool": "mail__send",
1610            "args_before": {},
1611            "args": {},
1612            "summary": "mail__send",
1613            "created_at": "2026-08-01T00:00:00Z",
1614        });
1615        sent["error"] = json!(null);
1616        std::fs::write(
1617            home.join("outbox").join("20260810-000001-bbb.json"),
1618            sent.to_string(),
1619        )
1620        .unwrap();
1621
1622        let findings = examine(&home, utc(NOW));
1623        let outbox = of(&findings, "outbox");
1624        assert_eq!(outbox.len(), 1, "{findings:#?}");
1625        assert_eq!(outbox[0].severity, Severity::Broken);
1626        assert!(
1627            outbox[0]
1628                .summary
1629                .contains("release failed: server unreachable"),
1630            "{}",
1631            outbox[0].summary
1632        );
1633        let remedy = outbox[0].remedy.as_ref().unwrap();
1634        assert_eq!(remedy.argv, vec!["mecha", "outbox", "review"]);
1635
1636        let _ = std::fs::remove_dir_all(&home);
1637    }
1638
1639    #[test]
1640    fn a_pending_draft_is_stale_at_49_hours_and_not_at_47() {
1641        let home = home("outbox-stale");
1642        // 49h before NOW.
1643        pending_item(&home, "20260812-110000-old", "2026-08-12T11:00:00Z", None);
1644        let findings = examine(&home, utc(NOW));
1645        let outbox = of(&findings, "outbox");
1646        assert_eq!(outbox.len(), 1, "{findings:#?}");
1647        assert_eq!(outbox[0].severity, Severity::Attention);
1648        assert!(outbox[0].summary.contains("pending for more than 48h"));
1649        assert_eq!(
1650            outbox[0].remedy.as_ref().unwrap().argv,
1651            vec!["mecha", "outbox", "review"],
1652            "the remedy is the review surface, never send"
1653        );
1654
1655        // 47h old: a person may simply not have reviewed yet.
1656        let fresh = home;
1657        let _ = std::fs::remove_dir_all(fresh.join("outbox"));
1658        pending_item(&fresh, "20260812-130000-new", "2026-08-12T13:00:00Z", None);
1659        let findings = examine(&fresh, utc(NOW));
1660        assert!(of(&findings, "outbox").is_empty(), "{findings:#?}");
1661
1662        let _ = std::fs::remove_dir_all(&fresh);
1663    }
1664
1665    #[test]
1666    fn a_failed_extraction_is_broken_at_any_age() {
1667        let home = home("frontdoor-failed");
1668        request(&home, 12, crate::frontdoor::EXTRACTION_FAILED, NOW);
1669
1670        let findings = examine(&home, utc(NOW));
1671        let front = of(&findings, "frontdoor");
1672        assert_eq!(front.len(), 1, "{findings:#?}");
1673        assert_eq!(front[0].severity, Severity::Broken);
1674        assert!(front[0].summary.contains("12"), "{}", front[0].summary);
1675        assert_eq!(
1676            front[0].remedy.as_ref().unwrap().argv,
1677            vec!["mecha", "frontdoor", "list"]
1678        );
1679
1680        let _ = std::fs::remove_dir_all(&home);
1681    }
1682
1683    #[test]
1684    fn a_request_waiting_on_me_is_stale_at_73_hours_and_not_at_71() {
1685        let home = home("frontdoor-stale");
1686        // 73h before NOW.
1687        request(
1688            &home,
1689            1,
1690            crate::frontdoor::AWAITING_ME,
1691            "2026-08-11T11:00:00Z",
1692        );
1693        let findings = examine(&home, utc(NOW));
1694        let front = of(&findings, "frontdoor");
1695        assert_eq!(front.len(), 1, "{findings:#?}");
1696        assert_eq!(front[0].severity, Severity::Attention);
1697        assert!(front[0].summary.contains("waiting on you"));
1698
1699        // 71h: not yet.
1700        let _ = std::fs::remove_dir_all(home.join("requests"));
1701        request(
1702            &home,
1703            2,
1704            crate::frontdoor::AWAITING_ME,
1705            "2026-08-11T13:00:00Z",
1706        );
1707        let findings = examine(&home, utc(NOW));
1708        assert!(of(&findings, "frontdoor").is_empty(), "{findings:#?}");
1709
1710        // And a state waiting on the *requester* is never the user's fault.
1711        let _ = std::fs::remove_dir_all(home.join("requests"));
1712        request(
1713            &home,
1714            3,
1715            crate::frontdoor::NEEDS_INFO,
1716            "2026-08-01T00:00:00Z",
1717        );
1718        let findings = examine(&home, utc(NOW));
1719        assert!(of(&findings, "frontdoor").is_empty(), "{findings:#?}");
1720
1721        let _ = std::fs::remove_dir_all(&home);
1722    }
1723
1724    /// `triaged` means "triage considered it and drafted nothing — a person
1725    /// has to decide", and nothing ever re-triages it: left off the
1726    /// waiting-on-me list it waits forever, invisibly.
1727    #[test]
1728    fn a_triaged_request_nothing_will_revisit_goes_stale() {
1729        let home = home("frontdoor-triaged");
1730        // 73h before NOW.
1731        request(&home, 4, crate::frontdoor::TRIAGED, "2026-08-11T11:00:00Z");
1732        // Older still, but waiting on the *stranger*: never the user's fault.
1733        request(
1734            &home,
1735            5,
1736            crate::frontdoor::NEEDS_INFO,
1737            "2026-08-01T00:00:00Z",
1738        );
1739
1740        let findings = examine(&home, utc(NOW));
1741        let front = of(&findings, "frontdoor");
1742        assert_eq!(front.len(), 1, "{findings:#?}");
1743        assert_eq!(front[0].severity, Severity::Attention);
1744        assert!(front[0].detail.contains("triaged"), "{}", front[0].detail);
1745        assert!(
1746            !front[0].detail.contains("needs_info"),
1747            "needs_info waits on the requester: {}",
1748            front[0].detail
1749        );
1750
1751        let _ = std::fs::remove_dir_all(&home);
1752    }
1753
1754    #[test]
1755    fn a_trigger_whose_last_run_failed_is_flagged_with_the_manual_probe() {
1756        let home = home("trigger-failed");
1757        trigger_file(&home, "morning", "");
1758        ledger_row(
1759            &home,
1760            &json!({
1761                "trigger": "morning",
1762                "slot": "2026-08-13T07:00:00Z",
1763                "started_at": "2026-08-13T07:00:01Z",
1764                "status": "ok",
1765                "summary": "fine",
1766            }),
1767        );
1768        ledger_row(
1769            &home,
1770            &json!({
1771                "trigger": "morning",
1772                "slot": "2026-08-14T07:00:00Z",
1773                "started_at": "2026-08-14T07:00:01Z",
1774                "status": "error",
1775                "error": "provider unreachable",
1776            }),
1777        );
1778
1779        let findings = examine(&home, utc(NOW));
1780        let triggers = of(&findings, "triggers");
1781        assert_eq!(triggers.len(), 1, "{findings:#?}");
1782        assert_eq!(triggers[0].severity, Severity::Attention);
1783        assert!(triggers[0].summary.contains("morning"));
1784        assert!(triggers[0].detail.contains("provider unreachable"));
1785        assert_eq!(
1786            triggers[0].remedy.as_ref().unwrap().argv,
1787            vec!["mecha", "trigger", "run", "morning"],
1788            "a manual run is the safe probe: it never advances the schedule"
1789        );
1790
1791        let _ = std::fs::remove_dir_all(&home);
1792    }
1793
1794    /// A skip is a row, not a run: the overlap/staleness bookkeeping the
1795    /// scheduler appends after a failure must not read as a recovery. The
1796    /// old check keyed on the literal last ledger row and reported nothing.
1797    #[test]
1798    fn a_skip_row_after_a_failed_run_does_not_hide_the_failure() {
1799        let home = home("trigger-skip-hides-error");
1800        trigger_file(&home, "morning", "");
1801        ledger_row(
1802            &home,
1803            &json!({
1804                "trigger": "morning",
1805                "slot": "2026-08-13T07:00:00Z",
1806                "started_at": "2026-08-13T07:00:01Z",
1807                "status": "error",
1808                "error": "provider unreachable",
1809            }),
1810        );
1811        ledger_row(
1812            &home,
1813            &json!({
1814                "trigger": "morning",
1815                "slot": "2026-08-14T07:00:00Z",
1816                "started_at": "2026-08-14T07:00:01Z",
1817                "status": "skipped-stale",
1818            }),
1819        );
1820
1821        let findings = examine(&home, utc(NOW));
1822        let triggers = of(&findings, "triggers");
1823        assert_eq!(triggers.len(), 1, "{findings:#?}");
1824        assert!(
1825            triggers[0].summary.contains("most recent run failed"),
1826            "{}",
1827            triggers[0].summary
1828        );
1829        assert!(triggers[0].detail.contains("provider unreachable"));
1830
1831        let _ = std::fs::remove_dir_all(&home);
1832    }
1833
1834    #[test]
1835    fn an_ok_run_followed_by_a_skip_is_healthy() {
1836        let home = home("trigger-ok-then-skip");
1837        trigger_file(&home, "morning", "");
1838        ledger_row(
1839            &home,
1840            &json!({
1841                "trigger": "morning",
1842                "slot": "2026-08-13T07:00:00Z",
1843                "started_at": "2026-08-13T07:00:01Z",
1844                "status": "ok",
1845            }),
1846        );
1847        ledger_row(
1848            &home,
1849            &json!({
1850                "trigger": "morning",
1851                "slot": "2026-08-14T07:00:00Z",
1852                "started_at": "2026-08-14T07:00:01Z",
1853                "status": "skipped-overlap",
1854            }),
1855        );
1856
1857        let findings = examine(&home, utc(NOW));
1858        assert!(of(&findings, "triggers").is_empty(), "{findings:#?}");
1859
1860        let _ = std::fs::remove_dir_all(&home);
1861    }
1862
1863    #[test]
1864    fn a_trigger_quietly_failing_a_third_of_its_calls_is_reported() {
1865        // Every run says `ok` and every briefing arrived. The only place the
1866        // degradation exists is the call counts, which nothing read before.
1867        let home = home("trigger-tool-errors");
1868        trigger_file(&home, "morning", "");
1869        for day in 10..15 {
1870            ledger_row(
1871                &home,
1872                &json!({
1873                    "trigger": "morning",
1874                    "slot": format!("2026-08-{day}T07:00:00Z"),
1875                    "started_at": format!("2026-08-{day}T07:00:01Z"),
1876                    "status": "ok",
1877                    "summary": "briefed",
1878                    "tool_calls": 6,
1879                    "tool_errors": 3,
1880                }),
1881            );
1882        }
1883
1884        let findings = examine(&home, utc(NOW));
1885        let triggers = of(&findings, "triggers");
1886        assert_eq!(triggers.len(), 1, "{findings:#?}");
1887        assert_eq!(triggers[0].severity, Severity::Attention);
1888        assert!(
1889            triggers[0].summary.contains("15 of 30"),
1890            "{}",
1891            triggers[0].summary
1892        );
1893        assert_eq!(
1894            triggers[0].remedy.as_ref().unwrap().argv,
1895            vec!["mecha", "trigger", "show", "morning"],
1896            "reading is the remedy — what to change is in the transcript"
1897        );
1898
1899        let _ = std::fs::remove_dir_all(&home);
1900    }
1901
1902    #[test]
1903    fn a_handful_of_failed_calls_is_not_a_trend() {
1904        // Two rules at once, and both are about not crying wolf. A rate over
1905        // three calls is noise, so the floor holds; and errors are how a run
1906        // learns about its environment, so a rate under the bar is silence
1907        // rather than a quieter finding.
1908        let home = home("trigger-tool-errors-quiet");
1909        trigger_file(&home, "morning", "");
1910        // Under the call floor, though every call failed.
1911        ledger_row(
1912            &home,
1913            &json!({
1914                "trigger": "morning",
1915                "slot": "2026-08-14T07:00:00Z",
1916                "started_at": "2026-08-14T07:00:01Z",
1917                "status": "ok",
1918                "tool_calls": 3,
1919                "tool_errors": 3,
1920            }),
1921        );
1922        assert!(of(&examine(&home, utc(NOW)), "triggers").is_empty());
1923
1924        // Over the floor, under the rate.
1925        ledger_row(
1926            &home,
1927            &json!({
1928                "trigger": "morning",
1929                "slot": "2026-08-15T07:00:00Z",
1930                "started_at": "2026-08-15T07:00:01Z",
1931                "status": "ok",
1932                "tool_calls": 40,
1933                "tool_errors": 4,
1934            }),
1935        );
1936        assert!(of(&examine(&home, utc(NOW)), "triggers").is_empty());
1937
1938        let _ = std::fs::remove_dir_all(&home);
1939    }
1940
1941    #[test]
1942    fn a_trigger_that_stopped_doing_anything_is_reported() {
1943        // The null run. Status `ok`, schedule advanced, answer delivered, and
1944        // no work done — invisible in every signal the ledger carried before.
1945        let home = home("trigger-stopped-working");
1946        trigger_file(&home, "morning", "");
1947        for day in 10..14 {
1948            ledger_row(
1949                &home,
1950                &json!({
1951                    "trigger": "morning",
1952                    "slot": format!("2026-08-{day}T07:00:00Z"),
1953                    "started_at": format!("2026-08-{day}T07:00:01Z"),
1954                    "status": "ok",
1955                    "tool_calls": 8,
1956                    "tool_errors": 0,
1957                }),
1958            );
1959        }
1960        ledger_row(
1961            &home,
1962            &json!({
1963                "trigger": "morning",
1964                "slot": "2026-08-14T07:00:00Z",
1965                "started_at": "2026-08-14T07:00:01Z",
1966                "status": "ok",
1967                "summary": "nothing to report",
1968                "tool_calls": 0,
1969                "tool_errors": 0,
1970            }),
1971        );
1972
1973        let findings = examine(&home, utc(NOW));
1974        let triggers = of(&findings, "triggers");
1975        assert_eq!(triggers.len(), 1, "{findings:#?}");
1976        assert!(
1977            triggers[0].summary.contains("did no work"),
1978            "{}",
1979            triggers[0].summary
1980        );
1981        assert!(triggers[0].detail.contains("made 32"));
1982
1983        let _ = std::fs::remove_dir_all(&home);
1984    }
1985
1986    #[test]
1987    fn a_trigger_that_never_needed_tools_is_not_broken_for_not_using_them() {
1988        // The reason this is measured against the trigger's own history and
1989        // never an absolute floor: a prompt that needs no tools makes zero
1990        // calls every morning, and calling that broken would be wrong about
1991        // the healthiest trigger on the machine.
1992        let home = home("trigger-never-used-tools");
1993        trigger_file(&home, "haiku", "");
1994        for day in 10..15 {
1995            ledger_row(
1996                &home,
1997                &json!({
1998                    "trigger": "haiku",
1999                    "slot": format!("2026-08-{day}T07:00:00Z"),
2000                    "started_at": format!("2026-08-{day}T07:00:01Z"),
2001                    "status": "ok",
2002                    "tool_calls": 0,
2003                    "tool_errors": 0,
2004                }),
2005            );
2006        }
2007        assert!(of(&examine(&home, utc(NOW)), "triggers").is_empty());
2008
2009        let _ = std::fs::remove_dir_all(&home);
2010    }
2011
2012    #[test]
2013    fn a_failed_run_that_did_no_work_is_reported_once_not_twice() {
2014        // An errored run already has a finding naming the error. Reporting the
2015        // absence of work on top of it would be two findings for one fact,
2016        // and a reader who has to decide which of two rows is the real one is
2017        // reading a worse report than one row.
2018        let home = home("trigger-failed-no-work");
2019        trigger_file(&home, "morning", "");
2020        for day in 10..14 {
2021            ledger_row(
2022                &home,
2023                &json!({
2024                    "trigger": "morning",
2025                    "slot": format!("2026-08-{day}T07:00:00Z"),
2026                    "started_at": format!("2026-08-{day}T07:00:01Z"),
2027                    "status": "ok",
2028                    "tool_calls": 8,
2029                    "tool_errors": 0,
2030                }),
2031            );
2032        }
2033        ledger_row(
2034            &home,
2035            &json!({
2036                "trigger": "morning",
2037                "slot": "2026-08-14T07:00:00Z",
2038                "started_at": "2026-08-14T07:00:01Z",
2039                "status": "error",
2040                "error": "provider unreachable",
2041                "tool_calls": 0,
2042                "tool_errors": 0,
2043            }),
2044        );
2045
2046        let triggers = of(&examine(&home, utc(NOW)), "triggers")
2047            .into_iter()
2048            .cloned()
2049            .collect::<Vec<_>>();
2050        assert_eq!(triggers.len(), 1, "{triggers:#?}");
2051        assert!(triggers[0].detail.contains("provider unreachable"));
2052
2053        let _ = std::fs::remove_dir_all(&home);
2054    }
2055
2056    /// Write `n` runs of one model into the session store, so the
2057    /// population checks have something to be a population of.
2058    fn runs_in(
2059        home: &Path,
2060        model: &str,
2061        n: usize,
2062        stats: impl Fn(usize) -> crate::session::RunStats,
2063    ) {
2064        let dir = home.join("sessions");
2065        std::fs::create_dir_all(&dir).unwrap();
2066        for i in 0..n {
2067            let session = crate::session::Session::create(
2068                &dir,
2069                crate::session::SessionMeta {
2070                    // The model rides in the id: two calls to this helper
2071                    // in one test must not collide, or the second silently
2072                    // rewrites the first's transcripts and the fixture stops
2073                    // describing what the test says it does.
2074                    id: format!("2026080{}T00000{i:03}-{model}", 1 + i % 9),
2075                    created_at: utc(NOW),
2076                    provider: "local".into(),
2077                    model: model.to_string(),
2078                    workspace: std::path::PathBuf::from("/tmp"),
2079                    title: None,
2080                },
2081            )
2082            .unwrap();
2083            session
2084                .append(&crate::session::Record::Outcome(stats(i)))
2085                .unwrap();
2086        }
2087    }
2088
2089    fn run_stats(
2090        calls: u32,
2091        errors: u32,
2092        ended_failed: bool,
2093        cause: crate::agent::StopCause,
2094    ) -> crate::session::RunStats {
2095        crate::session::RunStats {
2096            tool_calls: calls,
2097            tool_errors: errors,
2098            ended_on_failed_call: ended_failed,
2099            stop_cause: Some(cause),
2100            ..Default::default()
2101        }
2102    }
2103
2104    #[test]
2105    fn a_model_that_keeps_finishing_over_failures_is_reported() {
2106        use crate::agent::StopCause;
2107        let home = home("runs-ended-on-failure");
2108        // A third of runs end over a failure; everything else is healthy.
2109        runs_in(&home, "tiny-local", 30, |i| {
2110            run_stats(6, 0, i % 3 == 0, StopCause::Completed)
2111        });
2112
2113        let all = examine(&home, utc(NOW));
2114        let findings = of(&all, "runs");
2115        assert_eq!(findings.len(), 1, "{findings:#?}");
2116        assert!(
2117            findings[0].summary.contains("tiny-local"),
2118            "{}",
2119            findings[0].summary
2120        );
2121        assert!(
2122            findings[0].summary.contains("33%"),
2123            "{}",
2124            findings[0].summary
2125        );
2126        assert_eq!(
2127            findings[0].remedy.as_ref().unwrap().argv,
2128            vec!["mecha", "sessions", "health", "--days", "30"],
2129            "reading is the remedy; doctor never decides what to change"
2130        );
2131
2132        let _ = std::fs::remove_dir_all(&home);
2133    }
2134
2135    #[test]
2136    fn a_cancelled_run_is_not_the_harness_cutting_it_short() {
2137        use crate::agent::StopCause;
2138        // A person pressing Ctrl-C is the system working, and counting it
2139        // would make an attentive user look like a problem.
2140        let home = home("runs-interrupted");
2141        runs_in(&home, "tiny-local", 30, |_| {
2142            run_stats(6, 0, false, StopCause::Interrupted)
2143        });
2144        let findings = examine(&home, utc(NOW));
2145        assert!(of(&findings, "runs").is_empty());
2146        let _ = std::fs::remove_dir_all(&home);
2147    }
2148
2149    #[test]
2150    fn a_turn_ceiling_stopping_a_quarter_of_runs_is_a_finding() {
2151        use crate::agent::StopCause;
2152        let home = home("runs-max-turns");
2153        runs_in(&home, "tiny-local", 30, |_| {
2154            run_stats(6, 0, false, StopCause::MaxTurns)
2155        });
2156        let all = examine(&home, utc(NOW));
2157        let findings = of(&all, "runs");
2158        assert_eq!(findings.len(), 1, "{findings:#?}");
2159        assert!(
2160            findings[0].summary.contains("cut"),
2161            "{}",
2162            findings[0].summary
2163        );
2164        let _ = std::fs::remove_dir_all(&home);
2165    }
2166
2167    #[test]
2168    fn a_thin_sample_of_one_model_says_nothing_about_it() {
2169        use crate::agent::StopCause;
2170        // Every run terrible, and still silent: nineteen runs is not a
2171        // population, and unknown is not a finding.
2172        let home = home("runs-thin");
2173        runs_in(&home, "tiny-local", 19, |_| {
2174            run_stats(6, 6, true, StopCause::MaxTurns)
2175        });
2176        let all = examine(&home, utc(NOW));
2177        assert!(of(&all, "runs").is_empty());
2178        let _ = std::fs::remove_dir_all(&home);
2179    }
2180
2181    #[test]
2182    fn a_bad_model_does_not_drag_a_good_one_into_a_finding() {
2183        use crate::agent::StopCause;
2184        // The reason rates split: blended, these two average to a rate that
2185        // describes neither, and a threshold on it names the wrong model.
2186        let home = home("runs-two-models");
2187        runs_in(&home, "steady", 25, |_| {
2188            run_stats(10, 0, false, StopCause::Completed)
2189        });
2190        runs_in(&home, "flaky", 25, |_| {
2191            run_stats(10, 9, false, StopCause::Completed)
2192        });
2193
2194        let all = examine(&home, utc(NOW));
2195        let findings = of(&all, "runs");
2196        assert_eq!(findings.len(), 1, "{findings:#?}");
2197        assert!(
2198            findings[0].summary.contains("flaky"),
2199            "{}",
2200            findings[0].summary
2201        );
2202        assert!(
2203            !findings[0].summary.contains("steady"),
2204            "the healthy model was named in a finding about the other one"
2205        );
2206        let _ = std::fs::remove_dir_all(&home);
2207    }
2208
2209    #[test]
2210    fn a_ledger_written_before_the_counts_existed_reports_nothing() {
2211        // Not a perfect score, and not a division by zero: no data is not a
2212        // finding, which is the rule the whole module runs on.
2213        let home = home("trigger-tool-errors-bare");
2214        trigger_file(&home, "morning", "");
2215        ledger_row(
2216            &home,
2217            &json!({
2218                "trigger": "morning",
2219                "slot": "2026-08-14T07:00:00Z",
2220                "started_at": "2026-08-14T07:00:01Z",
2221                "status": "ok",
2222            }),
2223        );
2224        assert!(of(&examine(&home, utc(NOW)), "triggers").is_empty());
2225
2226        let _ = std::fs::remove_dir_all(&home);
2227    }
2228
2229    #[test]
2230    fn a_disabled_trigger_is_nobody_s_emergency() {
2231        let home = home("trigger-disabled");
2232        trigger_file(&home, "morning", "enabled = false\n");
2233        ledger_row(
2234            &home,
2235            &json!({
2236                "trigger": "morning",
2237                "started_at": "2026-08-14T07:00:01Z",
2238                "status": "error",
2239                "error": "boom",
2240            }),
2241        );
2242        let findings = examine(&home, utc(NOW));
2243        assert!(of(&findings, "triggers").is_empty(), "{findings:#?}");
2244        let _ = std::fs::remove_dir_all(&home);
2245    }
2246
2247    #[test]
2248    fn a_catch_up_trigger_whose_slots_stopped_advancing_names_the_daemon() {
2249        let home = home("trigger-stale");
2250        trigger_file(&home, "morning", "");
2251        // Last accounted slot five days ago; daily at 07:00 UTC, so slots on
2252        // the 10th..14th are all unaccounted — far more than two.
2253        ledger_row(
2254            &home,
2255            &json!({
2256                "trigger": "morning",
2257                "slot": "2026-08-09T07:00:00Z",
2258                "started_at": "2026-08-09T07:00:01Z",
2259                "status": "ok",
2260            }),
2261        );
2262
2263        let findings = examine(&home, utc(NOW));
2264        let triggers = of(&findings, "triggers");
2265        assert_eq!(triggers.len(), 1, "{findings:#?}");
2266        assert_eq!(triggers[0].severity, Severity::Attention);
2267        assert!(triggers[0].summary.contains("missed more than two slots"));
2268        assert!(
2269            triggers[0].detail.contains("daemon"),
2270            "{}",
2271            triggers[0].detail
2272        );
2273        assert!(
2274            triggers[0].remedy.is_none(),
2275            "running the trigger would not restart the scheduler"
2276        );
2277
2278        // A current ledger is healthy: this morning's 07:00 accounted for.
2279        ledger_row(
2280            &home,
2281            &json!({
2282                "trigger": "morning",
2283                "slot": "2026-08-14T07:00:00Z",
2284                "started_at": "2026-08-14T07:00:01Z",
2285                "status": "ok",
2286            }),
2287        );
2288        let findings = examine(&home, utc(NOW));
2289        assert!(of(&findings, "triggers").is_empty(), "{findings:#?}");
2290
2291        let _ = std::fs::remove_dir_all(&home);
2292    }
2293
2294    /// The observer rule, which matters most: one poisoned store must not
2295    /// suppress what the other checks found — and must itself be reported.
2296    #[cfg(unix)]
2297    #[test]
2298    fn one_poisoned_store_does_not_suppress_the_others() {
2299        use std::os::unix::fs::PermissionsExt;
2300        // Root reads through 0o000 like it is not there, and the test would
2301        // be vacuous.
2302        if unsafe { libc::geteuid() } == 0 {
2303            return;
2304        }
2305
2306        let home = home("poisoned");
2307        write_marker(&home, "personal", &valid_marker());
2308        let outbox = home.join("outbox");
2309        std::fs::create_dir_all(&outbox).unwrap();
2310        std::fs::set_permissions(&outbox, std::fs::Permissions::from_mode(0o000)).unwrap();
2311
2312        let findings = examine(&home, utc(NOW));
2313
2314        // Restore before asserting, so a failure can still clean up.
2315        std::fs::set_permissions(&outbox, std::fs::Permissions::from_mode(0o700)).unwrap();
2316
2317        let mail = of(&findings, "mail");
2318        assert_eq!(mail.len(), 1, "the mail finding survived: {findings:#?}");
2319        assert_eq!(mail[0].severity, Severity::Broken);
2320        let broken_store = of(&findings, "outbox");
2321        assert_eq!(broken_store.len(), 1, "{findings:#?}");
2322        assert!(
2323            broken_store[0].summary.starts_with("store unreadable:"),
2324            "{}",
2325            broken_store[0].summary
2326        );
2327
2328        let _ = std::fs::remove_dir_all(&home);
2329    }
2330
2331    /// Finding-6 drift pin, reader half. Twin test (same golden bytes):
2332    /// `mecha_mail::token::tests::record_auth_error_serialises_the_golden_marker_byte_for_byte`
2333    /// in mecha-mail/src/token.rs — the crates share no types on purpose
2334    /// (the seam is a file of JSON), so a field rename on either side would
2335    /// pass both suites separately and silently kill this finding at
2336    /// runtime. If this literal changes, change the twin's too.
2337    #[test]
2338    fn the_golden_marker_literal_parses_into_the_dead_auth_finding() {
2339        const GOLDEN: &str = r#"{
2340  "at": "2026-08-11T09:00:00Z",
2341  "message": "account `personal`: refresh token expired or revoked — run `mecha-mail auth personal --provider google` (invalid_grant: Token has been revoked.)"
2342}"#;
2343        let home = home("golden-marker");
2344        write_marker(&home, "personal", GOLDEN);
2345
2346        let findings = examine(&home, utc(NOW));
2347        let mail = of(&findings, "mail");
2348        assert_eq!(mail.len(), 1, "{findings:#?}");
2349        assert_eq!(mail[0].severity, Severity::Broken);
2350        assert!(
2351            mail[0].detail.contains("since 2026-08-11T09:00:00Z"),
2352            "the marker's `at` must reach the detail: {}",
2353            mail[0].detail
2354        );
2355        assert!(
2356            mail[0]
2357                .detail
2358                .contains("run `mecha-mail auth personal --provider google`"),
2359            "the marker's `message` must reach the detail: {}",
2360            mail[0].detail
2361        );
2362
2363        let _ = std::fs::remove_dir_all(&home);
2364    }
2365
2366    #[test]
2367    fn findings_sort_broken_first() {
2368        let mut findings = vec![
2369            Finding {
2370                component: "outbox".into(),
2371                severity: Severity::Attention,
2372                summary: "stale".into(),
2373                detail: String::new(),
2374                remedy: None,
2375            },
2376            Finding {
2377                component: "mail".into(),
2378                severity: Severity::Broken,
2379                summary: "dead".into(),
2380                detail: String::new(),
2381                remedy: None,
2382            },
2383        ];
2384        sort(&mut findings);
2385        assert_eq!(findings[0].severity, Severity::Broken);
2386    }
2387
2388    #[test]
2389    fn an_empty_home_is_healthy() {
2390        let home = home("empty");
2391        assert!(examine(&home, utc(NOW)).is_empty());
2392        let _ = std::fs::remove_dir_all(&home);
2393    }
2394
2395    // --- graph nightly silence ---
2396
2397    /// A graph store nested inside a unique scratch dir, so no test plants a
2398    /// `.mecha-graph` beside another test's home in the shared temp dir.
2399    fn graph_store(name: &str) -> PathBuf {
2400        let store = home(name).join(".mecha-graph");
2401        std::fs::create_dir_all(store.join("logs")).unwrap();
2402        store
2403    }
2404
2405    fn nightly_log(store: &Path, file: &str) {
2406        std::fs::write(store.join("logs").join(file), "ran\n").unwrap();
2407    }
2408
2409    // NOW is 2026-08-14: a 08-12 log is two days quiet (stale), 08-13 is
2410    // yesterday (the newest a healthy quiet morning can show).
2411
2412    #[test]
2413    fn a_graph_nightly_that_stopped_writing_logs_is_a_finding() {
2414        let store = graph_store("graph-stale");
2415        nightly_log(&store, "nightly-20260812.log");
2416        let findings = check_graph_nightly(&store, utc(NOW));
2417        assert_eq!(findings.len(), 1);
2418        assert_eq!(findings[0].component, "graph");
2419        assert_eq!(findings[0].severity, Severity::Attention);
2420        assert!(
2421            findings[0].summary.contains("2 days"),
2422            "{}",
2423            findings[0].summary
2424        );
2425        assert!(
2426            findings[0].detail.contains("nightly-20260812.log"),
2427            "{}",
2428            findings[0].detail
2429        );
2430    }
2431
2432    #[test]
2433    fn yesterdays_log_is_healthy_because_todays_slot_may_not_have_fired() {
2434        let store = graph_store("graph-yesterday");
2435        nightly_log(&store, "nightly-20260813.log");
2436        nightly_log(&store, "mecha-nightly-20260813.log");
2437        assert!(check_graph_nightly(&store, utc(NOW)).is_empty());
2438    }
2439
2440    /// The two families age independently: the sweep running every night must
2441    /// not vouch for the vet/gossip half — that is exactly how 2026-08-17
2442    /// stayed invisible.
2443    #[test]
2444    fn each_nightly_family_is_judged_alone() {
2445        let store = graph_store("graph-split");
2446        nightly_log(&store, "nightly-20260814.log");
2447        nightly_log(&store, "mecha-nightly-20260811.log");
2448        let findings = check_graph_nightly(&store, utc(NOW));
2449        assert_eq!(findings.len(), 1);
2450        assert!(
2451            findings[0].summary.contains("mecha-nightly"),
2452            "{}",
2453            findings[0].summary
2454        );
2455    }
2456
2457    /// The `nightly-` scan must not claim `mecha-nightly-` files as its own:
2458    /// a fresh mecha-nightly log would otherwise hide a dead sweep.
2459    #[test]
2460    fn the_shorter_prefix_does_not_claim_the_longer_familys_logs() {
2461        let store = graph_store("graph-prefix");
2462        nightly_log(&store, "mecha-nightly-20260814.log");
2463        nightly_log(&store, "nightly-20260810.log");
2464        let findings = check_graph_nightly(&store, utc(NOW));
2465        assert_eq!(findings.len(), 1);
2466        assert!(
2467            findings[0].detail.contains("nightly-20260810.log"),
2468            "{}",
2469            findings[0].detail
2470        );
2471    }
2472
2473    /// Absence is "not installed", never a finding — a missing store, an
2474    /// empty log directory, and names that parse to no date all stay quiet.
2475    #[test]
2476    fn a_graph_that_never_ran_is_not_a_finding() {
2477        let missing = home("graph-missing").join(".mecha-graph");
2478        assert!(check_graph_nightly(&missing, utc(NOW)).is_empty());
2479
2480        let empty = graph_store("graph-empty");
2481        assert!(check_graph_nightly(&empty, utc(NOW)).is_empty());
2482
2483        let odd = graph_store("graph-odd-names");
2484        nightly_log(&odd, "nightly-garbage.log");
2485        nightly_log(&odd, "gossip-20260812.jsonl");
2486        assert!(check_graph_nightly(&odd, utc(NOW)).is_empty());
2487    }
2488
2489    /// The examine wiring: the store is found as the home's hidden sibling.
2490    #[test]
2491    fn examine_reads_the_graph_store_beside_the_home() {
2492        let scratch = home("graph-sibling");
2493        let mecha_home = scratch.join(".mecha");
2494        std::fs::create_dir_all(&mecha_home).unwrap();
2495        let store = scratch.join(".mecha-graph");
2496        std::fs::create_dir_all(store.join("logs")).unwrap();
2497        nightly_log(&store, "nightly-20260810.log");
2498        let findings = examine(&mecha_home, utc(NOW));
2499        assert_eq!(findings.len(), 1);
2500        assert_eq!(findings[0].component, "graph");
2501        let _ = std::fs::remove_dir_all(&scratch);
2502    }
2503}