Skip to main content

mecha_core/
doctor.rs

1//! `mecha doctor` — every store's distress, read in one pass.
2//!
3//! The incident this exists to end: a revoked OAuth token killed the
4//! scheduling pipeline for three days, and every component recorded its
5//! trouble correctly *in its own store* — an `auth_error.json` marker beside
6//! the credentials, outbox items pending with a release error, frontdoor
7//! requests parked in `awaiting_me`, trigger-ledger rows — while the operator
8//! learned nothing, because nothing reads **across** the stores. Doctor is
9//! that read.
10//!
11//! Two rules carry the design:
12//!
13//! - **Doctor is an observer, never load-bearing.** No network, no model, no
14//!   tokens — and no writes: the stores are read directly rather than through
15//!   the store constructors, because those create and re-chmod their
16//!   directories on open, and an examination that heals the permissions it
17//!   was about to report is measuring itself. Every check is individually
18//!   best-effort: an unreadable or unparseable store is itself a finding
19//!   ("store unreadable: <why>"), never a crash, and one check's failure
20//!   never stops the others.
21//! - **Fixes go through existing commands only.** A [`Remedy`] is an argv —
22//!   `mecha-mail auth personal --provider google`, `mecha outbox review` —
23//!   never a direct mutation of a store. In particular doctor never releases
24//!   an outbox draft: the remedy for stuck drafts is opening the review
25//!   surface, full stop.
26//!
27//! The checks are pure functions over injected store roots and an injected
28//! `now`, which is what makes "a 49-hour-old pending draft" a unit test
29//! instead of a two-day wait.
30
31use chrono::{DateTime, Utc};
32use serde::{Deserialize, Serialize};
33use std::collections::BTreeMap;
34use std::path::Path;
35
36/// How bad a finding is. Declared broken-first so the derived order is the
37/// display order: what is broken outranks what merely wants attention.
38#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Serialize, Deserialize)]
39#[serde(rename_all = "lowercase")]
40pub enum Severity {
41    /// Something is failing right now — a dead login, a release that errored.
42    Broken,
43    /// Nothing is failing, but something has sat unresolved long enough that
44    /// silence is the more likely explanation than intent.
45    Attention,
46}
47
48impl Severity {
49    pub fn as_str(&self) -> &'static str {
50        match self {
51            Severity::Broken => "broken",
52            Severity::Attention => "attention",
53        }
54    }
55}
56
57/// A proposed fix: an existing command, never a store mutation.
58#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
59pub struct Remedy {
60    /// One line saying what running it does — and, where ordering matters,
61    /// what to do first.
62    pub description: String,
63    /// The command as an argv, ready to spawn. Never empty.
64    pub argv: Vec<String>,
65    /// Whether the command needs the real terminal — an OAuth flow, an
66    /// `$EDITOR` — and must therefore inherit stdin and the screen rather
67    /// than being run with its output captured.
68    pub needs_terminal: bool,
69}
70
71/// One observation: which component, how bad, what, and the way out if one
72/// is known.
73#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
74pub struct Finding {
75    pub component: String,
76    pub severity: Severity,
77    pub summary: String,
78    pub detail: String,
79    #[serde(default, skip_serializing_if = "Option::is_none")]
80    pub remedy: Option<Remedy>,
81}
82
83impl Finding {
84    /// The observer rule made a constructor: a store doctor cannot read is a
85    /// finding about that store, never an error that stops the other checks.
86    fn unreadable(component: &str, what: &str, why: impl std::fmt::Display) -> Finding {
87        Finding {
88            component: component.to_string(),
89            severity: Severity::Attention,
90            summary: format!("store unreadable: {what}"),
91            detail: why.to_string(),
92            remedy: None,
93        }
94    }
95}
96
97/// A pending draft older than this with no error has most likely been
98/// forgotten rather than deliberately parked.
99const STUCK_DRAFT_AFTER: chrono::Duration = chrono::Duration::hours(48);
100
101/// A frontdoor request waiting on the user for longer than this is the
102/// stranger-facing silence the front door exists to prevent.
103const STALE_REQUEST_AFTER: chrono::Duration = chrono::Duration::hours(72);
104
105/// Examine every store under `home` and report what is wrong.
106///
107/// `now` is injected for testability; nothing here consults the clock.
108/// Best-effort throughout: each check appends what it found, a failed check
109/// appends a finding about the failure, and no check can stop another.
110pub fn examine(home: &Path, now: DateTime<Utc>) -> Vec<Finding> {
111    let mut findings = Vec::new();
112    findings.extend(check_mail(&home.join("mail")));
113    findings.extend(check_legacy_mail(home));
114    findings.extend(check_outbox(&home.join("outbox"), now));
115    findings.extend(check_frontdoor(&home.join("requests"), now));
116    findings.extend(check_triggers(&home.join("triggers"), now));
117    findings.extend(check_runs(&home.join("sessions")));
118    findings.extend(check_harness(&home.join("learning").join("harness"), now));
119    findings.extend(check_learning(&home.join("learning"), now));
120    // The graph store is `~/.mecha-graph`, a hidden sibling of the mecha home
121    // by that store's own convention — resolved relative to `home` so a test
122    // (or a relocated home) carries its sibling with it.
123    if let Some(parent) = home.parent() {
124        findings.extend(check_graph_nightly(&parent.join(".mecha-graph"), now));
125    }
126    sort(&mut findings);
127    findings
128}
129
130/// Severity first, then component, then insertion order — the shape both the
131/// renderer and the JSON output present.
132pub fn sort(findings: &mut [Finding]) {
133    findings.sort_by(|a, b| {
134        a.severity
135            .cmp(&b.severity)
136            .then_with(|| a.component.cmp(&b.component))
137    });
138}
139
140// --- dead mail auth ---------------------------------------------------------
141
142/// `auth_error.json`, structurally. The writer is `mecha-mail`'s token
143/// lifecycle; the seam is a file of JSON exactly like the frontdoor's
144/// directory-of-JSON, which is why core takes no `mecha-mail` dependency to
145/// read it.
146#[derive(Debug, Deserialize)]
147struct AuthMarker {
148    at: String,
149    message: String,
150}
151
152/// `accounts.toml`, structurally, for the same reason. Only the fields doctor
153/// needs; unknown ones are ignored.
154#[derive(Debug, Default, Deserialize)]
155struct MailAccounts {
156    #[serde(default, rename = "account")]
157    accounts: Vec<MailAccount>,
158}
159
160#[derive(Debug, Deserialize)]
161struct MailAccount {
162    name: String,
163    provider: String,
164    /// Declared lifetime of the refresh credential, in days. See
165    /// `mecha_mail::accounts::AccountEntry::grant_lifetime_days` — absent
166    /// means no known expiry, and no warning.
167    #[serde(default)]
168    grant_lifetime_days: Option<u32>,
169}
170
171/// Scan `<mail>/*/auth_error.json`. Presence means a *permanent* refresh
172/// failure — the marker is written on `invalid_grant` and cleared by the next
173/// successful credential save — so a marker is Broken, not a maybe.
174fn check_mail(mail: &Path) -> Vec<Finding> {
175    let mut out = Vec::new();
176    if !mail.is_dir() {
177        return out;
178    }
179
180    // Provider per account, best-effort: an unparseable registry costs the
181    // `--provider` flag on the remedy, never the finding itself.
182    let declared: Vec<MailAccount> = std::fs::read_to_string(mail.join("accounts.toml"))
183        .ok()
184        .and_then(|text| toml::from_str::<MailAccounts>(&text).ok())
185        .map(|file| file.accounts)
186        .unwrap_or_default();
187    let providers: BTreeMap<String, String> = declared
188        .iter()
189        .map(|a| (a.name.clone(), a.provider.clone()))
190        .collect();
191    let lifetimes: BTreeMap<String, u32> = declared
192        .iter()
193        .filter_map(|a| a.grant_lifetime_days.map(|d| (a.name.clone(), d)))
194        .collect();
195
196    let entries = match std::fs::read_dir(mail) {
197        Ok(entries) => entries,
198        Err(e) => {
199            out.push(Finding::unreadable(
200                "mail",
201                "the mail directory",
202                format!("{}: {e}", mail.display()),
203            ));
204            return out;
205        }
206    };
207
208    for entry in entries.flatten() {
209        let dir = entry.path();
210        if !dir.is_dir() {
211            continue;
212        }
213        let Some(account) = dir.file_name().and_then(|n| n.to_str()).map(String::from) else {
214            continue;
215        };
216        // A grant that predates the triage scopes refreshes cleanly forever
217        // and then 403s the first time something archives. That failure has
218        // no marker — nothing has gone wrong yet — so it is read off the
219        // credential file directly, structurally, like everything else here.
220        out.extend(check_triage_scope(&dir, &account, providers.get(&account)));
221        out.extend(check_grant_age(
222            &dir,
223            &account,
224            providers.get(&account),
225            lifetimes.get(&account).copied(),
226        ));
227
228        let marker_path = dir.join("auth_error.json");
229        if !marker_path.is_file() {
230            continue;
231        }
232        let text = match std::fs::read_to_string(&marker_path) {
233            Ok(text) => text,
234            Err(e) => {
235                out.push(Finding::unreadable(
236                    "mail",
237                    &format!("auth_error.json for `{account}`"),
238                    format!("{}: {e}", marker_path.display()),
239                ));
240                continue;
241            }
242        };
243        match serde_json::from_str::<AuthMarker>(&text) {
244            Ok(marker) => {
245                let provider = providers.get(&account);
246                let mut argv = vec![
247                    "mecha-mail".to_string(),
248                    "auth".to_string(),
249                    account.clone(),
250                ];
251                if let Some(provider) = provider {
252                    argv.push("--provider".to_string());
253                    argv.push(provider.clone());
254                }
255                out.push(Finding {
256                    component: "mail".to_string(),
257                    severity: Severity::Broken,
258                    summary: format!("mail auth for `{account}` is dead"),
259                    // The marker's message already names the exact re-auth
260                    // command, so it rides in the detail — which also covers
261                    // the case where accounts.toml could not say which
262                    // provider the remedy needs.
263                    detail: format!(
264                        "permanent refresh failure since {}: {}",
265                        marker.at, marker.message
266                    ),
267                    remedy: Some(Remedy {
268                        description: format!(
269                            "re-authenticate the `{account}` account (opens an OAuth flow)"
270                        ),
271                        argv,
272                        needs_terminal: true,
273                    }),
274                });
275            }
276            Err(e) => out.push(Finding::unreadable(
277                "mail",
278                &format!("auth_error.json for `{account}` did not parse"),
279                format!("{}: {e}", marker_path.display()),
280            )),
281        }
282    }
283    out
284}
285
286/// The scope a grant was minted with, as `mecha-mail` records it.
287///
288/// Read structurally rather than through `mecha-mail`'s type, for the reason
289/// the whole module gives: doctor takes no dependency on the crates it
290/// examines, and a field it does not know about must not stop it reading the
291/// one it does.
292#[derive(Debug, serde::Deserialize)]
293struct StoredGrant {
294    #[serde(default)]
295    granted_scopes: Option<String>,
296    #[serde(default)]
297    granted_at: Option<String>,
298}
299
300/// How many days before a grant expires to start saying so.
301///
302/// Two, because the remedy is a two-minute re-auth that needs a human at a
303/// terminal — long enough to survive a weekend-adjacent lapse, short enough
304/// that it is not background noise on a 7-day cycle. A warning that fires
305/// for most of the grant's life is a warning nobody reads.
306const GRANT_WARN_WITHIN_DAYS: i64 = 2;
307
308/// Warn before a grant with a known, fixed lifetime expires.
309///
310/// This exists because of the 2026-08-11 outage: Google expires the refresh
311/// token of an app in *Testing* publishing status exactly 7 days after
312/// consent, returns `invalid_grant` when it does — indistinguishable from a
313/// revocation — and scheduling went down for three days. Doctor reported it
314/// correctly *after* the fact. A recurring, dated failure deserves to be
315/// reported before it happens, which is the one thing a marker written on
316/// failure can never do.
317///
318/// Silent unless the lifetime was declared: see
319/// `AccountEntry::grant_lifetime_days` for why this is not inferred.
320fn check_grant_age(
321    dir: &Path,
322    account: &str,
323    provider: Option<&String>,
324    lifetime_days: Option<u32>,
325) -> Vec<Finding> {
326    let Some(lifetime) = lifetime_days.filter(|d| *d > 0) else {
327        return Vec::new();
328    };
329    let Ok(text) = std::fs::read_to_string(dir.join("oauth.json")) else {
330        return Vec::new();
331    };
332    let Ok(grant) = serde_json::from_str::<StoredGrant>(&text) else {
333        return Vec::new(); // already reported by the scope check
334    };
335    // An un-stamped grant predates the field. Its age is genuinely unknown,
336    // and inventing one would either cry wolf or promise safety — so say
337    // nothing and let the next re-auth start the clock honestly.
338    let Some(granted_at) = grant.granted_at.as_deref() else {
339        return Vec::new();
340    };
341    let Ok(granted) = chrono::DateTime::parse_from_rfc3339(granted_at) else {
342        return Vec::new();
343    };
344    let expires = granted.with_timezone(&chrono::Utc) + chrono::Duration::days(lifetime as i64);
345    // Hours, then round *up* to whole days. `num_days()` truncates toward
346    // zero, so a grant with 47 hours left reports "1 day" — which is both
347    // wrong and the wrong direction, since it makes the warning look more
348    // urgent than it is and then says "1 day" again tomorrow.
349    let hours_left = (expires - chrono::Utc::now()).num_hours();
350    let left = (hours_left as f64 / 24.0).ceil() as i64;
351    if left > GRANT_WARN_WITHIN_DAYS {
352        return Vec::new();
353    }
354    let when = if hours_left < 0 {
355        "has expired".to_string()
356    } else if hours_left < 24 {
357        "expires within a day".to_string()
358    } else {
359        format!("expires in {left} days")
360    };
361    let mut argv = vec![
362        "mecha-mail".to_string(),
363        "auth".to_string(),
364        account.to_string(),
365    ];
366    if let Some(p) = provider {
367        argv.push("--provider".to_string());
368        argv.push(p.clone());
369    }
370    vec![Finding {
371        component: "mail".to_string(),
372        severity: Severity::Attention,
373        summary: format!("`{account}` sign-in {when}"),
374        detail: format!(
375            "this grant lasts {lifetime} days from consent ({granted_at}) and refreshing does \
376             not extend it. Re-authenticate before it lapses — once it does, the failure looks \
377             like a revoked token and every scheduled run using this account stops."
378        ),
379        remedy: Some(Remedy {
380            description: format!("re-authenticate `{account}` now (opens an OAuth flow)"),
381            argv,
382            needs_terminal: true,
383        }),
384    }]
385}
386
387/// Which scope each provider needs before the triage verbs work. Mirrors
388/// `mecha_mail::token::triage_scope_for`; duplicated rather than imported
389/// because the seam here is a directory of JSON, not a crate dependency.
390fn triage_scope_for(provider: &str) -> Option<&'static str> {
391    match provider {
392        "google" => Some("gmail.modify"),
393        "outlook" | "microsoft" => Some("Mail.ReadWrite"),
394        _ => None,
395    }
396}
397
398/// Report an account whose OAuth grant does not cover archive/spam/read-state.
399///
400/// Only reported when the provider is known: guessing which scope a grant
401/// should carry would turn an unrecognised provider into a permanent false
402/// finding, and a doctor that cries wolf stops being read. An **absent**
403/// `granted_scopes` counts as not covered, which is correct rather than
404/// harsh — every grant written before the field existed predates the scopes
405/// too.
406///
407/// `Attention`, not `Broken`: nothing is failing right now — mail reads,
408/// sends and stages drafts exactly as before — but the first archive will
409/// fail, and that is precisely the "silence is the likely explanation"
410/// shape this severity is for. On a managed Microsoft tenant the remedy may
411/// also need an administrator rather than the user, so the detail says so
412/// instead of implying a re-auth alone will fix it.
413fn check_triage_scope(dir: &Path, account: &str, provider: Option<&String>) -> Vec<Finding> {
414    let Some(provider) = provider else {
415        return Vec::new();
416    };
417    let Some(needed) = triage_scope_for(provider) else {
418        return Vec::new();
419    };
420    let path = dir.join("oauth.json");
421    let Ok(text) = std::fs::read_to_string(&path) else {
422        // No credentials is not a scope problem; the account simply is not
423        // signed in, which other checks and the first real call will say.
424        return Vec::new();
425    };
426    let Ok(grant) = serde_json::from_str::<StoredGrant>(&text) else {
427        return vec![Finding::unreadable(
428            "mail",
429            &format!("oauth.json for `{account}` did not parse"),
430            format!("{}", path.display()),
431        )];
432    };
433    if grant
434        .granted_scopes
435        .as_deref()
436        .is_some_and(|g| g.contains(needed))
437    {
438        return Vec::new();
439    }
440    let admin_note = if provider == "outlook" || provider == "microsoft" {
441        " Microsoft blocks `Mail.ReadWrite` from end-user consent under its \
442         recommended policy, so on a managed tenant an administrator has to \
443         grant it to the app registration before this can succeed."
444    } else {
445        ""
446    };
447    vec![Finding {
448        component: "mail".to_string(),
449        severity: Severity::Attention,
450        summary: format!("`{account}` cannot archive, spam or mark mail read"),
451        detail: format!(
452            "the stored grant does not include `{needed}`, so mail_triage will fail on this \
453             account. Reading, sending and calendar work are unaffected.{admin_note}"
454        ),
455        remedy: Some(Remedy {
456            description: format!(
457                "re-authenticate `{account}` to add the triage scope (opens an OAuth flow)"
458            ),
459            argv: vec![
460                "mecha-mail".to_string(),
461                "auth".to_string(),
462                account.to_string(),
463                "--provider".to_string(),
464                provider.clone(),
465            ],
466            needs_terminal: true,
467        }),
468    }]
469}
470
471#[cfg(test)]
472mod grant_age_tests {
473    use super::*;
474
475    fn store(dir: &Path, granted_at: Option<&str>) {
476        std::fs::create_dir_all(dir).unwrap();
477        let stamp = granted_at
478            .map(|g| format!(r#","granted_at":"{g}""#))
479            .unwrap_or_default();
480        std::fs::write(
481            dir.join("oauth.json"),
482            format!(r#"{{"client_id":"i","access_token":"a","refresh_token":"r","expires_at":1{stamp}}}"#),
483        )
484        .unwrap();
485    }
486
487    fn days_ago(n: i64) -> String {
488        (chrono::Utc::now() - chrono::Duration::days(n)).to_rfc3339()
489    }
490
491    /// The 7-day Testing clock, reported before it fires rather than after.
492    #[test]
493    fn a_grant_nearing_its_declared_lifetime_is_reported_early() {
494        let tmp = std::env::temp_dir().join(format!("mecha-grant-{}", std::process::id()));
495        let g = "google".to_string();
496
497        // Fresh: silent. A warning that fires all week is not a warning.
498        store(&tmp, Some(&days_ago(1)));
499        assert!(check_grant_age(&tmp, "personal", Some(&g), Some(7)).is_empty());
500
501        // Day 5 of 7 — two days left, inside the window.
502        store(&tmp, Some(&days_ago(5)));
503        let f = check_grant_age(&tmp, "personal", Some(&g), Some(7));
504        assert_eq!(f.len(), 1, "should warn with 2 days left");
505        assert!(
506            f[0].summary.contains("expires in 2 days"),
507            "{}",
508            f[0].summary
509        );
510        assert!(f[0].remedy.as_ref().unwrap().needs_terminal);
511
512        // Under 24h: worded without a misleading whole-day count.
513        store(&tmp, Some(&days_ago(7)));
514        let f = check_grant_age(&tmp, "personal", Some(&g), Some(7));
515        assert!(f[0].summary.contains("within a day"), "{}", f[0].summary);
516
517        // Past it: still a finding, worded as past.
518        store(&tmp, Some(&days_ago(9)));
519        let f = check_grant_age(&tmp, "personal", Some(&g), Some(7));
520        assert!(f[0].summary.contains("has expired"), "{}", f[0].summary);
521
522        // No declared lifetime: silent however old. Not inferred, ever.
523        assert!(check_grant_age(&tmp, "personal", Some(&g), None).is_empty());
524
525        // Un-stamped grant: age unknown, so no claim either way.
526        store(&tmp, None);
527        assert!(check_grant_age(&tmp, "personal", Some(&g), Some(7)).is_empty());
528
529        std::fs::remove_dir_all(&tmp).ok();
530    }
531}
532
533/// The legacy per-provider stores — `<home>/google/oauth.json` and
534/// `<home>/outlook/oauth.json`, still served by the shipped `mecha-google`
535/// and `mecha-outlook` binaries and what `mecha-mail import` exists to
536/// migrate — get the same marker written beside their credentials by the
537/// same token lifecycle. A doctor that reads only the registry layout
538/// reports "all clear" over a dead legacy login.
539fn check_legacy_mail(home: &Path) -> Vec<Finding> {
540    let mut out = Vec::new();
541    for provider in ["google", "outlook"] {
542        let marker_path = home.join(provider).join("auth_error.json");
543        if !marker_path.is_file() {
544            continue;
545        }
546        let text = match std::fs::read_to_string(&marker_path) {
547            Ok(text) => text,
548            Err(e) => {
549                out.push(Finding::unreadable(
550                    "mail",
551                    &format!("auth_error.json for the legacy {provider} store"),
552                    format!("{}: {e}", marker_path.display()),
553                ));
554                continue;
555            }
556        };
557        match serde_json::from_str::<AuthMarker>(&text) {
558            Ok(marker) => out.push(Finding {
559                component: "mail".to_string(),
560                severity: Severity::Broken,
561                summary: format!("legacy {provider} mail auth is dead"),
562                // The marker's message names the exact re-auth command (the
563                // writer derives it from the store's directory), so it rides
564                // in the detail.
565                detail: format!(
566                    "permanent refresh failure since {}: {}",
567                    marker.at, marker.message
568                ),
569                remedy: Some(Remedy {
570                    description: format!(
571                        "bring the legacy {provider} login into the unified registry — \
572                         and re-authenticate it per the detail, which no import fixes"
573                    ),
574                    argv: vec![
575                        "mecha-mail".to_string(),
576                        "import".to_string(),
577                        provider.to_string(),
578                        "--provider".to_string(),
579                        provider.to_string(),
580                    ],
581                    needs_terminal: false,
582                }),
583            }),
584            Err(e) => out.push(Finding::unreadable(
585                "mail",
586                &format!("auth_error.json for the legacy {provider} store did not parse"),
587                format!("{}: {e}", marker_path.display()),
588            )),
589        }
590    }
591    out
592}
593
594// --- stuck outbox items -----------------------------------------------------
595
596/// Read the outbox items directly — one JSON file per item, the store's own
597/// on-disk contract — so that examining the store never creates or re-chmods
598/// it the way [`crate::outbox::OutboxStore::open`] deliberately does.
599fn check_outbox(root: &Path, now: DateTime<Utc>) -> Vec<Finding> {
600    let mut out = Vec::new();
601    if !root.is_dir() {
602        return out;
603    }
604    let entries = match std::fs::read_dir(root) {
605        Ok(entries) => entries,
606        Err(e) => {
607            out.push(Finding::unreadable(
608                "outbox",
609                "the outbox directory",
610                format!("{}: {e}", root.display()),
611            ));
612            return out;
613        }
614    };
615
616    let review = Remedy {
617        description: "open the outbox review surface — doctor never releases a draft".to_string(),
618        argv: vec!["mecha".into(), "outbox".into(), "review".into()],
619        needs_terminal: true,
620    };
621
622    let mut stale: Vec<String> = Vec::new();
623    for entry in entries.flatten() {
624        let path = entry.path();
625        if path.extension().and_then(|e| e.to_str()) != Some("json") {
626            continue;
627        }
628        let item: crate::outbox::OutboxItem =
629            match std::fs::read_to_string(&path).map(|t| serde_json::from_str(&t)) {
630                Ok(Ok(item)) => item,
631                Ok(Err(e)) => {
632                    out.push(Finding::unreadable(
633                        "outbox",
634                        &format!(
635                            "item {} did not parse",
636                            path.file_name().unwrap_or_default().to_string_lossy()
637                        ),
638                        format!("{}: {e}", path.display()),
639                    ));
640                    continue;
641                }
642                Err(e) => {
643                    out.push(Finding::unreadable(
644                        "outbox",
645                        &format!(
646                            "item {} could not be read",
647                            path.file_name().unwrap_or_default().to_string_lossy()
648                        ),
649                        format!("{}: {e}", path.display()),
650                    ));
651                    continue;
652                }
653            };
654        if item.status != "pending" {
655            continue;
656        }
657        if let Some(error) = &item.error {
658            out.push(Finding {
659                component: "outbox".to_string(),
660                severity: Severity::Broken,
661                summary: format!("release failed: {error}"),
662                detail: format!(
663                    "{} · {} — still pending; the draft is good, the delivery was not",
664                    item.id, item.summary
665                ),
666                remedy: Some(review.clone()),
667            });
668        } else if age_of(&item.created_at, now).is_some_and(|age| age > STUCK_DRAFT_AFTER) {
669            stale.push(format!(
670                "{} · {} — staged {}",
671                item.id,
672                item.summary,
673                render_age(now, &item.created_at)
674            ));
675        }
676    }
677
678    if !stale.is_empty() {
679        // read_dir order is arbitrary; ids sort by staging time.
680        stale.sort();
681        out.push(Finding {
682            component: "outbox".to_string(),
683            severity: Severity::Attention,
684            summary: format!(
685                "{} draft{} pending for more than 48h",
686                stale.len(),
687                if stale.len() == 1 { "" } else { "s" }
688            ),
689            detail: stale.join("\n"),
690            remedy: Some(review),
691        });
692    }
693    out
694}
695
696// --- frontdoor --------------------------------------------------------------
697
698/// The states that mean a request is waiting on the user rather than on the
699/// requester: `extracted` awaits triage, `awaiting_me` awaits a draft review,
700/// and `triaged` is triage's "I drafted nothing — this needs a person":
701/// nothing ever re-triages it, so left alone it waits forever, invisibly.
702/// (`needs_info` waits on the stranger, and `drained` on the extraction pass.)
703const WAITING_ON_ME: [&str; 3] = [
704    crate::frontdoor::EXTRACTED,
705    crate::frontdoor::AWAITING_ME,
706    crate::frontdoor::TRIAGED,
707];
708
709/// Read the request records directly, for the same no-side-effects reason as
710/// the outbox — [`crate::frontdoor::Frontdoor::open`] creates the directory.
711fn check_frontdoor(root: &Path, now: DateTime<Utc>) -> Vec<Finding> {
712    let mut out = Vec::new();
713    if !root.is_dir() {
714        return out;
715    }
716    let entries = match std::fs::read_dir(root) {
717        Ok(entries) => entries,
718        Err(e) => {
719            out.push(Finding::unreadable(
720                "frontdoor",
721                "the request store",
722                format!("{}: {e}", root.display()),
723            ));
724            return out;
725        }
726    };
727
728    let list = Remedy {
729        description: "list the frontdoor queue".to_string(),
730        argv: vec!["mecha".into(), "frontdoor".into(), "list".into()],
731        needs_terminal: false,
732    };
733
734    let mut stale: Vec<(i64, String)> = Vec::new();
735    for entry in entries.flatten() {
736        let path = entry.path();
737        if path.extension().and_then(|e| e.to_str()) != Some("json") {
738            continue;
739        }
740        let Ok(Ok(record)) = std::fs::read_to_string(&path)
741            .map(|t| serde_json::from_str::<crate::frontdoor::Record>(&t))
742        else {
743            // The frontdoor store itself skips unreadable records; doctor
744            // says so instead, because silent skipping is the disease here.
745            out.push(Finding::unreadable(
746                "frontdoor",
747                &format!(
748                    "request {} did not parse",
749                    path.file_name().unwrap_or_default().to_string_lossy()
750                ),
751                path.display().to_string(),
752            ));
753            continue;
754        };
755        if record.state == crate::frontdoor::EXTRACTION_FAILED {
756            out.push(Finding {
757                component: "frontdoor".to_string(),
758                severity: Severity::Broken,
759                summary: format!(
760                    "request {} failed extraction and waits for a human",
761                    record.seq
762                ),
763                detail: format!(
764                    "{} ({}) — {}",
765                    record.seq,
766                    record.type_id,
767                    record
768                        .extraction_error
769                        .as_deref()
770                        .unwrap_or("no error recorded")
771                ),
772                remedy: Some(list.clone()),
773            });
774        } else if WAITING_ON_ME.contains(&record.state.as_str())
775            && request_age(&record, now).is_some_and(|age| age > STALE_REQUEST_AFTER)
776        {
777            stale.push((
778                record.seq,
779                format!(
780                    "{} ({}) — {}, received {}",
781                    record.seq,
782                    record.type_id,
783                    record.state,
784                    render_age(now, &record.created_at)
785                ),
786            ));
787        }
788    }
789
790    if !stale.is_empty() {
791        // read_dir order is arbitrary; the queue reads oldest-first by seq.
792        stale.sort_by_key(|(seq, _)| *seq);
793        out.push(Finding {
794            component: "frontdoor".to_string(),
795            severity: Severity::Attention,
796            summary: format!(
797                "{} request{} waiting on you for more than 72h",
798                stale.len(),
799                if stale.len() == 1 { "" } else { "s" }
800            ),
801            detail: stale
802                .into_iter()
803                .map(|(_, line)| line)
804                .collect::<Vec<_>>()
805                .join("\n"),
806            remedy: Some(list),
807        });
808    }
809    out
810}
811
812/// How long a request has waited: from when it arrived here (`drained_at`),
813/// falling back to when the stranger sent it. Unparseable stamps mean the age
814/// is unknown, and unknown never counts as stale — a doctor that guesses is
815/// worse than one that says nothing.
816fn request_age(record: &crate::frontdoor::Record, now: DateTime<Utc>) -> Option<chrono::Duration> {
817    age_of(&record.drained_at, now).or_else(|| age_of(&record.created_at, now))
818}
819
820// --- trigger health ---------------------------------------------------------
821
822/// How many recent runs the reliability check averages over.
823///
824/// Five, because one bad morning is not a trend and a long window would hide a
825/// trigger that broke this week behind a month of health.
826const HEALTH_WINDOW: usize = 5;
827
828/// Below this many calls in the window, no rate is reported.
829///
830/// A rate over three calls is noise, and a doctor that cries wolf stops being
831/// read — the same reasoning as the scope check declining to guess.
832const HEALTH_MIN_CALLS: u32 = 10;
833
834/// The share of failed calls that is worth a human's attention.
835///
836/// A third. Deliberately not near-zero: a model that tries a path, is told it
837/// does not exist, and tries the right one has done nothing wrong, and errors
838/// are how a run learns about its environment. What this is looking for is a
839/// trigger whose environment has moved out from under it.
840const HEALTH_ERROR_RATE: f64 = 1.0 / 3.0;
841
842/// Read the trigger files and the ledger directly — same reason as above:
843/// [`crate::trigger::TriggerStore::open`] creates and re-chmods the root.
844fn check_triggers(root: &Path, now: DateTime<Utc>) -> Vec<Finding> {
845    let mut out = Vec::new();
846    if !root.is_dir() {
847        return out;
848    }
849    let entries = match std::fs::read_dir(root) {
850        Ok(entries) => entries,
851        Err(e) => {
852            out.push(Finding::unreadable(
853                "triggers",
854                "the trigger store",
855                format!("{}: {e}", root.display()),
856            ));
857            return out;
858        }
859    };
860
861    let mut triggers: Vec<crate::trigger::Trigger> = Vec::new();
862    for entry in entries.flatten() {
863        let path = entry.path();
864        if path.extension().and_then(|e| e.to_str()) != Some("toml") {
865            continue;
866        }
867        let name = path
868            .file_stem()
869            .and_then(|s| s.to_str())
870            .unwrap_or_default()
871            .to_string();
872        match std::fs::read_to_string(&path).map(|t| toml::from_str::<crate::trigger::Trigger>(&t))
873        {
874            Ok(Ok(mut trigger)) => {
875                trigger.name = name;
876                triggers.push(trigger);
877            }
878            _ => out.push(Finding::unreadable(
879                "triggers",
880                &format!("trigger file `{name}.toml` did not parse"),
881                path.display().to_string(),
882            )),
883        }
884    }
885
886    // One ledger scan for both questions: the newest row that actually *ran*
887    // per trigger, and the newest *accounted slot* per trigger (manual runs
888    // carry no slot, so they are invisible to the schedule on purpose).
889    let mut recent: BTreeMap<String, Vec<crate::trigger::RunRecord>> = BTreeMap::new();
890    let mut last_slot: BTreeMap<String, DateTime<Utc>> = BTreeMap::new();
891    let ledger = root.join("runs.jsonl");
892    if ledger.is_file() {
893        match std::fs::read_to_string(&ledger) {
894            Ok(text) => {
895                for line in text.lines().filter(|l| !l.trim().is_empty()) {
896                    // A torn line is the store's problem, not this row's
897                    // neighbours': skip it the way the ledger reader does.
898                    let Ok(row) = serde_json::from_str::<crate::trigger::RunRecord>(line) else {
899                        continue;
900                    };
901                    if let Some(slot) = row.slot {
902                        let newest = last_slot.entry(row.trigger.clone()).or_insert(slot);
903                        if slot > *newest {
904                            *newest = slot;
905                        }
906                    }
907                    // A skip is a row, not a run: a skipped-stale or
908                    // skipped-overlap appended after an error is bookkeeping,
909                    // not a recovery, and keying on the literal last row let
910                    // it hide the failure the operator needed to see.
911                    if matches!(
912                        row.status,
913                        crate::trigger::RunStatus::Ok | crate::trigger::RunStatus::Error
914                    ) {
915                        let window = recent.entry(row.trigger.clone()).or_default();
916                        window.push(row);
917                        if window.len() > HEALTH_WINDOW {
918                            window.remove(0);
919                        }
920                    }
921                }
922            }
923            Err(e) => out.push(Finding::unreadable(
924                "triggers",
925                "the run ledger",
926                format!("{}: {e}", ledger.display()),
927            )),
928        }
929    }
930
931    for trigger in &triggers {
932        if !trigger.enabled {
933            continue;
934        }
935
936        // The most recent run failed: a manual run is the safe probe, because
937        // it records a row with no slot and so never advances the schedule.
938        let window = recent.get(&trigger.name);
939        if let Some(row) = window.and_then(|w| w.last()) {
940            if row.status == crate::trigger::RunStatus::Error {
941                out.push(Finding {
942                    component: "triggers".to_string(),
943                    severity: Severity::Attention,
944                    summary: format!("trigger `{}`'s most recent run failed", trigger.name),
945                    detail: format!(
946                        "started {}: {}",
947                        row.started_at.to_rfc3339(),
948                        row.error.as_deref().unwrap_or("no error recorded")
949                    ),
950                    remedy: Some(Remedy {
951                        description: format!(
952                            "run `{}` by hand — a manual run is evidence, not a fire; it never advances the schedule",
953                            trigger.name
954                        ),
955                        argv: vec![
956                            "mecha".into(),
957                            "trigger".into(),
958                            "run".into(),
959                            trigger.name.clone(),
960                        ],
961                        needs_terminal: false,
962                    }),
963                });
964            }
965        }
966
967        // Reliability across the window. An unattended run has nobody
968        // watching it fail: the briefing still arrives, the ledger still says
969        // `ok`, and a trigger failing a third of its calls looks exactly like
970        // one that works. Silent below a floor of calls, because a rate over
971        // three of them is noise, and unknown is never a finding.
972        let (calls, errors) = window
973            .map(|w| {
974                w.iter().fold((0u32, 0u32), |(c, e), r| {
975                    (c + r.tool_calls, e + r.tool_errors)
976                })
977            })
978            .unwrap_or((0, 0));
979        if calls >= HEALTH_MIN_CALLS && f64::from(errors) / f64::from(calls) >= HEALTH_ERROR_RATE {
980            let runs = window.map(Vec::len).unwrap_or(0);
981            out.push(Finding {
982                component: "triggers".to_string(),
983                severity: Severity::Attention,
984                summary: format!(
985                    "trigger `{}` failed {errors} of {calls} tool calls",
986                    trigger.name
987                ),
988                detail: format!(
989                    "across its last {runs} run(s){}. A run's answer arrives either way, so this is invisible in the ledger's status — and per-step reliability is what decides how long a task the run can finish, so a third of the calls failing is not a third of the work lost.",
990                    if window.is_some_and(|w| w.last().is_some_and(|r| r.ended_on_failed_call)) {
991                        ", and the most recent run answered with its last call failed"
992                    } else {
993                        ""
994                    }
995                ),
996                // Reading is the remedy: what to change is in the transcript,
997                // and doctor never decides that.
998                remedy: Some(Remedy {
999                    description: format!("read `{}`'s recent runs", trigger.name),
1000                    argv: vec![
1001                        "mecha".into(),
1002                        "trigger".into(),
1003                        "show".into(),
1004                        trigger.name.clone(),
1005                    ],
1006                    needs_terminal: false,
1007                }),
1008            });
1009        }
1010
1011        // The null run: it fired, it succeeded, and it did nothing. The rate
1012        // check above cannot see this one — a rate over zero calls is
1013        // undefined rather than bad — so a trigger that made thirty calls a
1014        // morning and now makes none is silent in every signal the ledger
1015        // carries. Found by a sibling arc hitting the same shape one layer
1016        // down, where `mecha mail classify` returned success having classified
1017        // 0 of 16.
1018        //
1019        // Measured against the trigger's *own* history, never an absolute
1020        // floor: a prompt that legitimately needs no tools makes zero calls
1021        // every morning, and a check that called that broken would be wrong
1022        // about the healthiest trigger on the machine. So the earlier runs in
1023        // the window have to show the work that stopped.
1024        if let Some(window) = window {
1025            let newest = window.last();
1026            let before: u32 = window[..window.len().saturating_sub(1)]
1027                .iter()
1028                .map(|r| r.tool_calls)
1029                .sum();
1030            // Only an `ok` run: an errored one already has a finding above,
1031            // and two findings for one fact leave neither meaning anything.
1032            let stopped = newest
1033                .is_some_and(|r| r.tool_calls == 0 && r.status == crate::trigger::RunStatus::Ok)
1034                && before >= HEALTH_MIN_CALLS;
1035            if stopped {
1036                out.push(Finding {
1037                    component: "triggers".to_string(),
1038                    severity: Severity::Attention,
1039                    summary: format!(
1040                        "trigger `{}`'s most recent run did no work",
1041                        trigger.name
1042                    ),
1043                    detail: format!(
1044                        "it succeeded having made no tool calls, where its previous {} run(s) made {before}. A run that does nothing and reports success is indistinguishable from a healthy one in every other signal — the status is `ok`, the schedule advanced, and the answer arrived.",
1045                        window.len() - 1
1046                    ),
1047                    remedy: Some(Remedy {
1048                        description: format!("read `{}`'s recent runs", trigger.name),
1049                        argv: vec![
1050                            "mecha".into(),
1051                            "trigger".into(),
1052                            "show".into(),
1053                            trigger.name.clone(),
1054                        ],
1055                        needs_terminal: false,
1056                    }),
1057                });
1058            }
1059        }
1060
1061        // A catch-up-always trigger whose accounted slots stopped advancing:
1062        // a healthy daemon fires the most recent slot every tick, so more
1063        // than two slots newer than the last accounted one means nothing is
1064        // ticking at all. Cheap by construction — three `prev_at_or_before`
1065        // calls, no schedule re-derivation.
1066        if trigger.catch_up != crate::trigger::CatchUp::Always {
1067            continue;
1068        }
1069        let Some(anchor) = last_slot.get(&trigger.name).copied().or(trigger.created_at) else {
1070            // No ledger row and no creation stamp: there is no baseline to
1071            // measure staleness against, and unknown is not stale.
1072            continue;
1073        };
1074        let tz = trigger.tz(None);
1075        let step = chrono::Duration::seconds(1);
1076        let missed_more_than_two = trigger
1077            .schedule
1078            .prev_at_or_before(now, tz)
1079            .and_then(|s0| trigger.schedule.prev_at_or_before(s0 - step, tz))
1080            .and_then(|s1| trigger.schedule.prev_at_or_before(s1 - step, tz))
1081            .is_some_and(|s2| s2 > anchor);
1082        if missed_more_than_two {
1083            out.push(Finding {
1084                component: "triggers".to_string(),
1085                severity: Severity::Attention,
1086                summary: format!("trigger `{}` has missed more than two slots", trigger.name),
1087                detail: format!(
1088                    "last accounted slot {}; with catch_up=always a healthy scheduler fires \
1089                     the most recent slot every tick, so the daemon or its timer may be down \
1090                     (systemctl --user status mecha-triggers)",
1091                    anchor.to_rfc3339()
1092                ),
1093                // No argv on purpose: running the trigger by hand would not
1094                // restart whatever stopped ticking.
1095                remedy: None,
1096            });
1097        }
1098    }
1099    out
1100}
1101
1102// --- run quality ------------------------------------------------------------
1103
1104/// How many sessions back a run-quality check reads.
1105///
1106/// Doctor runs in one pass with no network and no model, and each session is
1107/// a file read — so this is a budget, not a claim about relevance. Two hundred
1108/// covers weeks of ordinary use and stays well inside "fast enough to run
1109/// whenever you wonder".
1110const RUNS_WINDOW: usize = 200;
1111
1112/// Below this many runs *for one model*, no rate is reported.
1113///
1114/// Twenty rather than the trigger check's ten, because these rates are
1115/// population statistics across mixed work rather than one job doing the same
1116/// thing every morning, and the noise is correspondingly higher.
1117const RUNS_MIN: usize = 20;
1118
1119/// The share of runs finishing over a failed call that is worth saying out
1120/// loud. Deliberately high: rule-based evaluators are measured to *under*
1121/// report success — they mark good trajectories as failures more often than
1122/// humans do (AgentRewardBench) — so a low bar here would fire constantly on
1123/// runs that were fine, and a doctor that cries wolf stops being read.
1124const ENDED_ON_FAILURE_RATE: f64 = 0.20;
1125
1126/// The share of attempted tool calls the environment refuses.
1127const TOOL_ERROR_RATE: f64 = 0.25;
1128
1129/// And below this many *calls* across the window, no rate at all. Runs and
1130/// calls are different denominators: twenty runs can hold four calls.
1131const RUNS_MIN_CALLS: u64 = 20;
1132
1133/// The share of runs the *harness* cut short. `Interrupted` is excluded from
1134/// the numerator by [`cut_short`]: a person pressing Ctrl-C is the system
1135/// working, and counting it would make an attentive user look like a problem.
1136const CUT_SHORT_RATE: f64 = 0.25;
1137
1138/// Did the harness end this run? One definition, on [`crate::agent::StopCause`],
1139/// shared with the candidate gate's metric — see its doc for why there were two.
1140fn cut_short(stats: &crate::session::RunStats) -> bool {
1141    stats.stop_cause.is_some_and(|c| c.cut_short())
1142}
1143
1144/// Report population-level run quality: the signals that are invisible in any
1145/// single run and obvious across a few hundred.
1146///
1147/// Split by model, because a corpus spanning two has no single rate worth
1148/// quoting — the blend is true and useless, and a threshold on it fires for
1149/// the wrong model. Silent until there is enough of one model to say
1150/// anything, which is the same rule as everywhere else here: unknown is not a
1151/// finding.
1152fn check_runs(sessions: &Path) -> Vec<Finding> {
1153    use crate::runlog::{Corpus, Scan};
1154
1155    let mut out = Vec::new();
1156    if !sessions.is_dir() {
1157        return out;
1158    }
1159    let corpus = match Corpus::scan(
1160        sessions,
1161        &Scan {
1162            max_sessions: Some(RUNS_WINDOW),
1163            since: None,
1164        },
1165    ) {
1166        Ok(c) => c,
1167        Err(e) => {
1168            out.push(Finding::unreadable(
1169                "runs",
1170                "the session store",
1171                format!("{}: {e}", sessions.display()),
1172            ));
1173            return out;
1174        }
1175    };
1176
1177    let remedy = |what: &str| {
1178        Some(Remedy {
1179            description: format!("read the run-quality summary ({what})"),
1180            argv: vec![
1181                "mecha".into(),
1182                "sessions".into(),
1183                "health".into(),
1184                "--days".into(),
1185                "30".into(),
1186            ],
1187            needs_terminal: false,
1188        })
1189    };
1190
1191    for (model, runs) in corpus.by_model() {
1192        if runs.len() < RUNS_MIN {
1193            continue;
1194        }
1195        let n = runs.len();
1196
1197        if let Some(rate) = runs.rate_of(|r| r.stats.ended_on_failed_call) {
1198            if rate >= ENDED_ON_FAILURE_RATE {
1199                out.push(Finding {
1200                    component: "runs".to_string(),
1201                    severity: Severity::Attention,
1202                    summary: format!(
1203                        "{:.0}% of `{model}` runs finished on a failed tool call",
1204                        rate * 100.0
1205                    ),
1206                    detail: format!(
1207                        "{} of {n} recent run(s). The model stopped of its own accord with its last call failed, and the answer it wrote may report success over it — which nothing in the text or the stop reason can show.",
1208                        runs.ended_on_failed_call()
1209                    ),
1210                    remedy: remedy("which runs, and what failed"),
1211                });
1212            }
1213        }
1214
1215        if let Some(rate) = runs.tool_error_rate() {
1216            // The sibling trigger check states the rule this one omitted: a
1217            // rate over three calls is noise. Twenty conversational runs that
1218            // made four calls between them must not raise a finding because
1219            // one of them errored.
1220            if rate >= TOOL_ERROR_RATE && runs.tool_calls() >= RUNS_MIN_CALLS {
1221                out.push(Finding {
1222                    component: "runs".to_string(),
1223                    severity: Severity::Attention,
1224                    summary: format!(
1225                        "`{model}` runs fail {:.0}% of their tool calls",
1226                        rate * 100.0
1227                    ),
1228                    detail: format!(
1229                        "{} of {} call(s) across {n} run(s) were refused by the environment. Errors are how a run learns where it is, so some are healthy — a quarter of them says something moved: a renamed path, a revoked grant, a tool whose schema the model keeps mis-filling.",
1230                        runs.tool_errors(),
1231                        runs.tool_calls()
1232                    ),
1233                    remedy: remedy("which tool, and how it failed"),
1234                });
1235            }
1236        }
1237
1238        if let Some(rate) = runs.rate_of(|r| cut_short(&r.stats)) {
1239            if rate >= CUT_SHORT_RATE {
1240                let cut = runs.rows.iter().filter(|r| cut_short(&r.stats)).count();
1241                out.push(Finding {
1242                    component: "runs".to_string(),
1243                    severity: Severity::Attention,
1244                    summary: format!(
1245                        "the harness cut {:.0}% of `{model}` runs short",
1246                        rate * 100.0
1247                    ),
1248                    detail: format!(
1249                        "{cut} of {n} recent run(s) hit a turn, token or cost ceiling, or tripped the loop guard. A budget that stops a quarter of runs is measuring the budget rather than the work — the answers are truncated and say so only in `stop_cause`. Cancellations are not counted here.",
1250                    ),
1251                    remedy: remedy("which ceiling, and how often"),
1252                });
1253            }
1254        }
1255    }
1256    out
1257}
1258
1259// --- graph nightly silence --------------------------------------------------
1260
1261/// The two daily jobs that keep the knowledge graph current, each of which
1262/// writes `<prefix>YYYYMMDD.log` on *every* run — a deferred night says so in
1263/// the log — so a day with no file means the script never started. That is
1264/// exactly the failure cron cannot report: no MTA, and the script's own
1265/// logging begins after the point where an exec failure kills it (measured
1266/// 2026-08-17, when a missing execute bit cost a night of vet and gossip and
1267/// nothing anywhere said so).
1268const GRAPH_NIGHTLIES: &[(&str, &str)] = &[
1269    ("nightly-", "the graph's own sweep (ingest, extract, decay)"),
1270    ("mecha-nightly-", "the mecha half (vet, precheck, gossip)"),
1271];
1272
1273/// A staged harness candidate older than this is the nightly loop waiting on
1274/// a review nobody knows is due — the same shape as a stuck draft, one store
1275/// over. Not blocking anything, hence the longer leash.
1276const STALE_CANDIDATE_AFTER: chrono::Duration = chrono::Duration::hours(72);
1277
1278/// Below this many origin-excluded reflections, a learner that has not run is
1279/// a young install, not a starved one — the silence carries no information.
1280/// High on the doctor rule: a finding that fires on every fresh setup trains
1281/// the reader to skip the component it names.
1282const STARVED_LEARNER_MIN_EXCLUDED: usize = 10;
1283
1284/// The starved learner: reflections keep arriving, the origin gate keeps
1285/// excluding them, and no domain ever reaches `learn`'s floor — so the rule
1286/// learner reports success every night and has produced nothing for weeks.
1287///
1288/// This is the null-run bug one layer up from the trigger version: every
1289/// stage exits 0, the ledger says `ok`, and the only evidence is a
1290/// distribution across a file nothing was reading. The check counts and
1291/// never judges the gate itself — the exclusions are the provenance design
1292/// working as specified — so the finding proposes a *decision*, not a
1293/// command: accept the rate, or change what evidence the loop can use. That
1294/// is why its remedy is the dry-run that shows the classifications, never
1295/// anything that loosens the gate.
1296fn check_learning(root: &Path, now: DateTime<Utc>) -> Vec<Finding> {
1297    let mut out = Vec::new();
1298    let path = root.join("reflections.jsonl");
1299    if !path.is_file() {
1300        return out;
1301    }
1302    let text = match std::fs::read_to_string(&path) {
1303        Ok(t) => t,
1304        Err(e) => {
1305            out.push(Finding::unreadable(
1306                "learning",
1307                "the reflections archive",
1308                format!("{}: {e}", path.display()),
1309            ));
1310            return out;
1311        }
1312    };
1313
1314    let mut total = 0usize;
1315    let mut excluded = 0usize;
1316    let mut newest_excluded: Option<DateTime<Utc>> = None;
1317    // domain → clean unprocessed count.
1318    let mut waiting: std::collections::BTreeMap<String, usize> = Default::default();
1319    for line in text.lines().filter(|l| !l.trim().is_empty()) {
1320        let Ok(r) = serde_json::from_str::<crate::learning::Reflexion>(line) else {
1321            // One torn line is not the archive; the counts below are still
1322            // a lower bound, which is the fail-quiet direction for a check
1323            // whose finding asks a person to make a decision.
1324            continue;
1325        };
1326        total += 1;
1327        if r.origin == crate::learning::Origin::Clean {
1328            if !r.is_processed {
1329                *waiting.entry(r.domain.clone()).or_default() += 1;
1330            }
1331        } else {
1332            excluded += 1;
1333            if let Ok(t) = DateTime::parse_from_rfc3339(&r.created_at) {
1334                let t = t.with_timezone(&Utc);
1335                if newest_excluded.is_none_or(|n| t > n) {
1336                    newest_excluded = Some(t);
1337                }
1338            }
1339        }
1340    }
1341
1342    let floor = crate::learning::LEARN_MIN_REFLECTIONS;
1343    // Any domain at the floor means learn will consolidate on its next pass:
1344    // not starved, whatever the exclusion count says.
1345    if waiting.values().any(|&n| n >= floor) {
1346        return out;
1347    }
1348    if excluded < STARVED_LEARNER_MIN_EXCLUDED {
1349        return out;
1350    }
1351    // A loop nothing has fed for a month is dormant, not starved — the
1352    // distinction matters because the remedy for dormant is elsewhere
1353    // (triggers, reflect itself), and this finding must not shadow it.
1354    let alive = newest_excluded.is_some_and(|t| now.signed_duration_since(t).num_days() <= 30);
1355    if !alive {
1356        return out;
1357    }
1358
1359    let pool = if waiting.is_empty() {
1360        "none clean and unprocessed".to_string()
1361    } else {
1362        waiting
1363            .iter()
1364            .map(|(d, n)| format!("{d} {n}/{floor}"))
1365            .collect::<Vec<_>>()
1366            .join(", ")
1367    };
1368    out.push(Finding {
1369        component: "learning".to_string(),
1370        severity: Severity::Attention,
1371        summary: format!(
1372            "the rule learner is starved: {excluded} of {total} reflections excluded by \
1373             origin, and no domain reaches the learn floor of {floor}"
1374        ),
1375        detail: format!(
1376            "reflect keeps mining and the provenance gate keeps excluding — the gate working \
1377             as designed, every night, with nothing downstream to show for it. Clean pool: \
1378             {pool}. The excluded evidence stays readable in {}; the decision this proposes \
1379             is yours, not a command's: accept the rate, or change what evidence the loop \
1380             may consolidate.",
1381            path.display()
1382        ),
1383        remedy: Some(Remedy {
1384            description: "see how new interventions classify — doctor never loosens the gate"
1385                .to_string(),
1386            argv: vec!["mecha".into(), "reflect".into(), "--dry-run".into()],
1387            needs_terminal: false,
1388        }),
1389    });
1390    out
1391}
1392
1393/// Scan `<learning>/harness/candidates` for staged candidates waiting on the
1394/// user. Quiet when the store has never existed — the loop not being wired is
1395/// not a finding. Reads the files directly, on the rule that an examination
1396/// must not heal (or create) what it reports on.
1397fn check_harness(root: &Path, now: DateTime<Utc>) -> Vec<Finding> {
1398    let mut out = Vec::new();
1399    let dir = root.join("candidates");
1400    if !dir.is_dir() {
1401        return out;
1402    }
1403    let entries = match std::fs::read_dir(&dir) {
1404        Ok(entries) => entries,
1405        Err(e) => {
1406            out.push(Finding::unreadable(
1407                "harness",
1408                "the harness candidate directory",
1409                format!("{}: {e}", dir.display()),
1410            ));
1411            return out;
1412        }
1413    };
1414    let mut stale: Vec<String> = Vec::new();
1415    for entry in entries.flatten() {
1416        let path = entry.path();
1417        if path.extension().and_then(|e| e.to_str()) != Some("json") {
1418            continue;
1419        }
1420        let cand: crate::harness::HarnessCandidate =
1421            match std::fs::read_to_string(&path).map(|t| serde_json::from_str(&t)) {
1422                Ok(Ok(cand)) => cand,
1423                Ok(Err(e)) => {
1424                    out.push(Finding::unreadable(
1425                        "harness",
1426                        &format!(
1427                            "candidate {} did not parse",
1428                            path.file_name().unwrap_or_default().to_string_lossy()
1429                        ),
1430                        format!("{}: {e}", path.display()),
1431                    ));
1432                    continue;
1433                }
1434                Err(e) => {
1435                    out.push(Finding::unreadable(
1436                        "harness",
1437                        &format!(
1438                            "candidate {} could not be read",
1439                            path.file_name().unwrap_or_default().to_string_lossy()
1440                        ),
1441                        format!("{}: {e}", path.display()),
1442                    ));
1443                    continue;
1444                }
1445            };
1446        if !cand.pending() {
1447            continue;
1448        }
1449        let old_enough = chrono::DateTime::parse_from_rfc3339(&cand.created_at)
1450            .map(|t| {
1451                now.signed_duration_since(t.with_timezone(&chrono::Utc)) > STALE_CANDIDATE_AFTER
1452            })
1453            // An unparseable stamp cannot prove the candidate is fresh.
1454            .unwrap_or(true);
1455        if old_enough {
1456            stale.push(format!("{} · {:?} {}", cand.id, cand.class, cand.change));
1457        }
1458    }
1459    if !stale.is_empty() {
1460        stale.sort();
1461        out.push(Finding {
1462            component: "harness".to_string(),
1463            severity: Severity::Attention,
1464            summary: format!(
1465                "{} harness candidate(s) staged for more than {}h",
1466                stale.len(),
1467                STALE_CANDIDATE_AFTER.num_hours()
1468            ),
1469            detail: stale.join("\n"),
1470            remedy: Some(Remedy {
1471                description: "review the staged candidates — doctor never accepts one".to_string(),
1472                argv: vec!["mecha".into(), "harness".into(), "list".into()],
1473                needs_terminal: false,
1474            }),
1475        });
1476    }
1477    out
1478}
1479
1480/// Scan `<graph store>/logs` for each nightly family's newest dated log.
1481///
1482/// Quiet when the store, the logs directory, or a family has never existed —
1483/// absence is "not installed", which is not a finding. The bar is "newer than
1484/// the day before yesterday": today's file legitimately does not exist before
1485/// that job's cron slot, so yesterday's is the newest a healthy quiet morning
1486/// can show.
1487fn check_graph_nightly(store: &Path, now: DateTime<Utc>) -> Vec<Finding> {
1488    let mut out = Vec::new();
1489    let logs = store.join("logs");
1490    if !logs.is_dir() {
1491        return out;
1492    }
1493    let names: Vec<String> = match std::fs::read_dir(&logs) {
1494        Ok(entries) => entries
1495            .flatten()
1496            .filter_map(|e| e.file_name().to_str().map(String::from))
1497            .collect(),
1498        Err(e) => {
1499            out.push(Finding::unreadable(
1500                "graph",
1501                "the graph nightly logs",
1502                format!("{}: {e}", logs.display()),
1503            ));
1504            return out;
1505        }
1506    };
1507
1508    for (prefix, what) in GRAPH_NIGHTLIES {
1509        let newest = names
1510            .iter()
1511            .filter_map(|n| {
1512                n.strip_prefix(prefix)?
1513                    .strip_suffix(".log")
1514                    .and_then(|d| chrono::NaiveDate::parse_from_str(d, "%Y%m%d").ok())
1515            })
1516            .max();
1517        // Never ran at all: indistinguishable from "this half is not set up",
1518        // and a doctor that guesses teaches people to ignore it.
1519        let Some(newest) = newest else { continue };
1520        let days_quiet = (now.date_naive() - newest).num_days();
1521        if days_quiet > 1 {
1522            out.push(Finding {
1523                component: "graph".to_string(),
1524                severity: Severity::Attention,
1525                summary: format!(
1526                    "the graph nightly ({}) has not run for {days_quiet} days",
1527                    prefix.trim_end_matches('-'),
1528                ),
1529                detail: format!(
1530                    "{what} last wrote {}{}.log under {}; it logs every \
1531                     run including deferred ones, so a missing day means the \
1532                     script never started — cron reports that nowhere",
1533                    prefix,
1534                    newest.format("%Y%m%d"),
1535                    logs.display(),
1536                ),
1537                remedy: Some(Remedy {
1538                    description: "list the cron entries that fire the graph nightlies, \
1539                                  then run the silent one by hand and read its error"
1540                        .to_string(),
1541                    argv: vec!["crontab".into(), "-l".into()],
1542                    needs_terminal: false,
1543                }),
1544            });
1545        }
1546    }
1547    out
1548}
1549
1550// --- shared helpers ---------------------------------------------------------
1551
1552/// The age of an RFC 3339 stamp, or `None` when it does not parse — unknown
1553/// must never masquerade as old (or as fresh).
1554fn age_of(stamp: &str, now: DateTime<Utc>) -> Option<chrono::Duration> {
1555    DateTime::parse_from_rfc3339(stamp)
1556        .ok()
1557        .map(|at| now - at.with_timezone(&Utc))
1558}
1559
1560/// "49h ago", "3d ago", or the raw stamp when it does not parse.
1561fn render_age(now: DateTime<Utc>, stamp: &str) -> String {
1562    match age_of(stamp, now) {
1563        Some(age) if age >= chrono::Duration::days(2) => format!("{}d ago", age.num_days()),
1564        Some(age) if age >= chrono::Duration::hours(1) => format!("{}h ago", age.num_hours()),
1565        Some(age) => format!("{}m ago", age.num_minutes().max(0)),
1566        None => stamp.to_string(),
1567    }
1568}
1569
1570#[cfg(test)]
1571mod tests {
1572    use super::*;
1573    use crate::agent::Taint;
1574    use crate::outbox::{OutboxItem, OutboxKind};
1575    use serde_json::json;
1576    use std::path::PathBuf;
1577
1578    fn utc(s: &str) -> DateTime<Utc> {
1579        DateTime::parse_from_rfc3339(s).unwrap().with_timezone(&Utc)
1580    }
1581
1582    const NOW: &str = "2026-08-14T12:00:00Z";
1583
1584    /// A scratch mecha home, unique per test and thread.
1585    fn home(name: &str) -> PathBuf {
1586        let dir = std::env::temp_dir().join(format!(
1587            "mecha-doctor-test-{name}-{}-{:?}",
1588            std::process::id(),
1589            std::thread::current().id()
1590        ));
1591        let _ = std::fs::remove_dir_all(&dir);
1592        std::fs::create_dir_all(&dir).unwrap();
1593        dir
1594    }
1595
1596    fn write_marker(home: &Path, account: &str, body: &str) {
1597        let dir = home.join("mail").join(account);
1598        std::fs::create_dir_all(&dir).unwrap();
1599        std::fs::write(dir.join("auth_error.json"), body).unwrap();
1600    }
1601
1602    fn valid_marker() -> String {
1603        json!({
1604            "at": "2026-08-11T09:00:00Z",
1605            "message": "the refresh token was revoked — run `mecha-mail auth personal --provider google` to sign in again",
1606        })
1607        .to_string()
1608    }
1609
1610    fn pending_item(home: &Path, id: &str, created_at: &str, error: Option<&str>) {
1611        let item = OutboxItem {
1612            id: id.to_string(),
1613            status: "pending".into(),
1614            tool: "mail__send".into(),
1615            kind: OutboxKind::Message,
1616            args_before: json!({"to": "a@x.org"}),
1617            args: json!({"to": "a@x.org"}),
1618            summary: "mail__send to a@x.org".into(),
1619            session_id: None,
1620            workspace: None,
1621            taint: Taint::default(),
1622            created_at: created_at.to_string(),
1623            resolved_at: None,
1624            reason: None,
1625            error: error.map(String::from),
1626        };
1627        let dir = home.join("outbox");
1628        std::fs::create_dir_all(&dir).unwrap();
1629        std::fs::write(
1630            dir.join(format!("{id}.json")),
1631            serde_json::to_string_pretty(&item).unwrap(),
1632        )
1633        .unwrap();
1634    }
1635
1636    fn request(home: &Path, seq: i64, state: &str, drained_at: &str) {
1637        let dir = home.join("requests");
1638        std::fs::create_dir_all(&dir).unwrap();
1639        let record = json!({
1640            "seq": seq,
1641            "type_id": "meeting",
1642            "state": state,
1643            "created_at": drained_at,
1644            "drained_at": drained_at,
1645            "valid": true,
1646            "values": {},
1647            "free_text": [],
1648        });
1649        std::fs::write(
1650            dir.join(format!("{seq:010}-meeting.json")),
1651            record.to_string(),
1652        )
1653        .unwrap();
1654    }
1655
1656    fn trigger_file(home: &Path, name: &str, extra: &str) {
1657        let dir = home.join("triggers");
1658        std::fs::create_dir_all(&dir).unwrap();
1659        std::fs::write(
1660            dir.join(format!("{name}.toml")),
1661            format!(
1662                "schedule = \"0 7 * * *\"\nprompt = \"brief me\"\ntimezone = \"UTC\"\n\
1663                 created_at = \"2026-08-01T00:00:00Z\"\n{extra}"
1664            ),
1665        )
1666        .unwrap();
1667    }
1668
1669    fn ledger_row(home: &Path, row: &serde_json::Value) {
1670        use std::io::Write;
1671        let dir = home.join("triggers");
1672        std::fs::create_dir_all(&dir).unwrap();
1673        let mut file = std::fs::OpenOptions::new()
1674            .create(true)
1675            .append(true)
1676            .open(dir.join("runs.jsonl"))
1677            .unwrap();
1678        writeln!(file, "{row}").unwrap();
1679    }
1680
1681    fn of<'a>(findings: &'a [Finding], component: &str) -> Vec<&'a Finding> {
1682        findings
1683            .iter()
1684            .filter(|f| f.component == component)
1685            .collect()
1686    }
1687
1688    #[test]
1689    fn a_dead_auth_marker_is_found_and_an_absent_one_is_not() {
1690        let home = home("dead-auth");
1691        write_marker(&home, "personal", &valid_marker());
1692        // A healthy account: a directory with credentials and no marker.
1693        std::fs::create_dir_all(home.join("mail").join("dartmouth")).unwrap();
1694        std::fs::write(
1695            home.join("mail").join("accounts.toml"),
1696            "[[account]]\nname = \"personal\"\nprovider = \"google\"\n\
1697             [[account]]\nname = \"dartmouth\"\nprovider = \"outlook\"\n",
1698        )
1699        .unwrap();
1700
1701        let findings = examine(&home, utc(NOW));
1702        let mail = of(&findings, "mail");
1703        assert_eq!(mail.len(), 1, "{findings:#?}");
1704        assert_eq!(mail[0].severity, Severity::Broken);
1705        assert!(mail[0].summary.contains("personal"), "{}", mail[0].summary);
1706        let remedy = mail[0].remedy.as_ref().expect("a dead login has a way out");
1707        assert_eq!(
1708            remedy.argv,
1709            vec!["mecha-mail", "auth", "personal", "--provider", "google"]
1710        );
1711        assert!(
1712            remedy.needs_terminal,
1713            "an OAuth flow needs the real terminal"
1714        );
1715
1716        let _ = std::fs::remove_dir_all(&home);
1717    }
1718
1719    #[test]
1720    fn a_provider_the_registry_cannot_name_is_omitted_from_the_remedy_not_guessed() {
1721        let home = home("no-registry");
1722        // No accounts.toml at all.
1723        write_marker(&home, "personal", &valid_marker());
1724
1725        let findings = examine(&home, utc(NOW));
1726        let mail = of(&findings, "mail");
1727        assert_eq!(mail.len(), 1);
1728        let remedy = mail[0].remedy.as_ref().unwrap();
1729        assert_eq!(remedy.argv, vec!["mecha-mail", "auth", "personal"]);
1730        // The marker's message names the full command, and it rides in the
1731        // detail so the operator still sees the provider.
1732        assert!(
1733            mail[0].detail.contains("--provider google"),
1734            "{}",
1735            mail[0].detail
1736        );
1737
1738        let _ = std::fs::remove_dir_all(&home);
1739    }
1740
1741    /// Legacy per-provider stores (`<home>/google/oauth.json`, still served
1742    /// by the shipped `mecha-google` binary) get the same marker beside
1743    /// their credentials — and the old scan, which read only
1744    /// `<home>/mail/*/`, walked straight past it.
1745    #[test]
1746    fn a_marker_in_a_legacy_per_provider_store_is_found_and_proposes_import() {
1747        let home = home("legacy-auth");
1748        let dir = home.join("google");
1749        std::fs::create_dir_all(&dir).unwrap();
1750        std::fs::write(
1751            dir.join("auth_error.json"),
1752            json!({
1753                "at": "2026-08-11T09:00:00Z",
1754                "message": "account `google`: refresh token expired or revoked — run `mecha-mail auth google --provider google` (invalid_grant)",
1755            })
1756            .to_string(),
1757        )
1758        .unwrap();
1759
1760        let findings = examine(&home, utc(NOW));
1761        let mail = of(&findings, "mail");
1762        assert_eq!(mail.len(), 1, "{findings:#?}");
1763        assert_eq!(mail[0].severity, Severity::Broken);
1764        assert!(
1765            mail[0].summary.contains("legacy google"),
1766            "{}",
1767            mail[0].summary
1768        );
1769        // The marker's message names the exact re-auth command; it must ride
1770        // in the detail.
1771        assert!(
1772            mail[0]
1773                .detail
1774                .contains("run `mecha-mail auth google --provider google`"),
1775            "{}",
1776            mail[0].detail
1777        );
1778        let remedy = mail[0].remedy.as_ref().expect("a way out");
1779        assert_eq!(
1780            remedy.argv,
1781            vec!["mecha-mail", "import", "google", "--provider", "google"]
1782        );
1783
1784        let _ = std::fs::remove_dir_all(&home);
1785    }
1786
1787    #[test]
1788    fn an_unparseable_marker_is_a_store_unreadable_finding_not_a_crash() {
1789        let home = home("bad-marker");
1790        write_marker(&home, "personal", "{ this is not json");
1791
1792        let findings = examine(&home, utc(NOW));
1793        let mail = of(&findings, "mail");
1794        assert_eq!(mail.len(), 1, "{findings:#?}");
1795        assert!(
1796            mail[0].summary.starts_with("store unreadable:"),
1797            "{}",
1798            mail[0].summary
1799        );
1800        assert!(mail[0].summary.contains("personal"), "{}", mail[0].summary);
1801
1802        let _ = std::fs::remove_dir_all(&home);
1803    }
1804
1805    #[test]
1806    fn a_pending_item_with_an_error_is_broken_and_a_resolved_one_is_not() {
1807        let home = home("outbox-error");
1808        pending_item(
1809            &home,
1810            "20260814-000001-aaa",
1811            NOW,
1812            Some("server unreachable"),
1813        );
1814        // A sent item with an old date and even an error field: never flagged.
1815        let mut sent = json!({
1816            "id": "20260810-000001-bbb",
1817            "status": "sent",
1818            "tool": "mail__send",
1819            "args_before": {},
1820            "args": {},
1821            "summary": "mail__send",
1822            "created_at": "2026-08-01T00:00:00Z",
1823        });
1824        sent["error"] = json!(null);
1825        std::fs::write(
1826            home.join("outbox").join("20260810-000001-bbb.json"),
1827            sent.to_string(),
1828        )
1829        .unwrap();
1830
1831        let findings = examine(&home, utc(NOW));
1832        let outbox = of(&findings, "outbox");
1833        assert_eq!(outbox.len(), 1, "{findings:#?}");
1834        assert_eq!(outbox[0].severity, Severity::Broken);
1835        assert!(
1836            outbox[0]
1837                .summary
1838                .contains("release failed: server unreachable"),
1839            "{}",
1840            outbox[0].summary
1841        );
1842        let remedy = outbox[0].remedy.as_ref().unwrap();
1843        assert_eq!(remedy.argv, vec!["mecha", "outbox", "review"]);
1844
1845        let _ = std::fs::remove_dir_all(&home);
1846    }
1847
1848    #[test]
1849    fn a_pending_draft_is_stale_at_49_hours_and_not_at_47() {
1850        let home = home("outbox-stale");
1851        // 49h before NOW.
1852        pending_item(&home, "20260812-110000-old", "2026-08-12T11:00:00Z", None);
1853        let findings = examine(&home, utc(NOW));
1854        let outbox = of(&findings, "outbox");
1855        assert_eq!(outbox.len(), 1, "{findings:#?}");
1856        assert_eq!(outbox[0].severity, Severity::Attention);
1857        assert!(outbox[0].summary.contains("pending for more than 48h"));
1858        assert_eq!(
1859            outbox[0].remedy.as_ref().unwrap().argv,
1860            vec!["mecha", "outbox", "review"],
1861            "the remedy is the review surface, never send"
1862        );
1863
1864        // 47h old: a person may simply not have reviewed yet.
1865        let fresh = home;
1866        let _ = std::fs::remove_dir_all(fresh.join("outbox"));
1867        pending_item(&fresh, "20260812-130000-new", "2026-08-12T13:00:00Z", None);
1868        let findings = examine(&fresh, utc(NOW));
1869        assert!(of(&findings, "outbox").is_empty(), "{findings:#?}");
1870
1871        let _ = std::fs::remove_dir_all(&fresh);
1872    }
1873
1874    fn harness_candidate(home: &Path, id: &str, created_at: &str, status: &str) {
1875        let dir = home.join("learning").join("harness").join("candidates");
1876        std::fs::create_dir_all(&dir).unwrap();
1877        let cand = crate::harness::HarnessCandidate {
1878            id: id.into(),
1879            created_at: created_at.into(),
1880            class: crate::candidate::ChangeClass::Config,
1881            change: "compact_at_tokens=24000".into(),
1882            metric: crate::candidate::Metric::CutShort,
1883            rationale: "test".into(),
1884            evidence: String::new(),
1885            model: None,
1886            status: status.into(),
1887            measurement: None,
1888            resolved_at: None,
1889            reason: None,
1890        };
1891        std::fs::write(
1892            dir.join(format!("{id}.json")),
1893            serde_json::to_string_pretty(&cand).unwrap(),
1894        )
1895        .unwrap();
1896    }
1897
1898    fn reflection_line(id: &str, origin: &str, processed: bool, created_at: &str) -> String {
1899        serde_json::json!({
1900            "id": id,
1901            "domain": "behavior",
1902            "session_id": "s",
1903            "trigger": "steer",
1904            "context": "",
1905            "intervention": "",
1906            "reflexion_text": "test",
1907            "is_processed": processed,
1908            "created_at": created_at,
1909            "origin": origin,
1910        })
1911        .to_string()
1912    }
1913
1914    fn write_reflections(home: &Path, lines: &[String]) {
1915        let dir = home.join("learning");
1916        std::fs::create_dir_all(&dir).unwrap();
1917        std::fs::write(dir.join("reflections.jsonl"), lines.join("\n")).unwrap();
1918    }
1919
1920    #[test]
1921    fn a_learner_fed_only_excluded_evidence_is_starved_and_a_met_floor_is_not() {
1922        let home = home("learning-starved");
1923        // 12 recent exclusions, one clean reflection stuck below the floor.
1924        let mut lines: Vec<String> = (0..12)
1925            .map(|i| reflection_line(&format!("u{i}"), "untrusted", false, "2026-08-13T12:00:00Z"))
1926            .collect();
1927        lines.push(reflection_line(
1928            "c1",
1929            "clean",
1930            false,
1931            "2026-08-05T00:00:00Z",
1932        ));
1933        write_reflections(&home, &lines);
1934
1935        let findings = examine(&home, utc(NOW));
1936        let learning = of(&findings, "learning");
1937        assert_eq!(learning.len(), 1, "{findings:#?}");
1938        assert_eq!(learning[0].severity, Severity::Attention);
1939        assert!(
1940            learning[0].summary.contains("starved"),
1941            "{}",
1942            learning[0].summary
1943        );
1944        assert!(
1945            learning[0].summary.contains("12 of 13"),
1946            "{}",
1947            learning[0].summary
1948        );
1949        assert_eq!(
1950            learning[0].remedy.as_ref().unwrap().argv,
1951            vec!["mecha", "reflect", "--dry-run"],
1952            "the remedy shows classifications; nothing may loosen the gate"
1953        );
1954
1955        // A domain at the floor means learn runs tonight: not starved.
1956        lines.push(reflection_line(
1957            "c2",
1958            "clean",
1959            false,
1960            "2026-08-06T00:00:00Z",
1961        ));
1962        lines.push(reflection_line(
1963            "c3",
1964            "clean",
1965            false,
1966            "2026-08-07T00:00:00Z",
1967        ));
1968        write_reflections(&home, &lines);
1969        let findings = examine(&home, utc(NOW));
1970        assert!(of(&findings, "learning").is_empty(), "{findings:#?}");
1971
1972        let _ = std::fs::remove_dir_all(&home);
1973    }
1974
1975    #[test]
1976    fn thin_or_dormant_exclusion_is_not_starvation() {
1977        let home = home("learning-thin");
1978        // Nine exclusions: below the evidence floor, silence means nothing yet.
1979        let lines: Vec<String> = (0..9)
1980            .map(|i| reflection_line(&format!("u{i}"), "untrusted", false, "2026-08-13T12:00:00Z"))
1981            .collect();
1982        write_reflections(&home, &lines);
1983        assert!(of(&examine(&home, utc(NOW)), "learning").is_empty());
1984
1985        // Twelve exclusions, all long stale: a dormant loop, not a starved one
1986        // — the newest excluded reflection is months before NOW.
1987        let lines: Vec<String> = (0..12)
1988            .map(|i| reflection_line(&format!("u{i}"), "untrusted", false, "2026-05-01T12:00:00Z"))
1989            .collect();
1990        write_reflections(&home, &lines);
1991        assert!(of(&examine(&home, utc(NOW)), "learning").is_empty());
1992
1993        // Processed clean reflections do not count toward the waiting pool —
1994        // consumed evidence is not a pool the floor can be met from.
1995        let mut lines: Vec<String> = (0..12)
1996            .map(|i| reflection_line(&format!("u{i}"), "untrusted", false, "2026-08-13T12:00:00Z"))
1997            .collect();
1998        for i in 0..3 {
1999            lines.push(reflection_line(
2000                &format!("p{i}"),
2001                "clean",
2002                true,
2003                "2026-08-05T00:00:00Z",
2004            ));
2005        }
2006        write_reflections(&home, &lines);
2007        let findings = examine(&home, utc(NOW));
2008        assert_eq!(of(&findings, "learning").len(), 1, "{findings:#?}");
2009
2010        let _ = std::fs::remove_dir_all(&home);
2011    }
2012
2013    #[test]
2014    fn a_staged_harness_candidate_is_stale_at_73_hours_and_not_at_71() {
2015        let home = home("harness-stale");
2016        // 73h before NOW.
2017        harness_candidate(&home, "hc-old", "2026-08-11T11:00:00Z", "staged");
2018        // Resolved candidates never nag, however old.
2019        harness_candidate(&home, "hc-done", "2026-08-01T00:00:00Z", "rejected");
2020        let findings = examine(&home, utc(NOW));
2021        let harness = of(&findings, "harness");
2022        assert_eq!(harness.len(), 1, "{findings:#?}");
2023        assert_eq!(harness[0].severity, Severity::Attention);
2024        assert!(harness[0].summary.contains("staged for more than 72h"));
2025        assert!(
2026            harness[0].detail.contains("hc-old"),
2027            "{}",
2028            harness[0].detail
2029        );
2030        assert_eq!(
2031            harness[0].remedy.as_ref().unwrap().argv,
2032            vec!["mecha", "harness", "list"],
2033            "the remedy is the review surface, never accept"
2034        );
2035
2036        // 71h old: the person may simply not have looked yet.
2037        let _ = std::fs::remove_dir_all(home.join("learning"));
2038        harness_candidate(&home, "hc-new", "2026-08-11T13:00:00Z", "staged");
2039        let findings = examine(&home, utc(NOW));
2040        assert!(of(&findings, "harness").is_empty(), "{findings:#?}");
2041
2042        let _ = std::fs::remove_dir_all(&home);
2043    }
2044
2045    #[test]
2046    fn a_failed_extraction_is_broken_at_any_age() {
2047        let home = home("frontdoor-failed");
2048        request(&home, 12, crate::frontdoor::EXTRACTION_FAILED, NOW);
2049
2050        let findings = examine(&home, utc(NOW));
2051        let front = of(&findings, "frontdoor");
2052        assert_eq!(front.len(), 1, "{findings:#?}");
2053        assert_eq!(front[0].severity, Severity::Broken);
2054        assert!(front[0].summary.contains("12"), "{}", front[0].summary);
2055        assert_eq!(
2056            front[0].remedy.as_ref().unwrap().argv,
2057            vec!["mecha", "frontdoor", "list"]
2058        );
2059
2060        let _ = std::fs::remove_dir_all(&home);
2061    }
2062
2063    #[test]
2064    fn a_request_waiting_on_me_is_stale_at_73_hours_and_not_at_71() {
2065        let home = home("frontdoor-stale");
2066        // 73h before NOW.
2067        request(
2068            &home,
2069            1,
2070            crate::frontdoor::AWAITING_ME,
2071            "2026-08-11T11:00:00Z",
2072        );
2073        let findings = examine(&home, utc(NOW));
2074        let front = of(&findings, "frontdoor");
2075        assert_eq!(front.len(), 1, "{findings:#?}");
2076        assert_eq!(front[0].severity, Severity::Attention);
2077        assert!(front[0].summary.contains("waiting on you"));
2078
2079        // 71h: not yet.
2080        let _ = std::fs::remove_dir_all(home.join("requests"));
2081        request(
2082            &home,
2083            2,
2084            crate::frontdoor::AWAITING_ME,
2085            "2026-08-11T13:00:00Z",
2086        );
2087        let findings = examine(&home, utc(NOW));
2088        assert!(of(&findings, "frontdoor").is_empty(), "{findings:#?}");
2089
2090        // And a state waiting on the *requester* is never the user's fault.
2091        let _ = std::fs::remove_dir_all(home.join("requests"));
2092        request(
2093            &home,
2094            3,
2095            crate::frontdoor::NEEDS_INFO,
2096            "2026-08-01T00:00:00Z",
2097        );
2098        let findings = examine(&home, utc(NOW));
2099        assert!(of(&findings, "frontdoor").is_empty(), "{findings:#?}");
2100
2101        let _ = std::fs::remove_dir_all(&home);
2102    }
2103
2104    /// `triaged` means "triage considered it and drafted nothing — a person
2105    /// has to decide", and nothing ever re-triages it: left off the
2106    /// waiting-on-me list it waits forever, invisibly.
2107    #[test]
2108    fn a_triaged_request_nothing_will_revisit_goes_stale() {
2109        let home = home("frontdoor-triaged");
2110        // 73h before NOW.
2111        request(&home, 4, crate::frontdoor::TRIAGED, "2026-08-11T11:00:00Z");
2112        // Older still, but waiting on the *stranger*: never the user's fault.
2113        request(
2114            &home,
2115            5,
2116            crate::frontdoor::NEEDS_INFO,
2117            "2026-08-01T00:00:00Z",
2118        );
2119
2120        let findings = examine(&home, utc(NOW));
2121        let front = of(&findings, "frontdoor");
2122        assert_eq!(front.len(), 1, "{findings:#?}");
2123        assert_eq!(front[0].severity, Severity::Attention);
2124        assert!(front[0].detail.contains("triaged"), "{}", front[0].detail);
2125        assert!(
2126            !front[0].detail.contains("needs_info"),
2127            "needs_info waits on the requester: {}",
2128            front[0].detail
2129        );
2130
2131        let _ = std::fs::remove_dir_all(&home);
2132    }
2133
2134    #[test]
2135    fn a_trigger_whose_last_run_failed_is_flagged_with_the_manual_probe() {
2136        let home = home("trigger-failed");
2137        trigger_file(&home, "morning", "");
2138        ledger_row(
2139            &home,
2140            &json!({
2141                "trigger": "morning",
2142                "slot": "2026-08-13T07:00:00Z",
2143                "started_at": "2026-08-13T07:00:01Z",
2144                "status": "ok",
2145                "summary": "fine",
2146            }),
2147        );
2148        ledger_row(
2149            &home,
2150            &json!({
2151                "trigger": "morning",
2152                "slot": "2026-08-14T07:00:00Z",
2153                "started_at": "2026-08-14T07:00:01Z",
2154                "status": "error",
2155                "error": "provider unreachable",
2156            }),
2157        );
2158
2159        let findings = examine(&home, utc(NOW));
2160        let triggers = of(&findings, "triggers");
2161        assert_eq!(triggers.len(), 1, "{findings:#?}");
2162        assert_eq!(triggers[0].severity, Severity::Attention);
2163        assert!(triggers[0].summary.contains("morning"));
2164        assert!(triggers[0].detail.contains("provider unreachable"));
2165        assert_eq!(
2166            triggers[0].remedy.as_ref().unwrap().argv,
2167            vec!["mecha", "trigger", "run", "morning"],
2168            "a manual run is the safe probe: it never advances the schedule"
2169        );
2170
2171        let _ = std::fs::remove_dir_all(&home);
2172    }
2173
2174    /// A skip is a row, not a run: the overlap/staleness bookkeeping the
2175    /// scheduler appends after a failure must not read as a recovery. The
2176    /// old check keyed on the literal last ledger row and reported nothing.
2177    #[test]
2178    fn a_skip_row_after_a_failed_run_does_not_hide_the_failure() {
2179        let home = home("trigger-skip-hides-error");
2180        trigger_file(&home, "morning", "");
2181        ledger_row(
2182            &home,
2183            &json!({
2184                "trigger": "morning",
2185                "slot": "2026-08-13T07:00:00Z",
2186                "started_at": "2026-08-13T07:00:01Z",
2187                "status": "error",
2188                "error": "provider unreachable",
2189            }),
2190        );
2191        ledger_row(
2192            &home,
2193            &json!({
2194                "trigger": "morning",
2195                "slot": "2026-08-14T07:00:00Z",
2196                "started_at": "2026-08-14T07:00:01Z",
2197                "status": "skipped-stale",
2198            }),
2199        );
2200
2201        let findings = examine(&home, utc(NOW));
2202        let triggers = of(&findings, "triggers");
2203        assert_eq!(triggers.len(), 1, "{findings:#?}");
2204        assert!(
2205            triggers[0].summary.contains("most recent run failed"),
2206            "{}",
2207            triggers[0].summary
2208        );
2209        assert!(triggers[0].detail.contains("provider unreachable"));
2210
2211        let _ = std::fs::remove_dir_all(&home);
2212    }
2213
2214    #[test]
2215    fn an_ok_run_followed_by_a_skip_is_healthy() {
2216        let home = home("trigger-ok-then-skip");
2217        trigger_file(&home, "morning", "");
2218        ledger_row(
2219            &home,
2220            &json!({
2221                "trigger": "morning",
2222                "slot": "2026-08-13T07:00:00Z",
2223                "started_at": "2026-08-13T07:00:01Z",
2224                "status": "ok",
2225            }),
2226        );
2227        ledger_row(
2228            &home,
2229            &json!({
2230                "trigger": "morning",
2231                "slot": "2026-08-14T07:00:00Z",
2232                "started_at": "2026-08-14T07:00:01Z",
2233                "status": "skipped-overlap",
2234            }),
2235        );
2236
2237        let findings = examine(&home, utc(NOW));
2238        assert!(of(&findings, "triggers").is_empty(), "{findings:#?}");
2239
2240        let _ = std::fs::remove_dir_all(&home);
2241    }
2242
2243    #[test]
2244    fn a_trigger_quietly_failing_a_third_of_its_calls_is_reported() {
2245        // Every run says `ok` and every briefing arrived. The only place the
2246        // degradation exists is the call counts, which nothing read before.
2247        let home = home("trigger-tool-errors");
2248        trigger_file(&home, "morning", "");
2249        for day in 10..15 {
2250            ledger_row(
2251                &home,
2252                &json!({
2253                    "trigger": "morning",
2254                    "slot": format!("2026-08-{day}T07:00:00Z"),
2255                    "started_at": format!("2026-08-{day}T07:00:01Z"),
2256                    "status": "ok",
2257                    "summary": "briefed",
2258                    "tool_calls": 6,
2259                    "tool_errors": 3,
2260                }),
2261            );
2262        }
2263
2264        let findings = examine(&home, utc(NOW));
2265        let triggers = of(&findings, "triggers");
2266        assert_eq!(triggers.len(), 1, "{findings:#?}");
2267        assert_eq!(triggers[0].severity, Severity::Attention);
2268        assert!(
2269            triggers[0].summary.contains("15 of 30"),
2270            "{}",
2271            triggers[0].summary
2272        );
2273        assert_eq!(
2274            triggers[0].remedy.as_ref().unwrap().argv,
2275            vec!["mecha", "trigger", "show", "morning"],
2276            "reading is the remedy — what to change is in the transcript"
2277        );
2278
2279        let _ = std::fs::remove_dir_all(&home);
2280    }
2281
2282    #[test]
2283    fn a_handful_of_failed_calls_is_not_a_trend() {
2284        // Two rules at once, and both are about not crying wolf. A rate over
2285        // three calls is noise, so the floor holds; and errors are how a run
2286        // learns about its environment, so a rate under the bar is silence
2287        // rather than a quieter finding.
2288        let home = home("trigger-tool-errors-quiet");
2289        trigger_file(&home, "morning", "");
2290        // Under the call floor, though every call failed.
2291        ledger_row(
2292            &home,
2293            &json!({
2294                "trigger": "morning",
2295                "slot": "2026-08-14T07:00:00Z",
2296                "started_at": "2026-08-14T07:00:01Z",
2297                "status": "ok",
2298                "tool_calls": 3,
2299                "tool_errors": 3,
2300            }),
2301        );
2302        assert!(of(&examine(&home, utc(NOW)), "triggers").is_empty());
2303
2304        // Over the floor, under the rate.
2305        ledger_row(
2306            &home,
2307            &json!({
2308                "trigger": "morning",
2309                "slot": "2026-08-15T07:00:00Z",
2310                "started_at": "2026-08-15T07:00:01Z",
2311                "status": "ok",
2312                "tool_calls": 40,
2313                "tool_errors": 4,
2314            }),
2315        );
2316        assert!(of(&examine(&home, utc(NOW)), "triggers").is_empty());
2317
2318        let _ = std::fs::remove_dir_all(&home);
2319    }
2320
2321    #[test]
2322    fn a_trigger_that_stopped_doing_anything_is_reported() {
2323        // The null run. Status `ok`, schedule advanced, answer delivered, and
2324        // no work done — invisible in every signal the ledger carried before.
2325        let home = home("trigger-stopped-working");
2326        trigger_file(&home, "morning", "");
2327        for day in 10..14 {
2328            ledger_row(
2329                &home,
2330                &json!({
2331                    "trigger": "morning",
2332                    "slot": format!("2026-08-{day}T07:00:00Z"),
2333                    "started_at": format!("2026-08-{day}T07:00:01Z"),
2334                    "status": "ok",
2335                    "tool_calls": 8,
2336                    "tool_errors": 0,
2337                }),
2338            );
2339        }
2340        ledger_row(
2341            &home,
2342            &json!({
2343                "trigger": "morning",
2344                "slot": "2026-08-14T07:00:00Z",
2345                "started_at": "2026-08-14T07:00:01Z",
2346                "status": "ok",
2347                "summary": "nothing to report",
2348                "tool_calls": 0,
2349                "tool_errors": 0,
2350            }),
2351        );
2352
2353        let findings = examine(&home, utc(NOW));
2354        let triggers = of(&findings, "triggers");
2355        assert_eq!(triggers.len(), 1, "{findings:#?}");
2356        assert!(
2357            triggers[0].summary.contains("did no work"),
2358            "{}",
2359            triggers[0].summary
2360        );
2361        assert!(triggers[0].detail.contains("made 32"));
2362
2363        let _ = std::fs::remove_dir_all(&home);
2364    }
2365
2366    #[test]
2367    fn a_trigger_that_never_needed_tools_is_not_broken_for_not_using_them() {
2368        // The reason this is measured against the trigger's own history and
2369        // never an absolute floor: a prompt that needs no tools makes zero
2370        // calls every morning, and calling that broken would be wrong about
2371        // the healthiest trigger on the machine.
2372        let home = home("trigger-never-used-tools");
2373        trigger_file(&home, "haiku", "");
2374        for day in 10..15 {
2375            ledger_row(
2376                &home,
2377                &json!({
2378                    "trigger": "haiku",
2379                    "slot": format!("2026-08-{day}T07:00:00Z"),
2380                    "started_at": format!("2026-08-{day}T07:00:01Z"),
2381                    "status": "ok",
2382                    "tool_calls": 0,
2383                    "tool_errors": 0,
2384                }),
2385            );
2386        }
2387        assert!(of(&examine(&home, utc(NOW)), "triggers").is_empty());
2388
2389        let _ = std::fs::remove_dir_all(&home);
2390    }
2391
2392    #[test]
2393    fn a_failed_run_that_did_no_work_is_reported_once_not_twice() {
2394        // An errored run already has a finding naming the error. Reporting the
2395        // absence of work on top of it would be two findings for one fact,
2396        // and a reader who has to decide which of two rows is the real one is
2397        // reading a worse report than one row.
2398        let home = home("trigger-failed-no-work");
2399        trigger_file(&home, "morning", "");
2400        for day in 10..14 {
2401            ledger_row(
2402                &home,
2403                &json!({
2404                    "trigger": "morning",
2405                    "slot": format!("2026-08-{day}T07:00:00Z"),
2406                    "started_at": format!("2026-08-{day}T07:00:01Z"),
2407                    "status": "ok",
2408                    "tool_calls": 8,
2409                    "tool_errors": 0,
2410                }),
2411            );
2412        }
2413        ledger_row(
2414            &home,
2415            &json!({
2416                "trigger": "morning",
2417                "slot": "2026-08-14T07:00:00Z",
2418                "started_at": "2026-08-14T07:00:01Z",
2419                "status": "error",
2420                "error": "provider unreachable",
2421                "tool_calls": 0,
2422                "tool_errors": 0,
2423            }),
2424        );
2425
2426        let triggers = of(&examine(&home, utc(NOW)), "triggers")
2427            .into_iter()
2428            .cloned()
2429            .collect::<Vec<_>>();
2430        assert_eq!(triggers.len(), 1, "{triggers:#?}");
2431        assert!(triggers[0].detail.contains("provider unreachable"));
2432
2433        let _ = std::fs::remove_dir_all(&home);
2434    }
2435
2436    /// Write `n` runs of one model into the session store, so the
2437    /// population checks have something to be a population of.
2438    fn runs_in(
2439        home: &Path,
2440        model: &str,
2441        n: usize,
2442        stats: impl Fn(usize) -> crate::session::RunStats,
2443    ) {
2444        let dir = home.join("sessions");
2445        std::fs::create_dir_all(&dir).unwrap();
2446        for i in 0..n {
2447            let session = crate::session::Session::create(
2448                &dir,
2449                crate::session::SessionMeta {
2450                    // The model rides in the id: two calls to this helper
2451                    // in one test must not collide, or the second silently
2452                    // rewrites the first's transcripts and the fixture stops
2453                    // describing what the test says it does.
2454                    id: format!("2026080{}T00000{i:03}-{model}", 1 + i % 9),
2455                    created_at: utc(NOW),
2456                    provider: "local".into(),
2457                    model: model.to_string(),
2458                    workspace: std::path::PathBuf::from("/tmp"),
2459                    title: None,
2460                },
2461            )
2462            .unwrap();
2463            session
2464                .append(&crate::session::Record::Outcome(stats(i)))
2465                .unwrap();
2466        }
2467    }
2468
2469    fn run_stats(
2470        calls: u32,
2471        errors: u32,
2472        ended_failed: bool,
2473        cause: crate::agent::StopCause,
2474    ) -> crate::session::RunStats {
2475        crate::session::RunStats {
2476            tool_calls: calls,
2477            tool_errors: errors,
2478            ended_on_failed_call: ended_failed,
2479            stop_cause: Some(cause),
2480            ..Default::default()
2481        }
2482    }
2483
2484    #[test]
2485    fn a_model_that_keeps_finishing_over_failures_is_reported() {
2486        use crate::agent::StopCause;
2487        let home = home("runs-ended-on-failure");
2488        // A third of runs end over a failure; everything else is healthy.
2489        runs_in(&home, "tiny-local", 30, |i| {
2490            run_stats(6, 0, i % 3 == 0, StopCause::Completed)
2491        });
2492
2493        let all = examine(&home, utc(NOW));
2494        let findings = of(&all, "runs");
2495        assert_eq!(findings.len(), 1, "{findings:#?}");
2496        assert!(
2497            findings[0].summary.contains("tiny-local"),
2498            "{}",
2499            findings[0].summary
2500        );
2501        assert!(
2502            findings[0].summary.contains("33%"),
2503            "{}",
2504            findings[0].summary
2505        );
2506        assert_eq!(
2507            findings[0].remedy.as_ref().unwrap().argv,
2508            vec!["mecha", "sessions", "health", "--days", "30"],
2509            "reading is the remedy; doctor never decides what to change"
2510        );
2511
2512        let _ = std::fs::remove_dir_all(&home);
2513    }
2514
2515    #[test]
2516    fn a_cancelled_run_is_not_the_harness_cutting_it_short() {
2517        use crate::agent::StopCause;
2518        // A person pressing Ctrl-C is the system working, and counting it
2519        // would make an attentive user look like a problem.
2520        let home = home("runs-interrupted");
2521        runs_in(&home, "tiny-local", 30, |_| {
2522            run_stats(6, 0, false, StopCause::Interrupted)
2523        });
2524        let findings = examine(&home, utc(NOW));
2525        assert!(of(&findings, "runs").is_empty());
2526        let _ = std::fs::remove_dir_all(&home);
2527    }
2528
2529    #[test]
2530    fn a_turn_ceiling_stopping_a_quarter_of_runs_is_a_finding() {
2531        use crate::agent::StopCause;
2532        let home = home("runs-max-turns");
2533        runs_in(&home, "tiny-local", 30, |_| {
2534            run_stats(6, 0, false, StopCause::MaxTurns)
2535        });
2536        let all = examine(&home, utc(NOW));
2537        let findings = of(&all, "runs");
2538        assert_eq!(findings.len(), 1, "{findings:#?}");
2539        assert!(
2540            findings[0].summary.contains("cut"),
2541            "{}",
2542            findings[0].summary
2543        );
2544        let _ = std::fs::remove_dir_all(&home);
2545    }
2546
2547    #[test]
2548    fn a_thin_sample_of_one_model_says_nothing_about_it() {
2549        use crate::agent::StopCause;
2550        // Every run terrible, and still silent: nineteen runs is not a
2551        // population, and unknown is not a finding.
2552        let home = home("runs-thin");
2553        runs_in(&home, "tiny-local", 19, |_| {
2554            run_stats(6, 6, true, StopCause::MaxTurns)
2555        });
2556        let all = examine(&home, utc(NOW));
2557        assert!(of(&all, "runs").is_empty());
2558        let _ = std::fs::remove_dir_all(&home);
2559    }
2560
2561    #[test]
2562    fn a_bad_model_does_not_drag_a_good_one_into_a_finding() {
2563        use crate::agent::StopCause;
2564        // The reason rates split: blended, these two average to a rate that
2565        // describes neither, and a threshold on it names the wrong model.
2566        let home = home("runs-two-models");
2567        runs_in(&home, "steady", 25, |_| {
2568            run_stats(10, 0, false, StopCause::Completed)
2569        });
2570        runs_in(&home, "flaky", 25, |_| {
2571            run_stats(10, 9, false, StopCause::Completed)
2572        });
2573
2574        let all = examine(&home, utc(NOW));
2575        let findings = of(&all, "runs");
2576        assert_eq!(findings.len(), 1, "{findings:#?}");
2577        assert!(
2578            findings[0].summary.contains("flaky"),
2579            "{}",
2580            findings[0].summary
2581        );
2582        assert!(
2583            !findings[0].summary.contains("steady"),
2584            "the healthy model was named in a finding about the other one"
2585        );
2586        let _ = std::fs::remove_dir_all(&home);
2587    }
2588
2589    #[test]
2590    fn a_ledger_written_before_the_counts_existed_reports_nothing() {
2591        // Not a perfect score, and not a division by zero: no data is not a
2592        // finding, which is the rule the whole module runs on.
2593        let home = home("trigger-tool-errors-bare");
2594        trigger_file(&home, "morning", "");
2595        ledger_row(
2596            &home,
2597            &json!({
2598                "trigger": "morning",
2599                "slot": "2026-08-14T07:00:00Z",
2600                "started_at": "2026-08-14T07:00:01Z",
2601                "status": "ok",
2602            }),
2603        );
2604        assert!(of(&examine(&home, utc(NOW)), "triggers").is_empty());
2605
2606        let _ = std::fs::remove_dir_all(&home);
2607    }
2608
2609    #[test]
2610    fn a_disabled_trigger_is_nobody_s_emergency() {
2611        let home = home("trigger-disabled");
2612        trigger_file(&home, "morning", "enabled = false\n");
2613        ledger_row(
2614            &home,
2615            &json!({
2616                "trigger": "morning",
2617                "started_at": "2026-08-14T07:00:01Z",
2618                "status": "error",
2619                "error": "boom",
2620            }),
2621        );
2622        let findings = examine(&home, utc(NOW));
2623        assert!(of(&findings, "triggers").is_empty(), "{findings:#?}");
2624        let _ = std::fs::remove_dir_all(&home);
2625    }
2626
2627    #[test]
2628    fn a_catch_up_trigger_whose_slots_stopped_advancing_names_the_daemon() {
2629        let home = home("trigger-stale");
2630        trigger_file(&home, "morning", "");
2631        // Last accounted slot five days ago; daily at 07:00 UTC, so slots on
2632        // the 10th..14th are all unaccounted — far more than two.
2633        ledger_row(
2634            &home,
2635            &json!({
2636                "trigger": "morning",
2637                "slot": "2026-08-09T07:00:00Z",
2638                "started_at": "2026-08-09T07:00:01Z",
2639                "status": "ok",
2640            }),
2641        );
2642
2643        let findings = examine(&home, utc(NOW));
2644        let triggers = of(&findings, "triggers");
2645        assert_eq!(triggers.len(), 1, "{findings:#?}");
2646        assert_eq!(triggers[0].severity, Severity::Attention);
2647        assert!(triggers[0].summary.contains("missed more than two slots"));
2648        assert!(
2649            triggers[0].detail.contains("daemon"),
2650            "{}",
2651            triggers[0].detail
2652        );
2653        assert!(
2654            triggers[0].remedy.is_none(),
2655            "running the trigger would not restart the scheduler"
2656        );
2657
2658        // A current ledger is healthy: this morning's 07:00 accounted for.
2659        ledger_row(
2660            &home,
2661            &json!({
2662                "trigger": "morning",
2663                "slot": "2026-08-14T07:00:00Z",
2664                "started_at": "2026-08-14T07:00:01Z",
2665                "status": "ok",
2666            }),
2667        );
2668        let findings = examine(&home, utc(NOW));
2669        assert!(of(&findings, "triggers").is_empty(), "{findings:#?}");
2670
2671        let _ = std::fs::remove_dir_all(&home);
2672    }
2673
2674    /// The observer rule, which matters most: one poisoned store must not
2675    /// suppress what the other checks found — and must itself be reported.
2676    #[cfg(unix)]
2677    #[test]
2678    fn one_poisoned_store_does_not_suppress_the_others() {
2679        use std::os::unix::fs::PermissionsExt;
2680        // Root reads through 0o000 like it is not there, and the test would
2681        // be vacuous.
2682        if unsafe { libc::geteuid() } == 0 {
2683            return;
2684        }
2685
2686        let home = home("poisoned");
2687        write_marker(&home, "personal", &valid_marker());
2688        let outbox = home.join("outbox");
2689        std::fs::create_dir_all(&outbox).unwrap();
2690        std::fs::set_permissions(&outbox, std::fs::Permissions::from_mode(0o000)).unwrap();
2691
2692        let findings = examine(&home, utc(NOW));
2693
2694        // Restore before asserting, so a failure can still clean up.
2695        std::fs::set_permissions(&outbox, std::fs::Permissions::from_mode(0o700)).unwrap();
2696
2697        let mail = of(&findings, "mail");
2698        assert_eq!(mail.len(), 1, "the mail finding survived: {findings:#?}");
2699        assert_eq!(mail[0].severity, Severity::Broken);
2700        let broken_store = of(&findings, "outbox");
2701        assert_eq!(broken_store.len(), 1, "{findings:#?}");
2702        assert!(
2703            broken_store[0].summary.starts_with("store unreadable:"),
2704            "{}",
2705            broken_store[0].summary
2706        );
2707
2708        let _ = std::fs::remove_dir_all(&home);
2709    }
2710
2711    /// Finding-6 drift pin, reader half. Twin test (same golden bytes):
2712    /// `mecha_mail::token::tests::record_auth_error_serialises_the_golden_marker_byte_for_byte`
2713    /// in mecha-mail/src/token.rs — the crates share no types on purpose
2714    /// (the seam is a file of JSON), so a field rename on either side would
2715    /// pass both suites separately and silently kill this finding at
2716    /// runtime. If this literal changes, change the twin's too.
2717    #[test]
2718    fn the_golden_marker_literal_parses_into_the_dead_auth_finding() {
2719        const GOLDEN: &str = r#"{
2720  "at": "2026-08-11T09:00:00Z",
2721  "message": "account `personal`: refresh token expired or revoked — run `mecha-mail auth personal --provider google` (invalid_grant: Token has been revoked.)"
2722}"#;
2723        let home = home("golden-marker");
2724        write_marker(&home, "personal", GOLDEN);
2725
2726        let findings = examine(&home, utc(NOW));
2727        let mail = of(&findings, "mail");
2728        assert_eq!(mail.len(), 1, "{findings:#?}");
2729        assert_eq!(mail[0].severity, Severity::Broken);
2730        assert!(
2731            mail[0].detail.contains("since 2026-08-11T09:00:00Z"),
2732            "the marker's `at` must reach the detail: {}",
2733            mail[0].detail
2734        );
2735        assert!(
2736            mail[0]
2737                .detail
2738                .contains("run `mecha-mail auth personal --provider google`"),
2739            "the marker's `message` must reach the detail: {}",
2740            mail[0].detail
2741        );
2742
2743        let _ = std::fs::remove_dir_all(&home);
2744    }
2745
2746    #[test]
2747    fn findings_sort_broken_first() {
2748        let mut findings = vec![
2749            Finding {
2750                component: "outbox".into(),
2751                severity: Severity::Attention,
2752                summary: "stale".into(),
2753                detail: String::new(),
2754                remedy: None,
2755            },
2756            Finding {
2757                component: "mail".into(),
2758                severity: Severity::Broken,
2759                summary: "dead".into(),
2760                detail: String::new(),
2761                remedy: None,
2762            },
2763        ];
2764        sort(&mut findings);
2765        assert_eq!(findings[0].severity, Severity::Broken);
2766    }
2767
2768    #[test]
2769    fn an_empty_home_is_healthy() {
2770        let home = home("empty");
2771        assert!(examine(&home, utc(NOW)).is_empty());
2772        let _ = std::fs::remove_dir_all(&home);
2773    }
2774
2775    // --- graph nightly silence ---
2776
2777    /// A graph store nested inside a unique scratch dir, so no test plants a
2778    /// `.mecha-graph` beside another test's home in the shared temp dir.
2779    fn graph_store(name: &str) -> PathBuf {
2780        let store = home(name).join(".mecha-graph");
2781        std::fs::create_dir_all(store.join("logs")).unwrap();
2782        store
2783    }
2784
2785    fn nightly_log(store: &Path, file: &str) {
2786        std::fs::write(store.join("logs").join(file), "ran\n").unwrap();
2787    }
2788
2789    // NOW is 2026-08-14: a 08-12 log is two days quiet (stale), 08-13 is
2790    // yesterday (the newest a healthy quiet morning can show).
2791
2792    #[test]
2793    fn a_graph_nightly_that_stopped_writing_logs_is_a_finding() {
2794        let store = graph_store("graph-stale");
2795        nightly_log(&store, "nightly-20260812.log");
2796        let findings = check_graph_nightly(&store, utc(NOW));
2797        assert_eq!(findings.len(), 1);
2798        assert_eq!(findings[0].component, "graph");
2799        assert_eq!(findings[0].severity, Severity::Attention);
2800        assert!(
2801            findings[0].summary.contains("2 days"),
2802            "{}",
2803            findings[0].summary
2804        );
2805        assert!(
2806            findings[0].detail.contains("nightly-20260812.log"),
2807            "{}",
2808            findings[0].detail
2809        );
2810    }
2811
2812    #[test]
2813    fn yesterdays_log_is_healthy_because_todays_slot_may_not_have_fired() {
2814        let store = graph_store("graph-yesterday");
2815        nightly_log(&store, "nightly-20260813.log");
2816        nightly_log(&store, "mecha-nightly-20260813.log");
2817        assert!(check_graph_nightly(&store, utc(NOW)).is_empty());
2818    }
2819
2820    /// The two families age independently: the sweep running every night must
2821    /// not vouch for the vet/gossip half — that is exactly how 2026-08-17
2822    /// stayed invisible.
2823    #[test]
2824    fn each_nightly_family_is_judged_alone() {
2825        let store = graph_store("graph-split");
2826        nightly_log(&store, "nightly-20260814.log");
2827        nightly_log(&store, "mecha-nightly-20260811.log");
2828        let findings = check_graph_nightly(&store, utc(NOW));
2829        assert_eq!(findings.len(), 1);
2830        assert!(
2831            findings[0].summary.contains("mecha-nightly"),
2832            "{}",
2833            findings[0].summary
2834        );
2835    }
2836
2837    /// The `nightly-` scan must not claim `mecha-nightly-` files as its own:
2838    /// a fresh mecha-nightly log would otherwise hide a dead sweep.
2839    #[test]
2840    fn the_shorter_prefix_does_not_claim_the_longer_familys_logs() {
2841        let store = graph_store("graph-prefix");
2842        nightly_log(&store, "mecha-nightly-20260814.log");
2843        nightly_log(&store, "nightly-20260810.log");
2844        let findings = check_graph_nightly(&store, utc(NOW));
2845        assert_eq!(findings.len(), 1);
2846        assert!(
2847            findings[0].detail.contains("nightly-20260810.log"),
2848            "{}",
2849            findings[0].detail
2850        );
2851    }
2852
2853    /// Absence is "not installed", never a finding — a missing store, an
2854    /// empty log directory, and names that parse to no date all stay quiet.
2855    #[test]
2856    fn a_graph_that_never_ran_is_not_a_finding() {
2857        let missing = home("graph-missing").join(".mecha-graph");
2858        assert!(check_graph_nightly(&missing, utc(NOW)).is_empty());
2859
2860        let empty = graph_store("graph-empty");
2861        assert!(check_graph_nightly(&empty, utc(NOW)).is_empty());
2862
2863        let odd = graph_store("graph-odd-names");
2864        nightly_log(&odd, "nightly-garbage.log");
2865        nightly_log(&odd, "gossip-20260812.jsonl");
2866        assert!(check_graph_nightly(&odd, utc(NOW)).is_empty());
2867    }
2868
2869    /// The examine wiring: the store is found as the home's hidden sibling.
2870    #[test]
2871    fn examine_reads_the_graph_store_beside_the_home() {
2872        let scratch = home("graph-sibling");
2873        let mecha_home = scratch.join(".mecha");
2874        std::fs::create_dir_all(&mecha_home).unwrap();
2875        let store = scratch.join(".mecha-graph");
2876        std::fs::create_dir_all(store.join("logs")).unwrap();
2877        nightly_log(&store, "nightly-20260810.log");
2878        let findings = examine(&mecha_home, utc(NOW));
2879        assert_eq!(findings.len(), 1);
2880        assert_eq!(findings[0].component, "graph");
2881        let _ = std::fs::remove_dir_all(&scratch);
2882    }
2883}