use chrono::{DateTime, Utc};
use serde::{Deserialize, Serialize};
use std::collections::BTreeMap;
use std::path::Path;
#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Serialize, Deserialize)]
#[serde(rename_all = "lowercase")]
pub enum Severity {
Broken,
Attention,
}
impl Severity {
pub fn as_str(&self) -> &'static str {
match self {
Severity::Broken => "broken",
Severity::Attention => "attention",
}
}
}
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
pub struct Remedy {
pub description: String,
pub argv: Vec<String>,
pub needs_terminal: bool,
}
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
pub struct Finding {
pub component: String,
pub severity: Severity,
pub summary: String,
pub detail: String,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub remedy: Option<Remedy>,
}
impl Finding {
fn unreadable(component: &str, what: &str, why: impl std::fmt::Display) -> Finding {
Finding {
component: component.to_string(),
severity: Severity::Attention,
summary: format!("store unreadable: {what}"),
detail: why.to_string(),
remedy: None,
}
}
}
const STUCK_DRAFT_AFTER: chrono::Duration = chrono::Duration::hours(48);
const STALE_REQUEST_AFTER: chrono::Duration = chrono::Duration::hours(72);
pub fn examine(home: &Path, now: DateTime<Utc>) -> Vec<Finding> {
let mut findings = Vec::new();
findings.extend(check_mail(&home.join("mail")));
findings.extend(check_legacy_mail(home));
findings.extend(check_outbox(&home.join("outbox"), now));
findings.extend(check_questions(&home.join("questions"), now));
findings.extend(check_frontdoor(&home.join("requests"), now));
findings.extend(check_triggers(&home.join("triggers"), now));
findings.extend(check_charter(&home.join("charter.toml")));
findings.extend(check_runs(&home.join("sessions")));
findings.extend(check_harness(&home.join("learning").join("harness"), now));
findings.extend(check_learning(&home.join("learning"), now));
findings.extend(check_proposal_review(&home.join("learning"), now));
if let Some(parent) = home.parent() {
findings.extend(check_graph_nightly(&parent.join(".mecha-graph"), now));
}
sort(&mut findings);
findings
}
pub fn sort(findings: &mut [Finding]) {
findings.sort_by(|a, b| {
a.severity
.cmp(&b.severity)
.then_with(|| a.component.cmp(&b.component))
});
}
#[derive(Debug, Deserialize)]
struct AuthMarker {
at: String,
message: String,
}
#[derive(Debug, Default, Deserialize)]
struct MailAccounts {
#[serde(default, rename = "account")]
accounts: Vec<MailAccount>,
}
#[derive(Debug, Deserialize)]
struct MailAccount {
name: String,
provider: String,
#[serde(default)]
grant_lifetime_days: Option<u32>,
}
fn check_mail(mail: &Path) -> Vec<Finding> {
let mut out = Vec::new();
if !mail.is_dir() {
return out;
}
let declared: Vec<MailAccount> = std::fs::read_to_string(mail.join("accounts.toml"))
.ok()
.and_then(|text| toml::from_str::<MailAccounts>(&text).ok())
.map(|file| file.accounts)
.unwrap_or_default();
let providers: BTreeMap<String, String> = declared
.iter()
.map(|a| (a.name.clone(), a.provider.clone()))
.collect();
let lifetimes: BTreeMap<String, u32> = declared
.iter()
.filter_map(|a| a.grant_lifetime_days.map(|d| (a.name.clone(), d)))
.collect();
let entries = match std::fs::read_dir(mail) {
Ok(entries) => entries,
Err(e) => {
out.push(Finding::unreadable(
"mail",
"the mail directory",
format!("{}: {e}", mail.display()),
));
return out;
}
};
for entry in entries.flatten() {
let dir = entry.path();
if !dir.is_dir() {
continue;
}
let Some(account) = dir.file_name().and_then(|n| n.to_str()).map(String::from) else {
continue;
};
out.extend(check_triage_scope(&dir, &account, providers.get(&account)));
out.extend(check_grant_age(
&dir,
&account,
providers.get(&account),
lifetimes.get(&account).copied(),
));
let marker_path = dir.join("auth_error.json");
if !marker_path.is_file() {
continue;
}
let text = match std::fs::read_to_string(&marker_path) {
Ok(text) => text,
Err(e) => {
out.push(Finding::unreadable(
"mail",
&format!("auth_error.json for `{account}`"),
format!("{}: {e}", marker_path.display()),
));
continue;
}
};
match serde_json::from_str::<AuthMarker>(&text) {
Ok(marker) => {
let provider = providers.get(&account);
let mut argv = vec![
"mecha-mail".to_string(),
"auth".to_string(),
account.clone(),
];
if let Some(provider) = provider {
argv.push("--provider".to_string());
argv.push(provider.clone());
}
out.push(Finding {
component: "mail".to_string(),
severity: Severity::Broken,
summary: format!("mail auth for `{account}` is dead"),
detail: format!(
"permanent refresh failure since {}: {}",
marker.at, marker.message
),
remedy: Some(Remedy {
description: format!(
"re-authenticate the `{account}` account (opens an OAuth flow)"
),
argv,
needs_terminal: true,
}),
});
}
Err(e) => out.push(Finding::unreadable(
"mail",
&format!("auth_error.json for `{account}` did not parse"),
format!("{}: {e}", marker_path.display()),
)),
}
}
out
}
#[derive(Debug, serde::Deserialize)]
struct StoredGrant {
#[serde(default)]
granted_scopes: Option<String>,
#[serde(default)]
granted_at: Option<String>,
}
const GRANT_WARN_WITHIN_DAYS: i64 = 2;
fn check_grant_age(
dir: &Path,
account: &str,
provider: Option<&String>,
lifetime_days: Option<u32>,
) -> Vec<Finding> {
let Some(lifetime) = lifetime_days.filter(|d| *d > 0) else {
return Vec::new();
};
let Ok(text) = std::fs::read_to_string(dir.join("oauth.json")) else {
return Vec::new();
};
let Ok(grant) = serde_json::from_str::<StoredGrant>(&text) else {
return Vec::new(); };
let Some(granted_at) = grant.granted_at.as_deref() else {
return Vec::new();
};
let Ok(granted) = chrono::DateTime::parse_from_rfc3339(granted_at) else {
return Vec::new();
};
let expires = granted.with_timezone(&chrono::Utc) + chrono::Duration::days(lifetime as i64);
let hours_left = (expires - chrono::Utc::now()).num_hours();
let left = (hours_left as f64 / 24.0).ceil() as i64;
if left > GRANT_WARN_WITHIN_DAYS {
return Vec::new();
}
let when = if hours_left < 0 {
"has expired".to_string()
} else if hours_left < 24 {
"expires within a day".to_string()
} else {
format!("expires in {left} days")
};
let mut argv = vec![
"mecha-mail".to_string(),
"auth".to_string(),
account.to_string(),
];
if let Some(p) = provider {
argv.push("--provider".to_string());
argv.push(p.clone());
}
vec![Finding {
component: "mail".to_string(),
severity: Severity::Attention,
summary: format!("`{account}` sign-in {when}"),
detail: format!(
"this grant lasts {lifetime} days from consent ({granted_at}) and refreshing does \
not extend it. Re-authenticate before it lapses — once it does, the failure looks \
like a revoked token and every scheduled run using this account stops."
),
remedy: Some(Remedy {
description: format!("re-authenticate `{account}` now (opens an OAuth flow)"),
argv,
needs_terminal: true,
}),
}]
}
fn triage_scope_for(provider: &str) -> Option<&'static str> {
match provider {
"google" => Some("gmail.modify"),
"outlook" | "microsoft" => Some("Mail.ReadWrite"),
_ => None,
}
}
fn check_triage_scope(dir: &Path, account: &str, provider: Option<&String>) -> Vec<Finding> {
let Some(provider) = provider else {
return Vec::new();
};
let Some(needed) = triage_scope_for(provider) else {
return Vec::new();
};
let path = dir.join("oauth.json");
let Ok(text) = std::fs::read_to_string(&path) else {
return Vec::new();
};
let Ok(grant) = serde_json::from_str::<StoredGrant>(&text) else {
return vec![Finding::unreadable(
"mail",
&format!("oauth.json for `{account}` did not parse"),
format!("{}", path.display()),
)];
};
if grant
.granted_scopes
.as_deref()
.is_some_and(|g| g.contains(needed))
{
return Vec::new();
}
let admin_note = if provider == "outlook" || provider == "microsoft" {
" Microsoft blocks `Mail.ReadWrite` from end-user consent under its \
recommended policy, so on a managed tenant an administrator has to \
grant it to the app registration before this can succeed."
} else {
""
};
vec![Finding {
component: "mail".to_string(),
severity: Severity::Attention,
summary: format!("`{account}` cannot archive, spam or mark mail read"),
detail: format!(
"the stored grant does not include `{needed}`, so mail_triage will fail on this \
account. Reading, sending and calendar work are unaffected.{admin_note}"
),
remedy: Some(Remedy {
description: format!(
"re-authenticate `{account}` to add the triage scope (opens an OAuth flow)"
),
argv: vec![
"mecha-mail".to_string(),
"auth".to_string(),
account.to_string(),
"--provider".to_string(),
provider.clone(),
],
needs_terminal: true,
}),
}]
}
#[cfg(test)]
mod grant_age_tests {
use super::*;
fn store(dir: &Path, granted_at: Option<&str>) {
std::fs::create_dir_all(dir).unwrap();
let stamp = granted_at
.map(|g| format!(r#","granted_at":"{g}""#))
.unwrap_or_default();
std::fs::write(
dir.join("oauth.json"),
format!(r#"{{"client_id":"i","access_token":"a","refresh_token":"r","expires_at":1{stamp}}}"#),
)
.unwrap();
}
fn days_ago(n: i64) -> String {
(chrono::Utc::now() - chrono::Duration::days(n)).to_rfc3339()
}
#[test]
fn a_grant_nearing_its_declared_lifetime_is_reported_early() {
let tmp = std::env::temp_dir().join(format!("mecha-grant-{}", std::process::id()));
let g = "google".to_string();
store(&tmp, Some(&days_ago(1)));
assert!(check_grant_age(&tmp, "personal", Some(&g), Some(7)).is_empty());
store(&tmp, Some(&days_ago(5)));
let f = check_grant_age(&tmp, "personal", Some(&g), Some(7));
assert_eq!(f.len(), 1, "should warn with 2 days left");
assert!(
f[0].summary.contains("expires in 2 days"),
"{}",
f[0].summary
);
assert!(f[0].remedy.as_ref().unwrap().needs_terminal);
store(&tmp, Some(&days_ago(7)));
let f = check_grant_age(&tmp, "personal", Some(&g), Some(7));
assert!(f[0].summary.contains("within a day"), "{}", f[0].summary);
store(&tmp, Some(&days_ago(9)));
let f = check_grant_age(&tmp, "personal", Some(&g), Some(7));
assert!(f[0].summary.contains("has expired"), "{}", f[0].summary);
assert!(check_grant_age(&tmp, "personal", Some(&g), None).is_empty());
store(&tmp, None);
assert!(check_grant_age(&tmp, "personal", Some(&g), Some(7)).is_empty());
std::fs::remove_dir_all(&tmp).ok();
}
}
fn check_legacy_mail(home: &Path) -> Vec<Finding> {
let mut out = Vec::new();
for provider in ["google", "outlook"] {
let marker_path = home.join(provider).join("auth_error.json");
if !marker_path.is_file() {
continue;
}
let text = match std::fs::read_to_string(&marker_path) {
Ok(text) => text,
Err(e) => {
out.push(Finding::unreadable(
"mail",
&format!("auth_error.json for the legacy {provider} store"),
format!("{}: {e}", marker_path.display()),
));
continue;
}
};
match serde_json::from_str::<AuthMarker>(&text) {
Ok(marker) => out.push(Finding {
component: "mail".to_string(),
severity: Severity::Broken,
summary: format!("legacy {provider} mail auth is dead"),
detail: format!(
"permanent refresh failure since {}: {}",
marker.at, marker.message
),
remedy: Some(Remedy {
description: format!(
"bring the legacy {provider} login into the unified registry — \
and re-authenticate it per the detail, which no import fixes"
),
argv: vec![
"mecha-mail".to_string(),
"import".to_string(),
provider.to_string(),
"--provider".to_string(),
provider.to_string(),
],
needs_terminal: false,
}),
}),
Err(e) => out.push(Finding::unreadable(
"mail",
&format!("auth_error.json for the legacy {provider} store did not parse"),
format!("{}: {e}", marker_path.display()),
)),
}
}
out
}
fn check_outbox(root: &Path, now: DateTime<Utc>) -> Vec<Finding> {
let mut out = Vec::new();
if !root.is_dir() {
return out;
}
let entries = match std::fs::read_dir(root) {
Ok(entries) => entries,
Err(e) => {
out.push(Finding::unreadable(
"outbox",
"the outbox directory",
format!("{}: {e}", root.display()),
));
return out;
}
};
let review = Remedy {
description: "open the outbox review surface — doctor never releases a draft".to_string(),
argv: vec!["mecha".into(), "outbox".into(), "review".into()],
needs_terminal: true,
};
let mut stale: Vec<String> = Vec::new();
for entry in entries.flatten() {
let path = entry.path();
if path.extension().and_then(|e| e.to_str()) != Some("json") {
continue;
}
let item: crate::outbox::OutboxItem =
match std::fs::read_to_string(&path).map(|t| serde_json::from_str(&t)) {
Ok(Ok(item)) => item,
Ok(Err(e)) => {
out.push(Finding::unreadable(
"outbox",
&format!(
"item {} did not parse",
path.file_name().unwrap_or_default().to_string_lossy()
),
format!("{}: {e}", path.display()),
));
continue;
}
Err(e) => {
out.push(Finding::unreadable(
"outbox",
&format!(
"item {} could not be read",
path.file_name().unwrap_or_default().to_string_lossy()
),
format!("{}: {e}", path.display()),
));
continue;
}
};
if item.status != "pending" {
continue;
}
if let Some(error) = &item.error {
out.push(Finding {
component: "outbox".to_string(),
severity: Severity::Broken,
summary: format!("release failed: {error}"),
detail: format!(
"{} · {} — still pending; the draft is good, the delivery was not",
item.id, item.summary
),
remedy: Some(review.clone()),
});
} else if age_of(&item.created_at, now).is_some_and(|age| age > STUCK_DRAFT_AFTER) {
stale.push(format!(
"{} · {} — staged {}",
item.id,
item.summary,
render_age(now, &item.created_at)
));
}
}
if !stale.is_empty() {
stale.sort();
out.push(Finding {
component: "outbox".to_string(),
severity: Severity::Attention,
summary: format!(
"{} draft{} pending for more than 48h",
stale.len(),
if stale.len() == 1 { "" } else { "s" }
),
detail: stale.join("\n"),
remedy: Some(review),
});
}
out
}
const UNANSWERED_QUESTION_AFTER: chrono::Duration = chrono::Duration::hours(24);
fn check_questions(root: &Path, now: DateTime<Utc>) -> Vec<Finding> {
let mut out = Vec::new();
if !root.is_dir() {
return out;
}
let entries = match std::fs::read_dir(root) {
Ok(entries) => entries,
Err(e) => {
out.push(Finding::unreadable(
"questions",
"the question store",
format!("{}: {e}", root.display()),
));
return out;
}
};
let mut stale: Vec<String> = Vec::new();
for entry in entries.flatten() {
let path = entry.path();
if path.extension().and_then(|e| e.to_str()) != Some("json") {
continue;
}
let q: crate::questions::Question =
match std::fs::read_to_string(&path).map(|t| serde_json::from_str(&t)) {
Ok(Ok(q)) => q,
Ok(Err(e)) => {
out.push(Finding::unreadable(
"questions",
&format!(
"question {} did not parse",
path.file_name().unwrap_or_default().to_string_lossy()
),
format!("{}: {e}", path.display()),
));
continue;
}
Err(e) => {
out.push(Finding::unreadable(
"questions",
&format!(
"question {} could not be read",
path.file_name().unwrap_or_default().to_string_lossy()
),
format!("{}: {e}", path.display()),
));
continue;
}
};
if !q.is_open() {
continue;
}
if age_of(&q.asked_at, now).is_some_and(|age| age > UNANSWERED_QUESTION_AFTER) {
stale.push(format!(
"{} · {} — asked {}",
crate::questions::QuestionStore::short(&q.id),
q.summary(),
render_age(now, &q.asked_at)
));
}
}
if !stale.is_empty() {
stale.sort();
out.push(Finding {
component: "questions".to_string(),
severity: Severity::Attention,
summary: format!(
"{} question{} unanswered for more than 24h — {} run{} cannot continue",
stale.len(),
if stale.len() == 1 { "" } else { "s" },
stale.len(),
if stale.len() == 1 { "" } else { "s" }
),
detail: stale.join("\n"),
remedy: Some(Remedy {
description: "see what the agent is stuck on — doctor never answers for you"
.to_string(),
argv: vec!["mecha".into(), "questions".into(), "list".into()],
needs_terminal: true,
}),
});
}
out
}
const WAITING_ON_ME: [&str; 3] = [
crate::frontdoor::EXTRACTED,
crate::frontdoor::AWAITING_ME,
crate::frontdoor::TRIAGED,
];
fn check_frontdoor(root: &Path, now: DateTime<Utc>) -> Vec<Finding> {
let mut out = Vec::new();
if !root.is_dir() {
return out;
}
let entries = match std::fs::read_dir(root) {
Ok(entries) => entries,
Err(e) => {
out.push(Finding::unreadable(
"frontdoor",
"the request store",
format!("{}: {e}", root.display()),
));
return out;
}
};
let list = Remedy {
description: "list the frontdoor queue".to_string(),
argv: vec!["mecha".into(), "frontdoor".into(), "list".into()],
needs_terminal: false,
};
let mut stale: Vec<(i64, String)> = Vec::new();
for entry in entries.flatten() {
let path = entry.path();
if path.extension().and_then(|e| e.to_str()) != Some("json") {
continue;
}
let Ok(Ok(record)) = std::fs::read_to_string(&path)
.map(|t| serde_json::from_str::<crate::frontdoor::Record>(&t))
else {
out.push(Finding::unreadable(
"frontdoor",
&format!(
"request {} did not parse",
path.file_name().unwrap_or_default().to_string_lossy()
),
path.display().to_string(),
));
continue;
};
if record.state == crate::frontdoor::EXTRACTION_FAILED {
out.push(Finding {
component: "frontdoor".to_string(),
severity: Severity::Broken,
summary: format!(
"request {} failed extraction and waits for a human",
record.seq
),
detail: format!(
"{} ({}) — {}",
record.seq,
record.type_id,
record
.extraction_error
.as_deref()
.unwrap_or("no error recorded")
),
remedy: Some(list.clone()),
});
} else if WAITING_ON_ME.contains(&record.state.as_str())
&& request_age(&record, now).is_some_and(|age| age > STALE_REQUEST_AFTER)
{
stale.push((
record.seq,
format!(
"{} ({}) — {}, received {}",
record.seq,
record.type_id,
record.state,
render_age(now, &record.created_at)
),
));
}
}
if !stale.is_empty() {
stale.sort_by_key(|(seq, _)| *seq);
out.push(Finding {
component: "frontdoor".to_string(),
severity: Severity::Attention,
summary: format!(
"{} request{} waiting on you for more than 72h",
stale.len(),
if stale.len() == 1 { "" } else { "s" }
),
detail: stale
.into_iter()
.map(|(_, line)| line)
.collect::<Vec<_>>()
.join("\n"),
remedy: Some(list),
});
}
out
}
fn request_age(record: &crate::frontdoor::Record, now: DateTime<Utc>) -> Option<chrono::Duration> {
age_of(&record.drained_at, now).or_else(|| age_of(&record.created_at, now))
}
const HEALTH_WINDOW: usize = 5;
const HEALTH_MIN_CALLS: u32 = 10;
const HEALTH_ERROR_RATE: f64 = 1.0 / 3.0;
fn check_triggers(root: &Path, now: DateTime<Utc>) -> Vec<Finding> {
let mut out = Vec::new();
if !root.is_dir() {
return out;
}
let entries = match std::fs::read_dir(root) {
Ok(entries) => entries,
Err(e) => {
out.push(Finding::unreadable(
"triggers",
"the trigger store",
format!("{}: {e}", root.display()),
));
return out;
}
};
let mut triggers: Vec<crate::trigger::Trigger> = Vec::new();
for entry in entries.flatten() {
let path = entry.path();
if path.extension().and_then(|e| e.to_str()) != Some("toml") {
continue;
}
let name = path
.file_stem()
.and_then(|s| s.to_str())
.unwrap_or_default()
.to_string();
match std::fs::read_to_string(&path).map(|t| toml::from_str::<crate::trigger::Trigger>(&t))
{
Ok(Ok(mut trigger)) => {
trigger.name = name;
triggers.push(trigger);
}
_ => out.push(Finding::unreadable(
"triggers",
&format!("trigger file `{name}.toml` did not parse"),
path.display().to_string(),
)),
}
}
let mut recent: BTreeMap<String, Vec<crate::trigger::RunRecord>> = BTreeMap::new();
let mut last_slot: BTreeMap<String, DateTime<Utc>> = BTreeMap::new();
let ledger = root.join("runs.jsonl");
if ledger.is_file() {
match std::fs::read_to_string(&ledger) {
Ok(text) => {
for line in text.lines().filter(|l| !l.trim().is_empty()) {
let Ok(row) = serde_json::from_str::<crate::trigger::RunRecord>(line) else {
continue;
};
if let Some(slot) = row.slot {
let newest = last_slot.entry(row.trigger.clone()).or_insert(slot);
if slot > *newest {
*newest = slot;
}
}
if matches!(
row.status,
crate::trigger::RunStatus::Ok | crate::trigger::RunStatus::Error
) {
let window = recent.entry(row.trigger.clone()).or_default();
window.push(row);
if window.len() > HEALTH_WINDOW {
window.remove(0);
}
}
}
}
Err(e) => out.push(Finding::unreadable(
"triggers",
"the run ledger",
format!("{}: {e}", ledger.display()),
)),
}
}
for trigger in &triggers {
if !trigger.enabled {
continue;
}
let window = recent.get(&trigger.name);
if let Some(row) = window.and_then(|w| w.last()) {
if row.status == crate::trigger::RunStatus::Error {
out.push(Finding {
component: "triggers".to_string(),
severity: Severity::Attention,
summary: format!("trigger `{}`'s most recent run failed", trigger.name),
detail: format!(
"started {}: {}",
row.started_at.to_rfc3339(),
row.error.as_deref().unwrap_or("no error recorded")
),
remedy: Some(Remedy {
description: format!(
"run `{}` by hand — a manual run is evidence, not a fire; it never advances the schedule",
trigger.name
),
argv: vec![
"mecha".into(),
"trigger".into(),
"run".into(),
trigger.name.clone(),
],
needs_terminal: false,
}),
});
}
}
let (calls, errors) = window
.map(|w| {
w.iter().fold((0u32, 0u32), |(c, e), r| {
(c + r.tool_calls, e + r.tool_errors)
})
})
.unwrap_or((0, 0));
if calls >= HEALTH_MIN_CALLS && f64::from(errors) / f64::from(calls) >= HEALTH_ERROR_RATE {
let runs = window.map(Vec::len).unwrap_or(0);
out.push(Finding {
component: "triggers".to_string(),
severity: Severity::Attention,
summary: format!(
"trigger `{}` failed {errors} of {calls} tool calls",
trigger.name
),
detail: format!(
"across its last {runs} run(s){}. A run's answer arrives either way, so this is invisible in the ledger's status — and per-step reliability is what decides how long a task the run can finish, so a third of the calls failing is not a third of the work lost.",
if window.is_some_and(|w| w.last().is_some_and(|r| r.ended_on_failed_call)) {
", and the most recent run answered with its last call failed"
} else {
""
}
),
remedy: Some(Remedy {
description: format!("read `{}`'s recent runs", trigger.name),
argv: vec![
"mecha".into(),
"trigger".into(),
"show".into(),
trigger.name.clone(),
],
needs_terminal: false,
}),
});
}
if let Some(window) = window {
let newest = window.last();
let before: u32 = window[..window.len().saturating_sub(1)]
.iter()
.map(|r| r.tool_calls)
.sum();
let stopped = newest
.is_some_and(|r| r.tool_calls == 0 && r.status == crate::trigger::RunStatus::Ok)
&& before >= HEALTH_MIN_CALLS;
if stopped {
out.push(Finding {
component: "triggers".to_string(),
severity: Severity::Attention,
summary: format!(
"trigger `{}`'s most recent run did no work",
trigger.name
),
detail: format!(
"it succeeded having made no tool calls, where its previous {} run(s) made {before}. A run that does nothing and reports success is indistinguishable from a healthy one in every other signal — the status is `ok`, the schedule advanced, and the answer arrived.",
window.len() - 1
),
remedy: Some(Remedy {
description: format!("read `{}`'s recent runs", trigger.name),
argv: vec![
"mecha".into(),
"trigger".into(),
"show".into(),
trigger.name.clone(),
],
needs_terminal: false,
}),
});
}
}
if trigger.catch_up != crate::trigger::CatchUp::Always {
continue;
}
let Some(anchor) = last_slot.get(&trigger.name).copied().or(trigger.created_at) else {
continue;
};
let tz = trigger.tz(None);
let step = chrono::Duration::seconds(1);
let missed_more_than_two = trigger
.schedule
.prev_at_or_before(now, tz)
.and_then(|s0| trigger.schedule.prev_at_or_before(s0 - step, tz))
.and_then(|s1| trigger.schedule.prev_at_or_before(s1 - step, tz))
.is_some_and(|s2| s2 > anchor);
if missed_more_than_two {
out.push(Finding {
component: "triggers".to_string(),
severity: Severity::Attention,
summary: format!("trigger `{}` has missed more than two slots", trigger.name),
detail: format!(
"last accounted slot {}; with catch_up=always a healthy scheduler fires \
the most recent slot every tick, so the daemon or its timer may be down \
(systemctl --user status mecha-triggers)",
anchor.to_rfc3339()
),
remedy: None,
});
}
}
out
}
const RUNS_WINDOW: usize = 200;
const RUNS_MIN: usize = 20;
const ENDED_ON_FAILURE_RATE: f64 = 0.20;
const TOOL_ERROR_RATE: f64 = 0.25;
const RUNS_MIN_CALLS: u64 = 20;
const CUT_SHORT_RATE: f64 = 0.25;
fn cut_short(stats: &crate::session::RunStats) -> bool {
stats.stop_cause.is_some_and(|c| c.cut_short())
}
fn check_charter(path: &Path) -> Vec<Finding> {
if !path.exists() {
return Vec::new();
}
let remedy = |description: &str| {
Some(Remedy {
description: description.to_string(),
argv: vec!["mecha".to_string(), "charter".to_string()],
needs_terminal: false,
})
};
match crate::charter::Charter::load(path) {
Err(e) => vec![Finding {
component: "charter".to_string(),
severity: Severity::Broken,
summary: "charter did not load".to_string(),
detail: format!(
"{}: {e:#} — every run is proceeding un-chartered",
path.display()
),
remedy: remedy("see the parse error and fix charter.toml"),
}],
Ok(charter) if charter.over_budget() => vec![Finding {
component: "charter".to_string(),
severity: Severity::Attention,
summary: "charter is over its character budget".to_string(),
detail: format!(
"{} is {} characters, over the {}-character budget",
path.display(),
charter.char_count(),
crate::charter::CHARTER_CHAR_BUDGET,
),
remedy: remedy("review the charter and trim it"),
}],
Ok(charter) if charter.is_empty() => vec![Finding {
component: "charter".to_string(),
severity: Severity::Attention,
summary: "charter file exists but has no lines".to_string(),
detail: format!(
"{} parsed cleanly with zero `[[line]]` entries — nothing from it \
rides in any prompt",
path.display()
),
remedy: remedy("see what's actually in the charter file"),
}],
Ok(_) => Vec::new(),
}
}
fn check_runs(sessions: &Path) -> Vec<Finding> {
use crate::runlog::{Corpus, Scan};
let mut out = Vec::new();
if !sessions.is_dir() {
return out;
}
let corpus = match Corpus::scan(
sessions,
&Scan {
max_sessions: Some(RUNS_WINDOW),
since: None,
workspace: None,
},
) {
Ok(c) => c,
Err(e) => {
out.push(Finding::unreadable(
"runs",
"the session store",
format!("{}: {e}", sessions.display()),
));
return out;
}
};
if corpus.unreadable > 0 {
out.push(Finding::unreadable(
"runs",
&format!("{} transcript(s) in the session store", corpus.unreadable),
format!(
"{}: files with a .jsonl extension that could not be read \
or carry no session header; every reader silently skips them",
sessions.display()
),
));
}
let remedy = |what: &str| {
Some(Remedy {
description: format!("read the run-quality summary ({what})"),
argv: vec![
"mecha".into(),
"sessions".into(),
"health".into(),
"--days".into(),
"30".into(),
],
needs_terminal: false,
})
};
for (model, runs) in corpus.by_model() {
if runs.len() < RUNS_MIN {
continue;
}
let n = runs.len();
if let Some(rate) = runs.rate_of(|r| r.stats.ended_on_failed_call) {
if rate >= ENDED_ON_FAILURE_RATE {
out.push(Finding {
component: "runs".to_string(),
severity: Severity::Attention,
summary: format!(
"{:.0}% of `{model}` runs finished on a failed tool call",
rate * 100.0
),
detail: format!(
"{} of {n} recent run(s). The model stopped of its own accord with its last call failed, and the answer it wrote may report success over it — which nothing in the text or the stop reason can show.",
runs.ended_on_failed_call()
),
remedy: remedy("which runs, and what failed"),
});
}
}
if let Some(rate) = runs.tool_error_rate() {
if rate >= TOOL_ERROR_RATE && runs.tool_calls() >= RUNS_MIN_CALLS {
out.push(Finding {
component: "runs".to_string(),
severity: Severity::Attention,
summary: format!(
"`{model}` runs fail {:.0}% of their tool calls",
rate * 100.0
),
detail: format!(
"{} of {} call(s) across {n} run(s) were refused by the environment. Errors are how a run learns where it is, so some are healthy — a quarter of them says something moved: a renamed path, a revoked grant, a tool whose schema the model keeps mis-filling.",
runs.tool_errors(),
runs.tool_calls()
),
remedy: remedy("which tool, and how it failed"),
});
}
}
if let Some(rate) = runs.rate_of(|r| cut_short(&r.stats)) {
if rate >= CUT_SHORT_RATE {
let cut = runs.rows.iter().filter(|r| cut_short(&r.stats)).count();
out.push(Finding {
component: "runs".to_string(),
severity: Severity::Attention,
summary: format!(
"the harness cut {:.0}% of `{model}` runs short",
rate * 100.0
),
detail: format!(
"{cut} of {n} recent run(s) hit a turn, token or cost ceiling, or tripped the loop guard. A budget that stops a quarter of runs is measuring the budget rather than the work — the answers are truncated and say so only in `stop_cause`. Cancellations are not counted here.",
),
remedy: remedy("which ceiling, and how often"),
});
}
}
}
out
}
const GRAPH_NIGHTLIES: &[(&str, &str)] = &[
("nightly-", "the graph's own sweep (ingest, extract, decay)"),
("mecha-nightly-", "the mecha half (vet, precheck, gossip)"),
];
const RECENT_CONSOLIDATION: chrono::Duration = chrono::Duration::hours(48);
fn learned_within(root: &Path, now: DateTime<Utc>, window: chrono::Duration) -> bool {
let Ok(text) = std::fs::read_to_string(root.join("runs.jsonl")) else {
return false;
};
text.lines()
.filter(|l| !l.trim().is_empty())
.filter_map(|l| serde_json::from_str::<serde_json::Value>(l).ok())
.filter(|v| v["reflexions_processed"].as_u64().unwrap_or(0) > 0)
.filter_map(|v| {
v["created_at"]
.as_str()
.and_then(|s| DateTime::parse_from_rfc3339(s).ok())
})
.any(|t| now.signed_duration_since(t.with_timezone(&Utc)) < window)
}
fn read_learned_rules(root: &Path, domain: &str) -> Option<Vec<crate::learning::Rule>> {
let path = root.join("rules").join(format!("{domain}.learned.toml"));
if !path.exists() {
return Some(Vec::new());
}
let text = std::fs::read_to_string(&path).ok()?;
#[derive(serde::Deserialize)]
struct File {
#[serde(default)]
rules: Vec<crate::learning::Rule>,
}
toml::from_str::<File>(&text).ok().map(|f| f.rules)
}
fn same_rules_as_accept(a: &[crate::learning::Rule], b: &[crate::learning::Rule]) -> bool {
a.len() == b.len()
&& a.iter()
.zip(b)
.all(|(x, y)| x.text == y.text && x.enabled == y.enabled)
}
const STALE_PROPOSAL_AFTER: chrono::Duration = chrono::Duration::hours(48);
fn check_proposal_review(root: &Path, now: DateTime<Utc>) -> Vec<Finding> {
let mut out = Vec::new();
let dir = root.join("proposals");
let Ok(entries) = std::fs::read_dir(&dir) else {
return out;
};
let mut pending: Vec<(String, DateTime<Utc>, usize, bool)> = Vec::new();
for entry in entries.flatten() {
let Ok(text) = std::fs::read_to_string(entry.path()) else {
out.push(Finding::unreadable(
"learning",
"a rule proposal",
entry.path().display().to_string(),
));
continue;
};
let Ok(p) = serde_json::from_str::<crate::learning::Proposal>(&text) else {
continue;
};
if p.status != "pending" {
continue;
}
let Ok(at) = DateTime::parse_from_rfc3339(&p.created_at) else {
continue;
};
let unappliable = read_learned_rules(root, &p.domain)
.map(|live| !same_rules_as_accept(&live, &p.rules_before))
.unwrap_or(false);
pending.push((
p.id.clone(),
at.with_timezone(&Utc),
p.reflexion_ids.len(),
unappliable,
));
}
if pending.is_empty() {
return out;
}
pending.sort_by_key(|(_, at, _, _)| *at);
let held: usize = pending.iter().map(|(_, _, n, _)| n).sum();
let oldest = pending[0].1;
let age = now.signed_duration_since(oldest);
let unappliable: Vec<&str> = pending
.iter()
.filter(|(_, _, _, stale)| *stale)
.map(|(id, _, _, _)| id.as_str())
.collect();
if !unappliable.is_empty() {
out.push(Finding {
component: "learning".to_string(),
severity: Severity::Attention,
summary: format!(
"{} rule proposal(s) can no longer be applied — the live rules moved \
after they were measured",
unappliable.len()
),
detail: format!(
"`proposals accept` refuses a proposal whose baseline has changed, so these \
are not decisions waiting on you — they are paper, and they still hold \
their reflections out of `learn`. Superseding releases that evidence \
*unconsumed*; rejecting would mark it processed and lose corrections you \
never ruled on. Affected: {}.",
unappliable.join(", ")
),
remedy: Some(Remedy {
description: "release their reflections back to the pool".to_string(),
argv: vec![
"mecha".into(),
"proposals".into(),
"supersede".into(),
"--stale".into(),
],
needs_terminal: false,
}),
});
}
if age > STALE_PROPOSAL_AFTER && unappliable.len() < pending.len() {
out.push(Finding {
component: "learning".to_string(),
severity: Severity::Attention,
summary: format!(
"{} rule proposal(s) awaiting review, oldest {} day(s) — holding {held} \
reflection(s) out of `learn`",
pending.len(),
age.num_days().max(1),
),
detail: "A pending proposal claims its reflections, and `learn` skips claimed \
ones — so an unreviewed queue starves the pass that would replace it, \
and every night reports success with nothing to show. Only one of \
several can ever be applied: each is a full rewrite measured against \
the rules that were live when it was staged."
.to_string(),
remedy: Some(Remedy {
description: "read what is waiting".to_string(),
argv: vec!["mecha".into(), "proposals".into()],
needs_terminal: false,
}),
});
}
out
}
const STALE_CANDIDATE_AFTER: chrono::Duration = chrono::Duration::hours(72);
const STARVED_LEARNER_MIN_EXCLUDED: usize = 10;
fn check_learning(root: &Path, now: DateTime<Utc>) -> Vec<Finding> {
let mut out = Vec::new();
let path = root.join("reflections.jsonl");
if !path.is_file() {
return out;
}
let text = match std::fs::read_to_string(&path) {
Ok(t) => t,
Err(e) => {
out.push(Finding::unreadable(
"learning",
"the reflections archive",
format!("{}: {e}", path.display()),
));
return out;
}
};
let mut total = 0usize;
let mut excluded = 0usize;
let mut newest_excluded: Option<DateTime<Utc>> = None;
let mut waiting: std::collections::BTreeMap<String, usize> = Default::default();
for line in text.lines().filter(|l| !l.trim().is_empty()) {
let Ok(r) = serde_json::from_str::<crate::learning::Reflexion>(line) else {
continue;
};
total += 1;
if r.learnable() {
if !r.is_processed {
*waiting.entry(r.domain.clone()).or_default() += 1;
}
} else if r.dropped_at.is_none() {
excluded += 1;
if let Ok(t) = DateTime::parse_from_rfc3339(&r.created_at) {
let t = t.with_timezone(&Utc);
if newest_excluded.is_none_or(|n| t > n) {
newest_excluded = Some(t);
}
}
}
}
let floor = crate::learning::LEARN_MIN_REFLECTIONS;
if waiting.values().any(|&n| n >= floor) {
return out;
}
if learned_within(root, now, RECENT_CONSOLIDATION) {
return out;
}
if excluded < STARVED_LEARNER_MIN_EXCLUDED {
return out;
}
let alive = newest_excluded.is_some_and(|t| now.signed_duration_since(t).num_days() <= 30);
if !alive {
return out;
}
let pool = if waiting.is_empty() {
"none clean and unprocessed".to_string()
} else {
waiting
.iter()
.map(|(d, n)| format!("{d} {n}/{floor}"))
.collect::<Vec<_>>()
.join(", ")
};
out.push(Finding {
component: "learning".to_string(),
severity: Severity::Attention,
summary: format!(
"the rule learner is starved: {excluded} of {total} reflections excluded by \
origin, and no domain reaches the learn floor of {floor}"
),
detail: format!(
"reflect keeps mining and the provenance gate keeps excluding — the gate working \
as designed, every night, with nothing downstream to show for it. Clean pool: \
{pool}. The excluded records stay readable in {} — some are third-party evidence \
the gate held back, some may be mecha's own words correctly kept out of a \
feedback loop; the decision this proposes is yours, not a command's: read what \
got excluded, and change what evidence the loop may consolidate if the mix \
looks wrong.",
path.display()
),
remedy: Some(Remedy {
description: "see how new interventions classify — doctor never loosens the gate"
.to_string(),
argv: vec!["mecha".into(), "reflect".into(), "--dry-run".into()],
needs_terminal: false,
}),
});
out
}
fn check_harness(root: &Path, now: DateTime<Utc>) -> Vec<Finding> {
let mut out = Vec::new();
let dir = root.join("candidates");
if !dir.is_dir() {
return out;
}
let entries = match std::fs::read_dir(&dir) {
Ok(entries) => entries,
Err(e) => {
out.push(Finding::unreadable(
"harness",
"the harness candidate directory",
format!("{}: {e}", dir.display()),
));
return out;
}
};
let mut stale: Vec<String> = Vec::new();
for entry in entries.flatten() {
let path = entry.path();
if path.extension().and_then(|e| e.to_str()) != Some("json") {
continue;
}
let cand: crate::harness::HarnessCandidate =
match std::fs::read_to_string(&path).map(|t| serde_json::from_str(&t)) {
Ok(Ok(cand)) => cand,
Ok(Err(e)) => {
out.push(Finding::unreadable(
"harness",
&format!(
"candidate {} did not parse",
path.file_name().unwrap_or_default().to_string_lossy()
),
format!("{}: {e}", path.display()),
));
continue;
}
Err(e) => {
out.push(Finding::unreadable(
"harness",
&format!(
"candidate {} could not be read",
path.file_name().unwrap_or_default().to_string_lossy()
),
format!("{}: {e}", path.display()),
));
continue;
}
};
if !cand.pending() {
continue;
}
let old_enough = chrono::DateTime::parse_from_rfc3339(&cand.created_at)
.map(|t| {
now.signed_duration_since(t.with_timezone(&chrono::Utc)) > STALE_CANDIDATE_AFTER
})
.unwrap_or(true);
if old_enough {
stale.push(format!("{} · {:?} {}", cand.id, cand.class, cand.change));
}
}
if !stale.is_empty() {
stale.sort();
out.push(Finding {
component: "harness".to_string(),
severity: Severity::Attention,
summary: format!(
"{} harness candidate(s) staged for more than {}h",
stale.len(),
STALE_CANDIDATE_AFTER.num_hours()
),
detail: stale.join("\n"),
remedy: Some(Remedy {
description: "review the staged candidates — doctor never accepts one".to_string(),
argv: vec!["mecha".into(), "harness".into(), "list".into()],
needs_terminal: false,
}),
});
}
out
}
fn check_graph_nightly(store: &Path, now: DateTime<Utc>) -> Vec<Finding> {
let mut out = Vec::new();
let logs = store.join("logs");
if !logs.is_dir() {
return out;
}
let names: Vec<String> = match std::fs::read_dir(&logs) {
Ok(entries) => entries
.flatten()
.filter_map(|e| e.file_name().to_str().map(String::from))
.collect(),
Err(e) => {
out.push(Finding::unreadable(
"graph",
"the graph nightly logs",
format!("{}: {e}", logs.display()),
));
return out;
}
};
for (prefix, what) in GRAPH_NIGHTLIES {
let newest = names
.iter()
.filter_map(|n| {
n.strip_prefix(prefix)?
.strip_suffix(".log")
.and_then(|d| chrono::NaiveDate::parse_from_str(d, "%Y%m%d").ok())
})
.max();
let Some(newest) = newest else { continue };
let days_quiet = (now.date_naive() - newest).num_days();
if days_quiet > 1 {
out.push(Finding {
component: "graph".to_string(),
severity: Severity::Attention,
summary: format!(
"the graph nightly ({}) has not run for {days_quiet} days",
prefix.trim_end_matches('-'),
),
detail: format!(
"{what} last wrote {}{}.log under {}; it logs every \
run including deferred ones, so a missing day means the \
script never started — cron reports that nowhere",
prefix,
newest.format("%Y%m%d"),
logs.display(),
),
remedy: Some(Remedy {
description: "list the cron entries that fire the graph nightlies, \
then run the silent one by hand and read its error"
.to_string(),
argv: vec!["crontab".into(), "-l".into()],
needs_terminal: false,
}),
});
}
}
out
}
fn age_of(stamp: &str, now: DateTime<Utc>) -> Option<chrono::Duration> {
DateTime::parse_from_rfc3339(stamp)
.ok()
.map(|at| now - at.with_timezone(&Utc))
}
fn render_age(now: DateTime<Utc>, stamp: &str) -> String {
match age_of(stamp, now) {
Some(age) if age >= chrono::Duration::days(2) => format!("{}d ago", age.num_days()),
Some(age) if age >= chrono::Duration::hours(1) => format!("{}h ago", age.num_hours()),
Some(age) => format!("{}m ago", age.num_minutes().max(0)),
None => stamp.to_string(),
}
}
#[cfg(test)]
mod tests {
use super::*;
use crate::agent::Taint;
use crate::outbox::{OutboxItem, OutboxKind};
use serde_json::json;
use std::path::PathBuf;
fn utc(s: &str) -> DateTime<Utc> {
DateTime::parse_from_rfc3339(s).unwrap().with_timezone(&Utc)
}
const NOW: &str = "2026-08-14T12:00:00Z";
fn home(name: &str) -> PathBuf {
let dir = std::env::temp_dir().join(format!(
"mecha-doctor-test-{name}-{}-{:?}",
std::process::id(),
std::thread::current().id()
));
let _ = std::fs::remove_dir_all(&dir);
std::fs::create_dir_all(&dir).unwrap();
dir
}
fn write_marker(home: &Path, account: &str, body: &str) {
let dir = home.join("mail").join(account);
std::fs::create_dir_all(&dir).unwrap();
std::fs::write(dir.join("auth_error.json"), body).unwrap();
}
fn valid_marker() -> String {
json!({
"at": "2026-08-11T09:00:00Z",
"message": "the refresh token was revoked — run `mecha-mail auth personal --provider google` to sign in again",
})
.to_string()
}
fn pending_item(home: &Path, id: &str, created_at: &str, error: Option<&str>) {
let item = OutboxItem {
id: id.to_string(),
status: "pending".into(),
tool: "mail__send".into(),
kind: OutboxKind::Message,
args_before: json!({"to": "a@x.org"}),
args: json!({"to": "a@x.org"}),
summary: "mail__send to a@x.org".into(),
session_id: None,
workspace: None,
taint: Taint::default(),
created_at: created_at.to_string(),
resolved_at: None,
reason: None,
error: error.map(String::from),
};
let dir = home.join("outbox");
std::fs::create_dir_all(&dir).unwrap();
std::fs::write(
dir.join(format!("{id}.json")),
serde_json::to_string_pretty(&item).unwrap(),
)
.unwrap();
}
fn request(home: &Path, seq: i64, state: &str, drained_at: &str) {
let dir = home.join("requests");
std::fs::create_dir_all(&dir).unwrap();
let record = json!({
"seq": seq,
"type_id": "meeting",
"state": state,
"created_at": drained_at,
"drained_at": drained_at,
"valid": true,
"values": {},
"free_text": [],
});
std::fs::write(
dir.join(format!("{seq:010}-meeting.json")),
record.to_string(),
)
.unwrap();
}
fn trigger_file(home: &Path, name: &str, extra: &str) {
let dir = home.join("triggers");
std::fs::create_dir_all(&dir).unwrap();
std::fs::write(
dir.join(format!("{name}.toml")),
format!(
"schedule = \"0 7 * * *\"\nprompt = \"brief me\"\ntimezone = \"UTC\"\n\
created_at = \"2026-08-01T00:00:00Z\"\n{extra}"
),
)
.unwrap();
}
fn ledger_row(home: &Path, row: &serde_json::Value) {
use std::io::Write;
let dir = home.join("triggers");
std::fs::create_dir_all(&dir).unwrap();
let mut file = std::fs::OpenOptions::new()
.create(true)
.append(true)
.open(dir.join("runs.jsonl"))
.unwrap();
writeln!(file, "{row}").unwrap();
}
fn of<'a>(findings: &'a [Finding], component: &str) -> Vec<&'a Finding> {
findings
.iter()
.filter(|f| f.component == component)
.collect()
}
#[test]
fn a_dead_auth_marker_is_found_and_an_absent_one_is_not() {
let home = home("dead-auth");
write_marker(&home, "personal", &valid_marker());
std::fs::create_dir_all(home.join("mail").join("dartmouth")).unwrap();
std::fs::write(
home.join("mail").join("accounts.toml"),
"[[account]]\nname = \"personal\"\nprovider = \"google\"\n\
[[account]]\nname = \"dartmouth\"\nprovider = \"outlook\"\n",
)
.unwrap();
let findings = examine(&home, utc(NOW));
let mail = of(&findings, "mail");
assert_eq!(mail.len(), 1, "{findings:#?}");
assert_eq!(mail[0].severity, Severity::Broken);
assert!(mail[0].summary.contains("personal"), "{}", mail[0].summary);
let remedy = mail[0].remedy.as_ref().expect("a dead login has a way out");
assert_eq!(
remedy.argv,
vec!["mecha-mail", "auth", "personal", "--provider", "google"]
);
assert!(
remedy.needs_terminal,
"an OAuth flow needs the real terminal"
);
let _ = std::fs::remove_dir_all(&home);
}
#[test]
fn a_provider_the_registry_cannot_name_is_omitted_from_the_remedy_not_guessed() {
let home = home("no-registry");
write_marker(&home, "personal", &valid_marker());
let findings = examine(&home, utc(NOW));
let mail = of(&findings, "mail");
assert_eq!(mail.len(), 1);
let remedy = mail[0].remedy.as_ref().unwrap();
assert_eq!(remedy.argv, vec!["mecha-mail", "auth", "personal"]);
assert!(
mail[0].detail.contains("--provider google"),
"{}",
mail[0].detail
);
let _ = std::fs::remove_dir_all(&home);
}
#[test]
fn a_marker_in_a_legacy_per_provider_store_is_found_and_proposes_import() {
let home = home("legacy-auth");
let dir = home.join("google");
std::fs::create_dir_all(&dir).unwrap();
std::fs::write(
dir.join("auth_error.json"),
json!({
"at": "2026-08-11T09:00:00Z",
"message": "account `google`: refresh token expired or revoked — run `mecha-mail auth google --provider google` (invalid_grant)",
})
.to_string(),
)
.unwrap();
let findings = examine(&home, utc(NOW));
let mail = of(&findings, "mail");
assert_eq!(mail.len(), 1, "{findings:#?}");
assert_eq!(mail[0].severity, Severity::Broken);
assert!(
mail[0].summary.contains("legacy google"),
"{}",
mail[0].summary
);
assert!(
mail[0]
.detail
.contains("run `mecha-mail auth google --provider google`"),
"{}",
mail[0].detail
);
let remedy = mail[0].remedy.as_ref().expect("a way out");
assert_eq!(
remedy.argv,
vec!["mecha-mail", "import", "google", "--provider", "google"]
);
let _ = std::fs::remove_dir_all(&home);
}
#[test]
fn an_unparseable_marker_is_a_store_unreadable_finding_not_a_crash() {
let home = home("bad-marker");
write_marker(&home, "personal", "{ this is not json");
let findings = examine(&home, utc(NOW));
let mail = of(&findings, "mail");
assert_eq!(mail.len(), 1, "{findings:#?}");
assert!(
mail[0].summary.starts_with("store unreadable:"),
"{}",
mail[0].summary
);
assert!(mail[0].summary.contains("personal"), "{}", mail[0].summary);
let _ = std::fs::remove_dir_all(&home);
}
#[test]
fn a_pending_item_with_an_error_is_broken_and_a_resolved_one_is_not() {
let home = home("outbox-error");
pending_item(
&home,
"20260814-000001-aaa",
NOW,
Some("server unreachable"),
);
let mut sent = json!({
"id": "20260810-000001-bbb",
"status": "sent",
"tool": "mail__send",
"args_before": {},
"args": {},
"summary": "mail__send",
"created_at": "2026-08-01T00:00:00Z",
});
sent["error"] = json!(null);
std::fs::write(
home.join("outbox").join("20260810-000001-bbb.json"),
sent.to_string(),
)
.unwrap();
let findings = examine(&home, utc(NOW));
let outbox = of(&findings, "outbox");
assert_eq!(outbox.len(), 1, "{findings:#?}");
assert_eq!(outbox[0].severity, Severity::Broken);
assert!(
outbox[0]
.summary
.contains("release failed: server unreachable"),
"{}",
outbox[0].summary
);
let remedy = outbox[0].remedy.as_ref().unwrap();
assert_eq!(remedy.argv, vec!["mecha", "outbox", "review"]);
let _ = std::fs::remove_dir_all(&home);
}
fn question(home: &Path, id: &str, asked_at: &str, status: &str) {
let dir = home.join("questions");
std::fs::create_dir_all(&dir).unwrap();
let q = crate::questions::Question {
id: id.into(),
status: status.into(),
question: "Which address should the letter go to?".into(),
options: vec![],
session_id: "sess-1".into(),
task_id: Some("task-9".into()),
workspace: None,
taint: Default::default(),
asked_at: asked_at.into(),
answered_at: None,
answer: None,
};
std::fs::write(
dir.join(format!("{id}.json")),
serde_json::to_string_pretty(&q).unwrap(),
)
.unwrap();
}
#[test]
fn an_unanswered_question_is_stale_at_25_hours_and_not_at_23() {
let home = home("questions-stale");
question(
&home,
"20260813-100000-aaaaaaaa",
"2026-08-13T10:00:00Z",
"open",
);
let findings = examine(&home, utc(NOW));
let qs = of(&findings, "questions");
assert_eq!(qs.len(), 1, "{findings:#?}");
assert_eq!(qs[0].severity, Severity::Attention);
assert!(
qs[0].summary.contains("cannot continue"),
"{:?}",
qs[0].summary
);
assert_eq!(
qs[0].remedy.as_ref().unwrap().argv,
vec!["mecha", "questions", "list"],
"doctor lists what is stuck; it never answers for the owner"
);
let fresh = home;
let _ = std::fs::remove_dir_all(fresh.join("questions"));
question(
&fresh,
"20260813-130000-bbbbbbbb",
"2026-08-13T13:00:00Z",
"open",
);
assert!(of(&examine(&fresh, utc(NOW)), "questions").is_empty());
let _ = std::fs::remove_dir_all(&fresh);
}
#[test]
fn an_answered_question_never_ages_into_a_finding() {
let home = home("questions-answered");
question(
&home,
"20260701-100000-cccccccc",
"2026-07-01T10:00:00Z",
"answered",
);
question(
&home,
"20260701-100000-dddddddd",
"2026-07-01T10:00:00Z",
"abandoned",
);
assert!(of(&examine(&home, utc(NOW)), "questions").is_empty());
let _ = std::fs::remove_dir_all(&home);
}
#[test]
fn a_store_that_was_never_created_is_not_a_finding() {
let home = home("questions-absent");
assert!(of(&examine(&home, utc(NOW)), "questions").is_empty());
let _ = std::fs::remove_dir_all(&home);
}
#[test]
fn a_pending_draft_is_stale_at_49_hours_and_not_at_47() {
let home = home("outbox-stale");
pending_item(&home, "20260812-110000-old", "2026-08-12T11:00:00Z", None);
let findings = examine(&home, utc(NOW));
let outbox = of(&findings, "outbox");
assert_eq!(outbox.len(), 1, "{findings:#?}");
assert_eq!(outbox[0].severity, Severity::Attention);
assert!(outbox[0].summary.contains("pending for more than 48h"));
assert_eq!(
outbox[0].remedy.as_ref().unwrap().argv,
vec!["mecha", "outbox", "review"],
"the remedy is the review surface, never send"
);
let fresh = home;
let _ = std::fs::remove_dir_all(fresh.join("outbox"));
pending_item(&fresh, "20260812-130000-new", "2026-08-12T13:00:00Z", None);
let findings = examine(&fresh, utc(NOW));
assert!(of(&findings, "outbox").is_empty(), "{findings:#?}");
let _ = std::fs::remove_dir_all(&fresh);
}
fn harness_candidate(home: &Path, id: &str, created_at: &str, status: &str) {
let dir = home.join("learning").join("harness").join("candidates");
std::fs::create_dir_all(&dir).unwrap();
let cand = crate::harness::HarnessCandidate {
id: id.into(),
created_at: created_at.into(),
class: crate::candidate::ChangeClass::Config,
change: "compact_at_tokens=24000".into(),
metric: crate::candidate::Metric::CutShort,
rationale: "test".into(),
evidence: String::new(),
model: None,
status: status.into(),
measurement: None,
resolved_at: None,
reason: None,
};
std::fs::write(
dir.join(format!("{id}.json")),
serde_json::to_string_pretty(&cand).unwrap(),
)
.unwrap();
}
fn reflection_line(id: &str, origin: &str, processed: bool, created_at: &str) -> String {
reflection_line_with_intervention(id, origin, processed, created_at, "")
}
fn reflection_line_with_intervention(
id: &str,
origin: &str,
processed: bool,
created_at: &str,
intervention: &str,
) -> String {
serde_json::json!({
"id": id,
"domain": "behavior",
"session_id": "s",
"trigger": "steer",
"context": "",
"intervention": intervention,
"reflexion_text": "test",
"is_processed": processed,
"created_at": created_at,
"origin": origin,
})
.to_string()
}
fn reflection_line_dropped(id: &str, origin: &str, created_at: &str) -> String {
serde_json::json!({
"id": id,
"domain": "behavior",
"session_id": "s",
"trigger": "steer",
"context": "",
"intervention": "",
"reflexion_text": "test",
"is_processed": false,
"created_at": created_at,
"origin": origin,
"dropped_at": created_at,
})
.to_string()
}
fn write_reflections(home: &Path, lines: &[String]) {
let dir = home.join("learning");
std::fs::create_dir_all(&dir).unwrap();
std::fs::write(dir.join("reflections.jsonl"), lines.join("\n")).unwrap();
}
#[test]
fn a_learner_that_just_ran_is_not_starved() {
let home = home("learning-just-ran");
let root = home.join("learning");
std::fs::create_dir_all(&root).unwrap();
let mut lines = String::new();
for i in 0..12 {
lines.push_str(&format!(
r#"{{"id":"x{i}","domain":"behavior","session_id":"s","trigger":"steer","context":"c","intervention":"i","reflexion_text":"t","error_type":null,"confidence":null,"is_processed":false,"leap_run_id":null,"created_at":"2026-08-28T00:00:00Z","origin":"untrusted","evidence":"full"}}
"#
));
}
std::fs::write(root.join("reflections.jsonl"), &lines).unwrap();
let now = DateTime::parse_from_rfc3339("2026-08-29T12:00:00Z")
.unwrap()
.with_timezone(&Utc);
assert!(
check_learning(&root, now)
.iter()
.any(|f| f.summary.contains("starved")),
"an unfed learner with no recent pass is starved"
);
std::fs::write(
root.join("runs.jsonl"),
"{\"id\":\"r1\",\"domain\":\"behavior\",\"reflexions_processed\":28,\"rules_before\":0,\"rules_after\":12,\"created_at\":\"2026-08-29T09:00:00Z\"}\n",
)
.unwrap();
assert!(
check_learning(&root, now).is_empty(),
"a consolidation nine hours ago is the pool being consumed, not starvation"
);
std::fs::write(
root.join("runs.jsonl"),
"{\"id\":\"r1\",\"domain\":\"behavior\",\"reflexions_processed\":28,\"rules_before\":0,\"rules_after\":12,\"created_at\":\"2026-07-20T09:00:00Z\"}\n",
)
.unwrap();
assert!(
check_learning(&root, now)
.iter()
.any(|f| f.summary.contains("starved")),
"a pass five weeks old explains nothing about today"
);
std::fs::write(
root.join("runs.jsonl"),
"{\"id\":\"r2\",\"domain\":\"behavior\",\"reflexions_processed\":0,\"rules_before\":12,\"rules_after\":11,\"created_at\":\"2026-08-29T09:00:00Z\"}\n",
)
.unwrap();
assert!(
check_learning(&root, now)
.iter()
.any(|f| f.summary.contains("starved")),
"a retirement pass consumed no reflections and must not silence starvation"
);
std::fs::remove_dir_all(&home).ok();
}
#[test]
fn a_learner_fed_only_excluded_evidence_is_starved_and_a_met_floor_is_not() {
let home = home("learning-starved");
let mut lines: Vec<String> = (0..12)
.map(|i| reflection_line(&format!("u{i}"), "untrusted", false, "2026-08-13T12:00:00Z"))
.collect();
lines.push(reflection_line(
"c1",
"clean",
false,
"2026-08-05T00:00:00Z",
));
write_reflections(&home, &lines);
let findings = examine(&home, utc(NOW));
let learning = of(&findings, "learning");
assert_eq!(learning.len(), 1, "{findings:#?}");
assert_eq!(learning[0].severity, Severity::Attention);
assert!(
learning[0].summary.contains("starved"),
"{}",
learning[0].summary
);
assert!(
learning[0].summary.contains("12 of 13"),
"{}",
learning[0].summary
);
assert_eq!(
learning[0].remedy.as_ref().unwrap().argv,
vec!["mecha", "reflect", "--dry-run"],
"the remedy shows classifications; nothing may loosen the gate"
);
lines.push(reflection_line(
"c2",
"clean",
false,
"2026-08-06T00:00:00Z",
));
lines.push(reflection_line(
"c3",
"clean",
false,
"2026-08-07T00:00:00Z",
));
write_reflections(&home, &lines);
let findings = examine(&home, utc(NOW));
assert!(of(&findings, "learning").is_empty(), "{findings:#?}");
let _ = std::fs::remove_dir_all(&home);
}
#[test]
fn an_owners_drop_is_not_a_provenance_exclusion() {
let home = home("learning-dropped");
let lines: Vec<String> = (0..12)
.map(|i| reflection_line_dropped(&format!("d{i}"), "untrusted", "2026-08-13T12:00:00Z"))
.collect();
write_reflections(&home, &lines);
assert!(
of(&examine(&home, utc(NOW)), "learning").is_empty(),
"a dozen owner refusals must not read as a starved learner"
);
let mut lines = lines;
lines.extend((0..5).map(|i| {
reflection_line(&format!("u{i}"), "untrusted", false, "2026-08-13T12:00:00Z")
}));
write_reflections(&home, &lines);
assert!(
of(&examine(&home, utc(NOW)), "learning").is_empty(),
"5 genuine exclusions is below the floor even with 12 drops beside them"
);
lines.extend((5..10).map(|i| {
reflection_line(&format!("u{i}"), "untrusted", false, "2026-08-13T12:00:00Z")
}));
write_reflections(&home, &lines);
let findings = examine(&home, utc(NOW));
let learning = of(&findings, "learning");
assert_eq!(learning.len(), 1, "{findings:#?}");
assert!(
learning[0].summary.contains("10 of"),
"the 12 drops must not be counted as excluded: {}",
learning[0].summary
);
let _ = std::fs::remove_dir_all(&home);
}
#[test]
fn a_reflection_stored_clean_before_harness_voice_existed_does_not_count_as_waiting() {
let home = home("learning-harness-voice");
let mut lines: Vec<String> = (0..10)
.map(|i| reflection_line(&format!("u{i}"), "untrusted", false, "2026-08-13T12:00:00Z"))
.collect();
lines.push(reflection_line_with_intervention(
"h1",
"clean",
false,
"2026-08-05T00:00:00Z",
crate::agent::FINAL_ANSWER_NUDGE,
));
lines.push(reflection_line_with_intervention(
"h2",
"clean",
false,
"2026-08-06T00:00:00Z",
crate::agent::FINAL_ANSWER_NUDGE,
));
lines.push(reflection_line(
"c1",
"clean",
false,
"2026-08-07T00:00:00Z",
));
write_reflections(&home, &lines);
let findings = examine(&home, utc(NOW));
let learning = of(&findings, "learning");
assert_eq!(learning.len(), 1, "{findings:#?}");
assert!(
learning[0].summary.contains("starved"),
"the two harness-voice records must not read as met-floor evidence: {}",
learning[0].summary
);
let _ = std::fs::remove_dir_all(&home);
}
#[test]
fn thin_or_dormant_exclusion_is_not_starvation() {
let home = home("learning-thin");
let lines: Vec<String> = (0..9)
.map(|i| reflection_line(&format!("u{i}"), "untrusted", false, "2026-08-13T12:00:00Z"))
.collect();
write_reflections(&home, &lines);
assert!(of(&examine(&home, utc(NOW)), "learning").is_empty());
let lines: Vec<String> = (0..12)
.map(|i| reflection_line(&format!("u{i}"), "untrusted", false, "2026-05-01T12:00:00Z"))
.collect();
write_reflections(&home, &lines);
assert!(of(&examine(&home, utc(NOW)), "learning").is_empty());
let mut lines: Vec<String> = (0..12)
.map(|i| reflection_line(&format!("u{i}"), "untrusted", false, "2026-08-13T12:00:00Z"))
.collect();
for i in 0..3 {
lines.push(reflection_line(
&format!("p{i}"),
"clean",
true,
"2026-08-05T00:00:00Z",
));
}
write_reflections(&home, &lines);
let findings = examine(&home, utc(NOW));
assert_eq!(of(&findings, "learning").len(), 1, "{findings:#?}");
let _ = std::fs::remove_dir_all(&home);
}
#[test]
fn a_staged_harness_candidate_is_stale_at_73_hours_and_not_at_71() {
let home = home("harness-stale");
harness_candidate(&home, "hc-old", "2026-08-11T11:00:00Z", "staged");
harness_candidate(&home, "hc-done", "2026-08-01T00:00:00Z", "rejected");
let findings = examine(&home, utc(NOW));
let harness = of(&findings, "harness");
assert_eq!(harness.len(), 1, "{findings:#?}");
assert_eq!(harness[0].severity, Severity::Attention);
assert!(harness[0].summary.contains("staged for more than 72h"));
assert!(
harness[0].detail.contains("hc-old"),
"{}",
harness[0].detail
);
assert_eq!(
harness[0].remedy.as_ref().unwrap().argv,
vec!["mecha", "harness", "list"],
"the remedy is the review surface, never accept"
);
let _ = std::fs::remove_dir_all(home.join("learning"));
harness_candidate(&home, "hc-new", "2026-08-11T13:00:00Z", "staged");
let findings = examine(&home, utc(NOW));
assert!(of(&findings, "harness").is_empty(), "{findings:#?}");
let _ = std::fs::remove_dir_all(&home);
}
#[test]
fn a_failed_extraction_is_broken_at_any_age() {
let home = home("frontdoor-failed");
request(&home, 12, crate::frontdoor::EXTRACTION_FAILED, NOW);
let findings = examine(&home, utc(NOW));
let front = of(&findings, "frontdoor");
assert_eq!(front.len(), 1, "{findings:#?}");
assert_eq!(front[0].severity, Severity::Broken);
assert!(front[0].summary.contains("12"), "{}", front[0].summary);
assert_eq!(
front[0].remedy.as_ref().unwrap().argv,
vec!["mecha", "frontdoor", "list"]
);
let _ = std::fs::remove_dir_all(&home);
}
#[test]
fn a_request_waiting_on_me_is_stale_at_73_hours_and_not_at_71() {
let home = home("frontdoor-stale");
request(
&home,
1,
crate::frontdoor::AWAITING_ME,
"2026-08-11T11:00:00Z",
);
let findings = examine(&home, utc(NOW));
let front = of(&findings, "frontdoor");
assert_eq!(front.len(), 1, "{findings:#?}");
assert_eq!(front[0].severity, Severity::Attention);
assert!(front[0].summary.contains("waiting on you"));
let _ = std::fs::remove_dir_all(home.join("requests"));
request(
&home,
2,
crate::frontdoor::AWAITING_ME,
"2026-08-11T13:00:00Z",
);
let findings = examine(&home, utc(NOW));
assert!(of(&findings, "frontdoor").is_empty(), "{findings:#?}");
let _ = std::fs::remove_dir_all(home.join("requests"));
request(
&home,
3,
crate::frontdoor::NEEDS_INFO,
"2026-08-01T00:00:00Z",
);
let findings = examine(&home, utc(NOW));
assert!(of(&findings, "frontdoor").is_empty(), "{findings:#?}");
let _ = std::fs::remove_dir_all(&home);
}
#[test]
fn a_triaged_request_nothing_will_revisit_goes_stale() {
let home = home("frontdoor-triaged");
request(&home, 4, crate::frontdoor::TRIAGED, "2026-08-11T11:00:00Z");
request(
&home,
5,
crate::frontdoor::NEEDS_INFO,
"2026-08-01T00:00:00Z",
);
let findings = examine(&home, utc(NOW));
let front = of(&findings, "frontdoor");
assert_eq!(front.len(), 1, "{findings:#?}");
assert_eq!(front[0].severity, Severity::Attention);
assert!(front[0].detail.contains("triaged"), "{}", front[0].detail);
assert!(
!front[0].detail.contains("needs_info"),
"needs_info waits on the requester: {}",
front[0].detail
);
let _ = std::fs::remove_dir_all(&home);
}
#[test]
fn a_trigger_whose_last_run_failed_is_flagged_with_the_manual_probe() {
let home = home("trigger-failed");
trigger_file(&home, "morning", "");
ledger_row(
&home,
&json!({
"trigger": "morning",
"slot": "2026-08-13T07:00:00Z",
"started_at": "2026-08-13T07:00:01Z",
"status": "ok",
"summary": "fine",
}),
);
ledger_row(
&home,
&json!({
"trigger": "morning",
"slot": "2026-08-14T07:00:00Z",
"started_at": "2026-08-14T07:00:01Z",
"status": "error",
"error": "provider unreachable",
}),
);
let findings = examine(&home, utc(NOW));
let triggers = of(&findings, "triggers");
assert_eq!(triggers.len(), 1, "{findings:#?}");
assert_eq!(triggers[0].severity, Severity::Attention);
assert!(triggers[0].summary.contains("morning"));
assert!(triggers[0].detail.contains("provider unreachable"));
assert_eq!(
triggers[0].remedy.as_ref().unwrap().argv,
vec!["mecha", "trigger", "run", "morning"],
"a manual run is the safe probe: it never advances the schedule"
);
let _ = std::fs::remove_dir_all(&home);
}
#[test]
fn a_skip_row_after_a_failed_run_does_not_hide_the_failure() {
let home = home("trigger-skip-hides-error");
trigger_file(&home, "morning", "");
ledger_row(
&home,
&json!({
"trigger": "morning",
"slot": "2026-08-13T07:00:00Z",
"started_at": "2026-08-13T07:00:01Z",
"status": "error",
"error": "provider unreachable",
}),
);
ledger_row(
&home,
&json!({
"trigger": "morning",
"slot": "2026-08-14T07:00:00Z",
"started_at": "2026-08-14T07:00:01Z",
"status": "skipped-stale",
}),
);
let findings = examine(&home, utc(NOW));
let triggers = of(&findings, "triggers");
assert_eq!(triggers.len(), 1, "{findings:#?}");
assert!(
triggers[0].summary.contains("most recent run failed"),
"{}",
triggers[0].summary
);
assert!(triggers[0].detail.contains("provider unreachable"));
let _ = std::fs::remove_dir_all(&home);
}
#[test]
fn an_ok_run_followed_by_a_skip_is_healthy() {
let home = home("trigger-ok-then-skip");
trigger_file(&home, "morning", "");
ledger_row(
&home,
&json!({
"trigger": "morning",
"slot": "2026-08-13T07:00:00Z",
"started_at": "2026-08-13T07:00:01Z",
"status": "ok",
}),
);
ledger_row(
&home,
&json!({
"trigger": "morning",
"slot": "2026-08-14T07:00:00Z",
"started_at": "2026-08-14T07:00:01Z",
"status": "skipped-overlap",
}),
);
let findings = examine(&home, utc(NOW));
assert!(of(&findings, "triggers").is_empty(), "{findings:#?}");
let _ = std::fs::remove_dir_all(&home);
}
#[test]
fn a_trigger_quietly_failing_a_third_of_its_calls_is_reported() {
let home = home("trigger-tool-errors");
trigger_file(&home, "morning", "");
for day in 10..15 {
ledger_row(
&home,
&json!({
"trigger": "morning",
"slot": format!("2026-08-{day}T07:00:00Z"),
"started_at": format!("2026-08-{day}T07:00:01Z"),
"status": "ok",
"summary": "briefed",
"tool_calls": 6,
"tool_errors": 3,
}),
);
}
let findings = examine(&home, utc(NOW));
let triggers = of(&findings, "triggers");
assert_eq!(triggers.len(), 1, "{findings:#?}");
assert_eq!(triggers[0].severity, Severity::Attention);
assert!(
triggers[0].summary.contains("15 of 30"),
"{}",
triggers[0].summary
);
assert_eq!(
triggers[0].remedy.as_ref().unwrap().argv,
vec!["mecha", "trigger", "show", "morning"],
"reading is the remedy — what to change is in the transcript"
);
let _ = std::fs::remove_dir_all(&home);
}
#[test]
fn a_handful_of_failed_calls_is_not_a_trend() {
let home = home("trigger-tool-errors-quiet");
trigger_file(&home, "morning", "");
ledger_row(
&home,
&json!({
"trigger": "morning",
"slot": "2026-08-14T07:00:00Z",
"started_at": "2026-08-14T07:00:01Z",
"status": "ok",
"tool_calls": 3,
"tool_errors": 3,
}),
);
assert!(of(&examine(&home, utc(NOW)), "triggers").is_empty());
ledger_row(
&home,
&json!({
"trigger": "morning",
"slot": "2026-08-15T07:00:00Z",
"started_at": "2026-08-15T07:00:01Z",
"status": "ok",
"tool_calls": 40,
"tool_errors": 4,
}),
);
assert!(of(&examine(&home, utc(NOW)), "triggers").is_empty());
let _ = std::fs::remove_dir_all(&home);
}
#[test]
fn a_trigger_that_stopped_doing_anything_is_reported() {
let home = home("trigger-stopped-working");
trigger_file(&home, "morning", "");
for day in 10..14 {
ledger_row(
&home,
&json!({
"trigger": "morning",
"slot": format!("2026-08-{day}T07:00:00Z"),
"started_at": format!("2026-08-{day}T07:00:01Z"),
"status": "ok",
"tool_calls": 8,
"tool_errors": 0,
}),
);
}
ledger_row(
&home,
&json!({
"trigger": "morning",
"slot": "2026-08-14T07:00:00Z",
"started_at": "2026-08-14T07:00:01Z",
"status": "ok",
"summary": "nothing to report",
"tool_calls": 0,
"tool_errors": 0,
}),
);
let findings = examine(&home, utc(NOW));
let triggers = of(&findings, "triggers");
assert_eq!(triggers.len(), 1, "{findings:#?}");
assert!(
triggers[0].summary.contains("did no work"),
"{}",
triggers[0].summary
);
assert!(triggers[0].detail.contains("made 32"));
let _ = std::fs::remove_dir_all(&home);
}
#[test]
fn a_trigger_that_never_needed_tools_is_not_broken_for_not_using_them() {
let home = home("trigger-never-used-tools");
trigger_file(&home, "haiku", "");
for day in 10..15 {
ledger_row(
&home,
&json!({
"trigger": "haiku",
"slot": format!("2026-08-{day}T07:00:00Z"),
"started_at": format!("2026-08-{day}T07:00:01Z"),
"status": "ok",
"tool_calls": 0,
"tool_errors": 0,
}),
);
}
assert!(of(&examine(&home, utc(NOW)), "triggers").is_empty());
let _ = std::fs::remove_dir_all(&home);
}
#[test]
fn a_failed_run_that_did_no_work_is_reported_once_not_twice() {
let home = home("trigger-failed-no-work");
trigger_file(&home, "morning", "");
for day in 10..14 {
ledger_row(
&home,
&json!({
"trigger": "morning",
"slot": format!("2026-08-{day}T07:00:00Z"),
"started_at": format!("2026-08-{day}T07:00:01Z"),
"status": "ok",
"tool_calls": 8,
"tool_errors": 0,
}),
);
}
ledger_row(
&home,
&json!({
"trigger": "morning",
"slot": "2026-08-14T07:00:00Z",
"started_at": "2026-08-14T07:00:01Z",
"status": "error",
"error": "provider unreachable",
"tool_calls": 0,
"tool_errors": 0,
}),
);
let triggers = of(&examine(&home, utc(NOW)), "triggers")
.into_iter()
.cloned()
.collect::<Vec<_>>();
assert_eq!(triggers.len(), 1, "{triggers:#?}");
assert!(triggers[0].detail.contains("provider unreachable"));
let _ = std::fs::remove_dir_all(&home);
}
#[test]
fn an_unreadable_transcript_is_a_finding_not_an_empty_queue() {
let home = home("runs-unreadable");
let dir = home.join("sessions");
std::fs::create_dir_all(&dir).unwrap();
std::fs::write(dir.join("20260828T000000-torn.jsonl"), "not json\n").unwrap();
let all = examine(&home, utc(NOW));
let findings = of(&all, "runs");
assert_eq!(findings.len(), 1, "{findings:#?}");
assert!(
findings[0].summary.contains("unreadable") && findings[0].summary.contains('1'),
"{}",
findings[0].summary
);
let _ = std::fs::remove_dir_all(&home);
}
fn runs_in(
home: &Path,
model: &str,
n: usize,
stats: impl Fn(usize) -> crate::session::RunStats,
) {
let dir = home.join("sessions");
std::fs::create_dir_all(&dir).unwrap();
for i in 0..n {
let session = crate::session::Session::create(
&dir,
crate::session::SessionMeta {
id: format!("2026080{}T00000{i:03}-{model}", 1 + i % 9),
created_at: utc(NOW),
provider: "local".into(),
model: model.to_string(),
workspace: std::path::PathBuf::from("/tmp"),
title: None,
},
)
.unwrap();
session
.append(&crate::session::Record::Outcome(stats(i)))
.unwrap();
}
}
fn run_stats(
calls: u32,
errors: u32,
ended_failed: bool,
cause: crate::agent::StopCause,
) -> crate::session::RunStats {
crate::session::RunStats {
tool_calls: calls,
tool_errors: errors,
ended_on_failed_call: ended_failed,
stop_cause: Some(cause),
..Default::default()
}
}
#[test]
fn a_model_that_keeps_finishing_over_failures_is_reported() {
use crate::agent::StopCause;
let home = home("runs-ended-on-failure");
runs_in(&home, "tiny-local", 30, |i| {
run_stats(6, 0, i % 3 == 0, StopCause::Completed)
});
let all = examine(&home, utc(NOW));
let findings = of(&all, "runs");
assert_eq!(findings.len(), 1, "{findings:#?}");
assert!(
findings[0].summary.contains("tiny-local"),
"{}",
findings[0].summary
);
assert!(
findings[0].summary.contains("33%"),
"{}",
findings[0].summary
);
assert_eq!(
findings[0].remedy.as_ref().unwrap().argv,
vec!["mecha", "sessions", "health", "--days", "30"],
"reading is the remedy; doctor never decides what to change"
);
let _ = std::fs::remove_dir_all(&home);
}
#[test]
fn a_cancelled_run_is_not_the_harness_cutting_it_short() {
use crate::agent::StopCause;
let home = home("runs-interrupted");
runs_in(&home, "tiny-local", 30, |_| {
run_stats(6, 0, false, StopCause::Interrupted)
});
let findings = examine(&home, utc(NOW));
assert!(of(&findings, "runs").is_empty());
let _ = std::fs::remove_dir_all(&home);
}
#[test]
fn a_turn_ceiling_stopping_a_quarter_of_runs_is_a_finding() {
use crate::agent::StopCause;
let home = home("runs-max-turns");
runs_in(&home, "tiny-local", 30, |_| {
run_stats(6, 0, false, StopCause::MaxTurns)
});
let all = examine(&home, utc(NOW));
let findings = of(&all, "runs");
assert_eq!(findings.len(), 1, "{findings:#?}");
assert!(
findings[0].summary.contains("cut"),
"{}",
findings[0].summary
);
let _ = std::fs::remove_dir_all(&home);
}
#[test]
fn a_thin_sample_of_one_model_says_nothing_about_it() {
use crate::agent::StopCause;
let home = home("runs-thin");
runs_in(&home, "tiny-local", 19, |_| {
run_stats(6, 6, true, StopCause::MaxTurns)
});
let all = examine(&home, utc(NOW));
assert!(of(&all, "runs").is_empty());
let _ = std::fs::remove_dir_all(&home);
}
#[test]
fn a_bad_model_does_not_drag_a_good_one_into_a_finding() {
use crate::agent::StopCause;
let home = home("runs-two-models");
runs_in(&home, "steady", 25, |_| {
run_stats(10, 0, false, StopCause::Completed)
});
runs_in(&home, "flaky", 25, |_| {
run_stats(10, 9, false, StopCause::Completed)
});
let all = examine(&home, utc(NOW));
let findings = of(&all, "runs");
assert_eq!(findings.len(), 1, "{findings:#?}");
assert!(
findings[0].summary.contains("flaky"),
"{}",
findings[0].summary
);
assert!(
!findings[0].summary.contains("steady"),
"the healthy model was named in a finding about the other one"
);
let _ = std::fs::remove_dir_all(&home);
}
#[test]
fn a_ledger_written_before_the_counts_existed_reports_nothing() {
let home = home("trigger-tool-errors-bare");
trigger_file(&home, "morning", "");
ledger_row(
&home,
&json!({
"trigger": "morning",
"slot": "2026-08-14T07:00:00Z",
"started_at": "2026-08-14T07:00:01Z",
"status": "ok",
}),
);
assert!(of(&examine(&home, utc(NOW)), "triggers").is_empty());
let _ = std::fs::remove_dir_all(&home);
}
#[test]
fn a_disabled_trigger_is_nobody_s_emergency() {
let home = home("trigger-disabled");
trigger_file(&home, "morning", "enabled = false\n");
ledger_row(
&home,
&json!({
"trigger": "morning",
"started_at": "2026-08-14T07:00:01Z",
"status": "error",
"error": "boom",
}),
);
let findings = examine(&home, utc(NOW));
assert!(of(&findings, "triggers").is_empty(), "{findings:#?}");
let _ = std::fs::remove_dir_all(&home);
}
#[test]
fn a_catch_up_trigger_whose_slots_stopped_advancing_names_the_daemon() {
let home = home("trigger-stale");
trigger_file(&home, "morning", "");
ledger_row(
&home,
&json!({
"trigger": "morning",
"slot": "2026-08-09T07:00:00Z",
"started_at": "2026-08-09T07:00:01Z",
"status": "ok",
}),
);
let findings = examine(&home, utc(NOW));
let triggers = of(&findings, "triggers");
assert_eq!(triggers.len(), 1, "{findings:#?}");
assert_eq!(triggers[0].severity, Severity::Attention);
assert!(triggers[0].summary.contains("missed more than two slots"));
assert!(
triggers[0].detail.contains("daemon"),
"{}",
triggers[0].detail
);
assert!(
triggers[0].remedy.is_none(),
"running the trigger would not restart the scheduler"
);
ledger_row(
&home,
&json!({
"trigger": "morning",
"slot": "2026-08-14T07:00:00Z",
"started_at": "2026-08-14T07:00:01Z",
"status": "ok",
}),
);
let findings = examine(&home, utc(NOW));
assert!(of(&findings, "triggers").is_empty(), "{findings:#?}");
let _ = std::fs::remove_dir_all(&home);
}
#[cfg(unix)]
#[test]
fn one_poisoned_store_does_not_suppress_the_others() {
use std::os::unix::fs::PermissionsExt;
if unsafe { libc::geteuid() } == 0 {
return;
}
let home = home("poisoned");
write_marker(&home, "personal", &valid_marker());
let outbox = home.join("outbox");
std::fs::create_dir_all(&outbox).unwrap();
std::fs::set_permissions(&outbox, std::fs::Permissions::from_mode(0o000)).unwrap();
let findings = examine(&home, utc(NOW));
std::fs::set_permissions(&outbox, std::fs::Permissions::from_mode(0o700)).unwrap();
let mail = of(&findings, "mail");
assert_eq!(mail.len(), 1, "the mail finding survived: {findings:#?}");
assert_eq!(mail[0].severity, Severity::Broken);
let broken_store = of(&findings, "outbox");
assert_eq!(broken_store.len(), 1, "{findings:#?}");
assert!(
broken_store[0].summary.starts_with("store unreadable:"),
"{}",
broken_store[0].summary
);
let _ = std::fs::remove_dir_all(&home);
}
#[test]
fn the_golden_marker_literal_parses_into_the_dead_auth_finding() {
const GOLDEN: &str = r#"{
"at": "2026-08-11T09:00:00Z",
"message": "account `personal`: refresh token expired or revoked — run `mecha-mail auth personal --provider google` (invalid_grant: Token has been revoked.)"
}"#;
let home = home("golden-marker");
write_marker(&home, "personal", GOLDEN);
let findings = examine(&home, utc(NOW));
let mail = of(&findings, "mail");
assert_eq!(mail.len(), 1, "{findings:#?}");
assert_eq!(mail[0].severity, Severity::Broken);
assert!(
mail[0].detail.contains("since 2026-08-11T09:00:00Z"),
"the marker's `at` must reach the detail: {}",
mail[0].detail
);
assert!(
mail[0]
.detail
.contains("run `mecha-mail auth personal --provider google`"),
"the marker's `message` must reach the detail: {}",
mail[0].detail
);
let _ = std::fs::remove_dir_all(&home);
}
#[test]
fn findings_sort_broken_first() {
let mut findings = vec![
Finding {
component: "outbox".into(),
severity: Severity::Attention,
summary: "stale".into(),
detail: String::new(),
remedy: None,
},
Finding {
component: "mail".into(),
severity: Severity::Broken,
summary: "dead".into(),
detail: String::new(),
remedy: None,
},
];
sort(&mut findings);
assert_eq!(findings[0].severity, Severity::Broken);
}
#[test]
fn an_empty_home_is_healthy() {
let home = home("empty");
assert!(examine(&home, utc(NOW)).is_empty());
let _ = std::fs::remove_dir_all(&home);
}
fn graph_store(name: &str) -> PathBuf {
let store = home(name).join(".mecha-graph");
std::fs::create_dir_all(store.join("logs")).unwrap();
store
}
fn nightly_log(store: &Path, file: &str) {
std::fs::write(store.join("logs").join(file), "ran\n").unwrap();
}
#[test]
fn a_graph_nightly_that_stopped_writing_logs_is_a_finding() {
let store = graph_store("graph-stale");
nightly_log(&store, "nightly-20260812.log");
let findings = check_graph_nightly(&store, utc(NOW));
assert_eq!(findings.len(), 1);
assert_eq!(findings[0].component, "graph");
assert_eq!(findings[0].severity, Severity::Attention);
assert!(
findings[0].summary.contains("2 days"),
"{}",
findings[0].summary
);
assert!(
findings[0].detail.contains("nightly-20260812.log"),
"{}",
findings[0].detail
);
}
#[test]
fn yesterdays_log_is_healthy_because_todays_slot_may_not_have_fired() {
let store = graph_store("graph-yesterday");
nightly_log(&store, "nightly-20260813.log");
nightly_log(&store, "mecha-nightly-20260813.log");
assert!(check_graph_nightly(&store, utc(NOW)).is_empty());
}
#[test]
fn each_nightly_family_is_judged_alone() {
let store = graph_store("graph-split");
nightly_log(&store, "nightly-20260814.log");
nightly_log(&store, "mecha-nightly-20260811.log");
let findings = check_graph_nightly(&store, utc(NOW));
assert_eq!(findings.len(), 1);
assert!(
findings[0].summary.contains("mecha-nightly"),
"{}",
findings[0].summary
);
}
#[test]
fn the_shorter_prefix_does_not_claim_the_longer_familys_logs() {
let store = graph_store("graph-prefix");
nightly_log(&store, "mecha-nightly-20260814.log");
nightly_log(&store, "nightly-20260810.log");
let findings = check_graph_nightly(&store, utc(NOW));
assert_eq!(findings.len(), 1);
assert!(
findings[0].detail.contains("nightly-20260810.log"),
"{}",
findings[0].detail
);
}
#[test]
fn a_graph_that_never_ran_is_not_a_finding() {
let missing = home("graph-missing").join(".mecha-graph");
assert!(check_graph_nightly(&missing, utc(NOW)).is_empty());
let empty = graph_store("graph-empty");
assert!(check_graph_nightly(&empty, utc(NOW)).is_empty());
let odd = graph_store("graph-odd-names");
nightly_log(&odd, "nightly-garbage.log");
nightly_log(&odd, "gossip-20260812.jsonl");
assert!(check_graph_nightly(&odd, utc(NOW)).is_empty());
}
#[test]
fn examine_reads_the_graph_store_beside_the_home() {
let scratch = home("graph-sibling");
let mecha_home = scratch.join(".mecha");
std::fs::create_dir_all(&mecha_home).unwrap();
let store = scratch.join(".mecha-graph");
std::fs::create_dir_all(store.join("logs")).unwrap();
nightly_log(&store, "nightly-20260810.log");
let findings = examine(&mecha_home, utc(NOW));
assert_eq!(findings.len(), 1);
assert_eq!(findings[0].component, "graph");
let _ = std::fs::remove_dir_all(&scratch);
}
#[test]
fn a_malformed_charter_is_broken_and_names_the_remedy() {
let home = home("charter-broken");
std::fs::write(
home.join("charter.toml"),
"[[line]]\nid = \"a\"\ntext = \"one\"\n[[line]]\nid = \"a\"\ntext = \"two\"\n",
)
.unwrap();
let findings = check_charter(&home.join("charter.toml"));
assert_eq!(findings.len(), 1, "{findings:#?}");
assert_eq!(findings[0].severity, Severity::Broken);
assert!(
findings[0].detail.contains("used more than once"),
"{}",
findings[0].detail
);
assert_eq!(
findings[0].remedy.as_ref().unwrap().argv,
vec!["mecha", "charter"]
);
let _ = std::fs::remove_dir_all(&home);
}
#[test]
fn a_charter_over_budget_is_attention_not_broken_and_still_named_loaded() {
let home = home("charter-over-budget");
let long = "x".repeat(3000);
std::fs::write(
home.join("charter.toml"),
format!("[[line]]\nid = \"only\"\ntext = \"{long}\"\n"),
)
.unwrap();
let findings = check_charter(&home.join("charter.toml"));
assert_eq!(findings.len(), 1, "{findings:#?}");
assert_eq!(findings[0].severity, Severity::Attention);
assert!(
findings[0].summary.contains("budget"),
"{}",
findings[0].summary
);
let _ = std::fs::remove_dir_all(&home);
}
#[test]
fn a_healthy_charter_and_a_missing_one_are_both_silent() {
let home = home("charter-healthy");
assert!(
check_charter(&home.join("charter.toml")).is_empty(),
"no file at all"
);
std::fs::write(
home.join("charter.toml"),
"[[line]]\nid = \"a\"\ntext = \"protect the owner\"\n",
)
.unwrap();
assert!(check_charter(&home.join("charter.toml")).is_empty());
let _ = std::fs::remove_dir_all(&home);
}
#[test]
fn a_genuinely_empty_charter_file_is_flagged_not_silent() {
let home = home("charter-empty-comment");
std::fs::write(home.join("charter.toml"), "# no priorities written yet\n").unwrap();
let findings = check_charter(&home.join("charter.toml"));
assert_eq!(findings.len(), 1, "{findings:#?}");
assert_eq!(findings[0].severity, Severity::Attention);
assert!(
findings[0].summary.contains("no lines"),
"{}",
findings[0].summary
);
let _ = std::fs::remove_dir_all(&home);
}
#[test]
fn a_directory_at_the_charter_path_is_broken_not_silently_absent() {
let home = home("charter-is-a-directory");
std::fs::create_dir_all(home.join("charter.toml")).unwrap();
let findings = check_charter(&home.join("charter.toml"));
assert_eq!(findings.len(), 1, "{findings:#?}");
assert_eq!(findings[0].severity, Severity::Broken);
let _ = std::fs::remove_dir_all(&home);
}
#[test]
fn a_typo_d_table_name_beside_a_real_line_is_broken_not_silently_short() {
let home = home("charter-typo-table");
std::fs::write(
home.join("charter.toml"),
"[[line]]\nid = \"a\"\ntext = \"one\"\n\n[[lines]]\nid = \"b\"\ntext = \"two\"\n",
)
.unwrap();
let findings = check_charter(&home.join("charter.toml"));
assert_eq!(findings.len(), 1, "{findings:#?}");
assert_eq!(findings[0].severity, Severity::Broken);
let _ = std::fs::remove_dir_all(&home);
}
}
#[cfg(test)]
mod proposal_review_tests {
use super::*;
fn store_at(name: &str) -> std::path::PathBuf {
let dir = std::env::temp_dir()
.join("mecha-doctor-proposals")
.join(format!("{}-{name}", std::process::id()));
let _ = std::fs::remove_dir_all(&dir);
std::fs::create_dir_all(dir.join("proposals")).unwrap();
dir
}
fn write_proposal(root: &Path, id: &str, created: &str, before: &[&str], reflexions: usize) {
let p = serde_json::json!({
"id": id,
"domain": "behavior",
"status": "pending",
"reflexion_ids": (0..reflexions).map(|i| format!("r-{i}")).collect::<Vec<_>>(),
"rules_before": before.iter().map(|t| serde_json::json!({"text": t})).collect::<Vec<_>>(),
"rules": [{"text": "a new rule"}],
"evidence": "e",
"created_at": created,
"resolved_at": null,
"reason": null,
});
std::fs::write(
root.join("proposals").join(format!("{id}.json")),
serde_json::to_string(&p).unwrap(),
)
.unwrap();
}
fn now() -> DateTime<Utc> {
DateTime::parse_from_rfc3339("2026-08-29T12:00:00Z")
.unwrap()
.with_timezone(&Utc)
}
#[test]
fn the_stale_predicate_matches_accepts() {
let r = |text: &str, enabled: bool| crate::learning::Rule {
text: text.into(),
enabled,
..Default::default()
};
let base = vec![r("alpha", true), r("beta", true)];
assert!(same_rules_as_accept(&base, &base.clone()));
assert!(!same_rules_as_accept(
&base,
&[r("beta", true), r("alpha", true)]
));
assert!(!same_rules_as_accept(
&base,
&[r("alpha", true), r("beta", false)]
));
assert!(!same_rules_as_accept(&base, &[r("alpha", true)]));
}
#[test]
fn checking_an_absent_store_creates_nothing() {
let root =
std::env::temp_dir().join(format!("mecha-doctor-nostore-{}", std::process::id()));
let _ = std::fs::remove_dir_all(&root);
assert!(read_learned_rules(&root, "behavior").is_some_and(|r| r.is_empty()));
assert!(
!root.exists(),
"reading rules must not create the store directory"
);
}
#[test]
fn an_unreviewed_queue_is_reported_with_what_it_is_holding() {
let root = store_at("stale");
write_proposal(&root, "p-old", "2026-08-23T12:00:00Z", &[], 10);
let out = check_proposal_review(&root, now());
assert_eq!(out.len(), 1, "one latency finding: {out:?}");
assert!(out[0].summary.contains("6 day(s)"), "{}", out[0].summary);
assert!(
out[0].summary.contains("holding 10 reflection(s)"),
"the cost is the held evidence, not the count of proposals: {}",
out[0].summary
);
}
#[test]
fn a_fresh_proposal_is_not_a_finding() {
let root = store_at("fresh");
write_proposal(&root, "p-new", "2026-08-29T03:30:00Z", &[], 4);
assert!(check_proposal_review(&root, now()).is_empty());
}
#[test]
fn an_unappliable_proposal_is_named_with_the_verb_that_frees_it() {
let root = store_at("unappliable");
write_proposal(&root, "p-stale", "2026-08-29T03:30:00Z", &["was live"], 7);
let out = check_proposal_review(&root, now());
assert_eq!(out.len(), 1, "{out:?}");
assert!(out[0].summary.contains("can no longer be applied"));
assert!(out[0].detail.contains("p-stale"));
let argv = &out[0].remedy.as_ref().unwrap().argv;
assert!(argv.contains(&"supersede".to_string()), "{argv:?}");
assert!(!argv.contains(&"reject".to_string()), "{argv:?}");
}
#[test]
fn a_store_that_has_never_staged_one_is_silent() {
let dir = std::env::temp_dir().join("mecha-doctor-proposals-absent");
let _ = std::fs::remove_dir_all(&dir);
assert!(check_proposal_review(&dir, now()).is_empty());
}
}