use crate::cli::CliOutput;
use crate::db;
use crate::models::field_names;
use anyhow::{Context, Result};
use serde::Serialize;
use serde_json::Value;
use std::path::Path;
use std::time::Duration;
const FACT_DIM_VIOLATIONS: &str = "dim_violations";
const FACT_MAX_SKEW_SECS: &str = "max_skew_secs";
const FACT_RECALL_MODE_ACTIVE: &str = "recall_mode_active";
const FACT_RERANKER_ACTIVE: &str = "reranker_active";
const SECTION_LLM_REACHABILITY: &str = "LLM Reachability (#1146)";
const SECTION_EMBEDDINGS_REACHABILITY: &str = "Embeddings Reachability (#1598)";
const MSG_RAW_SQL_DB_MODE: &str = "raw SQL section — only available in --db mode";
const MSG_HTTP_CLIENT_BUILD_FAILED: &str = "http client build failed";
const NOT_IN_RESPONSE: &str = "not_in_response";
const NOT_OBSERVED_PRE_P3: &str = "not_observed (pre-P3 rolling counter)";
#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize)]
#[serde(rename_all = "lowercase")]
pub enum Severity {
Info,
Warning,
Critical,
NotAvailable,
}
impl Severity {
fn label(self) -> &'static str {
match self {
Severity::Info => "INFO",
Severity::Warning => "WARN",
Severity::Critical => "CRIT",
Severity::NotAvailable => "N/A ",
}
}
}
#[derive(Debug, Serialize)]
pub struct ReportSection {
pub name: String,
pub severity: Severity,
pub facts: Vec<(String, String)>,
#[serde(skip_serializing_if = "Option::is_none")]
pub note: Option<String>,
}
#[derive(Debug, Serialize)]
pub struct Report {
pub mode: String,
pub source: String,
pub generated_at: String,
pub sections: Vec<ReportSection>,
pub overall: Severity,
}
impl Report {
fn rank(s: Severity) -> u8 {
match s {
Severity::NotAvailable => 0,
Severity::Info => 1,
Severity::Warning => 2,
Severity::Critical => 3,
}
}
fn compute_overall(&mut self) {
self.overall = self
.sections
.iter()
.map(|s| s.severity)
.max_by_key(|s| Self::rank(*s))
.unwrap_or(Severity::Info);
}
}
pub struct DoctorArgs {
pub remote: Option<String>,
pub json: bool,
pub fail_on_warn: bool,
}
#[derive(Debug, Default)]
pub struct TokensArgs {
pub json: bool,
pub raw_table: bool,
pub profile: Option<String>,
pub hooks: bool,
}
#[derive(Debug, Default)]
pub struct HooksReportArgs {
pub json: bool,
}
pub fn run_tokens(args: TokensArgs, out: &mut CliOutput<'_>) -> Result<i32> {
use crate::profile::{Family, Profile};
use crate::sizes;
let profile = match Profile::parse(args.profile.as_deref().unwrap_or("core")) {
Ok(p) => p,
Err(e) => {
writeln!(out.stderr, "ai-memory doctor --tokens: {e}")?;
return Ok(2);
}
};
let table = sizes::tool_sizes();
let trimmed_table = sizes::trimmed_tool_sizes();
let full_total: usize = table.iter().map(|t| t.total_tokens).sum();
let active_total: usize = table
.iter()
.filter(|t| profile.loads(&t.name))
.map(|t| t.total_tokens)
.sum();
let trimmed_full_total: usize = trimmed_table.iter().map(|t| t.total_tokens).sum();
let trimmed_active_total: usize = trimmed_table
.iter()
.filter(|t| profile.loads(&t.name))
.map(|t| t.total_tokens)
.sum();
let savings = full_total.saturating_sub(active_total);
let pct = if full_total == 0 {
0.0
} else {
(f64::from(u32::try_from(savings).unwrap_or(u32::MAX))
/ f64::from(u32::try_from(full_total).unwrap_or(u32::MAX)))
* 100.0
};
let mut family_totals: Vec<(String, usize, usize)> = Family::all()
.iter()
.map(|f| {
let mut tool_count = 0usize;
let mut sum = 0usize;
for entry in table {
if Family::for_tool(&entry.name) == Some(*f) {
tool_count += 1;
sum += entry.total_tokens;
}
}
(f.name().to_string(), tool_count, sum)
})
.collect();
family_totals.sort_by_key(|(_, _, sum)| std::cmp::Reverse(*sum));
if args.json || args.raw_table {
let payload = serde_json::json!({
(field_names::SCHEMA_VERSION): "v0.6.4-tokens-1",
"tokenizer": "cl100k_base",
"active_profile": profile.families().iter().map(|f| f.name()).collect::<Vec<_>>(),
"active_total_tokens": active_total,
"full_profile_total_tokens": full_total,
"trimmed_active_total_tokens": trimmed_active_total,
"trimmed_full_profile_total_tokens": trimmed_full_total,
"savings_tokens": savings,
"savings_pct": format!("{pct:.1}"),
"families": family_totals.iter().map(|(name, count, sum)| {
let fam = Family::all()
.iter()
.find(|f| f.name() == name)
.copied()
.unwrap_or(Family::Other);
serde_json::json!({
"name": name,
"tool_count": count,
"tokens": sum,
"loaded": profile.includes(fam),
})
}).collect::<Vec<_>>(),
"tools": if args.raw_table {
serde_json::Value::Array(
table.iter().map(|t| serde_json::json!({
"name": t.name,
"tokens": t.total_tokens,
"family": Family::for_tool(&t.name).map(|f| f.name()),
"loaded_under_active_profile": profile.loads(&t.name),
})).collect()
)
} else {
serde_json::Value::Null
},
});
writeln!(out.stdout, "{}", serde_json::to_string_pretty(&payload)?)?;
return Ok(0);
}
writeln!(out.stdout, "ai-memory doctor --tokens")?;
writeln!(
out.stdout,
" Tokenizer: cl100k_base (Claude / GPT input accounting)"
)?;
writeln!(
out.stdout,
" Active profile: {}",
profile
.families()
.iter()
.map(|f| f.name())
.collect::<Vec<_>>()
.join(",")
)?;
writeln!(out.stdout)?;
writeln!(out.stdout, " Tool surface cost (verbose schema, ceiling):")?;
writeln!(
out.stdout,
" Active ({:>2} tools loaded): {:>6} tokens",
table.iter().filter(|t| profile.loads(&t.name)).count(),
active_total
)?;
writeln!(
out.stdout,
" Full ({:>2} tools loaded): {:>6} tokens",
table.len(),
full_total
)?;
writeln!(
out.stdout,
" Savings vs full: {:>6} tokens ({pct:.1}%)",
savings
)?;
writeln!(out.stdout)?;
writeln!(
out.stdout,
" Tools/list payload (v0.7 C4 + #859 trim — properties exposed, prose stripped):"
)?;
writeln!(
out.stdout,
" Active {:>6} tokens",
trimmed_active_total
)?;
writeln!(
out.stdout,
" Full {:>6} tokens",
trimmed_full_total
)?;
writeln!(out.stdout)?;
writeln!(out.stdout, " Per-family breakdown (sorted by total cost):")?;
for (name, count, sum) in &family_totals {
writeln!(
out.stdout,
" {name:<12} {count:>2} tools {sum:>6} tokens",
)?;
}
if args.hooks {
writeln!(out.stdout)?;
render_hooks_human(out)?;
}
Ok(0)
}
pub fn run_hooks(args: HooksReportArgs, out: &mut CliOutput<'_>) -> Result<i32> {
use crate::hooks::config::HookConfig;
let path_opt = HookConfig::default_path();
let hooks: Vec<HookConfig> = match path_opt.as_ref() {
Some(p) if p.exists() => match HookConfig::load_from_file(p) {
Ok(h) => h,
Err(e) => {
writeln!(out.stderr, "ai-memory doctor --hooks: {e}")?;
return Ok(2);
}
},
_ => Vec::new(),
};
if args.json {
let payload = serde_json::json!({
(field_names::SCHEMA_VERSION): "v0.7-hooks-1",
"config_path": path_opt.as_ref().map(|p| p.display().to_string()),
"hooks_loaded": hooks.len(),
"executors": hooks.iter().map(|h| serde_json::json!({
"event": h.event,
"command": h.command.display().to_string(),
"mode": h.mode,
"namespace": h.namespace,
"priority": h.priority,
"timeout_ms": h.timeout_ms,
"enabled": h.enabled,
"metrics": {
"events_fired": 0,
"events_dropped": 0,
"mean_latency_us": 0,
},
})).collect::<Vec<_>>(),
"timeout_violations": crate::hooks::timeouts::timeout_violations_total(),
"note": "metrics placeholders until G7-G11 wires the executor into the daemon",
});
writeln!(out.stdout, "{}", serde_json::to_string_pretty(&payload)?)?;
return Ok(0);
}
render_hooks_human_with(out, path_opt.as_deref(), &hooks)?;
Ok(0)
}
fn render_hooks_human(out: &mut CliOutput<'_>) -> Result<()> {
use crate::hooks::config::HookConfig;
let path_opt = HookConfig::default_path();
let hooks: Vec<HookConfig> = match path_opt.as_ref() {
Some(p) if p.exists() => HookConfig::load_from_file(p).unwrap_or_default(),
_ => Vec::new(),
};
render_hooks_human_with(out, path_opt.as_deref(), &hooks)
}
fn render_hooks_human_with(
out: &mut CliOutput<'_>,
path: Option<&Path>,
hooks: &[crate::hooks::config::HookConfig],
) -> Result<()> {
writeln!(out.stdout, "ai-memory doctor --hooks")?;
if let Some(p) = path {
writeln!(out.stdout, " Config path: {}", p.display())?;
}
writeln!(out.stdout, " Hooks loaded: {}", hooks.len())?;
if hooks.is_empty() {
writeln!(
out.stdout,
" (no hooks configured — drop a hooks.toml at the path above to enable)"
)?;
return Ok(());
}
writeln!(out.stdout)?;
writeln!(
out.stdout,
" {:<26} {:<8} {:<22} fired dropped mean_us",
"event", "mode", "command"
)?;
for h in hooks {
let event = format!("{:?}", h.event);
let mode = format!("{:?}", h.mode);
let cmd = h
.command
.file_name()
.map(|s| s.to_string_lossy().into_owned())
.unwrap_or_else(|| h.command.display().to_string());
let cmd_truncated: String = cmd.chars().take(22).collect();
writeln!(
out.stdout,
" {event:<26} {mode:<8} {cmd_truncated:<22} {:>5} {:>7} {:>7}",
0, 0, 0,
)?;
}
writeln!(out.stdout)?;
writeln!(
out.stdout,
" Chain class-deadline violations: {}",
crate::hooks::timeouts::timeout_violations_total()
)?;
writeln!(
out.stdout,
" note: live metrics land when G7-G11 wires the executor into the daemon."
)?;
Ok(())
}
pub fn run(db_path: &Path, args: &DoctorArgs, out: &mut CliOutput<'_>) -> Result<i32> {
let mut report = if let Some(url) = &args.remote {
run_remote(url, db_path)
} else {
run_local(db_path)
};
report.compute_overall();
if args.json {
writeln!(out.stdout, "{}", serde_json::to_string_pretty(&report)?)?;
} else {
render_text(&report, out)?;
}
let code = match report.overall {
Severity::Critical => 2,
Severity::Warning if args.fail_on_warn => 1,
_ => 0,
};
Ok(code)
}
fn run_local(db_path: &Path) -> Report {
let mut sections = Vec::with_capacity(7);
let conn = match db::open(db_path) {
Ok(c) => c,
Err(e) => {
sections.push(ReportSection {
name: "Storage".into(),
severity: Severity::Critical,
facts: vec![("error".into(), e.to_string())],
note: Some(format!(
"could not open database at {} — every other section is N/A",
db_path.display()
)),
});
return Report {
mode: "local".into(),
source: db_path.display().to_string(),
generated_at: chrono::Utc::now().to_rfc3339(),
sections,
overall: Severity::Critical,
};
}
};
sections.push(section_storage(&conn, db_path));
sections.push(section_index(&conn));
sections.push(section_recall_local());
sections.push(section_governance(&conn));
sections.push(section_sync(&conn));
sections.push(section_webhook(&conn));
sections.push(section_capabilities_local());
sections.push(section_reflection_health(&conn));
sections.push(section_llm_reachability_1146());
sections.push(section_embeddings_reachability_1598());
Report {
mode: "local".into(),
source: db_path.display().to_string(),
generated_at: chrono::Utc::now().to_rfc3339(),
sections,
overall: Severity::Info,
}
}
fn section_storage(conn: &rusqlite::Connection, db_path: &Path) -> ReportSection {
let mut facts = Vec::new();
let mut severity = Severity::Info;
let mut note: Option<String> = None;
match db::stats(conn, db_path) {
Ok(stats) => {
facts.push((field_names::TOTAL_MEMORIES.into(), stats.total.to_string()));
facts.push(("expiring_within_1h".into(), stats.expiring_soon.to_string()));
facts.push(("links".into(), stats.links_count.to_string()));
facts.push(("db_size_bytes".into(), stats.db_size_bytes.to_string()));
for tc in &stats.by_tier {
facts.push((format!("tier::{}", tc.tier), tc.count.to_string()));
}
for nc in stats.by_namespace.iter().take(10) {
facts.push((format!("ns::{}", nc.namespace), nc.count.to_string()));
}
}
Err(e) => {
severity = Severity::Warning;
facts.push(("stats_error".into(), e.to_string()));
}
}
match db::doctor_dim_violations(conn) {
Ok(Some(0)) => {
facts.push((FACT_DIM_VIOLATIONS.into(), "0".into()));
}
Ok(Some(n)) => {
facts.push((FACT_DIM_VIOLATIONS.into(), n.to_string()));
severity = Severity::Critical;
note = Some(format!(
"{n} memories have an embedding dim that disagrees with their namespace's modal dim"
));
}
Ok(None) => {
facts.push((
FACT_DIM_VIOLATIONS.into(),
"not_observed (pre-P2 schema)".into(),
));
}
Err(e) => {
facts.push(("dim_violations_error".into(), e.to_string()));
}
}
ReportSection {
name: "Storage".into(),
severity,
facts,
note,
}
}
fn section_index(conn: &rusqlite::Connection) -> ReportSection {
let mut facts = Vec::new();
let mut severity = Severity::Info;
let mut note: Option<String> = None;
let hnsw_size: i64 = conn
.query_row(
"SELECT COUNT(*) FROM memories WHERE embedding IS NOT NULL",
[],
|r| r.get(0),
)
.unwrap_or(0);
facts.push(("hnsw_size_estimate".into(), hnsw_size.to_string()));
let cold_start_secs = (hnsw_size as f64) / 50_000.0;
facts.push((
"cold_start_rebuild_secs_estimate".into(),
format!("{cold_start_secs:.2}"),
));
facts.push((
"index_evictions_total".into(),
"not_observed (pre-P3 surface)".into(),
));
if hnsw_size >= 95_000 {
severity = Severity::Warning;
note = Some(format!(
"HNSW is at {hnsw_size} embeddings, within 5% of the 100k MAX_ENTRIES cap; \
P3 will start emitting eviction events"
));
}
ReportSection {
name: "Index".into(),
severity,
facts,
note,
}
}
fn section_recall_local() -> ReportSection {
ReportSection {
name: "Recall".into(),
severity: Severity::Info,
facts: vec![
(
"recall_mode_distribution".into(),
NOT_OBSERVED_PRE_P3.into(),
),
(
"reranker_used_distribution".into(),
NOT_OBSERVED_PRE_P3.into(),
),
(
"hint".into(),
"use --remote to read the live capabilities endpoint".into(),
),
],
note: None,
}
}
fn section_governance(conn: &rusqlite::Connection) -> ReportSection {
let mut facts = Vec::new();
let mut severity = Severity::Info;
let mut note: Option<String> = None;
let mode = crate::config::active_permissions_mode();
facts.push(("permissions_mode".into(), mode.as_str().to_string()));
let counts = crate::config::permissions_decision_counts();
facts.push(("decisions::enforce".into(), counts.enforce.to_string()));
facts.push(("decisions::advisory".into(), counts.advisory.to_string()));
facts.push(("decisions::off".into(), counts.off.to_string()));
let (with, without) = db::doctor_governance_coverage(conn).unwrap_or((0, 0));
facts.push(("namespaces_with_policy".into(), with.to_string()));
facts.push(("namespaces_without_policy".into(), without.to_string()));
let dist = db::doctor_governance_depth_distribution(conn).unwrap_or_default();
let depth_summary: String = dist
.iter()
.enumerate()
.filter(|(_, n)| **n > 0)
.map(|(d, n)| format!("d{d}={n}"))
.collect::<Vec<_>>()
.join(",");
facts.push((
"inheritance_depth".into(),
if depth_summary.is_empty() {
"empty".into()
} else {
depth_summary
},
));
match db::doctor_oldest_pending_age_secs(conn) {
Ok(Some(age)) => {
facts.push(("oldest_pending_age_secs".into(), age.to_string()));
if age > crate::SECS_PER_DAY {
severity = Severity::Critical;
note = Some(format!(
"oldest pending action is {age}s old (>{} threshold = 24h)",
crate::SECS_PER_DAY,
));
}
}
Ok(None) => {
facts.push(("oldest_pending_age_secs".into(), "queue_empty".into()));
}
Err(e) => {
facts.push(("pending_query_error".into(), e.to_string()));
}
}
let pending_count = db::count_pending_actions_by_status(conn, "pending").unwrap_or(0);
facts.push(("pending_actions_total".into(), pending_count.to_string()));
ReportSection {
name: "Governance".into(),
severity,
facts,
note,
}
}
fn section_sync(conn: &rusqlite::Connection) -> ReportSection {
let mut facts = Vec::new();
let mut severity = Severity::Info;
let mut note: Option<String> = None;
let peer_count: i64 = conn
.query_row("SELECT COUNT(*) FROM sync_state", [], |r| r.get(0))
.unwrap_or(0);
facts.push(("peer_count".into(), peer_count.to_string()));
if peer_count == 0 {
facts.push((
FACT_MAX_SKEW_SECS.into(),
"not_observed (no peers registered)".into(),
));
return ReportSection {
name: "Sync".into(),
severity: Severity::NotAvailable,
facts,
note: Some("no sync_state rows — single-node deployment or T3+ not yet enabled".into()),
};
}
match db::doctor_max_sync_skew_secs(conn) {
Ok(Some(skew)) => {
facts.push((FACT_MAX_SKEW_SECS.into(), skew.to_string()));
if skew > 600 {
severity = Severity::Critical;
note = Some(format!(
"max sync skew is {skew}s (>600s threshold) — peer mesh is drifting"
));
}
}
Ok(None) => {
facts.push((FACT_MAX_SKEW_SECS.into(), "not_observed".into()));
}
Err(e) => {
facts.push(("sync_query_error".into(), e.to_string()));
}
}
ReportSection {
name: "Sync".into(),
severity,
facts,
note,
}
}
fn section_webhook(conn: &rusqlite::Connection) -> ReportSection {
let mut facts = Vec::new();
let mut severity = Severity::Info;
let mut note: Option<String> = None;
let sub_count = db::count_subscriptions(conn).unwrap_or(0);
facts.push(("subscription_count".into(), sub_count.to_string()));
let (dispatched, failed) = db::doctor_webhook_delivery_totals(conn).unwrap_or((0, 0));
facts.push(("dispatched_total".into(), dispatched.to_string()));
facts.push(("failed_total".into(), failed.to_string()));
if dispatched > 0 {
let success_rate = ((dispatched.saturating_sub(failed)) as f64 / dispatched as f64) * 100.0;
facts.push(("success_rate_pct".into(), format!("{success_rate:.2}")));
if success_rate < 95.0 {
severity = Severity::Warning;
note = Some(format!(
"lifetime delivery success {success_rate:.2}% < 95% threshold"
));
}
} else {
facts.push(("success_rate_pct".into(), "no_deliveries_yet".into()));
}
ReportSection {
name: "Webhook".into(),
severity,
facts,
note,
}
}
fn section_capabilities_local() -> ReportSection {
ReportSection {
name: "Capabilities".into(),
severity: Severity::NotAvailable,
facts: vec![(
field_names::CAPABILITIES.into(),
"use --remote <url> to query the live capabilities endpoint".into(),
)],
note: None,
}
}
fn section_llm_reachability_1146() -> ReportSection {
use crate::config::{AppConfig, ConfigSource, KeySource};
let app_config = AppConfig::load();
let resolved = app_config.resolve_llm(None, None, None);
let mut facts = vec![
("backend".into(), resolved.backend.clone()),
("model".into(), resolved.model.clone()),
("base_url".into(), resolved.base_url.clone()),
("config_source".into(), resolved.source.as_str().to_string()),
(
field_names::KEY_SOURCE.into(),
resolved.api_key_source.as_str().to_string(),
),
];
if let KeySource::Error(reason) = &resolved.api_key_source {
facts.push(("key_error".into(), reason.clone()));
}
if matches!(resolved.source, ConfigSource::CompiledDefault) {
return ReportSection {
name: SECTION_LLM_REACHABILITY.into(),
severity: Severity::Info,
facts,
note: Some(
"no operator LLM configuration found (CLI / env / [llm] section / \
legacy flat fields all absent); LLM-powered features will be \
inactive. To enable, set AI_MEMORY_LLM_BACKEND in the process \
env or write a [llm] section in config.toml. See \
docs/CONFIG_SCHEMA.md for the canonical schema."
.into(),
),
};
}
let (probe_url, bearer) = if resolved.is_ollama_native() {
(crate::llm::ollama_tags_url(&resolved.base_url), None)
} else {
(
format!("{}/models", resolved.base_url),
resolved.api_key().map(str::to_string),
)
};
facts.push(("probe_url".into(), probe_url.clone()));
let started = std::time::Instant::now();
let client = match reqwest::blocking::Client::builder()
.timeout(std::time::Duration::from_secs(5))
.connect_timeout(std::time::Duration::from_secs(5))
.build()
{
Ok(c) => c,
Err(e) => {
facts.push((
"error".into(),
format!("{MSG_HTTP_CLIENT_BUILD_FAILED}: {e}"),
));
return ReportSection {
name: SECTION_LLM_REACHABILITY.into(),
severity: Severity::Critical,
facts,
note: Some("could not build HTTP client for probe".into()),
};
}
};
let mut req = client.get(&probe_url);
if let Some(k) = bearer {
req = req.bearer_auth(k);
}
let (severity, note) = match req.send() {
Ok(resp) => {
let status = resp.status();
let elapsed_ms = started.elapsed().as_millis();
facts.push(("http_status".into(), status.as_u16().to_string()));
facts.push((field_names::LATENCY_MS.into(), elapsed_ms.to_string()));
if status.is_success() {
(Severity::Info, None)
} else if status.as_u16() == 401 || status.as_u16() == 403 {
(
Severity::Warning,
Some(format!(
"auth failed (status {}); URL is reachable but the \
resolved API key was rejected — check [llm].api_key_env / \
[llm].api_key_file / process env",
status.as_u16()
)),
)
} else if status.as_u16() == 429 {
(
Severity::Warning,
Some("rate-limited (status 429); vendor reachable but throttling".into()),
)
} else if status.is_server_error() {
(
Severity::Warning,
Some(format!(
"vendor 5xx (status {}); reachable but currently degraded",
status.as_u16()
)),
)
} else {
(
Severity::Critical,
Some(format!(
"unexpected status {} from {} — verify base_url + endpoint shape",
status.as_u16(),
probe_url
)),
)
}
}
Err(e) => {
let elapsed_ms = started.elapsed().as_millis();
facts.push((field_names::LATENCY_MS.into(), elapsed_ms.to_string()));
facts.push(("error".into(), e.to_string()));
let kind = if e.is_timeout() {
"timeout"
} else if e.is_connect() {
"connect"
} else {
"transport"
};
(
Severity::Critical,
Some(format!(
"network/{kind} error contacting {probe_url} — verify \
base_url and connectivity"
)),
)
}
};
ReportSection {
name: SECTION_LLM_REACHABILITY.into(),
severity,
facts,
note,
}
}
fn gpu_policy_warn_applicable(backend: &str, gpu_detected: bool) -> bool {
!crate::config::is_api_embed_backend(backend) && !gpu_detected
}
fn nvidia_gpu_detected() -> bool {
std::process::Command::new("nvidia-smi")
.arg("-L")
.output()
.map(|out| out.status.success())
.unwrap_or(false)
}
fn section_embeddings_reachability_1598() -> ReportSection {
use crate::config::{AppConfig, ConfigSource, KeySource};
let app_config = AppConfig::load();
let resolved = app_config.resolve_embeddings();
let mut facts = vec![
("backend".into(), resolved.backend.clone()),
("model".into(), resolved.model.clone()),
("base_url".into(), resolved.url.clone()),
("config_source".into(), resolved.source.as_str().to_string()),
(
field_names::KEY_SOURCE.into(),
resolved.key_source.as_str().to_string(),
),
];
if let KeySource::Error(reason) = &resolved.key_source {
facts.push(("key_error".into(), reason.clone()));
}
if matches!(resolved.source, ConfigSource::CompiledDefault) {
return ReportSection {
name: SECTION_EMBEDDINGS_REACHABILITY.into(),
severity: Severity::Info,
facts,
note: Some(
"no operator embeddings configuration found (env / [embeddings] \
section / legacy flat fields all absent); the tier preset \
governs the embedder. To wire an API embedding backend, set \
AI_MEMORY_EMBED_BACKEND or write an [embeddings] section in \
config.toml (#1598)."
.into(),
),
};
}
let is_api = crate::config::is_api_embed_backend(&resolved.backend);
let started = std::time::Instant::now();
let client = match reqwest::blocking::Client::builder()
.timeout(std::time::Duration::from_secs(5))
.connect_timeout(std::time::Duration::from_secs(5))
.build()
{
Ok(c) => c,
Err(e) => {
facts.push((
"error".into(),
format!("{MSG_HTTP_CLIENT_BUILD_FAILED}: {e}"),
));
return ReportSection {
name: SECTION_EMBEDDINGS_REACHABILITY.into(),
severity: Severity::Critical,
facts,
note: Some("could not build HTTP client for probe".into()),
};
}
};
let (probe_url, req) = if is_api {
let url = format!(
"{}{}",
resolved.url,
crate::llm::OPENAI_COMPAT_EMBEDDINGS_PATH
);
let mut req = client
.post(&url)
.json(&serde_json::json!({ "model": resolved.model, "input": "a" }));
if let Some(key) = resolved.api_key() {
req = req.bearer_auth(key);
}
(url, req)
} else {
let url = crate::llm::ollama_tags_url(&resolved.url);
let req = client.get(&url);
(url, req)
};
facts.push(("probe_url".into(), probe_url.clone()));
let (mut severity, mut note) = match req.send() {
Ok(resp) => {
let status = resp.status();
let elapsed_ms = started.elapsed().as_millis();
facts.push(("http_status".into(), status.as_u16().to_string()));
facts.push((field_names::LATENCY_MS.into(), elapsed_ms.to_string()));
if status.is_success() {
(Severity::Info, None)
} else if status.as_u16() == 401 || status.as_u16() == 403 {
(
Severity::Warning,
Some(format!(
"auth failed (status {}); URL is reachable but the \
resolved embedding API key was rejected — check \
[embeddings].api_key_env / [embeddings].api_key_file / process env",
status.as_u16()
)),
)
} else if status.as_u16() == 429 {
(
Severity::Warning,
Some("rate-limited (status 429); vendor reachable but throttling".into()),
)
} else if status.is_server_error() {
(
Severity::Warning,
Some(format!(
"vendor 5xx (status {}); reachable but currently degraded",
status.as_u16()
)),
)
} else {
(
Severity::Critical,
Some(format!(
"unexpected status {} from {} — verify base_url + endpoint shape",
status.as_u16(),
probe_url
)),
)
}
}
Err(e) => {
let elapsed_ms = started.elapsed().as_millis();
facts.push((field_names::LATENCY_MS.into(), elapsed_ms.to_string()));
facts.push(("error".into(), e.to_string()));
let kind = if e.is_timeout() {
"timeout"
} else if e.is_connect() {
"connect"
} else {
"transport"
};
(
Severity::Critical,
Some(format!(
"network/{kind} error contacting {probe_url} — verify \
base_url and connectivity"
)),
)
}
};
if gpu_policy_warn_applicable(&resolved.backend, nvidia_gpu_detected()) {
severity = severity_max(severity, Severity::Warning);
let gpu_note = format!(
"embeddings backend '{}' on a host with no compatible GPU — \
operator policy prefers API embeddings on CPU-only nodes (#1598)",
resolved.backend
);
facts.push(("gpu_policy".into(), gpu_note.clone()));
note = Some(match note {
Some(existing) => format!("{existing}; {gpu_note}"),
None => gpu_note,
});
}
ReportSection {
name: SECTION_EMBEDDINGS_REACHABILITY.into(),
severity,
facts,
note,
}
}
fn section_reflection_health(conn: &rusqlite::Connection) -> ReportSection {
let mut facts = Vec::new();
let mut severity = Severity::Info;
let mut notes: Vec<String> = Vec::new();
let dist_rows = db::doctor_reflection_depth_distribution(conn).unwrap_or_default();
if dist_rows.is_empty() {
facts.push(("reflections_observed".into(), "none".into()));
} else {
for row in &dist_rows {
facts.push((
format!("ns::{}::dist", row.namespace),
format!(
"depth-0={} depth-1={} depth-2={} depth-3+={} avg={:.2} max={}",
row.depth0,
row.depth1,
row.depth2,
row.depth3_plus,
row.avg_depth,
row.max_depth
),
));
const WARN_DEPTH_THRESHOLD: i64 = 2;
if row.max_depth >= WARN_DEPTH_THRESHOLD {
severity = severity_max(severity, Severity::Warning);
notes.push(format!(
"namespace '{}' max_depth={} approaches default cap (max_reflection_depth=3)",
row.namespace, row.max_depth
));
}
}
}
let totals = db::doctor_reflection_totals_by_namespace(conn).unwrap_or_default();
for (ns, last_24h, last_7d, all_time) in &totals {
facts.push((
format!("ns::{}::totals", ns),
format!("24h={last_24h} 7d={last_7d} all_time={all_time}"),
));
}
let last_day_cutoff = (chrono::Utc::now() - chrono::Duration::hours(24)).to_rfc3339();
let refusals_24h =
db::doctor_reflection_depth_exceeded_count(conn, &last_day_cutoff).unwrap_or(0);
facts.push(("depth_limit_refusals_24h".into(), refusals_24h.to_string()));
if refusals_24h > 0 {
severity = severity_max(severity, Severity::Warning);
notes.push(format!(
"{refusals_24h} depth-limit refusal(s) in the last 24h \
(event_type='reflection.depth_exceeded' in signed_events)"
));
}
let refusals_all =
db::doctor_reflection_depth_exceeded_count(conn, "1970-01-01T00:00:00Z").unwrap_or(0);
facts.push((
"depth_limit_refusals_all_time".into(),
refusals_all.to_string(),
));
let note = if notes.is_empty() {
None
} else {
Some(notes.join("; "))
};
ReportSection {
name: "Reflection Health".into(),
severity,
facts,
note,
}
}
pub(super) fn severity_max(a: Severity, b: Severity) -> Severity {
if Report::rank(b) > Report::rank(a) {
b
} else {
a
}
}
fn run_remote(url: &str, db_path: &Path) -> Report {
let mut sections = Vec::with_capacity(2);
let base = url.trim_end_matches('/');
let cap_url = format!("{base}{}", crate::handlers::routes::CAPABILITIES);
let stats_url = format!("{base}{}", crate::handlers::routes::STATS);
sections.push(section_capabilities_remote(&cap_url));
sections.push(section_recall_remote(&cap_url));
sections.push(section_storage_remote(&stats_url));
sections.push(ReportSection {
name: "Index".into(),
severity: Severity::NotAvailable,
facts: vec![("hint".into(), MSG_RAW_SQL_DB_MODE.into())],
note: None,
});
sections.push(ReportSection {
name: "Governance".into(),
severity: Severity::NotAvailable,
facts: vec![("hint".into(), MSG_RAW_SQL_DB_MODE.into())],
note: None,
});
sections.push(ReportSection {
name: "Sync".into(),
severity: Severity::NotAvailable,
facts: vec![("hint".into(), MSG_RAW_SQL_DB_MODE.into())],
note: None,
});
sections.push(ReportSection {
name: "Webhook".into(),
severity: Severity::NotAvailable,
facts: vec![("hint".into(), MSG_RAW_SQL_DB_MODE.into())],
note: None,
});
Report {
mode: "remote".into(),
source: format!("{base} (local db reference: {})", db_path.display()),
generated_at: chrono::Utc::now().to_rfc3339(),
sections,
overall: Severity::Info,
}
}
fn http_get_json(url: &str) -> Result<Value> {
let client = reqwest::blocking::Client::builder()
.timeout(Duration::from_secs(5))
.build()
.context("constructing HTTP client")?;
let resp = client.get(url).send().context("HTTP GET")?;
let status = resp.status();
if !status.is_success() {
anyhow::bail!("HTTP {status} from {url}");
}
resp.json::<Value>().context("decoding JSON response")
}
fn section_capabilities_remote(url: &str) -> ReportSection {
let mut facts = Vec::new();
let mut severity = Severity::Info;
let mut note: Option<String> = None;
match http_get_json(url) {
Ok(v) => {
let schema = v
.get(field_names::SCHEMA_VERSION)
.and_then(Value::as_str)
.unwrap_or("unknown");
facts.push((field_names::SCHEMA_VERSION.into(), schema.to_string()));
let recall_mode = v
.get("features")
.and_then(|f| f.get(FACT_RECALL_MODE_ACTIVE))
.and_then(Value::as_str)
.unwrap_or(NOT_IN_RESPONSE);
facts.push((FACT_RECALL_MODE_ACTIVE.into(), recall_mode.to_string()));
let reranker = v
.get("features")
.and_then(|f| f.get(FACT_RERANKER_ACTIVE))
.and_then(Value::as_str)
.unwrap_or(NOT_IN_RESPONSE);
facts.push((FACT_RERANKER_ACTIVE.into(), reranker.to_string()));
if matches!(recall_mode, "degraded" | "disabled" | "keyword_only") {
let tier = v.get("feature_tier").and_then(Value::as_str).unwrap_or("");
if [
crate::config::FeatureTier::Semantic.as_str(),
crate::config::FeatureTier::Smart.as_str(),
crate::config::FeatureTier::Autonomous.as_str(),
]
.contains(&tier)
{
severity = Severity::Warning;
note = Some(format!(
"tier={tier} but recall_mode_active={recall_mode} — silent degradation"
));
}
}
}
Err(e) => {
severity = Severity::Critical;
facts.push(("error".into(), e.to_string()));
note = Some(format!("could not reach {url}"));
}
}
ReportSection {
name: "Capabilities".into(),
severity,
facts,
note,
}
}
fn section_recall_remote(cap_url: &str) -> ReportSection {
let mut facts = Vec::new();
let severity = Severity::Info;
if let Ok(v) = http_get_json(cap_url) {
let recall_mode = v
.get("features")
.and_then(|f| f.get(FACT_RECALL_MODE_ACTIVE))
.and_then(Value::as_str)
.unwrap_or(NOT_IN_RESPONSE);
facts.push(("active_recall_mode".into(), recall_mode.to_string()));
let reranker = v
.get("features")
.and_then(|f| f.get(FACT_RERANKER_ACTIVE))
.and_then(Value::as_str)
.unwrap_or(NOT_IN_RESPONSE);
facts.push(("active_reranker".into(), reranker.to_string()));
facts.push((
"recall_mode_distribution".into(),
NOT_OBSERVED_PRE_P3.into(),
));
} else {
facts.push(("error".into(), "could not fetch capabilities".into()));
}
ReportSection {
name: "Recall".into(),
severity,
facts,
note: None,
}
}
fn section_storage_remote(stats_url: &str) -> ReportSection {
let mut facts = Vec::new();
let severity = Severity::Info;
match http_get_json(stats_url) {
Ok(v) => {
if let Some(total) = v.get("total").and_then(Value::as_u64) {
facts.push((field_names::TOTAL_MEMORIES.into(), total.to_string()));
}
if let Some(exp) = v.get("expiring_soon").and_then(Value::as_u64) {
facts.push(("expiring_within_1h".into(), exp.to_string()));
}
if let Some(links) = v.get("links_count").and_then(Value::as_u64) {
facts.push(("links".into(), links.to_string()));
}
facts.push((
FACT_DIM_VIOLATIONS.into(),
"not_in_remote_response (P2 surface lands at /api/v1/stats)".into(),
));
}
Err(e) => {
facts.push(("error".into(), e.to_string()));
}
}
ReportSection {
name: "Storage".into(),
severity,
facts,
note: None,
}
}
fn render_text(report: &Report, out: &mut CliOutput<'_>) -> Result<()> {
writeln!(out.stdout, "ai-memory doctor — {} mode", report.mode)?;
writeln!(out.stdout, " source: {}", report.source)?;
writeln!(out.stdout, " generated_at: {}", report.generated_at)?;
writeln!(out.stdout, " overall: {}", report.overall.label())?;
writeln!(out.stdout)?;
for section in &report.sections {
writeln!(
out.stdout,
"[{}] {}",
section.severity.label(),
section.name
)?;
for (k, v) in §ion.facts {
writeln!(out.stdout, " {k:<32} {v}")?;
}
if let Some(note) = §ion.note {
writeln!(out.stdout, " note: {note}")?;
}
writeln!(out.stdout)?;
}
Ok(())
}
#[cfg(test)]
#[allow(clippy::too_many_lines, clippy::similar_names)]
mod tests {
use super::*;
use crate::cli::CliOutput;
use crate::cli::test_utils::{TestEnv, seed_memory};
use rusqlite::params;
#[test]
fn severity_rank_orders_critical_highest() {
assert!(Report::rank(Severity::Critical) > Report::rank(Severity::Warning));
assert!(Report::rank(Severity::Warning) > Report::rank(Severity::Info));
assert!(Report::rank(Severity::Info) > Report::rank(Severity::NotAvailable));
}
#[test]
fn severity_label_renders_for_every_variant() {
assert_eq!(Severity::Info.label(), "INFO");
assert_eq!(Severity::Warning.label(), "WARN");
assert_eq!(Severity::Critical.label(), "CRIT");
assert_eq!(Severity::NotAvailable.label(), "N/A ");
}
#[test]
fn severity_serializes_lowercase_and_round_trips() {
let s = serde_json::to_value(Severity::Critical).unwrap();
assert_eq!(s, serde_json::Value::String("critical".into()));
let s = serde_json::to_value(Severity::NotAvailable).unwrap();
assert_eq!(s, serde_json::Value::String("notavailable".into()));
}
fn mk_section(name: &str, severity: Severity) -> ReportSection {
ReportSection {
name: name.into(),
severity,
facts: vec![("k".into(), "v".into())],
note: None,
}
}
fn mk_report(sections: Vec<ReportSection>) -> Report {
Report {
mode: "local".into(),
source: ":memory:".into(),
generated_at: "now".into(),
sections,
overall: Severity::Info,
}
}
#[test]
fn compute_overall_picks_critical_when_present() {
let mut r = mk_report(vec![
mk_section("A", Severity::Info),
mk_section("B", Severity::Critical),
mk_section("C", Severity::Warning),
]);
r.compute_overall();
assert_eq!(r.overall, Severity::Critical);
}
#[test]
fn compute_overall_picks_warning_when_no_critical() {
let mut r = mk_report(vec![
mk_section("A", Severity::Info),
mk_section("B", Severity::Warning),
]);
r.compute_overall();
assert_eq!(r.overall, Severity::Warning);
}
#[test]
fn compute_overall_picks_info_when_no_warnings_or_critical() {
let mut r = mk_report(vec![
mk_section("A", Severity::NotAvailable),
mk_section("B", Severity::Info),
]);
r.compute_overall();
assert_eq!(r.overall, Severity::Info);
}
#[test]
fn compute_overall_handles_empty_sections() {
let mut r = mk_report(vec![]);
r.compute_overall();
assert_eq!(r.overall, Severity::Info);
}
#[test]
fn compute_overall_only_n_a_yields_n_a() {
let mut r = mk_report(vec![
mk_section("A", Severity::NotAvailable),
mk_section("B", Severity::NotAvailable),
]);
r.compute_overall();
assert_eq!(r.overall, Severity::NotAvailable);
}
#[test]
fn report_section_serializes_with_expected_keys() {
let section = ReportSection {
name: "Storage".into(),
severity: Severity::Warning,
facts: vec![("total".into(), "5".into())],
note: Some("hello".into()),
};
let v = serde_json::to_value(§ion).unwrap();
assert_eq!(v["name"], "Storage");
assert_eq!(v["severity"], "warning");
assert!(v["facts"].is_array());
assert_eq!(v["facts"][0][0], "total");
assert_eq!(v["facts"][0][1], "5");
assert_eq!(v["note"], "hello");
}
#[test]
fn report_section_skips_note_when_none() {
let section = ReportSection {
name: "Recall".into(),
severity: Severity::Info,
facts: vec![],
note: None,
};
let v = serde_json::to_value(§ion).unwrap();
assert!(
v.get("note").is_none(),
"note=None must be skipped per #[serde(skip_serializing_if)]"
);
}
#[test]
fn report_top_level_serialization_has_all_fields() {
let r = mk_report(vec![mk_section("S", Severity::Info)]);
let v = serde_json::to_value(&r).unwrap();
for k in ["mode", "source", "generated_at", "sections", "overall"] {
assert!(v.get(k).is_some(), "expected key {k} in JSON");
}
assert_eq!(v["sections"].as_array().unwrap().len(), 1);
}
fn run_local_collect(db_path: &Path) -> Report {
let mut report = run_local(db_path);
report.compute_overall();
report
}
fn find<'a>(report: &'a Report, name: &str) -> &'a ReportSection {
report
.sections
.iter()
.find(|s| s.name == name)
.unwrap_or_else(|| panic!("section {name} not found"))
}
fn fact<'a>(section: &'a ReportSection, key: &str) -> &'a str {
section
.facts
.iter()
.find(|(k, _)| k == key)
.map(|(_, v)| v.as_str())
.unwrap_or_else(|| panic!("fact {key} not found in section {}", section.name))
}
#[test]
fn local_run_on_empty_db_produces_ten_sections() {
let env = TestEnv::fresh();
let report = run_local_collect(&env.db_path);
assert_eq!(report.mode, "local");
assert_eq!(report.sections.len(), 10);
let names: Vec<&str> = report.sections.iter().map(|s| s.name.as_str()).collect();
assert_eq!(
names,
vec![
"Storage",
"Index",
"Recall",
"Governance",
"Sync",
"Webhook",
"Capabilities",
"Reflection Health",
"LLM Reachability (#1146)",
"Embeddings Reachability (#1598)",
]
);
}
#[test]
fn gpu_policy_warn_applies_only_to_local_backend_without_gpu_1598() {
assert!(gpu_policy_warn_applicable(
crate::llm::BACKEND_OLLAMA,
false
));
assert!(!gpu_policy_warn_applicable(
crate::llm::BACKEND_OLLAMA,
true
));
assert!(!gpu_policy_warn_applicable("openrouter", false));
assert!(!gpu_policy_warn_applicable("openai-compatible", false));
assert!(!gpu_policy_warn_applicable("openrouter", true));
}
#[test]
fn embeddings_reachability_section_present_with_provenance_facts_1598() {
let env = TestEnv::fresh();
let report = run_local_collect(&env.db_path);
let emb = find(&report, SECTION_EMBEDDINGS_REACHABILITY);
for key in [
"backend",
"model",
"base_url",
"config_source",
"key_source",
] {
assert!(
emb.facts.iter().any(|(k, _)| k == key),
"missing fact {key} in {:?}",
emb.facts
);
}
assert!(emb.facts.iter().all(|(k, _)| k != "api_key"));
}
#[test]
fn local_run_empty_db_storage_section_is_info() {
let env = TestEnv::fresh();
let report = run_local_collect(&env.db_path);
let storage = find(&report, "Storage");
assert_eq!(storage.severity, Severity::Info);
assert_eq!(fact(storage, "total_memories"), "0");
let dim = fact(storage, "dim_violations");
assert!(
dim.contains("not_observed") || dim == "0",
"unexpected dim_violations value: {dim}"
);
}
#[test]
fn local_run_with_seeded_memory_reports_total() {
let env = TestEnv::fresh();
seed_memory(&env.db_path, "ns-a", "title-1", "content one");
seed_memory(&env.db_path, "ns-a", "title-2", "content two");
seed_memory(&env.db_path, "ns-b", "title-3", "content three");
let report = run_local_collect(&env.db_path);
let storage = find(&report, "Storage");
assert_eq!(fact(storage, "total_memories"), "3");
let tier_mid = storage
.facts
.iter()
.find(|(k, _)| k == "tier::mid")
.map(|(_, v)| v.as_str());
assert_eq!(tier_mid, Some("3"));
let ns_a = storage
.facts
.iter()
.find(|(k, _)| k == "ns::ns-a")
.map(|(_, v)| v.as_str());
let ns_b = storage
.facts
.iter()
.find(|(k, _)| k == "ns::ns-b")
.map(|(_, v)| v.as_str());
assert_eq!(ns_a, Some("2"));
assert_eq!(ns_b, Some("1"));
}
#[test]
fn local_run_index_section_reports_hnsw_estimate() {
let env = TestEnv::fresh();
seed_memory(&env.db_path, "ns", "t1", "c1");
let report = run_local_collect(&env.db_path);
let index = find(&report, "Index");
assert_eq!(fact(index, "hnsw_size_estimate"), "0");
let cs = fact(index, "cold_start_rebuild_secs_estimate");
assert!(
cs.contains('.'),
"cold_start_secs_estimate should be float-like, got {cs}"
);
assert_eq!(index.severity, Severity::Info);
}
#[test]
fn local_run_recall_section_documents_pre_p3_state() {
let env = TestEnv::fresh();
let report = run_local_collect(&env.db_path);
let recall = find(&report, "Recall");
assert_eq!(recall.severity, Severity::Info);
assert!(fact(recall, "recall_mode_distribution").contains("pre-P3"));
assert!(fact(recall, "reranker_used_distribution").contains("pre-P3"));
assert!(fact(recall, "hint").contains("--remote"));
}
#[test]
fn local_run_sync_section_n_a_when_no_peers() {
let env = TestEnv::fresh();
let report = run_local_collect(&env.db_path);
let sync = find(&report, "Sync");
assert_eq!(sync.severity, Severity::NotAvailable);
assert_eq!(fact(sync, "peer_count"), "0");
assert!(sync.note.is_some());
}
#[test]
fn local_run_capabilities_local_section_n_a() {
let env = TestEnv::fresh();
let report = run_local_collect(&env.db_path);
let cap = find(&report, "Capabilities");
assert_eq!(cap.severity, Severity::NotAvailable);
assert!(fact(cap, "capabilities").contains("--remote"));
}
#[test]
fn local_run_governance_section_empty_is_info() {
let env = TestEnv::fresh();
let report = run_local_collect(&env.db_path);
let gov = find(&report, "Governance");
assert_eq!(gov.severity, Severity::Info);
assert_eq!(fact(gov, "namespaces_with_policy"), "0");
assert_eq!(fact(gov, "namespaces_without_policy"), "0");
assert_eq!(fact(gov, "inheritance_depth"), "empty");
assert_eq!(fact(gov, "oldest_pending_age_secs"), "queue_empty");
assert_eq!(fact(gov, "pending_actions_total"), "0");
}
#[test]
fn local_run_webhook_section_empty_no_deliveries() {
let env = TestEnv::fresh();
let report = run_local_collect(&env.db_path);
let wh = find(&report, "Webhook");
assert_eq!(wh.severity, Severity::Info);
assert_eq!(fact(wh, "subscription_count"), "0");
assert_eq!(fact(wh, "dispatched_total"), "0");
assert_eq!(fact(wh, "failed_total"), "0");
assert_eq!(fact(wh, "success_rate_pct"), "no_deliveries_yet");
}
#[test]
fn governance_section_critical_when_pending_older_than_24h() {
let env = TestEnv::fresh();
{
let conn = crate::db::open(&env.db_path).unwrap();
let twenty_five_hours_ago =
(chrono::Utc::now() - chrono::Duration::hours(25)).to_rfc3339();
conn.execute(
"INSERT INTO pending_actions \
(id, action_type, namespace, payload, requested_by, requested_at, status) \
VALUES ('p1', 'store', 'ns', '{}', 'agent', ?1, 'pending')",
params![twenty_five_hours_ago],
)
.unwrap();
}
let report = run_local_collect(&env.db_path);
let gov = find(&report, "Governance");
assert_eq!(gov.severity, Severity::Critical);
assert!(gov.note.as_ref().unwrap().contains("24h"));
assert_eq!(fact(gov, "pending_actions_total"), "1");
assert_eq!(report.overall, Severity::Critical);
}
#[test]
fn governance_section_info_when_pending_younger_than_24h() {
let env = TestEnv::fresh();
{
let conn = crate::db::open(&env.db_path).unwrap();
let one_hour_ago = (chrono::Utc::now() - chrono::Duration::hours(1)).to_rfc3339();
conn.execute(
"INSERT INTO pending_actions \
(id, action_type, namespace, payload, requested_by, requested_at, status) \
VALUES ('p2', 'store', 'ns', '{}', 'agent', ?1, 'pending')",
params![one_hour_ago],
)
.unwrap();
}
let report = run_local_collect(&env.db_path);
let gov = find(&report, "Governance");
assert_eq!(gov.severity, Severity::Info);
assert_eq!(fact(gov, "pending_actions_total"), "1");
let age_str = fact(gov, "oldest_pending_age_secs");
assert!(
age_str.parse::<i64>().is_ok(),
"expected numeric age, got {age_str}"
);
}
#[test]
fn sync_section_critical_when_skew_exceeds_600s() {
let env = TestEnv::fresh();
{
let conn = crate::db::open(&env.db_path).unwrap();
let now = chrono::Utc::now();
let now_s = now.to_rfc3339();
let earlier = (now - chrono::Duration::seconds(crate::SECS_PER_HOUR)).to_rfc3339();
conn.execute(
"INSERT INTO sync_state (agent_id, peer_id, last_seen_at, last_pulled_at) \
VALUES ('me', 'peer-1', ?1, ?2)",
params![now_s, earlier],
)
.unwrap();
}
let report = run_local_collect(&env.db_path);
let sync = find(&report, "Sync");
assert_eq!(sync.severity, Severity::Critical);
assert!(sync.note.as_ref().unwrap().contains("600s"));
assert_eq!(fact(sync, "peer_count"), "1");
assert_eq!(report.overall, Severity::Critical);
}
#[test]
fn sync_section_info_when_skew_under_threshold() {
let env = TestEnv::fresh();
{
let conn = crate::db::open(&env.db_path).unwrap();
let now = chrono::Utc::now();
let now_s = now.to_rfc3339();
let close = (now - chrono::Duration::seconds(60)).to_rfc3339();
conn.execute(
"INSERT INTO sync_state (agent_id, peer_id, last_seen_at, last_pulled_at) \
VALUES ('me', 'peer-1', ?1, ?2)",
params![now_s, close],
)
.unwrap();
}
let report = run_local_collect(&env.db_path);
let sync = find(&report, "Sync");
assert_eq!(sync.severity, Severity::Info);
assert_eq!(fact(sync, "peer_count"), "1");
let skew = fact(sync, "max_skew_secs");
assert!(
skew.parse::<i64>().is_ok(),
"expected numeric skew, got {skew}"
);
}
#[test]
fn webhook_section_warning_when_success_rate_below_95() {
let env = TestEnv::fresh();
{
let conn = crate::db::open(&env.db_path).unwrap();
let now = chrono::Utc::now().to_rfc3339();
conn.execute(
"INSERT INTO subscriptions \
(id, url, events, created_at, dispatch_count, failure_count) \
VALUES ('s1', 'http://example/x', '*', ?1, 100, 10)",
params![now],
)
.unwrap();
}
let report = run_local_collect(&env.db_path);
let wh = find(&report, "Webhook");
assert_eq!(wh.severity, Severity::Warning);
assert!(wh.note.as_ref().unwrap().contains("95%"));
assert_eq!(fact(wh, "subscription_count"), "1");
assert_eq!(fact(wh, "dispatched_total"), "100");
assert_eq!(fact(wh, "failed_total"), "10");
assert_eq!(fact(wh, "success_rate_pct"), "90.00");
}
#[test]
fn webhook_section_info_when_success_rate_at_or_above_95() {
let env = TestEnv::fresh();
{
let conn = crate::db::open(&env.db_path).unwrap();
let now = chrono::Utc::now().to_rfc3339();
conn.execute(
"INSERT INTO subscriptions \
(id, url, events, created_at, dispatch_count, failure_count) \
VALUES ('s1', 'http://example/x', '*', ?1, 100, 3)",
params![now],
)
.unwrap();
}
let report = run_local_collect(&env.db_path);
let wh = find(&report, "Webhook");
assert_eq!(wh.severity, Severity::Info);
assert!(wh.note.is_none());
assert_eq!(fact(wh, "success_rate_pct"), "97.00");
}
#[test]
fn governance_section_with_namespace_chain_reports_depths() {
let env = TestEnv::fresh();
{
let conn = crate::db::open(&env.db_path).unwrap();
let now = chrono::Utc::now().to_rfc3339();
for (ns, parent) in [
("root", None::<&str>),
("a", Some("root")),
("a/b", Some("a")),
] {
conn.execute(
"INSERT INTO namespace_meta (namespace, parent_namespace, updated_at) \
VALUES (?1, ?2, ?3)",
params![ns, parent, now],
)
.unwrap();
}
}
let report = run_local_collect(&env.db_path);
let gov = find(&report, "Governance");
assert_eq!(gov.severity, Severity::Info);
let depth = fact(gov, "inheritance_depth");
assert!(depth.contains("d0=") && depth.contains("d1=") && depth.contains("d2="));
assert_eq!(fact(gov, "namespaces_without_policy"), "3");
}
fn seed_reflection(conn: &rusqlite::Connection, namespace: &str, depth: i32, title: &str) {
let now = chrono::Utc::now().to_rfc3339();
conn.execute(
"INSERT INTO memories \
(id, tier, namespace, title, content, tags, priority, confidence, source, \
access_count, created_at, updated_at, metadata, reflection_depth) \
VALUES (?, 'mid', ?, ?, 'content', '[]', 5, 1.0, 'test', 0, ?, ?, '{}', ?)",
rusqlite::params![
uuid::Uuid::new_v4().to_string(),
namespace,
title,
now,
now,
depth
],
)
.unwrap();
}
fn seed_depth_exceeded_event(conn: &rusqlite::Connection, timestamp: &str) {
let event = crate::signed_events::SignedEvent {
id: uuid::Uuid::new_v4().to_string(),
agent_id: "test-agent".to_string(),
event_type: crate::signed_events::event_types::REFLECTION_DEPTH_EXCEEDED.to_string(),
payload_hash: vec![0xaa],
signature: None,
attest_level: "unsigned".to_string(),
timestamp: timestamp.to_string(),
..crate::signed_events::SignedEvent::default()
};
crate::signed_events::append_signed_event(conn, &event).unwrap();
}
#[test]
fn reflection_health_section_empty_db_is_info_no_reflections() {
let env = TestEnv::fresh();
let report = run_local_collect(&env.db_path);
let rh = find(&report, "Reflection Health");
assert_eq!(rh.severity, Severity::Info);
assert_eq!(fact(rh, "reflections_observed"), "none");
assert_eq!(fact(rh, "depth_limit_refusals_24h"), "0");
assert_eq!(fact(rh, "depth_limit_refusals_all_time"), "0");
}
#[test]
fn reflection_health_section_depth_distribution_counts() {
let env = TestEnv::fresh();
{
let conn = crate::db::open(&env.db_path).unwrap();
seed_reflection(&conn, "ns-alpha", 0, "base-1");
seed_reflection(&conn, "ns-alpha", 0, "base-2");
seed_reflection(&conn, "ns-alpha", 0, "base-3");
seed_reflection(&conn, "ns-alpha", 1, "refl-1");
seed_reflection(&conn, "ns-alpha", 1, "refl-2");
seed_reflection(&conn, "ns-alpha", 2, "refl-3");
seed_reflection(&conn, "ns-beta", 1, "beta-refl-1");
}
let report = run_local_collect(&env.db_path);
let rh = find(&report, "Reflection Health");
assert!(
rh.facts.iter().all(|(k, _)| k != "reflections_observed"),
"reflections_observed key should be absent when reflections exist"
);
let alpha_dist = rh
.facts
.iter()
.find(|(k, _)| k == "ns::ns-alpha::dist")
.map(|(_, v)| v.as_str());
assert!(alpha_dist.is_some(), "ns::ns-alpha::dist fact missing");
let alpha_str = alpha_dist.unwrap();
assert!(
alpha_str.contains("depth-0=3"),
"expected depth-0=3 in '{alpha_str}'"
);
assert!(
alpha_str.contains("depth-1=2"),
"expected depth-1=2 in '{alpha_str}'"
);
assert!(
alpha_str.contains("depth-2=1"),
"expected depth-2=1 in '{alpha_str}'"
);
assert!(
alpha_str.contains("depth-3+=0"),
"expected depth-3+=0 in '{alpha_str}'"
);
let beta_dist = rh
.facts
.iter()
.find(|(k, _)| k == "ns::ns-beta::dist")
.map(|(_, v)| v.as_str());
assert!(beta_dist.is_some(), "ns::ns-beta::dist fact missing");
let beta_str = beta_dist.unwrap();
assert!(
beta_str.contains("depth-1=1"),
"expected depth-1=1 in '{beta_str}'"
);
}
#[test]
fn reflection_health_warn_when_max_depth_approaches_cap() {
let env = TestEnv::fresh();
{
let conn = crate::db::open(&env.db_path).unwrap();
seed_reflection(&conn, "deep-ns", 2, "depth2-refl");
}
let report = run_local_collect(&env.db_path);
let rh = find(&report, "Reflection Health");
assert_eq!(rh.severity, Severity::Warning);
let note = rh
.note
.as_ref()
.expect("expected a note when depth approaches cap");
assert!(
note.contains("deep-ns"),
"note should name the namespace, got: {note}"
);
assert!(note.contains("cap"), "note should mention cap, got: {note}");
}
#[test]
fn reflection_health_warn_on_depth_limit_refusals_24h() {
let env = TestEnv::fresh();
{
let conn = crate::db::open(&env.db_path).unwrap();
let one_hour_ago = (chrono::Utc::now() - chrono::Duration::hours(1)).to_rfc3339();
seed_depth_exceeded_event(&conn, &one_hour_ago);
}
let report = run_local_collect(&env.db_path);
let rh = find(&report, "Reflection Health");
assert_eq!(rh.severity, Severity::Warning);
assert_eq!(fact(rh, "depth_limit_refusals_24h"), "1");
assert_eq!(fact(rh, "depth_limit_refusals_all_time"), "1");
let note = rh.note.as_ref().expect("expected note on refusals");
assert!(
note.contains("refusal"),
"note should mention refusal, got: {note}"
);
}
#[test]
fn reflection_health_old_refusals_do_not_trigger_24h_warn() {
let env = TestEnv::fresh();
{
let conn = crate::db::open(&env.db_path).unwrap();
let old = (chrono::Utc::now() - chrono::Duration::hours(48)).to_rfc3339();
seed_depth_exceeded_event(&conn, &old);
}
let report = run_local_collect(&env.db_path);
let rh = find(&report, "Reflection Health");
assert_eq!(fact(rh, "depth_limit_refusals_24h"), "0");
assert_eq!(fact(rh, "depth_limit_refusals_all_time"), "1");
assert_eq!(rh.severity, Severity::Info);
}
#[test]
fn reflection_health_totals_per_namespace() {
let env = TestEnv::fresh();
let recent = (chrono::Utc::now() - chrono::Duration::minutes(30)).to_rfc3339();
let old = (chrono::Utc::now() - chrono::Duration::days(10)).to_rfc3339();
{
let conn = crate::db::open(&env.db_path).unwrap();
conn.execute(
"INSERT INTO memories \
(id, tier, namespace, title, content, tags, priority, confidence, source, \
access_count, created_at, updated_at, metadata, reflection_depth) \
VALUES (?, 'mid', 'ns-new', 'new-refl', 'c', '[]', 5, 1.0, 'test', 0, ?, ?, '{}', 1)",
rusqlite::params![uuid::Uuid::new_v4().to_string(), recent, recent],
)
.unwrap();
conn.execute(
"INSERT INTO memories \
(id, tier, namespace, title, content, tags, priority, confidence, source, \
access_count, created_at, updated_at, metadata, reflection_depth) \
VALUES (?, 'mid', 'ns-old', 'old-refl', 'c', '[]', 5, 1.0, 'test', 0, ?, ?, '{}', 1)",
rusqlite::params![uuid::Uuid::new_v4().to_string(), old, old],
)
.unwrap();
}
let report = run_local_collect(&env.db_path);
let rh = find(&report, "Reflection Health");
let new_totals = rh
.facts
.iter()
.find(|(k, _)| k == "ns::ns-new::totals")
.map(|(_, v)| v.as_str())
.expect("ns::ns-new::totals fact missing");
assert!(
new_totals.contains("24h=1"),
"expected 24h=1 in '{new_totals}'"
);
assert!(
new_totals.contains("7d=1"),
"expected 7d=1 in '{new_totals}'"
);
assert!(
new_totals.contains("all_time=1"),
"expected all_time=1 in '{new_totals}'"
);
let old_totals = rh
.facts
.iter()
.find(|(k, _)| k == "ns::ns-old::totals")
.map(|(_, v)| v.as_str())
.expect("ns::ns-old::totals fact missing");
assert!(
old_totals.contains("24h=0"),
"expected 24h=0 in '{old_totals}'"
);
assert!(
old_totals.contains("7d=0"),
"expected 7d=0 in '{old_totals}'"
);
assert!(
old_totals.contains("all_time=1"),
"expected all_time=1 in '{old_totals}'"
);
}
#[test]
fn reflection_health_json_output_parseable_and_has_section() {
let mut env = TestEnv::fresh();
{
let conn = crate::db::open(&env.db_path).unwrap();
seed_reflection(&conn, "ns-json", 1, "json-refl");
}
let db_path = env.db_path.clone();
let mut out = env.output();
let exit = run(
&db_path,
&DoctorArgs {
remote: None,
json: true,
fail_on_warn: false,
},
&mut out,
)
.unwrap();
assert_eq!(exit, 0);
let v: serde_json::Value = serde_json::from_str(env.stdout_str()).expect("JSON must parse");
let sections = v["sections"].as_array().expect("sections is array");
let rh_section = sections
.iter()
.find(|s| s["name"] == "Reflection Health")
.expect("Reflection Health section must be in JSON output");
assert_eq!(rh_section["severity"], "info");
assert!(rh_section["facts"].is_array(), "facts must be a JSON array");
}
#[test]
fn run_emits_json_when_json_flag_set() {
let mut env = TestEnv::fresh();
let db_path = env.db_path.clone();
let mut out = env.output();
let exit = run(
&db_path,
&DoctorArgs {
remote: None,
json: true,
fail_on_warn: false,
},
&mut out,
)
.unwrap();
assert_eq!(exit, 0);
let s = env.stdout_str();
let v: serde_json::Value = serde_json::from_str(s).expect("JSON output must parse");
assert_eq!(v["mode"], "local");
assert!(v["sections"].is_array());
assert!(v["overall"].is_string());
}
#[test]
fn run_emits_text_by_default() {
let mut env = TestEnv::fresh();
let db_path = env.db_path.clone();
let mut out = env.output();
let exit = run(
&db_path,
&DoctorArgs {
remote: None,
json: false,
fail_on_warn: false,
},
&mut out,
)
.unwrap();
assert_eq!(exit, 0);
let s = env.stdout_str();
assert!(s.contains("ai-memory doctor — local mode"));
assert!(s.contains("[INFO] Storage"));
assert!(s.contains("[INFO] Index"));
assert!(s.contains("[N/A ] Capabilities"));
assert!(s.contains("total_memories"));
}
#[test]
fn run_returns_exit_2_on_critical() {
let mut env = TestEnv::fresh();
{
let conn = crate::db::open(&env.db_path).unwrap();
let twenty_five_hours_ago =
(chrono::Utc::now() - chrono::Duration::hours(25)).to_rfc3339();
conn.execute(
"INSERT INTO pending_actions \
(id, action_type, namespace, payload, requested_by, requested_at, status) \
VALUES ('p1', 'store', 'ns', '{}', 'agent', ?1, 'pending')",
params![twenty_five_hours_ago],
)
.unwrap();
}
let db_path = env.db_path.clone();
let mut out = env.output();
let exit = run(
&db_path,
&DoctorArgs {
remote: None,
json: true,
fail_on_warn: false,
},
&mut out,
)
.unwrap();
assert_eq!(exit, 2);
let v: serde_json::Value = serde_json::from_str(env.stdout_str()).unwrap();
assert_eq!(v["overall"], "critical");
}
#[test]
fn run_warning_keeps_exit_0_without_fail_on_warn() {
let mut env = TestEnv::fresh();
{
let conn = crate::db::open(&env.db_path).unwrap();
let now = chrono::Utc::now().to_rfc3339();
conn.execute(
"INSERT INTO subscriptions \
(id, url, events, created_at, dispatch_count, failure_count) \
VALUES ('s1', 'http://x', '*', ?1, 10, 5)",
params![now],
)
.unwrap();
}
let db_path = env.db_path.clone();
let mut out = env.output();
let exit = run(
&db_path,
&DoctorArgs {
remote: None,
json: false,
fail_on_warn: false,
},
&mut out,
)
.unwrap();
assert_eq!(exit, 0, "warning without --fail-on-warn must keep exit 0");
assert!(env.stdout_str().contains("[WARN] Webhook"));
}
#[test]
fn run_warning_returns_exit_1_with_fail_on_warn() {
let mut env = TestEnv::fresh();
{
let conn = crate::db::open(&env.db_path).unwrap();
let now = chrono::Utc::now().to_rfc3339();
conn.execute(
"INSERT INTO subscriptions \
(id, url, events, created_at, dispatch_count, failure_count) \
VALUES ('s1', 'http://x', '*', ?1, 10, 5)",
params![now],
)
.unwrap();
}
let db_path = env.db_path.clone();
let mut out = env.output();
let exit = run(
&db_path,
&DoctorArgs {
remote: None,
json: false,
fail_on_warn: true,
},
&mut out,
)
.unwrap();
assert_eq!(exit, 1, "--fail-on-warn must promote warning to exit 1");
}
#[test]
fn run_critical_is_exit_2_even_without_fail_on_warn() {
let mut env = TestEnv::fresh();
{
let conn = crate::db::open(&env.db_path).unwrap();
let twenty_five_hours_ago =
(chrono::Utc::now() - chrono::Duration::hours(25)).to_rfc3339();
conn.execute(
"INSERT INTO pending_actions \
(id, action_type, namespace, payload, requested_by, requested_at, status) \
VALUES ('p1', 'store', 'ns', '{}', 'agent', ?1, 'pending')",
params![twenty_five_hours_ago],
)
.unwrap();
}
let db_path = env.db_path.clone();
let mut out = env.output();
let exit = run(
&db_path,
&DoctorArgs {
remote: None,
json: false,
fail_on_warn: false,
},
&mut out,
)
.unwrap();
assert_eq!(exit, 2);
}
#[test]
fn local_run_on_unopenable_db_returns_critical_storage_only() {
let tmp = tempfile::tempdir().unwrap();
let bad = tmp.path().join("not-a-db.db");
std::fs::write(&bad, b"this is not a sqlite database, it's just text").unwrap();
let report = run_local_collect(&bad);
assert_eq!(report.sections.len(), 1);
let storage = &report.sections[0];
assert_eq!(storage.name, "Storage");
assert_eq!(storage.severity, Severity::Critical);
assert_eq!(report.overall, Severity::Critical);
assert!(storage.note.as_ref().unwrap().contains("could not open"));
}
#[test]
fn render_text_emits_section_note_when_present() {
let r = mk_report(vec![ReportSection {
name: "Sync".into(),
severity: Severity::Critical,
facts: vec![("max_skew_secs".into(), "9999".into())],
note: Some("peer mesh is drifting".into()),
}]);
let mut stdout = Vec::<u8>::new();
let mut stderr = Vec::<u8>::new();
let mut out = CliOutput::from_std(&mut stdout, &mut stderr);
render_text(&r, &mut out).unwrap();
let s = String::from_utf8(stdout).unwrap();
assert!(s.contains("[CRIT] Sync"));
assert!(s.contains("note: peer mesh is drifting"));
assert!(s.contains("max_skew_secs"));
assert!(s.contains("9999"));
}
async fn run_remote_in_blocking(url: String, db_path: PathBuf) -> Report {
tokio::task::spawn_blocking(move || {
let mut r = run_remote(&url, &db_path);
r.compute_overall();
r
})
.await
.unwrap()
}
use std::path::PathBuf;
#[tokio::test(flavor = "multi_thread")]
async fn remote_section_capabilities_parses_v2_fields() {
use wiremock::matchers::{method, path};
use wiremock::{Mock, MockServer, ResponseTemplate};
let server = MockServer::start().await;
Mock::given(method("GET"))
.and(path("/api/v1/capabilities"))
.respond_with(ResponseTemplate::new(200).set_body_json(serde_json::json!({
"schema_version": "2",
"feature_tier": "smart",
"features": {
"recall_mode_active": "hybrid",
"reranker_active": "cross_encoder"
}
})))
.mount(&server)
.await;
Mock::given(method("GET"))
.and(path("/api/v1/stats"))
.respond_with(ResponseTemplate::new(200).set_body_json(serde_json::json!({
"total": 42,
"expiring_soon": 1,
"links_count": 3
})))
.mount(&server)
.await;
let env = TestEnv::fresh();
let report = run_remote_in_blocking(server.uri(), env.db_path.clone()).await;
assert_eq!(report.mode, "remote");
assert!(report.source.starts_with(&server.uri()));
assert_eq!(report.sections.len(), 7);
let cap = find(&report, "Capabilities");
assert_eq!(cap.severity, Severity::Info);
assert_eq!(fact(cap, "schema_version"), "2");
assert_eq!(fact(cap, "recall_mode_active"), "hybrid");
assert_eq!(fact(cap, "reranker_active"), "cross_encoder");
let recall = find(&report, "Recall");
assert_eq!(fact(recall, "active_recall_mode"), "hybrid");
assert_eq!(fact(recall, "active_reranker"), "cross_encoder");
let storage = find(&report, "Storage");
assert_eq!(fact(storage, "total_memories"), "42");
assert_eq!(fact(storage, "expiring_within_1h"), "1");
assert_eq!(fact(storage, "links"), "3");
for raw in ["Index", "Governance", "Sync", "Webhook"] {
let s = find(&report, raw);
assert_eq!(s.severity, Severity::NotAvailable);
assert!(fact(s, "hint").contains("--db mode"));
}
}
#[tokio::test(flavor = "multi_thread")]
async fn remote_capabilities_silent_degrade_warns_on_capable_tier() {
use wiremock::matchers::{method, path};
use wiremock::{Mock, MockServer, ResponseTemplate};
let server = MockServer::start().await;
Mock::given(method("GET"))
.and(path("/api/v1/capabilities"))
.respond_with(ResponseTemplate::new(200).set_body_json(serde_json::json!({
"schema_version": "2",
"feature_tier": "semantic",
"features": {
"recall_mode_active": "keyword_only",
"reranker_active": "none"
}
})))
.mount(&server)
.await;
let env = TestEnv::fresh();
let report = run_remote_in_blocking(server.uri(), env.db_path.clone()).await;
let cap = find(&report, "Capabilities");
assert_eq!(cap.severity, Severity::Warning);
assert!(cap.note.as_ref().unwrap().contains("silent degradation"));
}
#[tokio::test(flavor = "multi_thread")]
async fn remote_capabilities_degraded_on_keyword_tier_does_not_warn() {
use wiremock::matchers::{method, path};
use wiremock::{Mock, MockServer, ResponseTemplate};
let server = MockServer::start().await;
Mock::given(method("GET"))
.and(path("/api/v1/capabilities"))
.respond_with(ResponseTemplate::new(200).set_body_json(serde_json::json!({
"schema_version": "2",
"feature_tier": "keyword",
"features": {
"recall_mode_active": "keyword_only",
"reranker_active": "none"
}
})))
.mount(&server)
.await;
let env = TestEnv::fresh();
let report = run_remote_in_blocking(server.uri(), env.db_path.clone()).await;
let cap = find(&report, "Capabilities");
assert_eq!(cap.severity, Severity::Info);
assert!(cap.note.is_none());
}
#[tokio::test(flavor = "multi_thread")]
async fn remote_capabilities_unreachable_endpoint_is_critical() {
use std::net::TcpListener;
let listener = TcpListener::bind("127.0.0.1:0").unwrap();
let port = listener.local_addr().unwrap().port();
drop(listener);
let url = format!("http://127.0.0.1:{port}");
let env = TestEnv::fresh();
let report = run_remote_in_blocking(url, env.db_path.clone()).await;
let cap = find(&report, "Capabilities");
assert_eq!(cap.severity, Severity::Critical);
assert!(cap.note.as_ref().unwrap().contains("could not reach"));
assert_eq!(report.overall, Severity::Critical);
}
#[tokio::test(flavor = "multi_thread")]
async fn remote_capabilities_legacy_v1_renders_not_in_response() {
use wiremock::matchers::{method, path};
use wiremock::{Mock, MockServer, ResponseTemplate};
let server = MockServer::start().await;
Mock::given(method("GET"))
.and(path("/api/v1/capabilities"))
.respond_with(ResponseTemplate::new(200).set_body_json(serde_json::json!({
"schema_version": "1"
})))
.mount(&server)
.await;
let env = TestEnv::fresh();
let report = run_remote_in_blocking(server.uri(), env.db_path.clone()).await;
let cap = find(&report, "Capabilities");
assert_eq!(cap.severity, Severity::Info);
assert_eq!(fact(cap, "schema_version"), "1");
assert_eq!(fact(cap, "recall_mode_active"), "not_in_response");
assert_eq!(fact(cap, "reranker_active"), "not_in_response");
}
#[tokio::test(flavor = "multi_thread")]
async fn remote_run_via_run_entry_uses_remote_mode_string() {
use wiremock::matchers::{method, path};
use wiremock::{Mock, MockServer, ResponseTemplate};
let server = MockServer::start().await;
Mock::given(method("GET"))
.and(path("/api/v1/capabilities"))
.respond_with(ResponseTemplate::new(200).set_body_json(serde_json::json!({
"schema_version": "2",
"feature_tier": "semantic",
"features": {
"recall_mode_active": "hybrid",
"reranker_active": "none"
}
})))
.mount(&server)
.await;
Mock::given(method("GET"))
.and(path("/api/v1/stats"))
.respond_with(ResponseTemplate::new(200).set_body_json(serde_json::json!({
"total": 0
})))
.mount(&server)
.await;
let env_db = TestEnv::fresh().db_path;
let url = server.uri();
let (exit, stdout) = tokio::task::spawn_blocking(move || {
let mut stdout = Vec::<u8>::new();
let mut stderr = Vec::<u8>::new();
let mut out = CliOutput::from_std(&mut stdout, &mut stderr);
let exit = run(
&env_db,
&DoctorArgs {
remote: Some(url),
json: true,
fail_on_warn: false,
},
&mut out,
)
.unwrap();
(exit, stdout)
})
.await
.unwrap();
assert_eq!(exit, 0);
let v: serde_json::Value = serde_json::from_slice(&stdout).unwrap();
assert_eq!(v["mode"], "remote");
}
#[tokio::test(flavor = "multi_thread")]
async fn remote_url_trailing_slash_is_trimmed() {
use wiremock::matchers::{method, path};
use wiremock::{Mock, MockServer, ResponseTemplate};
let server = MockServer::start().await;
Mock::given(method("GET"))
.and(path("/api/v1/capabilities"))
.respond_with(ResponseTemplate::new(200).set_body_json(serde_json::json!({
"schema_version": "2",
"features": {}
})))
.mount(&server)
.await;
Mock::given(method("GET"))
.and(path("/api/v1/stats"))
.respond_with(ResponseTemplate::new(200).set_body_json(serde_json::json!({})))
.mount(&server)
.await;
let env = TestEnv::fresh();
let report =
run_remote_in_blocking(format!("{}/", server.uri()), env.db_path.clone()).await;
let cap = find(&report, "Capabilities");
assert_eq!(cap.severity, Severity::Info);
}
#[tokio::test(flavor = "multi_thread")]
async fn remote_storage_500_renders_error_without_severity_bump() {
use wiremock::matchers::{method, path};
use wiremock::{Mock, MockServer, ResponseTemplate};
let server = MockServer::start().await;
Mock::given(method("GET"))
.and(path("/api/v1/capabilities"))
.respond_with(ResponseTemplate::new(200).set_body_json(serde_json::json!({
"schema_version": "2",
"features": {}
})))
.mount(&server)
.await;
Mock::given(method("GET"))
.and(path("/api/v1/stats"))
.respond_with(ResponseTemplate::new(500))
.mount(&server)
.await;
let env = TestEnv::fresh();
let report = run_remote_in_blocking(server.uri(), env.db_path.clone()).await;
let storage = find(&report, "Storage");
assert_eq!(storage.severity, Severity::Info);
let err = fact(storage, "error");
assert!(
err.contains("HTTP 500"),
"expected HTTP 500 message, got {err}"
);
}
fn run_tokens_capture(args: TokensArgs) -> (i32, String, String) {
let mut stdout = Vec::<u8>::new();
let mut stderr = Vec::<u8>::new();
let exit;
{
let mut out = CliOutput::from_std(&mut stdout, &mut stderr);
exit = run_tokens(args, &mut out).expect("run_tokens");
}
(
exit,
String::from_utf8(stdout).unwrap(),
String::from_utf8(stderr).unwrap(),
)
}
#[test]
fn run_tokens_human_default_profile_is_core() {
let (exit, stdout, _stderr) = run_tokens_capture(TokensArgs::default());
assert_eq!(exit, 0);
assert!(
stdout.contains("Active profile: core"),
"default profile should be core; got: {stdout}"
);
let n = crate::profile::Profile::full().expected_tool_count();
let needle = format!("Full ({n} tools loaded)");
assert!(
stdout.contains(&needle),
"report should include full-profile baseline `{needle}` (canonical \
from Profile::full().expected_tool_count()); got: {stdout}"
);
assert!(
stdout.contains("Tokenizer: cl100k_base"),
"report should call out the tokenizer"
);
}
#[test]
fn run_tokens_json_emits_structured_payload() {
let args = TokensArgs {
json: true,
raw_table: false,
profile: Some("graph".to_string()),
hooks: false,
};
let (exit, stdout, _) = run_tokens_capture(args);
assert_eq!(exit, 0);
let v: serde_json::Value =
serde_json::from_str(&stdout).expect("--json must emit valid JSON");
assert_eq!(v["schema_version"], "v0.6.4-tokens-1");
assert_eq!(v["tokenizer"], "cl100k_base");
let total = v["full_profile_total_tokens"].as_u64().unwrap();
assert!(
(5_000..=17_000).contains(&total),
"full_profile_total_tokens out of honest range: {total}"
);
assert!(v["active_total_tokens"].as_u64().unwrap() > 0);
let families = v["families"].as_array().unwrap();
let core_row = families.iter().find(|r| r["name"] == "core").unwrap();
assert_eq!(core_row["loaded"], true);
let graph_row = families.iter().find(|r| r["name"] == "graph").unwrap();
assert_eq!(graph_row["loaded"], true);
let archive_row = families.iter().find(|r| r["name"] == "archive").unwrap();
assert_eq!(archive_row["loaded"], false);
}
#[test]
fn run_tokens_raw_table_includes_per_tool_rows() {
let args = TokensArgs {
json: false,
raw_table: true,
profile: None,
hooks: false,
};
let (exit, stdout, _) = run_tokens_capture(args);
assert_eq!(exit, 0);
let v: serde_json::Value = serde_json::from_str(&stdout).unwrap();
let tools = v["tools"].as_array().unwrap();
assert_eq!(
tools.len(),
crate::profile::Profile::full().expected_tool_count(),
"raw_table must include every tool — canonical count is the \
SSOT `Profile::full().expected_tool_count()` (derived from \
the per-Family `tool_names` slices); no literal is restated"
);
let store = tools
.iter()
.find(|t| t["name"] == "memory_store")
.expect("memory_store row");
assert_eq!(store["family"], "core");
assert_eq!(store["loaded_under_active_profile"], true);
}
#[test]
fn run_tokens_invalid_profile_exits_2_with_diagnostic() {
let args = TokensArgs {
json: false,
raw_table: false,
profile: Some("Core".to_string()),
hooks: false,
};
let (exit, _stdout, stderr) = run_tokens_capture(args);
assert_eq!(exit, 2, "malformed profile must exit 2");
assert!(
stderr.contains("case-sensitive lowercase"),
"diagnostic should mention case rule; got: {stderr}"
);
}
fn run_hooks_capture(args: HooksReportArgs) -> (i32, String, String) {
let mut stdout = Vec::<u8>::new();
let mut stderr = Vec::<u8>::new();
let exit;
{
let mut out = CliOutput::from_std(&mut stdout, &mut stderr);
exit = run_hooks(args, &mut out).expect("run_hooks");
}
(
exit,
String::from_utf8(stdout).unwrap(),
String::from_utf8(stderr).unwrap(),
)
}
fn mk_hook(command: &str) -> crate::hooks::config::HookConfig {
crate::hooks::config::HookConfig {
event: crate::hooks::HookEvent::PostStore,
command: std::path::PathBuf::from(command),
priority: 10,
timeout_ms: 1_000,
mode: crate::hooks::config::HookMode::Exec,
enabled: true,
namespace: "*".to_string(),
fail_mode: crate::hooks::config::FailMode::Open,
}
}
#[test]
fn run_hooks_human_default_no_config_lists_zero() {
let (exit, stdout, _stderr) = run_hooks_capture(HooksReportArgs { json: false });
assert_eq!(exit, 0);
assert!(stdout.contains("ai-memory doctor --hooks"));
assert!(stdout.contains("Hooks loaded:"));
}
#[test]
fn run_hooks_json_emits_schema_versioned_payload() {
let (exit, stdout, _) = run_hooks_capture(HooksReportArgs { json: true });
assert_eq!(exit, 0);
let v: serde_json::Value = serde_json::from_str(&stdout).expect("valid JSON");
assert_eq!(v["schema_version"], "v0.7-hooks-1");
assert!(v["hooks_loaded"].is_number());
assert!(v["executors"].is_array());
assert!(v["timeout_violations"].is_number());
}
#[test]
fn run_tokens_with_hooks_flag_appends_block() {
let args = TokensArgs {
json: false,
raw_table: false,
profile: None,
hooks: true,
};
let (exit, stdout, _stderr) = run_tokens_capture(args);
assert_eq!(exit, 0);
assert!(stdout.contains("ai-memory doctor --tokens"));
assert!(stdout.contains("ai-memory doctor --hooks"));
}
#[test]
fn render_hooks_human_with_synthetic_hook_renders_row() {
let toml_src = r#"
[[hook]]
event = "post_store"
command = "/usr/local/bin/echo-something-long"
mode = "exec"
namespace = "*"
priority = 5
timeout_ms = 1000
enabled = true
"#;
let hooks = crate::hooks::config::HookConfig::load_from_str(toml_src).expect("parse hooks");
let mut stdout = Vec::<u8>::new();
let mut stderr = Vec::<u8>::new();
let synthetic_path = std::path::PathBuf::from("/tmp/synthetic/hooks.toml");
{
let mut out = CliOutput::from_std(&mut stdout, &mut stderr);
render_hooks_human_with(&mut out, Some(&synthetic_path), &hooks).unwrap();
}
let s = String::from_utf8(stdout).unwrap();
assert!(s.contains("ai-memory doctor --hooks"));
assert!(s.contains("Config path:"));
assert!(s.contains("Hooks loaded: 1"));
assert!(s.contains("echo-something-long") || s.contains("event"));
assert!(s.contains("Chain class-deadline violations"));
assert!(s.contains("note: live metrics land"));
}
#[test]
fn render_hooks_human_with_no_hooks_emits_helpful_note() {
let mut stdout = Vec::<u8>::new();
let mut stderr = Vec::<u8>::new();
let synthetic_path = std::path::PathBuf::from("/some/path/hooks.toml");
{
let mut out = CliOutput::from_std(&mut stdout, &mut stderr);
render_hooks_human_with(&mut out, Some(&synthetic_path), &[]).unwrap();
}
let s = String::from_utf8(stdout).unwrap();
assert!(s.contains("ai-memory doctor --hooks"));
assert!(s.contains("Config path:"));
assert!(s.contains("Hooks loaded: 0"));
assert!(s.contains("(no hooks configured"));
}
#[test]
fn render_hooks_human_with_command_no_filename_falls_back_to_display() {
let toml_src = r#"
[[hook]]
event = "post_store"
command = "/"
mode = "exec"
namespace = "*"
priority = 1
timeout_ms = 500
enabled = true
"#;
let hooks = crate::hooks::config::HookConfig::load_from_str(toml_src).expect("parse hooks");
let mut stdout = Vec::<u8>::new();
let mut stderr = Vec::<u8>::new();
{
let mut out = CliOutput::from_std(&mut stdout, &mut stderr);
render_hooks_human_with(&mut out, None, &hooks).unwrap();
}
let s = String::from_utf8(stdout).unwrap();
assert!(!s.contains("Config path:"));
assert!(s.contains("Hooks loaded: 1"));
}
#[test]
fn render_hooks_human_with_rows_renders_each_hook() {
let hooks = vec![mk_hook("/usr/local/bin/notify-hook.sh")];
let mut stdout = Vec::<u8>::new();
let mut stderr = Vec::<u8>::new();
{
let mut out = CliOutput::from_std(&mut stdout, &mut stderr);
render_hooks_human_with(&mut out, None, &hooks).unwrap();
}
let s = String::from_utf8(stdout).unwrap();
assert!(s.contains("Hooks loaded: 1"), "got: {s}");
assert!(s.contains("notify-hook.sh"), "got: {s}");
}
#[test]
fn storage_section_warns_with_stats_error_on_missing_schema() {
let conn = rusqlite::Connection::open_in_memory().expect("open_in_memory");
let section = section_storage(&conn, Path::new("/nonexistent/doctor.db"));
assert_eq!(section.severity, Severity::Warning);
assert!(
section.facts.iter().any(|(k, _)| k == "stats_error"),
"facts: {:?}",
section.facts
);
assert!(
section
.facts
.iter()
.any(|(k, v)| k == "dim_violations" && v.contains("not_observed")),
"facts: {:?}",
section.facts
);
}
#[test]
fn index_section_warns_when_hnsw_within_5pct_of_cap() {
let conn = rusqlite::Connection::open_in_memory().expect("open_in_memory");
conn.execute_batch(
"CREATE TABLE memories(embedding BLOB);
INSERT INTO memories(embedding)
WITH RECURSIVE c(x) AS (SELECT 1 UNION ALL SELECT x + 1 FROM c WHERE x < 95000)
SELECT x FROM c;",
)
.expect("seed 95k embedded rows");
let section = section_index(&conn);
assert_eq!(section.severity, Severity::Warning);
let note = section.note.as_deref().expect("note must explain the cap");
assert!(note.contains("within 5%"), "note: {note}");
assert!(
section
.facts
.iter()
.any(|(k, v)| k == "hnsw_size_estimate" && v == "95000"),
"facts: {:?}",
section.facts
);
}
#[test]
fn sync_section_not_observed_when_peer_has_no_pull_timestamp() {
let conn = rusqlite::Connection::open_in_memory().expect("open_in_memory");
conn.execute_batch(
"CREATE TABLE sync_state(last_seen_at TEXT, last_pulled_at TEXT);
INSERT INTO sync_state(last_seen_at, last_pulled_at)
VALUES ('2026-01-01T00:00:00Z', NULL);",
)
.expect("seed peer row");
let section = section_sync(&conn);
assert_eq!(section.severity, Severity::Info);
assert!(
section
.facts
.iter()
.any(|(k, v)| k == "max_skew_secs" && v == "not_observed"),
"facts: {:?}",
section.facts
);
assert!(
section
.facts
.iter()
.any(|(k, v)| k == "peer_count" && v == "1"),
"facts: {:?}",
section.facts
);
}
fn reach_env_lock() -> &'static std::sync::Mutex<()> {
static L: std::sync::OnceLock<std::sync::Mutex<()>> = std::sync::OnceLock::new();
L.get_or_init(|| std::sync::Mutex::new(()))
}
struct EnvScope(Vec<(&'static str, Option<std::ffi::OsString>)>);
impl EnvScope {
fn set(pairs: &[(&'static str, &str)]) -> Self {
let mut prev = Vec::new();
for (k, v) in pairs {
prev.push((*k, std::env::var_os(k)));
unsafe { std::env::set_var(k, v) };
}
prev.push((
"AI_MEMORY_NO_CONFIG",
std::env::var_os("AI_MEMORY_NO_CONFIG"),
));
unsafe { std::env::set_var("AI_MEMORY_NO_CONFIG", "1") };
Self(prev)
}
}
impl Drop for EnvScope {
fn drop(&mut self) {
for (k, v) in &self.0 {
match v {
Some(val) => unsafe { std::env::set_var(k, val) },
None => unsafe { std::env::remove_var(k) },
}
}
}
}
fn clear_llm_embed_env() {
for k in [
"AI_MEMORY_LLM_BACKEND",
"AI_MEMORY_LLM_BASE_URL",
"AI_MEMORY_LLM_API_KEY",
"AI_MEMORY_LLM_MODEL",
"AI_MEMORY_EMBED_BACKEND",
"AI_MEMORY_EMBED_BASE_URL",
"AI_MEMORY_EMBED_API_KEY",
"AI_MEMORY_EMBED_MODEL",
] {
unsafe { std::env::remove_var(k) };
}
}
#[test]
fn gpu_policy_warn_applicable_matrix_1598() {
assert!(!gpu_policy_warn_applicable("openai", true));
assert!(!gpu_policy_warn_applicable("openai", false));
assert!(gpu_policy_warn_applicable("ollama", false));
assert!(!gpu_policy_warn_applicable("ollama", true));
}
#[tokio::test(flavor = "multi_thread")]
async fn llm_reachability_compiled_default_is_info_1146() {
let _g = reach_env_lock().lock().unwrap_or_else(|e| e.into_inner());
clear_llm_embed_env();
let _scope = EnvScope::set(&[]);
let section = tokio::task::spawn_blocking(section_llm_reachability_1146)
.await
.unwrap();
assert_eq!(section.severity, Severity::Info);
assert!(
section
.note
.as_deref()
.unwrap_or("")
.contains("no operator LLM configuration"),
"compiled-default note expected; got {:?}",
section.note
);
}
#[tokio::test(flavor = "multi_thread")]
async fn llm_reachability_probe_arms_1146() {
use wiremock::matchers::{method, path};
use wiremock::{Mock, MockServer, ResponseTemplate};
for (code, want) in [
(200u16, Severity::Info),
(401, Severity::Warning),
(503, Severity::Warning),
] {
let server = MockServer::start().await;
Mock::given(method("GET"))
.and(path("/models"))
.respond_with(ResponseTemplate::new(code))
.mount(&server)
.await;
let uri = server.uri();
let section = {
let _g = reach_env_lock().lock().unwrap_or_else(|e| e.into_inner());
clear_llm_embed_env();
let _scope = EnvScope::set(&[
("AI_MEMORY_LLM_BACKEND", "openai-compatible"),
("AI_MEMORY_LLM_BASE_URL", &uri),
("AI_MEMORY_LLM_API_KEY", "probe-key"),
("AI_MEMORY_LLM_MODEL", "probe-model"),
]);
tokio::task::spawn_blocking(section_llm_reachability_1146)
.await
.unwrap()
};
assert_eq!(section.severity, want, "LLM probe status {code}");
assert_eq!(fact(§ion, "http_status"), code.to_string());
}
}
#[tokio::test(flavor = "multi_thread")]
async fn embeddings_reachability_compiled_default_is_info_1598() {
let _g = reach_env_lock().lock().unwrap_or_else(|e| e.into_inner());
clear_llm_embed_env();
let _scope = EnvScope::set(&[]);
let section = tokio::task::spawn_blocking(section_embeddings_reachability_1598)
.await
.unwrap();
assert_eq!(section.severity, Severity::Info);
assert!(
section
.note
.as_deref()
.unwrap_or("")
.contains("no operator embeddings configuration"),
"compiled-default note expected; got {:?}",
section.note
);
}
#[tokio::test(flavor = "multi_thread")]
async fn embeddings_reachability_api_probe_arms_1598() {
use wiremock::matchers::{method, path};
use wiremock::{Mock, MockServer, ResponseTemplate};
for (code, want) in [
(200u16, Severity::Info),
(401, Severity::Warning),
(500, Severity::Warning),
] {
let server = MockServer::start().await;
Mock::given(method("POST"))
.and(path("/embeddings"))
.respond_with(
ResponseTemplate::new(code).set_body_json(serde_json::json!({"data": []})),
)
.mount(&server)
.await;
let uri = server.uri();
let section = {
let _g = reach_env_lock().lock().unwrap_or_else(|e| e.into_inner());
clear_llm_embed_env();
let _scope = EnvScope::set(&[
("AI_MEMORY_EMBED_BACKEND", "openai-compatible"),
("AI_MEMORY_EMBED_BASE_URL", &uri),
("AI_MEMORY_EMBED_API_KEY", "probe-key"),
("AI_MEMORY_EMBED_MODEL", "probe-embed-model"),
]);
tokio::task::spawn_blocking(section_embeddings_reachability_1598)
.await
.unwrap()
};
assert_eq!(section.severity, want, "embed probe status {code}");
assert_eq!(fact(§ion, "http_status"), code.to_string());
}
}
}