Skip to main content

ai_memory/cli/
doctor.rs

1// Copyright 2026 AlphaOne LLC
2// SPDX-License-Identifier: Apache-2.0
3
4//! `ai-memory doctor` (Phase P7 / R7) — operator-visible health dashboard.
5//!
6//! The doctor reads three v0.6.3.1 surfaces — Capabilities v2 (P1), data
7//! integrity (P2), and recall observability (P3) — plus the v0.6.3 stats /
8//! governance / subscription tables, and produces a human-readable health
9//! report with severity tagging. It also has a `--json` mode for CI usage
10//! and a `--remote <url>` mode that becomes the **fleet doctor** at T3+.
11//!
12//! Exit codes:
13//!   - `0` — healthy (no warnings or critical findings).
14//!   - `1` — at least one warning (and `--fail-on-warn` was passed; without
15//!     the flag, warnings still keep exit 0).
16//!   - `2` — at least one critical finding.
17//!
18//! ## Severity rules (initial)
19//!
20//! - **Critical:** dim_violations > 0; pending_actions older than 24h;
21//!   sync skew > 600s; HNSW evictions > 0.
22//! - **Warning:** silent-degrade flag from Capabilities v2
23//!   (recall_mode != "hybrid" on capable tiers); subscription delivery
24//!   success < 95% over the lifetime of the subscription.
25//! - **Info:** anything else worth reporting.
26//!
27//! ## What is stubbed pending P1/P2/P3
28//!
29//! - **dim_violations** (P2): pre-P2 schemas have no `embedding_dim` column.
30//!   `db::doctor_dim_violations` returns `Ok(None)` and the doctor renders
31//!   "not yet observed (pre-P2 schema)".
32//! - **HNSW evictions** (P3): the eviction counter has no SQL surface today.
33//!   The doctor reports the value as 0 from a NOT_AVAILABLE-tagged section
34//!   until P3 lands the in-memory counter.
35//! - **recall_mode / reranker_used distribution** (P3): no rolling window
36//!   has been wired yet. The doctor consults the Capabilities response
37//!   for the *active* mode at this instant and reports it as the only
38//!   data point.
39//! - **Sync mesh** (T3+): we report `last_pulled_at` skew across
40//!   `sync_state` rows when present, otherwise NOT_AVAILABLE.
41//!
42//! ## Anti-goals (per spec)
43//!
44//! - Do NOT add new monitoring infrastructure (no Prometheus, OTel exporters).
45//! - Do NOT make doctor write to the DB. Read-only.
46//! - Do NOT make doctor block the database. Indexed `COUNT(*)` queries only.
47
48use crate::cli::CliOutput;
49use crate::db;
50use crate::models::field_names;
51use anyhow::{Context, Result};
52use serde::Serialize;
53use serde_json::Value;
54use std::path::Path;
55use std::time::Duration;
56
57// ── #1558 batch 6 — repeated doctor fact / section labels ──────────────
58const FACT_DIM_VIOLATIONS: &str = "dim_violations";
59const FACT_MAX_SKEW_SECS: &str = "max_skew_secs";
60const FACT_RECALL_MODE_ACTIVE: &str = "recall_mode_active";
61const FACT_RERANKER_ACTIVE: &str = "reranker_active";
62const SECTION_LLM_REACHABILITY: &str = "LLM Reachability (#1146)";
63const SECTION_EMBEDDINGS_REACHABILITY: &str = "Embeddings Reachability (#1598)";
64const MSG_RAW_SQL_DB_MODE: &str = "raw SQL section — only available in --db mode";
65/// #1598 literal-dedup — shared probe-client failure fact prefix for
66/// the LLM + Embeddings reachability sections.
67const MSG_HTTP_CLIENT_BUILD_FAILED: &str = "http client build failed";
68
69/// #1558 batch 5 wave 3 — placeholder fact value rendered when the
70/// probed capabilities payload does not carry the requested feature
71/// key (older daemons).
72const NOT_IN_RESPONSE: &str = "not_in_response";
73
74/// #1558 batch 5 wave 3 — placeholder fact value for the recall-mode /
75/// reranker distribution rows, which need the P3 rolling counter that
76/// has not landed yet.
77const NOT_OBSERVED_PRE_P3: &str = "not_observed (pre-P3 rolling counter)";
78
79/// Severity bucket attached to every doctor finding.
80#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize)]
81#[serde(rename_all = "lowercase")]
82pub enum Severity {
83    Info,
84    Warning,
85    Critical,
86    /// The section couldn't be queried in this mode (e.g. raw SQL section
87    /// in remote mode, or P2-dependent section on pre-P2 schema).
88    NotAvailable,
89}
90
91impl Severity {
92    fn label(self) -> &'static str {
93        match self {
94            Severity::Info => "INFO",
95            Severity::Warning => "WARN",
96            Severity::Critical => "CRIT",
97            Severity::NotAvailable => "N/A ",
98        }
99    }
100}
101
102/// One section of the report. `facts` is a list of human-readable
103/// `(key, value)` lines so the JSON output stays structured and the text
104/// output stays scannable.
105#[derive(Debug, Serialize)]
106pub struct ReportSection {
107    pub name: String,
108    pub severity: Severity,
109    pub facts: Vec<(String, String)>,
110    /// Optional one-line explanation when severity != Info.
111    #[serde(skip_serializing_if = "Option::is_none")]
112    pub note: Option<String>,
113}
114
115/// The full doctor report.
116#[derive(Debug, Serialize)]
117pub struct Report {
118    pub mode: String,
119    pub source: String,
120    pub generated_at: String,
121    pub sections: Vec<ReportSection>,
122    pub overall: Severity,
123}
124
125impl Report {
126    /// Compute the overall severity as the max across sections (CRIT > WARN > INFO > N/A).
127    fn rank(s: Severity) -> u8 {
128        match s {
129            Severity::NotAvailable => 0,
130            Severity::Info => 1,
131            Severity::Warning => 2,
132            Severity::Critical => 3,
133        }
134    }
135
136    fn compute_overall(&mut self) {
137        self.overall = self
138            .sections
139            .iter()
140            .map(|s| s.severity)
141            .max_by_key(|s| Self::rank(*s))
142            .unwrap_or(Severity::Info);
143    }
144}
145
146/// Args from the CLI clap struct. Kept separate so `cli::doctor::run` can
147/// be called directly from tests without going through clap.
148pub struct DoctorArgs {
149    pub remote: Option<String>,
150    pub json: bool,
151    pub fail_on_warn: bool,
152}
153
154/// v0.6.4-004 — Args for `ai-memory doctor --tokens`. Routes to
155/// [`run_tokens`] instead of the regular health pass.
156#[derive(Debug, Default)]
157pub struct TokensArgs {
158    /// Emit structured JSON instead of human-readable.
159    pub json: bool,
160    /// Dump the full per-tool size table (implies `json`).
161    pub raw_table: bool,
162    /// Hypothetical profile to evaluate (defaults to `core` —
163    /// the v0.6.4 default).
164    pub profile: Option<String>,
165    /// v0.7-G3 — also append the hook-executor metrics block.
166    /// Operators running `--tokens --hooks` see both surfaces in
167    /// one pass.
168    pub hooks: bool,
169}
170
171/// v0.7-G3 — Args for `ai-memory doctor --hooks` (standalone).
172/// Routes to [`run_hooks`].
173#[derive(Debug, Default)]
174pub struct HooksReportArgs {
175    /// Emit structured JSON instead of human-readable.
176    pub json: bool,
177}
178
179/// v0.6.4-004 — token-cost report.
180///
181/// Walks `crate::sizes::tool_sizes()`, groups by family via
182/// `crate::profile::Family::for_tool`, rolls up per-profile totals,
183/// and emits either a human-readable table or a JSON document.
184///
185/// Returns 0 on success. Errors when the `--profile` flag is malformed
186/// (the doctor's job is to surface the same diagnostic the MCP server
187/// would, not to crash with a stack trace) — those exit code 2.
188pub fn run_tokens(args: TokensArgs, out: &mut CliOutput<'_>) -> Result<i32> {
189    use crate::profile::{Family, Profile};
190    use crate::sizes;
191
192    // Resolve the hypothetical profile. Default to `core` since that
193    // is what v0.6.4 ships and what the operator wants to see savings
194    // *against*.
195    let profile = match Profile::parse(args.profile.as_deref().unwrap_or("core")) {
196        Ok(p) => p,
197        Err(e) => {
198            writeln!(out.stderr, "ai-memory doctor --tokens: {e}")?;
199            return Ok(2);
200        }
201    };
202
203    let table = sizes::tool_sizes();
204    let trimmed_table = sizes::trimmed_tool_sizes();
205    let full_total: usize = table.iter().map(|t| t.total_tokens).sum();
206    let active_total: usize = table
207        .iter()
208        .filter(|t| profile.loads(&t.name))
209        .map(|t| t.total_tokens)
210        .sum();
211    // v0.7 C4 — also report the trimmed (default `tools/list`) cost
212    // because that's what an MCP host actually pays per request unless
213    // it opts into `memory_capabilities { verbose=true }`.
214    let trimmed_full_total: usize = trimmed_table.iter().map(|t| t.total_tokens).sum();
215    let trimmed_active_total: usize = trimmed_table
216        .iter()
217        .filter(|t| profile.loads(&t.name))
218        .map(|t| t.total_tokens)
219        .sum();
220    let savings = full_total.saturating_sub(active_total);
221    let pct = if full_total == 0 {
222        0.0
223    } else {
224        (f64::from(u32::try_from(savings).unwrap_or(u32::MAX))
225            / f64::from(u32::try_from(full_total).unwrap_or(u32::MAX)))
226            * 100.0
227    };
228
229    // Per-family rollup. Includes "always-on" pseudo bucket for tools
230    // that load regardless of profile (today: just memory_capabilities).
231    let mut family_totals: Vec<(String, usize, usize)> = Family::all()
232        .iter()
233        .map(|f| {
234            let mut tool_count = 0usize;
235            let mut sum = 0usize;
236            for entry in table {
237                if Family::for_tool(&entry.name) == Some(*f) {
238                    tool_count += 1;
239                    sum += entry.total_tokens;
240                }
241            }
242            (f.name().to_string(), tool_count, sum)
243        })
244        .collect();
245    family_totals.sort_by_key(|(_, _, sum)| std::cmp::Reverse(*sum));
246
247    if args.json || args.raw_table {
248        // Always include the full per-tool table when --raw-table is
249        // set; --json gives the rolled-up view.
250        let payload = serde_json::json!({
251            (field_names::SCHEMA_VERSION): "v0.6.4-tokens-1",
252            "tokenizer": "cl100k_base",
253            "active_profile": profile.families().iter().map(|f| f.name()).collect::<Vec<_>>(),
254            "active_total_tokens": active_total,
255            "full_profile_total_tokens": full_total,
256            // v0.7 C4 — actually-paid cost on the default tools/list path.
257            "trimmed_active_total_tokens": trimmed_active_total,
258            "trimmed_full_profile_total_tokens": trimmed_full_total,
259            "savings_tokens": savings,
260            "savings_pct": format!("{pct:.1}"),
261            "families": family_totals.iter().map(|(name, count, sum)| {
262                // Resolve family enum from the name to ask whether
263                // it is loaded under the active profile.
264                let fam = Family::all()
265                    .iter()
266                    .find(|f| f.name() == name)
267                    .copied()
268                    .unwrap_or(Family::Other);
269                serde_json::json!({
270                    "name": name,
271                    "tool_count": count,
272                    "tokens": sum,
273                    "loaded": profile.includes(fam),
274                })
275            }).collect::<Vec<_>>(),
276            "tools": if args.raw_table {
277                serde_json::Value::Array(
278                    table.iter().map(|t| serde_json::json!({
279                        "name": t.name,
280                        "tokens": t.total_tokens,
281                        "family": Family::for_tool(&t.name).map(|f| f.name()),
282                        "loaded_under_active_profile": profile.loads(&t.name),
283                    })).collect()
284                )
285            } else {
286                serde_json::Value::Null
287            },
288        });
289        writeln!(out.stdout, "{}", serde_json::to_string_pretty(&payload)?)?;
290        return Ok(0);
291    }
292
293    // Human-readable.
294    writeln!(out.stdout, "ai-memory doctor --tokens")?;
295    writeln!(
296        out.stdout,
297        "  Tokenizer: cl100k_base (Claude / GPT input accounting)"
298    )?;
299    writeln!(
300        out.stdout,
301        "  Active profile: {}",
302        profile
303            .families()
304            .iter()
305            .map(|f| f.name())
306            .collect::<Vec<_>>()
307            .join(",")
308    )?;
309    writeln!(out.stdout)?;
310    writeln!(out.stdout, "  Tool surface cost (verbose schema, ceiling):")?;
311    writeln!(
312        out.stdout,
313        "    Active ({:>2} tools loaded): {:>6} tokens",
314        table.iter().filter(|t| profile.loads(&t.name)).count(),
315        active_total
316    )?;
317    writeln!(
318        out.stdout,
319        "    Full   ({:>2} tools loaded): {:>6} tokens",
320        table.len(),
321        full_total
322    )?;
323    writeln!(
324        out.stdout,
325        "    Savings vs full:           {:>6} tokens ({pct:.1}%)",
326        savings
327    )?;
328    // v0.7 C4 — the bottom line an MCP host actually pays per request.
329    writeln!(out.stdout)?;
330    writeln!(
331        out.stdout,
332        "  Tools/list payload (v0.7 C4 + #859 trim — properties exposed, prose stripped):"
333    )?;
334    writeln!(
335        out.stdout,
336        "    Active                     {:>6} tokens",
337        trimmed_active_total
338    )?;
339    writeln!(
340        out.stdout,
341        "    Full                       {:>6} tokens",
342        trimmed_full_total
343    )?;
344    writeln!(out.stdout)?;
345    writeln!(out.stdout, "  Per-family breakdown (sorted by total cost):")?;
346    for (name, count, sum) in &family_totals {
347        writeln!(
348            out.stdout,
349            "    {name:<12} {count:>2} tools  {sum:>6} tokens",
350        )?;
351    }
352    if args.hooks {
353        writeln!(out.stdout)?;
354        render_hooks_human(out)?;
355    }
356    Ok(0)
357}
358
359/// v0.7-G3 — `ai-memory doctor --hooks` entry point. Renders the
360/// loaded `hooks.toml` shape plus zeroed metric placeholders.
361///
362/// The CLI process is *not* the running daemon — it can't reach the
363/// in-process `ExecutorRegistry`. Until G7-G11 wires the executor
364/// into the actual memory operation points, this surface reports
365/// the loaded config + a zeroed metrics row per hook so operators
366/// can sanity-check their `hooks.toml` (and so the doctor JSON
367/// schema stabilizes for the dashboard work that lands alongside).
368pub fn run_hooks(args: HooksReportArgs, out: &mut CliOutput<'_>) -> Result<i32> {
369    use crate::hooks::config::HookConfig;
370
371    let path_opt = HookConfig::default_path();
372    let hooks: Vec<HookConfig> = match path_opt.as_ref() {
373        Some(p) if p.exists() => match HookConfig::load_from_file(p) {
374            Ok(h) => h,
375            Err(e) => {
376                writeln!(out.stderr, "ai-memory doctor --hooks: {e}")?;
377                return Ok(2);
378            }
379        },
380        _ => Vec::new(),
381    };
382
383    if args.json {
384        let payload = serde_json::json!({
385            (field_names::SCHEMA_VERSION): "v0.7-hooks-1",
386            "config_path": path_opt.as_ref().map(|p| p.display().to_string()),
387            "hooks_loaded": hooks.len(),
388            "executors": hooks.iter().map(|h| serde_json::json!({
389                "event": h.event,
390                "command": h.command.display().to_string(),
391                "mode": h.mode,
392                "namespace": h.namespace,
393                "priority": h.priority,
394                "timeout_ms": h.timeout_ms,
395                "enabled": h.enabled,
396                "metrics": {
397                    "events_fired": 0,
398                    "events_dropped": 0,
399                    "mean_latency_us": 0,
400                },
401            })).collect::<Vec<_>>(),
402            // G6 — process-wide chain-deadline trip count. Bumped
403            // by `HookChain::fire` every time a class deadline
404            // expired (either before a hook even ran, or because
405            // the chain-shrunk per-hook timeout fired). Surfaced
406            // here so operators can spot a chronically over-budget
407            // chain without grepping logs.
408            "timeout_violations": crate::hooks::timeouts::timeout_violations_total(),
409            "note": "metrics placeholders until G7-G11 wires the executor into the daemon",
410        });
411        writeln!(out.stdout, "{}", serde_json::to_string_pretty(&payload)?)?;
412        return Ok(0);
413    }
414
415    render_hooks_human_with(out, path_opt.as_deref(), &hooks)?;
416    Ok(0)
417}
418
419/// Human-readable hooks block. Used by `--hooks` standalone *and*
420/// by the appended block when the operator combines `--tokens --hooks`.
421fn render_hooks_human(out: &mut CliOutput<'_>) -> Result<()> {
422    use crate::hooks::config::HookConfig;
423    let path_opt = HookConfig::default_path();
424    let hooks: Vec<HookConfig> = match path_opt.as_ref() {
425        Some(p) if p.exists() => HookConfig::load_from_file(p).unwrap_or_default(),
426        _ => Vec::new(),
427    };
428    render_hooks_human_with(out, path_opt.as_deref(), &hooks)
429}
430
431fn render_hooks_human_with(
432    out: &mut CliOutput<'_>,
433    path: Option<&Path>,
434    hooks: &[crate::hooks::config::HookConfig],
435) -> Result<()> {
436    writeln!(out.stdout, "ai-memory doctor --hooks")?;
437    if let Some(p) = path {
438        writeln!(out.stdout, "  Config path: {}", p.display())?;
439    }
440    writeln!(out.stdout, "  Hooks loaded: {}", hooks.len())?;
441    if hooks.is_empty() {
442        writeln!(
443            out.stdout,
444            "  (no hooks configured — drop a hooks.toml at the path above to enable)"
445        )?;
446        return Ok(());
447    }
448    writeln!(out.stdout)?;
449    writeln!(
450        out.stdout,
451        "  {:<26} {:<8} {:<22} fired dropped mean_us",
452        "event", "mode", "command"
453    )?;
454    for h in hooks {
455        let event = format!("{:?}", h.event);
456        let mode = format!("{:?}", h.mode);
457        let cmd = h
458            .command
459            .file_name()
460            .map(|s| s.to_string_lossy().into_owned())
461            .unwrap_or_else(|| h.command.display().to_string());
462        let cmd_truncated: String = cmd.chars().take(22).collect();
463        writeln!(
464            out.stdout,
465            "  {event:<26} {mode:<8} {cmd_truncated:<22} {:>5} {:>7} {:>7}",
466            0, 0, 0,
467        )?;
468    }
469    writeln!(out.stdout)?;
470    writeln!(
471        out.stdout,
472        "  Chain class-deadline violations: {}",
473        crate::hooks::timeouts::timeout_violations_total()
474    )?;
475    writeln!(
476        out.stdout,
477        "  note: live metrics land when G7-G11 wires the executor into the daemon."
478    )?;
479    Ok(())
480}
481
482/// Entry point. Returns the process exit code as a `i32` (0/1/2). The
483/// caller (daemon_runtime) must `std::process::exit(code)` after the WAL
484/// checkpoint has been skipped (doctor never writes).
485///
486/// # Errors
487///
488/// Returns `Err` only when the report itself cannot be written to the
489/// output stream — DB / HTTP errors are folded into NOT_AVAILABLE
490/// sections so a partial report still renders.
491pub fn run(db_path: &Path, args: &DoctorArgs, out: &mut CliOutput<'_>) -> Result<i32> {
492    let mut report = if let Some(url) = &args.remote {
493        run_remote(url, db_path)
494    } else {
495        run_local(db_path)
496    };
497    report.compute_overall();
498
499    if args.json {
500        writeln!(out.stdout, "{}", serde_json::to_string_pretty(&report)?)?;
501    } else {
502        render_text(&report, out)?;
503    }
504
505    let code = match report.overall {
506        Severity::Critical => 2,
507        Severity::Warning if args.fail_on_warn => 1,
508        _ => 0,
509    };
510    Ok(code)
511}
512
513// ---------------------------------------------------------------------------
514// Local (--db) mode
515// ---------------------------------------------------------------------------
516
517fn run_local(db_path: &Path) -> Report {
518    let mut sections = Vec::with_capacity(7);
519
520    // Open the connection once; failures bubble into a single Critical
521    // section and the rest of the report is N/A.
522    let conn = match db::open(db_path) {
523        Ok(c) => c,
524        Err(e) => {
525            sections.push(ReportSection {
526                name: "Storage".into(),
527                severity: Severity::Critical,
528                facts: vec![("error".into(), e.to_string())],
529                note: Some(format!(
530                    "could not open database at {} — every other section is N/A",
531                    db_path.display()
532                )),
533            });
534            return Report {
535                mode: "local".into(),
536                source: db_path.display().to_string(),
537                generated_at: chrono::Utc::now().to_rfc3339(),
538                sections,
539                overall: Severity::Critical,
540            };
541        }
542    };
543
544    sections.push(section_storage(&conn, db_path));
545    sections.push(section_index(&conn));
546    sections.push(section_recall_local());
547    sections.push(section_governance(&conn));
548    sections.push(section_sync(&conn));
549    sections.push(section_webhook(&conn));
550    sections.push(section_capabilities_local());
551    sections.push(section_reflection_health(&conn));
552    sections.push(section_llm_reachability_1146());
553    sections.push(section_embeddings_reachability_1598());
554
555    Report {
556        mode: "local".into(),
557        source: db_path.display().to_string(),
558        generated_at: chrono::Utc::now().to_rfc3339(),
559        sections,
560        overall: Severity::Info,
561    }
562}
563
564fn section_storage(conn: &rusqlite::Connection, db_path: &Path) -> ReportSection {
565    let mut facts = Vec::new();
566    let mut severity = Severity::Info;
567    let mut note: Option<String> = None;
568
569    match db::stats(conn, db_path) {
570        Ok(stats) => {
571            facts.push((field_names::TOTAL_MEMORIES.into(), stats.total.to_string()));
572            facts.push(("expiring_within_1h".into(), stats.expiring_soon.to_string()));
573            facts.push(("links".into(), stats.links_count.to_string()));
574            facts.push(("db_size_bytes".into(), stats.db_size_bytes.to_string()));
575            for tc in &stats.by_tier {
576                facts.push((format!("tier::{}", tc.tier), tc.count.to_string()));
577            }
578            for nc in stats.by_namespace.iter().take(10) {
579                facts.push((format!("ns::{}", nc.namespace), nc.count.to_string()));
580            }
581        }
582        Err(e) => {
583            severity = Severity::Warning;
584            facts.push(("stats_error".into(), e.to_string()));
585        }
586    }
587
588    // dim_violations (P2 surface). Pre-P2: Ok(None) -> render N/A line, no severity bump.
589    match db::doctor_dim_violations(conn) {
590        Ok(Some(0)) => {
591            facts.push((FACT_DIM_VIOLATIONS.into(), "0".into()));
592        }
593        Ok(Some(n)) => {
594            facts.push((FACT_DIM_VIOLATIONS.into(), n.to_string()));
595            severity = Severity::Critical;
596            note = Some(format!(
597                "{n} memories have an embedding dim that disagrees with their namespace's modal dim"
598            ));
599        }
600        Ok(None) => {
601            facts.push((
602                FACT_DIM_VIOLATIONS.into(),
603                "not_observed (pre-P2 schema)".into(),
604            ));
605        }
606        Err(e) => {
607            facts.push(("dim_violations_error".into(), e.to_string()));
608        }
609    }
610
611    ReportSection {
612        name: "Storage".into(),
613        severity,
614        facts,
615        note,
616    }
617}
618
619fn section_index(conn: &rusqlite::Connection) -> ReportSection {
620    let mut facts = Vec::new();
621    let mut severity = Severity::Info;
622    let mut note: Option<String> = None;
623
624    // HNSW size proxy: count of memories with an embedding (the in-memory
625    // index is rebuilt from this on startup).
626    let hnsw_size: i64 = conn
627        .query_row(
628            "SELECT COUNT(*) FROM memories WHERE embedding IS NOT NULL",
629            [],
630            |r| r.get(0),
631        )
632        .unwrap_or(0);
633    facts.push(("hnsw_size_estimate".into(), hnsw_size.to_string()));
634
635    // Cold-start cost: rough estimate of the time to rebuild HNSW on
636    // daemon restart, derived from the canonical-workload measured rate
637    // (~50k inserts/sec). Surfaced as a sanity-check signal, not a budget.
638    let cold_start_secs = (hnsw_size as f64) / 50_000.0;
639    facts.push((
640        "cold_start_rebuild_secs_estimate".into(),
641        format!("{cold_start_secs:.2}"),
642    ));
643
644    // Eviction counter (P3). Until P3 wires the in-memory counter into a
645    // queryable surface, render NOT_AVAILABLE without a severity bump.
646    facts.push((
647        "index_evictions_total".into(),
648        "not_observed (pre-P3 surface)".into(),
649    ));
650
651    // P3-aware path: when MAX_ENTRIES (100_000) is approached, advise the
652    // operator. This is a forward-leaning hint that becomes accurate once
653    // P3 lands the counter.
654    if hnsw_size >= 95_000 {
655        severity = Severity::Warning;
656        note = Some(format!(
657            "HNSW is at {hnsw_size} embeddings, within 5% of the 100k MAX_ENTRIES cap; \
658             P3 will start emitting eviction events"
659        ));
660    }
661
662    ReportSection {
663        name: "Index".into(),
664        severity,
665        facts,
666        note,
667    }
668}
669
670fn section_recall_local() -> ReportSection {
671    // Without P3's rolling window, the local doctor can only report the
672    // tier configuration that *would* drive recall today. The remote
673    // doctor (--remote) gets the live `recall_mode_active` from the v2
674    // capabilities endpoint when P1 lands.
675    ReportSection {
676        name: "Recall".into(),
677        severity: Severity::Info,
678        facts: vec![
679            (
680                "recall_mode_distribution".into(),
681                NOT_OBSERVED_PRE_P3.into(),
682            ),
683            (
684                "reranker_used_distribution".into(),
685                NOT_OBSERVED_PRE_P3.into(),
686            ),
687            (
688                "hint".into(),
689                "use --remote to read the live capabilities endpoint".into(),
690            ),
691        ],
692        note: None,
693    }
694}
695
696fn section_governance(conn: &rusqlite::Connection) -> ReportSection {
697    let mut facts = Vec::new();
698    let mut severity = Severity::Info;
699    let mut note: Option<String> = None;
700
701    // v0.7.0 K3 — surface the active permissions.mode + per-mode
702    // decision counts so operators can verify the gate is wired and
703    // observe drift between advertised and enforced policy.
704    let mode = crate::config::active_permissions_mode();
705    facts.push(("permissions_mode".into(), mode.as_str().to_string()));
706    let counts = crate::config::permissions_decision_counts();
707    facts.push(("decisions::enforce".into(), counts.enforce.to_string()));
708    facts.push(("decisions::advisory".into(), counts.advisory.to_string()));
709    facts.push(("decisions::off".into(), counts.off.to_string()));
710
711    let (with, without) = db::doctor_governance_coverage(conn).unwrap_or((0, 0));
712    facts.push(("namespaces_with_policy".into(), with.to_string()));
713    facts.push(("namespaces_without_policy".into(), without.to_string()));
714
715    let dist = db::doctor_governance_depth_distribution(conn).unwrap_or_default();
716    let depth_summary: String = dist
717        .iter()
718        .enumerate()
719        .filter(|(_, n)| **n > 0)
720        .map(|(d, n)| format!("d{d}={n}"))
721        .collect::<Vec<_>>()
722        .join(",");
723    facts.push((
724        "inheritance_depth".into(),
725        if depth_summary.is_empty() {
726            "empty".into()
727        } else {
728            depth_summary
729        },
730    ));
731
732    match db::doctor_oldest_pending_age_secs(conn) {
733        Ok(Some(age)) => {
734            facts.push(("oldest_pending_age_secs".into(), age.to_string()));
735            if age > crate::SECS_PER_DAY {
736                severity = Severity::Critical;
737                note = Some(format!(
738                    "oldest pending action is {age}s old (>{} threshold = 24h)",
739                    crate::SECS_PER_DAY,
740                ));
741            }
742        }
743        Ok(None) => {
744            facts.push(("oldest_pending_age_secs".into(), "queue_empty".into()));
745        }
746        Err(e) => {
747            facts.push(("pending_query_error".into(), e.to_string()));
748        }
749    }
750
751    let pending_count = db::count_pending_actions_by_status(conn, "pending").unwrap_or(0);
752    facts.push(("pending_actions_total".into(), pending_count.to_string()));
753
754    ReportSection {
755        name: "Governance".into(),
756        severity,
757        facts,
758        note,
759    }
760}
761
762fn section_sync(conn: &rusqlite::Connection) -> ReportSection {
763    let mut facts = Vec::new();
764    let mut severity = Severity::Info;
765    let mut note: Option<String> = None;
766
767    let peer_count: i64 = conn
768        .query_row("SELECT COUNT(*) FROM sync_state", [], |r| r.get(0))
769        .unwrap_or(0);
770    facts.push(("peer_count".into(), peer_count.to_string()));
771
772    if peer_count == 0 {
773        facts.push((
774            FACT_MAX_SKEW_SECS.into(),
775            "not_observed (no peers registered)".into(),
776        ));
777        return ReportSection {
778            name: "Sync".into(),
779            severity: Severity::NotAvailable,
780            facts,
781            note: Some("no sync_state rows — single-node deployment or T3+ not yet enabled".into()),
782        };
783    }
784
785    match db::doctor_max_sync_skew_secs(conn) {
786        Ok(Some(skew)) => {
787            facts.push((FACT_MAX_SKEW_SECS.into(), skew.to_string()));
788            if skew > 600 {
789                severity = Severity::Critical;
790                note = Some(format!(
791                    "max sync skew is {skew}s (>600s threshold) — peer mesh is drifting"
792                ));
793            }
794        }
795        Ok(None) => {
796            facts.push((FACT_MAX_SKEW_SECS.into(), "not_observed".into()));
797        }
798        Err(e) => {
799            facts.push(("sync_query_error".into(), e.to_string()));
800        }
801    }
802
803    ReportSection {
804        name: "Sync".into(),
805        severity,
806        facts,
807        note,
808    }
809}
810
811fn section_webhook(conn: &rusqlite::Connection) -> ReportSection {
812    let mut facts = Vec::new();
813    let mut severity = Severity::Info;
814    let mut note: Option<String> = None;
815
816    let sub_count = db::count_subscriptions(conn).unwrap_or(0);
817    facts.push(("subscription_count".into(), sub_count.to_string()));
818
819    let (dispatched, failed) = db::doctor_webhook_delivery_totals(conn).unwrap_or((0, 0));
820    facts.push(("dispatched_total".into(), dispatched.to_string()));
821    facts.push(("failed_total".into(), failed.to_string()));
822
823    if dispatched > 0 {
824        let success_rate = ((dispatched.saturating_sub(failed)) as f64 / dispatched as f64) * 100.0;
825        facts.push(("success_rate_pct".into(), format!("{success_rate:.2}")));
826        // 95% lifetime success threshold. P5 will refine this to a
827        // rolling-1h window when the dispatch table grows a timestamp
828        // log; for now we use the lifetime totals already present in
829        // `subscriptions.dispatch_count` / `failure_count`.
830        if success_rate < 95.0 {
831            severity = Severity::Warning;
832            note = Some(format!(
833                "lifetime delivery success {success_rate:.2}% < 95% threshold"
834            ));
835        }
836    } else {
837        facts.push(("success_rate_pct".into(), "no_deliveries_yet".into()));
838    }
839
840    ReportSection {
841        name: "Webhook".into(),
842        severity,
843        facts,
844        note,
845    }
846}
847
848fn section_capabilities_local() -> ReportSection {
849    // The local doctor doesn't construct a TierConfig (would require
850    // loading user config). Surface the capability state via the remote
851    // mode against `--remote http://localhost:9077` instead. This local
852    // section just documents the gap.
853    ReportSection {
854        name: "Capabilities".into(),
855        severity: Severity::NotAvailable,
856        facts: vec![(
857            field_names::CAPABILITIES.into(),
858            "use --remote <url> to query the live capabilities endpoint".into(),
859        )],
860        note: None,
861    }
862}
863
864/// v0.7.x (#1146) — LLM reachability probe.
865///
866/// Resolves the canonical LLM configuration via
867/// [`crate::config::AppConfig::resolve_llm`] (the same path used by
868/// MCP, HTTP daemon, atomise, curator, and the boot banner) and
869/// fires a lightweight HTTP probe at the resolved `base_url`. Maps
870/// the response to a Severity per the #1146 spec:
871///
872/// | Status   | HTTP outcomes                          |
873/// |----------|----------------------------------------|
874/// | INFO     | 200 (vendor reachable + auth OK)       |
875/// | WARN     | 401 / 403 (auth issue; URL reachable)  |
876/// | WARN     | 429 (rate-limited; reachable)          |
877/// | WARN     | 5xx (vendor outage; reachable)         |
878/// | CRIT     | 4xx other (likely wrong base_url)      |
879/// | CRIT     | network/DNS/connect-refused/TLS error  |
880///
881/// Probe endpoint per backend:
882/// - `ollama` → `GET <base_url>/api/tags` (no auth)
883/// - any OpenAI-compatible → `GET <base_url>/models` (Bearer auth)
884///
885/// The section's `facts` carry the resolver's full provenance
886/// (`backend`, `model`, `base_url`, `config_source`, `key_source`)
887/// plus the HTTP status code + observed latency, so operators can
888/// see WHERE the wiring came from and WHY the probe lands where it
889/// does.
890fn section_llm_reachability_1146() -> ReportSection {
891    use crate::config::{AppConfig, ConfigSource, KeySource};
892
893    let app_config = AppConfig::load();
894    let resolved = app_config.resolve_llm(None, None, None);
895
896    let mut facts = vec![
897        ("backend".into(), resolved.backend.clone()),
898        ("model".into(), resolved.model.clone()),
899        ("base_url".into(), resolved.base_url.clone()),
900        ("config_source".into(), resolved.source.as_str().to_string()),
901        (
902            field_names::KEY_SOURCE.into(),
903            resolved.api_key_source.as_str().to_string(),
904        ),
905    ];
906
907    // If the key resolution surfaced an error during resolve (file
908    // perms / missing env / etc.), call it out — but still try the
909    // probe so the operator sees if the URL is at least reachable.
910    if let KeySource::Error(reason) = &resolved.api_key_source {
911        facts.push(("key_error".into(), reason.clone()));
912    }
913
914    // Compiled-default — operator has no LLM configuration anywhere
915    // (a fresh install or a keyword-tier-only deployment). This is a
916    // legitimate state, not a misconfiguration: emit INFO with a
917    // pointer at how to enable LLM features rather than WARN (which
918    // would break the "fresh-DB doctor report = all INFO" invariant
919    // pinned by `tests/doctor_cli.rs::doctor_reports_clean_on_fresh_db`).
920    if matches!(resolved.source, ConfigSource::CompiledDefault) {
921        return ReportSection {
922            name: SECTION_LLM_REACHABILITY.into(),
923            severity: Severity::Info,
924            facts,
925            note: Some(
926                "no operator LLM configuration found (CLI / env / [llm] section / \
927                 legacy flat fields all absent); LLM-powered features will be \
928                 inactive. To enable, set AI_MEMORY_LLM_BACKEND in the process \
929                 env or write a [llm] section in config.toml. See \
930                 docs/CONFIG_SCHEMA.md for the canonical schema."
931                    .into(),
932            ),
933        };
934    }
935
936    // Build the probe URL.
937    let (probe_url, bearer) = if resolved.is_ollama_native() {
938        (crate::llm::ollama_tags_url(&resolved.base_url), None)
939    } else {
940        (
941            format!("{}/models", resolved.base_url),
942            resolved.api_key().map(str::to_string),
943        )
944    };
945    facts.push(("probe_url".into(), probe_url.clone()));
946
947    let started = std::time::Instant::now();
948    let client = match reqwest::blocking::Client::builder()
949        .timeout(std::time::Duration::from_secs(5))
950        .connect_timeout(std::time::Duration::from_secs(5))
951        .build()
952    {
953        Ok(c) => c,
954        Err(e) => {
955            facts.push((
956                "error".into(),
957                format!("{MSG_HTTP_CLIENT_BUILD_FAILED}: {e}"),
958            ));
959            return ReportSection {
960                name: SECTION_LLM_REACHABILITY.into(),
961                severity: Severity::Critical,
962                facts,
963                note: Some("could not build HTTP client for probe".into()),
964            };
965        }
966    };
967
968    let mut req = client.get(&probe_url);
969    if let Some(k) = bearer {
970        req = req.bearer_auth(k);
971    }
972
973    let (severity, note) = match req.send() {
974        Ok(resp) => {
975            let status = resp.status();
976            let elapsed_ms = started.elapsed().as_millis();
977            facts.push(("http_status".into(), status.as_u16().to_string()));
978            facts.push((field_names::LATENCY_MS.into(), elapsed_ms.to_string()));
979
980            if status.is_success() {
981                (Severity::Info, None)
982            } else if status.as_u16() == 401 || status.as_u16() == 403 {
983                (
984                    Severity::Warning,
985                    Some(format!(
986                        "auth failed (status {}); URL is reachable but the \
987                         resolved API key was rejected — check [llm].api_key_env / \
988                         [llm].api_key_file / process env",
989                        status.as_u16()
990                    )),
991                )
992            } else if status.as_u16() == 429 {
993                (
994                    Severity::Warning,
995                    Some("rate-limited (status 429); vendor reachable but throttling".into()),
996                )
997            } else if status.is_server_error() {
998                (
999                    Severity::Warning,
1000                    Some(format!(
1001                        "vendor 5xx (status {}); reachable but currently degraded",
1002                        status.as_u16()
1003                    )),
1004                )
1005            } else {
1006                (
1007                    Severity::Critical,
1008                    Some(format!(
1009                        "unexpected status {} from {} — verify base_url + endpoint shape",
1010                        status.as_u16(),
1011                        probe_url
1012                    )),
1013                )
1014            }
1015        }
1016        Err(e) => {
1017            let elapsed_ms = started.elapsed().as_millis();
1018            facts.push((field_names::LATENCY_MS.into(), elapsed_ms.to_string()));
1019            facts.push(("error".into(), e.to_string()));
1020            let kind = if e.is_timeout() {
1021                "timeout"
1022            } else if e.is_connect() {
1023                "connect"
1024            } else {
1025                "transport"
1026            };
1027            (
1028                Severity::Critical,
1029                Some(format!(
1030                    "network/{kind} error contacting {probe_url} — verify \
1031                     base_url and connectivity"
1032                )),
1033            )
1034        }
1035    };
1036
1037    ReportSection {
1038        name: SECTION_LLM_REACHABILITY.into(),
1039        severity,
1040        facts,
1041        note,
1042    }
1043}
1044
1045/// #1598 — should the operator GPU-policy WARN fire? Pure predicate
1046/// (unit-testable without probing the host): the warn applies when the
1047/// resolved embedding backend is the local Ollama wire shape on a host
1048/// with no compatible GPU — operator policy prefers API embeddings on
1049/// CPU-only nodes.
1050fn gpu_policy_warn_applicable(backend: &str, gpu_detected: bool) -> bool {
1051    !crate::config::is_api_embed_backend(backend) && !gpu_detected
1052}
1053
1054/// #1598 — best-effort NVIDIA GPU detection: `nvidia-smi -L` on PATH
1055/// returning success. Any failure (binary missing, driver absent,
1056/// non-zero exit) is treated as no-GPU. Deliberately simple — the
1057/// GPU-policy WARN is advisory, not load-bearing.
1058fn nvidia_gpu_detected() -> bool {
1059    std::process::Command::new("nvidia-smi")
1060        .arg("-L")
1061        .output()
1062        .map(|out| out.status.success())
1063        .unwrap_or(false)
1064}
1065
1066/// #1598 — Embeddings Reachability section. Mirrors
1067/// [`section_llm_reachability_1146`] for the embedding endpoint:
1068///
1069/// Resolves the canonical embeddings configuration via
1070/// [`crate::config::AppConfig::resolve_embeddings`] (the same ladder
1071/// the MCP stdio init + daemon `build_embedder` consume per #1598)
1072/// and fires a lightweight probe at the resolved URL:
1073///
1074/// - `ollama` backend → `GET <url>/api/tags` (no auth)
1075/// - API backends → `POST <url>/embeddings` with a 1-char input +
1076///   the resolved Bearer key
1077///
1078/// Severity mapping matches the LLM section: INFO on 2xx; WARN on
1079/// 401/403/429/5xx (reachable but degraded/auth issue); CRIT on
1080/// other 4xx / network / DNS errors. The section facts carry the
1081/// resolver's full provenance (backend / model / base_url /
1082/// config_source / key_source — NEVER the key itself).
1083///
1084/// Additionally emits the operator GPU-policy WARN
1085/// ([`gpu_policy_warn_applicable`]) when the resolved backend is
1086/// `ollama` on a host with no detectable NVIDIA GPU.
1087fn section_embeddings_reachability_1598() -> ReportSection {
1088    use crate::config::{AppConfig, ConfigSource, KeySource};
1089
1090    let app_config = AppConfig::load();
1091    let resolved = app_config.resolve_embeddings();
1092
1093    let mut facts = vec![
1094        ("backend".into(), resolved.backend.clone()),
1095        ("model".into(), resolved.model.clone()),
1096        ("base_url".into(), resolved.url.clone()),
1097        ("config_source".into(), resolved.source.as_str().to_string()),
1098        (
1099            field_names::KEY_SOURCE.into(),
1100            resolved.key_source.as_str().to_string(),
1101        ),
1102    ];
1103
1104    // If the key resolution surfaced an error during resolve (file
1105    // perms / missing env / etc.), call it out — but still try the
1106    // probe so the operator sees if the URL is at least reachable.
1107    if let KeySource::Error(reason) = &resolved.key_source {
1108        facts.push(("key_error".into(), reason.clone()));
1109    }
1110
1111    // Compiled-default — operator has no embeddings configuration
1112    // anywhere (a fresh install; the tier preset governs). Legitimate
1113    // state, not a misconfiguration: emit INFO without probing so the
1114    // "fresh-DB doctor report = all INFO" invariant holds (mirrors
1115    // the LLM section's early return).
1116    if matches!(resolved.source, ConfigSource::CompiledDefault) {
1117        return ReportSection {
1118            name: SECTION_EMBEDDINGS_REACHABILITY.into(),
1119            severity: Severity::Info,
1120            facts,
1121            note: Some(
1122                "no operator embeddings configuration found (env / [embeddings] \
1123                 section / legacy flat fields all absent); the tier preset \
1124                 governs the embedder. To wire an API embedding backend, set \
1125                 AI_MEMORY_EMBED_BACKEND or write an [embeddings] section in \
1126                 config.toml (#1598)."
1127                    .into(),
1128            ),
1129        };
1130    }
1131
1132    let is_api = crate::config::is_api_embed_backend(&resolved.backend);
1133
1134    let started = std::time::Instant::now();
1135    let client = match reqwest::blocking::Client::builder()
1136        .timeout(std::time::Duration::from_secs(5))
1137        .connect_timeout(std::time::Duration::from_secs(5))
1138        .build()
1139    {
1140        Ok(c) => c,
1141        Err(e) => {
1142            facts.push((
1143                "error".into(),
1144                format!("{MSG_HTTP_CLIENT_BUILD_FAILED}: {e}"),
1145            ));
1146            return ReportSection {
1147                name: SECTION_EMBEDDINGS_REACHABILITY.into(),
1148                severity: Severity::Critical,
1149                facts,
1150                note: Some("could not build HTTP client for probe".into()),
1151            };
1152        }
1153    };
1154
1155    // Build the probe request: a no-auth model listing for the
1156    // Ollama wire shape, a minimal 1-char embed for API backends.
1157    let (probe_url, req) = if is_api {
1158        let url = format!(
1159            "{}{}",
1160            resolved.url,
1161            crate::llm::OPENAI_COMPAT_EMBEDDINGS_PATH
1162        );
1163        let mut req = client
1164            .post(&url)
1165            .json(&serde_json::json!({ "model": resolved.model, "input": "a" }));
1166        if let Some(key) = resolved.api_key() {
1167            req = req.bearer_auth(key);
1168        }
1169        (url, req)
1170    } else {
1171        let url = crate::llm::ollama_tags_url(&resolved.url);
1172        let req = client.get(&url);
1173        (url, req)
1174    };
1175    facts.push(("probe_url".into(), probe_url.clone()));
1176
1177    let (mut severity, mut note) = match req.send() {
1178        Ok(resp) => {
1179            let status = resp.status();
1180            let elapsed_ms = started.elapsed().as_millis();
1181            facts.push(("http_status".into(), status.as_u16().to_string()));
1182            facts.push((field_names::LATENCY_MS.into(), elapsed_ms.to_string()));
1183
1184            if status.is_success() {
1185                (Severity::Info, None)
1186            } else if status.as_u16() == 401 || status.as_u16() == 403 {
1187                (
1188                    Severity::Warning,
1189                    Some(format!(
1190                        "auth failed (status {}); URL is reachable but the \
1191                         resolved embedding API key was rejected — check \
1192                         [embeddings].api_key_env / [embeddings].api_key_file / process env",
1193                        status.as_u16()
1194                    )),
1195                )
1196            } else if status.as_u16() == 429 {
1197                (
1198                    Severity::Warning,
1199                    Some("rate-limited (status 429); vendor reachable but throttling".into()),
1200                )
1201            } else if status.is_server_error() {
1202                (
1203                    Severity::Warning,
1204                    Some(format!(
1205                        "vendor 5xx (status {}); reachable but currently degraded",
1206                        status.as_u16()
1207                    )),
1208                )
1209            } else {
1210                (
1211                    Severity::Critical,
1212                    Some(format!(
1213                        "unexpected status {} from {} — verify base_url + endpoint shape",
1214                        status.as_u16(),
1215                        probe_url
1216                    )),
1217                )
1218            }
1219        }
1220        Err(e) => {
1221            let elapsed_ms = started.elapsed().as_millis();
1222            facts.push((field_names::LATENCY_MS.into(), elapsed_ms.to_string()));
1223            facts.push(("error".into(), e.to_string()));
1224            let kind = if e.is_timeout() {
1225                "timeout"
1226            } else if e.is_connect() {
1227                "connect"
1228            } else {
1229                "transport"
1230            };
1231            (
1232                Severity::Critical,
1233                Some(format!(
1234                    "network/{kind} error contacting {probe_url} — verify \
1235                     base_url and connectivity"
1236                )),
1237            )
1238        }
1239    };
1240
1241    // Operator GPU-policy WARN (#1598): local-Ollama embeddings on a
1242    // CPU-only host — operator policy prefers API embeddings there.
1243    if gpu_policy_warn_applicable(&resolved.backend, nvidia_gpu_detected()) {
1244        severity = severity_max(severity, Severity::Warning);
1245        let gpu_note = format!(
1246            "embeddings backend '{}' on a host with no compatible GPU — \
1247             operator policy prefers API embeddings on CPU-only nodes (#1598)",
1248            resolved.backend
1249        );
1250        facts.push(("gpu_policy".into(), gpu_note.clone()));
1251        note = Some(match note {
1252            Some(existing) => format!("{existing}; {gpu_note}"),
1253            None => gpu_note,
1254        });
1255    }
1256
1257    ReportSection {
1258        name: SECTION_EMBEDDINGS_REACHABILITY.into(),
1259        severity,
1260        facts,
1261        note,
1262    }
1263}
1264
1265/// L1-4 — Reflection Health section.
1266///
1267/// Reports:
1268/// - depth distribution per namespace (depth-0 / depth-1 / depth-2 / depth-3+)
1269/// - reflection totals last 24h, 7d, all-time per namespace
1270/// - depth-limit refusals in the last 24h (from `signed_events`)
1271/// - average + max reflection chain depth per namespace (informational)
1272///
1273/// Severity rules:
1274/// - INFO: any reflection activity (at least one reflected memory exists)
1275/// - WARN: depth-limit refusals > 0 last 24h
1276/// - WARN: any namespace where `max_depth` is within 1 of the compiled
1277///   default cap (max_reflection_depth = 3, i.e. max_depth >= 2)
1278///
1279/// An empty namespace set renders as INFO with a "no reflections" note.
1280fn section_reflection_health(conn: &rusqlite::Connection) -> ReportSection {
1281    let mut facts = Vec::new();
1282    let mut severity = Severity::Info;
1283    let mut notes: Vec<String> = Vec::new();
1284
1285    // ── depth-distribution per namespace ─────────────────────────────
1286    let dist_rows = db::doctor_reflection_depth_distribution(conn).unwrap_or_default();
1287
1288    if dist_rows.is_empty() {
1289        facts.push(("reflections_observed".into(), "none".into()));
1290    } else {
1291        // Per-namespace breakdown.
1292        for row in &dist_rows {
1293            facts.push((
1294                format!("ns::{}::dist", row.namespace),
1295                format!(
1296                    "depth-0={} depth-1={} depth-2={} depth-3+={} avg={:.2} max={}",
1297                    row.depth0,
1298                    row.depth1,
1299                    row.depth2,
1300                    row.depth3_plus,
1301                    row.avg_depth,
1302                    row.max_depth
1303                ),
1304            ));
1305            // WARN when max_depth approaches the compiled cap (cap=3, warn at >=2).
1306            // The cap value is the `GovernancePolicy` compiled-in default; namespaces
1307            // with a custom cap resolved via governance are out of scope here (we'd
1308            // need to query every namespace's policy chain, which is expensive).
1309            const WARN_DEPTH_THRESHOLD: i64 = 2;
1310            if row.max_depth >= WARN_DEPTH_THRESHOLD {
1311                severity = severity_max(severity, Severity::Warning);
1312                notes.push(format!(
1313                    "namespace '{}' max_depth={} approaches default cap (max_reflection_depth=3)",
1314                    row.namespace, row.max_depth
1315                ));
1316            }
1317        }
1318    }
1319
1320    // ── per-namespace totals (24h / 7d / all-time) ───────────────────
1321    let totals = db::doctor_reflection_totals_by_namespace(conn).unwrap_or_default();
1322    for (ns, last_24h, last_7d, all_time) in &totals {
1323        facts.push((
1324            format!("ns::{}::totals", ns),
1325            format!("24h={last_24h} 7d={last_7d} all_time={all_time}"),
1326        ));
1327    }
1328
1329    // ── depth-limit refusals last 24h ────────────────────────────────
1330    let last_day_cutoff = (chrono::Utc::now() - chrono::Duration::hours(24)).to_rfc3339();
1331    let refusals_24h =
1332        db::doctor_reflection_depth_exceeded_count(conn, &last_day_cutoff).unwrap_or(0);
1333    facts.push(("depth_limit_refusals_24h".into(), refusals_24h.to_string()));
1334
1335    if refusals_24h > 0 {
1336        severity = severity_max(severity, Severity::Warning);
1337        notes.push(format!(
1338            "{refusals_24h} depth-limit refusal(s) in the last 24h \
1339             (event_type='reflection.depth_exceeded' in signed_events)"
1340        ));
1341    }
1342
1343    // All-time refusals as an informational counter.
1344    let refusals_all =
1345        db::doctor_reflection_depth_exceeded_count(conn, "1970-01-01T00:00:00Z").unwrap_or(0);
1346    facts.push((
1347        "depth_limit_refusals_all_time".into(),
1348        refusals_all.to_string(),
1349    ));
1350
1351    let note = if notes.is_empty() {
1352        None
1353    } else {
1354        Some(notes.join("; "))
1355    };
1356
1357    ReportSection {
1358        name: "Reflection Health".into(),
1359        severity,
1360        facts,
1361        note,
1362    }
1363}
1364
1365/// Return the higher-severity value of `a` and `b`.
1366/// Defined pub(super) so the reflection-health helpers in this module
1367/// can share the ordering logic without duplicating the `rank` table.
1368pub(super) fn severity_max(a: Severity, b: Severity) -> Severity {
1369    if Report::rank(b) > Report::rank(a) {
1370        b
1371    } else {
1372        a
1373    }
1374}
1375
1376// ---------------------------------------------------------------------------
1377// Remote (--remote) mode
1378// ---------------------------------------------------------------------------
1379
1380fn run_remote(url: &str, db_path: &Path) -> Report {
1381    let mut sections = Vec::with_capacity(2);
1382
1383    let base = url.trim_end_matches('/');
1384    let cap_url = format!("{base}{}", crate::handlers::routes::CAPABILITIES);
1385    let stats_url = format!("{base}{}", crate::handlers::routes::STATS);
1386
1387    sections.push(section_capabilities_remote(&cap_url));
1388    sections.push(section_recall_remote(&cap_url));
1389    sections.push(section_storage_remote(&stats_url));
1390    sections.push(ReportSection {
1391        name: "Index".into(),
1392        severity: Severity::NotAvailable,
1393        facts: vec![("hint".into(), MSG_RAW_SQL_DB_MODE.into())],
1394        note: None,
1395    });
1396    sections.push(ReportSection {
1397        name: "Governance".into(),
1398        severity: Severity::NotAvailable,
1399        facts: vec![("hint".into(), MSG_RAW_SQL_DB_MODE.into())],
1400        note: None,
1401    });
1402    sections.push(ReportSection {
1403        name: "Sync".into(),
1404        severity: Severity::NotAvailable,
1405        facts: vec![("hint".into(), MSG_RAW_SQL_DB_MODE.into())],
1406        note: None,
1407    });
1408    sections.push(ReportSection {
1409        name: "Webhook".into(),
1410        severity: Severity::NotAvailable,
1411        facts: vec![("hint".into(), MSG_RAW_SQL_DB_MODE.into())],
1412        note: None,
1413    });
1414
1415    Report {
1416        mode: "remote".into(),
1417        source: format!("{base} (local db reference: {})", db_path.display()),
1418        generated_at: chrono::Utc::now().to_rfc3339(),
1419        sections,
1420        overall: Severity::Info,
1421    }
1422}
1423
1424/// Fetch a JSON document from `url` with a short timeout. Returns `Err`
1425/// on transport failure or non-2xx status.
1426fn http_get_json(url: &str) -> Result<Value> {
1427    let client = reqwest::blocking::Client::builder()
1428        .timeout(Duration::from_secs(5))
1429        .build()
1430        .context("constructing HTTP client")?;
1431    let resp = client.get(url).send().context("HTTP GET")?;
1432    let status = resp.status();
1433    if !status.is_success() {
1434        anyhow::bail!("HTTP {status} from {url}");
1435    }
1436    resp.json::<Value>().context("decoding JSON response")
1437}
1438
1439fn section_capabilities_remote(url: &str) -> ReportSection {
1440    let mut facts = Vec::new();
1441    let mut severity = Severity::Info;
1442    let mut note: Option<String> = None;
1443
1444    match http_get_json(url) {
1445        Ok(v) => {
1446            // schema_version: "1" (legacy v0.6.3) or "2" (post-P1).
1447            let schema = v
1448                .get(field_names::SCHEMA_VERSION)
1449                .and_then(Value::as_str)
1450                .unwrap_or("unknown");
1451            facts.push((field_names::SCHEMA_VERSION.into(), schema.to_string()));
1452
1453            // P1 v2 fields — best-effort lookup. The legacy v1 shape
1454            // doesn't carry these; we render the missing ones as
1455            // "not_in_response" rather than failing.
1456            let recall_mode = v
1457                .get("features")
1458                .and_then(|f| f.get(FACT_RECALL_MODE_ACTIVE))
1459                .and_then(Value::as_str)
1460                .unwrap_or(NOT_IN_RESPONSE);
1461            facts.push((FACT_RECALL_MODE_ACTIVE.into(), recall_mode.to_string()));
1462
1463            let reranker = v
1464                .get("features")
1465                .and_then(|f| f.get(FACT_RERANKER_ACTIVE))
1466                .and_then(Value::as_str)
1467                .unwrap_or(NOT_IN_RESPONSE);
1468            facts.push((FACT_RERANKER_ACTIVE.into(), reranker.to_string()));
1469
1470            // Severity hints. recall_mode in {"degraded", "disabled",
1471            // "keyword_only"} bumps to Warning when the tier is supposed
1472            // to support hybrid (semantic / smart / autonomous).
1473            if matches!(recall_mode, "degraded" | "disabled" | "keyword_only") {
1474                let tier = v.get("feature_tier").and_then(Value::as_str).unwrap_or("");
1475                if [
1476                    crate::config::FeatureTier::Semantic.as_str(),
1477                    crate::config::FeatureTier::Smart.as_str(),
1478                    crate::config::FeatureTier::Autonomous.as_str(),
1479                ]
1480                .contains(&tier)
1481                {
1482                    severity = Severity::Warning;
1483                    note = Some(format!(
1484                        "tier={tier} but recall_mode_active={recall_mode} — silent degradation"
1485                    ));
1486                }
1487            }
1488        }
1489        Err(e) => {
1490            severity = Severity::Critical;
1491            facts.push(("error".into(), e.to_string()));
1492            note = Some(format!("could not reach {url}"));
1493        }
1494    }
1495
1496    ReportSection {
1497        name: "Capabilities".into(),
1498        severity,
1499        facts,
1500        note,
1501    }
1502}
1503
1504fn section_recall_remote(cap_url: &str) -> ReportSection {
1505    let mut facts = Vec::new();
1506    let severity = Severity::Info;
1507
1508    if let Ok(v) = http_get_json(cap_url) {
1509        let recall_mode = v
1510            .get("features")
1511            .and_then(|f| f.get(FACT_RECALL_MODE_ACTIVE))
1512            .and_then(Value::as_str)
1513            .unwrap_or(NOT_IN_RESPONSE);
1514        facts.push(("active_recall_mode".into(), recall_mode.to_string()));
1515        let reranker = v
1516            .get("features")
1517            .and_then(|f| f.get(FACT_RERANKER_ACTIVE))
1518            .and_then(Value::as_str)
1519            .unwrap_or(NOT_IN_RESPONSE);
1520        facts.push(("active_reranker".into(), reranker.to_string()));
1521        facts.push((
1522            "recall_mode_distribution".into(),
1523            NOT_OBSERVED_PRE_P3.into(),
1524        ));
1525    } else {
1526        facts.push(("error".into(), "could not fetch capabilities".into()));
1527    }
1528
1529    ReportSection {
1530        name: "Recall".into(),
1531        severity,
1532        facts,
1533        note: None,
1534    }
1535}
1536
1537fn section_storage_remote(stats_url: &str) -> ReportSection {
1538    let mut facts = Vec::new();
1539    let severity = Severity::Info;
1540
1541    match http_get_json(stats_url) {
1542        Ok(v) => {
1543            if let Some(total) = v.get("total").and_then(Value::as_u64) {
1544                facts.push((field_names::TOTAL_MEMORIES.into(), total.to_string()));
1545            }
1546            if let Some(exp) = v.get("expiring_soon").and_then(Value::as_u64) {
1547                facts.push(("expiring_within_1h".into(), exp.to_string()));
1548            }
1549            if let Some(links) = v.get("links_count").and_then(Value::as_u64) {
1550                facts.push(("links".into(), links.to_string()));
1551            }
1552            facts.push((
1553                FACT_DIM_VIOLATIONS.into(),
1554                "not_in_remote_response (P2 surface lands at /api/v1/stats)".into(),
1555            ));
1556        }
1557        Err(e) => {
1558            facts.push(("error".into(), e.to_string()));
1559        }
1560    }
1561
1562    ReportSection {
1563        name: "Storage".into(),
1564        severity,
1565        facts,
1566        note: None,
1567    }
1568}
1569
1570// ---------------------------------------------------------------------------
1571// Text rendering
1572// ---------------------------------------------------------------------------
1573
1574fn render_text(report: &Report, out: &mut CliOutput<'_>) -> Result<()> {
1575    writeln!(out.stdout, "ai-memory doctor — {} mode", report.mode)?;
1576    writeln!(out.stdout, "  source:       {}", report.source)?;
1577    writeln!(out.stdout, "  generated_at: {}", report.generated_at)?;
1578    writeln!(out.stdout, "  overall:      {}", report.overall.label())?;
1579    writeln!(out.stdout)?;
1580    for section in &report.sections {
1581        writeln!(
1582            out.stdout,
1583            "[{}] {}",
1584            section.severity.label(),
1585            section.name
1586        )?;
1587        for (k, v) in &section.facts {
1588            writeln!(out.stdout, "    {k:<32} {v}")?;
1589        }
1590        if let Some(note) = &section.note {
1591            writeln!(out.stdout, "    note: {note}")?;
1592        }
1593        writeln!(out.stdout)?;
1594    }
1595    Ok(())
1596}
1597
1598// ---------------------------------------------------------------------------
1599// Tests (unit-level — full integration tests live in tests/doctor_cli.rs)
1600// ---------------------------------------------------------------------------
1601
1602#[cfg(test)]
1603#[allow(clippy::too_many_lines, clippy::similar_names)]
1604mod tests {
1605    use super::*;
1606    use crate::cli::CliOutput;
1607    use crate::cli::test_utils::{TestEnv, seed_memory};
1608    use rusqlite::params;
1609
1610    // -------------------------------------------------------------------
1611    // Severity / Report helpers (pure, no DB)
1612    // -------------------------------------------------------------------
1613
1614    #[test]
1615    fn severity_rank_orders_critical_highest() {
1616        assert!(Report::rank(Severity::Critical) > Report::rank(Severity::Warning));
1617        assert!(Report::rank(Severity::Warning) > Report::rank(Severity::Info));
1618        assert!(Report::rank(Severity::Info) > Report::rank(Severity::NotAvailable));
1619    }
1620
1621    #[test]
1622    fn severity_label_renders_for_every_variant() {
1623        assert_eq!(Severity::Info.label(), "INFO");
1624        assert_eq!(Severity::Warning.label(), "WARN");
1625        assert_eq!(Severity::Critical.label(), "CRIT");
1626        assert_eq!(Severity::NotAvailable.label(), "N/A ");
1627    }
1628
1629    #[test]
1630    fn severity_serializes_lowercase_and_round_trips() {
1631        // The Serialize derive uses `rename_all = "lowercase"`. We don't
1632        // derive Deserialize, so we round-trip via the JSON Value form.
1633        let s = serde_json::to_value(Severity::Critical).unwrap();
1634        assert_eq!(s, serde_json::Value::String("critical".into()));
1635        let s = serde_json::to_value(Severity::NotAvailable).unwrap();
1636        assert_eq!(s, serde_json::Value::String("notavailable".into()));
1637    }
1638
1639    fn mk_section(name: &str, severity: Severity) -> ReportSection {
1640        ReportSection {
1641            name: name.into(),
1642            severity,
1643            facts: vec![("k".into(), "v".into())],
1644            note: None,
1645        }
1646    }
1647
1648    fn mk_report(sections: Vec<ReportSection>) -> Report {
1649        Report {
1650            mode: "local".into(),
1651            source: ":memory:".into(),
1652            generated_at: "now".into(),
1653            sections,
1654            overall: Severity::Info,
1655        }
1656    }
1657
1658    #[test]
1659    fn compute_overall_picks_critical_when_present() {
1660        let mut r = mk_report(vec![
1661            mk_section("A", Severity::Info),
1662            mk_section("B", Severity::Critical),
1663            mk_section("C", Severity::Warning),
1664        ]);
1665        r.compute_overall();
1666        assert_eq!(r.overall, Severity::Critical);
1667    }
1668
1669    #[test]
1670    fn compute_overall_picks_warning_when_no_critical() {
1671        let mut r = mk_report(vec![
1672            mk_section("A", Severity::Info),
1673            mk_section("B", Severity::Warning),
1674        ]);
1675        r.compute_overall();
1676        assert_eq!(r.overall, Severity::Warning);
1677    }
1678
1679    #[test]
1680    fn compute_overall_picks_info_when_no_warnings_or_critical() {
1681        let mut r = mk_report(vec![
1682            mk_section("A", Severity::NotAvailable),
1683            mk_section("B", Severity::Info),
1684        ]);
1685        r.compute_overall();
1686        assert_eq!(r.overall, Severity::Info);
1687    }
1688
1689    #[test]
1690    fn compute_overall_handles_empty_sections() {
1691        let mut r = mk_report(vec![]);
1692        r.compute_overall();
1693        // unwrap_or fallback path — empty iterator collapses to Info.
1694        assert_eq!(r.overall, Severity::Info);
1695    }
1696
1697    #[test]
1698    fn compute_overall_only_n_a_yields_n_a() {
1699        let mut r = mk_report(vec![
1700            mk_section("A", Severity::NotAvailable),
1701            mk_section("B", Severity::NotAvailable),
1702        ]);
1703        r.compute_overall();
1704        assert_eq!(r.overall, Severity::NotAvailable);
1705    }
1706
1707    // -------------------------------------------------------------------
1708    // ReportSection / Report serde shape
1709    // -------------------------------------------------------------------
1710
1711    #[test]
1712    fn report_section_serializes_with_expected_keys() {
1713        let section = ReportSection {
1714            name: "Storage".into(),
1715            severity: Severity::Warning,
1716            facts: vec![("total".into(), "5".into())],
1717            note: Some("hello".into()),
1718        };
1719        let v = serde_json::to_value(&section).unwrap();
1720        assert_eq!(v["name"], "Storage");
1721        assert_eq!(v["severity"], "warning");
1722        // Facts is a list of 2-tuples encoded as JSON arrays.
1723        assert!(v["facts"].is_array());
1724        assert_eq!(v["facts"][0][0], "total");
1725        assert_eq!(v["facts"][0][1], "5");
1726        assert_eq!(v["note"], "hello");
1727    }
1728
1729    #[test]
1730    fn report_section_skips_note_when_none() {
1731        let section = ReportSection {
1732            name: "Recall".into(),
1733            severity: Severity::Info,
1734            facts: vec![],
1735            note: None,
1736        };
1737        let v = serde_json::to_value(&section).unwrap();
1738        assert!(
1739            v.get("note").is_none(),
1740            "note=None must be skipped per #[serde(skip_serializing_if)]"
1741        );
1742    }
1743
1744    #[test]
1745    fn report_top_level_serialization_has_all_fields() {
1746        let r = mk_report(vec![mk_section("S", Severity::Info)]);
1747        let v = serde_json::to_value(&r).unwrap();
1748        for k in ["mode", "source", "generated_at", "sections", "overall"] {
1749            assert!(v.get(k).is_some(), "expected key {k} in JSON");
1750        }
1751        assert_eq!(v["sections"].as_array().unwrap().len(), 1);
1752    }
1753
1754    // -------------------------------------------------------------------
1755    // Local-DB mode — basic happy path
1756    // -------------------------------------------------------------------
1757
1758    fn run_local_collect(db_path: &Path) -> Report {
1759        let mut report = run_local(db_path);
1760        report.compute_overall();
1761        report
1762    }
1763
1764    fn find<'a>(report: &'a Report, name: &str) -> &'a ReportSection {
1765        report
1766            .sections
1767            .iter()
1768            .find(|s| s.name == name)
1769            .unwrap_or_else(|| panic!("section {name} not found"))
1770    }
1771
1772    fn fact<'a>(section: &'a ReportSection, key: &str) -> &'a str {
1773        section
1774            .facts
1775            .iter()
1776            .find(|(k, _)| k == key)
1777            .map(|(_, v)| v.as_str())
1778            .unwrap_or_else(|| panic!("fact {key} not found in section {}", section.name))
1779    }
1780
1781    #[test]
1782    fn local_run_on_empty_db_produces_ten_sections() {
1783        let env = TestEnv::fresh();
1784        let report = run_local_collect(&env.db_path);
1785        assert_eq!(report.mode, "local");
1786        // L1-4 added "Reflection Health"; #1146 added
1787        // "LLM Reachability (#1146)"; #1598 added
1788        // "Embeddings Reachability (#1598)" — total is now 10.
1789        assert_eq!(report.sections.len(), 10);
1790        let names: Vec<&str> = report.sections.iter().map(|s| s.name.as_str()).collect();
1791        assert_eq!(
1792            names,
1793            vec![
1794                "Storage",
1795                "Index",
1796                "Recall",
1797                "Governance",
1798                "Sync",
1799                "Webhook",
1800                "Capabilities",
1801                "Reflection Health",
1802                "LLM Reachability (#1146)",
1803                "Embeddings Reachability (#1598)",
1804            ]
1805        );
1806    }
1807
1808    // -------------------------------------------------------------------
1809    // #1598 — Embeddings Reachability section
1810    // -------------------------------------------------------------------
1811
1812    #[test]
1813    fn gpu_policy_warn_applies_only_to_local_backend_without_gpu_1598() {
1814        // ollama + no GPU → warn fires.
1815        assert!(gpu_policy_warn_applicable(
1816            crate::llm::BACKEND_OLLAMA,
1817            false
1818        ));
1819        // ollama + GPU present → no warn.
1820        assert!(!gpu_policy_warn_applicable(
1821            crate::llm::BACKEND_OLLAMA,
1822            true
1823        ));
1824        // API backends never trigger the warn, GPU or not.
1825        assert!(!gpu_policy_warn_applicable("openrouter", false));
1826        assert!(!gpu_policy_warn_applicable("openai-compatible", false));
1827        assert!(!gpu_policy_warn_applicable("openrouter", true));
1828    }
1829
1830    #[test]
1831    fn embeddings_reachability_section_present_with_provenance_facts_1598() {
1832        let env = TestEnv::fresh();
1833        let report = run_local_collect(&env.db_path);
1834        let emb = find(&report, SECTION_EMBEDDINGS_REACHABILITY);
1835        // Provenance facts are always present, regardless of whether
1836        // the probe ran (compiled-default short-circuits pre-probe).
1837        for key in [
1838            "backend",
1839            "model",
1840            "base_url",
1841            "config_source",
1842            "key_source",
1843        ] {
1844            assert!(
1845                emb.facts.iter().any(|(k, _)| k == key),
1846                "missing fact {key} in {:?}",
1847                emb.facts
1848            );
1849        }
1850        // The resolved key value itself must NEVER appear as a fact key.
1851        assert!(emb.facts.iter().all(|(k, _)| k != "api_key"));
1852    }
1853
1854    #[test]
1855    fn local_run_empty_db_storage_section_is_info() {
1856        let env = TestEnv::fresh();
1857        let report = run_local_collect(&env.db_path);
1858        let storage = find(&report, "Storage");
1859        assert_eq!(storage.severity, Severity::Info);
1860        assert_eq!(fact(storage, "total_memories"), "0");
1861        // Pre-P2 schema (current release) has no `embedding_dim` column —
1862        // `db::doctor_dim_violations` returns Ok(None), rendered as
1863        // "not_observed (pre-P2 schema)".
1864        let dim = fact(storage, "dim_violations");
1865        assert!(
1866            dim.contains("not_observed") || dim == "0",
1867            "unexpected dim_violations value: {dim}"
1868        );
1869    }
1870
1871    #[test]
1872    fn local_run_with_seeded_memory_reports_total() {
1873        let env = TestEnv::fresh();
1874        seed_memory(&env.db_path, "ns-a", "title-1", "content one");
1875        seed_memory(&env.db_path, "ns-a", "title-2", "content two");
1876        seed_memory(&env.db_path, "ns-b", "title-3", "content three");
1877        let report = run_local_collect(&env.db_path);
1878        let storage = find(&report, "Storage");
1879        assert_eq!(fact(storage, "total_memories"), "3");
1880        // Tier breakdown — seed_memory inserts at tier=mid.
1881        let tier_mid = storage
1882            .facts
1883            .iter()
1884            .find(|(k, _)| k == "tier::mid")
1885            .map(|(_, v)| v.as_str());
1886        assert_eq!(tier_mid, Some("3"));
1887        // Namespace breakdown caps at 10 entries; 2 namespaces fit.
1888        let ns_a = storage
1889            .facts
1890            .iter()
1891            .find(|(k, _)| k == "ns::ns-a")
1892            .map(|(_, v)| v.as_str());
1893        let ns_b = storage
1894            .facts
1895            .iter()
1896            .find(|(k, _)| k == "ns::ns-b")
1897            .map(|(_, v)| v.as_str());
1898        assert_eq!(ns_a, Some("2"));
1899        assert_eq!(ns_b, Some("1"));
1900    }
1901
1902    #[test]
1903    fn local_run_index_section_reports_hnsw_estimate() {
1904        let env = TestEnv::fresh();
1905        seed_memory(&env.db_path, "ns", "t1", "c1");
1906        let report = run_local_collect(&env.db_path);
1907        let index = find(&report, "Index");
1908        // seed_memory does not write an embedding so hnsw_size_estimate=0.
1909        assert_eq!(fact(index, "hnsw_size_estimate"), "0");
1910        // Cold-start estimate is rendered with two decimals.
1911        let cs = fact(index, "cold_start_rebuild_secs_estimate");
1912        assert!(
1913            cs.contains('.'),
1914            "cold_start_secs_estimate should be float-like, got {cs}"
1915        );
1916        assert_eq!(index.severity, Severity::Info);
1917    }
1918
1919    #[test]
1920    fn local_run_recall_section_documents_pre_p3_state() {
1921        let env = TestEnv::fresh();
1922        let report = run_local_collect(&env.db_path);
1923        let recall = find(&report, "Recall");
1924        assert_eq!(recall.severity, Severity::Info);
1925        assert!(fact(recall, "recall_mode_distribution").contains("pre-P3"));
1926        assert!(fact(recall, "reranker_used_distribution").contains("pre-P3"));
1927        // Hint nudges the operator toward --remote for the live feed.
1928        assert!(fact(recall, "hint").contains("--remote"));
1929    }
1930
1931    #[test]
1932    fn local_run_sync_section_n_a_when_no_peers() {
1933        let env = TestEnv::fresh();
1934        let report = run_local_collect(&env.db_path);
1935        let sync = find(&report, "Sync");
1936        // Empty sync_state => NotAvailable + note.
1937        assert_eq!(sync.severity, Severity::NotAvailable);
1938        assert_eq!(fact(sync, "peer_count"), "0");
1939        assert!(sync.note.is_some());
1940    }
1941
1942    #[test]
1943    fn local_run_capabilities_local_section_n_a() {
1944        let env = TestEnv::fresh();
1945        let report = run_local_collect(&env.db_path);
1946        let cap = find(&report, "Capabilities");
1947        assert_eq!(cap.severity, Severity::NotAvailable);
1948        assert!(fact(cap, "capabilities").contains("--remote"));
1949    }
1950
1951    #[test]
1952    fn local_run_governance_section_empty_is_info() {
1953        let env = TestEnv::fresh();
1954        let report = run_local_collect(&env.db_path);
1955        let gov = find(&report, "Governance");
1956        assert_eq!(gov.severity, Severity::Info);
1957        assert_eq!(fact(gov, "namespaces_with_policy"), "0");
1958        assert_eq!(fact(gov, "namespaces_without_policy"), "0");
1959        assert_eq!(fact(gov, "inheritance_depth"), "empty");
1960        assert_eq!(fact(gov, "oldest_pending_age_secs"), "queue_empty");
1961        assert_eq!(fact(gov, "pending_actions_total"), "0");
1962    }
1963
1964    #[test]
1965    fn local_run_webhook_section_empty_no_deliveries() {
1966        let env = TestEnv::fresh();
1967        let report = run_local_collect(&env.db_path);
1968        let wh = find(&report, "Webhook");
1969        assert_eq!(wh.severity, Severity::Info);
1970        assert_eq!(fact(wh, "subscription_count"), "0");
1971        assert_eq!(fact(wh, "dispatched_total"), "0");
1972        assert_eq!(fact(wh, "failed_total"), "0");
1973        assert_eq!(fact(wh, "success_rate_pct"), "no_deliveries_yet");
1974    }
1975
1976    // -------------------------------------------------------------------
1977    // Severity rule cases — DB-backed
1978    // -------------------------------------------------------------------
1979
1980    #[test]
1981    fn governance_section_critical_when_pending_older_than_24h() {
1982        let env = TestEnv::fresh();
1983        // Open the DB once to materialize schema, then write a pending row.
1984        {
1985            let conn = crate::db::open(&env.db_path).unwrap();
1986            let twenty_five_hours_ago =
1987                (chrono::Utc::now() - chrono::Duration::hours(25)).to_rfc3339();
1988            conn.execute(
1989                "INSERT INTO pending_actions \
1990                 (id, action_type, namespace, payload, requested_by, requested_at, status) \
1991                 VALUES ('p1', 'store', 'ns', '{}', 'agent', ?1, 'pending')",
1992                params![twenty_five_hours_ago],
1993            )
1994            .unwrap();
1995        }
1996        let report = run_local_collect(&env.db_path);
1997        let gov = find(&report, "Governance");
1998        assert_eq!(gov.severity, Severity::Critical);
1999        assert!(gov.note.as_ref().unwrap().contains("24h"));
2000        // pending_actions_total reflects the row.
2001        assert_eq!(fact(gov, "pending_actions_total"), "1");
2002        // overall picks the Critical from Governance.
2003        assert_eq!(report.overall, Severity::Critical);
2004    }
2005
2006    #[test]
2007    fn governance_section_info_when_pending_younger_than_24h() {
2008        let env = TestEnv::fresh();
2009        {
2010            let conn = crate::db::open(&env.db_path).unwrap();
2011            let one_hour_ago = (chrono::Utc::now() - chrono::Duration::hours(1)).to_rfc3339();
2012            conn.execute(
2013                "INSERT INTO pending_actions \
2014                 (id, action_type, namespace, payload, requested_by, requested_at, status) \
2015                 VALUES ('p2', 'store', 'ns', '{}', 'agent', ?1, 'pending')",
2016                params![one_hour_ago],
2017            )
2018            .unwrap();
2019        }
2020        let report = run_local_collect(&env.db_path);
2021        let gov = find(&report, "Governance");
2022        // 1h pending — under the 24h threshold; Info, no critical bump.
2023        assert_eq!(gov.severity, Severity::Info);
2024        assert_eq!(fact(gov, "pending_actions_total"), "1");
2025        // The age fact is set to a numeric string, not "queue_empty".
2026        let age_str = fact(gov, "oldest_pending_age_secs");
2027        assert!(
2028            age_str.parse::<i64>().is_ok(),
2029            "expected numeric age, got {age_str}"
2030        );
2031    }
2032
2033    #[test]
2034    fn sync_section_critical_when_skew_exceeds_600s() {
2035        let env = TestEnv::fresh();
2036        {
2037            let conn = crate::db::open(&env.db_path).unwrap();
2038            // last_seen_at = now, last_pulled_at = 1 hour ago → 3600s skew.
2039            let now = chrono::Utc::now();
2040            let now_s = now.to_rfc3339();
2041            let earlier = (now - chrono::Duration::seconds(crate::SECS_PER_HOUR)).to_rfc3339();
2042            conn.execute(
2043                "INSERT INTO sync_state (agent_id, peer_id, last_seen_at, last_pulled_at) \
2044                 VALUES ('me', 'peer-1', ?1, ?2)",
2045                params![now_s, earlier],
2046            )
2047            .unwrap();
2048        }
2049        let report = run_local_collect(&env.db_path);
2050        let sync = find(&report, "Sync");
2051        assert_eq!(sync.severity, Severity::Critical);
2052        assert!(sync.note.as_ref().unwrap().contains("600s"));
2053        assert_eq!(fact(sync, "peer_count"), "1");
2054        assert_eq!(report.overall, Severity::Critical);
2055    }
2056
2057    #[test]
2058    fn sync_section_info_when_skew_under_threshold() {
2059        let env = TestEnv::fresh();
2060        {
2061            let conn = crate::db::open(&env.db_path).unwrap();
2062            let now = chrono::Utc::now();
2063            let now_s = now.to_rfc3339();
2064            let close = (now - chrono::Duration::seconds(60)).to_rfc3339();
2065            conn.execute(
2066                "INSERT INTO sync_state (agent_id, peer_id, last_seen_at, last_pulled_at) \
2067                 VALUES ('me', 'peer-1', ?1, ?2)",
2068                params![now_s, close],
2069            )
2070            .unwrap();
2071        }
2072        let report = run_local_collect(&env.db_path);
2073        let sync = find(&report, "Sync");
2074        assert_eq!(sync.severity, Severity::Info);
2075        // peer_count=1, skew column rendered as a numeric string.
2076        assert_eq!(fact(sync, "peer_count"), "1");
2077        let skew = fact(sync, "max_skew_secs");
2078        assert!(
2079            skew.parse::<i64>().is_ok(),
2080            "expected numeric skew, got {skew}"
2081        );
2082    }
2083
2084    #[test]
2085    fn webhook_section_warning_when_success_rate_below_95() {
2086        let env = TestEnv::fresh();
2087        {
2088            let conn = crate::db::open(&env.db_path).unwrap();
2089            // 100 dispatches, 10 failures = 90% success → < 95% threshold.
2090            let now = chrono::Utc::now().to_rfc3339();
2091            conn.execute(
2092                "INSERT INTO subscriptions \
2093                 (id, url, events, created_at, dispatch_count, failure_count) \
2094                 VALUES ('s1', 'http://example/x', '*', ?1, 100, 10)",
2095                params![now],
2096            )
2097            .unwrap();
2098        }
2099        let report = run_local_collect(&env.db_path);
2100        let wh = find(&report, "Webhook");
2101        assert_eq!(wh.severity, Severity::Warning);
2102        assert!(wh.note.as_ref().unwrap().contains("95%"));
2103        assert_eq!(fact(wh, "subscription_count"), "1");
2104        assert_eq!(fact(wh, "dispatched_total"), "100");
2105        assert_eq!(fact(wh, "failed_total"), "10");
2106        assert_eq!(fact(wh, "success_rate_pct"), "90.00");
2107    }
2108
2109    #[test]
2110    fn webhook_section_info_when_success_rate_at_or_above_95() {
2111        let env = TestEnv::fresh();
2112        {
2113            let conn = crate::db::open(&env.db_path).unwrap();
2114            let now = chrono::Utc::now().to_rfc3339();
2115            // 100 dispatches, 3 failures = 97% success.
2116            conn.execute(
2117                "INSERT INTO subscriptions \
2118                 (id, url, events, created_at, dispatch_count, failure_count) \
2119                 VALUES ('s1', 'http://example/x', '*', ?1, 100, 3)",
2120                params![now],
2121            )
2122            .unwrap();
2123        }
2124        let report = run_local_collect(&env.db_path);
2125        let wh = find(&report, "Webhook");
2126        assert_eq!(wh.severity, Severity::Info);
2127        assert!(wh.note.is_none());
2128        assert_eq!(fact(wh, "success_rate_pct"), "97.00");
2129    }
2130
2131    #[test]
2132    fn governance_section_with_namespace_chain_reports_depths() {
2133        let env = TestEnv::fresh();
2134        {
2135            let conn = crate::db::open(&env.db_path).unwrap();
2136            let now = chrono::Utc::now().to_rfc3339();
2137            for (ns, parent) in [
2138                ("root", None::<&str>),
2139                ("a", Some("root")),
2140                ("a/b", Some("a")),
2141            ] {
2142                conn.execute(
2143                    "INSERT INTO namespace_meta (namespace, parent_namespace, updated_at) \
2144                     VALUES (?1, ?2, ?3)",
2145                    params![ns, parent, now],
2146                )
2147                .unwrap();
2148            }
2149        }
2150        let report = run_local_collect(&env.db_path);
2151        let gov = find(&report, "Governance");
2152        assert_eq!(gov.severity, Severity::Info);
2153        let depth = fact(gov, "inheritance_depth");
2154        assert!(depth.contains("d0=") && depth.contains("d1=") && depth.contains("d2="));
2155        assert_eq!(fact(gov, "namespaces_without_policy"), "3");
2156    }
2157
2158    // -------------------------------------------------------------------
2159    // L1-4 — Reflection Health section tests
2160    // -------------------------------------------------------------------
2161
2162    /// Helper: insert a memory with a specific `reflection_depth` directly.
2163    fn seed_reflection(conn: &rusqlite::Connection, namespace: &str, depth: i32, title: &str) {
2164        let now = chrono::Utc::now().to_rfc3339();
2165        conn.execute(
2166            "INSERT INTO memories \
2167             (id, tier, namespace, title, content, tags, priority, confidence, source, \
2168              access_count, created_at, updated_at, metadata, reflection_depth) \
2169             VALUES (?, 'mid', ?, ?, 'content', '[]', 5, 1.0, 'test', 0, ?, ?, '{}', ?)",
2170            rusqlite::params![
2171                uuid::Uuid::new_v4().to_string(),
2172                namespace,
2173                title,
2174                now,
2175                now,
2176                depth
2177            ],
2178        )
2179        .unwrap();
2180    }
2181
2182    /// Helper: insert a `reflection.depth_exceeded` signed event.
2183    fn seed_depth_exceeded_event(conn: &rusqlite::Connection, timestamp: &str) {
2184        // Route through `append_signed_event` so the cross-row chain
2185        // (v34, #698 V-4 closeout) is populated correctly. The
2186        // helper used a raw INSERT before v34 because there was no
2187        // chain to populate; v34's UNIQUE INDEX on `sequence`
2188        // tolerates the raw NULL-sequence shape but the seeded
2189        // events would not chain-verify, which complicates any
2190        // downstream doctor probe that walks the chain.
2191        let event = crate::signed_events::SignedEvent {
2192            id: uuid::Uuid::new_v4().to_string(),
2193            agent_id: "test-agent".to_string(),
2194            event_type: crate::signed_events::event_types::REFLECTION_DEPTH_EXCEEDED.to_string(),
2195            payload_hash: vec![0xaa],
2196            signature: None,
2197            attest_level: "unsigned".to_string(),
2198            timestamp: timestamp.to_string(),
2199            ..crate::signed_events::SignedEvent::default()
2200        };
2201        crate::signed_events::append_signed_event(conn, &event).unwrap();
2202    }
2203
2204    #[test]
2205    fn reflection_health_section_empty_db_is_info_no_reflections() {
2206        let env = TestEnv::fresh();
2207        let report = run_local_collect(&env.db_path);
2208        let rh = find(&report, "Reflection Health");
2209        assert_eq!(rh.severity, Severity::Info);
2210        assert_eq!(fact(rh, "reflections_observed"), "none");
2211        assert_eq!(fact(rh, "depth_limit_refusals_24h"), "0");
2212        assert_eq!(fact(rh, "depth_limit_refusals_all_time"), "0");
2213    }
2214
2215    #[test]
2216    fn reflection_health_section_depth_distribution_counts() {
2217        let env = TestEnv::fresh();
2218        {
2219            let conn = crate::db::open(&env.db_path).unwrap();
2220            // ns-alpha: 3 depth-0, 2 depth-1, 1 depth-2
2221            seed_reflection(&conn, "ns-alpha", 0, "base-1");
2222            seed_reflection(&conn, "ns-alpha", 0, "base-2");
2223            seed_reflection(&conn, "ns-alpha", 0, "base-3");
2224            seed_reflection(&conn, "ns-alpha", 1, "refl-1");
2225            seed_reflection(&conn, "ns-alpha", 1, "refl-2");
2226            seed_reflection(&conn, "ns-alpha", 2, "refl-3");
2227            // ns-beta: 1 depth-1
2228            seed_reflection(&conn, "ns-beta", 1, "beta-refl-1");
2229        }
2230        let report = run_local_collect(&env.db_path);
2231        let rh = find(&report, "Reflection Health");
2232        // Both namespaces have reflected memories, so no "none" entry.
2233        assert!(
2234            rh.facts.iter().all(|(k, _)| k != "reflections_observed"),
2235            "reflections_observed key should be absent when reflections exist"
2236        );
2237        // ns-alpha dist fact should be present.
2238        let alpha_dist = rh
2239            .facts
2240            .iter()
2241            .find(|(k, _)| k == "ns::ns-alpha::dist")
2242            .map(|(_, v)| v.as_str());
2243        assert!(alpha_dist.is_some(), "ns::ns-alpha::dist fact missing");
2244        let alpha_str = alpha_dist.unwrap();
2245        assert!(
2246            alpha_str.contains("depth-0=3"),
2247            "expected depth-0=3 in '{alpha_str}'"
2248        );
2249        assert!(
2250            alpha_str.contains("depth-1=2"),
2251            "expected depth-1=2 in '{alpha_str}'"
2252        );
2253        assert!(
2254            alpha_str.contains("depth-2=1"),
2255            "expected depth-2=1 in '{alpha_str}'"
2256        );
2257        assert!(
2258            alpha_str.contains("depth-3+=0"),
2259            "expected depth-3+=0 in '{alpha_str}'"
2260        );
2261        // ns-beta dist fact.
2262        let beta_dist = rh
2263            .facts
2264            .iter()
2265            .find(|(k, _)| k == "ns::ns-beta::dist")
2266            .map(|(_, v)| v.as_str());
2267        assert!(beta_dist.is_some(), "ns::ns-beta::dist fact missing");
2268        let beta_str = beta_dist.unwrap();
2269        assert!(
2270            beta_str.contains("depth-1=1"),
2271            "expected depth-1=1 in '{beta_str}'"
2272        );
2273    }
2274
2275    #[test]
2276    fn reflection_health_warn_when_max_depth_approaches_cap() {
2277        // max_depth = 2 triggers WARN (cap=3, warn threshold >=2).
2278        let env = TestEnv::fresh();
2279        {
2280            let conn = crate::db::open(&env.db_path).unwrap();
2281            seed_reflection(&conn, "deep-ns", 2, "depth2-refl");
2282        }
2283        let report = run_local_collect(&env.db_path);
2284        let rh = find(&report, "Reflection Health");
2285        assert_eq!(rh.severity, Severity::Warning);
2286        let note = rh
2287            .note
2288            .as_ref()
2289            .expect("expected a note when depth approaches cap");
2290        assert!(
2291            note.contains("deep-ns"),
2292            "note should name the namespace, got: {note}"
2293        );
2294        assert!(note.contains("cap"), "note should mention cap, got: {note}");
2295    }
2296
2297    #[test]
2298    fn reflection_health_warn_on_depth_limit_refusals_24h() {
2299        let env = TestEnv::fresh();
2300        {
2301            let conn = crate::db::open(&env.db_path).unwrap();
2302            // One refusal 1h ago → within 24h window.
2303            let one_hour_ago = (chrono::Utc::now() - chrono::Duration::hours(1)).to_rfc3339();
2304            seed_depth_exceeded_event(&conn, &one_hour_ago);
2305        }
2306        let report = run_local_collect(&env.db_path);
2307        let rh = find(&report, "Reflection Health");
2308        assert_eq!(rh.severity, Severity::Warning);
2309        assert_eq!(fact(rh, "depth_limit_refusals_24h"), "1");
2310        assert_eq!(fact(rh, "depth_limit_refusals_all_time"), "1");
2311        let note = rh.note.as_ref().expect("expected note on refusals");
2312        assert!(
2313            note.contains("refusal"),
2314            "note should mention refusal, got: {note}"
2315        );
2316    }
2317
2318    #[test]
2319    fn reflection_health_old_refusals_do_not_trigger_24h_warn() {
2320        let env = TestEnv::fresh();
2321        {
2322            let conn = crate::db::open(&env.db_path).unwrap();
2323            // Refusal 48h ago — outside 24h window.
2324            let old = (chrono::Utc::now() - chrono::Duration::hours(48)).to_rfc3339();
2325            seed_depth_exceeded_event(&conn, &old);
2326        }
2327        let report = run_local_collect(&env.db_path);
2328        let rh = find(&report, "Reflection Health");
2329        // 24h count should be 0, no WARN.
2330        assert_eq!(fact(rh, "depth_limit_refusals_24h"), "0");
2331        // All-time counter still sees it.
2332        assert_eq!(fact(rh, "depth_limit_refusals_all_time"), "1");
2333        // No 24h refusal → severity stays Info (unless depth approaches cap).
2334        assert_eq!(rh.severity, Severity::Info);
2335    }
2336
2337    #[test]
2338    fn reflection_health_totals_per_namespace() {
2339        let env = TestEnv::fresh();
2340        let recent = (chrono::Utc::now() - chrono::Duration::minutes(30)).to_rfc3339();
2341        let old = (chrono::Utc::now() - chrono::Duration::days(10)).to_rfc3339();
2342        {
2343            let conn = crate::db::open(&env.db_path).unwrap();
2344            // ns-new: one reflection created 30 min ago (24h + 7d + all_time)
2345            conn.execute(
2346                "INSERT INTO memories \
2347                 (id, tier, namespace, title, content, tags, priority, confidence, source, \
2348                  access_count, created_at, updated_at, metadata, reflection_depth) \
2349                 VALUES (?, 'mid', 'ns-new', 'new-refl', 'c', '[]', 5, 1.0, 'test', 0, ?, ?, '{}', 1)",
2350                rusqlite::params![uuid::Uuid::new_v4().to_string(), recent, recent],
2351            )
2352            .unwrap();
2353            // ns-old: one reflection created 10 days ago (all_time only)
2354            conn.execute(
2355                "INSERT INTO memories \
2356                 (id, tier, namespace, title, content, tags, priority, confidence, source, \
2357                  access_count, created_at, updated_at, metadata, reflection_depth) \
2358                 VALUES (?, 'mid', 'ns-old', 'old-refl', 'c', '[]', 5, 1.0, 'test', 0, ?, ?, '{}', 1)",
2359                rusqlite::params![uuid::Uuid::new_v4().to_string(), old, old],
2360            )
2361            .unwrap();
2362        }
2363        let report = run_local_collect(&env.db_path);
2364        let rh = find(&report, "Reflection Health");
2365        // ns-new: 24h=1, 7d=1, all_time=1
2366        let new_totals = rh
2367            .facts
2368            .iter()
2369            .find(|(k, _)| k == "ns::ns-new::totals")
2370            .map(|(_, v)| v.as_str())
2371            .expect("ns::ns-new::totals fact missing");
2372        assert!(
2373            new_totals.contains("24h=1"),
2374            "expected 24h=1 in '{new_totals}'"
2375        );
2376        assert!(
2377            new_totals.contains("7d=1"),
2378            "expected 7d=1 in '{new_totals}'"
2379        );
2380        assert!(
2381            new_totals.contains("all_time=1"),
2382            "expected all_time=1 in '{new_totals}'"
2383        );
2384        // ns-old: 24h=0, 7d=0, all_time=1
2385        let old_totals = rh
2386            .facts
2387            .iter()
2388            .find(|(k, _)| k == "ns::ns-old::totals")
2389            .map(|(_, v)| v.as_str())
2390            .expect("ns::ns-old::totals fact missing");
2391        assert!(
2392            old_totals.contains("24h=0"),
2393            "expected 24h=0 in '{old_totals}'"
2394        );
2395        assert!(
2396            old_totals.contains("7d=0"),
2397            "expected 7d=0 in '{old_totals}'"
2398        );
2399        assert!(
2400            old_totals.contains("all_time=1"),
2401            "expected all_time=1 in '{old_totals}'"
2402        );
2403    }
2404
2405    #[test]
2406    fn reflection_health_json_output_parseable_and_has_section() {
2407        let mut env = TestEnv::fresh();
2408        // Seed one reflection so the section has content.
2409        {
2410            let conn = crate::db::open(&env.db_path).unwrap();
2411            seed_reflection(&conn, "ns-json", 1, "json-refl");
2412        }
2413        let db_path = env.db_path.clone();
2414        let mut out = env.output();
2415        let exit = run(
2416            &db_path,
2417            &DoctorArgs {
2418                remote: None,
2419                json: true,
2420                fail_on_warn: false,
2421            },
2422            &mut out,
2423        )
2424        .unwrap();
2425        // A depth-1 reflection does not warn (threshold is >=2).
2426        assert_eq!(exit, 0);
2427        let v: serde_json::Value = serde_json::from_str(env.stdout_str()).expect("JSON must parse");
2428        let sections = v["sections"].as_array().expect("sections is array");
2429        let rh_section = sections
2430            .iter()
2431            .find(|s| s["name"] == "Reflection Health")
2432            .expect("Reflection Health section must be in JSON output");
2433        assert_eq!(rh_section["severity"], "info");
2434        assert!(rh_section["facts"].is_array(), "facts must be a JSON array");
2435    }
2436
2437    // -------------------------------------------------------------------
2438    // run() entry point — JSON / text / exit code branches
2439    // -------------------------------------------------------------------
2440
2441    #[test]
2442    fn run_emits_json_when_json_flag_set() {
2443        let mut env = TestEnv::fresh();
2444        let db_path = env.db_path.clone();
2445        let mut out = env.output();
2446        let exit = run(
2447            &db_path,
2448            &DoctorArgs {
2449                remote: None,
2450                json: true,
2451                fail_on_warn: false,
2452            },
2453            &mut out,
2454        )
2455        .unwrap();
2456        // Healthy fresh DB → exit 0.
2457        assert_eq!(exit, 0);
2458        let s = env.stdout_str();
2459        let v: serde_json::Value = serde_json::from_str(s).expect("JSON output must parse");
2460        assert_eq!(v["mode"], "local");
2461        assert!(v["sections"].is_array());
2462        assert!(v["overall"].is_string());
2463    }
2464
2465    #[test]
2466    fn run_emits_text_by_default() {
2467        let mut env = TestEnv::fresh();
2468        let db_path = env.db_path.clone();
2469        let mut out = env.output();
2470        let exit = run(
2471            &db_path,
2472            &DoctorArgs {
2473                remote: None,
2474                json: false,
2475                fail_on_warn: false,
2476            },
2477            &mut out,
2478        )
2479        .unwrap();
2480        assert_eq!(exit, 0);
2481        let s = env.stdout_str();
2482        // Header + section labels.
2483        assert!(s.contains("ai-memory doctor — local mode"));
2484        assert!(s.contains("[INFO] Storage"));
2485        assert!(s.contains("[INFO] Index"));
2486        assert!(s.contains("[N/A ] Capabilities"));
2487        // The label-prefixed fact key column is left-padded to 32 chars
2488        // (smoke check that the format string compiles).
2489        assert!(s.contains("total_memories"));
2490    }
2491
2492    #[test]
2493    fn run_returns_exit_2_on_critical() {
2494        let mut env = TestEnv::fresh();
2495        // Inject a 25h-old pending action → Governance CRIT → overall CRIT.
2496        {
2497            let conn = crate::db::open(&env.db_path).unwrap();
2498            let twenty_five_hours_ago =
2499                (chrono::Utc::now() - chrono::Duration::hours(25)).to_rfc3339();
2500            conn.execute(
2501                "INSERT INTO pending_actions \
2502                 (id, action_type, namespace, payload, requested_by, requested_at, status) \
2503                 VALUES ('p1', 'store', 'ns', '{}', 'agent', ?1, 'pending')",
2504                params![twenty_five_hours_ago],
2505            )
2506            .unwrap();
2507        }
2508        let db_path = env.db_path.clone();
2509        let mut out = env.output();
2510        let exit = run(
2511            &db_path,
2512            &DoctorArgs {
2513                remote: None,
2514                json: true,
2515                fail_on_warn: false,
2516            },
2517            &mut out,
2518        )
2519        .unwrap();
2520        assert_eq!(exit, 2);
2521        // JSON overall is "critical".
2522        let v: serde_json::Value = serde_json::from_str(env.stdout_str()).unwrap();
2523        assert_eq!(v["overall"], "critical");
2524    }
2525
2526    #[test]
2527    fn run_warning_keeps_exit_0_without_fail_on_warn() {
2528        let mut env = TestEnv::fresh();
2529        {
2530            let conn = crate::db::open(&env.db_path).unwrap();
2531            let now = chrono::Utc::now().to_rfc3339();
2532            conn.execute(
2533                "INSERT INTO subscriptions \
2534                 (id, url, events, created_at, dispatch_count, failure_count) \
2535                 VALUES ('s1', 'http://x', '*', ?1, 10, 5)",
2536                params![now],
2537            )
2538            .unwrap();
2539        }
2540        let db_path = env.db_path.clone();
2541        let mut out = env.output();
2542        let exit = run(
2543            &db_path,
2544            &DoctorArgs {
2545                remote: None,
2546                json: false,
2547                fail_on_warn: false,
2548            },
2549            &mut out,
2550        )
2551        .unwrap();
2552        assert_eq!(exit, 0, "warning without --fail-on-warn must keep exit 0");
2553        assert!(env.stdout_str().contains("[WARN] Webhook"));
2554    }
2555
2556    #[test]
2557    fn run_warning_returns_exit_1_with_fail_on_warn() {
2558        let mut env = TestEnv::fresh();
2559        {
2560            let conn = crate::db::open(&env.db_path).unwrap();
2561            let now = chrono::Utc::now().to_rfc3339();
2562            conn.execute(
2563                "INSERT INTO subscriptions \
2564                 (id, url, events, created_at, dispatch_count, failure_count) \
2565                 VALUES ('s1', 'http://x', '*', ?1, 10, 5)",
2566                params![now],
2567            )
2568            .unwrap();
2569        }
2570        let db_path = env.db_path.clone();
2571        let mut out = env.output();
2572        let exit = run(
2573            &db_path,
2574            &DoctorArgs {
2575                remote: None,
2576                json: false,
2577                fail_on_warn: true,
2578            },
2579            &mut out,
2580        )
2581        .unwrap();
2582        assert_eq!(exit, 1, "--fail-on-warn must promote warning to exit 1");
2583    }
2584
2585    #[test]
2586    fn run_critical_is_exit_2_even_without_fail_on_warn() {
2587        let mut env = TestEnv::fresh();
2588        {
2589            let conn = crate::db::open(&env.db_path).unwrap();
2590            let twenty_five_hours_ago =
2591                (chrono::Utc::now() - chrono::Duration::hours(25)).to_rfc3339();
2592            conn.execute(
2593                "INSERT INTO pending_actions \
2594                 (id, action_type, namespace, payload, requested_by, requested_at, status) \
2595                 VALUES ('p1', 'store', 'ns', '{}', 'agent', ?1, 'pending')",
2596                params![twenty_five_hours_ago],
2597            )
2598            .unwrap();
2599        }
2600        let db_path = env.db_path.clone();
2601        let mut out = env.output();
2602        let exit = run(
2603            &db_path,
2604            &DoctorArgs {
2605                remote: None,
2606                json: false,
2607                fail_on_warn: false,
2608            },
2609            &mut out,
2610        )
2611        .unwrap();
2612        assert_eq!(exit, 2);
2613    }
2614
2615    // -------------------------------------------------------------------
2616    // run() — corrupt DB path: db::open() fails → CRITICAL Storage section.
2617    // -------------------------------------------------------------------
2618
2619    #[test]
2620    fn local_run_on_unopenable_db_returns_critical_storage_only() {
2621        let tmp = tempfile::tempdir().unwrap();
2622        let bad = tmp.path().join("not-a-db.db");
2623        // Write garbage so SQLite refuses to open it.
2624        std::fs::write(&bad, b"this is not a sqlite database, it's just text").unwrap();
2625        let report = run_local_collect(&bad);
2626        // The error path appends a single Storage section and returns.
2627        assert_eq!(report.sections.len(), 1);
2628        let storage = &report.sections[0];
2629        assert_eq!(storage.name, "Storage");
2630        assert_eq!(storage.severity, Severity::Critical);
2631        // overall is computed from the single section.
2632        assert_eq!(report.overall, Severity::Critical);
2633        assert!(storage.note.as_ref().unwrap().contains("could not open"));
2634    }
2635
2636    // -------------------------------------------------------------------
2637    // Render helpers
2638    // -------------------------------------------------------------------
2639
2640    #[test]
2641    fn render_text_emits_section_note_when_present() {
2642        let r = mk_report(vec![ReportSection {
2643            name: "Sync".into(),
2644            severity: Severity::Critical,
2645            facts: vec![("max_skew_secs".into(), "9999".into())],
2646            note: Some("peer mesh is drifting".into()),
2647        }]);
2648        let mut stdout = Vec::<u8>::new();
2649        let mut stderr = Vec::<u8>::new();
2650        let mut out = CliOutput::from_std(&mut stdout, &mut stderr);
2651        render_text(&r, &mut out).unwrap();
2652        let s = String::from_utf8(stdout).unwrap();
2653        assert!(s.contains("[CRIT] Sync"));
2654        assert!(s.contains("note: peer mesh is drifting"));
2655        assert!(s.contains("max_skew_secs"));
2656        assert!(s.contains("9999"));
2657    }
2658
2659    // -------------------------------------------------------------------
2660    // Remote (--remote) mode — wiremock-driven HTTP fixtures
2661    // -------------------------------------------------------------------
2662
2663    /// Helper: run `run_remote` from a multi-thread tokio test by spawning
2664    /// the blocking reqwest call onto the spawn_blocking pool.
2665    async fn run_remote_in_blocking(url: String, db_path: PathBuf) -> Report {
2666        tokio::task::spawn_blocking(move || {
2667            let mut r = run_remote(&url, &db_path);
2668            r.compute_overall();
2669            r
2670        })
2671        .await
2672        .unwrap()
2673    }
2674
2675    use std::path::PathBuf;
2676
2677    #[tokio::test(flavor = "multi_thread")]
2678    async fn remote_section_capabilities_parses_v2_fields() {
2679        use wiremock::matchers::{method, path};
2680        use wiremock::{Mock, MockServer, ResponseTemplate};
2681        let server = MockServer::start().await;
2682        Mock::given(method("GET"))
2683            .and(path("/api/v1/capabilities"))
2684            .respond_with(ResponseTemplate::new(200).set_body_json(serde_json::json!({
2685                "schema_version": "2",
2686                "feature_tier": "smart",
2687                "features": {
2688                    "recall_mode_active": "hybrid",
2689                    "reranker_active": "cross_encoder"
2690                }
2691            })))
2692            .mount(&server)
2693            .await;
2694        Mock::given(method("GET"))
2695            .and(path("/api/v1/stats"))
2696            .respond_with(ResponseTemplate::new(200).set_body_json(serde_json::json!({
2697                "total": 42,
2698                "expiring_soon": 1,
2699                "links_count": 3
2700            })))
2701            .mount(&server)
2702            .await;
2703
2704        let env = TestEnv::fresh();
2705        let report = run_remote_in_blocking(server.uri(), env.db_path.clone()).await;
2706        assert_eq!(report.mode, "remote");
2707        assert!(report.source.starts_with(&server.uri()));
2708        // Sections: 7 total — Capabilities, Recall, Storage, Index, Governance, Sync, Webhook.
2709        assert_eq!(report.sections.len(), 7);
2710
2711        let cap = find(&report, "Capabilities");
2712        assert_eq!(cap.severity, Severity::Info);
2713        assert_eq!(fact(cap, "schema_version"), "2");
2714        assert_eq!(fact(cap, "recall_mode_active"), "hybrid");
2715        assert_eq!(fact(cap, "reranker_active"), "cross_encoder");
2716
2717        let recall = find(&report, "Recall");
2718        assert_eq!(fact(recall, "active_recall_mode"), "hybrid");
2719        assert_eq!(fact(recall, "active_reranker"), "cross_encoder");
2720
2721        let storage = find(&report, "Storage");
2722        assert_eq!(fact(storage, "total_memories"), "42");
2723        assert_eq!(fact(storage, "expiring_within_1h"), "1");
2724        assert_eq!(fact(storage, "links"), "3");
2725
2726        // Raw-SQL sections must be NotAvailable in remote mode.
2727        for raw in ["Index", "Governance", "Sync", "Webhook"] {
2728            let s = find(&report, raw);
2729            assert_eq!(s.severity, Severity::NotAvailable);
2730            assert!(fact(s, "hint").contains("--db mode"));
2731        }
2732    }
2733
2734    #[tokio::test(flavor = "multi_thread")]
2735    async fn remote_capabilities_silent_degrade_warns_on_capable_tier() {
2736        use wiremock::matchers::{method, path};
2737        use wiremock::{Mock, MockServer, ResponseTemplate};
2738        let server = MockServer::start().await;
2739        Mock::given(method("GET"))
2740            .and(path("/api/v1/capabilities"))
2741            .respond_with(ResponseTemplate::new(200).set_body_json(serde_json::json!({
2742                "schema_version": "2",
2743                "feature_tier": "semantic",
2744                "features": {
2745                    "recall_mode_active": "keyword_only",
2746                    "reranker_active": "none"
2747                }
2748            })))
2749            .mount(&server)
2750            .await;
2751        // /api/v1/stats not mocked → 404 → Storage carries an error fact
2752        // but no severity bump (severity stays Info per the code path).
2753        let env = TestEnv::fresh();
2754        let report = run_remote_in_blocking(server.uri(), env.db_path.clone()).await;
2755        let cap = find(&report, "Capabilities");
2756        assert_eq!(cap.severity, Severity::Warning);
2757        assert!(cap.note.as_ref().unwrap().contains("silent degradation"));
2758    }
2759
2760    #[tokio::test(flavor = "multi_thread")]
2761    async fn remote_capabilities_degraded_on_keyword_tier_does_not_warn() {
2762        // recall_mode=degraded but feature_tier=keyword → no silent-degrade
2763        // (keyword tier was never expected to run hybrid in the first place).
2764        use wiremock::matchers::{method, path};
2765        use wiremock::{Mock, MockServer, ResponseTemplate};
2766        let server = MockServer::start().await;
2767        Mock::given(method("GET"))
2768            .and(path("/api/v1/capabilities"))
2769            .respond_with(ResponseTemplate::new(200).set_body_json(serde_json::json!({
2770                "schema_version": "2",
2771                "feature_tier": "keyword",
2772                "features": {
2773                    "recall_mode_active": "keyword_only",
2774                    "reranker_active": "none"
2775                }
2776            })))
2777            .mount(&server)
2778            .await;
2779        let env = TestEnv::fresh();
2780        let report = run_remote_in_blocking(server.uri(), env.db_path.clone()).await;
2781        let cap = find(&report, "Capabilities");
2782        assert_eq!(cap.severity, Severity::Info);
2783        assert!(cap.note.is_none());
2784    }
2785
2786    #[tokio::test(flavor = "multi_thread")]
2787    async fn remote_capabilities_unreachable_endpoint_is_critical() {
2788        // Reserve a free port and immediately drop the listener so the
2789        // connection refusal is deterministic. Doctor's HTTP timeout is
2790        // 5s; the kernel rejects almost immediately so the test stays
2791        // well under the per-test timeout.
2792        use std::net::TcpListener;
2793        let listener = TcpListener::bind("127.0.0.1:0").unwrap();
2794        let port = listener.local_addr().unwrap().port();
2795        drop(listener);
2796        let url = format!("http://127.0.0.1:{port}");
2797
2798        let env = TestEnv::fresh();
2799        let report = run_remote_in_blocking(url, env.db_path.clone()).await;
2800        let cap = find(&report, "Capabilities");
2801        assert_eq!(cap.severity, Severity::Critical);
2802        assert!(cap.note.as_ref().unwrap().contains("could not reach"));
2803        assert_eq!(report.overall, Severity::Critical);
2804    }
2805
2806    #[tokio::test(flavor = "multi_thread")]
2807    async fn remote_capabilities_legacy_v1_renders_not_in_response() {
2808        // Legacy v0.6.3 capabilities responses don't carry the v2 fields.
2809        use wiremock::matchers::{method, path};
2810        use wiremock::{Mock, MockServer, ResponseTemplate};
2811        let server = MockServer::start().await;
2812        Mock::given(method("GET"))
2813            .and(path("/api/v1/capabilities"))
2814            .respond_with(ResponseTemplate::new(200).set_body_json(serde_json::json!({
2815                "schema_version": "1"
2816            })))
2817            .mount(&server)
2818            .await;
2819        let env = TestEnv::fresh();
2820        let report = run_remote_in_blocking(server.uri(), env.db_path.clone()).await;
2821        let cap = find(&report, "Capabilities");
2822        // Legacy v1 → no severity bump, but missing fields are rendered.
2823        assert_eq!(cap.severity, Severity::Info);
2824        assert_eq!(fact(cap, "schema_version"), "1");
2825        assert_eq!(fact(cap, "recall_mode_active"), "not_in_response");
2826        assert_eq!(fact(cap, "reranker_active"), "not_in_response");
2827    }
2828
2829    #[tokio::test(flavor = "multi_thread")]
2830    async fn remote_run_via_run_entry_uses_remote_mode_string() {
2831        use wiremock::matchers::{method, path};
2832        use wiremock::{Mock, MockServer, ResponseTemplate};
2833        let server = MockServer::start().await;
2834        Mock::given(method("GET"))
2835            .and(path("/api/v1/capabilities"))
2836            .respond_with(ResponseTemplate::new(200).set_body_json(serde_json::json!({
2837                "schema_version": "2",
2838                "feature_tier": "semantic",
2839                "features": {
2840                    "recall_mode_active": "hybrid",
2841                    "reranker_active": "none"
2842                }
2843            })))
2844            .mount(&server)
2845            .await;
2846        Mock::given(method("GET"))
2847            .and(path("/api/v1/stats"))
2848            .respond_with(ResponseTemplate::new(200).set_body_json(serde_json::json!({
2849                "total": 0
2850            })))
2851            .mount(&server)
2852            .await;
2853
2854        let env_db = TestEnv::fresh().db_path;
2855        let url = server.uri();
2856        let (exit, stdout) = tokio::task::spawn_blocking(move || {
2857            let mut stdout = Vec::<u8>::new();
2858            let mut stderr = Vec::<u8>::new();
2859            let mut out = CliOutput::from_std(&mut stdout, &mut stderr);
2860            let exit = run(
2861                &env_db,
2862                &DoctorArgs {
2863                    remote: Some(url),
2864                    json: true,
2865                    fail_on_warn: false,
2866                },
2867                &mut out,
2868            )
2869            .unwrap();
2870            (exit, stdout)
2871        })
2872        .await
2873        .unwrap();
2874        assert_eq!(exit, 0);
2875        let v: serde_json::Value = serde_json::from_slice(&stdout).unwrap();
2876        assert_eq!(v["mode"], "remote");
2877        // Trailing slash on the URL must be normalized.
2878    }
2879
2880    #[tokio::test(flavor = "multi_thread")]
2881    async fn remote_url_trailing_slash_is_trimmed() {
2882        use wiremock::matchers::{method, path};
2883        use wiremock::{Mock, MockServer, ResponseTemplate};
2884        let server = MockServer::start().await;
2885        Mock::given(method("GET"))
2886            .and(path("/api/v1/capabilities"))
2887            .respond_with(ResponseTemplate::new(200).set_body_json(serde_json::json!({
2888                "schema_version": "2",
2889                "features": {}
2890            })))
2891            .mount(&server)
2892            .await;
2893        Mock::given(method("GET"))
2894            .and(path("/api/v1/stats"))
2895            .respond_with(ResponseTemplate::new(200).set_body_json(serde_json::json!({})))
2896            .mount(&server)
2897            .await;
2898        let env = TestEnv::fresh();
2899        // Append a trailing slash; format!("{base}/api/v1/...") would
2900        // otherwise produce a `//api/v1/` path that wiremock would 404.
2901        let report =
2902            run_remote_in_blocking(format!("{}/", server.uri()), env.db_path.clone()).await;
2903        let cap = find(&report, "Capabilities");
2904        assert_eq!(cap.severity, Severity::Info);
2905    }
2906
2907    #[tokio::test(flavor = "multi_thread")]
2908    async fn remote_storage_500_renders_error_without_severity_bump() {
2909        use wiremock::matchers::{method, path};
2910        use wiremock::{Mock, MockServer, ResponseTemplate};
2911        let server = MockServer::start().await;
2912        Mock::given(method("GET"))
2913            .and(path("/api/v1/capabilities"))
2914            .respond_with(ResponseTemplate::new(200).set_body_json(serde_json::json!({
2915                "schema_version": "2",
2916                "features": {}
2917            })))
2918            .mount(&server)
2919            .await;
2920        Mock::given(method("GET"))
2921            .and(path("/api/v1/stats"))
2922            .respond_with(ResponseTemplate::new(500))
2923            .mount(&server)
2924            .await;
2925        let env = TestEnv::fresh();
2926        let report = run_remote_in_blocking(server.uri(), env.db_path.clone()).await;
2927        let storage = find(&report, "Storage");
2928        // Storage section preserves Info severity even on 5xx — by spec
2929        // (remote storage is best-effort; sql truth is the local mode).
2930        assert_eq!(storage.severity, Severity::Info);
2931        let err = fact(storage, "error");
2932        assert!(
2933            err.contains("HTTP 500"),
2934            "expected HTTP 500 message, got {err}"
2935        );
2936    }
2937
2938    // ---- v0.6.4-004 — `--tokens` reporter ----
2939
2940    fn run_tokens_capture(args: TokensArgs) -> (i32, String, String) {
2941        let mut stdout = Vec::<u8>::new();
2942        let mut stderr = Vec::<u8>::new();
2943        let exit;
2944        {
2945            let mut out = CliOutput::from_std(&mut stdout, &mut stderr);
2946            exit = run_tokens(args, &mut out).expect("run_tokens");
2947        }
2948        (
2949            exit,
2950            String::from_utf8(stdout).unwrap(),
2951            String::from_utf8(stderr).unwrap(),
2952        )
2953    }
2954
2955    #[test]
2956    fn run_tokens_human_default_profile_is_core() {
2957        let (exit, stdout, _stderr) = run_tokens_capture(TokensArgs::default());
2958        assert_eq!(exit, 0);
2959        assert!(
2960            stdout.contains("Active profile: core"),
2961            "default profile should be core; got: {stdout}"
2962        );
2963        // v0.7.0 refactor PR-2 (#793) — tool-count SSOT. The "Full (NN
2964        // tools loaded)" string is generated from
2965        // `Profile::full().expected_tool_count()`, so anchor the
2966        // expected substring on the same constant.
2967        let n = crate::profile::Profile::full().expected_tool_count();
2968        let needle = format!("Full   ({n} tools loaded)");
2969        assert!(
2970            stdout.contains(&needle),
2971            "report should include full-profile baseline `{needle}` (canonical \
2972             from Profile::full().expected_tool_count()); got: {stdout}"
2973        );
2974        assert!(
2975            stdout.contains("Tokenizer: cl100k_base"),
2976            "report should call out the tokenizer"
2977        );
2978    }
2979
2980    #[test]
2981    fn run_tokens_json_emits_structured_payload() {
2982        let args = TokensArgs {
2983            json: true,
2984            raw_table: false,
2985            profile: Some("graph".to_string()),
2986            hooks: false,
2987        };
2988        let (exit, stdout, _) = run_tokens_capture(args);
2989        assert_eq!(exit, 0);
2990        let v: serde_json::Value =
2991            serde_json::from_str(&stdout).expect("--json must emit valid JSON");
2992        assert_eq!(v["schema_version"], "v0.6.4-tokens-1");
2993        assert_eq!(v["tokenizer"], "cl100k_base");
2994        // Token count grows as schemas evolve. Assert the honest
2995        // cl100k_base range from sizes.rs (5K-17K post-#987 D1.6 — see
2996        // `tests/token_budget_guard.rs` for the load-bearing ceilings).
2997        // The exact-figure invariant lives in
2998        // `sizes::tests::full_profile_total_in_honest_measured_range`.
2999        let total = v["full_profile_total_tokens"].as_u64().unwrap();
3000        assert!(
3001            (5_000..=17_000).contains(&total),
3002            "full_profile_total_tokens out of honest range: {total}"
3003        );
3004        assert!(v["active_total_tokens"].as_u64().unwrap() > 0);
3005        // graph profile loads core + graph; both flags true on those rows.
3006        let families = v["families"].as_array().unwrap();
3007        let core_row = families.iter().find(|r| r["name"] == "core").unwrap();
3008        assert_eq!(core_row["loaded"], true);
3009        let graph_row = families.iter().find(|r| r["name"] == "graph").unwrap();
3010        assert_eq!(graph_row["loaded"], true);
3011        let archive_row = families.iter().find(|r| r["name"] == "archive").unwrap();
3012        assert_eq!(archive_row["loaded"], false);
3013    }
3014
3015    #[test]
3016    fn run_tokens_raw_table_includes_per_tool_rows() {
3017        let args = TokensArgs {
3018            json: false,
3019            raw_table: true,
3020            profile: None,
3021            hooks: false,
3022        };
3023        let (exit, stdout, _) = run_tokens_capture(args);
3024        assert_eq!(exit, 0);
3025        let v: serde_json::Value = serde_json::from_str(&stdout).unwrap();
3026        let tools = v["tools"].as_array().unwrap();
3027        assert_eq!(
3028            tools.len(),
3029            crate::profile::Profile::full().expected_tool_count(),
3030            "raw_table must include every tool — canonical count is the \
3031             SSOT `Profile::full().expected_tool_count()` (derived from \
3032             the per-Family `tool_names` slices); no literal is restated"
3033        );
3034        // memory_store is in core and must be loaded under the default
3035        // (core) profile.
3036        let store = tools
3037            .iter()
3038            .find(|t| t["name"] == "memory_store")
3039            .expect("memory_store row");
3040        assert_eq!(store["family"], "core");
3041        assert_eq!(store["loaded_under_active_profile"], true);
3042    }
3043
3044    #[test]
3045    fn run_tokens_invalid_profile_exits_2_with_diagnostic() {
3046        let args = TokensArgs {
3047            json: false,
3048            raw_table: false,
3049            profile: Some("Core".to_string()),
3050            hooks: false,
3051        };
3052        let (exit, _stdout, stderr) = run_tokens_capture(args);
3053        assert_eq!(exit, 2, "malformed profile must exit 2");
3054        assert!(
3055            stderr.contains("case-sensitive lowercase"),
3056            "diagnostic should mention case rule; got: {stderr}"
3057        );
3058    }
3059
3060    // ---------- E1 coverage uplift -----------------------------------
3061    // Targets: run_hooks (json + human), render_hooks_human (config
3062    // present + missing), --tokens --hooks combo.
3063
3064    fn run_hooks_capture(args: HooksReportArgs) -> (i32, String, String) {
3065        let mut stdout = Vec::<u8>::new();
3066        let mut stderr = Vec::<u8>::new();
3067        let exit;
3068        {
3069            let mut out = CliOutput::from_std(&mut stdout, &mut stderr);
3070            exit = run_hooks(args, &mut out).expect("run_hooks");
3071        }
3072        (
3073            exit,
3074            String::from_utf8(stdout).unwrap(),
3075            String::from_utf8(stderr).unwrap(),
3076        )
3077    }
3078
3079    /// Builds an in-memory `HookConfig` row for the render tests (the
3080    /// G3 doctor block renders config shape + zeroed metric
3081    /// placeholders, so any valid event/mode pair exercises the row
3082    /// renderer).
3083    fn mk_hook(command: &str) -> crate::hooks::config::HookConfig {
3084        crate::hooks::config::HookConfig {
3085            event: crate::hooks::HookEvent::PostStore,
3086            command: std::path::PathBuf::from(command),
3087            priority: 10,
3088            timeout_ms: 1_000,
3089            mode: crate::hooks::config::HookMode::Exec,
3090            enabled: true,
3091            namespace: "*".to_string(),
3092            fail_mode: crate::hooks::config::FailMode::Open,
3093        }
3094    }
3095
3096    #[test]
3097    fn run_hooks_human_default_no_config_lists_zero() {
3098        // Default path: HookConfig::default_path() may or may not exist
3099        // on this system, but the loader either returns an empty list
3100        // (file absent) or whatever is present. With or without, the
3101        // human-mode header line surfaces.
3102        let (exit, stdout, _stderr) = run_hooks_capture(HooksReportArgs { json: false });
3103        assert_eq!(exit, 0);
3104        assert!(stdout.contains("ai-memory doctor --hooks"));
3105        assert!(stdout.contains("Hooks loaded:"));
3106    }
3107
3108    #[test]
3109    fn run_hooks_json_emits_schema_versioned_payload() {
3110        let (exit, stdout, _) = run_hooks_capture(HooksReportArgs { json: true });
3111        assert_eq!(exit, 0);
3112        let v: serde_json::Value = serde_json::from_str(&stdout).expect("valid JSON");
3113        assert_eq!(v["schema_version"], "v0.7-hooks-1");
3114        assert!(v["hooks_loaded"].is_number());
3115        assert!(v["executors"].is_array());
3116        assert!(v["timeout_violations"].is_number());
3117    }
3118
3119    #[test]
3120    fn run_tokens_with_hooks_flag_appends_block() {
3121        // Drives the `args.hooks` arm inside run_tokens (lines 329-331).
3122        let args = TokensArgs {
3123            json: false,
3124            raw_table: false,
3125            profile: None,
3126            hooks: true,
3127        };
3128        let (exit, stdout, _stderr) = run_tokens_capture(args);
3129        assert_eq!(exit, 0);
3130        // Token report + appended hooks block.
3131        assert!(stdout.contains("ai-memory doctor --tokens"));
3132        assert!(stdout.contains("ai-memory doctor --hooks"));
3133    }
3134
3135    // The `run_hooks` paths that depend on a loaded `hooks.toml` at the
3136    // operator's real `~/Library/Application Support/ai-memory/hooks.toml`
3137    // would violate the hermetic-test contract. We instead exercise the
3138    // inner renderer (`render_hooks_human_with`) directly via the
3139    // `HookConfig::load_from_str` API — no env mutation, no disk
3140    // writes to user-owned paths.
3141
3142    #[test]
3143    fn render_hooks_human_with_synthetic_hook_renders_row() {
3144        // Drives render_hooks_human_with lines 414-444 + 446-454 — the
3145        // hooks-present branch.
3146        let toml_src = r#"
3147[[hook]]
3148event = "post_store"
3149command = "/usr/local/bin/echo-something-long"
3150mode = "exec"
3151namespace = "*"
3152priority = 5
3153timeout_ms = 1000
3154enabled = true
3155"#;
3156        let hooks = crate::hooks::config::HookConfig::load_from_str(toml_src).expect("parse hooks");
3157        let mut stdout = Vec::<u8>::new();
3158        let mut stderr = Vec::<u8>::new();
3159        let synthetic_path = std::path::PathBuf::from("/tmp/synthetic/hooks.toml");
3160        {
3161            let mut out = CliOutput::from_std(&mut stdout, &mut stderr);
3162            render_hooks_human_with(&mut out, Some(&synthetic_path), &hooks).unwrap();
3163        }
3164        let s = String::from_utf8(stdout).unwrap();
3165        assert!(s.contains("ai-memory doctor --hooks"));
3166        assert!(s.contains("Config path:"));
3167        assert!(s.contains("Hooks loaded: 1"));
3168        // Row carries the truncated file_name.
3169        assert!(s.contains("echo-something-long") || s.contains("event"));
3170        assert!(s.contains("Chain class-deadline violations"));
3171        assert!(s.contains("note: live metrics land"));
3172    }
3173
3174    #[test]
3175    fn render_hooks_human_with_no_hooks_emits_helpful_note() {
3176        // Drives render_hooks_human_with's hooks.is_empty() branch
3177        // (lines 418-424) + the path-Some line (lines 414-416).
3178        let mut stdout = Vec::<u8>::new();
3179        let mut stderr = Vec::<u8>::new();
3180        let synthetic_path = std::path::PathBuf::from("/some/path/hooks.toml");
3181        {
3182            let mut out = CliOutput::from_std(&mut stdout, &mut stderr);
3183            render_hooks_human_with(&mut out, Some(&synthetic_path), &[]).unwrap();
3184        }
3185        let s = String::from_utf8(stdout).unwrap();
3186        assert!(s.contains("ai-memory doctor --hooks"));
3187        assert!(s.contains("Config path:"));
3188        assert!(s.contains("Hooks loaded: 0"));
3189        assert!(s.contains("(no hooks configured"));
3190    }
3191
3192    #[test]
3193    fn render_hooks_human_with_command_no_filename_falls_back_to_display() {
3194        // Drives the `.unwrap_or_else(|| h.command.display().to_string())`
3195        // arm (line 438) — fires when command.file_name() returns None.
3196        let toml_src = r#"
3197[[hook]]
3198event = "post_store"
3199command = "/"
3200mode = "exec"
3201namespace = "*"
3202priority = 1
3203timeout_ms = 500
3204enabled = true
3205"#;
3206        // `command = "/"` has no `file_name()`; the fallback uses
3207        // `display()`.
3208        let hooks = crate::hooks::config::HookConfig::load_from_str(toml_src).expect("parse hooks");
3209        let mut stdout = Vec::<u8>::new();
3210        let mut stderr = Vec::<u8>::new();
3211        {
3212            let mut out = CliOutput::from_std(&mut stdout, &mut stderr);
3213            render_hooks_human_with(&mut out, None, &hooks).unwrap();
3214        }
3215        let s = String::from_utf8(stdout).unwrap();
3216        // No `Config path` line because path is None.
3217        assert!(!s.contains("Config path:"));
3218        assert!(s.contains("Hooks loaded: 1"));
3219    }
3220
3221    /// The happy-path complement of the `/`-fallback test above: a
3222    /// hook whose command HAS a `file_name()` renders by basename.
3223    /// (Also the call site that keeps `mk_hook` honest — the three
3224    /// section tests below were nested inside the previous test fn by
3225    /// the ee00d8bb coverage lift, so rustc flagged them unnameable
3226    /// and `mk_hook` dead; this test + the un-nesting restore them.)
3227    #[test]
3228    fn render_hooks_human_with_rows_renders_each_hook() {
3229        let hooks = vec![mk_hook("/usr/local/bin/notify-hook.sh")];
3230        let mut stdout = Vec::<u8>::new();
3231        let mut stderr = Vec::<u8>::new();
3232        {
3233            let mut out = CliOutput::from_std(&mut stdout, &mut stderr);
3234            render_hooks_human_with(&mut out, None, &hooks).unwrap();
3235        }
3236        let s = String::from_utf8(stdout).unwrap();
3237        assert!(s.contains("Hooks loaded: 1"), "got: {s}");
3238        assert!(s.contains("notify-hook.sh"), "got: {s}");
3239    }
3240
3241    /// A connection with no schema at all: `db::stats` fails, the
3242    /// Storage section must downgrade to WARN with a `stats_error`
3243    /// fact, and `dim_violations` renders the pre-P2 `not_observed`
3244    /// line (prepare on the missing table fails → `Ok(None)`).
3245    #[test]
3246    fn storage_section_warns_with_stats_error_on_missing_schema() {
3247        let conn = rusqlite::Connection::open_in_memory().expect("open_in_memory");
3248        let section = section_storage(&conn, Path::new("/nonexistent/doctor.db"));
3249        assert_eq!(section.severity, Severity::Warning);
3250        assert!(
3251            section.facts.iter().any(|(k, _)| k == "stats_error"),
3252            "facts: {:?}",
3253            section.facts
3254        );
3255        assert!(
3256            section
3257                .facts
3258                .iter()
3259                .any(|(k, v)| k == "dim_violations" && v.contains("not_observed")),
3260            "facts: {:?}",
3261            section.facts
3262        );
3263    }
3264
3265    /// Index section near-capacity arm: ≥95k embedded rows must WARN
3266    /// with the MAX_ENTRIES note. The section only counts
3267    /// `embedding IS NOT NULL`, so a minimal single-column table keeps
3268    /// the fixture cheap (one recursive-CTE insert, no full schema).
3269    #[test]
3270    fn index_section_warns_when_hnsw_within_5pct_of_cap() {
3271        let conn = rusqlite::Connection::open_in_memory().expect("open_in_memory");
3272        conn.execute_batch(
3273            "CREATE TABLE memories(embedding BLOB);
3274             INSERT INTO memories(embedding)
3275             WITH RECURSIVE c(x) AS (SELECT 1 UNION ALL SELECT x + 1 FROM c WHERE x < 95000)
3276             SELECT x FROM c;",
3277        )
3278        .expect("seed 95k embedded rows");
3279        let section = section_index(&conn);
3280        assert_eq!(section.severity, Severity::Warning);
3281        let note = section.note.as_deref().expect("note must explain the cap");
3282        assert!(note.contains("within 5%"), "note: {note}");
3283        assert!(
3284            section
3285                .facts
3286                .iter()
3287                .any(|(k, v)| k == "hnsw_size_estimate" && v == "95000"),
3288            "facts: {:?}",
3289            section.facts
3290        );
3291    }
3292
3293    /// Sync section `Ok(None)` skew arm: a registered peer whose
3294    /// `last_pulled_at` is NULL yields peer_count ≥ 1 but no measurable
3295    /// skew — the section must render `not_observed` at INFO rather
3296    /// than N/A (the no-peers early return) or CRIT.
3297    #[test]
3298    fn sync_section_not_observed_when_peer_has_no_pull_timestamp() {
3299        let conn = rusqlite::Connection::open_in_memory().expect("open_in_memory");
3300        conn.execute_batch(
3301            "CREATE TABLE sync_state(last_seen_at TEXT, last_pulled_at TEXT);
3302             INSERT INTO sync_state(last_seen_at, last_pulled_at)
3303             VALUES ('2026-01-01T00:00:00Z', NULL);",
3304        )
3305        .expect("seed peer row");
3306        let section = section_sync(&conn);
3307        assert_eq!(section.severity, Severity::Info);
3308        assert!(
3309            section
3310                .facts
3311                .iter()
3312                .any(|(k, v)| k == "max_skew_secs" && v == "not_observed"),
3313            "facts: {:?}",
3314            section.facts
3315        );
3316        assert!(
3317            section
3318                .facts
3319                .iter()
3320                .any(|(k, v)| k == "peer_count" && v == "1"),
3321            "facts: {:?}",
3322            section.facts
3323        );
3324    }
3325    // ---------------------------------------------------------------
3326    // #1146 / #1598 reachability probes + #1598 GPU policy — coverage
3327    // lift (GA push). Driven by wiremock + spawn_blocking (the
3328    // reachability sections use reqwest::blocking) with the resolver
3329    // env vars serialised on a module-local lock. Mirrors the
3330    // remote-section test idiom above.
3331    // ---------------------------------------------------------------
3332
3333    fn reach_env_lock() -> &'static std::sync::Mutex<()> {
3334        static L: std::sync::OnceLock<std::sync::Mutex<()>> = std::sync::OnceLock::new();
3335        L.get_or_init(|| std::sync::Mutex::new(()))
3336    }
3337
3338    /// RAII env setter scoped to a test; restores prior values on drop.
3339    struct EnvScope(Vec<(&'static str, Option<std::ffi::OsString>)>);
3340    impl EnvScope {
3341        fn set(pairs: &[(&'static str, &str)]) -> Self {
3342            let mut prev = Vec::new();
3343            for (k, v) in pairs {
3344                prev.push((*k, std::env::var_os(k)));
3345                // SAFETY: serialised by reach_env_lock in every caller.
3346                unsafe { std::env::set_var(k, v) };
3347            }
3348            // AI_MEMORY_NO_CONFIG keeps AppConfig::load off any on-disk
3349            // config.toml so the probe sees ONLY our env.
3350            prev.push((
3351                "AI_MEMORY_NO_CONFIG",
3352                std::env::var_os("AI_MEMORY_NO_CONFIG"),
3353            ));
3354            unsafe { std::env::set_var("AI_MEMORY_NO_CONFIG", "1") };
3355            Self(prev)
3356        }
3357    }
3358    impl Drop for EnvScope {
3359        fn drop(&mut self) {
3360            for (k, v) in &self.0 {
3361                match v {
3362                    Some(val) => unsafe { std::env::set_var(k, val) },
3363                    None => unsafe { std::env::remove_var(k) },
3364                }
3365            }
3366        }
3367    }
3368
3369    fn clear_llm_embed_env() {
3370        for k in [
3371            "AI_MEMORY_LLM_BACKEND",
3372            "AI_MEMORY_LLM_BASE_URL",
3373            "AI_MEMORY_LLM_API_KEY",
3374            "AI_MEMORY_LLM_MODEL",
3375            "AI_MEMORY_EMBED_BACKEND",
3376            "AI_MEMORY_EMBED_BASE_URL",
3377            "AI_MEMORY_EMBED_API_KEY",
3378            "AI_MEMORY_EMBED_MODEL",
3379        ] {
3380            unsafe { std::env::remove_var(k) };
3381        }
3382    }
3383
3384    #[test]
3385    fn gpu_policy_warn_applicable_matrix_1598() {
3386        // API embed backend → never the GPU warn (GPU irrelevant).
3387        assert!(!gpu_policy_warn_applicable("openai", true));
3388        assert!(!gpu_policy_warn_applicable("openai", false));
3389        // ollama + no GPU → warn applies; ollama + GPU → no warn.
3390        assert!(gpu_policy_warn_applicable("ollama", false));
3391        assert!(!gpu_policy_warn_applicable("ollama", true));
3392    }
3393
3394    #[tokio::test(flavor = "multi_thread")]
3395    async fn llm_reachability_compiled_default_is_info_1146() {
3396        let _g = reach_env_lock().lock().unwrap_or_else(|e| e.into_inner());
3397        clear_llm_embed_env();
3398        let _scope = EnvScope::set(&[]);
3399        let section = tokio::task::spawn_blocking(section_llm_reachability_1146)
3400            .await
3401            .unwrap();
3402        assert_eq!(section.severity, Severity::Info);
3403        assert!(
3404            section
3405                .note
3406                .as_deref()
3407                .unwrap_or("")
3408                .contains("no operator LLM configuration"),
3409            "compiled-default note expected; got {:?}",
3410            section.note
3411        );
3412    }
3413
3414    #[tokio::test(flavor = "multi_thread")]
3415    async fn llm_reachability_probe_arms_1146() {
3416        use wiremock::matchers::{method, path};
3417        use wiremock::{Mock, MockServer, ResponseTemplate};
3418        for (code, want) in [
3419            (200u16, Severity::Info),
3420            (401, Severity::Warning),
3421            (503, Severity::Warning),
3422        ] {
3423            let server = MockServer::start().await;
3424            Mock::given(method("GET"))
3425                .and(path("/models"))
3426                .respond_with(ResponseTemplate::new(code))
3427                .mount(&server)
3428                .await;
3429            let uri = server.uri();
3430            let section = {
3431                let _g = reach_env_lock().lock().unwrap_or_else(|e| e.into_inner());
3432                clear_llm_embed_env();
3433                let _scope = EnvScope::set(&[
3434                    ("AI_MEMORY_LLM_BACKEND", "openai-compatible"),
3435                    ("AI_MEMORY_LLM_BASE_URL", &uri),
3436                    ("AI_MEMORY_LLM_API_KEY", "probe-key"),
3437                    ("AI_MEMORY_LLM_MODEL", "probe-model"),
3438                ]);
3439                tokio::task::spawn_blocking(section_llm_reachability_1146)
3440                    .await
3441                    .unwrap()
3442            };
3443            assert_eq!(section.severity, want, "LLM probe status {code}");
3444            assert_eq!(fact(&section, "http_status"), code.to_string());
3445        }
3446    }
3447
3448    #[tokio::test(flavor = "multi_thread")]
3449    async fn embeddings_reachability_compiled_default_is_info_1598() {
3450        let _g = reach_env_lock().lock().unwrap_or_else(|e| e.into_inner());
3451        clear_llm_embed_env();
3452        let _scope = EnvScope::set(&[]);
3453        let section = tokio::task::spawn_blocking(section_embeddings_reachability_1598)
3454            .await
3455            .unwrap();
3456        assert_eq!(section.severity, Severity::Info);
3457        assert!(
3458            section
3459                .note
3460                .as_deref()
3461                .unwrap_or("")
3462                .contains("no operator embeddings configuration"),
3463            "compiled-default note expected; got {:?}",
3464            section.note
3465        );
3466    }
3467
3468    #[tokio::test(flavor = "multi_thread")]
3469    async fn embeddings_reachability_api_probe_arms_1598() {
3470        use wiremock::matchers::{method, path};
3471        use wiremock::{Mock, MockServer, ResponseTemplate};
3472        for (code, want) in [
3473            (200u16, Severity::Info),
3474            (401, Severity::Warning),
3475            (500, Severity::Warning),
3476        ] {
3477            let server = MockServer::start().await;
3478            Mock::given(method("POST"))
3479                .and(path("/embeddings"))
3480                .respond_with(
3481                    ResponseTemplate::new(code).set_body_json(serde_json::json!({"data": []})),
3482                )
3483                .mount(&server)
3484                .await;
3485            let uri = server.uri();
3486            let section = {
3487                let _g = reach_env_lock().lock().unwrap_or_else(|e| e.into_inner());
3488                clear_llm_embed_env();
3489                let _scope = EnvScope::set(&[
3490                    ("AI_MEMORY_EMBED_BACKEND", "openai-compatible"),
3491                    ("AI_MEMORY_EMBED_BASE_URL", &uri),
3492                    ("AI_MEMORY_EMBED_API_KEY", "probe-key"),
3493                    ("AI_MEMORY_EMBED_MODEL", "probe-embed-model"),
3494                ]);
3495                tokio::task::spawn_blocking(section_embeddings_reachability_1598)
3496                    .await
3497                    .unwrap()
3498            };
3499            assert_eq!(section.severity, want, "embed probe status {code}");
3500            assert_eq!(fact(&section, "http_status"), code.to_string());
3501        }
3502    }
3503}