mdlens/

cli.rs

use anyhow::Result;
use clap::{Parser, Subcommand, ValueEnum};
use serde::Serialize;
use std::cmp::Reverse;

use std::collections::{BTreeSet, HashMap, HashSet};
use std::io::{self, BufRead};
use std::path::Path;

use crate::errors;
use crate::model::Section;
use crate::pack::{pack_by_ids, PackSearchOptions};
use crate::parse::{load_markdown, parse_markdown};
use crate::render::{
    render_pack, render_read, render_search, render_sections, render_stats, render_tree,
    FileSectionsMap, PackIncluded, SectionsEntry, StatsEntry,
};
use crate::search::{discover_markdown_files, get_doc_section_summaries, search_files};
use crate::tokens::{estimate_tokens, truncate_to_tokens};

const TRUNCATION_NOTICE: &str = "\n\n<!-- mdlens: truncated at token budget -->";

#[derive(Parser)]
#[command(name = "mdlens")]
#[command(about = "Token-efficient Markdown structure CLI for AI agents")]
#[command(
    long_about = "mdlens parses Markdown files into a hierarchical section tree with\ndotted IDs, token estimates, and bounded-context packing.\n\nDesigned for AI agents that need to navigate, search, and pack\nMarkdown documentation into context windows efficiently.\n\nAgent quickstart:\n  1. For question answering over a Markdown directory, start with:\n       mdlens scout <dir> \"<question>\" --max-tokens 1000\n  2. Answer from scout when [highlights] and [evidence] are sufficient.\n  3. If one detail is missing, use a listed section id:\n       mdlens read <file> --id <N.N> --max-tokens 1200\n  4. Use search/tree/sections only when scout points at the wrong file or you\n     need broader navigation.\n\nScout is the recommended first command for arbitrary messy English markdown.\nIt returns query expansion, a compact file map, ranked highlights, and bounded\nevidence sections with parent heading/status context.\n\nAnswering from scout:\n  - Read [highlights] first, then [evidence].\n  - Preserve distinctive evidence terms: flags, IDs, metrics, option names,\n    labels, row values, and short policy/risk phrases.\n  - Copy short source phrases exactly when they are likely answer terms; avoid\n    changing singular/plural or rewriting concise labels into paraphrases.\n  - If scout already names the answer plus its rule, risk, command, or policy,\n    answer directly instead of continuing broad retrieval.\n  - For current-vs-stale questions, prefer current/current loader sections and\n    treat Do Not Use, copied tables, stale notes, and old runbooks as\n    distractors.\n  - For table questions, keep the table header with the selected row; do not\n    average unrelated rows unless the document says to.\n  - For why, policy, safety, privacy, negative, or tradeoff questions, include\n    the compact rule/risk/rationale bullets, not only the command or metric.\n  - For multi-file comparisons, answer each named entity separately, then\n    summarize the shared pattern.\n  - If evidence is missing, say the corpus does not specify the fact.\n\nRun `mdlens scout --help` for detailed scout-specific guidance."
)]
struct Cli {
    #[command(subcommand)]
    command: Commands,
}

#[derive(Subcommand)]
enum Commands {
    /// Show section hierarchy with token estimates for a file or directory
    Tree(TreeArgs),
    /// Extract a section by ID, heading path, or line range
    Read(ReadArgs),
    /// Search files and return section-level matches with snippets
    Search(SearchArgs),
    /// One-shot agent evidence pack: find files, show section maps, and include likely evidence
    Scout(ScoutArgs),
    /// Pack selected sections into a bounded token budget
    Pack(PackArgs),
    /// Inspect file sizes, word counts, and token estimates
    Stats(StatsArgs),
    /// Read file paths from stdin and output structured section metadata
    Sections(SectionsArgs),
    /// Wire mdlens guidance into AI coding harnesses (CLAUDE.md, AGENTS.md, ...)
    Init(InitArgs),
    /// Show accumulated token savings from scout/read usage (set MDLENS_NO_GAIN=1 to disable recording)
    Gain(GainArgs),
}

#[derive(clap::Args)]
struct TreeArgs {
    /// File or directory to analyze
    path: String,
    /// Output JSON (machine-readable with schema_version)
    #[arg(long)]
    json: bool,
    /// Limit section depth shown
    #[arg(long)]
    max_depth: Option<usize>,
    /// Show preamble section (content before first heading)
    #[arg(long)]
    include_preamble: bool,
    /// For directory input, include per-file summaries
    #[arg(long)]
    files: bool,
}

#[derive(clap::Args)]
struct ReadArgs {
    /// File to read from
    file: String,
    /// Section ID to extract (e.g., "1.2.3" — dotted hierarchy)
    #[arg(long, conflicts_with_all = ["heading_path", "lines"])]
    id: Option<String>,
    /// Heading path to extract (e.g., "Usage>Configuration"; escape literal > as \>)
    #[arg(long, conflicts_with_all = ["id", "lines"])]
    heading_path: Option<String>,
    /// Line range to extract (e.g., "120:190")
    #[arg(long, conflicts_with_all = ["id", "heading_path"])]
    lines: Option<String>,
    /// Include parent headings above the section excerpt
    #[arg(long)]
    parents: bool,
    /// Include all child sections (default: true unless --no-children)
    #[arg(long, conflicts_with = "no_children")]
    children: bool,
    /// Only include heading and direct body before first child heading
    #[arg(long, conflicts_with = "children")]
    no_children: bool,
    /// Truncate output to approximate token budget
    #[arg(long)]
    max_tokens: Option<usize>,
    /// Output JSON (machine-readable with schema_version)
    #[arg(long)]
    json: bool,
}

#[derive(clap::Args)]
struct SearchArgs {
    /// File or directory to search
    path: String,
    /// Search query (plain text or regex with --regex)
    query: String,
    /// Output JSON (machine-readable with schema_version)
    #[arg(long)]
    json: bool,
    /// Use regex for the query
    #[arg(long)]
    regex: bool,
    /// Case-sensitive search (default: case-insensitive)
    #[arg(long)]
    case_sensitive: bool,
    /// Maximum number of results (default: 20)
    #[arg(long, default_value_t = 20)]
    max_results: usize,
    /// Context lines around each match (default: 2)
    #[arg(long, default_value_t = 2)]
    context_lines: usize,
    /// Include full section body text for each result
    #[arg(long)]
    content: bool,
    /// Show first N non-empty lines of each matched section inline
    #[arg(long)]
    preview: Option<usize>,
    /// Cap total output tokens across included search results
    #[arg(long)]
    max_tokens: Option<usize>,
}

#[derive(clap::Args)]
#[command(
    long_about = "One-shot agent evidence pack for answering a natural-language question over Markdown.\n\n`scout` is optimized for agent workflows: fewer shell calls, bounded output,\nand enough section context to answer without dumping whole files. It searches\nsection text, headings, paths, parent context, and table rows; ranks likely\nevidence; then emits a compact pack."
)]
#[command(
    after_help = "Agent workflow:\n  - Use scout as the first retrieval call for QA over a directory:\n      mdlens scout docs/ \"What policy changed between the old and current loader?\" --max-tokens 1000\n  - Use --json when a harness wants structured metadata plus the same rendered evidence pack.\n  - Read [highlights] first. They are globally ranked compact evidence lines.\n  - Then read [evidence]. Each block names file, section id, heading path, line\n    span, token estimate, and ranking reason.\n  - If the answer is present, stop and answer directly. Preserve distinctive\n    terms: flags, IDs, metrics, option names, row values, labels, and short\n    policy phrases.\n  - Copy short source phrases exactly when they are likely answer terms; avoid\n    changing singular/plural or rewriting concise labels into paraphrases.\n  - If exactly one fact is missing, use the section map from [files] and read\n    one section:\n      mdlens read <file> --id <section-id> --max-tokens 1200\n  - Use `mdlens search` only when scout clearly found the wrong file or when\n    you need a second independent query.\n\nHow to interpret scout output:\n  [queries]   Search expansions derived from the question.\n  [files]     Candidate files, picked section ids, and nearby unread sections.\n  [focus]     Dominant file when the question appears single-file.\n  [highlights] Globally ranked lines/table rows likely to answer the question.\n  [evidence]  Bounded excerpts from the selected sections.\n\nQuestion-shape guidance:\n  - Current-vs-stale questions: prefer sections marked current/current loader;\n    treat Do Not Use, stale notes, copied tables, and old runbooks as distractors.\n  - Table questions: keep the table header with the selected row; do not average\n    unrelated rows unless the document says to.\n  - Why, policy, safety, privacy, negative, or tradeoff questions: include the\n    compact rule/risk/rationale bullets, not only the command or metric.\n  - Multi-file comparison: answer each named entity separately, then summarize\n    the shared pattern.\n  - Missing evidence: say the corpus does not specify the fact rather than\n    guessing from file names.\n\nUseful defaults:\n  --max-tokens 1000 keeps scout cheap for most agent turns.\n  --max-sections 12 gives enough diversity before packing.\n  --max-files 4 keeps the file map readable."
)]
struct ScoutArgs {
    /// File or directory to scout
    path: String,
    /// Natural-language question or retrieval goal
    question: String,
    /// Output JSON (machine-readable with schema_version)
    #[arg(long)]
    json: bool,
    /// Approximate evidence-token budget (default: 1000)
    #[arg(long, default_value_t = 1000)]
    max_tokens: usize,
    /// Maximum candidate sections to consider before packing (default: 12)
    #[arg(long, default_value_t = 12)]
    max_sections: usize,
    /// Maximum files to include in the file map (default: 4)
    #[arg(long, default_value_t = 4)]
    max_files: usize,
}

#[derive(clap::Args)]
struct PackArgs {
    /// File or directory to pack from
    path: String,
    /// Comma-separated section IDs to include
    #[arg(long, conflicts_with_all = ["paths", "search"])]
    ids: Option<String>,
    /// Semicolon-separated heading paths to include
    #[arg(long, conflicts_with_all = ["ids", "search"])]
    paths: Option<String>,
    /// Search query to find sections to pack
    #[arg(long, conflicts_with_all = ["ids", "paths"])]
    search: Option<String>,
    /// Required: maximum token budget
    #[arg(long)]
    max_tokens: usize,
    /// Include parent heading context above selected sections
    #[arg(long)]
    parents: bool,
    /// Avoid duplicate nested sections (default)
    #[arg(long, conflicts_with = "no_dedupe")]
    dedupe: bool,
    /// Allow duplicate sections in the final pack
    #[arg(long, conflicts_with = "dedupe")]
    no_dedupe: bool,
    /// Use regex when selecting sections via --search
    #[arg(long)]
    regex: bool,
    /// Case-sensitive search when selecting sections via --search
    #[arg(long)]
    case_sensitive: bool,
    /// Maximum number of search results to consider for --search (default: 20)
    #[arg(long, default_value_t = 20)]
    max_results: usize,
    /// Context lines when searching via --search (default: 2)
    #[arg(long, default_value_t = 2)]
    context_lines: usize,
    /// Output JSON (machine-readable with schema_version)
    #[arg(long)]
    json: bool,
}

#[derive(Clone, ValueEnum)]
enum StatsSort {
    Path,
    Tokens,
    Lines,
}

#[derive(clap::Args)]
struct StatsArgs {
    /// File or directory to analyze
    path: String,
    /// Output JSON (machine-readable with schema_version)
    #[arg(long)]
    json: bool,
    /// Sort by field: path, tokens, or lines (default: path)
    #[arg(long, value_enum, default_value_t = StatsSort::Path)]
    sort: StatsSort,
    /// Show top N results
    #[arg(long)]
    top: Option<usize>,
}

#[derive(clap::Args)]
struct SectionsArgs {
    /// File paths to process (alternative or supplement to stdin)
    #[arg(value_name = "FILE")]
    files: Vec<String>,
    /// Include full section body text (default: metadata only)
    #[arg(long)]
    content: bool,
    /// Include descendant subsection text inside each section body
    #[arg(long)]
    children: bool,
    /// Show first N lines of each section body inline (cheaper than --content; helps pick the right section before a full read)
    #[arg(long)]
    preview: Option<usize>,
    /// Limit section hierarchy depth shown (default: unlimited)
    #[arg(long)]
    max_depth: Option<usize>,
    /// Cap total output tokens (truncates last section if exceeded)
    #[arg(long)]
    max_tokens: Option<usize>,
    /// Cap the number of sections emitted after selection/ranking
    #[arg(long)]
    max_sections: Option<usize>,
    /// Reject input if more than N files are piped (prevents accidental large reads; recommended: 5)
    #[arg(long)]
    max_files: Option<usize>,
    /// Machine-readable JSON output
    #[arg(long)]
    json: bool,
    /// Include heading path (e.g. "SGOCR Champion > Candidate Quality")
    #[arg(long)]
    heading_paths: bool,
    /// Include original line numbers (start-end)
    #[arg(long)]
    lines: bool,
    /// Deduplicate sections if same section matches multiple lines (default: true)
    #[arg(long, default_value_t = true)]
    dedupe: bool,
    /// Allow duplicate sections in output
    #[arg(long, conflicts_with = "dedupe")]
    no_dedupe: bool,
}

#[derive(clap::Args)]
struct InitArgs {
    /// Write to user-level config files (e.g. ~/.claude/CLAUDE.md) instead of the project
    #[arg(long, short = 'g')]
    global: bool,
    /// Wire into Claude Code (CLAUDE.md)
    #[arg(long)]
    claude: bool,
    /// Wire into Codex / AGENTS.md (also covers opencode)
    #[arg(long)]
    codex: bool,
    /// Wire into Gemini CLI (GEMINI.md)
    #[arg(long)]
    gemini: bool,
    /// Wire into GitHub Copilot (.github/copilot-instructions.md)
    #[arg(long)]
    copilot: bool,
    /// Wire into Cursor (.cursor/rules/mdlens.md)
    #[arg(long)]
    cursor: bool,
    /// Select a harness by name: claude, codex, gemini, copilot, cursor (repeatable)
    #[arg(long, value_name = "NAME")]
    agent: Vec<String>,
    /// Project root to write into (default: current directory)
    #[arg(long, default_value = ".")]
    path: String,
    /// Show what would change without writing
    #[arg(long)]
    dry_run: bool,
}

#[derive(clap::Args)]
struct GainArgs {
    /// Output JSON (machine-readable with schema_version)
    #[arg(long)]
    json: bool,
    /// Reset accumulated savings history to zero (requires --yes to confirm)
    #[arg(long)]
    reset: bool,
    /// Confirm a destructive --reset without a prompt
    #[arg(long)]
    yes: bool,
}

#[derive(Clone)]
struct SectionHit {
    path: String,
    line: usize,
}

enum SectionInput {
    File(String),
    Hit(SectionHit),
}

pub fn run() -> Result<()> {
    let cli = Cli::parse();

    match cli.command {
        Commands::Tree(args) => cmd_tree(args),
        Commands::Read(args) => cmd_read(args),
        Commands::Search(args) => cmd_search(args),
        Commands::Scout(args) => cmd_scout(args),
        Commands::Pack(args) => cmd_pack(args),
        Commands::Stats(args) => cmd_stats(args),
        Commands::Sections(args) => cmd_sections(args),
        Commands::Init(args) => cmd_init(args),
        Commands::Gain(args) => cmd_gain(args),
    }
}

fn cmd_gain(args: GainArgs) -> Result<()> {
    crate::gain::run_gain(args.json, args.reset, args.yes)
}

fn cmd_init(args: InitArgs) -> Result<()> {
    use crate::init::{self, Change, Harness};

    // Collect explicitly-selected harnesses from both flag and --agent forms.
    let mut selected: Vec<Harness> = Vec::new();
    let push = |h: Harness, v: &mut Vec<Harness>| {
        if !v.contains(&h) {
            v.push(h);
        }
    };
    if args.claude {
        push(Harness::Claude, &mut selected);
    }
    if args.codex {
        push(Harness::Codex, &mut selected);
    }
    if args.gemini {
        push(Harness::Gemini, &mut selected);
    }
    if args.copilot {
        push(Harness::Copilot, &mut selected);
    }
    if args.cursor {
        push(Harness::Cursor, &mut selected);
    }
    for name in &args.agent {
        match Harness::from_name(name) {
            Some(h) => push(h, &mut selected),
            None => {
                return Err(anyhow::anyhow!(
                    "unknown harness '{}' (expected: claude, codex, gemini, copilot, cursor)",
                    name
                ))
            }
        }
    }
    if selected.is_empty() {
        selected = init::default_harnesses();
    }

    let root = std::path::PathBuf::from(&args.path);
    let outcomes = init::run_init(&selected, args.global, args.dry_run, root)?;

    if args.dry_run {
        println!("mdlens init (dry run — no files written)");
    } else {
        println!("mdlens init");
    }
    for o in &outcomes {
        let target = o
            .path
            .as_ref()
            .map(|p| p.display().to_string())
            .unwrap_or_else(|| "(no global config — run without -g for this harness)".to_string());
        let status = match o.change {
            Change::Created => "created",
            Change::UpdatedBlock => "updated",
            Change::AlreadyCurrent => "up to date",
            Change::SkippedNoGlobal => "skipped",
        };
        println!("  [{}] {}  ->  {}", status, o.harness.label(), target);
    }

    // If everything was skipped (e.g. `init -g --cursor`), nothing was done —
    // surface that as an error rather than a silent success.
    if outcomes
        .iter()
        .all(|o| matches!(o.change, Change::SkippedNoGlobal))
    {
        return Err(anyhow::anyhow!(
            "nothing to do: the selected harness(es) have no global config file — re-run without -g"
        ));
    }
    Ok(())
}

fn cmd_tree(args: TreeArgs) -> Result<()> {
    let files = crate::search::discover_markdown_files(&args.path)?;

    if files.len() == 1 {
        let doc = parse_markdown(&files[0])?;
        if args.json {
            let output = TreeJsonOutput {
                schema_version: 1,
                path: doc.path.clone(),
                line_count: doc.line_count,
                byte_count: doc.byte_count,
                char_count: doc.char_count,
                word_count: doc.word_count,
                token_estimate: doc.token_estimate,
                sections: serialize_sections(
                    &doc.sections,
                    args.max_depth,
                    args.include_preamble,
                    0,
                ),
            };
            println!("{}", serde_json::to_string_pretty(&output)?);
        } else {
            println!(
                "{}",
                render_tree(&doc, args.max_depth, args.include_preamble)
            );
        }
    } else {
        // Multiple files — cap depth at 1 by default to keep directory output manageable
        let depth_capped = args.max_depth.is_none();
        let effective_depth = args.max_depth.or(Some(1));

        if args.json {
            let mut file_outputs = Vec::new();
            for file in &files {
                let doc = parse_markdown(file)?;
                file_outputs.push(TreeFileJsonOutput {
                    path: doc.path.clone(),
                    line_count: doc.line_count,
                    byte_count: doc.byte_count,
                    char_count: doc.char_count,
                    word_count: doc.word_count,
                    token_estimate: doc.token_estimate,
                    sections: serialize_sections(
                        &doc.sections,
                        effective_depth,
                        args.include_preamble,
                        0,
                    ),
                });
            }
            let output = TreeMultiJsonOutput {
                schema_version: 1,
                files: file_outputs,
            };
            println!("{}", serde_json::to_string_pretty(&output)?);
        } else {
            for file in &files {
                let doc = parse_markdown(file)?;
                println!(
                    "\n{}",
                    render_tree(&doc, effective_depth, args.include_preamble)
                );
            }
            if depth_capped {
                eprintln!("[tree] directory mode: showing depth ≤1 by default; use --max-depth N for more");
            }
        }
    }

    Ok(())
}

fn cmd_read(args: ReadArgs) -> Result<()> {
    let parsed = load_markdown(&args.file)?;
    let doc = &parsed.doc;
    let lines = &parsed.lines;
    let include_children = !args.no_children || args.children;

    let (section_text, section_meta, selector_type, selector_value, section_ref) =
        if let Some(ref id) = args.id {
            let section = doc
                .find_section_by_id(id)
                .ok_or_else(|| anyhow::anyhow!("section id not found: {id}"))?;
            let content = if include_children {
                section.extract_content(lines)
            } else {
                section.extract_direct_content(lines)
            }
            .join("\n");
            (
                content,
                SectionMeta::from(section),
                "id",
                id.clone(),
                Some(section),
            )
        } else if let Some(ref path_str) = args.heading_path {
            let section = find_unique_section_by_path(doc, path_str)?;
            let content = if include_children {
                section.extract_content(lines)
            } else {
                section.extract_direct_content(lines)
            }
            .join("\n");
            (
                content,
                SectionMeta::from(section),
                "path",
                path_str.clone(),
                Some(section),
            )
        } else if let Some(ref lines_str) = args.lines {
            let parts: Vec<&str> = lines_str.split(':').collect();
            if parts.len() != 2 {
                return Err(anyhow::anyhow!(
                    "invalid line range: {}; expected format START:END",
                    lines_str
                ));
            }
            let start: usize = parts[0].trim().parse()?;
            let end: usize = parts[1].trim().parse()?;
            if start > end {
                return Err(errors::invalid_line_range(start, end));
            }
            if start < 1 || end > lines.len() {
                return Err(anyhow::anyhow!(
                    "line range {}:{} out of bounds (file has {} lines)",
                    start,
                    end,
                    lines.len()
                ));
            }
            let content = lines[(start - 1)..end].join("\n");
            let token_est = estimate_tokens(&content);
            (
                content,
                SectionMeta {
                    id: format!("lines:{}:{}", start, end),
                    title: format!("Lines {}-{}", start, end),
                    level: 0,
                    path: vec![format!("Lines {}-{}", start, end)],
                    line_start: start,
                    line_end: end,
                    token_estimate: token_est,
                },
                "lines",
                format!("{}:{}", start, end),
                None,
            )
        } else {
            return Err(anyhow::anyhow!(
                "exactly one of --id, --heading-path, or --lines is required"
            ));
        };

    let mut full_content = String::new();

    if args.parents {
        if let Some(sec) = section_ref {
            let parents = find_parent_headings(doc, sec);
            for line_idx in parents {
                if !full_content.is_empty() {
                    full_content.push_str("\n\n");
                }
                full_content.push_str(&lines[line_idx - 1]);
            }
        }
    }

    if !full_content.is_empty() && !section_text.is_empty() {
        full_content.push_str("\n\n");
    }
    full_content.push_str(&section_text);

    let truncated = if let Some(max_tokens) = args.max_tokens {
        if estimate_tokens(&full_content) > max_tokens {
            full_content = truncate_content_to_tokens(&full_content, max_tokens);
            true
        } else {
            false
        }
    } else {
        false
    };

    if args.json {
        let output = ReadJsonOutput {
            schema_version: 1,
            path: doc.path.clone(),
            selector: ReadSelector {
                r#type: selector_type.to_string(),
                value: selector_value.to_string(),
            },
            section: SectionJsonOutput {
                id: section_meta.id.clone(),
                title: section_meta.title.clone(),
                level: section_meta.level,
                path: section_meta.path.clone(),
                line_start: section_meta.line_start,
                line_end: section_meta.line_end,
                token_estimate: section_meta.token_estimate,
                children: Vec::new(),
            },
            content: full_content,
            truncated,
        };
        // Record against the JSON the agent actually receives.
        let json = serde_json::to_string_pretty(&output)?;
        crate::gain::record("read", doc.token_estimate, estimate_tokens(&json));
        println!("{json}");
    } else {
        let section = Section {
            id: section_meta.id.clone(),
            slug: Section::slugify(&section_meta.title),
            title: section_meta.title.clone(),
            level: section_meta.level,
            path: section_meta.path.clone(),
            line_start: section_meta.line_start,
            line_end: section_meta.line_end,
            content_line_start: section_meta.line_start,
            byte_start: 0,
            byte_end: 0,
            char_count: 0,
            word_count: 0,
            token_estimate: section_meta.token_estimate,
            children: Vec::new(),
        };
        let rendered = render_read(&section, &full_content, truncated);
        crate::gain::record("read", doc.token_estimate, estimate_tokens(&rendered));
        println!("{rendered}");
    }

    Ok(())
}

struct SectionMeta {
    id: String,
    title: String,
    level: u8,
    path: Vec<String>,
    line_start: usize,
    line_end: usize,
    token_estimate: usize,
}

impl From<&Section> for SectionMeta {
    fn from(s: &Section) -> Self {
        SectionMeta {
            id: s.id.clone(),
            title: s.title.clone(),
            level: s.level,
            path: s.path.clone(),
            line_start: s.line_start,
            line_end: s.line_end,
            token_estimate: s.token_estimate,
        }
    }
}

/// Find parent heading line numbers for a section.
fn find_parent_headings(doc: &crate::model::Document, section: &Section) -> Vec<usize> {
    let mut parent_map: std::collections::HashMap<String, Option<String>> =
        std::collections::HashMap::new();
    build_parent_map(&doc.sections, None, &mut parent_map);
    let mut chain = Vec::new();
    let mut current_id = section.id.clone();
    while let Some(Some(pid)) = parent_map.get(&current_id) {
        if let Some(parent_sec) = doc.find_section_by_id(pid) {
            chain.push(parent_sec.line_start);
        }
        current_id = pid.clone();
    }
    chain.reverse();
    chain
}

fn find_unique_section_by_path<'a>(
    doc: &'a crate::model::Document,
    path_str: &str,
) -> Result<&'a Section> {
    let path = parse_heading_path(path_str);
    let matches = doc.find_sections_by_path(&path);
    match matches.len() {
        0 => Err(anyhow::anyhow!("path not found: {path_str}")),
        1 => Ok(matches[0]),
        _ => Err(errors::ambiguous_path(path_str, &matches)),
    }
}

fn parse_heading_path(path: &str) -> Vec<String> {
    let mut parts = Vec::new();
    let mut current = String::new();
    let mut escaped = false;

    for ch in path.chars() {
        if escaped {
            current.push(ch);
            escaped = false;
            continue;
        }

        match ch {
            '\\' => escaped = true,
            '>' => {
                let part = current.trim();
                if !part.is_empty() {
                    parts.push(part.to_string());
                }
                current.clear();
            }
            _ => current.push(ch),
        }
    }

    let part = current.trim();
    if !part.is_empty() {
        parts.push(part.to_string());
    }

    parts
}

fn build_parent_map(
    sections: &[Section],
    parent_id: Option<String>,
    map: &mut std::collections::HashMap<String, Option<String>>,
) {
    for section in sections {
        map.insert(section.id.clone(), parent_id.clone());
        build_parent_map(&section.children, Some(section.id.clone()), map);
    }
}

fn cmd_search(args: SearchArgs) -> Result<()> {
    let mut results = search_files(
        &args.path,
        &args.query,
        args.case_sensitive,
        args.regex,
        args.max_results,
        args.context_lines,
    )?;

    if args.content || args.preview.is_some() || args.max_tokens.is_some() {
        enrich_search_results(&mut results, args.content, args.preview)?;
    }

    if let Some(max_tokens) = args.max_tokens {
        let mut kept = Vec::new();
        let mut total_tokens = 0usize;
        for result in results {
            let item_tokens = if args.content {
                result
                    .body
                    .as_ref()
                    .map(|body| estimate_tokens(body))
                    .unwrap_or(result.token_estimate)
            } else if let Some(preview) = &result.preview {
                estimate_tokens(preview)
            } else {
                result.token_estimate
            };
            if total_tokens + item_tokens > max_tokens {
                break;
            }
            total_tokens += item_tokens;
            kept.push(result);
        }
        results = kept;
    }

    if args.json {
        let output = SearchJsonOutput {
            schema_version: 1,
            query: args.query,
            root: args.path,
            results: results
                .iter()
                .map(|r| SearchJsonResult {
                    path: r.path.clone(),
                    section_id: r.section_id.clone(),
                    section_title: r.section_title.clone(),
                    section_path: r.section_path.clone(),
                    line_start: r.line_start,
                    line_end: r.line_end,
                    token_estimate: r.token_estimate,
                    match_count: r.match_count,
                    body: r.body.clone(),
                    preview: r.preview.clone(),
                    snippets: r
                        .snippets
                        .iter()
                        .map(|s| SearchJsonSnippet {
                            line_start: s.line_start,
                            line_end: s.line_end,
                            text: s.text.clone(),
                        })
                        .collect(),
                })
                .collect(),
        };
        println!("{}", serde_json::to_string_pretty(&output)?);
    } else {
        let file_sections = build_file_sections_map(&results);
        println!("{}", render_search(&results, args.content, &file_sections));
    }

    Ok(())
}

fn build_file_sections_map(results: &[crate::render::SearchResult]) -> FileSectionsMap {
    let unique_files: std::collections::HashSet<&str> =
        results.iter().map(|r| r.path.as_str()).collect();
    let mut map = FileSectionsMap::new();
    for path in unique_files {
        if let Ok(summaries) = get_doc_section_summaries(path) {
            map.insert(path.to_string(), summaries);
        }
    }
    map
}

#[derive(Clone, Serialize)]
struct ScoutCandidate {
    path: String,
    section_id: String,
    score: i32,
    reason: String,
}

struct ScoutHighlight {
    score: i32,
    path: String,
    section_id: String,
    line_no: usize,
    line: String,
}

fn cmd_scout(args: ScoutArgs) -> Result<()> {
    let queries = scout_queries(&args.question);
    let mut candidates: Vec<ScoutCandidate> = Vec::new();
    let per_query_results = (args.max_sections * 3).max(args.max_sections).min(60);

    for query in &queries {
        let results = search_files(&args.path, query, false, false, per_query_results, 2)?;
        for result in results {
            let query_tokens = signal_tokens(query);
            let normalized_path = normalize_for_match(&result.path);
            let path_quality_score = scout_path_quality_score(&result.path);
            let path_hits = query_tokens
                .iter()
                .filter(|token| normalized_path.contains(&normalize_for_match(token)))
                .count() as i32;
            let path_boost = if path_hits > 0 {
                180 + path_hits * 45
            } else {
                0
            };
            let broad_penalty = if path_hits == 0 && query_tokens.len() <= 1 {
                60
            } else {
                0
            };
            candidates.push(ScoutCandidate {
                path: result.path,
                section_id: result.section_id,
                score: 100
                    + path_boost
                    + path_quality_score
                    + result.match_count as i32 * 5
                    + scout_heading_score(
                        &result.section_path,
                        &result.section_title,
                        &args.question,
                    )
                    - result.token_estimate as i32 / 250
                    - broad_penalty,
                reason: format!("content match: {query}"),
            });
        }
    }

    add_lexical_scout_candidates(
        &args.path,
        &args.question,
        &mut candidates,
        args.max_sections * 4,
    )?;
    add_path_match_candidates(&args.path, &args.question, &mut candidates)?;
    add_named_target_candidates(&args.path, &args.question, &mut candidates)?;
    add_neighbor_candidates(&mut candidates)?;

    candidates.sort_by(|lhs, rhs| {
        rhs.score
            .cmp(&lhs.score)
            .then(lhs.path.cmp(&rhs.path))
            .then(lhs.section_id.cmp(&rhs.section_id))
    });
    dedupe_scout_candidates(&mut candidates);
    prune_parent_scout_candidates(&mut candidates);
    let candidate_pool = candidates.clone();
    diversify_scout_candidates(&mut candidates, args.max_sections, &args.question);
    ensure_named_target_coverage(
        &mut candidates,
        &candidate_pool,
        args.max_sections,
        &args.question,
    )?;
    candidates.truncate(args.max_sections);

    let mut out = String::new();
    out.push_str(&format!(
        "[scout] question=\"{}\" budget=~{}t candidates={}\n",
        args.question,
        args.max_tokens,
        candidates.len()
    ));
    if !queries.is_empty() {
        out.push_str(&format!("[queries] {}\n", queries.join(" | ")));
    }
    out.push('\n');
    let evidence_candidates = order_scout_evidence(
        focused_scout_candidates(&candidates, &args.question),
        &args.question,
    )?;
    let map_candidates = if evidence_candidates.len() < candidates.len() {
        &evidence_candidates
    } else {
        &candidates
    };
    render_scout_file_maps(&mut out, map_candidates, args.max_files)?;
    if !evidence_candidates.is_empty() && evidence_candidates.len() < candidates.len() {
        out.push_str(&format!("\n[focus] {}\n", evidence_candidates[0].path));
    }
    out.push_str("\n[highlights]\n");
    render_scout_highlights(&mut out, &evidence_candidates, &args.question, 7)?;
    out.push_str("\n[evidence]\n");
    // baseline_tokens: total tokens of the distinct files scout pulled evidence
    // from (what the agent would otherwise read in full). Filled from the parse
    // cache render_scout_evidence already builds, so no file is parsed twice.
    let mut baseline_tokens = 0usize;
    render_scout_evidence(
        &mut out,
        &evidence_candidates,
        &args.question,
        args.max_tokens,
        &mut baseline_tokens,
    )?;

    if args.json {
        let output = ScoutJsonOutput {
            schema_version: 1,
            root: args.path,
            question: args.question,
            token_budget: args.max_tokens,
            candidate_count: candidates.len(),
            queries,
            candidates: evidence_candidates,
            rendered_text: out,
        };
        // Record against what the agent actually receives (the full JSON wrapper).
        let json = serde_json::to_string_pretty(&output)?;
        crate::gain::record("scout", baseline_tokens, estimate_tokens(&json));
        println!("{json}");
    } else {
        crate::gain::record("scout", baseline_tokens, estimate_tokens(&out));
        print!("{out}");
    }
    Ok(())
}

fn scout_queries(question: &str) -> Vec<String> {
    let mut queries = Vec::new();
    let phrases = extract_capitalized_phrases(question);
    for phrase in phrases {
        let cleaned = clean_query_phrase(&phrase);
        push_unique_query(&mut queries, cleaned.clone());
        if cleaned.contains('-') {
            push_unique_query(&mut queries, cleaned.replace('-', " "));
        }
    }

    let signal_tokens = signal_tokens(question);
    for token in signal_tokens.into_iter().take(8) {
        if token.len() >= 8
            || token.contains('-')
            || token.contains('_')
            || token.chars().any(|c| c.is_ascii_digit())
        {
            push_unique_query(&mut queries, token);
        }
    }

    if queries.is_empty() {
        push_unique_query(&mut queries, question.to_string());
    }
    queries.truncate(12);
    queries
}

fn add_lexical_scout_candidates(
    root: &str,
    question: &str,
    candidates: &mut Vec<ScoutCandidate>,
    limit: usize,
) -> Result<()> {
    let query_terms = lexical_query_terms(question);
    if query_terms.is_empty() {
        return Ok(());
    }

    struct LexicalSection {
        path: String,
        section_id: String,
        section_path: Vec<String>,
        section_title: String,
        token_estimate: usize,
        len: usize,
        terms: HashMap<String, usize>,
        title_terms: HashSet<String>,
        path_terms: HashSet<String>,
    }

    let files = discover_markdown_files(root)?;
    let mut sections = Vec::new();
    let mut df: HashMap<String, usize> = HashMap::new();
    let mut total_len = 0usize;

    for file in files {
        let parsed = load_markdown(&file)?;
        let path_terms = lexical_terms(&file).into_iter().collect::<HashSet<_>>();
        for section in flatten_doc_sections(&parsed.doc.sections) {
            if section.title == "<preamble>" {
                continue;
            }
            let content = section.extract_content(&parsed.lines).join("\n");
            let title_text = section.path.join(" ");
            let mut terms = lexical_terms(&format!("{title_text}\n{content}"));
            if terms.is_empty() {
                continue;
            }
            let title_terms = lexical_terms(&title_text)
                .into_iter()
                .collect::<HashSet<_>>();
            let mut tf = HashMap::new();
            let mut unique = HashSet::new();
            for term in terms.drain(..) {
                *tf.entry(term.clone()).or_insert(0) += 1;
                unique.insert(term);
            }
            for term in unique {
                *df.entry(term).or_insert(0) += 1;
            }
            let len = tf.values().sum::<usize>().max(1);
            total_len += len;
            sections.push(LexicalSection {
                path: file.clone(),
                section_id: section.id.clone(),
                section_path: section.path.clone(),
                section_title: section.title.clone(),
                token_estimate: section.token_estimate,
                len,
                terms: tf,
                title_terms,
                path_terms: path_terms.clone(),
            });
        }
    }

    let n = sections.len();
    if n == 0 {
        return Ok(());
    }
    let avg_len = total_len as f64 / n as f64;
    let unique_query_terms = query_terms.into_iter().collect::<BTreeSet<_>>();
    let mut scored = Vec::new();

    for section in sections {
        let mut score = 0.0f64;
        let mut matched = 0usize;
        for term in &unique_query_terms {
            let tf = section.terms.get(term).copied().unwrap_or(0) as f64;
            let title_hit = section.title_terms.contains(term);
            let path_hit = section.path_terms.contains(term);
            if tf == 0.0 && !title_hit && !path_hit {
                continue;
            }
            matched += 1;
            let doc_freq = df.get(term).copied().unwrap_or(1) as f64;
            let idf = ((n as f64 - doc_freq + 0.5) / (doc_freq + 0.5) + 1.0).ln();
            let k1 = 1.2;
            let b = 0.75;
            let bm25 = if tf > 0.0 {
                idf * (tf * (k1 + 1.0)) / (tf + k1 * (1.0 - b + b * section.len as f64 / avg_len))
            } else {
                0.0
            };
            score += bm25;
            if title_hit {
                score += idf * 1.8;
            }
            if path_hit {
                score += idf * 1.1;
            }
        }
        if matched == 0 {
            continue;
        }
        let coverage = matched as f64 / unique_query_terms.len().max(1) as f64;
        let structural_prior =
            scout_heading_score(&section.section_path, &section.section_title, question) as f64
                / 25.0;
        let path_prior = scout_path_quality_score(&section.path) as f64 / 20.0;
        let authority_prior =
            scout_source_authority_score(&section.path, &section.section_path, "", question) as f64
                / 15.0;
        let compactness = -(section.token_estimate as f64 / 900.0);
        let final_score = (score * (0.75 + coverage)
            + structural_prior
            + path_prior
            + authority_prior
            + compactness)
            * 100.0;
        scored.push((
            final_score.round() as i32,
            section.path,
            section.section_id,
            matched,
        ));
    }

    scored.sort_by(|lhs, rhs| {
        rhs.0
            .cmp(&lhs.0)
            .then(rhs.3.cmp(&lhs.3))
            .then(lhs.1.cmp(&rhs.1))
            .then(lhs.2.cmp(&rhs.2))
    });
    for (score, path, section_id, matched) in scored.into_iter().take(limit.max(1)) {
        candidates.push(ScoutCandidate {
            path,
            section_id,
            score,
            reason: format!("lexical relevance: {matched} query terms"),
        });
    }
    Ok(())
}

fn lexical_query_terms(text: &str) -> Vec<String> {
    let mut out = Vec::new();
    for token in lexical_terms(text) {
        if token.len() >= 3
            && !matches!(
                token.as_str(),
                "answer" | "doc" | "docs" | "file" | "markdown" | "readme" | "section"
            )
            && !out.contains(&token)
        {
            out.push(token);
        }
    }
    out
}

fn lexical_terms(text: &str) -> Vec<String> {
    text.split(|c: char| !c.is_ascii_alphanumeric() && c != '_' && c != '-')
        .filter_map(normalize_lexical_term)
        .collect()
}

fn normalize_lexical_term(raw: &str) -> Option<String> {
    let mut token = raw.trim().trim_matches('-').to_ascii_lowercase();
    if token.len() < 3 || is_stopword(&token) {
        return None;
    }
    if token.chars().all(|c| c.is_ascii_digit()) {
        return Some(token);
    }
    for suffix in ["ing", "edly", "edly", "ed", "es", "s"] {
        if token.len() > suffix.len() + 3 && token.ends_with(suffix) {
            token.truncate(token.len() - suffix.len());
            break;
        }
    }
    Some(token)
}

fn scout_heading_score(section_path: &[String], section_title: &str, question: &str) -> i32 {
    let question_l = question.to_ascii_lowercase();
    let heading_l = format!("{} {}", section_path.join(" "), section_title).to_ascii_lowercase();
    let mut score = 0;

    for token in signal_tokens(question).iter().take(8) {
        if heading_l.contains(&token.to_ascii_lowercase()) {
            score += 20;
        }
    }
    for (needle, heading, weight) in [
        ("install", "install", 90),
        ("command", "install", 45),
        ("usage", "usage", 70),
        ("example", "example", 55),
        ("configure", "configuration", 70),
        ("config", "configuration", 70),
        ("option", "option", 65),
        ("hyperparameter", "hyperparameter", 75),
        ("limitation", "limitation", 90),
        ("caveat", "caveat", 90),
        ("external", "external", 45),
        ("conclude", "conclude", 70),
        ("why", "conclude", 35),
        ("analysis", "analysis", 45),
        ("failure", "failure", 55),
        ("recommend", "recommendation", 95),
        ("policy", "recommendation", 65),
        ("policy", "policy", 95),
        ("privacy", "privacy", 95),
        ("mask", "privacy", 75),
        ("masking", "privacy", 75),
        ("rule", "rule", 90),
        ("rules", "rule", 90),
        ("counting", "counting", 100),
        ("safety", "safety", 100),
        ("hazard", "safety", 75),
        ("hazard", "hazard", 85),
        ("risk", "risk", 80),
        ("why", "policy", 70),
        ("why", "rule", 70),
        ("why", "risk", 65),
        ("treat", "policy", 70),
        ("treat", "rule", 70),
        ("treat", "risk", 65),
        ("direction", "recommendation", 45),
    ] {
        if question_l.contains(needle) && heading_l.contains(heading) {
            score += weight;
        }
    }
    for (low_value, penalty) in [
        ("license", 70),
        ("citation", 80),
        ("cite", 80),
        ("contact", 55),
        ("contribute", 55),
        ("acknowledg", 55),
    ] {
        if heading_l.contains(low_value) && !question_l.contains(low_value) {
            score -= penalty;
        }
    }
    score
}

fn scout_path_quality_score(path: &str) -> i32 {
    let stem = Path::new(path)
        .file_stem()
        .and_then(|name| name.to_str())
        .unwrap_or(path)
        .to_ascii_lowercase();
    let mut score = 0;
    for marker in [
        "policy",
        "runbook",
        "guide",
        "manual",
        "spec",
        "reference",
        "card",
        "schema",
        "protocol",
    ] {
        if stem.contains(marker) {
            score += 45;
        }
    }
    for marker in [
        "scratch",
        "tmp",
        "temp",
        "draft",
        "random",
        "copied",
        "copy",
        "chat",
        "conversation",
    ] {
        if stem.contains(marker) {
            score -= 180;
        }
    }
    score
}

fn scout_source_authority_score(
    path: &str,
    section_path: &[String],
    content: &str,
    question: &str,
) -> i32 {
    let mut score = scout_path_quality_score(path);
    let question_l = question.to_ascii_lowercase();
    let heading_l = section_path.join(" ").to_ascii_lowercase();
    let content_l = content.to_ascii_lowercase();
    let combined = format!("{heading_l}\n{content_l}");

    for marker in [
        "source of truth",
        "current",
        "locked",
        "policy",
        "rule",
        "spec",
        "reference",
        "runbook",
        "known risk",
    ] {
        if combined.contains(marker) {
            score += 28;
        }
    }

    let asks_for_informal = [
        "scratch",
        "draft",
        "old note",
        "old notes",
        "stale",
        "historical",
        "outdated",
        "do not use",
    ]
    .iter()
    .any(|needle| question_l.contains(needle));
    let low_authority_multiplier = if asks_for_informal { 1 } else { 2 };
    for (marker, penalty) in [
        ("not authoritative", 180),
        ("maybe stale", 140),
        ("random copied", 120),
        ("todo maybe", 110),
        ("scratch note", 100),
        ("copied wrong", 80),
        ("old notes disagree", 75),
    ] {
        if combined.contains(marker) {
            score -= penalty * low_authority_multiplier;
        }
    }

    score
}

fn wants_multi_file_evidence(question: &str) -> bool {
    let question_l = question.to_ascii_lowercase();
    [
        " across ",
        " between ",
        " compare ",
        " compares ",
        " comparing ",
        " contrast ",
        " both ",
        " each ",
        " multiple ",
        " multi-file ",
    ]
    .iter()
    .any(|needle| format!(" {question_l} ").contains(needle))
}

fn push_unique_query(queries: &mut Vec<String>, query: String) {
    let query = query
        .trim()
        .trim_matches(|c: char| !c.is_alphanumeric())
        .to_string();
    if query.len() < 3 {
        return;
    }
    if is_stopword(&query) {
        return;
    }
    if !queries
        .iter()
        .any(|existing| existing.eq_ignore_ascii_case(&query))
    {
        queries.push(query);
    }
}

fn clean_query_phrase(phrase: &str) -> String {
    phrase
        .split_whitespace()
        .filter_map(|token| {
            let cleaned =
                token.trim_matches(|c: char| !c.is_alphanumeric() && c != '-' && c != '/');
            if cleaned.eq_ignore_ascii_case("readme") || is_stopword(cleaned) {
                None
            } else {
                Some(cleaned.to_string())
            }
        })
        .collect::<Vec<_>>()
        .join(" ")
}

fn extract_capitalized_phrases(text: &str) -> Vec<String> {
    let mut phrases = Vec::new();
    let mut current: Vec<String> = Vec::new();
    for raw in text.split_whitespace() {
        let word = raw.trim_matches(|c: char| !c.is_alphanumeric() && c != '-' && c != '/');
        let is_signal = word
            .chars()
            .next()
            .is_some_and(|c| c.is_ascii_uppercase() || c.is_ascii_digit())
            || word.chars().any(|c| c.is_ascii_digit())
            || word.contains('-')
            || word.contains('/');
        if is_signal && word.len() > 1 {
            current.push(word.to_string());
            if raw.ends_with(',') || raw.ends_with(';') {
                if current.len() >= 2 || current[0].len() >= 5 {
                    phrases.push(current.join(" "));
                }
                current.clear();
            }
        } else if !current.is_empty() {
            if current.len() >= 2 || current[0].len() >= 5 {
                phrases.push(current.join(" "));
            }
            current.clear();
        }
    }
    if !current.is_empty() && (current.len() >= 2 || current[0].len() >= 5) {
        phrases.push(current.join(" "));
    }
    phrases
}

fn signal_tokens(text: &str) -> Vec<String> {
    let mut out = Vec::new();
    for raw in text.split(|c: char| !c.is_ascii_alphanumeric() && c != '_' && c != '-') {
        let token = raw.trim().trim_matches('-');
        if token.len() < 3 {
            continue;
        }
        if is_stopword(token) {
            continue;
        }
        if !out
            .iter()
            .any(|existing: &String| existing.eq_ignore_ascii_case(token))
        {
            out.push(token.to_string());
        }
    }
    out
}

fn is_stopword(token: &str) -> bool {
    matches!(
        token.to_ascii_lowercase().as_str(),
        "about"
            | "according"
            | "added"
            | "after"
            | "against"
            | "answer"
            | "are"
            | "across"
            | "before"
            | "between"
            | "can"
            | "compared"
            | "complete"
            | "does"
            | "during"
            | "explain"
            | "fit"
            | "for"
            | "from"
            | "given"
            | "good"
            | "has"
            | "have"
            | "how"
            | "in"
            | "instead"
            | "into"
            | "its"
            | "list"
            | "provide"
            | "readme"
            | "row"
            | "run"
            | "should"
            | "than"
            | "that"
            | "the"
            | "their"
            | "there"
            | "these"
            | "they"
            | "this"
            | "toolbox"
            | "using"
            | "user"
            | "wants"
            | "what"
            | "when"
            | "where"
            | "which"
            | "while"
            | "with"
            | "without"
            | "would"
            | "yourself"
            | "and"
    )
}

fn add_path_match_candidates(
    root: &str,
    question: &str,
    candidates: &mut Vec<ScoutCandidate>,
) -> Result<()> {
    let files = discover_markdown_files(root)?;
    let question_tokens = signal_tokens(question);
    if question_tokens.is_empty() {
        return Ok(());
    }
    for path in files {
        let normalized = normalize_for_match(&path);
        let mut hits = 0;
        for token in &question_tokens {
            if normalized.contains(&normalize_for_match(token)) {
                hits += 1;
            }
        }
        let source_like_path = scout_path_quality_score(&path) > 0;
        let policy_or_multi_question = wants_multi_file_evidence(question)
            || question.to_ascii_lowercase().contains("why")
            || question.to_ascii_lowercase().contains("rule")
            || question.to_ascii_lowercase().contains("policy")
            || question.to_ascii_lowercase().contains("safety")
            || question.to_ascii_lowercase().contains("privacy");
        let required_hits = if source_like_path && policy_or_multi_question {
            1
        } else {
            2
        };
        if hits < required_hits {
            continue;
        }
        let parsed = load_markdown(&path)?;
        for section in parsed.doc.sections.iter().take(2) {
            candidates.push(ScoutCandidate {
                path: path.clone(),
                section_id: section.id.clone(),
                score: 240 + hits * 30,
                reason: "path/name match".to_string(),
            });
        }
        if let Some(best) = best_named_section(&parsed.doc.sections, question) {
            candidates.push(ScoutCandidate {
                path: path.clone(),
                section_id: best.id.clone(),
                score: 300
                    + hits * 45
                    + scout_path_quality_score(&path)
                    + scout_heading_score(&best.path, &best.title, question),
                reason: "path/name match + relevant heading".to_string(),
            });
        }
    }
    Ok(())
}

fn add_named_target_candidates(
    root: &str,
    question: &str,
    candidates: &mut Vec<ScoutCandidate>,
) -> Result<()> {
    let targets = target_phrases_from_question(question);
    if targets.len() < 2 {
        return Ok(());
    }

    for target in targets {
        let results = search_files(root, &target, false, false, 12, 2)?;
        let mut seen_files = HashSet::new();
        for result in results.into_iter().take(8) {
            let content_authority =
                scout_source_authority_score(&result.path, &result.section_path, "", question);
            candidates.push(ScoutCandidate {
                path: result.path.clone(),
                section_id: result.section_id.clone(),
                score: 620
                    + content_authority
                    + result.match_count as i32 * 20
                    + scout_heading_score(&result.section_path, &result.section_title, question),
                reason: format!("named target: {target}"),
            });

            if seen_files.insert(result.path.clone()) {
                let parsed = load_markdown(&result.path)?;
                if let Some(best) = best_named_section(&parsed.doc.sections, question) {
                    candidates.push(ScoutCandidate {
                        path: result.path.clone(),
                        section_id: best.id.clone(),
                        score: 760
                            + scout_source_authority_score(&result.path, &best.path, "", question)
                            + scout_heading_score(&best.path, &best.title, question),
                        reason: format!("named target + relevant heading: {target}"),
                    });
                }
            }
        }
    }
    Ok(())
}

fn normalize_for_match(text: &str) -> String {
    text.chars()
        .map(|c| {
            if c.is_ascii_alphanumeric() {
                c.to_ascii_lowercase()
            } else {
                ' '
            }
        })
        .collect::<String>()
}

fn best_named_section<'a>(sections: &'a [Section], question: &str) -> Option<&'a Section> {
    let mut best: Option<(&Section, i32)> = None;
    score_named_sections(sections, question, &mut best);
    best.map(|(section, _)| section)
}

fn score_named_sections<'a>(
    sections: &'a [Section],
    question: &str,
    best: &mut Option<(&'a Section, i32)>,
) {
    for section in sections {
        let title = section.title.to_ascii_lowercase();
        let mut score = 0;
        for (needle, weight) in [
            ("usage", 30),
            ("install", 30),
            ("quick", 20),
            ("example", 20),
            ("configuration", 20),
            ("training", 20),
            ("preprocess", 20),
            ("limitation", 25),
            ("caveat", 25),
            ("documentation", 10),
            ("overview", 10),
            ("policy", 120),
            ("privacy", 110),
            ("rule", 115),
            ("counting", 110),
            ("safety", 115),
            ("risk", 90),
            ("current", 75),
            ("stale", 75),
        ] {
            if title.contains(needle) {
                score += weight;
            }
        }
        for token in signal_tokens(question).iter().take(8) {
            if title.contains(&token.to_ascii_lowercase()) {
                score += 25;
            }
        }
        if score > 0 && best.is_none_or(|(_, best_score)| score > best_score) {
            *best = Some((section, score));
        }
        score_named_sections(&section.children, question, best);
    }
}

fn add_neighbor_candidates(candidates: &mut Vec<ScoutCandidate>) -> Result<()> {
    let originals = candidates.to_vec();
    let mut by_file: HashMap<String, HashSet<String>> = HashMap::new();
    for candidate in &originals {
        by_file
            .entry(candidate.path.clone())
            .or_default()
            .insert(candidate.section_id.clone());
    }
    // Deterministic iteration: HashMap order is unspecified, so sort by path.
    // (Downstream sorting currently masks this, but relying on that is fragile.)
    let mut by_file: Vec<(String, HashSet<String>)> = by_file.into_iter().collect();
    by_file.sort_by(|a, b| a.0.cmp(&b.0));
    for (path, ids) in by_file {
        let parsed = load_markdown(&path)?;
        let flat = flatten_doc_sections(&parsed.doc.sections);
        for (idx, section) in flat.iter().enumerate() {
            if !ids.contains(&section.id) {
                continue;
            }
            let start = idx.saturating_sub(1);
            let end = (idx + 1).min(flat.len().saturating_sub(1));
            for neighbor in flat.iter().take(end + 1).skip(start) {
                if neighbor.id == section.id {
                    continue;
                }
                candidates.push(ScoutCandidate {
                    path: path.clone(),
                    section_id: neighbor.id.clone(),
                    score: 70,
                    reason: format!("neighbor of §{}", section.id),
                });
            }
        }
    }
    Ok(())
}

fn flatten_doc_sections(sections: &[Section]) -> Vec<&Section> {
    let mut out = Vec::new();
    collect_flat_sections(sections, &mut out);
    out.sort_by_key(|section| section.line_start);
    out
}

fn collect_flat_sections<'a>(sections: &'a [Section], out: &mut Vec<&'a Section>) {
    for section in sections {
        out.push(section);
        collect_flat_sections(&section.children, out);
    }
}

fn dedupe_scout_candidates(candidates: &mut Vec<ScoutCandidate>) {
    let mut seen = HashSet::new();
    candidates
        .retain(|candidate| seen.insert(format!("{}::{}", candidate.path, candidate.section_id)));
}

fn prune_parent_scout_candidates(candidates: &mut Vec<ScoutCandidate>) {
    let ids_by_file: HashMap<String, Vec<String>> =
        candidates
            .iter()
            .fold(HashMap::new(), |mut by_file, candidate| {
                by_file
                    .entry(candidate.path.clone())
                    .or_default()
                    .push(candidate.section_id.clone());
                by_file
            });

    candidates.retain(|candidate| {
        !ids_by_file.get(&candidate.path).is_some_and(|ids| {
            ids.iter()
                .any(|id| is_child_section_id(&candidate.section_id, id))
        })
    });
}

fn diversify_scout_candidates(
    candidates: &mut Vec<ScoutCandidate>,
    max_sections: usize,
    question: &str,
) {
    if !wants_multi_file_evidence(question) || candidates.len() <= max_sections {
        return;
    }

    let mut targets = target_phrases_from_question(question);
    if targets.len() < 2 {
        targets = target_tokens_from_question(question);
    }
    if let Some(selected) =
        target_coverage_scout_candidates(candidates, max_sections, &targets, question)
    {
        *candidates = selected;
        return;
    }

    let mut selected = Vec::new();
    let mut selected_keys = HashSet::new();
    let mut per_file_count: HashMap<String, usize> = HashMap::new();

    for candidate in candidates.iter() {
        if selected.len() >= max_sections {
            break;
        }
        let count = per_file_count.get(&candidate.path).copied().unwrap_or(0);
        if count >= 2 {
            continue;
        }
        let key = format!("{}::{}", candidate.path, candidate.section_id);
        if selected_keys.insert(key) {
            selected.push(candidate.clone());
            *per_file_count.entry(candidate.path.clone()).or_default() += 1;
        }
    }

    for candidate in candidates.iter() {
        if selected.len() >= max_sections {
            break;
        }
        let key = format!("{}::{}", candidate.path, candidate.section_id);
        if selected_keys.insert(key) {
            selected.push(candidate.clone());
        }
    }

    if selected.len() >= 2 {
        *candidates = selected;
    }
}

fn target_coverage_scout_candidates(
    candidates: &[ScoutCandidate],
    max_sections: usize,
    targets: &[String],
    question: &str,
) -> Option<Vec<ScoutCandidate>> {
    if targets.len() < 2 || max_sections == 0 {
        return None;
    }

    let mut cache: HashMap<String, crate::parse::ParsedMarkdown> = HashMap::new();
    let mut selected = Vec::new();
    let mut selected_keys = HashSet::new();
    let mut covered_targets: HashSet<String> = HashSet::new();
    let mut per_file_count: HashMap<String, usize> = HashMap::new();

    while selected.len() < max_sections {
        let mut best_idx = None;
        let mut best_score = i32::MIN;
        let mut best_new_targets = HashSet::new();

        for (idx, candidate) in candidates.iter().enumerate() {
            let key = format!("{}::{}", candidate.path, candidate.section_id);
            if selected_keys.contains(&key) {
                continue;
            }
            let Ok((target_hits, authority)) =
                scout_candidate_target_hits(candidate, targets, question, &mut cache)
            else {
                continue;
            };
            let new_targets = target_hits
                .difference(&covered_targets)
                .cloned()
                .collect::<HashSet<_>>();
            if new_targets.is_empty() && covered_targets.len() < targets.len() {
                continue;
            }
            let same_file_penalty =
                per_file_count.get(&candidate.path).copied().unwrap_or(0) as i32 * 160;
            let coverage_gain = new_targets.len() as i32 * 420 + target_hits.len() as i32 * 35;
            let score = candidate.score + authority + coverage_gain - same_file_penalty;
            if score > best_score {
                best_score = score;
                best_idx = Some(idx);
                best_new_targets = new_targets;
            }
        }

        let Some(idx) = best_idx else {
            break;
        };
        let candidate = candidates[idx].clone();
        let key = format!("{}::{}", candidate.path, candidate.section_id);
        selected_keys.insert(key);
        for target in best_new_targets {
            covered_targets.insert(target);
        }
        *per_file_count.entry(candidate.path.clone()).or_default() += 1;
        selected.push(candidate);

        if covered_targets.len() >= targets.len() {
            break;
        }
    }

    if selected.len() < 2 {
        return None;
    }

    for candidate in candidates {
        if selected.len() >= max_sections {
            break;
        }
        let key = format!("{}::{}", candidate.path, candidate.section_id);
        if selected_keys.contains(&key) {
            continue;
        }
        let Ok((_, authority)) =
            scout_candidate_target_hits(candidate, targets, question, &mut cache)
        else {
            continue;
        };
        if authority < -250 && selected.len() >= 2 {
            continue;
        }
        selected_keys.insert(key);
        selected.push(candidate.clone());
    }

    Some(selected)
}

fn ensure_named_target_coverage(
    selected: &mut Vec<ScoutCandidate>,
    pool: &[ScoutCandidate],
    max_sections: usize,
    question: &str,
) -> Result<()> {
    let targets = target_phrases_from_question(question);
    if targets.len() < 2 || max_sections == 0 {
        return Ok(());
    }

    let mut cache: HashMap<String, crate::parse::ParsedMarkdown> = HashMap::new();
    let mut selected_keys = selected
        .iter()
        .map(|candidate| format!("{}::{}", candidate.path, candidate.section_id))
        .collect::<HashSet<_>>();
    let mut covered = HashSet::new();
    for candidate in selected.iter() {
        let (hits, _) = scout_candidate_target_hits(candidate, &targets, question, &mut cache)?;
        covered.extend(hits);
    }

    for target in targets {
        if covered.contains(&target) {
            continue;
        }

        let mut best: Option<(ScoutCandidate, i32)> = None;
        for candidate in pool {
            let key = format!("{}::{}", candidate.path, candidate.section_id);
            if selected_keys.contains(&key) {
                continue;
            }
            let (hits, authority) = scout_candidate_target_hits(
                candidate,
                std::slice::from_ref(&target),
                question,
                &mut cache,
            )?;
            if hits.is_empty() {
                continue;
            }
            let score = candidate.score + authority;
            if best
                .as_ref()
                .is_none_or(|(_, best_score)| score > *best_score)
            {
                best = Some((candidate.clone(), score));
            }
        }

        let Some((candidate, _)) = best else {
            continue;
        };
        let key = format!("{}::{}", candidate.path, candidate.section_id);
        if selected.len() >= max_sections {
            selected.pop();
        }
        selected_keys.insert(key);
        covered.insert(target);
        selected.push(candidate);
    }

    Ok(())
}

fn scout_candidate_target_hits(
    candidate: &ScoutCandidate,
    targets: &[String],
    question: &str,
    cache: &mut HashMap<String, crate::parse::ParsedMarkdown>,
) -> Result<(HashSet<String>, i32)> {
    if !cache.contains_key(&candidate.path) {
        cache.insert(candidate.path.clone(), load_markdown(&candidate.path)?);
    }
    let parsed = cache.get(&candidate.path).expect("cached parsed markdown");
    let Some(section) = parsed.doc.find_section_by_id(&candidate.section_id) else {
        return Ok((HashSet::new(), scout_path_quality_score(&candidate.path)));
    };
    let content = section.extract_content(&parsed.lines).join("\n");
    let source_haystack =
        normalize_compact(&format!("{}\n{}", candidate.path, section.path.join(" ")));
    let haystack = normalize_compact(&format!(
        "{}\n{}\n{}",
        candidate.path,
        section.path.join(" "),
        content
    ));
    let hits = targets
        .iter()
        .filter(|target| haystack.contains(&normalize_compact(target)))
        .cloned()
        .collect::<HashSet<_>>();
    let source_hit_count = targets
        .iter()
        .filter(|target| source_haystack.contains(&normalize_compact(target)))
        .count() as i32;
    let mut authority =
        scout_source_authority_score(&candidate.path, &section.path, &content, question);
    authority += source_hit_count * 360;
    if source_hit_count == 0 && !hits.is_empty() {
        authority -= 120;
    }
    Ok((hits, authority))
}

fn is_child_section_id(parent: &str, child: &str) -> bool {
    child.len() > parent.len()
        && child.starts_with(parent)
        && child[parent.len()..].starts_with('.')
}

fn focused_scout_candidates(candidates: &[ScoutCandidate], question: &str) -> Vec<ScoutCandidate> {
    let Some(top) = candidates.first() else {
        return Vec::new();
    };
    if wants_multi_file_evidence(question) {
        let targets = target_tokens_from_question(question);
        if !targets.is_empty() {
            let focused = candidates
                .iter()
                .filter(|candidate| path_matches_any_target(&candidate.path, &targets))
                .cloned()
                .collect::<Vec<_>>();
            if focused.len() >= 2 {
                return focused;
            }
        }
        return candidates.to_vec();
    }
    let top_path_tokens = distinctive_path_tokens(&top.path);
    if scout_path_quality_score(&top.path) > 0 && !top_path_tokens.is_empty() {
        let focused = candidates
            .iter()
            .filter(|candidate| {
                candidate.path == top.path
                    || distinctive_path_tokens(&candidate.path)
                        .iter()
                        .any(|token| top_path_tokens.contains(token))
            })
            .cloned()
            .collect::<Vec<_>>();
        if focused.len() >= 2 {
            return focused;
        }
    }
    let best_other_score = candidates
        .iter()
        .find(|candidate| candidate.path != top.path)
        .map(|candidate| candidate.score);
    let dominant_file =
        top.score >= 280 && best_other_score.is_none_or(|score| top.score - score >= 80);
    if dominant_file {
        candidates
            .iter()
            .filter(|candidate| candidate.path == top.path)
            .cloned()
            .collect()
    } else {
        candidates.to_vec()
    }
}

fn order_scout_evidence(
    mut candidates: Vec<ScoutCandidate>,
    question: &str,
) -> Result<Vec<ScoutCandidate>> {
    let question_l = question.to_ascii_lowercase();
    if !wants_rationale_or_policy_evidence(&question_l) {
        return Ok(candidates);
    }

    let mut cache: HashMap<String, crate::parse::ParsedMarkdown> = HashMap::new();
    let mut scored = Vec::new();
    for (idx, candidate) in candidates.drain(..).enumerate() {
        if !cache.contains_key(&candidate.path) {
            cache.insert(candidate.path.clone(), load_markdown(&candidate.path)?);
        }
        let parsed = cache.get(&candidate.path).expect("cached parsed markdown");
        let score = parsed
            .doc
            .find_section_by_id(&candidate.section_id)
            .map(|section| {
                let content = section.extract_content(&parsed.lines).join("\n");
                candidate.score
                    + scout_rationale_evidence_score(&section.path, &content, &question_l)
            })
            .unwrap_or(candidate.score);
        scored.push((score, idx, candidate));
    }
    scored.sort_by(|lhs, rhs| rhs.0.cmp(&lhs.0).then(lhs.1.cmp(&rhs.1)));
    Ok(scored
        .into_iter()
        .map(|(_, _, candidate)| candidate)
        .collect())
}

fn wants_rationale_or_policy_evidence(question_l: &str) -> bool {
    [
        "why",
        "what makes",
        "rather than",
        "policy",
        "privacy",
        "safety",
        "allow",
        "allows",
        "exporting",
        "mask",
        "masking",
        "rationale",
        "reason",
    ]
    .iter()
    .any(|needle| question_l.contains(needle))
}

fn asks_for_metric_or_table(question_l: &str) -> bool {
    [
        "metric",
        "score",
        "baseline",
        "table",
        "row",
        "0.",
        "current score",
    ]
    .iter()
    .any(|needle| question_l.contains(needle))
}

fn scout_rationale_evidence_score(section_path: &[String], content: &str, question_l: &str) -> i32 {
    let text = format!("{}\n{}", section_path.join(" "), content).to_ascii_lowercase();
    let mut score = 0;
    score += scout_rationale_marker_score(&text);
    score += scout_question_token_overlap_score(&text, question_l, 28, 220);
    if !asks_for_metric_or_table(question_l) {
        for needle in [
            "metric | score",
            "| score |",
            "baseline",
            "current metric",
            "benchmark",
            "leaderboard",
        ] {
            if text.contains(needle) {
                score -= 220;
            }
        }
    }
    score
}

fn distinctive_path_tokens(path: &str) -> HashSet<String> {
    let stem = Path::new(path)
        .file_stem()
        .and_then(|name| name.to_str())
        .unwrap_or(path);
    stem.split(|c: char| !c.is_ascii_alphanumeric())
        .map(str::to_ascii_lowercase)
        .filter(|token| {
            token.len() >= 4
                && !matches!(
                    token.as_str(),
                    "readme"
                        | "index"
                        | "docs"
                        | "doc"
                        | "notes"
                        | "note"
                        | "eval"
                        | "scene"
                        | "card"
                        | "annotation"
                        | "policy"
                        | "scratch"
                        | "draft"
                        | "copy"
                        | "copied"
                        | "tmp"
                        | "temp"
                        | "anchor"
                )
        })
        .collect()
}

fn target_tokens_from_question(question: &str) -> Vec<String> {
    let mut out = Vec::new();
    for phrase in extract_capitalized_phrases(question) {
        for token in signal_tokens(&phrase) {
            for part in token.split('-') {
                let part = part.to_ascii_lowercase();
                if part.len() >= 4 && !is_stopword(&part) && !out.contains(&part) {
                    out.push(part);
                }
            }
        }
    }
    out
}

fn target_phrases_from_question(question: &str) -> Vec<String> {
    let mut out = Vec::new();
    for phrase in extract_capitalized_phrases(question) {
        if !phrase
            .chars()
            .any(|ch| ch.is_ascii_uppercase() || ch.is_ascii_digit())
        {
            continue;
        }
        let tokens = signal_tokens(&phrase)
            .into_iter()
            .filter(|token| {
                !matches!(
                    token.to_ascii_lowercase().as_str(),
                    "compare" | "contrast" | "across" | "between" | "which"
                )
            })
            .collect::<Vec<_>>();
        if tokens.is_empty() {
            continue;
        }
        let phrase = tokens.join(" ");
        if phrase.len() >= 4 && !out.iter().any(|existing| existing == &phrase) {
            out.push(phrase);
        }
    }
    out
}

#[cfg(test)]
mod scout_tests {
    use super::{scout_adaptive_score_floor, target_phrases_from_question, ScoutCandidate};

    fn cands(scores: &[i32]) -> Vec<ScoutCandidate> {
        scores
            .iter()
            .map(|&s| ScoutCandidate {
                path: "p.md".into(),
                section_id: "s".into(),
                score: s,
                reason: "r".into(),
            })
            .collect()
    }

    #[test]
    fn adaptive_floor_cuts_a_clear_cliff() {
        // Strong head, then a hard fall to a flat tail.
        let floor = scout_adaptive_score_floor(&cands(&[800, 760, 740, 120, 100, 90, 80]));
        assert!(floor > 120, "should cut the tail at the cliff, got {floor}");
        assert!(floor <= 740, "should keep the head, got {floor}");
    }

    #[test]
    fn adaptive_floor_keeps_smooth_tail() {
        // Gentle linear decay has no knee: keep everything (i32::MIN).
        let floor = scout_adaptive_score_floor(&cands(&[300, 280, 260, 240, 220, 200, 180]));
        assert_eq!(floor, i32::MIN, "smooth decay should not be cut");
    }

    #[test]
    fn adaptive_floor_no_cut_when_few_candidates() {
        assert_eq!(scout_adaptive_score_floor(&cands(&[900, 100])), i32::MIN);
    }

    #[test]
    fn adaptive_floor_handles_unsorted_input() {
        // Same multiset as the cliff case but in non-monotonic order (as the
        // real input is) — the floor must be identical.
        let floor = scout_adaptive_score_floor(&cands(&[100, 800, 90, 740, 120, 760, 80]));
        assert!(floor > 120 && floor <= 740, "got {floor}");
    }

    #[test]
    fn target_phrases_keep_hyphenated_entities() {
        let targets = target_phrases_from_question(
            "Across Harbor-17, Rainy Rail Depot, and Night Bus Stop, how do the docs treat reflected or glare-corrupted text?",
        );
        assert!(targets.contains(&"Harbor-17".to_string()), "{targets:?}");
        assert!(
            targets.contains(&"Rainy Rail Depot".to_string()),
            "{targets:?}"
        );
        assert!(
            targets.contains(&"Night Bus Stop".to_string()),
            "{targets:?}"
        );
    }
}

fn path_matches_any_target(path: &str, targets: &[String]) -> bool {
    let path_l = normalize_compact(path);
    targets
        .iter()
        .any(|target| path_l.contains(&normalize_compact(target)))
}

fn normalize_compact(text: &str) -> String {
    text.chars()
        .filter(|c| c.is_ascii_alphanumeric())
        .map(|c| c.to_ascii_lowercase())
        .collect()
}

fn render_scout_file_maps(
    out: &mut String,
    candidates: &[ScoutCandidate],
    max_files: usize,
) -> Result<()> {
    let mut files = Vec::new();
    let mut seen = HashSet::new();
    for candidate in candidates {
        if seen.insert(candidate.path.clone()) {
            files.push(candidate.path.clone());
        }
        if files.len() >= max_files {
            break;
        }
    }
    out.push_str("[files]\n");
    for path in files {
        let summaries = get_doc_section_summaries(&path)?;
        let picked: HashSet<&str> = candidates
            .iter()
            .filter(|c| c.path == path)
            .map(|c| c.section_id.as_str())
            .collect();
        let sections = summaries
            .iter()
            .filter(|(id, title)| title != "<preamble>" && picked.contains(id.as_str()))
            .map(|(id, title)| format!("§{} {}", id, title))
            .take(6)
            .collect::<Vec<_>>();
        let also = summaries
            .iter()
            .filter(|(id, title)| title != "<preamble>" && !picked.contains(id.as_str()))
            .take(6)
            .map(|(id, title)| format!("§{} {}", id, title))
            .collect::<Vec<_>>();
        out.push_str(&format!("- {}\n", path));
        if !sections.is_empty() {
            out.push_str(&format!("  picked: {}\n", sections.join(" · ")));
        }
        if !also.is_empty() {
            out.push_str(&format!("  also: {}\n", also.join(" · ")));
        }
    }
    Ok(())
}

fn render_scout_highlights(
    out: &mut String,
    candidates: &[ScoutCandidate],
    question: &str,
    max_lines: usize,
) -> Result<()> {
    let tokens: Vec<String> = signal_tokens(question)
        .into_iter()
        .map(|token| token.to_ascii_lowercase())
        .collect();
    let question_l = question.to_ascii_lowercase();
    let wants_code = ["cli", "command", "install", "invoke"]
        .iter()
        .any(|needle| question_l.contains(needle));
    let mut emitted = 0usize;
    let mut seen = HashSet::new();
    let mut highlights = Vec::new();
    let mut cache: HashMap<String, crate::parse::ParsedMarkdown> = HashMap::new();

    for candidate in candidates {
        if !cache.contains_key(&candidate.path) {
            cache.insert(candidate.path.clone(), load_markdown(&candidate.path)?);
        }
        let parsed = cache.get(&candidate.path).expect("cached parsed markdown");
        let Some(section) = parsed.doc.find_section_by_id(&candidate.section_id) else {
            continue;
        };
        if is_low_value_section_for_question(section, &question_l) {
            continue;
        }
        let lines = section.extract_content(&parsed.lines);
        for (idx, line) in lines.iter().enumerate() {
            if emitted >= max_lines {
                break;
            }
            let trimmed = line.trim();
            let lower = trimmed.to_ascii_lowercase();
            if is_noisy_highlight_line(trimmed) && !is_relevant_table_line(trimmed, &tokens) {
                continue;
            }
            let token_hits = tokens.iter().filter(|token| lower.contains(*token)).count();
            let useful_code_line = trimmed.contains("--")
                || (wants_code
                    && (trimmed.contains('`')
                        || trimmed.starts_with("pip ")
                        || trimmed.starts_with("conda ")
                        || trimmed.starts_with("python ")
                        || trimmed.starts_with("git ")
                        || trimmed.starts_with("cmake ")
                        || trimmed.starts_with("make ")));
            let useful_table_line = is_relevant_table_line(trimmed, &tokens);
            if token_hits == 0 && !useful_code_line && !useful_table_line {
                continue;
            }
            let mut score = token_hits as i32 * 20;
            if useful_table_line {
                score += 80;
            }
            if wants_rationale_or_policy_evidence(&question_l) {
                score += scout_rationale_highlight_score(&lower, &question_l);
            }
            for (needle, weight) in [
                ("--", 70),
                ("cpu", 45),
                ("gpu", 45),
                ("warning", 45),
                ("disable", 45),
                ("configuration", 30),
                ("header", 30),
                ("human-readable", 30),
                ("supported formats", 30),
                ("convert", 30),
            ] {
                if lower.contains(needle) {
                    score += weight;
                }
            }
            highlights.push(ScoutHighlight {
                score,
                path: candidate.path.clone(),
                section_id: section.id.clone(),
                line_no: section.line_start + idx,
                line: if useful_table_line {
                    scout_table_context(lines, idx)
                } else {
                    scout_highlight_context(lines, idx, &lower)
                },
            });
        }
    }

    highlights.sort_by(|lhs, rhs| {
        rhs.score
            .cmp(&lhs.score)
            .then(lhs.path.cmp(&rhs.path))
            .then(lhs.line_no.cmp(&rhs.line_no))
    });
    for highlight in highlights {
        if emitted >= max_lines {
            break;
        }
        emit_scout_highlight(out, &mut seen, &mut emitted, &highlight);
    }

    if emitted == 0 {
        out.push_str("- no compact highlights; read evidence sections below\n");
    }
    Ok(())
}

fn scout_rationale_highlight_score(lower: &str, question_l: &str) -> i32 {
    let mut score = 0;
    score += scout_rationale_marker_score(lower) / 2;
    score += scout_question_token_overlap_score(lower, question_l, 18, 120);
    if !asks_for_metric_or_table(question_l) {
        for needle in ["| score |", "baseline", "current metric", "benchmark", "0."] {
            if lower.contains(needle) {
                score -= 140;
            }
        }
    }
    score
}

fn scout_rationale_marker_score(lower: &str) -> i32 {
    let mut score = 0;
    for (needles, weight) in [
        (
            &["rule:", "rule ", "policy", "guideline", "standard"][..],
            180,
        ),
        (
            &[
                "known risk",
                "risk",
                "unsafe",
                "wrong answer",
                "misread",
                "confus",
                "ambiguous",
            ][..],
            160,
        ),
        (
            &[
                "privacy",
                "personal data",
                "identifiable",
                "redact",
                "mask",
                "export",
                "leak",
            ][..],
            150,
        ),
        (
            &[
                "must",
                "should",
                "requires",
                "allow",
                "not enough",
                "do not",
                "rather than",
            ][..],
            100,
        ),
        (
            &["because", "reason", "rationale", "therefore", "so that"][..],
            80,
        ),
    ] {
        if needles.iter().any(|needle| lower.contains(needle)) {
            score += weight;
        }
    }
    score
}

fn scout_question_token_overlap_score(
    lower: &str,
    question_l: &str,
    per_token: i32,
    cap: i32,
) -> i32 {
    let hits = signal_tokens(question_l)
        .into_iter()
        .map(|token| token.to_ascii_lowercase())
        .filter(|token| lower.contains(token))
        .count() as i32;
    (hits * per_token).min(cap)
}

fn is_noisy_highlight_line(line: &str) -> bool {
    line.is_empty()
        || line.starts_with('|')
        || line == "```"
        || line == "```shell"
        || line.trim_matches('~') == "```"
        || line.trim_matches('~') == "```shell"
        || line.starts_with("<!--")
        || line.starts_with("[!")
        || line.starts_with("![")
        || line.starts_with("[![")
        || line.starts_with("@article")
        || line.starts_with("@inproceedings")
        || (line.starts_with('[') && line.contains("]: "))
        || line.len() > 1000
}

fn is_relevant_table_line(line: &str, tokens: &[String]) -> bool {
    line.starts_with('|')
        && line.matches('|').count() >= 3
        && !is_table_separator_line(line)
        && tokens
            .iter()
            .any(|token| line.to_ascii_lowercase().contains(token))
}

fn is_table_separator_line(line: &str) -> bool {
    line.chars()
        .all(|ch| ch == '|' || ch == '-' || ch == ':' || ch.is_whitespace())
}

fn scout_table_context(lines: &[String], idx: usize) -> String {
    let row = lines[idx].trim();
    let header = (1..idx).rev().find_map(|candidate_idx| {
        let separator = lines[candidate_idx].trim();
        if !separator.starts_with('|') || !is_table_separator_line(separator) {
            return None;
        }
        let header = lines[candidate_idx - 1].trim();
        header.starts_with('|').then_some(header)
    });

    match header {
        Some(header) if header != row => format!("{header} => {row}"),
        _ => row.to_string(),
    }
}

fn scout_highlight_context(lines: &[String], idx: usize, lower: &str) -> String {
    let radius = if lower.contains("disable") || lower.contains("warning") {
        5
    } else if lines[idx].trim().len() < 300 {
        2
    } else {
        0
    };
    let start = idx.saturating_sub(radius);
    let end = (idx + radius).min(lines.len().saturating_sub(1));
    let mut parts = Vec::new();
    for line in &lines[start..=end] {
        let trimmed = line.trim();
        if is_noisy_highlight_line(trimmed) && !trimmed.starts_with('|') {
            continue;
        }
        parts.push(trimmed);
    }
    let mut joined = parts.join(" ");
    if joined.len() > 900 {
        joined.truncate(900);
        joined.push_str("...");
    }
    joined
}

fn is_low_value_section_for_question(section: &Section, question_l: &str) -> bool {
    let section_path = section.path.join(" ").to_ascii_lowercase();
    let citation_section = section_path.contains("citation")
        || section_path.contains("cite")
        || section_path.contains("references");
    citation_section
        && !["citation", "cite", "doi", "reference", "paper"]
            .iter()
            .any(|needle| question_l.contains(needle))
}

fn emit_scout_highlight(
    out: &mut String,
    seen: &mut HashSet<String>,
    emitted: &mut usize,
    highlight: &ScoutHighlight,
) {
    let key = format!(
        "{}:{}:{}",
        highlight.path, highlight.line_no, highlight.line
    );
    if !seen.insert(key) {
        return;
    }
    out.push_str(&format!(
        "- {} §{} l{}: {}\n",
        highlight.path, highlight.section_id, highlight.line_no, highlight.line
    ));
    *emitted += 1;
}

/// Tail-aware adaptive-k cutoff for evidence emission.
///
/// The candidates handed to `render_scout_evidence` are re-ordered by
/// `order_scout_evidence` and are NOT monotonically descending by score, so we
/// must never `break` the emission loop on raw order. Instead we look at the
/// *distribution* of scores: sort a copy descending, find the sharpest relative
/// cliff between consecutive scores (a kneedle-style knee), and return the score
/// FLOOR at-or-above which a candidate is "in the head". Emission then filters
/// each candidate by `score >= floor` regardless of its position in the list.
///
/// Model-free and deterministic. The cliff threshold is derived from the score
/// distribution itself (relative drops compared against the median relative
/// drop), not a hardcoded constant. Returns `i32::MIN` when no clear knee
/// exists, which keeps every candidate (no behavioral change on smooth tails).
fn scout_adaptive_score_floor(candidates: &[ScoutCandidate]) -> i32 {
    // Need enough points for a tail to exist and a knee to be meaningful.
    if candidates.len() < 4 {
        return i32::MIN;
    }
    let mut scores: Vec<i32> = candidates.iter().map(|c| c.score).collect();
    scores.sort_unstable_by(|a, b| b.cmp(a));

    let top = scores[0];
    // A degenerate (flat or non-positive) head has no cliff to find.
    if top <= 0 || scores[scores.len() - 1] == top {
        return i32::MIN;
    }

    // Relative drop across each adjacent pair, normalized by the head score so
    // the measure is scale-free. Restrict cut points to the latter portion of
    // the curve so we never amputate the genuine head on a single early step.
    let n = scores.len();
    let min_keep = (n / 4).max(2); // never cut before keeping at least this many
    let mut drops: Vec<f64> = Vec::with_capacity(n - 1);
    for w in scores.windows(2) {
        drops.push((w[0] - w[1]) as f64 / top as f64);
    }

    // Typical (median) drop magnitude characterises the curve's smooth decay.
    let mut sorted_drops = drops.clone();
    sorted_drops.sort_by(|a, b| a.partial_cmp(b).unwrap_or(std::cmp::Ordering::Equal));
    let median_drop = sorted_drops[sorted_drops.len() / 2];

    // The knee is the largest drop occurring after `min_keep`, but only if it is
    // a genuine cliff: clearly larger than the typical drop AND a sizeable share
    // of the full head score. Both gates are relative to the data, not constants
    // tuned to the eval.
    let mut best_idx: Option<usize> = None;
    let mut best_drop = 0.0_f64;
    for (i, &drop) in drops.iter().enumerate() {
        // `i` is the gap between kept[i] and kept[i+1]; keeping i+1 sections.
        if i + 1 < min_keep {
            continue;
        }
        if drop > best_drop {
            best_drop = drop;
            best_idx = Some(i);
        }
    }

    match best_idx {
        Some(i)
            if best_drop >= 0.20 // cliff spans >=20% of the head score
                && best_drop >= median_drop * 3.0 + 1e-9 // and dwarfs typical decay
                && drops.iter().filter(|&&d| d >= best_drop - 1e-9).count() == 1 =>
        {
            // Keep everything strictly above the cliff: score floor is the value
            // just *before* the drop (scores[i]); the next score (scores[i+1])
            // and below are tail and get cut. Ties at the floor are kept.
            scores[i]
        }
        _ => i32::MIN,
    }
}

fn render_scout_evidence(
    out: &mut String,
    candidates: &[ScoutCandidate],
    question: &str,
    max_tokens: usize,
    baseline_out: &mut usize,
) -> Result<()> {
    let mut total_tokens = 0usize;
    let mut emitted_sigs: Vec<HashSet<String>> = Vec::new();
    let mut cache: HashMap<String, crate::parse::ParsedMarkdown> = HashMap::new();
    let mut emitted_ranges: HashMap<String, Vec<(usize, usize)>> = HashMap::new();
    let question_l = question.to_ascii_lowercase();
    // Tail-aware cutoff: filter low-relevance tail candidates by score floor
    // derived from the score distribution. NOT a positional break — the input
    // order is non-monotonic, so we gate per-candidate by score instead.
    // Skip the cutoff for comparative/multi-file questions, where a low-scored
    // section can be the sole evidence for one entity and the global floor can't
    // see per-file need. Otherwise trim the low-relevance tail (distractors).
    let score_floor = if wants_multi_file_evidence(question) {
        i32::MIN
    } else {
        scout_adaptive_score_floor(candidates)
    };
    let mut tail_cut = 0usize;
    for candidate in candidates {
        if total_tokens >= max_tokens {
            out.push_str("\n<!-- mdlens: scout budget exhausted -->\n");
            break;
        }
        if candidate.score < score_floor {
            tail_cut += 1;
            continue;
        }
        if !cache.contains_key(&candidate.path) {
            cache.insert(candidate.path.clone(), load_markdown(&candidate.path)?);
        }
        let parsed = cache.get(&candidate.path).expect("cached parsed markdown");
        let Some(section) = parsed.doc.find_section_by_id(&candidate.section_id) else {
            continue;
        };
        if is_low_value_section_for_question(section, &question_l) {
            continue;
        }
        let ranges = emitted_ranges.entry(candidate.path.clone()).or_default();
        if ranges.iter().any(|(start, end)| {
            section.line_start <= *start
                && section.line_end >= *end
                && (section.line_end - section.line_start) > (*end - *start)
        }) {
            continue;
        }
        let remaining = max_tokens.saturating_sub(total_tokens);
        let section_budget = remaining.min(650);
        let ancestors = section_ancestors(&parsed.doc.sections, &section.id);
        let (content, truncated) =
            scout_section_content(section, &ancestors, &parsed.lines, question, section_budget);
        let emitted_tokens = estimate_tokens(&content);
        if emitted_tokens == 0 {
            continue;
        }
        // MMR: skip a section that near-duplicates one already emitted (e.g. a
        // block copied across files), reinvesting the budget in new evidence.
        // The size floor avoids false-positive dedup on tiny sections that share
        // a couple of long tokens.
        let sig: HashSet<String> = content
            .split_whitespace()
            .filter(|w| w.len() >= 4)
            .map(|w| w.to_ascii_lowercase())
            .collect();
        if sig.len() >= 12
            && emitted_sigs.iter().any(|e| {
                let inter = sig.intersection(e).count();
                let uni = (sig.len() + e.len()).saturating_sub(inter).max(1);
                inter as f64 / uni as f64 > 0.6
            })
        {
            // Signal the omission so the agent knows the pack was de-duplicated.
            out.push_str("\n<!-- mdlens: omitted a near-duplicate section -->\n");
            continue;
        }
        out.push_str(&format!(
            "\n--- {} §{} {} l{}-{} ~{}t reason={} ---\n",
            candidate.path,
            section.id,
            section.path.join(" > "),
            section.line_start,
            section.line_end,
            section.token_estimate,
            candidate.reason
        ));
        out.push_str(&content);
        if !content.ends_with('\n') {
            out.push('\n');
        }
        ranges.push((section.line_start, section.line_end));
        total_tokens += emitted_tokens;
        emitted_sigs.push(sig);
        if truncated {
            continue;
        }
    }
    if tail_cut > 0 {
        out.push_str(&format!(
            "\n<!-- mdlens: tail-aware cutoff dropped {tail_cut} low-relevance section(s) -->\n"
        ));
    }
    // Baseline = full-text tokens of the distinct files we opened to build the
    // pack. Reuses the parse cache above, so each file is parsed only once.
    *baseline_out = cache.values().map(|p| p.doc.token_estimate).sum();
    Ok(())
}

fn scout_section_content(
    section: &Section,
    ancestors: &[&Section],
    lines: &[String],
    question: &str,
    max_tokens: usize,
) -> (String, bool) {
    let parent_context = scout_parent_context(ancestors, lines, max_tokens.min(220));
    let content_lines = section.extract_content(lines);
    let full = content_lines.join("\n");
    let full_with_context = if parent_context.trim().is_empty() {
        full.clone()
    } else {
        format!("{parent_context}\n...\n{full}")
    };
    let full_tokens = estimate_tokens(&full_with_context);
    if full_tokens <= max_tokens {
        return (full_with_context, false);
    }

    let focused_budget = max_tokens
        .saturating_sub(estimate_tokens(&parent_context))
        .max(max_tokens / 2);
    let focused = scout_focused_excerpt(content_lines, question, focused_budget);
    if !focused.trim().is_empty() {
        if parent_context.trim().is_empty() {
            return (focused, true);
        }
        return (format!("{parent_context}\n...\n{focused}"), true);
    }

    (
        truncate_to_tokens(&full_with_context, max_tokens, TRUNCATION_NOTICE),
        true,
    )
}

fn section_ancestors<'a>(sections: &'a [Section], target_id: &str) -> Vec<&'a Section> {
    let mut path = Vec::new();
    collect_section_ancestors(sections, target_id, &mut path);
    path
}

fn collect_section_ancestors<'a>(
    sections: &'a [Section],
    target_id: &str,
    path: &mut Vec<&'a Section>,
) -> bool {
    for section in sections {
        if section.id == target_id {
            return true;
        }
        path.push(section);
        if collect_section_ancestors(&section.children, target_id, path) {
            return true;
        }
        path.pop();
    }
    false
}

fn scout_parent_context(ancestors: &[&Section], lines: &[String], max_tokens: usize) -> String {
    if ancestors.is_empty() || max_tokens == 0 {
        return String::new();
    }

    let mut parts = Vec::new();
    for ancestor in ancestors {
        let direct = ancestor.extract_direct_content(lines);
        let cleaned = direct
            .iter()
            .map(|line| line.trim_end())
            .filter(|line| !line.trim().is_empty() && !is_noisy_highlight_line(line.trim()))
            .collect::<Vec<_>>()
            .join("\n");
        if cleaned.trim().is_empty() {
            continue;
        }
        parts.push(cleaned);
    }

    let joined = parts.join("\n");
    if estimate_tokens(&joined) <= max_tokens {
        joined
    } else {
        truncate_to_tokens(&joined, max_tokens, TRUNCATION_NOTICE)
    }
}

fn scout_focused_excerpt(lines: &[String], question: &str, max_tokens: usize) -> String {
    let tokens: Vec<String> = signal_tokens(question)
        .into_iter()
        .map(|token| token.to_ascii_lowercase())
        .collect();
    let question_l = question.to_ascii_lowercase();
    let wants_code = ["cli", "command", "install", "invoke"]
        .iter()
        .any(|needle| question_l.contains(needle));

    let mut selected = BTreeSet::new();
    for (idx, line) in lines.iter().enumerate() {
        let trimmed = line.trim();
        let lower = trimmed.to_ascii_lowercase();
        if is_noisy_highlight_line(trimmed) && !is_relevant_table_line(trimmed, &tokens) {
            continue;
        }
        let token_hits = tokens.iter().filter(|token| lower.contains(*token)).count();
        let code_hit = trimmed.contains("--")
            || (wants_code
                && (trimmed.contains('`')
                    || trimmed.starts_with("pip ")
                    || trimmed.starts_with("conda ")
                    || trimmed.starts_with("python ")
                    || trimmed.starts_with("git ")
                    || trimmed.starts_with("cmake ")
                    || trimmed.starts_with("make ")));
        let table_hit = is_relevant_table_line(trimmed, &tokens);
        if token_hits == 0 && !code_hit && !table_hit {
            continue;
        }
        let radius = if table_hit {
            2
        } else if lower.contains("disable") || lower.contains("warning") || code_hit {
            5
        } else if token_hits >= 2 {
            2
        } else {
            1
        };
        for context_idx in idx.saturating_sub(radius)..=(idx + radius).min(lines.len() - 1) {
            selected.insert(context_idx);
        }
    }

    let mut out = String::new();
    let mut last_idx = None;
    for idx in selected {
        let line = lines[idx].trim_end();
        if line.trim().is_empty() {
            continue;
        }
        if let Some(last) = last_idx {
            if idx > last + 1 && !out.ends_with("\n...\n") {
                out.push_str("...\n");
            }
        }
        let candidate = format!("{out}{line}\n");
        if estimate_tokens(&candidate) > max_tokens {
            out.push_str(TRUNCATION_NOTICE);
            break;
        }
        out = candidate;
        last_idx = Some(idx);
    }

    out
}

fn cmd_pack(args: PackArgs) -> Result<()> {
    let dedupe = args.dedupe && !args.no_dedupe;
    let result = if let Some(ref ids_str) = args.ids {
        let ids: Vec<String> = ids_str.split(',').map(|s| s.trim().to_string()).collect();
        pack_by_ids(&args.path, &ids, args.max_tokens, args.parents, dedupe)?
    } else if let Some(ref paths_str) = args.paths {
        let doc = parse_markdown(&args.path)?;
        let path_list: Vec<&str> = paths_str.split(';').collect();
        let mut ids = Vec::new();
        for p in path_list {
            ids.push(find_unique_section_by_path(&doc, p)?.id.clone());
        }
        pack_by_ids(&args.path, &ids, args.max_tokens, args.parents, dedupe)?
    } else if let Some(ref query) = args.search {
        crate::pack::pack_by_search(
            &args.path,
            query,
            args.max_tokens,
            PackSearchOptions {
                include_parents: args.parents,
                dedupe,
                case_sensitive: args.case_sensitive,
                use_regex: args.regex,
                max_results: args.max_results,
                context_lines: args.context_lines,
            },
        )?
    } else {
        return Err(anyhow::anyhow!(
            "exactly one of --ids, --paths, or --search is required"
        ));
    };

    if args.json {
        let output = PackJsonOutput {
            schema_version: 1,
            token_budget: result.token_budget,
            token_estimate: result.token_estimate,
            truncated: result.truncated,
            included: result
                .included
                .iter()
                .map(|inc| PackJsonIncluded {
                    path: inc.path.clone(),
                    section_id: inc.section_id.clone(),
                    section_path: inc.section_path.clone(),
                    line_start: inc.line_start,
                    line_end: inc.line_end,
                    token_estimate: inc.token_estimate,
                    truncated: inc.truncated,
                })
                .collect(),
            content: result.content.clone(),
        };
        println!("{}", serde_json::to_string_pretty(&output)?);
    } else {
        let included_render: Vec<PackIncluded> = result
            .included
            .iter()
            .map(|inc| PackIncluded {
                section_id: inc.section_id.clone(),
                section_title: inc.section_path.last().cloned().unwrap_or_default(),
                line_range: format!("{}-{}", inc.line_start, inc.line_end),
                token_estimate: inc.token_estimate,
            })
            .collect();
        println!(
            "{}",
            render_pack(
                &args.path,
                result.token_budget,
                &included_render,
                &result.content,
                result.truncated
            )
        );
    }

    Ok(())
}

fn cmd_stats(args: StatsArgs) -> Result<()> {
    let files = crate::search::discover_markdown_files(&args.path)?;
    let mut entries = Vec::new();

    for file in &files {
        let doc = parse_markdown(file)?;
        entries.push(StatsEntry {
            path: doc.path,
            lines: doc.line_count,
            words: doc.word_count,
            tokens: doc.token_estimate,
        });
    }

    // Sort
    match args.sort {
        StatsSort::Tokens => entries.sort_by_key(|entry| Reverse(entry.tokens)),
        StatsSort::Lines => entries.sort_by_key(|entry| Reverse(entry.lines)),
        StatsSort::Path => entries.sort_by(|lhs, rhs| lhs.path.cmp(&rhs.path)),
    }

    // Apply top limit
    let entries = if let Some(top) = args.top {
        &entries[..std::cmp::min(top, entries.len())]
    } else {
        &entries
    };

    if args.json {
        let output = StatsJsonOutput {
            schema_version: 1,
            entries: entries
                .iter()
                .map(|e| StatsJsonEntry {
                    path: e.path.clone(),
                    lines: e.lines,
                    words: e.words,
                    tokens: e.tokens,
                })
                .collect(),
        };
        println!("{}", serde_json::to_string_pretty(&output)?);
    } else {
        println!("{}", render_stats(entries));
    }

    Ok(())
}

fn cmd_sections(args: SectionsArgs) -> Result<()> {
    let stdin = io::stdin();
    let mut inputs: Vec<SectionInput> = Vec::new();

    // Read from stdin only when it is not a tty (i.e. piped input)
    if !args.files.is_empty() {
        // Positional args provided — use those, skip stdin
        for f in &args.files {
            let trimmed = f.trim().to_string();
            if !trimmed.is_empty() {
                inputs.push(SectionInput::File(trimmed));
            }
        }
    } else {
        for line in stdin.lock().lines() {
            let line = line?;
            if let Some(input) = parse_sections_input_line(&line) {
                inputs.push(input);
            }
        }
    }

    if inputs.is_empty() {
        return Ok(());
    }

    let dedupe = args.dedupe && !args.no_dedupe;
    let has_hit_input = inputs
        .iter()
        .any(|input| matches!(input, SectionInput::Hit(_)));

    if !has_hit_input {
        let mut paths: Vec<String> = inputs
            .into_iter()
            .filter_map(|input| match input {
                SectionInput::File(path) => Some(path),
                SectionInput::Hit(_) => None,
            })
            .collect();

        if dedupe {
            let mut seen = HashSet::new();
            paths.retain(|p| seen.insert(p.clone()));
        }

        return render_sections_from_paths(args, paths);
    }

    let mut file_order: Vec<String> = Vec::new();
    let mut file_hits: HashMap<String, Vec<usize>> = HashMap::new();

    for input in inputs {
        match input {
            SectionInput::File(path) => {
                if !file_order.iter().any(|existing| existing == &path) {
                    file_order.push(path.clone());
                }
                file_hits.entry(path).or_default();
            }
            SectionInput::Hit(hit) => {
                let entry = file_hits.entry(hit.path.clone()).or_default();
                if !dedupe || !entry.contains(&hit.line) {
                    entry.push(hit.line);
                }
                if !file_order.iter().any(|existing| existing == &hit.path) {
                    file_order.push(hit.path);
                }
            }
        }
    }

    if let Some(max_files) = args.max_files {
        if file_order.len() > max_files {
            anyhow::bail!(
                "[error] {} files exceed --max-files {}; narrow with a more specific grep or raise the limit",
                file_order.len(),
                max_files
            );
        }
    } else if args.max_tokens.is_none() && file_order.len() > 8 {
        eprintln!(
            "[warn] {} files piped without --max-tokens or --max-files; output may be large",
            file_order.len()
        );
    }

    let mut file_outputs: Vec<SectionsFileOutput> = Vec::new();
    let mut total_tokens: usize = 0;
    let mut omitted: usize = 0;

    for path in &file_order {
        let parsed = match load_markdown(path) {
            Ok(p) => p,
            Err(e) => {
                eprintln!("Warning: could not read {}: {}", path, e);
                continue;
            }
        };

        let doc = &parsed.doc;
        let lines = &parsed.lines;

        let mut sections: Vec<SectionsSectionOutput> =
            if let Some(hit_lines) = file_hits.get(path).filter(|lines| !lines.is_empty()) {
                collect_hit_sections(
                    &doc.sections,
                    lines,
                    hit_lines,
                    args.children,
                    args.preview,
                    dedupe,
                )
            } else {
                let mut collected = Vec::new();
                collect_all_sections(
                    &doc.sections,
                    lines,
                    args.children,
                    args.preview,
                    args.max_depth,
                    0,
                    &mut collected,
                );
                collected
            };

        if sections.is_empty() {
            continue;
        }

        if let Some(max_sections) = args.max_sections {
            if sections.len() > max_sections {
                omitted += sections.len() - max_sections;
                sections.truncate(max_sections);
            }
        }

        // Apply max-tokens cap
        if let Some(max_tokens) = args.max_tokens {
            let mut kept: Vec<SectionsSectionOutput> = Vec::new();
            for sec in sections {
                if total_tokens + sec.token_estimate > max_tokens {
                    omitted += 1;
                } else {
                    total_tokens += sec.token_estimate;
                    kept.push(sec);
                }
            }
            sections = kept;
        }

        if !sections.is_empty() {
            file_outputs.push(SectionsFileOutput {
                path: path.clone(),
                sections,
            });
        }
    }

    emit_sections_output(&args, file_outputs, omitted)
}

fn render_sections_from_paths(args: SectionsArgs, paths: Vec<String>) -> Result<()> {
    if paths.is_empty() {
        return Ok(());
    }

    let depth_capped = args.max_depth.is_none() && (!args.content || args.preview.is_some());
    let effective_depth = if depth_capped {
        Some(2)
    } else {
        args.max_depth
    };

    if let Some(max_files) = args.max_files {
        if paths.len() > max_files {
            anyhow::bail!(
                "[error] {} files exceed --max-files {}; narrow with a more specific grep or raise the limit",
                paths.len(),
                max_files
            );
        }
    } else if args.max_tokens.is_none() && paths.len() > 8 {
        eprintln!(
            "[warn] {} files piped without --max-tokens or --max-files; output may be large",
            paths.len()
        );
    }

    let mut file_outputs: Vec<SectionsFileOutput> = Vec::new();
    let mut total_tokens: usize = 0;
    let mut omitted: usize = 0;

    for path in &paths {
        let parsed = match load_markdown(path) {
            Ok(p) => p,
            Err(e) => {
                eprintln!("Warning: could not read {}: {}", path, e);
                continue;
            }
        };

        let doc = &parsed.doc;
        let lines = &parsed.lines;
        let mut sections: Vec<SectionsSectionOutput> = Vec::new();
        collect_all_sections(
            &doc.sections,
            lines,
            args.children,
            args.preview,
            effective_depth,
            0,
            &mut sections,
        );

        if sections.is_empty() {
            continue;
        }

        if let Some(max_sections) = args.max_sections {
            if sections.len() > max_sections {
                omitted += sections.len() - max_sections;
                sections.truncate(max_sections);
            }
        }

        if let Some(max_tokens) = args.max_tokens {
            let mut kept: Vec<SectionsSectionOutput> = Vec::new();
            for sec in sections {
                if total_tokens + sec.token_estimate > max_tokens {
                    omitted += 1;
                } else {
                    total_tokens += sec.token_estimate;
                    kept.push(sec);
                }
            }
            sections = kept;
        }

        if !sections.is_empty() {
            file_outputs.push(SectionsFileOutput {
                path: path.clone(),
                sections,
            });
        }
    }

    if depth_capped {
        eprintln!(
            "[sections] whole-file mode: showing depth ≤2 by default; use --max-depth N for more"
        );
    }

    emit_sections_output(&args, file_outputs, omitted)
}

fn emit_sections_output(
    args: &SectionsArgs,
    file_outputs: Vec<SectionsFileOutput>,
    omitted: usize,
) -> Result<()> {
    if omitted > 0 {
        if let Some(max_tokens) = args.max_tokens {
            eprintln!(
                "[warn] {} sections omitted by limits (budget ~{}t)",
                omitted, max_tokens
            );
        } else {
            eprintln!("[warn] {} sections omitted by limits", omitted);
        }
    }

    if file_outputs.is_empty() {
        return Ok(());
    }

    if args.json {
        let output = SectionsJsonOutput {
            schema_version: 1,
            files: file_outputs
                .iter()
                .map(|fo| SectionsJsonFile {
                    path: fo.path.clone(),
                    sections: fo
                        .sections
                        .iter()
                        .map(|s| SectionsJsonSection {
                            id: s.id.clone(),
                            title: s.title.clone(),
                            heading_path: if args.heading_paths {
                                Some(s.heading_path.clone())
                            } else {
                                None
                            },
                            line_start: if args.lines { Some(s.line_start) } else { None },
                            line_end: if args.lines { Some(s.line_end) } else { None },
                            token_estimate: s.token_estimate,
                            body: if args.content {
                                Some(s.body.clone())
                            } else {
                                None
                            },
                            preview: s.preview.clone(),
                        })
                        .collect(),
                })
                .collect(),
        };
        println!("{}", serde_json::to_string_pretty(&output)?);
    } else {
        let entries: Vec<SectionsEntry> = file_outputs
            .iter()
            .flat_map(|fo| {
                fo.sections.iter().map(|s| SectionsEntry {
                    file_path: fo.path.clone(),
                    id: s.id.clone(),
                    title: s.title.clone(),
                    heading_path: if args.heading_paths {
                        Some(s.heading_path.clone())
                    } else {
                        None
                    },
                    line_start: if args.lines { Some(s.line_start) } else { None },
                    line_end: if args.lines { Some(s.line_end) } else { None },
                    token_estimate: s.token_estimate,
                    body: if args.content {
                        Some(s.body.clone())
                    } else {
                        None
                    },
                    preview: s.preview.clone(),
                })
            })
            .collect();
        println!("{}", render_sections(&entries, args.content));
    }

    Ok(())
}

struct SectionsSectionOutput {
    id: String,
    title: String,
    heading_path: Vec<String>,
    line_start: usize,
    line_end: usize,
    token_estimate: usize,
    body: String,
    preview: Option<String>,
}

struct SectionsFileOutput {
    path: String,
    sections: Vec<SectionsSectionOutput>,
}

#[derive(Clone)]
struct HitSectionAggregate<'a> {
    section: &'a Section,
    hit_count: usize,
    first_line: usize,
}

fn parse_sections_input_line(line: &str) -> Option<SectionInput> {
    let trimmed = line.trim();
    if trimmed.is_empty() {
        return None;
    }

    if let Some((path, line_num)) = parse_grep_hit(trimmed) {
        return Some(SectionInput::Hit(SectionHit {
            path: path.to_string(),
            line: line_num,
        }));
    }

    Some(SectionInput::File(trimmed.to_string()))
}

fn parse_grep_hit(line: &str) -> Option<(&str, usize)> {
    let first = line.find(':')?;
    let rest = &line[(first + 1)..];
    let second = rest.find(':')?;
    let path = &line[..first];
    let line_num = rest[..second].parse().ok()?;
    Some((path, line_num))
}

fn collect_hit_sections(
    sections: &[Section],
    lines: &[String],
    hit_lines: &[usize],
    include_children: bool,
    preview_lines: Option<usize>,
    dedupe: bool,
) -> Vec<SectionsSectionOutput> {
    let mut by_section: HashMap<String, HitSectionAggregate<'_>> = HashMap::new();
    let mut ordered_hits: Vec<(usize, &Section)> = Vec::new();

    for line_num in hit_lines {
        let Some(section) = find_deepest_section_for_line(sections, *line_num) else {
            continue;
        };
        if dedupe {
            by_section
                .entry(section.id.clone())
                .and_modify(|entry| entry.hit_count += 1)
                .or_insert(HitSectionAggregate {
                    section,
                    hit_count: 1,
                    first_line: *line_num,
                });
        } else {
            ordered_hits.push((*line_num, section));
        }
    }

    let aggregates: Vec<HitSectionAggregate<'_>> = if dedupe {
        let mut ranked: Vec<HitSectionAggregate<'_>> = by_section.into_values().collect();
        ranked.sort_by(|lhs, rhs| {
            rhs.hit_count
                .cmp(&lhs.hit_count)
                .then(lhs.section.token_estimate.cmp(&rhs.section.token_estimate))
                .then(lhs.first_line.cmp(&rhs.first_line))
                .then(lhs.section.line_start.cmp(&rhs.section.line_start))
        });
        ranked
    } else {
        ordered_hits.sort_by(|lhs, rhs| {
            lhs.0
                .cmp(&rhs.0)
                .then(lhs.1.line_start.cmp(&rhs.1.line_start))
                .then(lhs.1.id.cmp(&rhs.1.id))
        });
        ordered_hits
            .into_iter()
            .map(|(first_line, section)| HitSectionAggregate {
                section,
                hit_count: 1,
                first_line,
            })
            .collect()
    };

    let mut collected = Vec::new();
    for aggregate in aggregates {
        let section = aggregate.section;
        let body_lines = if include_children {
            section.extract_content(lines)
        } else {
            section.extract_direct_content(lines)
        };
        let body = body_lines.join("\n");
        let preview = preview_lines.map(|n| {
            body_lines
                .iter()
                .filter(|l| !l.trim().is_empty())
                .take(n)
                .cloned()
                .collect::<Vec<_>>()
                .join("\n")
        });

        collected.push(SectionsSectionOutput {
            id: section.id.clone(),
            title: section.title.clone(),
            heading_path: section.path.clone(),
            line_start: section.line_start,
            line_end: section.line_end,
            token_estimate: estimate_tokens(&body),
            body,
            preview,
        });
    }

    collected
}

fn collect_all_sections(
    sections: &[Section],
    lines: &[String],
    include_children: bool,
    preview_lines: Option<usize>,
    max_depth: Option<usize>,
    current_depth: usize,
    result: &mut Vec<SectionsSectionOutput>,
) {
    for section in sections {
        if section.title == "<preamble>" {
            continue;
        }
        if let Some(max) = max_depth {
            if current_depth >= max {
                continue;
            }
        }
        let body_lines = if include_children {
            section.extract_content(lines)
        } else {
            section.extract_direct_content(lines)
        };
        let body = body_lines.join("\n");
        let preview = preview_lines.map(|n| {
            body_lines
                .iter()
                .filter(|l| !l.trim().is_empty())
                .take(n)
                .cloned()
                .collect::<Vec<_>>()
                .join("\n")
        });
        result.push(SectionsSectionOutput {
            id: section.id.clone(),
            title: section.title.clone(),
            heading_path: section.path.clone(),
            line_start: section.line_start,
            line_end: section.line_end,
            token_estimate: estimate_tokens(&body),
            body,
            preview,
        });
        collect_all_sections(
            &section.children,
            lines,
            include_children,
            preview_lines,
            max_depth,
            current_depth + 1,
            result,
        );
    }
}

fn enrich_search_results(
    results: &mut [crate::render::SearchResult],
    with_content: bool,
    preview_lines: Option<usize>,
) -> Result<()> {
    let mut docs: HashMap<String, crate::parse::ParsedMarkdown> = HashMap::new();

    for result in results.iter_mut() {
        let parsed = if let Some(parsed) = docs.get(&result.path) {
            parsed
        } else {
            let loaded = load_markdown(&result.path)?;
            docs.insert(result.path.clone(), loaded);
            docs.get(&result.path).expect("inserted parsed markdown")
        };

        let Some(section) = parsed.doc.find_section_by_id(&result.section_id) else {
            continue;
        };
        let body_lines = section.extract_direct_content(&parsed.lines);
        if with_content {
            result.body = Some(body_lines.join("\n"));
        }
        if let Some(n) = preview_lines {
            result.preview = Some(
                body_lines
                    .iter()
                    .filter(|line| !line.trim().is_empty())
                    .take(n)
                    .cloned()
                    .collect::<Vec<_>>()
                    .join("\n"),
            );
        }
    }

    Ok(())
}

fn find_deepest_section_for_line(sections: &[Section], line_num: usize) -> Option<&Section> {
    for section in sections {
        if line_num < section.line_start || line_num > section.line_end {
            continue;
        }
        if let Some(child) = find_deepest_section_for_line(&section.children, line_num) {
            return Some(child);
        }
        return Some(section);
    }
    None
}

// --- JSON output types ---

#[derive(Serialize)]
struct TreeJsonOutput {
    schema_version: u32,
    path: String,
    line_count: usize,
    byte_count: usize,
    char_count: usize,
    word_count: usize,
    token_estimate: usize,
    sections: Vec<SectionJsonOutput>,
}

#[derive(Serialize)]
struct TreeFileJsonOutput {
    path: String,
    line_count: usize,
    byte_count: usize,
    char_count: usize,
    word_count: usize,
    token_estimate: usize,
    sections: Vec<SectionJsonOutput>,
}

#[derive(Serialize)]
struct TreeMultiJsonOutput {
    schema_version: u32,
    files: Vec<TreeFileJsonOutput>,
}

#[derive(Serialize)]
struct SectionJsonOutput {
    id: String,
    title: String,
    level: u8,
    path: Vec<String>,
    line_start: usize,
    line_end: usize,
    token_estimate: usize,
    #[serde(skip_serializing_if = "Vec::is_empty")]
    children: Vec<SectionJsonOutput>,
}

#[derive(Serialize)]
struct ReadJsonOutput {
    schema_version: u32,
    path: String,
    selector: ReadSelector,
    section: SectionJsonOutput,
    content: String,
    truncated: bool,
}

#[derive(Serialize)]
struct ReadSelector {
    #[serde(rename = "type")]
    r#type: String,
    value: String,
}

#[derive(Serialize)]
struct SearchJsonOutput {
    schema_version: u32,
    query: String,
    root: String,
    results: Vec<SearchJsonResult>,
}

#[derive(Serialize)]
struct SearchJsonResult {
    path: String,
    section_id: String,
    section_title: String,
    section_path: Vec<String>,
    line_start: usize,
    line_end: usize,
    token_estimate: usize,
    match_count: usize,
    body: Option<String>,
    preview: Option<String>,
    snippets: Vec<SearchJsonSnippet>,
}

#[derive(Serialize)]
struct SearchJsonSnippet {
    line_start: usize,
    line_end: usize,
    text: String,
}

#[derive(Serialize)]
struct ScoutJsonOutput {
    schema_version: u32,
    root: String,
    question: String,
    token_budget: usize,
    candidate_count: usize,
    queries: Vec<String>,
    candidates: Vec<ScoutCandidate>,
    rendered_text: String,
}

#[derive(Serialize)]
struct PackJsonOutput {
    schema_version: u32,
    token_budget: usize,
    token_estimate: usize,
    truncated: bool,
    included: Vec<PackJsonIncluded>,
    content: String,
}

#[derive(Serialize)]
struct PackJsonIncluded {
    path: String,
    section_id: String,
    section_path: Vec<String>,
    line_start: usize,
    line_end: usize,
    token_estimate: usize,
    truncated: bool,
}

#[derive(Serialize)]
struct StatsJsonOutput {
    schema_version: u32,
    entries: Vec<StatsJsonEntry>,
}

#[derive(Serialize)]
struct StatsJsonEntry {
    path: String,
    lines: usize,
    words: usize,
    tokens: usize,
}

#[derive(Serialize)]
struct SectionsJsonOutput {
    schema_version: u32,
    files: Vec<SectionsJsonFile>,
}

#[derive(Serialize)]
struct SectionsJsonFile {
    path: String,
    sections: Vec<SectionsJsonSection>,
}

#[derive(Serialize)]
struct SectionsJsonSection {
    id: String,
    title: String,
    #[serde(skip_serializing_if = "Option::is_none")]
    heading_path: Option<Vec<String>>,
    #[serde(skip_serializing_if = "Option::is_none")]
    line_start: Option<usize>,
    #[serde(skip_serializing_if = "Option::is_none")]
    line_end: Option<usize>,
    token_estimate: usize,
    #[serde(skip_serializing_if = "Option::is_none")]
    body: Option<String>,
    #[serde(skip_serializing_if = "Option::is_none")]
    preview: Option<String>,
}

// --- Helper functions ---

fn serialize_sections(
    sections: &[Section],
    max_depth: Option<usize>,
    include_preamble: bool,
    current_depth: usize,
) -> Vec<SectionJsonOutput> {
    let mut result = Vec::new();
    for section in sections {
        if section.title == "<preamble>" && !include_preamble {
            continue;
        }
        let children = if let Some(max) = max_depth {
            if current_depth + 1 < max {
                serialize_sections(
                    &section.children,
                    max_depth,
                    include_preamble,
                    current_depth + 1,
                )
            } else {
                Vec::new()
            }
        } else {
            serialize_sections(
                &section.children,
                max_depth,
                include_preamble,
                current_depth + 1,
            )
        };

        result.push(SectionJsonOutput {
            id: section.id.clone(),
            title: section.title.clone(),
            level: section.level,
            path: section.path.clone(),
            line_start: section.line_start,
            line_end: section.line_end,
            token_estimate: section.token_estimate,
            children,
        });
    }
    result
}

fn truncate_content_to_tokens(content: &str, max_tokens: usize) -> String {
    truncate_to_tokens(content, max_tokens, TRUNCATION_NOTICE)
}