hypersteeldb 0.5.4

//! `steel` — the three verbs in a terminal.
//!
//! ```text
//! steel ./corpus                 ingest a directory of .md / .txt and open the UI
//! steel ./corpus --artifact .hsdb   follow a saved vocabulary instead of rediscovering
//! steel --help
//! ```
//!
//! The point of the UI is the thing a REPL hides: **what you are allowed to ask.** Categories are discovered
//! from your documents, so their names are not guessable, and the most common way to waste time with this engine
//! is to type a query against a category that does not exist. So the categories are always on screen, a refusal
//! is rendered as prominently as an answer, and the expression the engine actually ran is shown next to the
//! count it produced.
//!
//! `learn` is a key press rather than a startup step, because it is the one verb that needs a model.

use ratatui::crossterm::event::{self, Event, KeyCode, KeyEventKind, KeyModifiers};
use ratatui::crossterm::execute;
use ratatui::crossterm::terminal::{
    disable_raw_mode, enable_raw_mode, EnterAlternateScreen, LeaveAlternateScreen,
};
use ratatui::prelude::*;
use ratatui::widgets::{Block, Borders, Clear, List, ListItem, Paragraph, Wrap};
use std::io::stdout;
use std::path::PathBuf;
use std::time::Duration;
use steeldb::learn::Teacher;
use steeldb::SteelDb;

/// Local models that can actually drive `learn`, best first.
///
/// Measured, not guessed. `learn` needs structured output; asked for a tool call, most small models answer in
/// prose or not at all, and the ones here were the ones that came back with a usable ontology. Order is by
/// quality of what they proposed, not by size: a 1.7B produced coherent facet names where a 3B produced the
/// corpus's proper nouns. Anything not on the list still works — pass `--model` — this is only the default.
const PREFERRED: &[&str] = &[
    "qwen3:1.7b",
    "qwen3.5:0.8b",
    "granite3-moe:3b",
    "functiongemma:latest",
    "granite3-moe:1b",
    "qwen2.5:0.5b",
];

#[derive(PartialEq)]
enum Focus {
    Query,
    Categories,
}

enum Popup {
    None,
    Help,
    Learn(Vec<String>),
}

struct App {
    db: SteelDb,
    corpus: PathBuf,
    /// where the vocabulary came from — discovered now, or read from a saved set
    following: Option<PathBuf>,
    model: Option<String>,
    focus: Focus,
    input: String,
    cat_sel: usize,
    /// the expression that was run, and what came back
    ran: Option<String>,
    hits: Vec<(u32, String)>,
    count: usize,
    micros: f64,
    /// a refusal, rendered as a first-class outcome rather than an error line
    refused: Vec<String>,
    status: String,
    popup: Popup,
    scroll: u16,
}

impl App {
    fn new(db: SteelDb, corpus: PathBuf, following: Option<PathBuf>, model: Option<String>) -> App {
        let n = db.len();
        let origin = match &following {
            Some(d) => format!("following {}", d.display()),
            None => "vocabulary discovered from the text".to_string(),
        };
        App {
            db,
            corpus,
            following,
            model,
            focus: Focus::Query,
            input: String::new(),
            cat_sel: 0,
            ran: None,
            hits: Vec::new(),
            count: 0,
            micros: 0.0,
            refused: Vec::new(),
            status: format!("{n} situations · {origin} · ? for help"),
            popup: Popup::None,
            scroll: 0,
        }
    }

    fn run_query(&mut self, ikl: &str) {
        self.ran = Some(ikl.to_string());
        self.scroll = 0;
        match self.db.query(ikl) {
            Ok(answer) => {
                self.refused.clear();
                self.count = answer.len();
                self.micros = answer.micros();
                self.hits = self
                    .db
                    .resolve(&answer)
                    .take(200)
                    .map(|(id, text)| (id, text.chars().take(400).collect()))
                    .collect();
                self.status = format!("{} situations", self.count);
            }
            Err(r) => {
                // A refusal is the outcome, not a failure of the tool. Keep the problems AND the alternatives:
                // the problems say why, the alternatives say what to type instead.
                self.hits.clear();
                self.count = 0;
                let mut lines = r.problems.clone();
                if !r.alternatives.is_empty() {
                    lines.push(String::new());
                    lines.push(format!("try: {}", r.alternatives.join("  ")));
                }
                self.refused = lines;
                self.status = "refused".into();
            }
        }
    }

    fn learn(&mut self, rt: &tokio::runtime::Runtime) {
        let Some(model) = self.model.clone() else {
            self.status = "no local model found — is ollama running? (`ollama serve`)".into();
            return;
        };
        let teacher = match Teacher::ollama(&model) {
            Ok(t) => t,
            Err(e) => {
                self.status = format!("{e}");
                return;
            }
        };
        let proposal = match rt.block_on(teacher.propose_categories(&self.db)) {
            Ok(p) => p,
            Err(e) => {
                self.status = format!("{model}: {e}");
                return;
            }
        };
        if proposal.candidates.is_empty() {
            self.popup = Popup::Learn(vec![
                format!("{model} proposed nothing."),
                String::new(),
                "That is a real answer: the model found no category the".into(),
                "deterministic pass had missed.".into(),
            ]);
            return;
        }
        // The gate runs the same test `ingest` uses, so show the verdict per candidate — a rejection is the
        // interesting part, and the reason is what makes it reviewable.
        let verdicts = self.db.adopt(&proposal);
        let mut lines = vec![format!("{model} proposed {}:", verdicts.len()), String::new()];
        for v in &verdicts {
            lines.push(format!(
                "{}  {:20} coverage {:.2}  overlap {:.2}",
                if v.kept { "KEEP" } else { "drop" },
                v.name,
                v.coverage,
                v.overlap
            ));
            lines.push(format!("      {}", v.reason));
        }
        let kept = verdicts.iter().filter(|v| v.kept).count();
        lines.push(String::new());
        lines.push(if kept == 0 {
            "Nothing adopted. A suggestion the gate refuses is not absorbed.".into()
        } else {
            format!("{kept} adopted — press s to save the artefact set.")
        });
        self.popup = Popup::Learn(lines);
        self.cat_sel = 0;
    }

    fn save(&mut self, with_training: bool) {
        let dir = self.corpus.join(".hypersteeldb");
        let out = if with_training {
            self.db.save_with_training(&dir).map(|n| format!("saved + {n} training spans"))
        } else {
            self.db.save(&dir).map(|_| "saved".to_string())
        };
        self.status = match out {
            Ok(m) => {
                self.following = Some(dir.clone());
                format!("{m} → {} — later runs follow it", dir.display())
            }
            Err(e) => format!("save failed: {e}"),
        };
    }
}

/// Ask the local server what it has, and pick the best one we know works.
fn pick_model(explicit: Option<String>) -> Option<String> {
    if explicit.is_some() {
        return explicit;
    }
    let body = std::process::Command::new("curl")
        .args(["-s", "-m", "2", "http://localhost:11434/api/tags"])
        .output()
        .ok()?;
    let v: serde_json::Value = serde_json::from_slice(&body.stdout).ok()?;
    let have: Vec<String> = v
        .get("models")?
        .as_array()?
        .iter()
        .filter_map(|m| m.get("name")?.as_str().map(str::to_string))
        .collect();
    PREFERRED
        .iter()
        .find(|p| have.iter().any(|h| h == *p))
        .map(|s| s.to_string())
        .or_else(|| have.first().cloned())
}

fn main() {
    // Report with Display, not Debug. Returning Result from main prints the Debug form, which renders a
    // multi-line message with literal `\n` in it — and these messages exist to be read by a person who is
    // stuck, so the escaping defeats their whole purpose.
    if let Err(e) = run_cli() {
        eprintln!("{e}");
        std::process::exit(1);
    }
}

fn run_cli() -> Result<(), Box<dyn std::error::Error>> {
    let args: Vec<String> = std::env::args().skip(1).collect();
    if args.is_empty() || args.iter().any(|a| a == "-h" || a == "--help") {
        println!(
            "steel — ask a pile of documents a question it can refuse\n\n\
             USAGE:\n  \
               steel <corpus-dir> [--neural [--sample <n>]] [--model <name>]\n  \
               steel <corpus-dir> [--artifact <dir>] [--rediscover]\n\n\
             The corpus directory is read for .md and .txt files. Categories are discovered from the text,\n\
             so run it and look at them before writing a query.\n\n\
             OPTIONS:\n  \
               --artifact <dir>   follow a specific saved vocabulary\n  \
               --rediscover       ignore <corpus>/.hypersteeldb and discover afresh\n  \
               --neural           discover the vocabulary with the span tagger + a curator model\n  \
               --sample <n>       documents the neural pass reads (default 100)\n  \
               --model <name>     curator model; `bedrock:<id>` for Bedrock, else a local\n  \
                                  ollama model (default: best one installed)\n\n\
             A saved set at <corpus>/.hypersteeldb is followed automatically, so a category you\n\
             learn and save is still there next time.\n"
        );
        return Ok(());
    }

    let mut corpus: Option<PathBuf> = None;
    let mut artifact: Option<PathBuf> = None;
    let mut model: Option<String> = None;
    let mut rediscover = false;
    let mut neural = false;
    let mut neural_sample = 100usize;
    let mut it = args.iter();
    while let Some(a) = it.next() {
        match a.as_str() {
            "--artifact" => artifact = it.next().map(PathBuf::from),
            "--model" => model = it.next().cloned(),
            "--rediscover" => rediscover = true,
            "--neural" => neural = true,
            "--sample" => neural_sample = it.next().and_then(|v| v.parse().ok()).unwrap_or(100),
            other => corpus = Some(PathBuf::from(other)),
        }
    }
    let corpus = corpus.ok_or("no corpus directory given (try --help)")?;

    // Ingest before touching the terminal: a failure here should print plainly, not inside an alternate screen
    // that is about to be torn down.
    eprint!("reading {} … ", corpus.display());
    // A saved set next to the corpus is followed by default. `s` writes it to `<corpus>/.hypersteeldb`, and the
    // contract the README states is that every run afterwards follows those files — so rediscovering here would
    // mean a category learned and saved in one session silently vanished in the next, which is precisely the
    // drift the artefact set exists to stop. `--rediscover` opts out.
    let default_set = corpus.join(".hypersteeldb");
    let follow = artifact.clone().or_else(|| {
        (!rediscover && default_set.join("manifest.json").is_file()).then_some(default_set)
    });
    let db = match (&follow, neural) {
        // A saved set always wins: following it is the whole point, and re-running the models would be both
        // slower and a different answer.
        (Some(dir), _) => {
            let docs = read_dir_docs(&corpus)?;
            SteelDb::ingest_using(docs, dir)?
        }
        (None, true) => neural_ingest(&corpus, neural_sample, model.clone())?,
        (None, false) => SteelDb::open(&corpus)?,
    };
    let ncat = db.categories().len();
    eprintln!(
        "{} situations, {ncat} categor{} ({})",
        db.len(),
        if ncat == 1 { "y" } else { "ies" },
        match (&follow, neural) {
            (Some(d), _) => format!("following {}", d.display()),
            (None, true) => "discovered by the tagger".to_string(),
            (None, false) => "discovered".to_string(),
        }
    );

    // A clear refusal beats `Os { code: 6, message: "Device not configured" }`, which is what the terminal
    // layer returns when there is no tty — piping this into `less` or running it from a script otherwise gives
    // an error that names nothing the user can act on.
    if !std::io::IsTerminal::is_terminal(&std::io::stdout()) {
        eprintln!(
            "steel needs an interactive terminal (stdout is not a tty).\n\
             Nothing is wrong with the corpus: {} situations and {} categories were indexed.",
            db.len(),
            db.categories().len()
        );
        std::process::exit(2);
    }

    // A terminal reporting zero size cannot show anything, and the event loop would sit there drawing nothing
    // and waiting for keys — a black hole with no way to tell it is running. `script` reports exactly this.
    if let Ok(sz) = ratatui::crossterm::terminal::size() {
        if sz.0 == 0 || sz.1 == 0 {
            eprintln!(
                "steel needs a terminal with a size; this one reports {}x{}.\n\
                 Nothing is wrong with the corpus: {} situations and {} categories were indexed.",
                sz.0,
                sz.1,
                db.len(),
                db.categories().len()
            );
            std::process::exit(2);
        }
    }

    let model = pick_model(model);
    let rt = tokio::runtime::Builder::new_current_thread().enable_all().build()?;
    let mut app = App::new(db, corpus, follow, model);

    enable_raw_mode()?;
    let mut out = stdout();
    execute!(out, EnterAlternateScreen)?;
    let mut terminal = Terminal::new(CrosstermBackend::new(out))?;
    let res = run(&mut terminal, &mut app, &rt);
    disable_raw_mode()?;
    execute!(terminal.backend_mut(), LeaveAlternateScreen)?;
    terminal.show_cursor()?;
    res?;
    Ok(())
}

/// Discover the vocabulary with the span tagger and a curator model, then index the whole corpus by it.
///
/// Three stages, each a different kind of work: the tagger marks typed spans and transport groups them into raw
/// clusters; a model merges synonyms, drops noise and names each surviving facet as a KIND; then the gate and
/// the model-free matcher do the rest. The models run once, over the sample.
#[cfg(all(feature = "onnx", feature = "embed", feature = "paddock"))]
fn neural_ingest(
    corpus: &std::path::Path,
    sample: usize,
    model: Option<String>,
) -> Result<SteelDb, Box<dyn std::error::Error>> {
    let docs = read_dir_docs(corpus)?;
    if docs.is_empty() {
        return Err(format!("no .md or .txt files under {}", corpus.display()).into());
    }
    let model = pick_model(model).ok_or(
        "no local model found for curation — start one (`ollama serve`) or pass --model",
    )?;
    let head: Vec<String> = docs.iter().take(sample.max(1)).cloned().collect();

    // Build the curator BEFORE tagging. Tagging a hundred documents takes the best part of a minute, and
    // discovering afterwards that there is no usable curator wastes all of it — the check is free, so do it
    // while the user is still looking at a blank screen rather than a progress line.
    let teacher = teacher_for(&model)?;

    eprint!("\n  tagging {} of {} documents … ", head.len(), docs.len());
    let raw = steeldb::tagger_discover::discover(&head, 12, 8)?;
    eprintln!(
        "{} entity clusters, {} relation clusters (routed away: {:?})",
        raw.entity_clusters.len(),
        raw.relation_clusters.len(),
        raw.routed_away
    );

    eprint!("  curating with {model} … ");
    let rt = tokio::runtime::Builder::new_current_thread().enable_all().build()?;
    let curated = rt.block_on(teacher.curate(&raw))?;
    eprintln!("{} facet types", curated.candidates.len());

    eprint!("  indexing … ");
    Ok(SteelDb::ingest_curated(docs, &curated, &raw.surfaces())?)
}

/// The flag exists in every build so it can explain itself rather than being silently absent.
#[cfg(not(all(feature = "onnx", feature = "embed", feature = "paddock")))]
fn neural_ingest(
    _corpus: &std::path::Path,
    _sample: usize,
    _model: Option<String>,
) -> Result<SteelDb, Box<dyn std::error::Error>> {
    Err("--neural needs the span tagger and a curator, which are not in this build.\n\
         Rebuild with:  cargo install hypersteeldb --features cli,onnx,embed\n\
         and fetch the weights:\n  \
         huggingface-cli download cp500/steeldb-models --local-dir ~/.steeldb/models"
        .into())
}

/// Build a teacher from a model name, routing `bedrock:<id>` to Bedrock and anything else to the local server.
///
/// Bedrock is the markedly better curator — on the same clusters it abstracts values up to kinds
/// (`6g` becomes `network-generation`) where a small local model keeps the value as the facet name — so it needs
/// to be reachable from the tool, not only from the library.
fn teacher_for(model: &str) -> Result<steeldb::learn::Teacher, Box<dyn std::error::Error>> {
    match model.strip_prefix("bedrock:") {
        Some(id) => {
            #[cfg(feature = "bedrock")]
            {
                return Ok(steeldb::learn::Teacher::bedrock(id)?);
            }
            #[cfg(not(feature = "bedrock"))]
            {
                let _ = id;
                Err("this build has no Bedrock support.\n\
                     Rebuild with:  cargo install hypersteeldb --features cli,onnx,embed,bedrock\n\
                     and set AWS_REGION with credentials on the standard chain."
                    .into())
            }
        }
        None => Ok(steeldb::learn::Teacher::ollama(model)?),
    }
}

fn read_dir_docs(dir: &std::path::Path) -> std::io::Result<Vec<String>> {
    let mut docs = Vec::new();
    let mut entries: Vec<PathBuf> = std::fs::read_dir(dir)?.filter_map(|e| e.ok()).map(|e| e.path()).collect();
    entries.sort();
    for p in entries {
        if matches!(p.extension().and_then(|e| e.to_str()), Some("md") | Some("txt")) {
            if let Ok(t) = std::fs::read_to_string(&p) {
                docs.push(t);
            }
        }
    }
    Ok(docs)
}

fn run<B: Backend>(
    terminal: &mut Terminal<B>,
    app: &mut App,
    rt: &tokio::runtime::Runtime,
) -> std::io::Result<()> {
    loop {
        terminal.draw(|f| draw(f, app))?;
        if !event::poll(Duration::from_millis(200))? {
            continue;
        }
        let Event::Key(k) = event::read()? else { continue };
        if k.kind != KeyEventKind::Press {
            continue;
        }
        if k.modifiers.contains(KeyModifiers::CONTROL) && matches!(k.code, KeyCode::Char('c')) {
            return Ok(());
        }
        // a popup swallows keys, so Esc always means "back" rather than "quit"
        if !matches!(app.popup, Popup::None) {
            if matches!(k.code, KeyCode::Esc | KeyCode::Enter | KeyCode::Char('q') | KeyCode::Char('?')) {
                app.popup = Popup::None;
            }
            continue;
        }
        match k.code {
            KeyCode::Esc => return Ok(()),
            KeyCode::Char('?') => app.popup = Popup::Help,
            KeyCode::Tab => {
                app.focus = if app.focus == Focus::Query { Focus::Categories } else { Focus::Query };
            }
            KeyCode::Enter => match app.focus {
                Focus::Query => {
                    let q = app.input.trim().to_string();
                    if !q.is_empty() {
                        app.run_query(&q);
                    }
                }
                Focus::Categories => {
                    if let Some(c) = app.db.categories().get(app.cat_sel) {
                        let w = c.wildcard();
                        app.input = w.clone();
                        app.run_query(&w);
                        app.focus = Focus::Query;
                    }
                }
            },
            KeyCode::Up => match app.focus {
                Focus::Categories => app.cat_sel = app.cat_sel.saturating_sub(1),
                Focus::Query => app.scroll = app.scroll.saturating_sub(1),
            },
            KeyCode::Down => match app.focus {
                Focus::Categories => {
                    let n = app.db.categories().len();
                    if n > 0 {
                        app.cat_sel = (app.cat_sel + 1).min(n - 1);
                    }
                }
                Focus::Query => app.scroll = app.scroll.saturating_add(1),
            },
            KeyCode::PageDown => app.scroll = app.scroll.saturating_add(10),
            KeyCode::PageUp => app.scroll = app.scroll.saturating_sub(10),
            KeyCode::Backspace if app.focus == Focus::Query => {
                app.input.pop();
            }
            KeyCode::Char(c) if app.focus == Focus::Query => app.input.push(c),
            KeyCode::Char('l') if app.focus == Focus::Categories => {
                app.status = "asking the model…".into();
                terminal.draw(|f| draw(f, app))?;
                app.learn(rt);
            }
            KeyCode::Char('s') if app.focus == Focus::Categories => app.save(false),
            KeyCode::Char('S') if app.focus == Focus::Categories => app.save(true),
            KeyCode::Char('q') if app.focus == Focus::Categories => return Ok(()),
            _ => {}
        }
    }
}

fn draw(f: &mut Frame, app: &App) {
    // A terminal can report zero size — `script` and some CI harnesses do, and a window can be dragged to
    // nothing. Laying out into a zero-area rect makes ratatui index an empty buffer and panic, which is a crash
    // in the one place the user cannot be shown a message. There is nothing to draw, so don't.
    if f.area().width == 0 || f.area().height == 0 {
        return;
    }

    let rows = Layout::default()
        .direction(Direction::Vertical)
        .constraints([Constraint::Length(1), Constraint::Min(3), Constraint::Length(3), Constraint::Length(1)])
        .split(f.area());

    // ── header
    let model = app.model.as_deref().unwrap_or("no model");
    let header = format!(
        " steel · {} · {} situations · {} categories · {} · learn: {} ",
        app.corpus.display(),
        app.db.len(),
        app.db.categories().len(),
        if app.following.is_some() { "artefact" } else { "discovered" },
        model
    );
    f.render_widget(
        Paragraph::new(Line::from(Span::styled(
            header,
            Style::default().fg(Color::Black).bg(Color::Cyan).add_modifier(Modifier::BOLD),
        )))
        .style(Style::default().bg(Color::Cyan)),
        rows[0],
    );

    let cols = Layout::default()
        .direction(Direction::Horizontal)
        .constraints([Constraint::Length(34), Constraint::Min(20)])
        .split(rows[1]);

    // ── categories: always visible, because their names are not guessable
    let cats = app.db.categories();
    let items: Vec<ListItem> = if cats.is_empty() {
        vec![ListItem::new(Line::from(Span::styled(
            "none — a corpus needs terms that recur across several documents",
            Style::default().fg(Color::DarkGray),
        )))]
    } else {
        cats.iter()
            .enumerate()
            .map(|(i, c)| {
                let sel = app.focus == Focus::Categories && i == app.cat_sel;
                let mark = if sel { "▸ " } else { "  " };
                let words = c.words.iter().skip(1).take(3).cloned().collect::<Vec<_>>().join(", ");
                ListItem::new(vec![
                    Line::from(vec![
                        Span::raw(mark),
                        Span::styled(
                            c.wildcard(),
                            Style::default()
                                .fg(if sel { Color::Yellow } else { Color::White })
                                .add_modifier(Modifier::BOLD),
                        ),
                    ]),
                    Line::from(Span::styled(format!("    {words}"), Style::default().fg(Color::DarkGray))),
                ])
            })
            .collect()
    };
    let cat_border = if app.focus == Focus::Categories { Color::Yellow } else { Color::DarkGray };
    f.render_widget(
        List::new(items).block(
            Block::default()
                .borders(Borders::ALL)
                .border_style(Style::default().fg(cat_border))
                .title(" you can ask about "),
        ),
        cols[0],
    );

    // ── results, or the refusal
    let mut body: Vec<Line> = Vec::new();
    if !app.refused.is_empty() {
        body.push(Line::from(Span::styled(
            format!("refused: {}", app.ran.clone().unwrap_or_default()),
            Style::default().fg(Color::Red).add_modifier(Modifier::BOLD),
        )));
        body.push(Line::from(""));
        for l in &app.refused {
            body.push(Line::from(Span::styled(l.clone(), Style::default().fg(Color::Yellow))));
        }
        body.push(Line::from(""));
        body.push(Line::from(Span::styled(
            "An empty result and an unanswerable question are different facts.",
            Style::default().fg(Color::DarkGray),
        )));
    } else if let Some(ran) = &app.ran {
        body.push(Line::from(vec![
            Span::styled(ran.clone(), Style::default().fg(Color::Cyan)),
            Span::raw("  →  "),
            Span::styled(
                format!("{} situations", app.count),
                Style::default().fg(Color::Green).add_modifier(Modifier::BOLD),
            ),
            Span::styled(format!("  ({:.0} µs)", app.micros), Style::default().fg(Color::DarkGray)),
        ]));
        body.push(Line::from(""));
        for (id, text) in &app.hits {
            body.push(Line::from(vec![
                Span::styled(format!("[{id}] "), Style::default().fg(Color::DarkGray)),
                Span::raw(text.clone()),
            ]));
        }
        if app.count > app.hits.len() {
            body.push(Line::from(Span::styled(
                format!("… {} more (the count is exact; the listing is capped)", app.count - app.hits.len()),
                Style::default().fg(Color::DarkGray),
            )));
        }
    } else {
        body.push(Line::from(Span::styled(
            "Type an expression and press Enter, or Tab to pick a category.",
            Style::default().fg(Color::DarkGray),
        )));
        body.push(Line::from(""));
        for ex in [
            "battle/*                      every value in a category",
            "(and battle/* (not state/negated))   both, excluding denials",
            "(num length_m gt 1000)        a numeric comparison",
            "(s-path :s 2 (source A) (target B))  linked by 2+ shared tags",
        ] {
            body.push(Line::from(Span::styled(format!("  {ex}"), Style::default().fg(Color::DarkGray))));
        }
    }
    f.render_widget(
        Paragraph::new(body)
            .wrap(Wrap { trim: false })
            .scroll((app.scroll, 0))
            .block(Block::default().borders(Borders::ALL).border_style(Style::default().fg(Color::DarkGray)).title(" result ")),
        cols[1],
    );

    // ── query line
    let q_border = if app.focus == Focus::Query { Color::Yellow } else { Color::DarkGray };
    f.render_widget(
        Paragraph::new(Line::from(vec![
            Span::styled("› ", Style::default().fg(Color::Yellow)),
            Span::raw(app.input.clone()),
            Span::styled(if app.focus == Focus::Query { "█" } else { "" }, Style::default().fg(Color::Yellow)),
        ]))
        .block(
            Block::default()
                .borders(Borders::ALL)
                .border_style(Style::default().fg(q_border))
                .title(" query "),
        ),
        rows[2],
    );

    // ── status
    // The hints are focus-dependent, because `l` and `s` type letters on the query line. Advertising them
    // regardless would be telling the user something untrue about the key under their finger.
    let hints = match app.focus {
        Focus::Query => "  Tab categories · Enter run · ↑↓ scroll · ? help · Esc quit",
        Focus::Categories => "  Tab query · Enter select · ↑↓ move · l learn · s save · S +training · ? help · q quit",
    };
    f.render_widget(
        Paragraph::new(Line::from(vec![
            Span::styled(format!(" {} ", app.status), Style::default().fg(Color::Black).bg(Color::Gray)),
            Span::styled(hints, Style::default().fg(Color::DarkGray)),
        ])),
        rows[3],
    );

    match &app.popup {
        Popup::None => {}
        Popup::Help => popup(f, " help ", &help_lines()),
        Popup::Learn(lines) => popup(f, " learn ", lines),
    }
}

fn help_lines() -> Vec<String> {
    [
        "Categories are discovered from your documents, so their names",
        "come from the words your text uses. Read them before querying.",
        "",
        "  Tab        move between the query line and the category list",
        "  Enter      run the query, or query the selected category",
        "  ↑ ↓        scroll results, or move in the category list",
        "  l          learn: ask a local model for a missed category",
        "  s / S      save the artefact set / and a finetune set",
        "  ? Esc      this help / quit",
        "",
        "Query forms:",
        "  cat/value            one exact tag",
        "  cat/*                any value in a category",
        "  (and A B) (or A B) (not A)",
        "  (num field gt 100)   numeric comparison",
        "  (evidence A :min-bel 0.8)",
        "  (s-path :s 2 (source A) (target B))",
        "",
        "A query naming something the corpus does not have is REFUSED,",
        "with the categories that do exist. That is not an error — an",
        "empty result and an unanswerable question are different facts.",
    ]
    .iter()
    .map(|s| s.to_string())
    .collect()
}

fn popup(f: &mut Frame, title: &str, lines: &[String]) {
    let area = f.area();
    if area.width < 4 || area.height < 3 {
        return;
    }
    let w = area.width.min(74).max(20);
    let h = (lines.len() as u16 + 4).min(area.height.saturating_sub(2)).max(5);
    let rect = Rect {
        x: area.x + (area.width.saturating_sub(w)) / 2,
        y: area.y + (area.height.saturating_sub(h)) / 2,
        width: w,
        height: h,
    };
    f.render_widget(Clear, rect);
    let body: Vec<Line> = lines
        .iter()
        .map(|l| {
            let style = if l.starts_with("KEEP") {
                Style::default().fg(Color::Green)
            } else if l.starts_with("drop") {
                Style::default().fg(Color::Red)
            } else {
                Style::default()
            };
            Line::from(Span::styled(l.clone(), style))
        })
        .collect();
    f.render_widget(
        Paragraph::new(body).wrap(Wrap { trim: false }).block(
            Block::default()
                .borders(Borders::ALL)
                .border_style(Style::default().fg(Color::Yellow))
                .title(title.to_string()),
        ),
        rect,
    );
}

#[cfg(test)]
mod tests {
    use super::*;
    use ratatui::backend::TestBackend;

    fn app() -> App {
        let docs: Vec<String> = [
            "Morty Shade defeated Wallace Gale at Ecruteak City during the Indigo Invitational in 2025.",
            "Bea Strike defeated Iris Draco at Ecruteak City during the Indigo Invitational in 2025.",
            "Lance Wing defeated Karen Dusk at Ecruteak City during the Indigo Invitational in 2025.",
            "A habitat survey recorded Aggron near Sootopolis City at an elevation of 1082 m.",
            "A habitat survey recorded Salamence near Sootopolis City at an elevation of 2369 m.",
            "A habitat survey recorded Metagross near Sootopolis City at an elevation of 640 m.",
            "Milotic is not permitted in Series 1 play for the 2025 season.",
            "Registeel is not permitted in Series 1 play for the 2025 season.",
        ]
        .iter()
        .map(|s| s.to_string())
        .collect();
        let db = SteelDb::ingest(docs).expect("ingest");
        App::new(db, PathBuf::from("/tmp/corp"), None, Some("test-model".into()))
    }

    /// Render to a buffer and return it as text, so assertions can be about what a user would see.
    fn render(app: &App, w: u16, h: u16) -> String {
        let mut term = Terminal::new(TestBackend::new(w, h)).expect("backend");
        term.draw(|f| draw(f, app)).expect("draw");
        let buf = term.backend().buffer().clone();
        (0..buf.area.height)
            .map(|y| {
                (0..buf.area.width).map(|x| buf[(x, y)].symbol().to_string()).collect::<String>()
            })
            .collect::<Vec<_>>()
            .join("\n")
    }

    #[test]
    fn the_categories_are_on_screen_before_anything_is_typed() {
        // The whole reason this is a UI and not a REPL: category names are discovered, so they cannot be
        // guessed, and the commonest way to waste time here is querying one that does not exist.
        let app = app();
        let screen = render(&app, 120, 30);
        assert!(screen.contains("you can ask about"), "{screen}");
        for c in app.db.categories() {
            assert!(screen.contains(&c.wildcard()), "category {} is not shown:\n{screen}", c.wildcard());
        }
    }

    #[test]
    fn a_refusal_is_rendered_as_prominently_as_an_answer() {
        let mut app = app();
        app.run_query("gene/brca1");
        let screen = render(&app, 120, 30);
        assert!(screen.contains("refused"), "{screen}");
        assert!(screen.contains("gene/brca1"), "the refused query must be echoed:\n{screen}");
        // and it must tell the user what to type instead
        assert!(screen.contains("try:"), "no alternatives offered:\n{screen}");
    }

    #[test]
    fn an_answer_shows_the_expression_that_ran_and_an_exact_count() {
        let mut app = app();
        let cat = app.db.categories().first().map(|c| c.wildcard()).expect("a category");
        app.run_query(&cat);
        let screen = render(&app, 120, 30);
        assert!(screen.contains(&cat), "{screen}");
        assert!(screen.contains("situations"), "{screen}");
        assert!(app.count > 0, "the first category should match something");
        assert!(app.refused.is_empty());
    }

    #[test]
    fn rendering_survives_a_tiny_terminal() {
        // A layout that panics on a small window is a crash in the one place a user cannot see a message.
        let mut app = app();
        app.run_query("gene/brca1");
        // 0x0 is not hypothetical: running under `script` reports it, and rendering into a zero-area rect
        // panicked inside ratatui's buffer indexing until `draw` learned to decline.
        for (w, h) in [(20u16, 6u16), (40, 10), (1, 1), (2, 2), (200, 80)] {
            let _ = render(&app, w, h);
        }
    }

    #[test]
    fn popups_render_without_overflowing_the_screen() {
        let mut app = app();
        app.popup = Popup::Help;
        let screen = render(&app, 100, 40);
        assert!(screen.contains("help"), "{screen}");
        app.popup = Popup::Learn(vec!["KEEP  x".into(), "drop  y".into()]);
        let screen = render(&app, 100, 40);
        assert!(screen.contains("KEEP"), "{screen}");
        // and in a window too small to hold the text
        app.popup = Popup::Help;
        let _ = render(&app, 24, 7);
    }

    #[test]
    fn the_preferred_model_list_is_ordered_and_unique() {
        let mut seen = std::collections::HashSet::new();
        for m in PREFERRED {
            assert!(seen.insert(*m), "{m} listed twice");
            assert!(m.contains(':'), "{m} should be a concrete ollama tag, not a family");
        }
        // an explicit choice always wins over discovery
        assert_eq!(pick_model(Some("mine:1b".into())), Some("mine:1b".to_string()));
    }

    /// The artefact contract, end to end: save it, and the next run must follow it.
    ///
    /// `steel` wrote the set to `<corpus>/.hypersteeldb` and then rediscovered on the next run, ignoring it
    /// unless `--artifact` was passed by hand. So a category learned in one session and saved was silently gone
    /// in the next — the opposite of what the artefact set is for.
    #[test]
    fn a_saved_set_beside_the_corpus_is_followed_on_the_next_run() {
        let dir = std::env::temp_dir().join(format!("steel_follow_{}", std::process::id()));
        let _ = std::fs::remove_dir_all(&dir);
        std::fs::create_dir_all(&dir).expect("mkdir");
        for (i, d) in [
            "Morty Shade defeated Wallace Gale at Ecruteak City in 2025.",
            "Bea Strike defeated Iris Draco at Ecruteak City in 2025.",
            "Lance Wing defeated Karen Dusk at Ecruteak City in 2025.",
            "A habitat survey recorded Aggron near Sootopolis City at 1082 m.",
            "A habitat survey recorded Salamence near Sootopolis City at 2369 m.",
            "A habitat survey recorded Metagross near Sootopolis City at 640 m.",
            "Milotic is not permitted in Series 1 play for the 2025 season.",
            "Registeel is not permitted in Series 1 play for the 2025 season.",
        ]
        .iter()
        .enumerate()
        {
            std::fs::write(dir.join(format!("d{i}.md")), d).expect("write");
        }

        let first = SteelDb::open(&dir).expect("open");
        let set = dir.join(".hypersteeldb");
        first.save(&set).expect("save");
        assert!(set.join("manifest.json").is_file(), "save should write a manifest");

        // exactly the detection main() performs
        let follows = set.join("manifest.json").is_file();
        assert!(follows, "a saved set must be detected beside the corpus");

        let docs = read_dir_docs(&dir).expect("read");
        let second = SteelDb::ingest_using(docs, &set).expect("follow");
        assert_eq!(second.askable(), first.askable(), "the followed vocabulary must match");
        assert_eq!(second.tags(), first.tags(), "and index identically");

        let _ = std::fs::remove_dir_all(&dir);
    }

    #[test]
    fn the_header_says_which_vocabulary_is_on_screen() {
        // Discovered and recorded vocabularies look identical once rendered, so the mode has to be stated or a
        // user cannot tell whether `learn` results are in play.
        let mut a = app();
        assert!(render(&a, 120, 20).contains("discovered"), "discovered mode is not labelled");

        a.following = Some(PathBuf::from("/tmp/corp/.hypersteeldb"));
        assert!(render(&a, 120, 20).contains("artefact"), "artefact mode is not labelled");
    }
}