hypersteeldb 0.5.5

//! SteelDB — a single terminal app with three workflows over ONE resident corpus (models + local Qwen
//! stay hot across views):
//!   • Ask       — agentic Q&A over the corpus (retrieval-as-reasoning), with live file-by-file ingest.
//!   • Schema    — browse facets and their top tokens (the queryable vocabulary).
//!   • Ontology  — Sinkhorn-OT ontology discovery over the corpus vocabulary, animated to convergence.
//! Tab / Shift-Tab switch views; Ctrl-C or Esc quits. `/add <path>` (Ask view) ingests more at runtime.
//!
//!   cargo run --features tui,paddock,onnx,docs --bin steeldb -- <dir|file>

use ratatui::crossterm::event::{self, Event, KeyCode, KeyModifiers};
use ratatui::crossterm::execute;
use ratatui::crossterm::terminal::{disable_raw_mode, enable_raw_mode, EnterAlternateScreen, LeaveAlternateScreen};
use ratatui::prelude::*;
use ratatui::widgets::{Block, Borders, Paragraph, Wrap};
use std::io::stdout;
use std::path::{Path, PathBuf};
use std::sync::mpsc;
use std::time::Duration;
use steeldb::agent::{run_agent, ProviderConfig};
use steeldb::discover_ontology::{candidate_terms, Cluster, OtDiscover};
use steeldb::projectors::{CsvProjector, JsonProjector, JsonlProjector, TextEngine};
use steeldb::text::Model2Vec;
use steeldb::{Corpus, CorpusKind, Projector};

const SPINNER: [&str; 8] = ["⠋", "⠙", "⠹", "⠸", "⠼", "⠴", "⠦", "⠧"];
const PALETTE: [Color; 8] = [Color::Cyan, Color::Green, Color::Yellow, Color::Magenta, Color::Blue, Color::Red, Color::LightGreen, Color::LightMagenta];
const OT_SCHEDULE: [usize; 14] = [1, 2, 3, 5, 8, 12, 18, 26, 40, 60, 90, 130, 180, 200];

#[derive(Clone, Copy, PartialEq)]
enum Mode {
    Ask,
    Schema,
    Ontology,
}
impl Mode {
    fn next(self) -> Mode {
        match self {
            Mode::Ask => Mode::Schema,
            Mode::Schema => Mode::Ontology,
            Mode::Ontology => Mode::Ask,
        }
    }
    fn prev(self) -> Mode {
        self.next().next()
    }
    fn title(self) -> &'static str {
        match self {
            Mode::Ask => "Ask",
            Mode::Schema => "Schema",
            Mode::Ontology => "Ontology",
        }
    }
}

fn provider_config() -> ProviderConfig {
    // Complete on-device default: Needle2 drives tool selection and deterministic templates synthesise
    // the answer — no external model provider required. A config file (steeldb.json) or env can opt into
    // an external OpenAI-compatible provider (OpenRouter/OpenAI/…) or Bedrock for natural-language
    // synthesis, but that is never needed for SteelDB to work.
    ProviderConfig::resolve("http://localhost:11434/v1", "qwen3:1.7b")
}

fn hot_engine() -> Option<TextEngine> {
    let ml = steeldb::paths::model_dir("step0_bundle_ml", "STEELDB_ML_BUNDLE", "spo.onnx")?;
    let splade = steeldb::paths::model_dir("splade", "STEELDB_SPLADE_DIR", "splade.onnx");
    let mut eng = TextEngine::load(&ml, splade.as_deref()).ok()?;
    // Prefer the TUNED tagger (step 2) when a checkpoint is available: it emits typed facet URIs and the
    // epistemic `state/…` cue that carries infon polarity, which the untyped ONNX tagger cannot.
    #[cfg(feature = "native")]
    if let Some((dir, base, tokenizer)) = tuned_tagger_paths() {
        match eng.enable_tuned(&dir, &base, &tokenizer, 128) {
            Ok(()) => eprintln!("tagger: tuned ({})", dir.display()),
            Err(e) => eprintln!("tagger: tuned unavailable ({e}) — using base ONNX tagger"),
        }
    }
    Some(eng)
}

/// Resolve `(tuned_dir, encoder_base, tokenizer)` for the tuned tagger, or `None` when not set up.
/// `STEELDB_TAGGER_MODEL` overrides; otherwise `models/tagger-tuned` via the standard model roots.
#[cfg(feature = "native")]
fn tuned_tagger_paths() -> Option<(PathBuf, PathBuf, PathBuf)> {
    let home = std::env::var("HOME").unwrap_or_default();
    let pick = |pat: String| -> Option<PathBuf> {
        let (d, _) = pat.rsplit_once('/')?;
        std::fs::read_dir(d).ok()?.filter_map(|e| e.ok()).map(|e| e.path()).find(|p| p.is_dir())
    };
    let dir = std::env::var("STEELDB_TAGGER_MODEL")
        .ok()
        .map(PathBuf::from)
        .or_else(|| steeldb::paths::model_dir("tagger-tuned", "STEELDB_TAGGER_MODEL", "tagger.json"))
        .filter(|d| d.join("tagger.json").exists())?;
    let base = std::env::var("STEELDB_TAGGER_BASE")
        .ok()
        .map(PathBuf::from)
        .or_else(|| pick(format!("{home}/.cache/huggingface/hub/models--google--bert_uncased_L-2_H-128_A-2/snapshots/*")))?;
    let tokenizer = std::env::var("STEELDB_TAGGER_TOKENIZER")
        .ok()
        .map(PathBuf::from)
        .or_else(|| pick(format!("{home}/.cache/huggingface/hub/models--bert-base-uncased/snapshots/*")).map(|p| p.join("tokenizer.json")))?;
    Some((dir, base, tokenizer))
}

fn collect(path: &Path) -> Vec<PathBuf> {
    if path.is_file() {
        return vec![path.to_path_buf()];
    }
    let mut out = Vec::new();
    let mut stack = vec![path.to_path_buf()];
    while let Some(d) = stack.pop() {
        let Ok(rd) = std::fs::read_dir(&d) else { continue };
        for e in rd.flatten() {
            let name = e.file_name().to_string_lossy().to_string();
            if name.starts_with('.') || name == "node_modules" || name == "target" {
                continue;
            }
            let p = e.path();
            if p.is_dir() {
                stack.push(p);
            } else {
                out.push(p);
            }
        }
    }
    out.sort();
    out
}

/// Overlay file for the growing gazetteer, stored beside the index in the scanned directory.
fn overlay_path_for(dir: &str) -> PathBuf {
    let p = Path::new(dir);
    let base = if p.is_dir() { p.to_path_buf() } else { p.parent().map(|x| x.to_path_buf()).unwrap_or_else(|| PathBuf::from(".")) };
    base.join(".steeldb-gazetteer.json")
}

fn ingest_file(corpus: &mut Corpus, engine: &mut Option<TextEngine>, path: &Path) -> usize {
    let ext = path.extension().and_then(|e| e.to_str()).unwrap_or("").to_lowercase();
    let name = path.file_name().map(|n| n.to_string_lossy().to_string()).unwrap_or_default();
    let src_tok = format!("src/{}", steeldb::projector::slug(&name));
    let mut n = 0usize;
    let mut sink = |s: steeldb::Situation, disp: String| {
        let mut toks = s.tokens;
        toks.push(src_tok.clone());
        corpus.add_situation_polar(toks, vec![name.clone(), disp], s.numbers, s.beliefs);
        n += 1;
    };
    match ext.as_str() {
        "csv" | "tsv" => {
            if let Ok(p) = CsvProjector::open(path) {
                let _ = Box::new(p).project(&mut |s| {
                    let d = s.display.join(" · ");
                    sink(s, d);
                });
            }
        }
        "json" | "ndjson" => {
            let _ = Box::new(JsonProjector::open(path)).project(&mut |s| {
                let d = s.display.join(" · ");
                sink(s, d);
            });
        }
        "jsonl" => {
            let _ = Box::new(JsonlProjector::open(path, None)).project(&mut |s| {
                let d = s.display.join(" · ");
                sink(s, d);
            });
        }
        _ => {
            #[cfg(feature = "docs")]
            if steeldb::docs::is_doc_ext(&ext) {
                if let Some(eng) = engine.as_mut() {
                    if let Ok(Some(text)) = steeldb::docs::extract_text(path) {
                        eng.project_text(&text, &mut |s| {
                            let d = s.display.join(" ");
                            sink(s, d);
                        });
                    }
                }
            }
            #[cfg(not(feature = "docs"))]
            let _ = engine;
        }
    }
    n
}

enum Job {
    Ask(String),
    Add(String),
    Schema,
    Ontology,
}
enum Reply {
    Status(String),
    Line(Line<'static>),
    Stats { sits: u32, toks: usize },
    Schema(Vec<(String, usize, Vec<(String, usize)>)>),
    Ontology(Box<OtDiscover>),
    Done,
}

fn main() -> Result<(), Box<dyn std::error::Error>> {
    let args: Vec<String> = std::env::args().collect();
    let dir = args.get(1).filter(|a| !a.starts_with("--")).cloned();

    // Headless mode: `steeldb <dir> --ask "question"` scans the folder, answers, and exits (the
    // DuckDB `-c` equivalent — scriptable, pipeable, testable without the alternate-screen UI).
    if let Some(pos) = args.iter().position(|a| a == "--ask") {
        let question = args.get(pos + 1).cloned().unwrap_or_default();
        let dir = dir.clone().unwrap_or_else(|| ".".to_string());
        return run_headless(&dir, &question);
    }
    // STEP 0 — `steeldb <dir> --init`: discover the corpus's facet spec (Vocabulary Space V) BEFORE
    // ingest and persist it beside the data. Ported from the reference `design.py` step-0 flow.
    if args.iter().any(|a| a == "--init") {
        let dir = dir.clone().unwrap_or_else(|| ".".to_string());
        return run_init(&dir);
    }
    // STEP 1 — `steeldb <dir> --gen-tagger-data <out.jsonl>`: synthesise the tagger's finetuning set
    // from the corpus's discovered spec (typed spans + epistemic flags + bound relations), validating
    // every example against the spec before keeping it.
    if let Some(pos) = args.iter().position(|a| a == "--gen-tagger-data") {
        let out = args.get(pos + 1).cloned().unwrap_or_else(|| "tagger_data.jsonl".to_string());
        let dir = dir.clone().unwrap_or_else(|| ".".to_string());
        return run_gen_tagger_data(&dir, &out);
    }
    // STEP 2 — `steeldb <dir> --train-tagger <data.jsonl> [--out <dir>]`: finetune the multi-head tagger
    // (Head A typed BIO + Head B epistemic/polarity) on the step-1 dataset, in-process via candle.
    #[cfg(feature = "native")]
    if let Some(pos) = args.iter().position(|a| a == "--train-tagger") {
        let data = args.get(pos + 1).cloned().unwrap_or_else(|| "tagger_data.jsonl".to_string());
        let out = args.iter().position(|a| a == "--out").and_then(|i| args.get(i + 1)).cloned().unwrap_or_else(|| "models/tagger-tuned".to_string());
        let dir = dir.clone().unwrap_or_else(|| ".".to_string());
        return run_train_tagger(&dir, &data, &out);
    }
    // `steeldb <dir> --tag "text"` — run the tuned tagger (step 2 output) over a sentence and print the
    // typed spans + epistemic polarity it predicts. The honest check on a finetune: held-out text.
    #[cfg(feature = "native")]
    if let Some(pos) = args.iter().position(|a| a == "--tag") {
        let text = args.get(pos + 1).cloned().unwrap_or_default();
        let model = args.iter().position(|a| a == "--model").and_then(|i| args.get(i + 1)).cloned().unwrap_or_else(|| "models/tagger-tuned".to_string());
        return run_tag(&model, &text);
    }
    // `steeldb <dir> --project "text"` — full projection: tuned tagger → Vocabulary-Space tokens +
    // infon polarity, then ingest into a corpus and report the DS belief interval per token.
    #[cfg(feature = "native")]
    if let Some(pos) = args.iter().position(|a| a == "--project") {
        let text = args.get(pos + 1).cloned().unwrap_or_default();
        let model = args.iter().position(|a| a == "--model").and_then(|i| args.get(i + 1)).cloned().unwrap_or_else(|| "models/tagger-tuned".to_string());
        return run_project(&model, &text);
    }
    // `steeldb <dir> --ikl "(and org/* (not state/negated))"` — the engine's native query surface: raw
    // IKL set-algebra over the bitmap, linted first. No model in the loop (paper §3).
    if let Some(pos) = args.iter().position(|a| a == "--ikl") {
        let expr = args.get(pos + 1).cloned().unwrap_or_default();
        let dir = dir.clone().unwrap_or_else(|| ".".to_string());
        return run_ikl(&dir, &expr);
    }
    // STEP 5 — `steeldb <dir> --train-relations <data.jsonl> --model <tagger-dir>`: train Head C (the
    // biaffine relation scorer) on top of the tuned encoder, giving dimension 2 (`rel/x/+`, `rel/x/-`).
    #[cfg(feature = "native")]
    if let Some(pos) = args.iter().position(|a| a == "--train-relations") {
        let data = args.get(pos + 1).cloned().unwrap_or_default();
        let model = args.iter().position(|a| a == "--model").and_then(|i| args.get(i + 1)).cloned().unwrap_or_else(|| "models/tagger-tuned".to_string());
        let dir = dir.clone().unwrap_or_else(|| ".".to_string());
        return run_train_relations(&dir, &data, &model);
    }
    // STEPS 3-4 — `steeldb <dir> --grow [--rounds N] [--gain X]`: grow the ontology one agent-proposed
    // facet at a time, gated by the MECE criterion, then register the version with its metrics.
    if args.iter().any(|a| a == "--grow") {
        let dir = dir.clone().unwrap_or_else(|| ".".to_string());
        let rounds = args.iter().position(|a| a == "--rounds").and_then(|i| args.get(i + 1)).and_then(|v| v.parse().ok()).unwrap_or(3);
        let gain = args.iter().position(|a| a == "--gain").and_then(|i| args.get(i + 1)).and_then(|v| v.parse().ok()).unwrap_or(0.05);
        return run_grow(&dir, rounds, gain);
    }
    // STEP 1c — `steeldb <dir> --mine-gazetteer <data.jsonl>`: derive the high-resolution whole-entity
    // gazetteer from step-1's labelled spans (grounded by construction) and merge it into the spec.
    if let Some(pos) = args.iter().position(|a| a == "--mine-gazetteer") {
        let data = args.get(pos + 1).cloned().unwrap_or_default();
        let dir = dir.clone().unwrap_or_else(|| ".".to_string());
        return run_mine_gazetteer(&dir, &data);
    }
    // `steeldb <dir> --gen-ikl-data <out.jsonl>`: verified (question → compound IKL) training data for the
    // model that writes S-expressions. Every target is linted and executed against this corpus first.
    if let Some(pos) = args.iter().position(|a| a == "--gen-ikl-data") {
        let out = args.get(pos + 1).cloned().unwrap_or_else(|| "ikl_data.jsonl".to_string());
        let dir = dir.clone().unwrap_or_else(|| ".".to_string());
        return run_gen_ikl(&dir, &out);
    }
    // Pure retrieval — no LLM. Scan files, resolve the query to corpus tokens, print matching passages.
    // The DuckDB-scan layer on its own: `steeldb <dir> --search "keywords"`.
    if let Some(pos) = args.iter().position(|a| a == "--search") {
        let query = args.get(pos + 1).cloned().unwrap_or_default();
        let dir = dir.clone().unwrap_or_else(|| ".".to_string());
        return run_search(&dir, &query);
    }

    let (job_tx, job_rx) = mpsc::channel::<Job>();
    let (rep_tx, rep_rx) = mpsc::channel::<Reply>();
    let cfg = provider_config();

    std::thread::spawn(move || {
        let rt = tokio::runtime::Builder::new_current_thread().enable_all().build().expect("rt");
        // Model init can be slow (native inference builds/loads a local model); tell the UI so it doesn't
        // look frozen. The first native run quantizes the tuned model once (~a few minutes on CPU).
        let init_msg = match &cfg {
            ProviderConfig::Native { .. } => "preparing local model (first run builds a quantized model, ~a few min)…",
            _ => "connecting to model…",
        };
        let _ = rep_tx.send(Reply::Status(init_msg.into()));
        let provider = rt.block_on(cfg.build());
        match &provider {
            Ok(p) => {
                let _ = rep_tx.send(Reply::Status(format!("model ready ({}) · scanning…", p.name())));
            }
            Err(e) => {
                let _ = rep_tx.send(Reply::Status(format!("model unavailable: {e}")));
            }
        }
        let mut engine = hot_engine();
        let mut m2v: Option<Model2Vec> = None;
        let mut corpus = Corpus::new_incremental("(live)", vec!["file".into(), "record".into()], CorpusKind::Csv);

        let send_schema = |corpus: &Corpus, tx: &mpsc::Sender<Reply>| {
            let facets: Vec<(String, usize, Vec<(String, usize)>)> = corpus
                .stats()
                .facets
                .into_iter()
                .filter(|(f, _)| f != "src")
                .map(|(f, n)| (f.clone(), n, corpus.facet_tokens(&f, 20)))
                .collect();
            let _ = tx.send(Reply::Schema(facets));
        };

        while let Ok(job) = job_rx.recv() {
            match job {
                Job::Add(path) => {
                    let overlay = overlay_path_for(&path);
                    if let Some(eng) = engine.as_mut() {
                        eng.enable_growth(&overlay);
                    }
                    corpus.set_gazetteer_overlay(&overlay);
                    for f in collect(Path::new(&path)) {
                        let fname = f.file_name().map(|n| n.to_string_lossy().to_string()).unwrap_or_default();
                        let _ = rep_tx.send(Reply::Status(format!("ingesting {fname}…")));
                        let added = ingest_file(&mut corpus, &mut engine, &f);
                        let s = corpus.stats();
                        let _ = rep_tx.send(Reply::Line(dim(&format!("+ {fname}  ({added} situations)"))));
                        let _ = rep_tx.send(Reply::Stats { sits: s.situations, toks: s.vocab });
                    }
                    if let Some(eng) = engine.as_ref() {
                        let n = eng.save_overlay();
                        if n > 0 {
                            let _ = rep_tx.send(Reply::Line(dim(&format!("· gazetteer overlay: {n} learned entities"))));
                        }
                    }
                    send_schema(&corpus, &rep_tx);
                }
                Job::Ask(q) => match &provider {
                    Ok(p) => match rt.block_on(run_agent(p.as_ref(), &corpus, &q, 14)) {
                        Ok(ans) => {
                            for l in answer_lines(&ans.answer) {
                                let _ = rep_tx.send(Reply::Line(l));
                            }
                            let tools: Vec<String> = ans.trace.iter().map(|t| t.name.clone()).collect();
                            if !tools.is_empty() {
                                let _ = rep_tx.send(Reply::Line(dim(&format!("· {} tool calls: {}", ans.trace.len(), tools.join(", ")))));
                            }
                        }
                        Err(e) => {
                            let _ = rep_tx.send(Reply::Line(errln(&e)));
                        }
                    },
                    Err(e) => {
                        let _ = rep_tx.send(Reply::Line(errln(&format!("provider: {e}"))));
                    }
                },
                Job::Schema => send_schema(&corpus, &rep_tx),
                Job::Ontology => {
                    if m2v.is_none() {
                        m2v = steeldb::paths::model_dir("model2vec", "STEELDB_MODEL2VEC", "potion.f32")
                            .and_then(|dir| Model2Vec::load(&dir).ok());
                    }
                    match &m2v {
                        Some(m) => {
                            // terms from the corpus vocabulary, plus any display text, embedded once
                            let mut terms = corpus.top_token_leaves(220);
                            terms.extend(candidate_terms(&corpus.stats().facets.iter().map(|(f, _)| f.clone()).collect::<Vec<_>>().join(" "), 40));
                            terms.sort();
                            terms.dedup();
                            let mut kept = Vec::new();
                            let mut embs = Vec::new();
                            for t in terms {
                                if let Some(e) = m.embed(&t) {
                                    kept.push(t);
                                    embs.push(e);
                                }
                            }
                            if kept.len() >= 4 {
                                let k = 8.min(kept.len() / 3).max(2);
                                let _ = rep_tx.send(Reply::Ontology(Box::new(OtDiscover::new(kept, embs, k))));
                            } else {
                                let _ = rep_tx.send(Reply::Status("ontology: not enough vocabulary yet".into()));
                            }
                        }
                        None => {
                            let _ = rep_tx.send(Reply::Status("ontology: model2vec unavailable (set STEELDB_MODEL2VEC)".into()));
                        }
                    }
                }
            }
            let _ = rep_tx.send(Reply::Done);
        }
    });

    let model = match provider_config() {
        ProviderConfig::Paddock { model, .. } => model,
        ProviderConfig::Bedrock { model_id, .. } => model_id,
        ProviderConfig::Native { .. } => "native:qwen3-1.7b+lora".to_string(),
        ProviderConfig::Needle { .. } => "needle-26m".to_string(),
    };
    let mut app = App::new(model);
    if let Some(d) = &dir {
        app.busy = true;
        app.status = "ingesting…".into();
        let _ = job_tx.send(Job::Add(d.clone()));
        app.push(dim(&format!("ingesting {d} … then ask a question, or Tab to Schema / Ontology")));
    } else {
        app.push(dim("empty corpus — /add <path> to ingest, then ask. Tab switches views."));
    }

    enable_raw_mode()?;
    let mut out = stdout();
    execute!(out, EnterAlternateScreen)?;
    let mut terminal = Terminal::new(CrosstermBackend::new(out))?;
    let res = run_ui(&mut terminal, &mut app, &job_tx, &rep_rx);
    disable_raw_mode()?;
    execute!(terminal.backend_mut(), LeaveAlternateScreen)?;
    terminal.show_cursor()?;
    res.map_err(Into::into)
}

/// Headless scan-and-answer: ingest a folder/file (any readable type), run the agent once, print the
/// grounded answer + tool trace to stdout, exit. No TUI.
fn run_headless(dir: &str, question: &str) -> Result<(), Box<dyn std::error::Error>> {
    let rt = tokio::runtime::Builder::new_current_thread().enable_all().build()?;
    let mut engine = hot_engine();
    let overlay = overlay_path_for(dir);
    if let Some(eng) = engine.as_mut() {
        eng.enable_growth(&overlay);
    }
    let mut corpus = Corpus::new_incremental(dir, vec!["file".into(), "record".into()], CorpusKind::Csv);
    corpus.set_gazetteer_overlay(&overlay);
    let files = collect(Path::new(dir));
    eprint!("scanning {} file(s) in {dir} … ", files.len());
    for f in &files {
        ingest_file(&mut corpus, &mut engine, f);
    }
    if let Some(eng) = engine.as_ref() {
        let n = eng.save_overlay();
        if n > 0 {
            eprintln!("gazetteer overlay: {n} learned entities → {}", overlay.display());
        }
    }
    let s = corpus.stats();
    eprintln!("{} situations, {} tokens", s.situations, s.vocab);
    if question.trim().is_empty() {
        eprintln!("(no --ask question; corpus built)");
        return Ok(());
    }
    let provider = rt.block_on(provider_config().build())?;
    eprintln!("provider: {} · asking: {question}", provider.name());
    match rt.block_on(run_agent(provider.as_ref(), &corpus, question, 14)) {
        Ok(ans) => {
            eprintln!("── {} tool calls: {} ──", ans.trace.len(), ans.trace.iter().map(|t| t.name.clone()).collect::<Vec<_>>().join(", "));
            println!("{}", ans.answer);
            Ok(())
        }
        Err(e) => Err(e.into()),
    }
}

/// STEP 0: facet-spec discovery. Samples the corpus (as `design.py::load_docs` does — scrubbed, truncated,
/// spread across the corpus), derives a seed facet set, validates the MECE/disjointness invariants, and
/// persists `.steeldb-facets.json` beside the data. This spec is what types entity spans, binds relation
/// polarity, gives wildcards their hierarchy, and gives the linter its authoritative vocabulary.
fn run_init(dir: &str) -> Result<(), Box<dyn std::error::Error>> {
    use steeldb::vocabulary::{sample_docs, seed_from_sample, spec_path, VocabularySpace};
    let path = Path::new(dir);
    eprint!("step 0 · sampling {dir} … ");
    let samples = sample_docs(path, 96, 4000);
    eprintln!("{} docs", samples.len());
    if samples.is_empty() {
        return Err(format!("no sampleable documents under {dir}").into());
    }

    // Seed: structural discovery from the sample (model-free). An LLM pass (PROPOSAL_SYSTEM /
    // proposal_schema) refines this into a disjoint hierarchy with directed relations.
    let min_support = 3; // absolute floor: heterogeneous corpora spread fields thinly across shapes
    let mut spec = seed_from_sample(dir, &samples, min_support);
    if let Some(existing) = VocabularySpace::for_corpus(path) {
        // keep any hand-authored or LLM-proposed hierarchy/relations already on disk
        if !existing.relation_facets.is_empty() || existing.entity_facets.iter().any(|f| f.parent.is_some()) {
            eprintln!("step 0 · merging with existing spec ({} relations)", existing.relation_facets.len());
        }
        spec = steeldb::vocabulary::merge_existing(spec, &existing);
    }

    // Python-faithful step 0: an agent proposes the minimal DISJOINT taxonomy with directed relations.
    // `--llm` opts in; the structural seed remains the model-free default and the merge base.
    if std::env::args().any(|a| a == "--llm") {
        let rt = tokio::runtime::Builder::new_current_thread().enable_all().build()?;
        eprint!("step 0 · proposing ontology via model … ");
        let proposed = rt.block_on(async {
            let provider = provider_config().build().await?;
            steeldb::vocabulary::propose(provider.as_ref(), dir, &samples).await
        });
        match proposed {
            Ok(p) => {
                eprintln!("{} facets, {} relations", p.entity_facets.len(), p.relation_facets.len());
                spec = steeldb::vocabulary::merge(p, &spec);
            }
            Err(e) => eprintln!("failed ({e}) — keeping structural seed"),
        }
    }
    spec.validate().map_err(|e| format!("invalid spec: {e}"))?;
    let out = spec_path(path);
    spec.save(&out)?;

    println!("facet spec → {}", out.display());
    println!("{} entity facets (min_support {min_support} of {} sampled docs):", spec.entity_facets.len(), samples.len());
    for f in spec.entity_facets.iter().take(30) {
        println!("  {:<28} {}", spec.facet_path(&f.name), f.description);
    }
    if !spec.relation_facets.is_empty() {
        println!("{} relations:", spec.relation_facets.len());
        for r in &spec.relation_facets {
            println!("  rel/{}/+  {} → {}", r.name, r.head, r.tail);
        }
    }
    println!("\nwildcard stems: {}", spec.valid_prefixes().join("  "));
    Ok(())
}

/// STEP 1: synthesise the tagger finetuning dataset for this corpus's discovered spec.
fn run_gen_tagger_data(dir: &str, out: &str) -> Result<(), Box<dyn std::error::Error>> {
    use steeldb::tagger_data::{generate, head_a_labels, head_c_labels, to_jsonl, GenPlan};
    use steeldb::vocabulary::VocabularySpace;
    let path = Path::new(dir);
    let spec = VocabularySpace::for_corpus(path).ok_or_else(|| format!("no facet spec for {dir} — run `steeldb {dir} --init --llm` first (step 0)"))?;
    if spec.relation_facets.is_empty() {
        return Err("spec has no relations — step 0 needs the --llm pass to propose directed relations".into());
    }
    // --grounded: label REAL corpus passages instead of inventing sentences (the tune_ontology.py
    // correction — invented text teaches the heads a vocabulary the corpus does not have).
    if std::env::args().any(|a| a == "--grounded") {
        use steeldb::tagger_data::{generate_grounded, passages_from_docs};
        use steeldb::vocabulary::sample_docs;
        let n_docs: usize = std::env::var("STEELDB_GEN_DOCS").ok().and_then(|v| v.parse().ok()).unwrap_or(120);
        let docs = sample_docs(path, n_docs, 6000);
        let mut passages = passages_from_docs(&docs, 10, 400);
        let cap: usize = std::env::var("STEELDB_GEN_PASSAGES").ok().and_then(|v| v.parse().ok()).unwrap_or(240);
        passages.truncate(cap);
        eprintln!("step 1 (grounded) · {} passages from {} docs · Head A {} labels", passages.len(), docs.len(), head_a_labels(&spec).len());
        let rt = tokio::runtime::Builder::new_current_thread().enable_all().build()?;
        let (examples, report) = rt.block_on(async {
            let provider = provider_config().build().await?;
            Ok::<_, String>(generate_grounded(provider.as_ref(), &spec, &passages, 8).await)
        })?;
        std::fs::write(out, to_jsonl(&examples))?;
        println!("wrote {} grounded examples → {out}", examples.len());
        println!("{}", serde_json::to_string_pretty(&report)?);
        return Ok(());
    }
    // STEELDB_GEN_SCALE multiplies every case count — the lever for a production-sized dataset.
    let scale: usize = std::env::var("STEELDB_GEN_SCALE").ok().and_then(|v| v.parse().ok()).unwrap_or(1);
    let d = GenPlan::default();
    let plan = GenPlan {
        normal: d.normal * scale,
        coref: d.coref * scale,
        hedged: d.hedged * scale,
        negated: d.negated * scale,
        adversarial: d.adversarial * scale,
    };
    eprintln!(
        "step 1 · {} relations x {} cases = up to {} examples · Head A {} labels, Head C {} labels",
        spec.relation_facets.len(),
        plan.cases().len(),
        spec.relation_facets.len() * plan.total_per_relation(),
        head_a_labels(&spec).len(),
        head_c_labels(&spec).len()
    );
    let rt = tokio::runtime::Builder::new_current_thread().enable_all().build()?;
    let (examples, report) = rt.block_on(async {
        let provider = provider_config().build().await?;
        Ok::<_, String>(generate(provider.as_ref(), &spec, plan).await)
    })?;
    std::fs::write(out, to_jsonl(&examples))?;
    println!("wrote {} examples → {out}", examples.len());
    println!("{}", serde_json::to_string_pretty(&report)?);
    Ok(())
}

/// STEP 2: finetune the tagger on the generated dataset (candle, in-process).
#[cfg(feature = "native")]
fn run_train_tagger(dir: &str, data: &str, out: &str) -> Result<(), Box<dyn std::error::Error>> {
    use steeldb::tagger_data::TaggerExample;
    use steeldb::tagger_train::{save, train, TrainConfig};
    use steeldb::vocabulary::VocabularySpace;
    let spec = VocabularySpace::for_corpus(Path::new(dir)).ok_or_else(|| format!("no facet spec for {dir} — run step 0 first"))?;
    let examples: Vec<TaggerExample> = std::fs::read_to_string(data)?
        .lines()
        .filter(|l| !l.trim().is_empty())
        .filter_map(|l| serde_json::from_str(l).ok())
        .collect();
    if examples.is_empty() {
        return Err(format!("no examples in {data}").into());
    }
    // bases resolve from the HF cache; override with STEELDB_TAGGER_BASE / STEELDB_TAGGER_TOKENIZER
    let home = std::env::var("HOME").unwrap_or_default();
    let pick = |pat: String| -> Option<std::path::PathBuf> {
        let (d, _) = pat.rsplit_once('/')?;
        std::fs::read_dir(d).ok()?.filter_map(|e| e.ok()).map(|e| e.path()).find(|p| p.is_dir())
    };
    let base = std::env::var("STEELDB_TAGGER_BASE").ok().map(std::path::PathBuf::from)
        .or_else(|| pick(format!("{home}/.cache/huggingface/hub/models--google--bert_uncased_L-2_H-128_A-2/snapshots/*")))
        .ok_or("no base model — set STEELDB_TAGGER_BASE to an HF snapshot dir (config.json + model.safetensors)")?;
    let tokenizer = std::env::var("STEELDB_TAGGER_TOKENIZER").ok().map(std::path::PathBuf::from)
        .or_else(|| pick(format!("{home}/.cache/huggingface/hub/models--bert-base-uncased/snapshots/*")).map(|p| p.join("tokenizer.json")))
        .ok_or("no tokenizer.json — set STEELDB_TAGGER_TOKENIZER")?;

    let epochs = std::env::var("STEELDB_TAGGER_EPOCHS").ok().and_then(|v| v.parse().ok()).unwrap_or(20);
    // A small randomly-initialised head tolerates 1e-3, but finetuning a large pretrained encoder at that
    // rate destroys it — override with STEELDB_TAGGER_LR (3e-5 is the usual range for BERT-base).
    let lr: f64 = std::env::var("STEELDB_TAGGER_LR").ok().and_then(|v| v.parse().ok()).unwrap_or(1e-3);
    let batch: usize = std::env::var("STEELDB_TAGGER_BATCH").ok().and_then(|v| v.parse().ok()).unwrap_or(8);
    let cfg = TrainConfig { base_dir: base, tokenizer, epochs, lr, batch, max_len: 128, ..Default::default() };
    eprintln!("step 2 · training on {} examples · base {} · {} epochs", examples.len(), cfg.base_dir.display(), cfg.epochs);
    let (varmap, report, labels) = train(&spec, &examples, &cfg)?;
    save(&varmap, &labels, Path::new(out))?;
    println!("tagger → {out}");
    println!("{}", serde_json::to_string_pretty(&report)?);
    Ok(())
}

/// Run the tuned tagger over one sentence (held-out evaluation of step 2).
#[cfg(feature = "native")]
fn run_tag(model_dir: &str, text: &str) -> Result<(), Box<dyn std::error::Error>> {
    use steeldb::tagger_train::TunedTagger;
    let home = std::env::var("HOME").unwrap_or_default();
    let pick = |pat: String| -> Option<std::path::PathBuf> {
        let (d, _) = pat.rsplit_once('/')?;
        std::fs::read_dir(d).ok()?.filter_map(|e| e.ok()).map(|e| e.path()).find(|p| p.is_dir())
    };
    let base = std::env::var("STEELDB_TAGGER_BASE").ok().map(std::path::PathBuf::from)
        .or_else(|| pick(format!("{home}/.cache/huggingface/hub/models--google--bert_uncased_L-2_H-128_A-2/snapshots/*")))
        .ok_or("no base model")?;
    let tokenizer = std::env::var("STEELDB_TAGGER_TOKENIZER").ok().map(std::path::PathBuf::from)
        .or_else(|| pick(format!("{home}/.cache/huggingface/hub/models--bert-base-uncased/snapshots/*")).map(|p| p.join("tokenizer.json")))
        .ok_or("no tokenizer")?;
    let tt = TunedTagger::load(Path::new(model_dir), &base, &tokenizer, 128)?;
    for span in tt.tag(text)? {
        let flags = match (span.negated, span.hedged) {
            (false, false) => "asserted",
            (false, true) => "hedged",
            (true, true) => "negated+hedged",
            (true, false) => "negated",
        };
        println!("  [{:>3}:{:<3}] {:<10} {:<34} {:<15} i={:+.1}", span.start, span.end, span.facet, format!("{:?}", span.text), flags, span.belief);
    }
    Ok(())
}

/// Project text through the tuned tagger into the bitmap and report DS belief per token.
#[cfg(feature = "native")]
fn run_project(model_dir: &str, text: &str) -> Result<(), Box<dyn std::error::Error>> {
    use steeldb::tagger_train::TunedTagger;
    let home = std::env::var("HOME").unwrap_or_default();
    let pick = |pat: String| -> Option<std::path::PathBuf> {
        let (d, _) = pat.rsplit_once('/')?;
        std::fs::read_dir(d).ok()?.filter_map(|e| e.ok()).map(|e| e.path()).find(|p| p.is_dir())
    };
    let base = std::env::var("STEELDB_TAGGER_BASE").ok().map(std::path::PathBuf::from)
        .or_else(|| pick(format!("{home}/.cache/huggingface/hub/models--google--bert_uncased_L-2_H-128_A-2/snapshots/*"))).ok_or("no base")?;
    let tokenizer = std::env::var("STEELDB_TAGGER_TOKENIZER").ok().map(std::path::PathBuf::from)
        .or_else(|| pick(format!("{home}/.cache/huggingface/hub/models--bert-base-uncased/snapshots/*")).map(|p| p.join("tokenizer.json"))).ok_or("no tokenizer")?;
    let mut tt = TunedTagger::load(Path::new(model_dir), &base, &tokenizer, 128)?;
    // Head C (dim 2) needs the spec for its type mask; look beside the corpus, then the model dir.
    if let Some(spec) = steeldb::vocabulary::VocabularySpace::for_corpus(Path::new("records"))
        .or_else(|| steeldb::vocabulary::VocabularySpace::for_corpus(Path::new(".")))
    {
        match tt.enable_relations(Path::new(model_dir), &spec) {
            Ok(()) => eprintln!("head C: enabled ({} relations)", spec.relation_facets.len()),
            Err(e) => eprintln!("head C: not enabled ({e})"),
        }
    }
    let sit = tt.project(text)?;
    let mut corpus = Corpus::new_incremental("(project)", vec!["text".into()], CorpusKind::Csv);
    corpus.add_situation_polar(sit.tokens.clone(), sit.display.clone(), sit.numbers.clone(), sit.beliefs.clone());
    println!("tokens:");
    for t in &sit.tokens {
        let (bel, pl) = corpus.belief_interval(t);
        let i = sit.beliefs.iter().find(|(k, _)| k == t).map(|(_, v)| *v).unwrap_or(1.0);
        println!("  {t:<34} i={i:+.1}  Bel={bel:.2} Pl={pl:.2}");
    }
    if !sit.numbers.is_empty() {
        println!("numeric: {:?}", sit.numbers);
    }
    Ok(())
}

/// Raw IKL query: ingest, LINT the expression against the corpus vocabulary, then evaluate as bitwise
/// set-algebra. This is the compile-time boundary from the paper — an unknown atom is a caught error with
/// a suggestion, not a silently empty result.
fn run_ikl(dir: &str, expr: &str) -> Result<(), Box<dyn std::error::Error>> {
    let mut engine = hot_engine();
    let overlay = overlay_path_for(dir);
    if let Some(eng) = engine.as_mut() {
        eng.enable_growth(&overlay);
    }
    let mut corpus = Corpus::new_incremental(dir, vec!["file".into(), "record".into()], CorpusKind::Csv);
    corpus.set_gazetteer_overlay(&overlay);
    let files = collect(Path::new(dir));
    eprint!("scanning {} file(s) … ", files.len());
    for f in &files {
        ingest_file(&mut corpus, &mut engine, f);
    }
    let s = corpus.stats();
    eprintln!("{} situations, {} tokens", s.situations, s.vocab);

    // lint first (paper §2): unknown atoms fail closed with a correction
    let report = corpus.linter().lint(expr);
    if let Some(fixed) = &report.repaired {
        eprintln!("linter: repaired syntax → {fixed}");
    }
    if !report.ok {
        for e in &report.errors {
            eprintln!("linter: {}", e.message);
        }
        return Err("query rejected by the linter".into());
    }
    let out = corpus.query(expr, 20);
    println!("{} matches in {:.1}µs", out.count, out.micros);
    for h in out.hits.iter().take(10) {
        let cells: String = h.cells.join(" · ").chars().take(150).collect();
        println!("  [{}] {}", h.sid, cells);
    }
    Ok(())
}

/// STEP 5: train the biaffine relation head (Head C) over the tuned encoder.
#[cfg(feature = "native")]
fn run_train_relations(dir: &str, data: &str, model: &str) -> Result<(), Box<dyn std::error::Error>> {
    use steeldb::relation_train::{save, train_relations};
    use steeldb::tagger_data::TaggerExample;
    use steeldb::tagger_train::TrainConfig;
    use steeldb::vocabulary::VocabularySpace;
    let spec = VocabularySpace::for_corpus(Path::new(dir)).ok_or_else(|| format!("no facet spec for {dir}"))?;
    let examples: Vec<TaggerExample> = std::fs::read_to_string(data)?
        .lines()
        .filter(|l| !l.trim().is_empty())
        .filter_map(|l| serde_json::from_str(l).ok())
        .collect();
    let (dir_t, base, tokenizer) = tuned_tagger_paths().ok_or("no tuned tagger — run step 2 first")?;
    let tagger_dir = if model == "models/tagger-tuned" { dir_t } else { PathBuf::from(model) };
    let epochs: usize = std::env::var("STEELDB_REL_EPOCHS").ok().and_then(|v| v.parse().ok()).unwrap_or(40);
    let cfg = TrainConfig { base_dir: base, tokenizer, lr: 1e-3, max_len: 128, ..Default::default() };
    eprintln!("step 5 · Head C on {} examples · encoder {} · {epochs} epochs", examples.len(), tagger_dir.display());
    let (varmap, report) = train_relations(&spec, &examples, &cfg, &tagger_dir, epochs)?;
    save(&varmap, &spec, &tagger_dir)?;
    println!("head C → {}", tagger_dir.display());
    println!("{}", serde_json::to_string_pretty(&report)?);
    Ok(())
}

/// STEPS 3-4: ontology growth under the MECE gate, then registration.
fn run_grow(dir: &str, rounds: usize, gain: f64) -> Result<(), Box<dyn std::error::Error>> {
    use steeldb::vocabulary::{sample_docs, spec_path, VocabularySpace};
    let path = Path::new(dir);
    let mut spec = VocabularySpace::for_corpus(path).ok_or_else(|| format!("no facet spec for {dir} — run `--init --llm` first (step 0)"))?;
    let docs = sample_docs(path, 96, 4000);
    if docs.is_empty() {
        return Err(format!("no sampleable documents under {dir}").into());
    }
    eprintln!("steps 3-4 · growing from {} facets · {} docs · gain ≥ {gain}", spec.taggable_facets().len(), docs.len());

    let rt = tokio::runtime::Builder::new_current_thread().enable_all().build()?;
    let (grown, log) = rt.block_on(async {
        let provider = provider_config().build().await?;
        Ok::<_, String>(steeldb::grow::grow(provider.as_ref(), &spec, &docs, rounds, gain).await)
    })?;
    spec = grown;

    println!("growth decisions:");
    for e in &log {
        println!("  [{}] {:<18} parent={:<10} coverage={:.3} maxcos={:.3} gain={:.3} → {}",
            e.round, e.name, e.parent.clone().unwrap_or_else(|| "-".into()), e.coverage, e.maxcos, e.gain,
            if e.kept { "KEPT".to_string() } else { format!("rejected ({})", e.reason) });
    }
    spec.save(&spec_path(path))?;

    // register the version with its MECE numbers + the decision log
    let metrics = serde_json::json!({ "growth": log, "gain_threshold": gain, "facets": spec.entity_facets.len() });
    let root = PathBuf::from(dir).join(".steeldb-registry");
    let v = steeldb::registry::register(&root, &spec, &metrics, Some("grow"))?;
    println!("\nregistered {} → {}", v.id, root.display());
    println!("  facets {} ({} hierarchical) · relations {}", v.entity_facets, v.hierarchical_facets, v.relation_facets);
    Ok(())
}

/// STEP 1c: mine the gazetteer from labelled spans and merge it into the corpus spec.
fn run_mine_gazetteer(dir: &str, data: &str) -> Result<(), Box<dyn std::error::Error>> {
    use steeldb::tagger_data::{mine_gazetteer, TaggerExample};
    use steeldb::vocabulary::{spec_path, VocabularySpace};
    let path = Path::new(dir);
    let mut spec = VocabularySpace::for_corpus(path).ok_or_else(|| format!("no facet spec for {dir} — run step 0 first"))?;
    let examples: Vec<TaggerExample> = std::fs::read_to_string(data)?
        .lines()
        .filter(|l| !l.trim().is_empty())
        .filter_map(|l| serde_json::from_str(l).ok())
        .collect();
    let min_count: usize = std::env::var("STEELDB_GAZ_MIN").ok().and_then(|v| v.parse().ok()).unwrap_or(2);
    let mined = mine_gazetteer(&spec, &examples, min_count);
    let before = spec.gazetteer.len();
    for e in mined {
        if !spec.gazetteer.iter().any(|g| g.surface.eq_ignore_ascii_case(&e.surface)) {
            spec.gazetteer.push(e);
        }
    }
    spec.qualify_gazetteer();
    spec.validate()?;
    spec.save(&spec_path(path))?;
    println!("gazetteer: {} → {} entries (min_count {min_count}, from {} examples)", before, spec.gazetteer.len(), examples.len());
    for g in spec.gazetteer.iter().rev().take(12) {
        println!("  {:<40} → {}", g.surface, g.token);
    }
    Ok(())
}

/// Generate engine-verified (question → IKL s-expression) pairs for finetuning the query model.
fn run_gen_ikl(dir: &str, out: &str) -> Result<(), Box<dyn std::error::Error>> {
    use steeldb::ikl_trajectories::{coverage, generate, to_needle_jsonl};
    use steeldb::vocabulary::VocabularySpace;
    let path = Path::new(dir);
    let spec = VocabularySpace::for_corpus(path).ok_or_else(|| format!("no facet spec for {dir} — run step 0 first"))?;
    let mut engine = hot_engine();
    let overlay = overlay_path_for(dir);
    if let Some(eng) = engine.as_mut() {
        eng.enable_growth(&overlay);
    }
    let mut corpus = Corpus::new_incremental(dir, vec!["file".into(), "record".into()], CorpusKind::Csv);
    corpus.set_gazetteer_overlay(&overlay);
    // A sample is enough: the generator needs a realistic VOCABULARY and real row counts, not every file.
    let limit: usize = std::env::var("STEELDB_IKL_FILES").ok().and_then(|v| v.parse().ok()).unwrap_or(60);
    let files = collect(path);
    let step = (files.len() / limit.max(1)).max(1);
    let sampled: Vec<_> = files.iter().step_by(step).take(limit).collect();
    eprint!("ingesting {} of {} file(s) … ", sampled.len(), files.len());
    for f in sampled {
        ingest_file(&mut corpus, &mut engine, f);
    }
    let st = corpus.stats();
    eprintln!("{} situations, {} tokens, {} numeric fields", st.situations, st.vocab, st.numeric_fields.len());

    let per: usize = std::env::var("STEELDB_IKL_PER").ok().and_then(|v| v.parse().ok()).unwrap_or(4);
    let trajs = generate(&corpus, &spec, per);
    std::fs::write(out, to_needle_jsonl(&spec, &trajs))?;
    println!("wrote {} verified IKL examples → {out}", trajs.len());
    println!("coverage: {}", serde_json::to_string(&coverage(&trajs))?);
    Ok(())
}

/// Pure-retrieval scan: ingest, resolve the query to precise corpus tokens, print matching passages.
/// No LLM — this is the retrieval layer standalone (grep-in-natural-language over any doc type).
fn run_search(dir: &str, query: &str) -> Result<(), Box<dyn std::error::Error>> {
    let mut engine = hot_engine();
    let overlay = overlay_path_for(dir);
    if let Some(eng) = engine.as_mut() {
        eng.enable_growth(&overlay);
    }
    let mut corpus = Corpus::new_incremental(dir, vec!["file".into(), "record".into()], CorpusKind::Csv);
    corpus.set_gazetteer_overlay(&overlay);
    let files = collect(Path::new(dir));
    eprint!("scanning {} file(s) … ", files.len());
    for f in &files {
        ingest_file(&mut corpus, &mut engine, f);
    }
    if let Some(eng) = engine.as_ref() {
        eng.save_overlay();
    }
    let s = corpus.stats();
    eprintln!("{} situations, {} tokens", s.situations, s.vocab);
    let linked = corpus.entity_link(query);
    if linked.is_empty() {
        println!("no matches for \"{query}\"");
        return Ok(());
    }
    let t = std::time::Instant::now();
    let ranked = corpus.search_ranked(&linked, 10);
    eprintln!("matched tokens: {} · {} hits ({:.0}µs, ranked by relevance)\n", linked.join(", "), ranked.len(), t.elapsed().as_secs_f64() * 1e6);
    for (i, (_sid, cov, cells)) in ranked.iter().enumerate() {
        let file = cells.first().map(|s| s.as_str()).unwrap_or("");
        let text: String = cells.get(1).cloned().unwrap_or_default().chars().take(260).collect();
        println!("{:>2}. [{cov}★ {file}] {text}", i + 1);
    }
    Ok(())
}

fn dim(s: &str) -> Line<'static> {
    Line::from(Span::styled(s.to_string(), Style::default().fg(Color::DarkGray)))
}
fn errln(s: &str) -> Line<'static> {
    Line::from(vec![Span::styled("! ".to_string(), Style::default().fg(Color::Red)), Span::raw(s.to_string())])
}
fn answer_lines(text: &str) -> Vec<Line<'static>> {
    text.split('\n')
        .enumerate()
        .map(|(i, seg)| Line::from(vec![Span::styled(if i == 0 { "‹ " } else { "  " }.to_string(), Style::default().fg(Color::Green)), Span::raw(seg.to_string())]))
        .collect()
}

struct App {
    mode: Mode,
    // shared
    model: String,
    sits: u32,
    toks: usize,
    busy: bool,
    status: String,
    tick: usize,
    // ask
    lines: Vec<Line<'static>>,
    input: String,
    scroll: u16,
    stick: bool,
    // schema
    schema: Vec<(String, usize, Vec<(String, usize)>)>,
    sel: usize,
    // ontology
    onto: Option<OtDiscover>,
    onto_step: usize,
    clusters: Vec<Cluster>,
    cost: f32,
    cost_hist: Vec<f32>,
    onto_requested: bool,
}

impl App {
    fn new(model: String) -> App {
        App {
            mode: Mode::Ask,
            model,
            sits: 0,
            toks: 0,
            busy: false,
            status: String::new(),
            tick: 0,
            lines: Vec::new(),
            input: String::new(),
            scroll: 0,
            stick: true,
            schema: Vec::new(),
            sel: 0,
            onto: None,
            onto_step: 0,
            clusters: Vec::new(),
            cost: 0.0,
            cost_hist: Vec::new(),
            onto_requested: false,
        }
    }
    fn push(&mut self, l: Line<'static>) {
        self.lines.push(l);
        self.stick = true;
    }
    fn onto_advance(&mut self) {
        if let Some(d) = &self.onto {
            let iters = OT_SCHEDULE[self.onto_step.min(OT_SCHEDULE.len() - 1)];
            let (assign, cost) = d.assign(iters);
            self.clusters = d.clusters(&assign, 8);
            self.cost = cost;
            self.cost_hist.push(cost);
            if self.onto_step < OT_SCHEDULE.len() - 1 {
                self.onto_step += 1;
            }
        }
    }
}

fn run_ui<B: Backend>(terminal: &mut Terminal<B>, app: &mut App, job_tx: &mpsc::Sender<Job>, rep_rx: &mpsc::Receiver<Reply>) -> std::io::Result<()> {
    loop {
        while let Ok(rep) = rep_rx.try_recv() {
            match rep {
                Reply::Status(s) => app.status = s,
                Reply::Line(l) => app.push(l),
                Reply::Stats { sits, toks } => {
                    app.sits = sits;
                    app.toks = toks;
                }
                Reply::Schema(s) => {
                    app.schema = s;
                    app.sel = app.sel.min(app.schema.len().saturating_sub(1));
                }
                Reply::Ontology(d) => {
                    app.onto = Some(*d);
                    app.onto_step = 0;
                    app.cost_hist.clear();
                    app.onto_advance();
                }
                Reply::Done => {
                    app.busy = false;
                    app.status.clear();
                }
            }
        }

        terminal.draw(|f| draw(f, app))?;

        // animate ontology
        if app.mode == Mode::Ontology && app.onto.is_some() && app.onto_step < OT_SCHEDULE.len() - 1 && app.tick % 2 == 0 {
            app.onto_advance();
        }

        if event::poll(Duration::from_millis(120))? {
            if let Event::Key(k) = event::read()? {
                if k.modifiers.contains(KeyModifiers::CONTROL) && matches!(k.code, KeyCode::Char('c')) {
                    return Ok(());
                }
                match k.code {
                    KeyCode::Esc => return Ok(()),
                    KeyCode::Tab => {
                        app.mode = app.mode.next();
                        on_enter_mode(app, job_tx);
                    }
                    KeyCode::BackTab => {
                        app.mode = app.mode.prev();
                        on_enter_mode(app, job_tx);
                    }
                    _ => handle_mode_key(app, job_tx, k.code),
                }
            }
        }
        app.tick = app.tick.wrapping_add(1);
    }
}

fn on_enter_mode(app: &mut App, job_tx: &mpsc::Sender<Job>) {
    match app.mode {
        Mode::Schema => {
            let _ = job_tx.send(Job::Schema);
        }
        Mode::Ontology => {
            if !app.onto_requested {
                app.onto_requested = true;
                app.busy = true;
                app.status = "embedding vocabulary…".into();
                let _ = job_tx.send(Job::Ontology);
            }
        }
        Mode::Ask => {}
    }
}

fn handle_mode_key(app: &mut App, job_tx: &mpsc::Sender<Job>, code: KeyCode) {
    match app.mode {
        Mode::Ask => match code {
            KeyCode::Enter => {
                let line = app.input.trim().to_string();
                app.input.clear();
                if line.is_empty() || app.busy {
                } else if let Some(rest) = line.strip_prefix("/add ").or_else(|| line.strip_prefix(":add ")) {
                    app.push(Line::from(Span::styled(format!("/add {}", rest.trim()), Style::default().fg(Color::Yellow))));
                    app.busy = true;
                    app.status = "ingesting…".into();
                    let _ = job_tx.send(Job::Add(rest.trim().to_string()));
                } else {
                    app.push(Line::from(vec![Span::styled("› ".to_string(), Style::default().fg(Color::Cyan).add_modifier(Modifier::BOLD)), Span::raw(line.clone())]));
                    app.busy = true;
                    app.status = "thinking…".into();
                    let _ = job_tx.send(Job::Ask(line));
                }
            }
            KeyCode::Char(c) => app.input.push(c),
            KeyCode::Backspace => {
                app.input.pop();
            }
            KeyCode::PageUp => {
                app.stick = false;
                app.scroll = app.scroll.saturating_sub(8);
            }
            KeyCode::PageDown => app.scroll = app.scroll.saturating_add(8),
            _ => {}
        },
        Mode::Schema => match code {
            KeyCode::Up => app.sel = app.sel.saturating_sub(1),
            KeyCode::Down => app.sel = (app.sel + 1).min(app.schema.len().saturating_sub(1)),
            _ => {}
        },
        Mode::Ontology => {
            if let KeyCode::Char('r') = code {
                app.onto_step = 0;
                app.cost_hist.clear();
                app.onto_advance();
            }
        }
    }
}

fn draw(f: &mut Frame, app: &mut App) {
    let chunks = Layout::default()
        .direction(Direction::Vertical)
        .constraints([Constraint::Length(1), Constraint::Min(1), Constraint::Length(3)])
        .split(f.area());

    // tab bar + stats
    let mut spans = vec![Span::raw(" ")];
    for m in [Mode::Ask, Mode::Schema, Mode::Ontology] {
        let style = if m == app.mode {
            Style::default().fg(Color::Black).bg(Color::Cyan).add_modifier(Modifier::BOLD)
        } else {
            Style::default().fg(Color::Cyan)
        };
        spans.push(Span::styled(format!(" {} ", m.title()), style));
        spans.push(Span::raw(" "));
    }
    spans.push(Span::styled(format!("· {} situations, {} tokens · {} ●hot", app.sits, app.toks, app.model), Style::default().fg(Color::DarkGray)));
    f.render_widget(Paragraph::new(Line::from(spans)), chunks[0]);

    match app.mode {
        Mode::Ask => draw_ask(f, app, chunks[1]),
        Mode::Schema => draw_schema(f, app, chunks[1]),
        Mode::Ontology => draw_ontology(f, app, chunks[1]),
    }

    // footer: input (Ask) or keybindings
    let foot = match app.mode {
        Mode::Ask if !app.busy => Line::from(vec![Span::styled(" › ", Style::default().fg(Color::Cyan)), Span::raw(app.input.clone()), Span::styled("▏", Style::default().fg(Color::Cyan))]),
        _ if app.busy => Line::from(vec![Span::styled(format!(" {} ", SPINNER[app.tick % SPINNER.len()]), Style::default().fg(Color::Yellow)), Span::styled(app.status.clone(), Style::default().fg(Color::DarkGray))]),
        Mode::Schema => dim(" ↑/↓ select facet · Tab switch view · Ctrl-C quit"),
        Mode::Ontology => dim(" r replay · Tab switch view · Ctrl-C quit"),
        Mode::Ask => dim(" thinking…"),
    };
    let title = match app.mode {
        Mode::Ask => " ask (Enter · /add <path>) ",
        _ => " keys ",
    };
    f.render_widget(Paragraph::new(foot).block(Block::default().borders(Borders::ALL).title(title)), chunks[2]);
}

fn draw_ask(f: &mut Frame, app: &mut App, area: Rect) {
    let view_h = area.height.saturating_sub(2);
    let follow = (app.lines.len() as u16).saturating_sub(view_h);
    let scroll = if app.stick { follow } else { app.scroll.min(follow) };
    app.scroll = scroll;
    if scroll >= follow {
        app.stick = true;
    }
    f.render_widget(
        Paragraph::new(app.lines.clone()).block(Block::default().borders(Borders::ALL).title(" conversation ")).wrap(Wrap { trim: false }).scroll((scroll, 0)),
        area,
    );
}

fn draw_schema(f: &mut Frame, app: &App, area: Rect) {
    let cols = Layout::default().direction(Direction::Horizontal).constraints([Constraint::Length(24), Constraint::Min(1)]).split(area);
    // facet list
    let items: Vec<Line> = app
        .schema
        .iter()
        .enumerate()
        .map(|(i, (f, n, _))| {
            let sel = i == app.sel;
            let style = if sel { Style::default().fg(Color::Black).bg(Color::Cyan) } else { Style::default().fg(Color::Cyan) };
            Line::from(vec![Span::styled(format!(" {f} "), style), Span::styled(format!("({n})"), Style::default().fg(Color::DarkGray))])
        })
        .collect();
    f.render_widget(Paragraph::new(items).block(Block::default().borders(Borders::ALL).title(" facets ")), cols[0]);
    // tokens of the selected facet
    let toks: Vec<Line> = app
        .schema
        .get(app.sel)
        .map(|(_, _, ts)| {
            let max = ts.first().map(|(_, n)| *n).unwrap_or(1).max(1);
            ts.iter()
                .map(|(t, n)| {
                    let w = (n * 16 / max).max(1);
                    Line::from(vec![
                        Span::styled("█".repeat(w), Style::default().fg(Color::Green)),
                        Span::raw(" "),
                        Span::styled(format!("{:>5} ", n), Style::default().fg(Color::DarkGray)),
                        Span::raw(t.clone()),
                    ])
                })
                .collect()
        })
        .unwrap_or_default();
    let title = app.schema.get(app.sel).map(|(f, _, _)| format!(" {f} — top tokens ")).unwrap_or_else(|| " tokens ".into());
    f.render_widget(Paragraph::new(toks).block(Block::default().borders(Borders::ALL).title(title)), cols[1]);
}

fn spark(hist: &[f32]) -> String {
    if hist.is_empty() {
        return String::new();
    }
    let bars = ['▁', '▂', '▃', '▄', '▅', '▆', '▇', '█'];
    let (lo, hi) = hist.iter().fold((f32::INFINITY, f32::NEG_INFINITY), |(l, h), &v| (l.min(v), h.max(v)));
    let rng = (hi - lo).max(1e-9);
    hist.iter().map(|&v| bars[(((v - lo) / rng) * 7.0).round() as usize]).collect()
}

fn draw_ontology(f: &mut Frame, app: &App, area: Rect) {
    let rows = Layout::default().direction(Direction::Vertical).constraints([Constraint::Min(1), Constraint::Length(3)]).split(area);
    if app.onto.is_none() {
        f.render_widget(Paragraph::new(dim("embedding the corpus vocabulary…")).block(Block::default().borders(Borders::ALL).title(" discovered facets ")), rows[0]);
        return;
    }
    let maxsize = app.clusters.iter().map(|c| c.size).max().unwrap_or(1).max(1);
    let lines: Vec<Line> = app
        .clusters
        .iter()
        .enumerate()
        .map(|(i, c)| {
            let color = PALETTE[i % PALETTE.len()];
            let filled = (c.size * 16 / maxsize).max(1);
            Line::from(vec![
                Span::styled(format!(" {}", "█".repeat(filled)), Style::default().fg(color)),
                Span::styled("·".repeat(16 - filled), Style::default().fg(Color::DarkGray)),
                Span::styled(format!(" {:>3} ", c.size), Style::default().fg(Color::DarkGray)),
                Span::styled(format!("{:<16}", c.label), Style::default().fg(color).add_modifier(Modifier::BOLD)),
                Span::styled(c.terms.iter().skip(1).cloned().collect::<Vec<_>>().join(" "), Style::default().fg(Color::Gray)),
            ])
        })
        .collect();
    let iters = OT_SCHEDULE[app.onto_step.min(OT_SCHEDULE.len() - 1)];
    let ttl = format!(" discovered facets · iter {iters} · cost {:.4} ", app.cost);
    f.render_widget(Paragraph::new(lines).block(Block::default().borders(Borders::ALL).title(ttl)), rows[0]);
    let foot = Line::from(vec![Span::styled(" transport cost ", Style::default().fg(Color::DarkGray)), Span::styled(spark(&app.cost_hist), Style::default().fg(Color::Green))]);
    f.render_widget(Paragraph::new(foot).block(Block::default().borders(Borders::ALL).title(" convergence ")), rows[1]);
}