steeldb/

api.rs

//! **The public API.** One type to learn, and a return type that makes refusal impossible to ignore.
//!
//! ```no_run
//! use steeldb::SteelDb;
//!
//! let db = SteelDb::ingest(["Morty Shade defeated Wallace Gale at Ecruteak City in 2025."])?;
//!
//! match db.query("(and defeated/* (not state/negated))") {
//!     Ok(answer)  => println!("{} situations", answer.len()),
//!     Err(refused) => println!("{refused}"),   // names what the data *does* contain
//! }
//! # Ok::<(), steeldb::api::Error>(())
//! ```
//!
//! ## Why `query` returns a `Result`
//!
//! Most engines answer everything. Ask for something absent and you get an empty list, which is
//! indistinguishable from "there are genuinely none" — and for an automated caller, indistinguishable from a
//! correct answer. The whole argument of this engine is that it can say **no**, so the API makes that outcome a
//! separate branch you have to look at rather than a value you can skim past.
//!
//! [`Refused`] carries what *does* exist, so a caller — human or agent — can repair the question instead of
//! guessing again.

use crate::bitmap::{Postings, RoarPostings};
use crate::db::Corpus;
use crate::projector::CorpusKind;
use std::collections::BTreeMap;
use std::path::Path;

type P = RoarPostings;

// ── results ───────────────────────────────────────────────────────────────────────────────────────────

/// A complete set of matching situations.
///
/// Complete, not ranked: every situation satisfying the query is here, so counting is meaningful. Iterate it
/// directly, or read [`Answer::ids`].
#[derive(Debug, Clone, Default)]
pub struct Answer {
    ids: Vec<u32>,
    micros: f64,
}

impl Answer {
    /// How many situations matched.
    pub fn len(&self) -> usize {
        self.ids.len()
    }
    pub fn is_empty(&self) -> bool {
        self.ids.is_empty()
    }
    /// The matching situation ids, ascending.
    pub fn ids(&self) -> &[u32] {
        &self.ids
    }
    /// Wall-clock microseconds for the bitmap program. Zero on targets without a monotonic clock (wasm).
    pub fn micros(&self) -> f64 {
        self.micros
    }
}

impl IntoIterator for Answer {
    type Item = u32;
    type IntoIter = std::vec::IntoIter<u32>;
    fn into_iter(self) -> Self::IntoIter {
        self.ids.into_iter()
    }
}

impl<'a> IntoIterator for &'a Answer {
    type Item = &'a u32;
    type IntoIter = std::slice::Iter<'a, u32>;
    fn into_iter(self) -> Self::IntoIter {
        self.ids.iter()
    }
}

/// A query the data cannot answer, and what it can answer instead.
///
/// This is the type that carries the engine's central promise. It is an error rather than an empty result
/// because those are different facts, and conflating them is how a confident wrong answer gets produced.
#[derive(Debug, Clone)]
pub struct Refused {
    /// the query as given
    pub query: String,
    /// one line per problem, already phrased for a reader
    pub problems: Vec<String>,
    /// tags or categories that do exist and are close to what was asked
    pub alternatives: Vec<String>,
}

/// Break a long line at comma boundaries so a refusal stays readable in a terminal and in a README.
///
/// A refusal that lists a corpus's dimensions runs well past any sensible width, and the one thing it must not
/// be is hard to read — it is the message a user gets at precisely the moment they are confused.
fn wrap_indented(text: &str, width: usize, indent: &str) -> String {
    let mut out = String::new();
    let mut line = String::new();
    for (i, piece) in text.split(", ").enumerate() {
        let sep = if i == 0 { "" } else { ", " };
        if !line.is_empty() && line.chars().count() + sep.len() + piece.chars().count() > width {
            // the comma stays at the end of the line it broke, or the list reads as if an item were dropped
            out.push_str(&line);
            out.push(',');
            out.push('\n');
            out.push_str(indent);
            line = piece.to_string();
        } else {
            line.push_str(sep);
            line.push_str(piece);
        }
    }
    out.push_str(&line);
    out
}

impl std::fmt::Display for Refused {
    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
        write!(f, "refused: {}", self.query)?;
        for p in &self.problems {
            write!(f, "\n  {}", wrap_indented(p, 70, "    "))?;
        }
        if !self.alternatives.is_empty() {
            write!(f, "\n  available: {}", wrap_indented(&self.alternatives.join(", "), 59, "             "))?;
        }
        Ok(())
    }
}

impl std::error::Error for Refused {}

/// The evidential bound on a claim: `[belief, plausibility]`.
///
/// Two numbers rather than one, because a single probability cannot separate a contested claim from an
/// unexamined one — both come out near the middle. See the module docs of [`crate::evidence`].
#[derive(Debug, Clone, Copy, PartialEq)]
pub struct Interval {
    /// evidence positively supporting the claim — the floor
    pub belief: f64,
    /// evidence not ruling it out — the ceiling
    pub plausibility: f64,
}

impl Interval {
    /// The width of the interval: how much the corpus simply does not say.
    pub fn ignorance(&self) -> f64 {
        (self.plausibility - self.belief).max(0.0)
    }
    /// Established: supported and unrefuted.
    pub fn is_certain(&self) -> bool {
        self.belief >= 1.0 - f64::EPSILON
    }
    /// Refuted: nothing supports it and something rules it out.
    pub fn is_refuted(&self) -> bool {
        self.plausibility <= f64::EPSILON
    }
    /// Nobody said: no support, nothing against.
    pub fn is_unknown(&self) -> bool {
        self.belief <= f64::EPSILON && self.plausibility >= 1.0 - f64::EPSILON
    }
}

impl std::fmt::Display for Interval {
    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
        write!(f, "[{:.2}, {:.2}]", self.belief, self.plausibility)
    }
}

/// One discovered category and the words it claims.
#[derive(Debug, Clone, Copy)]
pub struct Category<'a> {
    /// the category name, which is also its query stem: `name/*`
    pub name: &'a str,
    /// the words in the text that put a document in this category
    pub words: &'a [String],
}

impl Category<'_> {
    /// The wildcard that matches every value in this category.
    pub fn wildcard(&self) -> String {
        format!("{}/*", self.name)
    }
}

/// Setup failures, kept separate from query refusals.
#[derive(Debug)]
pub enum Error {
    /// nothing usable was found to index
    Empty(String),
    /// an artefact set could not be read or written
    Artifact(crate::artifact::ArtifactError),
    Io(std::io::Error),
}

impl std::fmt::Display for Error {
    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
        match self {
            Error::Empty(what) => write!(f, "nothing to index: {what}"),
            Error::Artifact(e) => write!(f, "{e}"),
            Error::Io(e) => write!(f, "{e}"),
        }
    }
}

impl std::error::Error for Error {}

impl From<std::io::Error> for Error {
    fn from(e: std::io::Error) -> Self {
        Error::Io(e)
    }
}

// ── the handle ────────────────────────────────────────────────────────────────────────────────────────

/// How to build the vocabulary. The defaults are tuned for prose and rarely need changing.
#[derive(Debug, Clone)]
pub struct Options {
    /// how much candidate vocabulary to consider
    pub terms: usize,
    /// how many categories to look for
    pub categories: usize,
    /// a candidate must clear `coverage × (1 − overlap)` to be kept
    pub min_gain: f64,
}

impl Default for Options {
    fn default() -> Self {
        Options { terms: 90, categories: 6, min_gain: 0.05 }
    }
}

/// An indexed corpus you can ask questions of.
///
/// Read-only after construction and `Send + Sync`, so one instance can serve many threads.
pub struct SteelDb {
    corpus: Corpus,
    categories: Vec<(String, Vec<String>)>,
    /// latent motifs: the themes `motif/*` tokens come from (the paper's sixth dimension)
    motifs: Vec<(String, Vec<String>)>,
    documents: Vec<String>,
    /// retained so `learn` gates adopted candidates on the same threshold ingest used
    min_gain: f64,
}

/// Terms considered when looking for latent motifs. Fewer than for categories on purpose: a motif is a broad
/// theme, and the long tail of rare terms adds noise rather than themes.
const MOTIF_TERMS: usize = 40;
/// How many motifs to look for. The transport solver clamps this down on a small corpus.
const MOTIF_GROUPS: usize = 3;

impl SteelDb {
    /// **Ingest.** Discover a vocabulary from documents and index them.
    ///
    /// Offline and deterministic: no credentials, no network, no model files. The same documents always give
    /// the same vocabulary.
    pub fn ingest<I, S>(docs: I) -> Result<Self, Error>
    where
        I: IntoIterator<Item = S>,
        S: AsRef<str>,
    {
        Self::ingest_with(docs, Options::default())
    }

    /// As [`SteelDb::ingest`], with explicit discovery settings.
    pub fn ingest_with<I, S>(docs: I, opts: Options) -> Result<Self, Error>
    where
        I: IntoIterator<Item = S>,
        S: AsRef<str>,
    {
        let documents: Vec<String> =
            docs.into_iter().map(|d| d.as_ref().trim().to_string()).filter(|d| !d.is_empty()).collect();
        if documents.is_empty() {
            return Err(Error::Empty("no non-empty documents".into()));
        }

        // discover categories, keeping only those that earn their place
        let clusters = crate::emergent::discover(&documents, opts.terms, opts.categories);
        let mut spec = crate::vocabulary::VocabularySpace {
            version: 1,
            corpus: "documents".into(),
            entity_facets: Vec::new(),
            relation_facets: Vec::new(),
            gazetteer: Vec::new(),
            metrics: None,
        };
        let mut categories: Vec<(String, Vec<String>)> = Vec::new();
        for (round, c) in clusters.iter().enumerate() {
            let cand = crate::grow::Candidate {
                name: c.label.clone(),
                parent: None,
                description: String::new(),
                examples: c.terms.clone(),
                worth_adding: true,
            };
            let scored = crate::grow::score_candidate_full(&spec, &documents, &cand);
            let (score, dup) = match scored {
                Some((s, d)) => (Some(s), d),
                None => (None, None),
            };
            if crate::grow::gate_full(&spec, &cand, score.as_ref(), dup, opts.min_gain, round).kept {
                crate::grow::adopt(&mut spec, &cand);
                categories.push((c.label.clone(), c.terms.clone()));
            }
        }

        // Latent motifs are not gated the way categories are: they are not competing for a retrieval head, they
        // are themes a situation can join through terms that travel together, and nothing is refused on their
        // account. They do have to be LATENT, though. Run over the same geometry, transport often recovers the
        // groups the categories already name, and on a small corpus every motif came back as an exact restatement
        // of a category — `motif/defeated` beside `defeated/*`, carrying the same situations under a second name.
        // A motif that a category already expresses is dropped, so `motif/*` means "structure the categories did
        // not capture" and an empty motif set is the honest answer rather than a padded one.
        let motifs: Vec<(String, Vec<String>)> =
            crate::emergent::discover_motifs(&documents, MOTIF_TERMS, MOTIF_GROUPS)
                .into_iter()
                .filter(|(name, _)| !categories.iter().any(|(cat, _)| cat == name))
                .collect();

        let corpus = Self::project(&documents, &categories, &motifs);
        Ok(SteelDb { corpus, categories, motifs, documents, min_gain: opts.min_gain })
    }

    /// **Ingest, following an existing artefact set.**
    ///
    /// Reads the vocabulary from `artifact_dir` instead of rediscovering it, so the result is reproducible and
    /// no model is involved even if a model produced the vocabulary originally. This is the pairing that makes
    /// `learn` worth running: the expensive, non-deterministic step happens once, and every run afterwards is
    /// offline and identical.
    pub fn ingest_using<I, S>(docs: I, artifact_dir: impl AsRef<Path>) -> Result<Self, Error>
    where
        I: IntoIterator<Item = S>,
        S: AsRef<str>,
    {
        let set = crate::artifact::Artifacts::load(artifact_dir).map_err(Error::Artifact)?;
        let documents: Vec<String> =
            docs.into_iter().map(|d| d.as_ref().trim().to_string()).filter(|d| !d.is_empty()).collect();
        if documents.is_empty() {
            return Err(Error::Empty("no non-empty documents".into()));
        }
        let categories: Vec<(String, Vec<String>)> =
            set.categories.into_iter().map(|c| (c.name, c.words)).collect();
        // motifs come from the artefact rather than being rediscovered, so a reload cannot drift from the run
        // that produced the files
        let motifs: Vec<(String, Vec<String>)> =
            set.motifs.into_iter().map(|m| (m.name, m.words)).collect();
        let corpus = Self::project_with(&documents, &categories, &set.gazetteer, &motifs);
        Ok(SteelDb { corpus, categories, motifs, documents, min_gain: Options::default().min_gain })
    }

    /// Write this database's vocabulary to an artefact directory.
    ///
    /// Only derived vocabulary is written — categories, mention surfaces, relation verbs. No document text, so
    /// the directory is safe to commit alongside code.
    pub fn save(&self, artifact_dir: impl AsRef<Path>) -> Result<(), Error> {
        let categories = self
            .categories
            .iter()
            .map(|(name, words)| crate::artifact::CategoryRecord {
                name: name.clone(),
                words: words.clone(),
            })
            .collect();
        let registrations = Self::registrations(&self.documents);
        let surfaces: Vec<String> = registrations.iter().map(|r| r.surface.clone()).collect();
        let mut relations: Vec<String> = Vec::new();
        for doc in &self.documents {
            for r in crate::emergent::relation_spans(doc, &surfaces) {
                if !relations.contains(&r.verb) {
                    relations.push(r.verb);
                }
            }
        }
        let motifs = self
            .motifs
            .iter()
            .map(|(name, words)| crate::artifact::CategoryRecord {
                name: name.clone(),
                words: words.clone(),
            })
            .collect();
        crate::artifact::Artifacts::new("discovery", categories, registrations, relations)
            .with_motifs(motifs)
            .save(artifact_dir)
            .map_err(Error::Artifact)
    }

    /// As [`SteelDb::save`], and additionally write a **finetuning set** derived from the corpus.
    ///
    /// The set is weak supervision: every span the discovered vocabulary can locate, labelled with the category
    /// that claims it. It is what you would hand to a tagger finetune so the model learns to find these spans in
    /// text it has not seen.
    ///
    /// Unlike the vocabulary files, this one **contains document text** — a span label is meaningless without
    /// the words it points at. It is written to a `training/` subdirectory which
    /// [`crate::artifact::Artifacts::save`] excludes with a `.gitignore`, and the manifest records that the
    /// subdirectory is unsafe to publish.
    pub fn save_with_training(&self, artifact_dir: impl AsRef<Path>) -> Result<usize, Error> {
        let gazetteer = crate::emergent::mine_gazetteer(&self.documents, 2);
        let mut jsonl = String::new();
        for doc in &self.documents {
            let mut spans: Vec<serde_json::Value> = Vec::new();
            let mut push = |s: usize, e: usize, facet: &str| {
                if let Some(surface) = doc.get(s..e) {
                    spans.push(serde_json::json!({
                        "start": s, "end": e, "facet": facet, "surface": surface,
                    }));
                }
            };
            for m in &gazetteer {
                for (s, e) in crate::emergent::word_spans(doc, m) {
                    push(s, e, "entity");
                }
            }
            for (s, e, field) in crate::emergent::quantity_spans(doc) {
                push(s, e, &format!("qty/{field}"));
            }
            for (s, e, tok) in crate::emergent::temporal_spans(doc) {
                let _ = tok;
                push(s, e, "time");
            }
            for (cat, words) in &self.categories {
                for w in words {
                    for (s, e) in crate::emergent::word_spans(doc, w) {
                        push(s, e, cat);
                    }
                }
            }
            // overlapping labels would teach the tagger contradictory boundaries
            spans.sort_by_key(|v| (v["start"].as_u64().unwrap_or(0), v["end"].as_u64().unwrap_or(0)));
            let mut kept: Vec<serde_json::Value> = Vec::new();
            let mut cursor = 0u64;
            for sp in spans {
                let (s, e) = (sp["start"].as_u64().unwrap_or(0), sp["end"].as_u64().unwrap_or(0));
                if s >= cursor {
                    cursor = e;
                    kept.push(sp);
                }
            }
            if kept.is_empty() {
                continue; // a passage with no labels teaches nothing
            }
            let line = serde_json::json!({ "text": doc, "spans": kept });
            jsonl.push_str(&line.to_string());
            jsonl.push('\n');
        }

        let categories = self
            .categories
            .iter()
            .map(|(name, words)| crate::artifact::CategoryRecord {
                name: name.clone(),
                words: words.clone(),
            })
            .collect();
        let mut relations: Vec<String> = Vec::new();
        for doc in &self.documents {
            for r in crate::emergent::relation_spans(doc, &gazetteer) {
                if !relations.contains(&r.verb) {
                    relations.push(r.verb);
                }
            }
        }
        let set = crate::artifact::Artifacts::new(
            "discovery",
            categories,
            gazetteer.iter().map(|s| crate::artifact::Registration {
                surface: s.clone(),
                token: format!("entity/{}", crate::projector::slug(s)),
            }).collect(),
            relations,
        )
        .with_motifs(
            self.motifs
                .iter()
                .map(|(name, words)| crate::artifact::CategoryRecord {
                    name: name.clone(),
                    words: words.clone(),
                })
                .collect(),
        )
        .with_training(jsonl);
        let n = set.training_examples;
        set.save(artifact_dir).map_err(Error::Artifact)?;
        Ok(n)
    }

    /// Index a directory of documents, discovering the vocabulary from what it finds.
    pub fn open(dir: impl AsRef<Path>) -> Result<Self, Error> {
        let dir = dir.as_ref();
        let mut docs: Vec<String> = Vec::new();
        for entry in std::fs::read_dir(dir)? {
            let path = entry?.path();
            if path.extension().map(|e| e == "md" || e == "txt").unwrap_or(false) {
                if let Ok(text) = std::fs::read_to_string(&path) {
                    docs.push(text);
                }
            }
        }
        if docs.is_empty() {
            return Err(Error::Empty(format!("no .md or .txt files under {}", dir.display())));
        }
        Self::ingest(docs)
    }

    /// Project documents into tagged situations. One situation per document, carrying every dimension it
    /// supports: entities, relation roles with polarity, temporal buckets, quantities, epistemic state.
    fn project(
        docs: &[String],
        categories: &[(String, Vec<String>)],
        motifs: &[(String, Vec<String>)],
    ) -> Corpus {
        Self::project_with(docs, categories, &Self::registrations(docs), motifs)
    }

    /// Mine mentions and register each under its canonical token, so discovery and an artefact reload agree.
    fn registrations(docs: &[String]) -> Vec<crate::artifact::Registration> {
        crate::emergent::mine_gazetteer(docs, 2)
            .into_iter()
            .map(|surface| {
                let token = format!("entity/{}", crate::projector::slug(&surface));
                crate::artifact::Registration { surface, token }
            })
            .collect()
    }

    /// As [`Self::project`], with the mention list supplied rather than mined — so an artefact set fully
    /// determines the projection and a later run cannot drift from the recorded vocabulary.
    fn project_with(
        docs: &[String],
        categories: &[(String, Vec<String>)],
        registrations: &[crate::artifact::Registration],
        motifs: &[(String, Vec<String>)],
    ) -> Corpus {
        let gazetteer: Vec<String> = registrations.iter().map(|r| r.surface.clone()).collect();
        // surface -> canonical token, so a normalised registration is honoured rather than re-derived
        let canonical: std::collections::HashMap<&str, &str> =
            registrations.iter().map(|r| (r.surface.as_str(), r.token.as_str())).collect();
        let mut corpus = Corpus::new_incremental("documents", vec!["document".into()], CorpusKind::Text);

        for doc in docs {
            let mut tags: Vec<String> = Vec::new();
            let mut numbers: Vec<(String, f64)> = Vec::new();
            // Both cue sets, because the two projections had drifted apart here too: the browser copy knew
            // "provisional" and "no longer" and this one did not. Keeping `belief_level` keeps the fourth
            // polarity level — a claim that is both denied and hedged is neither a flat denial nor a hedge.
            let lower = doc.to_lowercase();
            let level = crate::dimensions::belief_level(
                lower.contains("not permitted") || lower.contains("is not ") || lower.contains("no longer"),
                lower.contains("under review") || lower.contains("may be") || lower.contains("provisional"),
            );

            for mention in &gazetteer {
                if crate::emergent::contains_term(doc, mention) {
                    // use the registered token; slugging the surface here is what lost normalisation in v1
                    match canonical.get(mention.as_str()) {
                        Some(tok) => tags.push((*tok).to_string()),
                        None => tags.push(format!("entity/{}", crate::projector::slug(mention))),
                    }
                }
            }
            for r in crate::emergent::relation_spans(doc, &gazetteer) {
                let verb = crate::projector::slug(&r.verb);
                // The bare polarity tags say a relation of this kind happened in this situation. The
                // argument-bound ones say WHO was on each side, which is what makes a reversed relation
                // unmatchable rather than merely unranked: "Morty defeated Wallace" carries
                // `rel/defeated/+/morty-shade`, and the reverse claim has no tag to hide inside.
                tags.push(format!("rel/{verb}/+"));
                tags.push(format!("rel/{verb}/+/{}", crate::projector::slug(&r.actor)));
                tags.push(format!("rel/{verb}/-"));
                tags.push(format!("rel/{verb}/-/{}", crate::projector::slug(&r.target)));
            }
            for (_, _, tok) in crate::emergent::temporal_spans(doc) {
                tags.push(tok);
            }
            for (st, en, field) in crate::emergent::quantity_spans(doc) {
                tags.push(format!("quantity/{field}"));
                let digits: String = doc[st..en]
                    .chars()
                    .enumerate()
                    .take_while(|(i, c)| c.is_ascii_digit() || *c == '.' || (*i == 0 && *c == '-'))
                    .map(|(_, c)| c)
                    .collect();
                if let Ok(v) = digits.parse::<f64>() {
                    numbers.push((field, v));
                }
            }
            for (cat, terms) in categories {
                for t in terms {
                    if crate::emergent::contains_term(doc, t) {
                        tags.push(format!("{cat}/{}", crate::projector::slug(t)));
                    }
                }
            }
            // A situation joins a motif through any member term, so two situations can share a theme without
            // sharing a word — which is the point of the dimension.
            for (name, terms) in motifs {
                if terms.iter().any(|t| crate::emergent::contains_term(doc, t)) {
                    tags.push(format!("motif/{}", crate::projector::slug(name)));
                }
            }
            tags.push(
                match level {
                    l if l < 0.0 => "state/negated",
                    l if l < 1.0 => "state/hedged",
                    _ => "state/asserted",
                }
                .to_string(),
            );

            let beliefs: Vec<(String, f32)> = tags.iter().map(|t| (t.clone(), level)).collect();
            let display = vec![doc.chars().take(160).collect::<String>()];
            numbers.dedup_by(|a, b| a.0 == b.0);
            corpus.add_situation_polar(tags, display, numbers, beliefs);
        }
        corpus
    }

    /// Run a query, or refuse it.
    ///
    /// Every tag is checked against the vocabulary before anything executes, so an unsupported query costs
    /// nothing and comes back with alternatives.
    pub fn query(&self, ikl: &str) -> Result<Answer, Refused> {
        let report = self.corpus.linter().lint(ikl);
        if let Some(fixed) = &report.repaired {
            // The linter can repair unbalanced parentheses, and reports the repair. Answering the repaired
            // expression would mean answering a question the caller did not ask — `(and a b` is not `(and a b)`
            // until someone decides it is. So a query that needed repair is refused, with the repair offered as
            // a suggestion.
            return Err(Refused {
                query: ikl.to_string(),
                problems: vec![if fixed.trim().is_empty() {
                    "unbalanced parentheses: a ')' with no matching '(' leaves nothing to run".into()
                } else {
                    format!("unbalanced parentheses; did you mean: {fixed}")
                }],
                alternatives: self.categories.iter().map(|(c, _)| format!("{c}/*")).collect(),
            });
        }
        if !report.ok {
            return Err(Refused {
                query: ikl.to_string(),
                problems: report.errors.iter().map(|e| e.message.clone()).collect(),
                alternatives: self.categories.iter().map(|(c, _)| format!("{c}/*")).collect(),
            });
        }
        match crate::tokenql::try_evaluate(self.corpus.index(), ikl) {
            Ok(set) => Ok(Answer { ids: set.to_sorted(), micros: 0.0 }),
            Err(e) => Err(Refused {
                query: ikl.to_string(),
                problems: vec![e.to_string()],
                alternatives: self.categories.iter().map(|(c, _)| format!("{c}/*")).collect(),
            }),
        }
    }

    /// Check a query without running it. Cheap, and the same check `query` performs.
    pub fn check(&self, ikl: &str) -> Result<(), Refused> {
        let report = self.corpus.linter().lint(ikl);
        if let Some(fixed) = &report.repaired {
            // The linter can repair unbalanced parentheses, and reports the repair. Answering the repaired
            // expression would mean answering a question the caller did not ask — `(and a b` is not `(and a b)`
            // until someone decides it is. So a query that needed repair is refused, with the repair offered as
            // a suggestion.
            return Err(Refused {
                query: ikl.to_string(),
                problems: vec![if fixed.trim().is_empty() {
                    "unbalanced parentheses: a ')' with no matching '(' leaves nothing to run".into()
                } else {
                    format!("unbalanced parentheses; did you mean: {fixed}")
                }],
                alternatives: self.categories.iter().map(|(c, _)| format!("{c}/*")).collect(),
            });
        }
        if report.ok {
            Ok(())
        } else {
            Err(Refused {
                query: ikl.to_string(),
                problems: report.errors.iter().map(|e| e.message.clone()).collect(),
                alternatives: self.categories.iter().map(|(c, _)| format!("{c}/*")).collect(),
            })
        }
    }

    /// The evidential bound on a tag across the whole corpus.
    pub fn belief(&self, tag: &str) -> Interval {
        let (belief, plausibility) = self.corpus.belief_interval(tag);
        Interval { belief, plausibility }
    }

    /// Situations on a chain from `from` to `to` where each step shares at least `s` tags.
    ///
    /// Raising `s` demands more agreement per step, which is what stops a walk drifting somewhere unrelated.
    pub fn s_path(&self, from: &str, to: &str, s: usize) -> Answer {
        let set = self.corpus.index().s_path_tokens(from, to, s).map(|chain| {
            let mut out = P::empty();
            for tok in &chain {
                out.or_inplace(&crate::tokenql::TokenStore::atom(self.corpus.index(), tok));
            }
            out
        });
        Answer { ids: set.map(|s| s.to_sorted()).unwrap_or_default(), micros: 0.0 }
    }

    /// Both readings of the incidence matrix, swept over the overlap threshold.
    pub fn filtration(&self, max_s: usize) -> Vec<crate::programs::Level> {
        crate::programs::s_filtration(self.corpus.index(), max_s, &Default::default(), 128)
    }

    /// The discovered categories and the words each claims.
    pub fn categories(&self) -> Vec<Category<'_>> {
        self.categories.iter().map(|(name, words)| Category { name, words }).collect()
    }

    /// The wildcard for every discovered category — the set of things you can ask about.
    pub fn askable(&self) -> Vec<String> {
        self.categories.iter().map(|(c, _)| format!("{c}/*")).collect()
    }

    /// Every tag in the index, grouped by its category and sorted within each group.
    ///
    /// Sorted because the engine is otherwise deterministic and a caller should not have to defend against
    /// index iteration order — two runs over the same documents return byte-identical output.
    pub fn tags(&self) -> BTreeMap<String, Vec<String>> {
        let mut out: BTreeMap<String, Vec<String>> = BTreeMap::new();
        for tag in self.corpus.index().tokens() {
            let stem = tag.split('/').next().unwrap_or("").to_string();
            out.entry(stem).or_default().push(tag.clone());
        }
        for v in out.values_mut() {
            v.sort();
            v.dedup();
        }
        out
    }

    /// The document behind a situation id.
    ///
    /// Without this an [`Answer`] is a list of integers. Every result is traceable back to the text that
    /// produced it, which is what makes an answer checkable rather than merely plausible.
    pub fn text(&self, situation: u32) -> Option<&str> {
        self.documents.get(situation as usize).map(|s| s.as_str())
    }

    /// The documents an answer refers to, in id order.
    pub fn resolve<'a>(&'a self, answer: &'a Answer) -> impl Iterator<Item = (u32, &'a str)> + 'a {
        answer.ids().iter().filter_map(move |id| self.text(*id).map(|t| (*id, t)))
    }

    /// How many situations are indexed.
    pub fn len(&self) -> usize {
        self.documents.len()
    }
    pub fn is_empty(&self) -> bool {
        self.documents.is_empty()
    }
    /// The documents as given, for tracing a result back to its source.
    pub fn documents(&self) -> &[String] {
        &self.documents
    }

    // ── internals used by `learn`, which needs to re-run the same gate ──

    /// A vocabulary spec matching the currently accepted categories.
    pub(crate) fn spec_snapshot(&self) -> crate::vocabulary::VocabularySpace {
        crate::vocabulary::VocabularySpace {
            version: 1,
            corpus: "documents".into(),
            entity_facets: self
                .categories
                .iter()
                .map(|(name, words)| crate::vocabulary::EntityFacet {
                    name: name.clone(),
                    parent: None,
                    description: String::new(),
                    examples: words.clone(),
                    structural: false,
                })
                .collect(),
            relation_facets: Vec::new(),
            gazetteer: Vec::new(),
            metrics: None,
        }
    }

    /// Hand over the index. Used by the WebAssembly layer so the browser demo runs the same projection the
    /// library does, rather than a second implementation that has to be kept in step by hand.
    #[cfg(feature = "wasm")]
    pub(crate) fn into_corpus(self) -> Corpus {
        self.corpus
    }

    pub(crate) fn min_gain(&self) -> f64 {
        self.min_gain
    }

    pub(crate) fn push_category(&mut self, name: String, words: Vec<String>) {
        self.categories.push((name, words));
    }

    /// Rebuild the index. Required after adopting a category, because a new category changes what every
    /// document projects to — leaving the old index would answer with a vocabulary the spec no longer matches.
    pub(crate) fn reproject(&mut self) {
        self.corpus = Self::project(&self.documents, &self.categories, &self.motifs);
    }
}

#[cfg(test)]
mod tests {
    use super::*;

    fn corpus() -> Vec<String> {
        [
            "Morty Shade defeated Wallace Gale at Ecruteak City during the Indigo Invitational in 2025.",
            "Bea Strike defeated Iris Draco at Ecruteak City during the Indigo Invitational in 2025.",
            "A habitat survey recorded Aggron near Sootopolis City at an elevation of 1082 m.",
            "A habitat survey recorded Salamence near Sootopolis City at an elevation of 2369 m.",
            "Milotic is not permitted in Series 1 play for the 2025 season.",
            "Metagross is permitted in Series 4 play for the 2026 season.",
        ]
        .iter()
        .map(|s| s.to_string())
        .collect()
    }

    #[test]
    fn three_lines_to_a_working_database() {
        let db = SteelDb::ingest(corpus()).expect("index");
        assert_eq!(db.len(), 6);
        assert!(!db.categories().is_empty(), "should discover at least one category");
    }

    #[test]
    fn an_unsupported_query_is_refused_with_alternatives() {
        let db = SteelDb::ingest(corpus()).unwrap();
        let err = db.query("gene/brca1").expect_err("must refuse a category the data lacks");
        assert!(!err.alternatives.is_empty(), "a refusal must say what does exist");
        let shown = err.to_string();
        assert!(shown.contains("refused"), "{shown}");
        assert!(shown.contains("available"), "{shown}");
    }

    #[test]
    fn a_supported_query_returns_a_complete_set() {
        let db = SteelDb::ingest(corpus()).unwrap();
        let cat = db.categories()[0].name.to_string();
        let answer = db.query(&format!("{cat}/*")).expect("a discovered category must be queryable");
        assert!(!answer.is_empty());
        // complete, not sampled: every id is a real situation
        assert!(answer.ids().iter().all(|id| (*id as usize) < db.len()));
        // and iterable directly
        assert_eq!(answer.ids().len(), (&answer).into_iter().count());
    }

    #[test]
    fn negation_narrows_rather_than_widens() {
        let db = SteelDb::ingest(corpus()).unwrap();
        let cat = db.categories()[0].name.to_string();
        let all = db.query(&format!("{cat}/*")).unwrap().len();
        let some = db.query(&format!("(and {cat}/* (not state/negated))")).unwrap().len();
        assert!(some <= all, "excluding something cannot return more: {some} vs {all}");
    }

    #[test]
    fn belief_separates_asserted_from_negated() {
        let db = SteelDb::ingest(corpus()).unwrap();
        let asserted = db.belief("state/asserted");
        let negated = db.belief("state/negated");
        assert!(asserted.belief > negated.belief, "{asserted} vs {negated}");
        // and the interval exposes its own meaning rather than making the caller compare floats
        assert!(asserted.ignorance() >= 0.0);
        assert!(db.belief("state/nonexistent").is_unknown(), "an absent tag is unknown, not refuted");
    }

    #[test]
    fn check_costs_nothing_and_agrees_with_query() {
        let db = SteelDb::ingest(corpus()).unwrap();
        assert!(db.check("gene/brca1").is_err());
        assert!(db.query("gene/brca1").is_err());
        let cat = db.categories()[0].name.to_string();
        assert!(db.check(&format!("{cat}/*")).is_ok());
    }

    #[test]
    fn the_filtration_thins_as_the_threshold_rises() {
        let db = SteelDb::ingest(corpus()).unwrap();
        let levels = db.filtration(4);
        assert_eq!(levels.len(), 4);
        // more required agreement can only remove edges
        for w in levels.windows(2) {
            assert!(w[1].primal.edges <= w[0].primal.edges, "edges must not grow with s");
            assert!(w[1].dual.edges <= w[0].dual.edges);
        }
    }

    #[test]
    fn empty_input_is_an_error_not_an_empty_database() {
        assert!(matches!(SteelDb::ingest(Vec::<String>::new()), Err(Error::Empty(_))));
        assert!(matches!(SteelDb::ingest(vec!["   ", ""]), Err(Error::Empty(_))));
    }

    #[test]
    fn artefacts_make_a_later_ingest_reproducible() {
        // The pairing that justifies `learn`: the vocabulary is recorded once, and a later run reproduces it
        // exactly without a model.
        let dir = std::env::temp_dir().join(format!("hsdb_api_repro_{}", std::process::id()));
        let _ = std::fs::remove_dir_all(&dir);

        let first = SteelDb::ingest(corpus()).unwrap();
        first.save(&dir).unwrap();
        let cat = first.categories()[0].name.to_string();
        let expected = first.query(&format!("{cat}/*")).unwrap().len();

        let second = SteelDb::ingest_using(corpus(), &dir).unwrap();
        assert_eq!(
            second.categories().iter().map(|c| c.name.to_string()).collect::<Vec<_>>(),
            first.categories().iter().map(|c| c.name.to_string()).collect::<Vec<_>>(),
            "the recorded vocabulary must be reproduced exactly"
        );
        assert_eq!(second.query(&format!("{cat}/*")).unwrap().len(), expected, "and answer identically");
        let _ = std::fs::remove_dir_all(&dir);
    }

    #[test]
    fn saved_artefacts_contain_no_document_text() {
        // Leakage guard at the API level: what `save` writes must be vocabulary, never the corpus.
        let dir = std::env::temp_dir().join(format!("hsdb_api_leak_{}", std::process::id()));
        let _ = std::fs::remove_dir_all(&dir);
        let db = SteelDb::ingest(corpus()).unwrap();
        db.save(&dir).unwrap();

        for entry in std::fs::read_dir(&dir).unwrap() {
            let p = entry.unwrap().path();
            let text = std::fs::read_to_string(&p).unwrap();
            for doc in corpus() {
                assert!(
                    !text.contains(doc.as_str()),
                    "{} contains a whole document",
                    p.display()
                );
                // a distinctive multi-word fragment is enough to prove a copy
                let frag: String = doc.split_whitespace().take(6).collect::<Vec<_>>().join(" ");
                assert!(!text.contains(&frag), "{} contains the fragment {frag:?}", p.display());
            }
        }
        let _ = std::fs::remove_dir_all(&dir);
    }

    #[test]
    fn ingesting_against_a_missing_artefact_set_is_an_error() {
        let missing = std::env::temp_dir().join("hsdb_definitely_absent_dir");
        let _ = std::fs::remove_dir_all(&missing);
        assert!(matches!(
            SteelDb::ingest_using(corpus(), &missing),
            Err(Error::Artifact(_))
        ));
    }

    #[test]
    fn a_finetuning_set_is_written_separately_from_the_vocabulary() {
        let dir = std::env::temp_dir().join(format!("hsdb_api_train_{}", std::process::id()));
        let _ = std::fs::remove_dir_all(&dir);
        let db = SteelDb::ingest(corpus()).unwrap();
        let n = db.save_with_training(&dir).unwrap();
        assert!(n > 0, "the corpus should yield labelled passages");

        // the committable parts still contain no document text, even though a training set exists
        for f in ["manifest.json", "vocabulary.json", "gazetteer.json", "relations.json"] {
            let text = std::fs::read_to_string(dir.join(f)).unwrap();
            for doc in corpus() {
                let frag: String = doc.split_whitespace().take(6).collect::<Vec<_>>().join(" ");
                assert!(!text.contains(&frag), "{f} leaked: {frag:?}");
            }
        }
        // and the training file does carry text, which is the point of keeping it apart
        let train = std::fs::read_to_string(dir.join("training").join("spans.jsonl")).unwrap();
        assert!(train.contains("Morty Shade"), "the finetuning set needs the words");
        assert!(dir.join(".gitignore").exists(), "and must be excluded from commits");
        let _ = std::fs::remove_dir_all(&dir);
    }

    #[test]
    fn training_spans_do_not_overlap() {
        // Overlapping labels would teach a tagger contradictory boundaries for the same characters.
        let dir = std::env::temp_dir().join(format!("hsdb_api_ovl_{}", std::process::id()));
        let _ = std::fs::remove_dir_all(&dir);
        SteelDb::ingest(corpus()).unwrap().save_with_training(&dir).unwrap();
        let train = std::fs::read_to_string(dir.join("training").join("spans.jsonl")).unwrap();
        for line in train.lines().filter(|l| !l.trim().is_empty()) {
            let v: serde_json::Value = serde_json::from_str(line).unwrap();
            let spans = v["spans"].as_array().unwrap();
            let mut last_end = 0u64;
            for sp in spans {
                let s = sp["start"].as_u64().unwrap();
                let e = sp["end"].as_u64().unwrap();
                assert!(s >= last_end, "span {s}..{e} overlaps the previous one ending at {last_end}");
                assert!(e > s, "empty span");
                last_end = e;
            }
        }
        let _ = std::fs::remove_dir_all(&dir);
    }

    #[test]
    fn registered_variants_still_merge_after_an_artefact_reload() {
        // The v1 bug: the artefact recorded only surfaces, so a reload re-derived the token by slugging the raw
        // text and two variants of one entity stopped sharing a token. Registration NORMALISES, and that has to
        // survive the round trip or ingest is not really following the artefacts.
        let dir = std::env::temp_dir().join(format!("hsdb_api_canon_{}", std::process::id()));
        let _ = std::fs::remove_dir_all(&dir);

        let db = SteelDb::ingest(corpus()).unwrap();
        db.save(&dir).unwrap();

        let set = crate::artifact::Artifacts::load(&dir).unwrap();
        assert!(!set.gazetteer.is_empty(), "the corpus should register some mentions");
        for r in &set.gazetteer {
            assert!(!r.token.is_empty(), "every registration needs a canonical token");
            assert!(r.token.contains('/'), "a token is facet-qualified: {}", r.token);
        }

        // reloading must reproduce the same entity tags, not re-derive different ones
        let reloaded = SteelDb::ingest_using(corpus(), &dir).unwrap();
        let tags_before: Vec<String> =
            db.tags().get("entity").cloned().unwrap_or_default();
        let tags_after: Vec<String> =
            reloaded.tags().get("entity").cloned().unwrap_or_default();
        assert_eq!(tags_before, tags_after, "entity tags must survive the round trip unchanged");
        let _ = std::fs::remove_dir_all(&dir);
    }

    #[test]
    fn an_artefact_reload_indexes_identically_to_discovery() {
        // The promise `ingest_using` makes is that the expensive step happens once and every run afterwards is
        // the same. That only holds if the artefact captures everything the projection needs — motifs were
        // missing in schema 2, so a reload silently produced no motif/* tokens and the same documents indexed
        // two different ways depending on which path they came through.
        let docs = corpus();
        let db = SteelDb::ingest(docs.clone()).expect("ingest");

        let dir = std::env::temp_dir().join(format!("steeldb-reload-{}", std::process::id()));
        let _ = std::fs::remove_dir_all(&dir);
        db.save(&dir).expect("save");
        let reloaded = SteelDb::ingest_using(docs, &dir).expect("reload");

        assert_eq!(db.tags(), reloaded.tags(), "an artefact reload must index identically");
        assert_eq!(db.askable(), reloaded.askable());
        let _ = std::fs::remove_dir_all(&dir);
    }

    #[test]
    fn a_relation_records_who_was_on_each_side() {
        // Bare polarity says a defeat happened; it does not say who won. Without the argument-bound tokens the
        // reverse claim matches exactly the same set, which is the one thing a directional relation must not do.
        let db = SteelDb::ingest(corpus()).expect("ingest");
        let rel = db.tags().get("rel").cloned().unwrap_or_default();

        let bound: Vec<&String> = rel.iter().filter(|t| t.matches('/').count() == 3).collect();
        assert!(!bound.is_empty(), "no argument-bound relation tokens: {rel:?}");
        assert!(
            bound.iter().any(|t| t.contains("/+/") ) && bound.iter().any(|t| t.contains("/-/")),
            "both sides must be recorded: {bound:?}"
        );
        // and the mention must survive whole — `shade` instead of `morty-shade` loses the person
        assert!(
            rel.iter().any(|t| t.ends_with("morty-shade")),
            "a name opening a sentence must not be truncated: {rel:?}"
        );
    }

    #[test]
    fn a_refusal_stays_readable_rather_than_running_off_the_line() {
        // This is the message a user sees at the moment they are confused, so width matters. It listed every
        // dimension in the corpus on one line, which came to 122 characters and wrapped wherever the terminal
        // happened to end.
        let db = SteelDb::ingest(corpus()).unwrap();
        let shown = db.query("gene/brca1").unwrap_err().to_string();

        for line in shown.lines() {
            assert!(line.chars().count() <= 78, "line is {} chars: {line:?}", line.chars().count());
        }
        // wrapping must not drop or mangle an item
        assert!(shown.contains("defeated"), "{shown}");
        assert!(!shown.contains(",,") && !shown.contains(" ,"), "mangled list: {shown}");
        // a broken line keeps its comma, so the list does not read as if an entry were missing
        for line in shown.lines() {
            let t = line.trim_end();
            if t.ends_with("defeated") || t.ends_with("elevation") {
                panic!("a wrapped list line lost its comma: {shown}");
            }
        }
    }

    #[test]
    fn wrapping_leaves_a_short_message_untouched() {
        assert_eq!(wrap_indented("a, b", 70, "  "), "a, b");
        assert_eq!(wrap_indented("", 70, "  "), "");
        assert_eq!(wrap_indented("single", 2, "  "), "single", "one oversized item cannot be split");
    }

    #[test]
    fn unbalanced_parentheses_are_refused_rather_than_repaired_or_crashed() {
        // Two faults found by stressing the published crate. `query(")")` PANICKED: the linter repairs
        // parentheses and reported the query clean, then evaluation parsed the raw string and hit an
        // `expect("unbalanced )")`. In the wasm build a panic takes the whole module down.
        //
        // The second fault was quieter and worse. Because the linter repairs, `(and a b` was reported clean and
        // then evaluated as `(and a b)` — answering a question the caller had not asked. A repair is a
        // suggestion, not a licence to rewrite.
        let db = SteelDb::ingest(corpus()).unwrap();

        for q in [")", "(", "(and", "))))", "((((", "(and a b", "(or (not x"] {
            let e = db
                .query(q)
                .err()
                .unwrap_or_else(|| panic!("{q:?} was answered instead of refused"));
            assert!(
                e.problems.iter().any(|p| p.contains("unbalanced")),
                "{q:?} refused for the wrong reason: {:?}",
                e.problems
            );
            // check() must agree, or one path answers what the other rejects
            assert!(db.check(q).is_err(), "check accepted {q:?} while query refused it");
        }

        // a balanced expression is unaffected
        assert!(db.query("state/asserted").is_ok());
        assert!(db.query("(not state/negated)").is_ok());
    }

    #[test]
    fn the_parser_never_panics_on_hostile_input() {
        // `tokenql::parse` is public, so it is reachable with any string at all.
        for q in [")", "((", "()", "\"", "\"unclosed", "(\")\")", "", "(((((((((((((((((((("] {
            let _ = crate::tokenql::parse(q);
        }
    }
}