hypersteeldb 0.1.0

//! **The public API.** One type to learn, and a return type that makes refusal impossible to ignore.
//!
//! ```no_run
//! use steeldb::SteelDb;
//!
//! let db = SteelDb::from_documents(["Morty Shade defeated Wallace Gale at Ecruteak City in 2025."])?;
//!
//! match db.query("(and battle/* (not state/negated))") {
//!     Ok(answer)  => println!("{} situations", answer.len()),
//!     Err(refused) => println!("{refused}"),   // names what the data *does* contain
//! }
//! # Ok::<(), steeldb::api::Error>(())
//! ```
//!
//! ## Why `query` returns a `Result`
//!
//! Most engines answer everything. Ask for something absent and you get an empty list, which is
//! indistinguishable from "there are genuinely none" — and for an automated caller, indistinguishable from a
//! correct answer. The whole argument of this engine is that it can say **no**, so the API makes that outcome a
//! separate branch you have to look at rather than a value you can skim past.
//!
//! [`Refused`] carries what *does* exist, so a caller — human or agent — can repair the question instead of
//! guessing again.

use crate::bitmap::{Postings, RoarPostings};
use crate::db::Corpus;
use crate::projector::CorpusKind;
use std::collections::BTreeMap;
use std::path::Path;

type P = RoarPostings;

// ── results ───────────────────────────────────────────────────────────────────────────────────────────

/// A complete set of matching situations.
///
/// Complete, not ranked: every situation satisfying the query is here, so counting is meaningful. Iterate it
/// directly, or read [`Answer::ids`].
#[derive(Debug, Clone, Default)]
pub struct Answer {
    ids: Vec<u32>,
    micros: f64,
}

impl Answer {
    /// How many situations matched.
    pub fn len(&self) -> usize {
        self.ids.len()
    }
    pub fn is_empty(&self) -> bool {
        self.ids.is_empty()
    }
    /// The matching situation ids, ascending.
    pub fn ids(&self) -> &[u32] {
        &self.ids
    }
    /// Wall-clock microseconds for the bitmap program. Zero on targets without a monotonic clock (wasm).
    pub fn micros(&self) -> f64 {
        self.micros
    }
}

impl IntoIterator for Answer {
    type Item = u32;
    type IntoIter = std::vec::IntoIter<u32>;
    fn into_iter(self) -> Self::IntoIter {
        self.ids.into_iter()
    }
}

impl<'a> IntoIterator for &'a Answer {
    type Item = &'a u32;
    type IntoIter = std::slice::Iter<'a, u32>;
    fn into_iter(self) -> Self::IntoIter {
        self.ids.iter()
    }
}

/// A query the data cannot answer, and what it can answer instead.
///
/// This is the type that carries the engine's central promise. It is an error rather than an empty result
/// because those are different facts, and conflating them is how a confident wrong answer gets produced.
#[derive(Debug, Clone)]
pub struct Refused {
    /// the query as given
    pub query: String,
    /// one line per problem, already phrased for a reader
    pub problems: Vec<String>,
    /// tags or categories that do exist and are close to what was asked
    pub alternatives: Vec<String>,
}

impl std::fmt::Display for Refused {
    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
        write!(f, "refused: {}", self.query)?;
        for p in &self.problems {
            write!(f, "\n  {p}")?;
        }
        if !self.alternatives.is_empty() {
            write!(f, "\n  available: {}", self.alternatives.join(", "))?;
        }
        Ok(())
    }
}

impl std::error::Error for Refused {}

/// The evidential bound on a claim: `[belief, plausibility]`.
///
/// Two numbers rather than one, because a single probability cannot separate a contested claim from an
/// unexamined one — both come out near the middle. See the module docs of [`crate::evidence`].
#[derive(Debug, Clone, Copy, PartialEq)]
pub struct Interval {
    /// evidence positively supporting the claim — the floor
    pub belief: f64,
    /// evidence not ruling it out — the ceiling
    pub plausibility: f64,
}

impl Interval {
    /// The width of the interval: how much the corpus simply does not say.
    pub fn ignorance(&self) -> f64 {
        (self.plausibility - self.belief).max(0.0)
    }
    /// Established: supported and unrefuted.
    pub fn is_certain(&self) -> bool {
        self.belief >= 1.0 - f64::EPSILON
    }
    /// Refuted: nothing supports it and something rules it out.
    pub fn is_refuted(&self) -> bool {
        self.plausibility <= f64::EPSILON
    }
    /// Nobody said: no support, nothing against.
    pub fn is_unknown(&self) -> bool {
        self.belief <= f64::EPSILON && self.plausibility >= 1.0 - f64::EPSILON
    }
}

impl std::fmt::Display for Interval {
    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
        write!(f, "[{:.2}, {:.2}]", self.belief, self.plausibility)
    }
}

/// One discovered category and the words it claims.
#[derive(Debug, Clone, Copy)]
pub struct Category<'a> {
    /// the category name, which is also its query stem: `name/*`
    pub name: &'a str,
    /// the words in the text that put a document in this category
    pub words: &'a [String],
}

impl Category<'_> {
    /// The wildcard that matches every value in this category.
    pub fn wildcard(&self) -> String {
        format!("{}/*", self.name)
    }
}

/// Setup failures, kept separate from query refusals.
#[derive(Debug)]
pub enum Error {
    /// nothing usable was found to index
    Empty(String),
    /// an artefact set could not be read or written
    Artifact(crate::artifact::ArtifactError),
    Io(std::io::Error),
}

impl std::fmt::Display for Error {
    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
        match self {
            Error::Empty(what) => write!(f, "nothing to index: {what}"),
            Error::Artifact(e) => write!(f, "{e}"),
            Error::Io(e) => write!(f, "{e}"),
        }
    }
}

impl std::error::Error for Error {}

impl From<std::io::Error> for Error {
    fn from(e: std::io::Error) -> Self {
        Error::Io(e)
    }
}

// ── the handle ────────────────────────────────────────────────────────────────────────────────────────

/// How to build the vocabulary. The defaults are tuned for prose and rarely need changing.
#[derive(Debug, Clone)]
pub struct Options {
    /// how much candidate vocabulary to consider
    pub terms: usize,
    /// how many categories to look for
    pub categories: usize,
    /// a candidate must clear `coverage × (1 − overlap)` to be kept
    pub min_gain: f64,
}

impl Default for Options {
    fn default() -> Self {
        Options { terms: 90, categories: 6, min_gain: 0.05 }
    }
}

/// An indexed corpus you can ask questions of.
///
/// Read-only after construction and `Send + Sync`, so one instance can serve many threads.
pub struct SteelDb {
    corpus: Corpus,
    categories: Vec<(String, Vec<String>)>,
    documents: Vec<String>,
    /// retained so `learn` gates adopted candidates on the same threshold ingest used
    min_gain: f64,
}

impl SteelDb {
    /// **Ingest.** Discover a vocabulary from documents and index them.
    ///
    /// Offline and deterministic: no credentials, no network, no model files. The same documents always give
    /// the same vocabulary.
    pub fn ingest<I, S>(docs: I) -> Result<Self, Error>
    where
        I: IntoIterator<Item = S>,
        S: AsRef<str>,
    {
        Self::ingest_with(docs, Options::default())
    }

    /// As [`SteelDb::ingest`], with explicit discovery settings.
    pub fn ingest_with<I, S>(docs: I, opts: Options) -> Result<Self, Error>
    where
        I: IntoIterator<Item = S>,
        S: AsRef<str>,
    {
        let documents: Vec<String> =
            docs.into_iter().map(|d| d.as_ref().trim().to_string()).filter(|d| !d.is_empty()).collect();
        if documents.is_empty() {
            return Err(Error::Empty("no non-empty documents".into()));
        }

        // discover categories, keeping only those that earn their place
        let clusters = crate::emergent::discover(&documents, opts.terms, opts.categories);
        let mut spec = crate::vocabulary::VocabularySpace {
            version: 1,
            corpus: "documents".into(),
            entity_facets: Vec::new(),
            relation_facets: Vec::new(),
            gazetteer: Vec::new(),
            metrics: None,
        };
        let mut categories: Vec<(String, Vec<String>)> = Vec::new();
        for (round, c) in clusters.iter().enumerate() {
            let cand = crate::grow::Candidate {
                name: c.label.clone(),
                parent: None,
                description: String::new(),
                examples: c.terms.clone(),
                worth_adding: true,
            };
            let scored = crate::grow::score_candidate_full(&spec, &documents, &cand);
            let (score, dup) = match scored {
                Some((s, d)) => (Some(s), d),
                None => (None, None),
            };
            if crate::grow::gate_full(&spec, &cand, score.as_ref(), dup, opts.min_gain, round).kept {
                crate::grow::adopt(&mut spec, &cand);
                categories.push((c.label.clone(), c.terms.clone()));
            }
        }

        let corpus = Self::project(&documents, &categories);
        Ok(SteelDb { corpus, categories, documents, min_gain: opts.min_gain })
    }

    /// **Ingest, following an existing artefact set.**
    ///
    /// Reads the vocabulary from `artifact_dir` instead of rediscovering it, so the result is reproducible and
    /// no model is involved even if a model produced the vocabulary originally. This is the pairing that makes
    /// `learn` worth running: the expensive, non-deterministic step happens once, and every run afterwards is
    /// offline and identical.
    pub fn ingest_using<I, S>(docs: I, artifact_dir: impl AsRef<Path>) -> Result<Self, Error>
    where
        I: IntoIterator<Item = S>,
        S: AsRef<str>,
    {
        let set = crate::artifact::Artifacts::load(artifact_dir).map_err(Error::Artifact)?;
        let documents: Vec<String> =
            docs.into_iter().map(|d| d.as_ref().trim().to_string()).filter(|d| !d.is_empty()).collect();
        if documents.is_empty() {
            return Err(Error::Empty("no non-empty documents".into()));
        }
        let categories: Vec<(String, Vec<String>)> =
            set.categories.into_iter().map(|c| (c.name, c.words)).collect();
        let corpus = Self::project_with(&documents, &categories, &set.gazetteer);
        Ok(SteelDb { corpus, categories, documents, min_gain: Options::default().min_gain })
    }

    /// Write this database's vocabulary to an artefact directory.
    ///
    /// Only derived vocabulary is written — categories, mention surfaces, relation verbs. No document text, so
    /// the directory is safe to commit alongside code.
    pub fn save(&self, artifact_dir: impl AsRef<Path>) -> Result<(), Error> {
        let categories = self
            .categories
            .iter()
            .map(|(name, words)| crate::artifact::CategoryRecord {
                name: name.clone(),
                words: words.clone(),
            })
            .collect();
        let registrations = Self::registrations(&self.documents);
        let surfaces: Vec<String> = registrations.iter().map(|r| r.surface.clone()).collect();
        let mut relations: Vec<String> = Vec::new();
        for doc in &self.documents {
            for r in crate::emergent::relation_spans(doc, &surfaces) {
                if !relations.contains(&r.verb) {
                    relations.push(r.verb);
                }
            }
        }
        crate::artifact::Artifacts::new("discovery", categories, registrations, relations)
            .save(artifact_dir)
            .map_err(Error::Artifact)
    }

    /// As [`SteelDb::save`], and additionally write a **finetuning set** derived from the corpus.
    ///
    /// The set is weak supervision: every span the discovered vocabulary can locate, labelled with the category
    /// that claims it. It is what you would hand to a tagger finetune so the model learns to find these spans in
    /// text it has not seen.
    ///
    /// Unlike the vocabulary files, this one **contains document text** — a span label is meaningless without
    /// the words it points at. It is written to a `training/` subdirectory which
    /// [`crate::artifact::Artifacts::save`] excludes with a `.gitignore`, and the manifest records that the
    /// subdirectory is unsafe to publish.
    pub fn save_with_training(&self, artifact_dir: impl AsRef<Path>) -> Result<usize, Error> {
        let gazetteer = crate::emergent::mine_gazetteer(&self.documents, 2);
        let mut jsonl = String::new();
        for doc in &self.documents {
            let mut spans: Vec<serde_json::Value> = Vec::new();
            let mut push = |s: usize, e: usize, facet: &str| {
                if let Some(surface) = doc.get(s..e) {
                    spans.push(serde_json::json!({
                        "start": s, "end": e, "facet": facet, "surface": surface,
                    }));
                }
            };
            for m in &gazetteer {
                for (s, e) in crate::emergent::word_spans(doc, m) {
                    push(s, e, "entity");
                }
            }
            for (s, e, field) in crate::emergent::quantity_spans(doc) {
                push(s, e, &format!("qty/{field}"));
            }
            for (s, e, tok) in crate::emergent::temporal_spans(doc) {
                let _ = tok;
                push(s, e, "time");
            }
            for (cat, words) in &self.categories {
                for w in words {
                    for (s, e) in crate::emergent::word_spans(doc, w) {
                        push(s, e, cat);
                    }
                }
            }
            // overlapping labels would teach the tagger contradictory boundaries
            spans.sort_by_key(|v| (v["start"].as_u64().unwrap_or(0), v["end"].as_u64().unwrap_or(0)));
            let mut kept: Vec<serde_json::Value> = Vec::new();
            let mut cursor = 0u64;
            for sp in spans {
                let (s, e) = (sp["start"].as_u64().unwrap_or(0), sp["end"].as_u64().unwrap_or(0));
                if s >= cursor {
                    cursor = e;
                    kept.push(sp);
                }
            }
            if kept.is_empty() {
                continue; // a passage with no labels teaches nothing
            }
            let line = serde_json::json!({ "text": doc, "spans": kept });
            jsonl.push_str(&line.to_string());
            jsonl.push('\n');
        }

        let categories = self
            .categories
            .iter()
            .map(|(name, words)| crate::artifact::CategoryRecord {
                name: name.clone(),
                words: words.clone(),
            })
            .collect();
        let mut relations: Vec<String> = Vec::new();
        for doc in &self.documents {
            for r in crate::emergent::relation_spans(doc, &gazetteer) {
                if !relations.contains(&r.verb) {
                    relations.push(r.verb);
                }
            }
        }
        let set = crate::artifact::Artifacts::new(
            "discovery",
            categories,
            gazetteer.iter().map(|s| crate::artifact::Registration {
                surface: s.clone(),
                token: format!("entity/{}", crate::projector::slug(s)),
            }).collect(),
            relations,
        )
        .with_training(jsonl);
        let n = set.training_examples;
        set.save(artifact_dir).map_err(Error::Artifact)?;
        Ok(n)
    }

    /// Index a directory of documents, discovering the vocabulary from what it finds.
    pub fn open(dir: impl AsRef<Path>) -> Result<Self, Error> {
        let dir = dir.as_ref();
        let mut docs: Vec<String> = Vec::new();
        for entry in std::fs::read_dir(dir)? {
            let path = entry?.path();
            if path.extension().map(|e| e == "md" || e == "txt").unwrap_or(false) {
                if let Ok(text) = std::fs::read_to_string(&path) {
                    docs.push(text);
                }
            }
        }
        if docs.is_empty() {
            return Err(Error::Empty(format!("no .md or .txt files under {}", dir.display())));
        }
        Self::ingest(docs)
    }

    /// Project documents into tagged situations. One situation per document, carrying every dimension it
    /// supports: entities, relation roles with polarity, temporal buckets, quantities, epistemic state.
    fn project(docs: &[String], categories: &[(String, Vec<String>)]) -> Corpus {
        Self::project_with(docs, categories, &Self::registrations(docs))
    }

    /// Mine mentions and register each under its canonical token, so discovery and an artefact reload agree.
    fn registrations(docs: &[String]) -> Vec<crate::artifact::Registration> {
        crate::emergent::mine_gazetteer(docs, 2)
            .into_iter()
            .map(|surface| {
                let token = format!("entity/{}", crate::projector::slug(&surface));
                crate::artifact::Registration { surface, token }
            })
            .collect()
    }

    /// As [`Self::project`], with the mention list supplied rather than mined — so an artefact set fully
    /// determines the projection and a later run cannot drift from the recorded vocabulary.
    fn project_with(
        docs: &[String],
        categories: &[(String, Vec<String>)],
        registrations: &[crate::artifact::Registration],
    ) -> Corpus {
        let gazetteer: Vec<String> = registrations.iter().map(|r| r.surface.clone()).collect();
        // surface -> canonical token, so a normalised registration is honoured rather than re-derived
        let canonical: std::collections::HashMap<&str, &str> =
            registrations.iter().map(|r| (r.surface.as_str(), r.token.as_str())).collect();
        let mut corpus = Corpus::new_incremental("documents", vec!["document".into()], CorpusKind::Text);

        for doc in docs {
            let mut tags: Vec<String> = Vec::new();
            let mut numbers: Vec<(String, f64)> = Vec::new();
            let level = crate::dimensions::belief_level(
                doc.to_lowercase().contains("not permitted") || doc.to_lowercase().contains("is not "),
                doc.to_lowercase().contains("under review") || doc.to_lowercase().contains("may be"),
            );

            for mention in &gazetteer {
                if crate::emergent::contains_term(doc, mention) {
                    // use the registered token; slugging the surface here is what lost normalisation in v1
                    match canonical.get(mention.as_str()) {
                        Some(tok) => tags.push((*tok).to_string()),
                        None => tags.push(format!("entity/{}", crate::projector::slug(mention))),
                    }
                }
            }
            for r in crate::emergent::relation_spans(doc, &gazetteer) {
                let verb = crate::projector::slug(&r.verb);
                tags.push(format!("rel/{verb}/+"));
                tags.push(format!("rel/{verb}/-"));
            }
            for (_, _, tok) in crate::emergent::temporal_spans(doc) {
                tags.push(tok);
            }
            for (st, en, field) in crate::emergent::quantity_spans(doc) {
                tags.push(format!("quantity/{field}"));
                let digits: String = doc[st..en]
                    .chars()
                    .enumerate()
                    .take_while(|(i, c)| c.is_ascii_digit() || *c == '.' || (*i == 0 && *c == '-'))
                    .map(|(_, c)| c)
                    .collect();
                if let Ok(v) = digits.parse::<f64>() {
                    numbers.push((field, v));
                }
            }
            for (cat, terms) in categories {
                for t in terms {
                    if crate::emergent::contains_term(doc, t) {
                        tags.push(format!("{cat}/{}", crate::projector::slug(t)));
                    }
                }
            }
            tags.push(
                match level {
                    l if l < 0.0 => "state/negated",
                    l if l < 1.0 => "state/hedged",
                    _ => "state/asserted",
                }
                .to_string(),
            );

            let beliefs: Vec<(String, f32)> = tags.iter().map(|t| (t.clone(), level)).collect();
            let display = vec![doc.chars().take(160).collect::<String>()];
            numbers.dedup_by(|a, b| a.0 == b.0);
            corpus.add_situation_polar(tags, display, numbers, beliefs);
        }
        corpus
    }

    /// Run a query, or refuse it.
    ///
    /// Every tag is checked against the vocabulary before anything executes, so an unsupported query costs
    /// nothing and comes back with alternatives.
    pub fn query(&self, ikl: &str) -> Result<Answer, Refused> {
        let report = self.corpus.linter().lint(ikl);
        if !report.ok {
            return Err(Refused {
                query: ikl.to_string(),
                problems: report.errors.iter().map(|e| e.message.clone()).collect(),
                alternatives: self.categories.iter().map(|(c, _)| format!("{c}/*")).collect(),
            });
        }
        match crate::tokenql::try_evaluate(self.corpus.index(), ikl) {
            Ok(set) => Ok(Answer { ids: set.to_sorted(), micros: 0.0 }),
            Err(e) => Err(Refused {
                query: ikl.to_string(),
                problems: vec![e.to_string()],
                alternatives: self.categories.iter().map(|(c, _)| format!("{c}/*")).collect(),
            }),
        }
    }

    /// Check a query without running it. Cheap, and the same check `query` performs.
    pub fn check(&self, ikl: &str) -> Result<(), Refused> {
        let report = self.corpus.linter().lint(ikl);
        if report.ok {
            Ok(())
        } else {
            Err(Refused {
                query: ikl.to_string(),
                problems: report.errors.iter().map(|e| e.message.clone()).collect(),
                alternatives: self.categories.iter().map(|(c, _)| format!("{c}/*")).collect(),
            })
        }
    }

    /// The evidential bound on a tag across the whole corpus.
    pub fn belief(&self, tag: &str) -> Interval {
        let (belief, plausibility) = self.corpus.belief_interval(tag);
        Interval { belief, plausibility }
    }

    /// Situations on a chain from `from` to `to` where each step shares at least `s` tags.
    ///
    /// Raising `s` demands more agreement per step, which is what stops a walk drifting somewhere unrelated.
    pub fn s_path(&self, from: &str, to: &str, s: usize) -> Answer {
        let set = self.corpus.index().s_path_tokens(from, to, s).map(|chain| {
            let mut out = P::empty();
            for tok in &chain {
                out.or_inplace(&crate::tokenql::TokenStore::atom(self.corpus.index(), tok));
            }
            out
        });
        Answer { ids: set.map(|s| s.to_sorted()).unwrap_or_default(), micros: 0.0 }
    }

    /// Both readings of the incidence matrix, swept over the overlap threshold.
    pub fn filtration(&self, max_s: usize) -> Vec<crate::programs::Level> {
        crate::programs::s_filtration(self.corpus.index(), max_s, &Default::default(), 128)
    }

    /// The discovered categories and the words each claims.
    pub fn categories(&self) -> Vec<Category<'_>> {
        self.categories.iter().map(|(name, words)| Category { name, words }).collect()
    }

    /// The wildcard for every discovered category — the set of things you can ask about.
    pub fn askable(&self) -> Vec<String> {
        self.categories.iter().map(|(c, _)| format!("{c}/*")).collect()
    }

    /// Every tag in the index, grouped by its category and sorted within each group.
    ///
    /// Sorted because the engine is otherwise deterministic and a caller should not have to defend against
    /// index iteration order — two runs over the same documents return byte-identical output.
    pub fn tags(&self) -> BTreeMap<String, Vec<String>> {
        let mut out: BTreeMap<String, Vec<String>> = BTreeMap::new();
        for tag in self.corpus.index().tokens() {
            let stem = tag.split('/').next().unwrap_or("").to_string();
            out.entry(stem).or_default().push(tag.clone());
        }
        for v in out.values_mut() {
            v.sort();
            v.dedup();
        }
        out
    }

    /// The document behind a situation id.
    ///
    /// Without this an [`Answer`] is a list of integers. Every result is traceable back to the text that
    /// produced it, which is what makes an answer checkable rather than merely plausible.
    pub fn text(&self, situation: u32) -> Option<&str> {
        self.documents.get(situation as usize).map(|s| s.as_str())
    }

    /// The documents an answer refers to, in id order.
    pub fn resolve<'a>(&'a self, answer: &'a Answer) -> impl Iterator<Item = (u32, &'a str)> + 'a {
        answer.ids().iter().filter_map(move |id| self.text(*id).map(|t| (*id, t)))
    }

    /// How many situations are indexed.
    pub fn len(&self) -> usize {
        self.documents.len()
    }
    pub fn is_empty(&self) -> bool {
        self.documents.is_empty()
    }
    /// The documents as given, for tracing a result back to its source.
    pub fn documents(&self) -> &[String] {
        &self.documents
    }

    // ── internals used by `learn`, which needs to re-run the same gate ──

    /// A vocabulary spec matching the currently accepted categories.
    pub(crate) fn spec_snapshot(&self) -> crate::vocabulary::VocabularySpace {
        crate::vocabulary::VocabularySpace {
            version: 1,
            corpus: "documents".into(),
            entity_facets: self
                .categories
                .iter()
                .map(|(name, words)| crate::vocabulary::EntityFacet {
                    name: name.clone(),
                    parent: None,
                    description: String::new(),
                    examples: words.clone(),
                    structural: false,
                })
                .collect(),
            relation_facets: Vec::new(),
            gazetteer: Vec::new(),
            metrics: None,
        }
    }

    pub(crate) fn min_gain(&self) -> f64 {
        self.min_gain
    }

    pub(crate) fn push_category(&mut self, name: String, words: Vec<String>) {
        self.categories.push((name, words));
    }

    /// Rebuild the index. Required after adopting a category, because a new category changes what every
    /// document projects to — leaving the old index would answer with a vocabulary the spec no longer matches.
    pub(crate) fn reproject(&mut self) {
        self.corpus = Self::project(&self.documents, &self.categories);
    }
}

#[cfg(test)]
mod tests {
    use super::*;

    fn corpus() -> Vec<String> {
        [
            "Morty Shade defeated Wallace Gale at Ecruteak City during the Indigo Invitational in 2025.",
            "Bea Strike defeated Iris Draco at Ecruteak City during the Indigo Invitational in 2025.",
            "A habitat survey recorded Aggron near Sootopolis City at an elevation of 1082 m.",
            "A habitat survey recorded Salamence near Sootopolis City at an elevation of 2369 m.",
            "Milotic is not permitted in Series 1 play for the 2025 season.",
            "Metagross is permitted in Series 4 play for the 2026 season.",
        ]
        .iter()
        .map(|s| s.to_string())
        .collect()
    }

    #[test]
    fn three_lines_to_a_working_database() {
        let db = SteelDb::ingest(corpus()).expect("index");
        assert_eq!(db.len(), 6);
        assert!(!db.categories().is_empty(), "should discover at least one category");
    }

    #[test]
    fn an_unsupported_query_is_refused_with_alternatives() {
        let db = SteelDb::ingest(corpus()).unwrap();
        let err = db.query("gene/brca1").expect_err("must refuse a category the data lacks");
        assert!(!err.alternatives.is_empty(), "a refusal must say what does exist");
        let shown = err.to_string();
        assert!(shown.contains("refused"), "{shown}");
        assert!(shown.contains("available"), "{shown}");
    }

    #[test]
    fn a_supported_query_returns_a_complete_set() {
        let db = SteelDb::ingest(corpus()).unwrap();
        let cat = db.categories()[0].name.to_string();
        let answer = db.query(&format!("{cat}/*")).expect("a discovered category must be queryable");
        assert!(!answer.is_empty());
        // complete, not sampled: every id is a real situation
        assert!(answer.ids().iter().all(|id| (*id as usize) < db.len()));
        // and iterable directly
        assert_eq!(answer.ids().len(), (&answer).into_iter().count());
    }

    #[test]
    fn negation_narrows_rather_than_widens() {
        let db = SteelDb::ingest(corpus()).unwrap();
        let cat = db.categories()[0].name.to_string();
        let all = db.query(&format!("{cat}/*")).unwrap().len();
        let some = db.query(&format!("(and {cat}/* (not state/negated))")).unwrap().len();
        assert!(some <= all, "excluding something cannot return more: {some} vs {all}");
    }

    #[test]
    fn belief_separates_asserted_from_negated() {
        let db = SteelDb::ingest(corpus()).unwrap();
        let asserted = db.belief("state/asserted");
        let negated = db.belief("state/negated");
        assert!(asserted.belief > negated.belief, "{asserted} vs {negated}");
        // and the interval exposes its own meaning rather than making the caller compare floats
        assert!(asserted.ignorance() >= 0.0);
        assert!(db.belief("state/nonexistent").is_unknown(), "an absent tag is unknown, not refuted");
    }

    #[test]
    fn check_costs_nothing_and_agrees_with_query() {
        let db = SteelDb::ingest(corpus()).unwrap();
        assert!(db.check("gene/brca1").is_err());
        assert!(db.query("gene/brca1").is_err());
        let cat = db.categories()[0].name.to_string();
        assert!(db.check(&format!("{cat}/*")).is_ok());
    }

    #[test]
    fn the_filtration_thins_as_the_threshold_rises() {
        let db = SteelDb::ingest(corpus()).unwrap();
        let levels = db.filtration(4);
        assert_eq!(levels.len(), 4);
        // more required agreement can only remove edges
        for w in levels.windows(2) {
            assert!(w[1].primal.edges <= w[0].primal.edges, "edges must not grow with s");
            assert!(w[1].dual.edges <= w[0].dual.edges);
        }
    }

    #[test]
    fn empty_input_is_an_error_not_an_empty_database() {
        assert!(matches!(SteelDb::ingest(Vec::<String>::new()), Err(Error::Empty(_))));
        assert!(matches!(SteelDb::ingest(vec!["   ", ""]), Err(Error::Empty(_))));
    }

    #[test]
    fn artefacts_make_a_later_ingest_reproducible() {
        // The pairing that justifies `learn`: the vocabulary is recorded once, and a later run reproduces it
        // exactly without a model.
        let dir = std::env::temp_dir().join(format!("hsdb_api_repro_{}", std::process::id()));
        let _ = std::fs::remove_dir_all(&dir);

        let first = SteelDb::ingest(corpus()).unwrap();
        first.save(&dir).unwrap();
        let cat = first.categories()[0].name.to_string();
        let expected = first.query(&format!("{cat}/*")).unwrap().len();

        let second = SteelDb::ingest_using(corpus(), &dir).unwrap();
        assert_eq!(
            second.categories().iter().map(|c| c.name.to_string()).collect::<Vec<_>>(),
            first.categories().iter().map(|c| c.name.to_string()).collect::<Vec<_>>(),
            "the recorded vocabulary must be reproduced exactly"
        );
        assert_eq!(second.query(&format!("{cat}/*")).unwrap().len(), expected, "and answer identically");
        let _ = std::fs::remove_dir_all(&dir);
    }

    #[test]
    fn saved_artefacts_contain_no_document_text() {
        // Leakage guard at the API level: what `save` writes must be vocabulary, never the corpus.
        let dir = std::env::temp_dir().join(format!("hsdb_api_leak_{}", std::process::id()));
        let _ = std::fs::remove_dir_all(&dir);
        let db = SteelDb::ingest(corpus()).unwrap();
        db.save(&dir).unwrap();

        for entry in std::fs::read_dir(&dir).unwrap() {
            let p = entry.unwrap().path();
            let text = std::fs::read_to_string(&p).unwrap();
            for doc in corpus() {
                assert!(
                    !text.contains(doc.as_str()),
                    "{} contains a whole document",
                    p.display()
                );
                // a distinctive multi-word fragment is enough to prove a copy
                let frag: String = doc.split_whitespace().take(6).collect::<Vec<_>>().join(" ");
                assert!(!text.contains(&frag), "{} contains the fragment {frag:?}", p.display());
            }
        }
        let _ = std::fs::remove_dir_all(&dir);
    }

    #[test]
    fn ingesting_against_a_missing_artefact_set_is_an_error() {
        let missing = std::env::temp_dir().join("hsdb_definitely_absent_dir");
        let _ = std::fs::remove_dir_all(&missing);
        assert!(matches!(
            SteelDb::ingest_using(corpus(), &missing),
            Err(Error::Artifact(_))
        ));
    }

    #[test]
    fn a_finetuning_set_is_written_separately_from_the_vocabulary() {
        let dir = std::env::temp_dir().join(format!("hsdb_api_train_{}", std::process::id()));
        let _ = std::fs::remove_dir_all(&dir);
        let db = SteelDb::ingest(corpus()).unwrap();
        let n = db.save_with_training(&dir).unwrap();
        assert!(n > 0, "the corpus should yield labelled passages");

        // the committable parts still contain no document text, even though a training set exists
        for f in ["manifest.json", "vocabulary.json", "gazetteer.json", "relations.json"] {
            let text = std::fs::read_to_string(dir.join(f)).unwrap();
            for doc in corpus() {
                let frag: String = doc.split_whitespace().take(6).collect::<Vec<_>>().join(" ");
                assert!(!text.contains(&frag), "{f} leaked: {frag:?}");
            }
        }
        // and the training file does carry text, which is the point of keeping it apart
        let train = std::fs::read_to_string(dir.join("training").join("spans.jsonl")).unwrap();
        assert!(train.contains("Morty Shade"), "the finetuning set needs the words");
        assert!(dir.join(".gitignore").exists(), "and must be excluded from commits");
        let _ = std::fs::remove_dir_all(&dir);
    }

    #[test]
    fn training_spans_do_not_overlap() {
        // Overlapping labels would teach a tagger contradictory boundaries for the same characters.
        let dir = std::env::temp_dir().join(format!("hsdb_api_ovl_{}", std::process::id()));
        let _ = std::fs::remove_dir_all(&dir);
        SteelDb::ingest(corpus()).unwrap().save_with_training(&dir).unwrap();
        let train = std::fs::read_to_string(dir.join("training").join("spans.jsonl")).unwrap();
        for line in train.lines().filter(|l| !l.trim().is_empty()) {
            let v: serde_json::Value = serde_json::from_str(line).unwrap();
            let spans = v["spans"].as_array().unwrap();
            let mut last_end = 0u64;
            for sp in spans {
                let s = sp["start"].as_u64().unwrap();
                let e = sp["end"].as_u64().unwrap();
                assert!(s >= last_end, "span {s}..{e} overlaps the previous one ending at {last_end}");
                assert!(e > s, "empty span");
                last_end = e;
            }
        }
        let _ = std::fs::remove_dir_all(&dir);
    }

    #[test]
    fn registered_variants_still_merge_after_an_artefact_reload() {
        // The v1 bug: the artefact recorded only surfaces, so a reload re-derived the token by slugging the raw
        // text and two variants of one entity stopped sharing a token. Registration NORMALISES, and that has to
        // survive the round trip or ingest is not really following the artefacts.
        let dir = std::env::temp_dir().join(format!("hsdb_api_canon_{}", std::process::id()));
        let _ = std::fs::remove_dir_all(&dir);

        let db = SteelDb::ingest(corpus()).unwrap();
        db.save(&dir).unwrap();

        let set = crate::artifact::Artifacts::load(&dir).unwrap();
        assert!(!set.gazetteer.is_empty(), "the corpus should register some mentions");
        for r in &set.gazetteer {
            assert!(!r.token.is_empty(), "every registration needs a canonical token");
            assert!(r.token.contains('/'), "a token is facet-qualified: {}", r.token);
        }

        // reloading must reproduce the same entity tags, not re-derive different ones
        let reloaded = SteelDb::ingest_using(corpus(), &dir).unwrap();
        let tags_before: Vec<String> =
            db.tags().get("entity").cloned().unwrap_or_default();
        let tags_after: Vec<String> =
            reloaded.tags().get("entity").cloned().unwrap_or_default();
        assert_eq!(tags_before, tags_after, "entity tags must survive the round trip unchanged");
        let _ = std::fs::remove_dir_all(&dir);
    }
}