hypersteeldb 0.1.0

//! **Step 0 in the browser** — the same Rust that runs in the CLI, compiled to WebAssembly.
//!
//! Paper §2 (Domain Typing via the Bitmap Symbol Table) is model-free and pure: given document text, field
//! sensing counts recurring labelled fields and returns the Vocabulary Space **V**. No file I/O, no network,
//! no model — which is exactly the property that lets it run client-side.
//!
//! This exists so the public demo can be a *static* page while still executing the real engine. A JavaScript
//! reimplementation of step 0 would demonstrate nothing: the claim under test is that the same code returns
//! the same vocabulary every time, so the demo has to run that code.
//!
//! Build (see `spaces/build-wasm.sh`):
//! ```text
//! cargo build --release --target wasm32-unknown-unknown --no-default-features --features wasm --lib
//! wasm-bindgen --target web --out-dir <out> target/wasm32-unknown-unknown/release/steeldb.wasm
//! ```

use wasm_bindgen::prelude::*;

use crate::linter::Linter;
use crate::vocabulary::{candidate_fields, field_spans, field_value_tokens, seed_from_sample};

/// Derive the Vocabulary Space from documents, returning the same JSON shape the `sense_ontology` binary
/// prints. `docs_text` holds the documents separated by a line containing `---`.
#[wasm_bindgen]
pub fn sense_ontology(docs_text: &str, min_support: usize) -> String {
    let samples: Vec<String> = docs_text
        .split("\n---")
        .map(|d| d.trim().to_string())
        .filter(|d| !d.is_empty())
        .collect();

    if samples.is_empty() {
        return serde_json::json!({ "error": "no documents" }).to_string();
    }

    let candidates = candidate_fields(&samples);
    let spec = seed_from_sample("pasted", &samples, min_support);

    serde_json::json!({
        "corpus": "pasted",
        "sampled_docs": samples.len(),
        "min_support": min_support,
        "entity_facets": spec.entity_facets.iter().map(|f| serde_json::json!({
            "name": f.name,
            "path": spec.facet_path(&f.name),
            "parent": f.parent,
            "description": f.description,
            "structural": f.structural,
            "examples": f.examples,
        })).collect::<Vec<_>>(),
        "relation_facets": spec.relation_facets.iter().map(|r| serde_json::json!({
            "name": r.name, "head": r.head, "tail": r.tail, "uri": format!("rel/{}/+", r.name),
        })).collect::<Vec<_>>(),
        "wildcard_stems": spec.valid_prefixes(),
        "taggable_facets": spec.taggable_facets(),
        "candidate_fields": candidates.iter().map(|(name, n)| serde_json::json!({
            "field": name, "support": n, "kept": *n >= min_support,
        })).collect::<Vec<_>>(),
        "valid": spec.validate().is_ok(),
        "validation_error": spec.validate().err().map(|e| e.to_string()),
    })
    .to_string()
}

/// Lint an IKL s-expression against a vocabulary derived from the same documents.
///
/// This is the payoff of step 0 and the reason the two are shown together: an atom can only be checked
/// because the previous step decided what exists. An unknown term comes back as a rejection *with
/// in-vocabulary alternatives*, never as a guess.
#[wasm_bindgen]
pub fn lint_expression(docs_text: &str, min_support: usize, expression: &str) -> String {
    let samples: Vec<String> = docs_text
        .split("\n---")
        .map(|d| d.trim().to_string())
        .filter(|d| !d.is_empty())
        .collect();
    let spec = seed_from_sample("pasted", &samples, min_support);

    // The symbol table a real corpus exposes: concrete `facet/value` tokens, not just stems. Seeding only
    // stems would make a perfectly valid `type/fire` look unknown, which misrepresents the linter.
    let facets: Vec<String> = spec.entity_facets.iter().map(|f| f.name.clone()).collect();
    let mut tokens = field_value_tokens(&samples, &facets);
    tokens.extend(facets.iter().map(|f| format!("{f}/*")));
    let linter = Linter::from_tokens(tokens.clone());
    let report = linter.lint(expression);

    serde_json::json!({
        "expression": expression,
        "ok": report.ok,
        "repaired": report.repaired,
        "errors": report.errors.iter().map(|e| serde_json::json!({
            "message": e.message,
        })).collect::<Vec<_>>(),
        "facets": linter.facet_names(),
        "symbol_table_size": tokens.len(),
    })
    .to_string()
}

/// Annotate the documents for a displaCy-style inline rendering.
///
/// Returns each document already **segmented** into an ordered list of chunks — `{text}` for plain text and
/// `{text, facet}` for a matched field value. Byte offsets are deliberately NOT exposed: Rust counts bytes
/// while JavaScript indexes UTF-16 code units, so any document containing `é` or `°` would render shifted
/// text. Segmenting here makes that class of bug unrepresentable in the caller.
#[wasm_bindgen]
pub fn annotate(docs_text: &str, min_support: usize) -> String {
    let samples: Vec<String> = docs_text
        .split("\n---")
        .map(|d| d.trim().to_string())
        .filter(|d| !d.is_empty())
        .collect();
    let spec = seed_from_sample("pasted", &samples, min_support);
    let kept: std::collections::HashSet<&str> = spec.entity_facets.iter().map(|f| f.name.as_str()).collect();

    let docs: Vec<serde_json::Value> = samples
        .iter()
        .map(|doc| {
            let mut spans = field_spans(doc);
            // only facets that cleared the support floor are part of the vocabulary
            spans.retain(|s| kept.contains(s.facet.as_str()));
            spans.sort_by_key(|s| s.start);

            let mut segments: Vec<serde_json::Value> = Vec::new();
            let mut cursor = 0usize;
            for s in &spans {
                if s.start < cursor || s.end > doc.len() {
                    continue; // overlapping or out of range: skip rather than corrupt the text
                }
                if s.start > cursor {
                    segments.push(serde_json::json!({ "text": &doc[cursor..s.start] }));
                }
                segments.push(serde_json::json!({ "text": &doc[s.start..s.end], "facet": s.facet }));
                cursor = s.end;
            }
            if cursor < doc.len() {
                segments.push(serde_json::json!({ "text": &doc[cursor..] }));
            }
            serde_json::json!({ "segments": segments, "matched": spans.len() })
        })
        .collect();

    serde_json::json!({
        "documents": docs,
        "facets": spec.entity_facets.iter().map(|f| f.name.clone()).collect::<Vec<_>>(),
        "sampled_docs": samples.len(),
        "min_support": min_support,
    })
    .to_string()
}

/// The discovered taxonomy as a dendrogram: leaves are the facets kept at the cut, internal nodes are the
/// merges that produced them.
#[wasm_bindgen]
pub fn hierarchy(docs_text: &str, n_terms: usize, n_clusters: usize) -> String {
    let docs: Vec<String> = docs_text
        .split("\n---")
        .map(|d| d.trim().to_string())
        .filter(|d| !d.is_empty())
        .collect();
    match crate::emergent::discover_hierarchy(&docs, n_terms, n_clusters) {
        Some(tree) => serde_json::to_string(&tree).unwrap_or_else(|_| "null".into()),
        None => "null".to_string(),
    }
}

/// Run the **real** ontology codebook on real typed spans: per kind, k-means prototypes then Sinkhorn
/// optimal transport on a `1 - cosine` cost, exactly as `spo_sinkhorn.py` does.
///
/// `spans_json` is the output of the native `export_spans` binary: the SPO tagger's typed spans with their
/// model2vec embeddings. Those two stages cannot compile to WebAssembly (ONNX Runtime, and a tokenizer that
/// links a C regex library), so they run natively and arrive here as data. Everything the ontology discovery
/// actually consists of runs here.
///
/// Clustering is per kind because the kinds ARE the facet axes: each within-kind cluster is a facet value.
/// Pooling them would cluster a place against a verb, which is the failure the earlier co-occurrence demo had.
///
/// The epsilon schedule is returned as frames so a viewer can watch the plan sharpen. Lowering epsilon is
/// what concentrates the assignment; more iterations only make it satisfy the marginals.
#[wasm_bindgen]
pub fn codebook(spans_json: &str, k_per_kind: usize) -> String {
    use crate::text::ot;

    let Ok(payload) = serde_json::from_str::<serde_json::Value>(spans_json) else {
        return serde_json::json!({ "error": "could not parse spans" }).to_string();
    };
    let empty: Vec<serde_json::Value> = Vec::new();
    let spans = payload["spans"].as_array().unwrap_or(&empty);

    // group by kind, l2-normalising as we go so cosine is a dot product
    let mut by_kind: std::collections::BTreeMap<String, (Vec<String>, Vec<Vec<f32>>, Vec<usize>)> =
        std::collections::BTreeMap::new();
    for sp in spans {
        let kind = sp["kind"].as_str().unwrap_or("?").to_string();
        let text = sp["text"].as_str().unwrap_or("").to_string();
        let count = sp["count"].as_u64().unwrap_or(1) as usize;
        let v: Vec<f32> = sp["vec"]
            .as_array()
            .map(|a| a.iter().filter_map(|x| x.as_f64()).map(|x| x as f32).collect())
            .unwrap_or_default();
        if text.is_empty() || v.is_empty() {
            continue;
        }
        let norm = v.iter().map(|x| x * x).sum::<f32>().sqrt() + 1e-9;
        let unit: Vec<f32> = v.into_iter().map(|x| x / norm).collect();
        let e = by_kind.entry(kind).or_insert_with(|| (Vec::new(), Vec::new(), Vec::new()));
        e.0.push(text);
        e.1.push(unit);
        e.2.push(count);
    }

    let entropy = |plan: &Vec<Vec<f32>>, k: usize| -> f64 {
        let mut h = 0.0f64;
        let mut rows = 0usize;
        for row in plan {
            let sum: f32 = row.iter().sum();
            if sum <= 0.0 {
                continue;
            }
            for &x in row {
                let p = (x / sum) as f64;
                if p > 1e-12 {
                    h -= p * p.ln();
                }
            }
            rows += 1;
        }
        if rows == 0 { 0.0 } else { h / rows as f64 / (k as f64).ln().max(1e-9) }
    };

    let schedule: [f32; 6] = [0.40, 0.26, 0.17, 0.11, 0.07, 0.045];
    let mut kinds_out: Vec<serde_json::Value> = Vec::new();

    for (kind, (terms, vecs, counts)) in by_kind {
        let k = k_per_kind.clamp(2, terms.len().max(2)).min(terms.len());
        if terms.len() < 2 {
            continue;
        }
        let protos = ot::kmeans(&vecs, k, 40, 7);
        // cost = 1 - cosine, the reference's `1.0 - X @ C.T`
        let cost: Vec<Vec<f32>> = vecs
            .iter()
            .map(|v| protos.iter().map(|c| 1.0 - v.iter().zip(c).map(|(a, b)| a * b).sum::<f32>()).collect())
            .collect();

        let mut frames: Vec<serde_json::Value> = Vec::new();
        let mut final_plan: Vec<Vec<f32>> = Vec::new();
        for &eps in &schedule {
            let (plan, c) = ot::sinkhorn(&cost, eps, 200);
            frames.push(serde_json::json!({
                "epsilon": eps,
                "entropy": entropy(&plan, k),
                "cost": c,
                "plan": plan.iter().map(|row| {
                    let sum: f32 = row.iter().sum::<f32>().max(1e-9);
                    row.iter().map(|x| (x / sum * 100.0).round() as i32).collect::<Vec<_>>()
                }).collect::<Vec<_>>(),
            }));
            final_plan = plan;
        }

        // each term joins the prototype it sends most mass to; the cluster is labelled by its nearest member
        let mut members: Vec<Vec<(String, usize, f32)>> = vec![Vec::new(); k];
        for (i, row) in final_plan.iter().enumerate() {
            let (best, mass) = row
                .iter()
                .enumerate()
                .max_by(|a, b| a.1.partial_cmp(b.1).unwrap_or(std::cmp::Ordering::Equal))
                .map(|(j, m)| (j, *m))
                .unwrap_or((0, 0.0));
            // proximity to the prototype decides the label, so it is the most representative member
            let closeness = 1.0 - cost[i][best];
            members[best].push((terms[i].clone(), counts[i], closeness.max(0.0)));
            let _ = mass;
        }

        let clusters: Vec<serde_json::Value> = members
            .into_iter()
            .enumerate()
            .filter(|(_, m)| !m.is_empty())
            .map(|(j, mut m)| {
                m.sort_by(|a, b| b.2.partial_cmp(&a.2).unwrap_or(std::cmp::Ordering::Equal));
                serde_json::json!({
                    "index": j,
                    "label": m.first().map(|(t, _, _)| t.clone()).unwrap_or_default(),
                    "size": m.len(),
                    "members": m.iter().take(8).map(|(t, c, _)| serde_json::json!({ "text": t, "count": c }))
                        .collect::<Vec<_>>(),
                })
            })
            .collect();

        kinds_out.push(serde_json::json!({
            "kind": kind,
            "terms": terms,
            "k": k,
            "clusters": clusters,
            "frames": frames,
        }));
    }

    serde_json::json!({
        "documents": payload["documents"].clone(),
        "dim": payload["dim"].clone(),
        "routed_away": payload["routed_away"].clone(),
        "numeric_rules": payload["numeric_rules"].clone(),
        "kinds": kinds_out,
        "source": payload["source"].clone(),
    })
    .to_string()
}

/// Sinkhorn optimal transport over the co-occurrence geometry, reported at increasing iteration counts so a
/// viewer can watch the transport plan sharpen and its entropy fall.
///
/// Rows are terms, columns are emerging topics, and a cell is how much of that term's mass is assigned to
/// that topic. Early on the mass is spread out (high entropy, the algorithm is undecided); as it iterates the
/// plan concentrates, and that concentration IS the ontology crystallising. Balanced marginals are what stop
/// one topic from swallowing everything.
#[wasm_bindgen]
pub fn transport(docs_text: &str, n_terms: usize, k: usize, eps: f32) -> String {
    use crate::text::ot;
    let docs: Vec<String> = docs_text
        .split("\n---")
        .map(|d| d.trim().to_string())
        .filter(|d| !d.is_empty())
        .collect();

    let (names, vecs) = crate::emergent::term_vectors(&docs, n_terms);
    if names.len() < k || k == 0 {
        return serde_json::json!({ "error": "not enough salient terms for that many topics" }).to_string();
    }

    // topic centroids from k-means over the same geometry
    let centroids = ot::kmeans(&vecs, k, 25, 7);
    // cost = 1 - cosine; vectors are L2-normalised so cosine is the dot product
    let cost: Vec<Vec<f32>> = vecs
        .iter()
        .map(|v| centroids.iter().map(|c| 1.0 - v.iter().zip(c).map(|(a, b)| a * b).sum::<f32>()).collect())
        .collect();

    // entropy of a plan, normalised so 1.0 = maximally spread and 0.0 = fully decided
    let entropy = |plan: &Vec<Vec<f32>>| -> f64 {
        let mut h = 0.0f64;
        let mut cells = 0usize;
        for row in plan {
            let sum: f32 = row.iter().sum();
            if sum <= 0.0 {
                continue;
            }
            for &x in row {
                let p = (x / sum) as f64;
                if p > 1e-12 {
                    h -= p * p.ln();
                }
            }
            cells += 1;
        }
        if cells == 0 { 0.0 } else { h / cells as f64 / (k as f64).ln().max(1e-9) }
    };

    // Anneal the regularisation rather than the iteration count. More iterations make Sinkhorn SATISFY the
    // marginals, which spreads mass and RAISES entropy — measured here rising 0.17 to 0.28, the opposite of
    // convergence. Sharpening comes from lowering eps: as eps falls the plan approaches a hard assignment.
    // Each step is run to convergence so what the viewer sees is the effect of eps alone.
    let schedule: Vec<f32> = vec![0.50, 0.34, 0.22, 0.15, 0.10, 0.065, 0.042, 0.028];
    let mut steps: Vec<serde_json::Value> = Vec::new();
    let mut final_plan: Vec<Vec<f32>> = Vec::new();
    let mut final_plans: Vec<Vec<Vec<f32>>> = Vec::new();
    for &e in &schedule {
        let kmat: Vec<Vec<f32>> = cost.iter().map(|row| row.iter().map(|v| (-v / e).exp()).collect()).collect();
        let _ = &kmat;
        let (plan, c) = ot::sinkhorn(&cost, e, 160);
        steps.push(serde_json::json!({ "epsilon": e, "entropy": entropy(&plan), "cost": c }));
        final_plans.push(plan.clone());
        final_plan = plan;
    }
    let _ = eps;

    // label each topic by the term that sends it the most mass
    let topic_labels: Vec<String> = (0..k)
        .map(|j| {
            let mut best = (0usize, -1f32);
            for (i, row) in final_plan.iter().enumerate() {
                if row.get(j).copied().unwrap_or(0.0) > best.1 {
                    best = (i, row[j]);
                }
            }
            names.get(best.0).cloned().unwrap_or_default()
        })
        .collect();

    serde_json::json!({
        "terms": names,
        "topics": topic_labels,
        "steps": steps,
        // row-normalised plans as percentages, one per annealing step, so a viewer can animate the sharpening
        "frames": final_plans.iter().map(|plan| plan.iter().map(|row| {
            let sum: f32 = row.iter().sum::<f32>().max(1e-9);
            row.iter().map(|x| (x / sum * 100.0).round() as i32).collect::<Vec<_>>()
        }).collect::<Vec<_>>()).collect::<Vec<_>>(),
        "plan": final_plan.iter().map(|row| {
            let sum: f32 = row.iter().sum::<f32>().max(1e-9);
            row.iter().map(|x| (x / sum * 100.0).round() as i32).collect::<Vec<_>>()
        }).collect::<Vec<_>>(),
    })
    .to_string()
}

/// The salience step alone: which words carry information about how this corpus divides up.
#[wasm_bindgen]
pub fn salient_terms(docs_text: &str, n_terms: usize) -> String {
    let docs: Vec<String> = docs_text
        .split("\n---")
        .map(|d| d.trim().to_string())
        .filter(|d| !d.is_empty())
        .collect();
    let terms: Vec<serde_json::Value> = crate::emergent::salient(&docs, n_terms)
        .into_iter()
        .map(|(term, idf, df)| serde_json::json!({ "term": term, "idf": idf, "documents": df }))
        .collect();
    serde_json::json!({ "documents": docs.len(), "terms": terms }).to_string()
}

/// Discover facets from PROSE, with no field markers and no model: TF-IDF salience, co-occurrence
/// clustering, then a TF-IDF label per cluster. Each cluster is then judged by the real MECE gate, so the
/// page shows a proposal and an independent decision rather than one blurred step.
#[wasm_bindgen]
pub fn discover_prose(docs_text: &str, n_terms: usize, n_clusters: usize, gain_threshold: f64) -> String {
    let docs: Vec<String> = docs_text
        .split("\n---")
        .map(|d| d.trim().to_string())
        .filter(|d| !d.is_empty())
        .collect();

    let clusters = crate::emergent::discover(&docs, n_terms, n_clusters);

    // Start from an empty vocabulary and let each cluster face the gate in turn. Adopting as we go is what
    // makes the test meaningful: a later cluster is measured against what has already been accepted, so a
    // near-duplicate of an earlier facet is rejected as redundant.
    let mut spec = crate::vocabulary::VocabularySpace {
        version: 1,
        corpus: "pasted".into(),
        entity_facets: Vec::new(),
        relation_facets: Vec::new(),
        gazetteer: Vec::new(),
        metrics: None,
    };

    let mut events: Vec<serde_json::Value> = Vec::new();
    for (round, c) in clusters.iter().enumerate() {
        let cand = crate::grow::Candidate {
            name: c.label.clone(),
            parent: None,
            description: format!("terms co-occurring with '{}'", c.label),
            examples: c.terms.clone(),
            worth_adding: true,
        };
        let scored = crate::grow::score_candidate_full(&spec, &docs, &cand);
        let (score, dup) = match scored {
            Some((s, d)) => (Some(s), d),
            None => (None, None),
        };
        let ev = crate::grow::gate_full(&spec, &cand, score.as_ref(), dup, gain_threshold, round);
        let accepted = ev.kept;
        let _ = &score;
        if accepted {
            crate::grow::adopt(&mut spec, &cand);
        }
        events.push(serde_json::json!({
            "label": c.label,
            "terms": c.terms,
            "coverage": c.coverage,
            "cohesion": c.cohesion,
            "accepted": accepted,
            "reason": ev.reason,
            "gain": ev.gain,
            "maxcos": ev.maxcos,
            "nearest": ev.nearest,
            "threshold": ev.threshold,
        }));
    }

    serde_json::json!({
        "documents": docs.len(),
        "clusters": events,
        "accepted": spec.entity_facets.iter().map(|f| f.name.clone()).collect::<Vec<_>>(),
        "gain_threshold": gain_threshold,
    })
    .to_string()
}

/// Run the engine against fixed inputs with known-correct answers, so a page can show that the real logic is
/// loaded and behaving rather than merely claiming it.
///
/// These exercise the path this demo actually runs. An earlier version asserted things about the
/// labelled-field parser, which the demo stopped using when it moved to prose — a passing check for code
/// nothing calls is worse than no check, because it reads as evidence.
#[wasm_bindgen]
pub fn self_test() -> String {
    let mut checks: Vec<serde_json::Value> = Vec::new();
    let mut all_ok = true;
    let mut check = |name: &str, ok: bool, detail: String| {
        if !ok {
            all_ok = false;
        }
        checks.push(serde_json::json!({ "name": name, "ok": ok, "detail": detail }));
    };

    let docs: Vec<String> = [
        "A survey in Sootopolis City recorded Aggron at an elevation of 1082 m",
        "Another survey in Sootopolis City measured 28 degrees at the same site",
        "Morty Shade defeated Wallace Gale during the Indigo Invitational",
    ]
    .iter()
    .map(|s| s.to_string())
    .collect();

    // whole mentions survive; their parts are not promoted on their own
    let gaz = crate::emergent::mine_gazetteer(&docs, 2);
    check(
        "whole entities",
        gaz.contains(&"Sootopolis City".to_string()) && !gaz.contains(&"City".to_string()),
        format!("{} kept intact, no fragments", gaz.len()),
    );

    // quantities keep their unit, and km is not read as m
    let q = crate::emergent::quantity_spans("at 1082 m and 500 km and 7 minutes");
    let fields: Vec<&str> = q.iter().map(|(_, _, f)| f.as_str()).collect();
    check(
        "quantities + units",
        fields.contains(&"length_m") && fields.contains(&"length_km") && fields.contains(&"minutes"),
        fields.join(", "),
    );

    // relation direction follows word order and reverses when the sentence does
    let mentions: Vec<String> = vec!["Morty Shade".into(), "Wallace Gale".into()];
    let fwd = crate::emergent::relation_spans("Morty Shade defeated Wallace Gale", &mentions);
    let rev = crate::emergent::relation_spans("Wallace Gale defeated Morty Shade", &mentions);
    check(
        "relation direction",
        fwd.first().map(|r| r.actor == "Morty Shade").unwrap_or(false)
            && rev.first().map(|r| r.actor == "Wallace Gale").unwrap_or(false),
        "actor and target swap with word order".into(),
    );

    // dates bucket, and a measurement is not mistaken for a year
    let t = crate::emergent::temporal_spans("held in Q3 2026, at 2369 m");
    let toks: Vec<&str> = t.iter().map(|(_, _, x)| x.as_str()).collect();
    check(
        "date buckets",
        toks.contains(&"time/2026/q3") && !toks.contains(&"time/2369"),
        toks.join(", "),
    );

    // the linter refuses a dimension the corpus does not have
    let linter = Linter::from_tokens(vec!["entity/sootopolis-city".to_string(), "quantity/temp_c".to_string()]);
    check(
        "refuses the unknown",
        !linter.lint("gene/brca1").ok && linter.lint("entity/sootopolis-city").ok,
        "unknown refused, known accepted".into(),
    );

    serde_json::json!({ "ok": all_ok, "checks": checks, "version": version() }).to_string()
}

#[wasm_bindgen]
pub struct Paper {
    corpus: crate::db::Corpus,
    facets: Vec<(String, Vec<String>)>,
    /// multi-word mentions mined from the corpus, longest first
    gazetteer: Vec<String>,
    /// latent motif labels from the Sinkhorn transport plan
    motifs: Vec<String>,
    documents: Vec<String>,
    doc_count: usize,
}

/// Epistemic cues, in the paper's four-level polarity scheme (§4). A ruling that is negated or hedged must
/// not be counted as asserted, which is what makes belief and plausibility differ.
fn belief_for(text: &str) -> f32 {
    let t = text.to_lowercase();
    if t.contains("under review") || t.contains("provisional") || t.contains("may be") {
        0.5 // hedged: possible, not established
    } else if t.contains("not permitted") || t.contains("is not ") || t.contains("no longer") {
        -1.0 // refuted
    } else {
        1.0 // asserted
    }
}

/// Pull `<number> <unit>` pairs out of prose so numeric range predicates have something to range over.
fn numbers_in(text: &str) -> Vec<(String, f64)> {
    let mut out: Vec<(String, f64)> = Vec::new();
    let words: Vec<&str> = text.split_whitespace().collect();
    for w in words.windows(2) {
        let raw = w[0].trim_matches(|c: char| !c.is_ascii_digit() && c != '.' && c != '-');
        let Ok(v) = raw.parse::<f64>() else { continue };
        let unit = w[1].trim_matches(|c: char| !c.is_alphanumeric() && c != '°').to_lowercase();
        let field = match unit.as_str() {
            "minutes" | "minute" | "min" => "minutes",
            "m" | "metres" | "meters" => "elevation_m",
            "°c" | "c" | "celsius" => "temp_c",
            "kg" => "weight_kg",
            _ => continue,
        };
        // keep the first occurrence of each field per document
        if !out.iter().any(|(f, _)| f == field) {
            out.push((field.to_string(), v));
        }
    }
    out
}

#[wasm_bindgen]
impl Paper {
    /// Discover the vocabulary from prose, then project every document into the index.
    #[wasm_bindgen(constructor)]
    pub fn new(docs_text: &str, n_terms: usize, n_clusters: usize, gain_threshold: f64) -> Paper {
        let docs: Vec<String> = docs_text
            .split("\n---")
            .map(|d| d.trim().to_string())
            .filter(|d| !d.is_empty())
            .collect();

        // step 1-4: discover, then keep only what the MECE gate accepts
        let clusters = crate::emergent::discover(&docs, n_terms, n_clusters);
        let mut spec = crate::vocabulary::VocabularySpace {
            version: 1,
            corpus: "pasted".into(),
            entity_facets: Vec::new(),
            relation_facets: Vec::new(),
            gazetteer: Vec::new(),
            metrics: None,
        };
        let mut facets: Vec<(String, Vec<String>)> = Vec::new();
        for (round, c) in clusters.iter().enumerate() {
            let cand = crate::grow::Candidate {
                name: c.label.clone(),
                parent: None,
                description: String::new(),
                examples: c.terms.clone(),
                worth_adding: true,
            };
            let scored = crate::grow::score_candidate_full(&spec, &docs, &cand);
            let (score, dup) = match scored {
                Some((s, d)) => (Some(s), d),
                None => (None, None),
            };
            let ev = crate::grow::gate_full(&spec, &cand, score.as_ref(), dup, gain_threshold, round);
            if ev.kept {
                crate::grow::adopt(&mut spec, &cand);
                facets.push((c.label.clone(), c.terms.clone()));
            }
        }

        // project: one situation per document, tokens are `facet/term` for terms actually present
        let mut corpus = crate::db::Corpus::new_incremental(
            "prose",
            vec!["document".into()],
            crate::projector::CorpusKind::Text,
        );
        // Latent motifs. The paper allows SPLADE or Sinkhorn optimal transport; this uses transport over the
        // co-occurrence geometry, so a theme is discovered rather than keyword-matched — a situation can join
        // a motif through terms that travel together even with no word in common with another member.
        let (motif_terms, motif_vecs) = crate::emergent::term_vectors(&docs, 40);
        let k_motif = 3usize.min(motif_terms.len());
        let mut motifs: Vec<String> = Vec::new();
        let mut motif_members: Vec<Vec<String>> = Vec::new();
        if k_motif > 0 {
            let centroids = crate::text::ot::kmeans(&motif_vecs, k_motif, 25, 7);
            let cost: Vec<Vec<f32>> = motif_vecs
                .iter()
                .map(|v| centroids.iter()
                    .map(|c| 1.0 - v.iter().zip(c).map(|(a, b)| a * b).sum::<f32>())
                    .collect())
                .collect();
            // low epsilon, so each term commits to one theme rather than hedging across all of them
            let (plan, _) = crate::text::ot::sinkhorn(&cost, 0.03, 160);
            let mut members: Vec<Vec<String>> = vec![Vec::new(); k_motif];
            for (i, row) in plan.iter().enumerate() {
                let best = row.iter().enumerate()
                    .max_by(|a, b| a.1.partial_cmp(b.1).unwrap_or(std::cmp::Ordering::Equal))
                    .map(|(j, _)| j).unwrap_or(0);
                members[best].push(motif_terms[i].clone());
            }
            let commons = crate::emergent::common_nouns(&docs);
            for group in &members {
                motifs.push(
                    group.iter().find(|t| commons.contains(*t)).or_else(|| group.first())
                        .cloned().unwrap_or_default(),
                );
            }
            motif_members = members;
        }

        // A gazetteer grown from the corpus, so multi-word mentions stay whole. Without it "Sootopolis City"
        // is indexed as two unrelated words and the engine believes it saw two things.
        let gazetteer = crate::emergent::mine_gazetteer(&docs, 2);

        for doc in &docs {
            let mut tokens: Vec<String> = Vec::new();
            let mut beliefs: Vec<(String, f32)> = Vec::new();
            let mut numeric_extra: Vec<(String, f64)> = Vec::new();
            let level = belief_for(doc);

            // whole-mention entities
            for mention in &gazetteer {
                if !crate::emergent::word_spans(doc, mention).is_empty() {
                    let tok = format!("entity/{}", crate::projector::slug(mention));
                    tokens.push(tok.clone());
                    beliefs.push((tok, level));
                }
            }
            // Relational roles with explicit polarity: + on the acting side, - on the receiving side, stored
            // as separate tags so a reversed relation has no tag to hide inside.
            for r in crate::emergent::relation_spans(doc, &gazetteer) {
                let verb = crate::projector::slug(&r.verb);
                for (role, who) in [("+", &r.actor), ("-", &r.target)] {
                    for tok in [
                        format!("rel/{verb}/{role}"),
                        format!("rel/{verb}/{role}/{}", crate::projector::slug(who)),
                    ] {
                        tokens.push(tok.clone());
                        beliefs.push((tok, level));
                    }
                }
            }
            // the motifs this situation participates in
            for (mi, members) in motif_members.iter().enumerate() {
                if members.iter().any(|t| crate::emergent::contains_term(doc, t)) {
                    if let Some(name) = motifs.get(mi) {
                        let tok = format!("motif/{}", crate::projector::slug(name));
                        tokens.push(tok.clone());
                        beliefs.push((tok, level));
                    }
                }
            }
            // temporal buckets, so a date is a set members can intersect on rather than an instant
            for (_, _, tok) in crate::emergent::temporal_spans(doc) {
                tokens.push(tok.clone());
                beliefs.push((tok, level));
            }
            // quantities carry their unit, so numeric predicates have a field to range over
            for (st, en, field) in crate::emergent::quantity_spans(doc) {
                let text = &doc[st..en];
                // the leading sign is part of the value
                let num: String = text
                    .chars()
                    .enumerate()
                    .take_while(|(i, c)| c.is_ascii_digit() || *c == '.' || (*i == 0 && *c == '-'))
                    .map(|(_, c)| c)
                    .collect();
                if let Ok(v) = num.parse::<f64>() {
                    let tok = format!("quantity/{field}");
                    tokens.push(tok.clone());
                    beliefs.push((tok, level));
                    numeric_extra.push((field, v));
                }
            }
            for (facet, terms) in &facets {
                for term in terms {
                    if crate::grow::contains_word(doc, term) {
                        let tok = format!("{facet}/{}", crate::projector::slug(term));
                        tokens.push(tok.clone());
                        beliefs.push((tok, level));
                    }
                }
            }
            // the epistemic state itself is a token, so it can be filtered on (paper dimension 5)
            tokens.push(
                match level {
                    l if l < 0.0 => "state/negated",
                    l if l < 1.0 => "state/hedged",
                    _ => "state/asserted",
                }
                .to_string(),
            );
            let display = vec![doc.chars().take(150).collect::<String>()];
            let mut nums = numbers_in(doc);
            nums.append(&mut numeric_extra);
            nums.dedup_by(|a, b| a.0 == b.0);
            corpus.add_situation_polar(tokens, display, nums, beliefs);
        }

        Paper { corpus, facets, gazetteer, motifs, doc_count: docs.len(), documents: docs }
    }

    /// Vocabulary + index summary: what was discovered and what got stored.
    pub fn summary(&self) -> String {
        let st = self.corpus.stats();
        serde_json::json!({
            "documents": self.doc_count,
            "situations": st.situations,
            "tokens": st.vocab,
            "facets": self.facets.iter().map(|(f, t)| serde_json::json!({
                "name": f, "terms": t, "wildcard": format!("{f}/*"),
            })).collect::<Vec<_>>(),
            "numeric_fields": st.numeric_fields,
        })
        .to_string()
    }

    /// The document-by-token incidence matrix — the picture of what a bitmap index actually is.
    pub fn incidence(&self, limit_tokens: usize) -> String {
        let mut rows: Vec<serde_json::Value> = Vec::new();
        for (facet, _) in &self.facets {
            for (tok, _n) in self.corpus.facet_tokens(facet, limit_tokens) {
                let out = self.corpus.query(&tok, usize::MAX);
                let docs: Vec<u32> = out.hits.iter().map(|h| h.sid).collect();
                rows.push(serde_json::json!({ "token": tok, "facet": facet, "documents": docs }));
            }
        }
        serde_json::json!({ "total_documents": self.doc_count, "rows": rows }).to_string()
    }

    /// Run an IKL s-expression: the compiled bitmap program.
    pub fn query(&self, ikl: &str) -> String {
        let lint = self.corpus.linter().lint(ikl);
        if !lint.ok {
            return serde_json::json!({
                "ok": false,
                "errors": lint.errors.iter().map(|e| e.message.clone()).collect::<Vec<_>>(),
            })
            .to_string();
        }
        let out = self.corpus.query(ikl, 200);
        serde_json::json!({
            "ok": true,
            "count": out.count,
            "documents": out.hits.iter().map(|h| h.sid).collect::<Vec<_>>(),
            "preview": out.hits.iter().take(3).map(|h| h.cells.join(" ")).collect::<Vec<_>>(),
        })
        .to_string()
    }

    /// Belief and plausibility for a token, from the polarity layer (§4).
    pub fn belief(&self, token: &str) -> String {
        let (bel, pl) = self.corpus.belief_interval(token);
        serde_json::json!({ "token": token, "belief": bel, "plausibility": pl,
                            "ignorance": (pl - bel).max(0.0) })
            .to_string()
    }

    /// Tokens that co-occur with this one — the neighbourhood an s-path walks.
    pub fn cooccurs(&self, token: &str, k: usize) -> String {
        self.corpus.cooccurs(token, k).to_string()
    }

    /// Shortest connection between two tokens through shared documents (§3).
    pub fn s_path(&self, a: &str, b: &str, s: usize) -> String {
        self.corpus.s_path(a, b, s).to_string()
    }

    /// Annotate every document for a displaCy-style viewer: the text split into ordered chunks, where a
    /// chunk carrying a `facet` is a term the vocabulary actually matched.
    ///
    /// Returns segments rather than offsets on purpose. Rust counts bytes and JavaScript indexes UTF-16 code
    /// units, so handing offsets across would render shifted text on any document containing an accented
    /// character — and these documents contain "Pokémon".
    pub fn annotate(&self) -> String {
        let docs: Vec<serde_json::Value> = self
            .documents
            .iter()
            .map(|doc| {
                // Three layers, most specific first: quantities, then whole multi-word mentions, then the
                // single facet words. Sorting by start and then by DESCENDING length means a longer, more
                // specific match always claims the span, so "Sootopolis City" is never split into two.
                let mut hits: Vec<(usize, usize, String)> = Vec::new();
                for (s, e, field) in crate::emergent::quantity_spans(doc) {
                    hits.push((s, e, format!("quantity:{field}")));
                }
                for (s, e, _) in crate::emergent::temporal_spans(doc) {
                    hits.push((s, e, "time".to_string()));
                }
                for mention in &self.gazetteer {
                    for (s, e) in crate::emergent::word_spans(doc, mention) {
                        hits.push((s, e, "entity".to_string()));
                    }
                }
                for (facet, terms) in &self.facets {
                    for term in terms {
                        for (s, e) in crate::emergent::word_spans(doc, term) {
                            hits.push((s, e, facet.clone()));
                        }
                    }
                }
                hits.sort_by(|a, b| a.0.cmp(&b.0).then((b.1 - b.0).cmp(&(a.1 - a.0))));

                let mut segments: Vec<serde_json::Value> = Vec::new();
                let mut cursor = 0usize;
                for (s, e, facet) in hits {
                    if s < cursor || e > doc.len() {
                        continue; // overlapping match already covered
                    }
                    if s > cursor {
                        segments.push(serde_json::json!({ "text": &doc[cursor..s] }));
                    }
                    segments.push(serde_json::json!({ "text": &doc[s..e], "facet": facet }));
                    cursor = e;
                }
                if cursor < doc.len() {
                    segments.push(serde_json::json!({ "text": &doc[cursor..] }));
                }
                serde_json::json!({ "segments": segments })
            })
            .collect();

        let mut legend: Vec<String> = vec!["entity".into(), "time".into(), "quantity".into()];
        legend.extend(self.facets.iter().map(|(f, _)| f.clone()));
        serde_json::json!({
            "documents": docs,
            "facets": legend,
            "gazetteer": self.gazetteer.iter().take(20).collect::<Vec<_>>(),
            "gazetteer_size": self.gazetteer.len(),
        })
        .to_string()
    }

    /// The Vocabulary Space **V**, grouped into the paper's six dimensions, each token carrying the set of
    /// situations whose bit is set.
    ///
    /// This is the picture of what a bitmap index holds: one row per vocabulary term, one column per
    /// situation. Grouping the rows by dimension shows why the taxonomy is uniform — an agent combines
    /// `entity/...` and `time/...` and `state/...` with the same three operators, without learning a schema.
    ///
    /// Dimensions the in-browser pipeline cannot populate are reported as empty with a stated reason rather
    /// than quietly omitted: relational roles need the span tagger, and latent motifs need SPLADE. Showing a
    /// dimension as present when nothing fills it would misrepresent what is running.
    pub fn dimensions(&self) -> String {
        let row = |token: &str| -> serde_json::Value {
            let out = self.corpus.query(token, usize::MAX);
            serde_json::json!({
                "token": token,
                "situations": out.hits.iter().map(|h| h.sid).collect::<Vec<_>>(),
                "count": out.count,
            })
        };
        let collect = |prefixes: &[&str], limit: usize| -> Vec<serde_json::Value> {
            let mut rows: Vec<serde_json::Value> = Vec::new();
            for p in prefixes {
                for (tok, _) in self.corpus.facet_tokens(p, limit) {
                    rows.push(row(&tok));
                }
            }
            rows
        };

        // the discovered category facets are entity-like: concrete kinds the corpus talks about
        let mut entity_prefixes: Vec<&str> = vec!["entity"];
        let facet_names: Vec<String> = self.facets.iter().map(|(f, _)| f.clone()).collect();
        for f in &facet_names {
            entity_prefixes.push(f.as_str());
        }

        let dims = serde_json::json!([
            {
                "n": "2.1.1", "name": "Entities & artifacts",
                "example": "org/toyota · artifact/battery_cell",
                "note": "concrete nouns: the things the corpus is about",
                "rows": collect(&entity_prefixes, 6),
                "source": "mined gazetteer + discovered categories",
            },
            {
                "n": "2.1.2", "name": "Relational roles",
                "example": "rel/supplies/+ · rel/supplies/-",
                "note": "who acted on what: + marks the actor, - the target, so direction is explicit",
                "rows": collect(&["rel"], 8),
                "source": "pattern extraction: mention, relation verb, mention — direction from word order",
            },
            {
                "n": "2.1.3", "name": "Spatial & temporal loci",
                "example": "time/2026/q3 · geo/apac/brisbane",
                "note": "when and where, bucketed so a date becomes a set that can be intersected",
                "rows": collect(&["time", "geo"], 8),
                "source": "extracted deterministically from the text",
            },
            {
                "n": "2.1.4", "name": "Quantities & tolerances",
                "example": "qty/temp/celsius/20_to_30",
                "note": "measurements quantised into range buckets, so a comparison becomes a set match",
                "rows": collect(&["quantity"], 8),
                "source": "number + unit extraction",
            },
            {
                "n": "2.1.5", "name": "Epistemic modifiers",
                "example": "state/negated · trend/cost/decrease",
                "note": "whether the corpus asserts, hedges or denies the fact",
                "rows": collect(&["state", "trend"], 6),
                "source": "cue detection over the text",
            },
            {
                "n": "2.1.6", "name": "Latent motifs",
                "example": "motif/hazard/thermal",
                "note": "implicit themes with no shared keyword, from sparse neural activations",
                "rows": collect(&["motif"], 6),
                "source": "Sinkhorn optimal transport over the co-occurrence geometry",
            },
        ]);

        serde_json::json!({
            "dimensions": dims,
            "total_situations": self.doc_count,
            "total_tokens": self.corpus.stats().vocab,
        })
        .to_string()
    }

    /// The reified hyperedge for one document: its situation id and every tag it asserts, grouped by the
    /// dimension the tag belongs to.
    ///
    /// This is the output of annotation, made visible. The highlighted spans in a document are not the end
    /// product — they are evidence for a single fact that binds all of them at once, and this returns that
    /// fact. Grouping by dimension shows that one situation carries entities, roles, time, quantities and
    /// epistemic state together, which is exactly what a binary edge cannot represent.
    pub fn situation(&self, index: usize) -> String {
        let i = index.min(self.doc_count.saturating_sub(1));
        let Some(doc) = self.documents.get(i) else {
            return serde_json::json!({ "error": "no such situation" }).to_string();
        };

        // Recover the tags for this situation by asking the index which of its tokens contain this sid.
        // Reading them back out of the index rather than recomputing them means the panel shows what was
        // actually stored, not a second opinion about what should have been.
        let mut groups: std::collections::BTreeMap<&str, Vec<String>> = std::collections::BTreeMap::new();
        let dim_of = |tok: &str| -> &'static str {
            let stem = tok.split('/').next().unwrap_or("");
            match stem {
                "entity" => "entities",
                "rel" => "roles",
                "time" | "geo" => "loci",
                "quantity" => "quantities",
                "state" | "trend" => "epistemic",
                "motif" => "motifs",
                _ => "categories",
            }
        };

        let mut prefixes: Vec<String> =
            ["entity", "rel", "time", "geo", "quantity", "state", "trend", "motif"]
                .iter()
                .map(|s| s.to_string())
                .collect();
        prefixes.extend(self.facets.iter().map(|(f, _)| f.clone()));

        for p in &prefixes {
            for (tok, _) in self.corpus.facet_tokens(p, 200) {
                let out = self.corpus.query(&tok, usize::MAX);
                if out.hits.iter().any(|h| h.sid as usize == i) {
                    groups.entry(dim_of(&tok)).or_default().push(tok);
                }
            }
        }

        // the relations again, so the panel can show who acted on whom rather than only the bare role tags
        let rels: Vec<serde_json::Value> = crate::emergent::relation_spans(doc, &self.gazetteer)
            .into_iter()
            .map(|r| serde_json::json!({ "verb": r.verb, "actor": r.actor, "target": r.target }))
            .collect();

        let total: usize = groups.values().map(|v| v.len()).sum();
        serde_json::json!({
            "sid": i,
            "of": self.doc_count,
            "groups": groups,
            "relations": rels,
            "total_tags": total,
        })
        .to_string()
    }

    /// Plan a free-text question into slots, then run the whole chain.
    ///
    /// The planner here is deliberately dumb and deterministic: it finds negation cues and assigns the
    /// vocabulary terms it can link to either the include or the exclude slot. A tuned needle3 does this job
    /// properly — it reads intent rather than keywords — but it cannot run in a browser, so a page must either
    /// ship recorded model output or plan locally. This is the local option, and it is labelled as such
    /// wherever it appears.
    ///
    /// The important property is that swapping the planner changes nothing downstream. Expansion, type
    /// checking, compilation and traversal are identical whichever produced the slots, because the planner's
    /// only output is which constraints apply.
    pub fn plan_text(&self, question: &str, s_threshold: usize) -> String {
        const NEG_CUES: &[&str] = &["not", "without", "excluding", "except", "exclude", "no", "never"];
        let lower = question.to_lowercase();
        let words: Vec<&str> = lower.split(|c: char| !c.is_alphanumeric() && c != '-').filter(|w| !w.is_empty()).collect();

        // vocabulary the question may refer to: facet names, plus the epistemic states
        let mut include: Vec<String> = Vec::new();
        let mut exclude: Vec<String> = Vec::new();
        let mut categories: Vec<String> = Vec::new();

        // a cue puts the NEXT few recognised terms into exclude, until the clause ends
        let mut negating_for = 0usize;
        for w in &words {
            if NEG_CUES.contains(w) {
                // a cue governs the rest of its clause, approximated as the next three terms
                negating_for = 3;
                continue;
            }
            let facet = self.facets.iter().find(|(f, _)| f == w).map(|(f, _)| f.clone());
            let state = match *w {
                "negated" | "denied" | "refuted" => Some("negated".to_string()),
                "hedged" | "provisional" | "uncertain" => Some("hedged".to_string()),
                "asserted" | "stated" | "confirmed" => Some("asserted".to_string()),
                _ => None,
            };
            if let Some(st) = state {
                if negating_for > 0 {
                    exclude.push(st);
                    negating_for -= 1;
                } else {
                    include.push(st);
                }
                continue;
            }
            if let Some(f) = facet {
                if negating_for > 0 {
                    exclude.push(f);
                    negating_for -= 1;
                } else {
                    categories.push(f);
                }
                continue;
            }
            // an unrecognised word is not a constraint; it also does not consume the cue
            if negating_for > 0 && self.corpus.entity_link(w).is_empty() {
                continue;
            }
            if !self.corpus.entity_link(w).is_empty() {
                if negating_for > 0 {
                    exclude.push(w.to_string());
                    negating_for -= 1;
                } else {
                    include.push(w.to_string());
                }
            }
        }

        let plan = serde_json::json!({
            "include": include, "exclude": exclude, "any_of": Vec::<String>::new(), "categories": categories,
        });
        if include.is_empty() && exclude.is_empty() && categories.is_empty() {
            return serde_json::json!({
                "ok": false,
                "stage": "plan",
                "plan": plan,
                "reason": "nothing in the question matched this corpus vocabulary",
                "available": self.facets.iter().map(|(f, _)| f.clone()).collect::<Vec<_>>(),
            })
            .to_string();
        }

        let mut result: serde_json::Value = serde_json::from_str(&self.plan_and_traverse(
            &serde_json::to_string(&include).unwrap_or_default(),
            &serde_json::to_string(&exclude).unwrap_or_default(),
            "[]",
            &serde_json::to_string(&categories).unwrap_or_default(),
            s_threshold,
        ))
        .unwrap_or_default();
        result["plan"] = plan;
        result["question"] = serde_json::json!(question);
        result.to_string()
    }

    /// The full execution order: a plan of slots becomes an expression, the expression is type-checked and
    /// compiled to a row filter, and only then do the higher-order tools run — inside the filtered set.
    ///
    /// This is the paper's predicate pushdown, and the order is a requirement rather than a preference. Running
    /// traversal first means walking a structure whose size is the whole corpus; running it after the filter
    /// means walking whatever survived. The planner never touches bitmaps and never decides what is
    /// reachable — it only proposes which constraints apply.
    ///
    /// Returns each stage separately so a caller can show the row count collapsing before any traversal
    /// happens, and the neighbourhood computed only within the survivors.
    pub fn plan_and_traverse(
        &self,
        include: &str,
        exclude: &str,
        any_of: &str,
        categories: &str,
        s_threshold: usize,
    ) -> String {
        // stage 1-3: link the slots, compose the expression, lint it, run it
        let composed: serde_json::Value =
            serde_json::from_str(&self.compose(include, exclude, any_of, categories)).unwrap_or_default();
        if !composed["ok"].as_bool().unwrap_or(false) {
            return serde_json::json!({
                "ok": false,
                "stage": "check",
                "reason": composed["reason"].clone(),
                "unresolved": composed["unresolved"].clone(),
                "available": composed["available"].clone(),
            })
            .to_string();
        }
        let ikl = composed["ikl"].as_str().unwrap_or("").to_string();

        // stage 4: the surviving rows
        let out = self.corpus.query(&ikl, usize::MAX);
        let survivors: std::collections::BTreeSet<usize> =
            out.hits.iter().map(|h| h.sid as usize).collect();

        // stage 5: higher-order work, restricted to the survivors. Tags are collected only from those rows,
        // so the traversal cannot leave the filtered set even in principle.
        let mut tags_of: std::collections::BTreeMap<usize, std::collections::BTreeSet<String>> =
            std::collections::BTreeMap::new();
        let mut prefixes: Vec<String> =
            ["entity", "rel", "time", "geo", "quantity", "state", "motif"].iter().map(|x| x.to_string()).collect();
        prefixes.extend(self.facets.iter().map(|(f, _)| f.clone()));
        for p in &prefixes {
            for (tok, _) in self.corpus.facet_tokens(p, 300) {
                for h in self.corpus.query(&tok, usize::MAX).hits {
                    let sid = h.sid as usize;
                    if survivors.contains(&sid) {
                        tags_of.entry(sid).or_default().insert(tok.clone());
                    }
                }
            }
        }

        let ids: Vec<usize> = survivors.iter().copied().collect();
        let mut edges: Vec<serde_json::Value> = Vec::new();
        for i in 0..ids.len() {
            for j in (i + 1)..ids.len() {
                let (a, b) = (ids[i], ids[j]);
                let empty = std::collections::BTreeSet::new();
                let ta = tags_of.get(&a).unwrap_or(&empty);
                let tb = tags_of.get(&b).unwrap_or(&empty);
                let shared: Vec<String> = ta.intersection(tb).cloned().collect();
                if shared.len() >= s_threshold.max(1) {
                    edges.push(serde_json::json!({
                        "a": a, "b": b, "shared": shared.len(),
                        "via": shared.iter().take(3).collect::<Vec<_>>(),
                    }));
                }
            }
        }

        serde_json::json!({
            "ok": true,
            "ikl": ikl,
            "total_situations": self.doc_count,
            "survivors": ids,
            "survivor_count": ids.len(),
            "s": s_threshold.max(1),
            "edges": edges,
            // the comparison that justifies the ordering
            "pairs_if_unfiltered": self.doc_count * (self.doc_count.saturating_sub(1)) / 2,
            "pairs_examined": ids.len() * (ids.len().saturating_sub(1)) / 2,
        })
        .to_string()
    }

    /// Both topologies of the incidence matrix, swept over the overlap threshold `s`.
    ///
    /// The index is an incidence matrix: rows are tags, columns are situations. That single structure can be
    /// read two ways, and both are useful:
    ///
    /// * **primal** — situations are nodes, joined when they share at least `s` tags. "Which events are
    ///   related?"
    /// * **dual** — tags are nodes, joined when they co-occur in at least `s` situations. "Which concepts
    ///   belong together?"
    ///
    /// Nothing is rebuilt to switch between them; it is the same matrix transposed, which is why a hypergraph
    /// engine gets the dual for free where a pairwise graph would need a second index.
    ///
    /// Sweeping `s` is the **s-filtration**. At `s = 1` a single shared tag is enough and almost everything is
    /// connected, which is the regime where naive graph walks drift. Raising `s` demands more agreement per
    /// step and the graph thins toward only the genuinely related. Reporting connected components alongside
    /// edges shows structure appearing as the noise is removed.
    pub fn topology(&self, max_s: usize) -> String {
        // tags per situation (primal rows) and situations per tag (dual rows)
        let mut tags_of: Vec<std::collections::BTreeSet<String>> =
            vec![std::collections::BTreeSet::new(); self.doc_count];
        let mut sits_of: std::collections::BTreeMap<String, std::collections::BTreeSet<usize>> =
            std::collections::BTreeMap::new();

        let mut prefixes: Vec<String> =
            ["entity", "rel", "time", "geo", "quantity", "state", "motif"].iter().map(|s| s.to_string()).collect();
        prefixes.extend(self.facets.iter().map(|(f, _)| f.clone()));
        for p in &prefixes {
            for (tok, _) in self.corpus.facet_tokens(p, 300) {
                let out = self.corpus.query(&tok, usize::MAX);
                let sids: std::collections::BTreeSet<usize> =
                    out.hits.iter().map(|h| h.sid as usize).collect();
                for &i in &sids {
                    if i < self.doc_count {
                        tags_of[i].insert(tok.clone());
                    }
                }
                sits_of.insert(tok, sids);
            }
        }

        // count edges and connected components of a graph given an adjacency predicate
        fn components(n: usize, edges: &[(usize, usize)]) -> usize {
            let mut parent: Vec<usize> = (0..n).collect();
            fn find(p: &mut Vec<usize>, x: usize) -> usize {
                if p[x] != x {
                    let r = find(p, p[x]);
                    p[x] = r;
                }
                p[x]
            }
            for &(a, b) in edges {
                let (ra, rb) = (find(&mut parent, a), find(&mut parent, b));
                if ra != rb {
                    parent[ra] = rb;
                }
            }
            let mut seen = std::collections::BTreeSet::new();
            for i in 0..n {
                let r = find(&mut parent, i);
                seen.insert(r);
            }
            seen.len()
        }

        let tag_names: Vec<String> = sits_of.keys().cloned().collect();
        let mut levels: Vec<serde_json::Value> = Vec::new();

        for s_thr in 1..=max_s.clamp(1, 8) {
            // primal: situations sharing >= s tags
            let mut p_edges: Vec<(usize, usize)> = Vec::new();
            for i in 0..self.doc_count {
                for j in (i + 1)..self.doc_count {
                    if tags_of[i].intersection(&tags_of[j]).count() >= s_thr {
                        p_edges.push((i, j));
                    }
                }
            }
            // dual: tags co-occurring in >= s situations
            let mut d_edges: Vec<(usize, usize)> = Vec::new();
            for a in 0..tag_names.len() {
                for b in (a + 1)..tag_names.len() {
                    let (sa, sb) = (&sits_of[&tag_names[a]], &sits_of[&tag_names[b]]);
                    if sa.intersection(sb).count() >= s_thr {
                        d_edges.push((a, b));
                    }
                }
            }

            levels.push(serde_json::json!({
                "s": s_thr,
                "primal": {
                    "nodes": self.doc_count,
                    "edges": p_edges.len(),
                    "components": components(self.doc_count, &p_edges),
                    // a small sample so a viewer can draw the graph at this level
                    "sample": p_edges.iter().take(120).map(|(a, b)| vec![a, b]).collect::<Vec<_>>(),
                },
                "dual": {
                    "nodes": tag_names.len(),
                    "edges": d_edges.len(),
                    "components": components(tag_names.len(), &d_edges),
                    "sample": d_edges.iter().take(120).map(|(a, b)| vec![a, b]).collect::<Vec<_>>(),
                },
            }));
        }

        serde_json::json!({
            "situations": self.doc_count,
            "tags": tag_names.len(),
            "tag_names": tag_names,
            "levels": levels,
        })
        .to_string()
    }

    /// Link a free-text span to real vocabulary tokens.
    ///
    /// This is the join that makes a small extraction model useful. A 121M tool-calling model grounds its
    /// arguments by copying spans out of the question — it returns "sinnoh region", not `sinnoh/region`.
    /// Mapping a span onto the vocabulary is a deterministic lookup, so the engine does it rather than asking
    /// the model to memorise the token set.
    pub fn link(&self, text: &str) -> String {
        serde_json::json!(self.corpus.entity_link(text)).to_string()
    }

    /// Compose an s-expression from extracted slots, then lint and run it.
    ///
    /// The model supplies intent as slots; the ALGEBRA is built here, in code. That split is the point of the
    /// paper: a language model is unreliable at nesting brackets correctly but good at spotting which phrase
    /// is a requirement and which is an exclusion, so composition stays deterministic and the result is
    /// auditable.
    pub fn compose(&self, include: &str, exclude: &str, any_of: &str, categories: &str) -> String {
        let parse = |s: &str| -> Vec<String> {
            serde_json::from_str::<Vec<String>>(s).unwrap_or_default()
        };
        // every span is linked to real tokens; a span that links to nothing is reported, not guessed at
        let mut parts: Vec<String> = Vec::new();
        let mut unresolved: Vec<String> = Vec::new();
        let mut resolve = |spans: Vec<String>, out: &mut Vec<String>| {
            for span in spans {
                let linked: Vec<String> = self.corpus.entity_link(&span);
                if linked.is_empty() {
                    unresolved.push(span);
                } else {
                    out.extend(linked);
                }
            }
        };

        let (mut inc, mut exc, mut anyv) = (Vec::new(), Vec::new(), Vec::new());
        resolve(parse(include), &mut inc);
        resolve(parse(exclude), &mut exc);
        resolve(parse(any_of), &mut anyv);
        for c in parse(categories) {
            let name = c.split('/').next().unwrap_or(&c).trim().to_lowercase();
            if self.facets.iter().any(|(f, _)| *f == name) {
                parts.push(format!("{name}/*"));
            } else {
                unresolved.push(c);
            }
        }

        parts.extend(inc);
        if !anyv.is_empty() {
            parts.push(format!("(or {})", anyv.join(" ")));
        }
        for e in &exc {
            parts.push(format!("(not {e})"));
        }

        if parts.is_empty() {
            return serde_json::json!({
                "ok": false,
                "ikl": null,
                "unresolved": unresolved,
                "reason": "nothing in the question could be linked to this corpus",
                "available": self.facets.iter().map(|(f, _)| f.clone()).collect::<Vec<_>>(),
            })
            .to_string();
        }

        // ANY unlinkable slot must refuse. The query is a conjunction of constraints, so dropping one -
        // positive or negative - removes a filter and returns MORE documents than were asked for. My first
        // version only guarded exclusions, on the reasoning that a missing positive term merely narrows the
        // result; that was simply wrong, and a test showed it: dropping the unlinked term `species` turned
        // "species and not negated" into "not negated", widening 8 documents to 19. Silently answering a
        // weaker question than the one asked is the failure this whole system exists to prevent.
        if !unresolved.is_empty() {
            return serde_json::json!({
                "ok": false,
                "ikl": null,
                "unresolved": unresolved,
                "reason": "some terms could not be linked to this corpus; answering without them would \
                           return more documents than were asked for",
                "available": self.facets.iter().map(|(f, _)| f.clone()).collect::<Vec<_>>(),
            })
            .to_string();
        }

        let ikl = if parts.len() == 1 { parts[0].clone() } else { format!("(and {})", parts.join(" ")) };
        let lint = self.corpus.linter().lint(&ikl);
        let out = if lint.ok { Some(self.corpus.query(&ikl, 5)) } else { None };

        serde_json::json!({
            "ok": lint.ok,
            "ikl": ikl,
            "unresolved": unresolved,
            "errors": lint.errors.iter().map(|e| e.message.clone()).collect::<Vec<_>>(),
            "count": out.as_ref().map(|o| o.count),
            "available": self.facets.iter().map(|(f, _)| f.clone()).collect::<Vec<_>>(),
        })
        .to_string()
    }

    /// Every token in the index, so a page can offer real choices instead of invented ones.
    pub fn tokens(&self) -> String {
        let mut all: Vec<String> = Vec::new();
        for (facet, _) in &self.facets {
            for (tok, _) in self.corpus.facet_tokens(facet, 200) {
                all.push(tok);
            }
        }
        serde_json::json!(all).to_string()
    }
}

/// Version string, so a stale cached `.wasm` is visible rather than silently wrong.
#[wasm_bindgen]
pub fn version() -> String {
    format!("steeldb {} (wasm, step 0)", env!("CARGO_PKG_VERSION"))
}