hypersteeldb 0.5.5

//! **Step 3: ontology growth** — the MDL-style split criterion from `design.py`.
//!
//! The seed spec (step 0) is deliberately minimal ("3-4 SEED entity facets … growth adds more later").
//! Growth then asks an agent for one candidate facet covering *what the existing facets leave
//! unexplained*, and keeps it **only if it earns its place**:
//!
//! ```text
//!   gain = coverage × (1 − maxcos)         (Collectively-Exhaustive × Mutually-Exclusive)
//!   keep ⟺ gain ≥ threshold
//! ```
//!
//! The reference measures orthogonality between *trained SPLADE decoder weights*, which means finetuning
//! a head per candidate before you can score it. That is the expensive part, and it is not necessary to
//! decide the question the gate asks: *does this facet pick out corpus content the others miss?* Here each
//! facet is given a **detector** — its name plus the example surfaces the proposing agent supplied — and
//! the candidate is scored on the real corpus incidence those detectors produce ([`crate::mece`]).
//! Cheap, deterministic, and measured on the corpus as it actually is rather than on model weights.
//!
//! The honest limitation of that substitution: a detector built from a handful of example surfaces
//! under-measures a facet whose vocabulary is broad, so `gain` is a lower bound. A candidate that clears
//! the threshold has definitely earned its place; one that narrowly fails might have earned it with a
//! trained head.

use crate::mece;
use crate::vocabulary::{EntityFacet, VocabularySpace};
use crate::{InfonIndex, RoarPostings};
use serde::{Deserialize, Serialize};
use std::collections::HashMap;

/// A growth candidate — mirrors `design.py::propose_candidate`'s schema.
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct Candidate {
    pub name: String,
    /// the existing facet this specialises — the source of the URI hierarchy
    #[serde(default)]
    pub parent: Option<String>,
    #[serde(default)]
    pub description: String,
    #[serde(default)]
    pub examples: Vec<String>,
    #[serde(default = "yes")]
    pub worth_adding: bool,
}

fn yes() -> bool {
    true
}

/// One decision in the growth log — kept for the registry so a spec's provenance is auditable.
#[derive(Debug, Clone, Serialize)]
pub struct GrowEvent {
    pub round: usize,
    pub name: String,
    pub parent: Option<String>,
    pub coverage: f64,
    pub maxcos: f64,
    pub nearest: String,
    pub gain: f64,
    pub threshold: f64,
    pub kept: bool,
    pub reason: String,
}

/// Detector terms for a facet: its own name plus any example surfaces, lowercased. This is what stands in
/// for a trained retrieval head when scoring the MECE gate.
fn detectors(f: &EntityFacet) -> Vec<String> {
    let mut v = vec![f.name.replace(['-', '_'], " ")];
    v.extend(f.examples.iter().map(|e| e.to_lowercase()));
    v.into_iter().map(|s| s.trim().to_lowercase()).filter(|s| s.len() >= 3).collect()
}

/// Project sampled documents into a facet-incidence index using detector matching: one situation per
/// document, one `facet/term` token per detector hit. Word-boundary matched so short names don't fire
/// inside longer words.
fn incidence_index(spec: &VocabularySpace, docs: &[String]) -> InfonIndex<RoarPostings> {
    let mut raw: HashMap<String, Vec<u32>> = HashMap::new();
    for (sid, doc) in docs.iter().enumerate() {
        let low = doc.to_lowercase();
        for f in &spec.entity_facets {
            for term in detectors(f) {
                if contains_word(&low, &term) {
                    raw.entry(format!("{}/{}", f.name, crate::projector::slug(&term))).or_default().push(sid as u32);
                }
            }
        }
    }
    for v in raw.values_mut() {
        v.sort_unstable();
        v.dedup();
    }
    InfonIndex::from_postings(raw, docs.len() as u32)
}

/// Word-boundary containment (so `org` doesn't match inside `organic`).
/// Whole-word containment. A substring test would match "it" inside "submitted", which silently
/// corrupts both training labels and projected tokens.
pub fn contains_word(hay: &str, needle: &str) -> bool {
    let mut from = 0usize;
    while let Some(rel) = hay[from..].find(needle) {
        let s = from + rel;
        let e = s + needle.len();
        let before_ok = s == 0 || !hay[..s].chars().next_back().map(|c| c.is_alphanumeric()).unwrap_or(false);
        let after_ok = e >= hay.len() || !hay[e..].chars().next().map(|c| c.is_alphanumeric()).unwrap_or(false);
        if before_ok && after_ok {
            return true;
        }
        from = s + needle.len().max(1);
        if from >= hay.len() {
            break;
        }
    }
    false
}

/// Score a candidate against the current spec on a document sample. Returns the MECE numbers for the
/// candidate facet, or `None` when its detectors never fire (nothing to measure).
pub fn score_candidate(spec: &VocabularySpace, docs: &[String], cand: &Candidate) -> Option<mece::FacetScore> {
    score_candidate_full(spec, docs, cand).map(|(s, _)| s)
}

/// Score a candidate and also report **parent duplication**: `(candidate_coverage / parent_coverage,
/// maxcos_against_parent)`. A specialisation must be strictly narrower than the facet it refines — if it
/// reproduces the parent's coverage *and* its incidence, it is a rename, not a new node type.
pub fn score_candidate_full(spec: &VocabularySpace, docs: &[String], cand: &Candidate) -> Option<(mece::FacetScore, Option<(f64, f64)>)> {
    let mut trial = spec.clone();
    trial.entity_facets.push(EntityFacet {
        name: cand.name.clone(),
        parent: cand.parent.clone(),
        description: cand.description.clone(),
        examples: cand.examples.clone(),
        structural: false,
    });
    let ix = incidence_index(&trial, docs);
    // A candidate declares a `parent` it SPECIALISES, so overlap with that parent (and its ancestors) is by
    // design, not redundancy — the reference gates a new head against its siblings. Excluding the ancestor
    // chain from the comparison is what keeps a legitimate child facet from being scored as a duplicate of
    // the thing it refines.
    let mut excluded: Vec<String> = vec!["src".to_string()];
    if let Some(p) = &cand.parent {
        excluded.push(p.clone());
        excluded.extend(trial.ancestors(p));
    }
    let skip: Vec<&str> = excluded.iter().map(|s| s.as_str()).collect();
    let rep = mece::report(&ix, &skip);
    let score = rep.facets.into_iter().find(|f| f.facet == cand.name)?;

    // Parent-duplication check. Measured on an index containing ONLY the candidate and its parent, so the
    // reported `maxcos` is necessarily against the parent — relying on the global `nearest` would hide
    // parent duplication whenever some unrelated facet happened to be closer.
    let dup = cand.parent.as_ref().and_then(|p| {
        let parent_facet = trial.entity_facets.iter().find(|f| f.name == *p)?.clone();
        let cand_facet = trial.entity_facets.iter().find(|f| f.name == cand.name)?.clone();
        let pair_spec = VocabularySpace {
            version: trial.version,
            corpus: trial.corpus.clone(),
            entity_facets: vec![parent_facet, cand_facet],
            relation_facets: Vec::new(),
            gazetteer: Vec::new(),
            metrics: None,
        };
        let pix = incidence_index(&pair_spec, docs);
        let rep2 = mece::report(&pix, &["src"]);
        let c = rep2.facets.iter().find(|f| f.facet == cand.name)?;
        let par = rep2.facets.iter().find(|f| f.facet == *p)?;
        let ratio = if par.coverage > 0.0 { c.coverage / par.coverage } else { 0.0 };
        Some((ratio, c.maxcos))
    });
    Some((score, dup))
}

/// Apply the gate to a scored candidate: keep iff `gain ≥ threshold`, the candidate is wanted, its parent
/// exists, and its name isn't already taken.
pub fn gate(spec: &VocabularySpace, cand: &Candidate, score: Option<&mece::FacetScore>, threshold: f64, round: usize) -> GrowEvent {
    gate_full(spec, cand, score, None, threshold, round)
}

/// Full gate including the parent-duplication signal from [`score_candidate_full`].
pub fn gate_full(
    spec: &VocabularySpace,
    cand: &Candidate,
    score: Option<&mece::FacetScore>,
    parent_dup: Option<(f64, f64)>,
    threshold: f64,
    round: usize,
) -> GrowEvent {
    let s = score.cloned().unwrap_or(mece::FacetScore {
        facet: cand.name.clone(),
        tokens: 0,
        coverage: 0.0,
        maxcos: 1.0,
        nearest: String::new(),
        gain: 0.0,
    });
    let (kept, reason) = if !cand.worth_adding {
        (false, "agent reported the corpus already covered".to_string())
    } else if cand.name.trim().is_empty() {
        (false, "empty name".to_string())
    } else if spec.has_entity_facet(&cand.name) {
        (false, format!("facet '{}' already declared", cand.name))
    } else if cand.parent.as_deref().map(|p| !spec.has_entity_facet(p)).unwrap_or(false) {
        (false, format!("parent '{}' is not an existing facet", cand.parent.clone().unwrap_or_default()))
    } else if s.tokens == 0 {
        (false, "detectors never fired on the sample".to_string())
    } else if parent_dup.map(|(ratio, cos)| cos > 0.95 || (ratio > 0.9 && cos > 0.9)).unwrap_or(false) {
        let (ratio, cos) = parent_dup.unwrap();
        (false, format!("duplicates its parent '{}' (cos {cos:.2}, coverage ratio {ratio:.2}) — a specialisation must add vocabulary, not restate the parent", cand.parent.clone().unwrap_or_default()))
    } else if s.gain < threshold {
        (false, format!("gain {:.3} < threshold {:.3} (coverage {:.3}, maxcos {:.3} vs '{}')", s.gain, threshold, s.coverage, s.maxcos, s.nearest))
    } else {
        (true, format!("gain {:.3} ≥ {:.3}", s.gain, threshold))
    };
    GrowEvent {
        round,
        name: cand.name.clone(),
        parent: cand.parent.clone(),
        coverage: s.coverage,
        maxcos: s.maxcos,
        nearest: s.nearest,
        gain: s.gain,
        threshold,
        kept,
        reason,
    }
}

/// Add a kept candidate to the spec.
pub fn adopt(spec: &mut VocabularySpace, cand: &Candidate) {
    spec.entity_facets.push(EntityFacet {
        name: cand.name.clone(),
        parent: cand.parent.clone(),
        description: cand.description.clone(),
        examples: cand.examples.clone(),
        structural: false,
    });
}

/// Run the growth loop: propose → score → gate → adopt, stopping when a round keeps nothing (the
/// reference's "Else stop"). Returns the grown spec and the full decision log.
#[cfg(feature = "agent")]
pub async fn grow(
    provider: &dyn crate::agent::provider::LlmProvider,
    spec: &VocabularySpace,
    docs: &[String],
    rounds: usize,
    threshold: f64,
) -> (VocabularySpace, Vec<GrowEvent>) {
    use crate::agent::types::{Msg, ToolSpec};
    let mut spec = spec.clone();
    let mut log: Vec<GrowEvent> = Vec::new();
    let tools = vec![ToolSpec {
        name: "emit_candidate".into(),
        description: "Emit one candidate facet that specialises an existing facet.".into(),
        schema: crate::vocabulary::candidate_schema(),
    }];
    for round in 1..=rounds {
        let existing: Vec<String> = spec
            .taggable_facets()
            .iter()
            .map(|f| match &f.parent {
                Some(p) => format!("{} (parent {p})", f.name),
                None => f.name.clone(),
            })
            .collect();
        let sample: Vec<&str> = docs.iter().take(10).map(|s| s.as_str()).collect();
        let prompt = format!(
            "Existing facets: {}\nEach new facet specialises one of these (set 'parent').\n\nCorpus sample:\n{}",
            existing.join(", "),
            sample.join("\n---\n")
        );
        let turn = match provider.chat(crate::vocabulary::CANDIDATE_SYSTEM, &[Msg::user_text(prompt)], &tools).await {
            Ok(t) => t,
            Err(e) => {
                log.push(GrowEvent {
                    round,
                    name: String::new(),
                    parent: None,
                    coverage: 0.0,
                    maxcos: 1.0,
                    nearest: String::new(),
                    gain: 0.0,
                    threshold,
                    kept: false,
                    reason: format!("provider error: {e}"),
                });
                break;
            }
        };
        let payload = turn
            .tool_uses
            .first()
            .map(|(_, _, v)| v.clone())
            .or_else(|| crate::vocabulary::extract_json(&turn.text));
        let Some(v) = payload else { break };
        let mut cand: Candidate = match serde_json::from_value(v) {
            Ok(c) => c,
            Err(_) => break,
        };
        cand.name = crate::projector::slug(&cand.name);
        cand.parent = cand.parent.map(|p| crate::projector::slug(&p)).filter(|p| !p.is_empty());

        let scored = score_candidate_full(&spec, docs, &cand);
        let (score, dup) = match &scored {
            Some((s, d)) => (Some(s), *d),
            None => (None, None),
        };
        let ev = gate_full(&spec, &cand, score, dup, threshold, round);
        let kept = ev.kept;
        log.push(ev);
        if kept {
            adopt(&mut spec, &cand);
        } else {
            break; // the reference stops as soon as a round earns nothing
        }
    }
    spec.metrics = Some(serde_json::json!({
        "source": "grow",
        "hierarchy": spec.entity_facets.iter().any(|f| f.parent.is_some()),
        "rounds": log.len(),
        "kept": log.iter().filter(|e| e.kept).count(),
    }));
    (spec, log)
}

#[cfg(test)]
mod tests {
    use super::*;
    use crate::vocabulary::RelationFacet;

    fn spec() -> VocabularySpace {
        VocabularySpace {
            version: 1,
            corpus: "defence".into(),
            entity_facets: vec![
                EntityFacet { name: "org".into(), parent: None, description: "companies".into(), examples: vec!["Boeing".into(), "Airbus".into()], structural: false },
                EntityFacet { name: "system".into(), parent: None, description: "platforms".into(), examples: vec!["drone".into(), "radar".into()], structural: false },
            ],
            relation_facets: vec![RelationFacet { name: "develops".into(), head: "org".into(), tail: "system".into() }],
            gazetteer: vec![],
            metrics: None,
        }
    }

    fn docs() -> Vec<String> {
        vec![
            "Boeing builds a drone with new radar and a lithium battery pack.".into(),
            "Airbus tested the radar under a thermal battery fault.".into(),
            "A drone carried a battery to altitude; Boeing observed.".into(),
            "Airbus and Boeing both use radar.".into(),
        ]
    }

    #[test]
    fn a_novel_facet_earns_its_place() {
        let s = spec();
        let d = docs();
        // "battery" appears in 3/4 docs and is not what org/system detect → real gain
        let cand = Candidate { name: "battery".into(), parent: Some("system".into()), description: "cells".into(), examples: vec!["battery".into()], worth_adding: true };
        let score = score_candidate(&s, &d, &cand).expect("scored");
        eprintln!("candidate score: {score:?}");
        assert!(score.coverage > 0.5, "battery covers most docs: {score:?}");
        let ev = gate(&s, &cand, Some(&score), 0.1, 1);
        assert!(ev.kept, "{}", ev.reason);
        assert_eq!(ev.parent.as_deref(), Some("system"));
    }

    #[test]
    fn a_child_facet_is_not_penalised_for_overlapping_its_parent() {
        // `battery` specialises `system`; its detectors necessarily co-occur with system's. Judged against
        // siblings (not the parent) it must still be able to earn its place.
        let s = spec();
        let d = docs();
        let child = Candidate { name: "battery".into(), parent: Some("system".into()), description: "cells".into(), examples: vec!["battery".into()], worth_adding: true };
        let scored = score_candidate(&s, &d, &child).expect("scored");
        eprintln!("child score (parent excluded): {scored:?}");
        assert_ne!(scored.nearest, "system", "the parent must be excluded from the redundancy comparison");
        assert!(gate(&s, &child, Some(&scored), 0.1, 1).kept);
    }

    #[test]
    fn a_redundant_facet_is_rejected() {
        let s = spec();
        let d = docs();
        // a facet whose detectors duplicate `system`'s ("radar") must be squeezed out by maxcos
        let cand = Candidate { name: "sensor".into(), parent: Some("system".into()), description: "dupe".into(), examples: vec!["radar".into()], worth_adding: true };
        let (score, dup) = score_candidate_full(&s, &d, &cand).expect("scored");
        eprintln!("redundant score: {score:?} parent_dup={dup:?}");
        let ev = gate_full(&s, &cand, Some(&score), dup, 0.1, 1);
        assert!(!ev.kept, "a facet that merely renames its parent must be rejected");
        assert!(ev.reason.contains("duplicates its parent") || ev.reason.contains("gain"), "{}", ev.reason);
    }

    #[test]
    fn gate_enforces_structural_invariants() {
        let s = spec();
        let d = docs();
        let mk = |name: &str, parent: Option<&str>, worth: bool| Candidate {
            name: name.into(),
            parent: parent.map(String::from),
            description: String::new(),
            examples: vec!["battery".into()],
            worth_adding: worth,
        };
        // agent says stop
        assert!(!gate(&s, &mk("battery", Some("system"), false), None, 0.1, 1).kept);
        // duplicate name
        assert!(!gate(&s, &mk("org", Some("system"), true), None, 0.1, 1).kept);
        // dangling parent
        let c = mk("battery", Some("nonexistent"), true);
        let ev = gate(&s, &c, score_candidate(&s, &d, &c).as_ref(), 0.1, 1);
        assert!(!ev.kept && ev.reason.contains("parent"), "{}", ev.reason);
        // detectors that never fire
        let c2 = Candidate { name: "quantum".into(), parent: Some("system".into()), description: String::new(), examples: vec!["tachyon".into()], worth_adding: true };
        let ev2 = gate(&s, &c2, score_candidate(&s, &d, &c2).as_ref(), 0.1, 1);
        assert!(!ev2.kept, "{}", ev2.reason);
    }

    #[test]
    fn adopt_extends_the_hierarchy() {
        let mut s = spec();
        let cand = Candidate { name: "battery".into(), parent: Some("system".into()), description: "cells".into(), examples: vec![], worth_adding: true };
        adopt(&mut s, &cand);
        assert_eq!(s.facet_path("battery"), "system/battery");
        assert!(s.valid_prefixes().contains(&"system/battery".to_string()));
        assert!(s.validate().is_ok());
    }

    #[test]
    fn word_boundary_detectors() {
        assert!(contains_word("a battery pack", "battery"));
        assert!(!contains_word("organic material", "org"));
        assert!(contains_word("the org chart", "org"));
    }
}