//! **Step 0: the facet spec** — the corpus's Vocabulary Space `V`, discovered before ingest and
//! persisted beside the data. Ported from the reference `design.py` (Step-0 ontology design + growth).
//!
//! The spec is the authoritative taxonomy that everything else reads:
//! * **ingest** types entity spans (`org/…`, `artifact/…`) and binds relation polarity (`rel/x/+`,
//! `rel/x/-`) from each relation's declared `head`/`tail` facets;
//! * the **linter** validates agent atoms against it (paper §2) and suggests corrections;
//! * **wildcards** expand over its `parent` hierarchy — `artifact/battery/*` is only meaningful
//! because the spec says `battery`'s parent is `artifact`.
//!
//! Two ways to obtain one: `propose` (an LLM reads a corpus sample and emits a disjoint facet set with
//! directed relations + a seed gazetteer — forced structured output, as in `design.py`), or
//! `seed_from_vocab` (deterministic fallback from tokens already observed, no model required).
//! `design.py`'s GROW step (information-gain-gated facet addition) is future work; the `parent` field and
//! `metrics` slot exist so growth can be layered on without changing the format.
use serde::{Deserialize, Serialize};
use std::collections::HashMap;
use std::path::{Path, PathBuf};
/// One entity facet = one *kind of thing*. Facets must be mutually disjoint (each becomes a retrieval
/// head); `parent` gives the subtree hierarchy that makes prefix wildcards expandable.
#[derive(Debug, Clone, Serialize, Deserialize, PartialEq)]
pub struct EntityFacet {
pub name: String,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub parent: Option<String>,
#[serde(default, skip_serializing_if = "String::is_empty")]
pub description: String,
#[serde(default, skip_serializing_if = "Vec::is_empty")]
pub examples: Vec<String>,
/// True when this facet was mined from the corpus's *field labels* (a CSV-ish column such as `duns`
/// or `award-amount`) rather than proposed as a semantic KIND of thing. Structural facets are valid
/// query dimensions but are not text-taggable, so the tagger's label space excludes them.
#[serde(default, skip_serializing_if = "std::ops::Not::not")]
pub structural: bool,
}
/// A *directed* relation: `head` acts on `tail`. Direction cannot live in a bag-of-words head (a SPLADE
/// facet has no notion of argument order), so it is declared here and the projector reads it to emit
/// `rel/<name>/+` on the head-side argument and `rel/<name>/-` on the tail-side one.
#[derive(Debug, Clone, Serialize, Deserialize, PartialEq)]
pub struct RelationFacet {
pub name: String,
pub head: String,
pub tail: String,
}
#[derive(Debug, Clone, Serialize, Deserialize, PartialEq)]
pub struct GazEntry {
pub surface: String,
pub token: String,
}
#[derive(Debug, Clone, Serialize, Deserialize, Default)]
pub struct VocabularySpace {
#[serde(default = "one")]
pub version: u32,
#[serde(default, skip_serializing_if = "String::is_empty")]
pub corpus: String,
#[serde(default)]
pub entity_facets: Vec<EntityFacet>,
#[serde(default)]
pub relation_facets: Vec<RelationFacet>,
#[serde(default, skip_serializing_if = "Vec::is_empty")]
pub gazetteer: Vec<GazEntry>,
/// Free-form provenance/quality metrics (discovery source, coverage, orthogonality) — the slot
/// `design.py`'s growth gate writes its MDL-style gain into.
#[serde(default, skip_serializing_if = "Option::is_none")]
pub metrics: Option<serde_json::Value>,
}
fn one() -> u32 {
1
}
/// Canonical filename for a corpus's spec, stored beside the data like the gazetteer overlay.
pub const SPEC_FILE: &str = ".steeldb-facets.json";
/// Where the spec for `dir` lives (directory → `<dir>/.steeldb-facets.json`; file → beside it).
pub fn spec_path(dir: &Path) -> PathBuf {
let base = if dir.is_dir() { dir.to_path_buf() } else { dir.parent().map(|p| p.to_path_buf()).unwrap_or_else(|| PathBuf::from(".")) };
base.join(SPEC_FILE)
}
impl VocabularySpace {
pub fn load(path: &Path) -> std::io::Result<VocabularySpace> {
let bytes = std::fs::read(path)?;
serde_json::from_slice(&bytes).map_err(std::io::Error::other)
}
/// Load the spec for a corpus dir, if one has been initialised.
pub fn for_corpus(dir: &Path) -> Option<VocabularySpace> {
VocabularySpace::load(&spec_path(dir)).ok()
}
pub fn save(&self, path: &Path) -> std::io::Result<()> {
std::fs::write(path, serde_json::to_vec_pretty(self).map_err(std::io::Error::other)?)
}
/// Semantic facets only — the tagger's label space (excludes structurally-mined field names).
pub fn taggable_facets(&self) -> Vec<&EntityFacet> {
self.entity_facets.iter().filter(|f| !f.structural).collect()
}
pub fn entity_names(&self) -> Vec<&str> {
self.entity_facets.iter().map(|f| f.name.as_str()).collect()
}
/// The declared relation, by name.
pub fn relation(&self, name: &str) -> Option<&RelationFacet> {
self.relation_facets.iter().find(|r| r.name == name)
}
/// Is `facet` a declared entity facet?
pub fn has_entity_facet(&self, facet: &str) -> bool {
self.entity_facets.iter().any(|f| f.name == facet)
}
/// Ancestor chain of a facet, nearest first (`battery` → `[artifact]`). Cycle-safe.
pub fn ancestors(&self, facet: &str) -> Vec<String> {
let by_name: HashMap<&str, &EntityFacet> = self.entity_facets.iter().map(|f| (f.name.as_str(), f)).collect();
let mut out = Vec::new();
let mut cur = facet.to_string();
for _ in 0..16 {
match by_name.get(cur.as_str()).and_then(|f| f.parent.clone()) {
Some(p) if !out.contains(&p) => {
out.push(p.clone());
cur = p;
}
_ => break,
}
}
out
}
/// The **hierarchical URI prefix** for a facet — the path from root to leaf, which is what makes
/// subtree wildcards work: `battery` (parent `artifact`) → `artifact/battery`.
pub fn facet_path(&self, facet: &str) -> String {
let mut parts = self.ancestors(facet);
parts.reverse();
parts.push(facet.to_string());
parts.join("/")
}
/// Fully-qualified entity URI for a span of this facet: `artifact/battery/cell`.
pub fn entity_uri(&self, facet: &str, surface: &str) -> String {
format!("{}/{}", self.facet_path(facet), crate::projector::slug(surface))
}
/// Every valid atom prefix implied by the spec — the linter's allowlist roots and the set of
/// expandable wildcard stems.
pub fn valid_prefixes(&self) -> Vec<String> {
let mut v: Vec<String> = self.entity_facets.iter().map(|f| self.facet_path(&f.name)).collect();
v.extend(self.relation_facets.iter().map(|r| format!("rel/{}", r.name)));
v.sort();
v.dedup();
v
}
/// Deterministic fallback seed: take the facet names already present in a corpus's vocabulary. No
/// model, no hierarchy — a flat but valid spec so `init` always produces something usable.
pub fn seed_from_vocab<I: IntoIterator<Item = String>>(corpus: &str, tokens: I) -> VocabularySpace {
let mut names: Vec<String> = Vec::new();
for t in tokens {
let f = t.split('/').next().unwrap_or(&t).to_string();
if !f.is_empty() && f != "src" && !names.contains(&f) {
names.push(f);
}
}
names.sort();
VocabularySpace {
version: 1,
corpus: corpus.to_string(),
entity_facets: names.into_iter().map(|n| EntityFacet { name: n, parent: None, description: String::new(), examples: Vec::new(), structural: true }).collect(),
relation_facets: Vec::new(),
gazetteer: Vec::new(),
metrics: Some(serde_json::json!({ "source": "seed_from_vocab", "hierarchy": false })),
}
}
/// Ensure every gazetteer token is **facet-qualified** (`facet/value`). A bare token cannot be
/// resolved by the linter or reached by a wildcard, so unqualified entries are qualified against a
/// declared facet when one is implied, and dropped otherwise.
pub fn qualify_gazetteer(&mut self) {
let names: Vec<String> = self.entity_facets.iter().map(|f| f.name.clone()).collect();
let paths: std::collections::HashMap<String, String> = names.iter().map(|n| (n.clone(), self.facet_path(n))).collect();
self.gazetteer.retain_mut(|g| {
if let Some((head, _)) = g.token.split_once('/') {
if names.iter().any(|n| n == head) {
// rewrite to the full hierarchical path so subtree wildcards reach it
if let Some(p) = paths.get(head) {
let leaf = g.token.split_once('/').map(|x| x.1.to_string()).unwrap_or_default();
g.token = format!("{p}/{leaf}");
}
return true;
}
}
false // unqualified / unknown facet → not addressable, drop
});
}
/// Reject a spec whose facets are not disjoint / are self-parented — the invariant `design.py`
/// enforces by prompt ("must be DISJOINT") and we enforce structurally.
/// Drop repeated entity facets, keeping the first of each name, and return the names dropped.
///
/// [`Self::validate`] rejects a duplicate, which is right for a committed spec — two facets with one name
/// cannot both become a retrieval head. It is the wrong answer for a model's suggestion: `granite3-moe:1b`
/// proposed `competitor` twice and lost all five of its facets over the repeat. Same reasoning as
/// [`Self::prune_invalid_relations`]: the caller still reviews, and the gate still re-tests.
pub fn dedupe_entity_facets(&mut self) -> Vec<String> {
let mut seen = std::collections::HashSet::new();
let mut dropped = Vec::new();
self.entity_facets.retain(|f| {
if seen.insert(f.name.clone()) {
true
} else {
dropped.push(f.name.clone());
false
}
});
dropped
}
/// Drop relation facets whose head or tail is not a declared entity facet, returning what went.
///
/// [`Self::validate`] rejects such a relation, which is right for a spec about to be committed and wrong
/// for a spec a language model just proposed. `qwen3.5:0.8b` emitted a perfectly good set of entity facets
/// alongside one relation whose tail was `2025`; validation threw away the entire proposal over it, even
/// though `learn::propose_categories` reads only the entity facets and never looks at relations.
///
/// Pruning a model's suggestion is not the same as repairing a user's query. The caller still reviews the
/// proposal and still decides, and every surviving candidate is re-tested by the MECE gate.
pub fn prune_invalid_relations(&mut self) -> Vec<String> {
let declared: std::collections::HashSet<String> =
self.entity_facets.iter().map(|f| f.name.clone()).collect();
let mut dropped = Vec::new();
self.relation_facets.retain(|r| {
let ok = declared.contains(&r.head) && declared.contains(&r.tail);
if !ok {
dropped.push(format!("{} ({} -> {})", r.name, r.head, r.tail));
}
ok
});
dropped
}
pub fn validate(&self) -> Result<(), String> {
let mut seen = std::collections::HashSet::new();
for f in &self.entity_facets {
if f.name.trim().is_empty() {
return Err("entity facet with empty name".into());
}
if !seen.insert(f.name.as_str()) {
return Err(format!("duplicate entity facet '{}' (facets must be disjoint)", f.name));
}
if f.parent.as_deref() == Some(f.name.as_str()) {
return Err(format!("facet '{}' is its own parent", f.name));
}
}
for r in &self.relation_facets {
for (side, f) in [("head", &r.head), ("tail", &r.tail)] {
if !self.has_entity_facet(f) {
return Err(format!("relation '{}' {side} facet '{f}' is not a declared entity facet", r.name));
}
}
}
Ok(())
}
}
// ── step-0 discovery inputs: sample the corpus, extract candidate facets ────────────────────────
/// Sample up to `n` text-ish documents from a corpus dir (recursive, deterministic order) — the input to
/// facet discovery. Kept small: discovery reads a *sample*, not the corpus.
pub fn sample_docs(dir: &Path, n: usize, max_chars: usize) -> Vec<String> {
let mut files: Vec<PathBuf> = Vec::new();
let mut stack = vec![dir.to_path_buf()];
while let Some(d) = stack.pop() {
let Ok(rd) = std::fs::read_dir(&d) else { continue };
for e in rd.flatten() {
let p = e.path();
let name = e.file_name().to_string_lossy().to_string();
if name.starts_with('.') { continue; }
if p.is_dir() { stack.push(p); }
else if matches!(p.extension().and_then(|x| x.to_str()), Some("txt" | "md" | "csv" | "json" | "jsonl")) { files.push(p); }
}
}
files.sort();
// spread the sample across the corpus rather than taking a contiguous prefix
let step = (files.len() / n.max(1)).max(1);
files
.iter()
.step_by(step)
.take(n)
.filter_map(|p| std::fs::read_to_string(p).ok())
.map(|s| s.chars().take(max_chars).collect())
.collect()
}
/// Candidate facets mined structurally from a sample: labelled fields (`**Key:**`, `Key:`, `"key":`) with
/// their frequency. This is the model-free half of step 0 — it discovers the corpus's *own* field
/// vocabulary, which an LLM then organises into a disjoint hierarchy (or which stands alone as a seed).
/// Not every labelled line names a dimension of the data: list scaffolding, markdown residue, URLs and
/// boilerplate headings all look like fields.
const BOILERPLATE: &[&str] = &[
"item", "abstract", "http", "https", "note", "notes", "text", "help", "answer", "example",
"examples", "disclaimer", "summary", "overview", "description", "see", "source", "sources",
"reference", "references", "a", "an", "the",
];
/// Parse one line as a labelled field, returning `(facet slug, raw value)`.
///
/// Shared by [`candidate_fields`] (which counts the slugs) and [`field_value_tokens`] (which needs the
/// values too). Keeping one parser means the tokens a corpus projects can never disagree with the facets
/// discovered from it.
fn field_pair(line: &str) -> Option<(String, String)> {
let t = line.trim().trim_start_matches(['-', '*', ' ']);
// `**Key:** value` / `"key": value` / `Key: value`
let (key, value) = if let Some(rest) = t.strip_prefix("**") {
let mut it = rest.splitn(2, "**");
let k = it.next()?.trim_end_matches(':').to_string();
let v = it.next().unwrap_or("").trim().trim_start_matches(':').trim().to_string();
(k, v)
} else if let Some(q) = t.strip_prefix('"') {
let mut it = q.splitn(2, '"');
let k = it.next()?.to_string();
let v = it
.next()
.unwrap_or("")
.trim()
.trim_start_matches(':')
.trim()
.trim_end_matches(',')
.trim_matches('"')
.to_string();
(k, v)
} else {
// Leading `-`/`*` were stripped above, so a markdown line like `- **Species:** Snorlax` arrives as
// `Species:** Snorlax` and the CLOSING `**` lands in the value. Harmless while only the key was
// used; it corrupts the value (and any offset derived from it), so clear the residue.
let (k, v) = t.split_once(':')?;
(k.to_string(), v.trim().trim_start_matches('*').trim().to_string())
};
let k = key.trim();
// a field label: short, alphabetic-ish, not a sentence
if k.is_empty() || k.len() > 32 || k.split_whitespace().count() > 3 {
return None;
}
if !k.chars().next().map(|c| c.is_alphabetic()).unwrap_or(false) {
return None;
}
let slug = crate::projector::slug(k);
if slug.len() < 3 || BOILERPLATE.contains(&slug.as_str()) {
return None;
}
if !slug.chars().next().map(|c| c.is_alphabetic()).unwrap_or(false) {
return None;
}
if !slug.chars().all(|c| c.is_alphanumeric() || c == '-') {
return None;
}
if slug.starts_with("item-") && slug[5..].chars().all(|c| c.is_ascii_digit()) {
return None;
}
Some((slug, value))
}
pub fn candidate_fields(samples: &[String]) -> Vec<(String, usize)> {
let mut freq: HashMap<String, usize> = HashMap::new();
for doc in samples {
for line in doc.lines() {
if let Some((slug, _)) = field_pair(line) {
*freq.entry(slug).or_default() += 1;
}
}
}
let mut v: Vec<(String, usize)> = freq.into_iter().collect();
v.sort_by(|a, b| b.1.cmp(&a.1).then(a.0.cmp(&b.0)));
v
}
/// Rank a cluster's distinctive terms by TF-IDF against the rest of the corpus.
///
/// Used to NAME a cluster without a model. A label has to be the term that distinguishes this group from
/// every other group, which is what the IDF factor measures; raw frequency would just return whichever word
/// is commonest everywhere. Deliberately model-free so it runs wherever the engine runs, including wasm —
/// an embedding-based labeller (KeyBERT and friends) would drag in a tokenizer that cannot target wasm32.
pub fn tfidf_terms(cluster: &[String], corpus: &[String], k: usize) -> Vec<(String, f64)> {
fn words(s: &str) -> Vec<String> {
s.split(|c: char| !c.is_alphanumeric() && c != '-' && c != '\'')
.map(|w| w.trim_matches('-').to_lowercase())
.filter(|w| w.len() >= 3 && w.len() <= 28 && w.chars().next().map(|c| c.is_alphabetic()).unwrap_or(false))
.collect()
}
const STOP: &[&str] = &[
"the", "and", "for", "with", "was", "were", "this", "that", "from", "into", "are", "has", "had",
"his", "her", "its", "not", "but", "all", "any", "may", "can", "will", "each", "than", "then",
"during", "under", "over", "also", "which", "while", "their", "there", "been", "being", "who",
"when", "what", "how", "why", "per", "via", "such", "more", "most", "less", "other", "some",
];
// term frequency within the cluster
let mut tf: HashMap<String, usize> = HashMap::new();
for doc in cluster {
for w in words(doc) {
if !STOP.contains(&w.as_str()) {
*tf.entry(w).or_default() += 1;
}
}
}
if tf.is_empty() {
return Vec::new();
}
// document frequency across the whole corpus
let mut df: HashMap<String, usize> = HashMap::new();
for doc in corpus {
let seen: std::collections::HashSet<String> = words(doc).into_iter().collect();
for w in seen {
*df.entry(w).or_default() += 1;
}
}
let n = corpus.len().max(1) as f64;
let mut scored: Vec<(String, f64)> = tf
.into_iter()
.map(|(term, count)| {
let d = *df.get(&term).unwrap_or(&1) as f64;
// smoothed IDF: a term in every document carries no information about this cluster
let idf = ((n + 1.0) / (d + 1.0)).ln() + 1.0;
(term, (count as f64).sqrt() * idf)
})
.collect();
scored.sort_by(|a, b| b.1.partial_cmp(&a.1).unwrap_or(std::cmp::Ordering::Equal).then(a.0.cmp(&b.0)));
scored.truncate(k);
scored
}
/// A labelled field located in the original text: `[start, end)` are byte offsets into the document.
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct FieldSpan {
pub start: usize,
pub end: usize,
pub facet: String,
pub value: String,
}
/// Locate each labelled field's VALUE in a document, with byte offsets into the original text.
///
/// Offsets are computed by walking the document, so they index the text the caller passed in rather than a
/// normalised copy — the same discipline the tagger's span alignment needs, and the reason a highlighted
/// rendering cannot drift from what was actually matched.
pub fn field_spans(doc: &str) -> Vec<FieldSpan> {
let mut out = Vec::new();
let mut offset = 0usize;
for line in doc.split_inclusive('\n') {
if let Some((facet, value)) = field_pair(line) {
// the value is taken verbatim from this line, so it can be located within it
if !value.is_empty() {
if let Some(rel) = line.find(value.as_str()) {
out.push(FieldSpan {
start: offset + rel,
end: offset + rel + value.len(),
facet,
value,
});
}
}
}
offset += line.len();
}
out
}
/// Project the sampled documents' labelled fields into `facet/value` tokens, for the facets in `facets`.
///
/// This is the model-free half of what the text projector does with a model: turn a field's value into a
/// concrete symbol so `type/fire` is a real token and not merely a plausible one. Without it a linter can
/// only check facet STEMS, which would make a perfectly valid `type/fire` look unknown.
///
/// Comma-separated values become separate tokens (`ice, dragon, fairy` is three symbols, not one), and
/// sentence-length values are skipped — a paragraph is not an identifier.
pub fn field_value_tokens(samples: &[String], facets: &[String]) -> Vec<String> {
let keep: std::collections::HashSet<&str> = facets.iter().map(|f| f.as_str()).collect();
let mut out: std::collections::BTreeSet<String> = std::collections::BTreeSet::new();
for doc in samples {
for line in doc.lines() {
let Some((slug, value)) = field_pair(line) else { continue };
if !keep.contains(slug.as_str()) || value.is_empty() {
continue;
}
for part in value.split([',', ';', '/']) {
let p = part.trim().trim_matches('*').trim();
// an identifier, not prose
if p.is_empty() || p.len() > 48 || p.split_whitespace().count() > 5 {
continue;
}
let v = crate::projector::slug(p);
if v.is_empty() {
continue;
}
out.insert(format!("{slug}/{v}"));
}
}
}
out.into_iter().collect()
}
/// Model-free step 0: structural discovery from a sample. Fields appearing in at least `min_support`
/// sampled docs become entity facets. Flat (no parents) — an LLM pass adds hierarchy + relations.
pub fn seed_from_sample(corpus: &str, samples: &[String], min_support: usize) -> VocabularySpace {
let cands = candidate_fields(samples);
let entity_facets: Vec<EntityFacet> = cands
.iter()
.filter(|(_, n)| *n >= min_support)
.take(40)
.map(|(name, n)| EntityFacet { name: name.clone(), parent: None, description: format!("observed field ({n} occurrences in sample)"), examples: Vec::new(), structural: true })
.collect();
VocabularySpace {
version: 1,
corpus: corpus.to_string(),
entity_facets,
relation_facets: Vec::new(),
gazetteer: Vec::new(),
metrics: Some(serde_json::json!({ "source": "seed_from_sample", "hierarchy": false, "sampled_docs": samples.len(), "candidates": cands.len() })),
}
}
// ── step-0 agent proposal (design.py::step0) ────────────────────────────────────────────────────
/// Ask a model to propose the seed ontology from a corpus sample — the Rust counterpart of
/// `design.py::step0`. Uses forced-ish structured output: a single `emit_ontology` tool whose schema is
/// `proposal_schema()`. Falls back to parsing a JSON object out of the reply text if the model answers in
/// prose instead of calling the tool, so a weaker local model still yields a spec.
#[cfg(feature = "agent")]
pub async fn propose(
provider: &dyn crate::agent::provider::LlmProvider,
corpus: &str,
docs: &[String],
) -> Result<VocabularySpace, String> {
use crate::agent::types::{Msg, ToolSpec};
// one schema, used as the tool's parameters AND as the decoding grammar, so the two paths cannot drift
let schema = proposal_schema();
let tools = vec![ToolSpec {
name: "emit_ontology".into(),
description: "Emit the proposed seed ontology for this corpus: disjoint entity facets, directed relation facets, and a seed gazetteer.".into(),
schema: schema.clone(),
}];
// design.py sends the first 12 sampled docs, separated by ---
let sample: Vec<&str> = docs.iter().take(12).map(|s| s.as_str()).collect();
let prompt = format!("Corpus sample:\n\n{}", sample.join("\n---\n"));
let turn = provider.chat(PROPOSAL_SYSTEM, &[Msg::user_text(prompt.clone())], &tools).await?;
if let Some((_, _, input)) = turn.tool_uses.first() {
let mut spec = spec_from_proposal(corpus, input, &format!("propose:{}", provider.name()));
let _dup = spec.dedupe_entity_facets();
let _dropped = spec.prune_invalid_relations();
spec.validate()?;
return Ok(spec);
}
// No tool call. Usually that means the model cannot make one — `qwen2.5:0.5b` returns `tool_calls: null`
// for every request — so asking again the same way is pointless. Instead, constrain DECODING to the schema:
// the sampler can only emit tokens the grammar permits, which needs no capability from the model at all.
// Models that produced prose or nothing through the tool path return usable JSON through this one.
if let Some(v) = provider.chat_json(PROPOSAL_SYSTEM, &[Msg::user_text(prompt)], &schema, "ontology").await? {
let mut spec = spec_from_proposal(corpus, &v, &format!("propose-json:{}", provider.name()));
let _dup = spec.dedupe_entity_facets();
let _dropped = spec.prune_invalid_relations();
spec.validate()?;
return Ok(spec);
}
// prose fallback: first balanced JSON object in the text
let v = extract_json(&turn.text).ok_or_else(|| {
// The usual cause is a model that cannot call tools, which replies with prose or with nothing at all.
// Saying so beats echoing an empty reply: `qwen2.5:0.5b` returns `tool_calls: null` for every request,
// and the failure is indistinguishable from a bad prompt unless the message names the requirement.
format!(
"model did not emit an ontology. `learn` needs a model that supports tool calling; \
one that does not will answer in prose or not at all. Reply was: {:?}",
turn.text.chars().take(160).collect::<String>()
)
})?;
let mut spec = spec_from_proposal(corpus, &v, &format!("propose-text:{}", provider.name()));
let _dropped = spec.prune_invalid_relations();
spec.validate()?;
Ok(spec)
}
/// First balanced `{...}` JSON object in a string (tolerant of prose or code fences around it).
pub fn extract_json(text: &str) -> Option<serde_json::Value> {
let bytes = text.as_bytes();
let start = text.find('{')?;
let mut depth = 0usize;
let mut in_str = false;
let mut esc = false;
for i in start..bytes.len() {
let c = bytes[i] as char;
if in_str {
if esc {
esc = false;
} else if c == '\\' {
esc = true;
} else if c == '"' {
in_str = false;
}
continue;
}
match c {
'"' => in_str = true,
'{' => depth += 1,
'}' => {
depth -= 1;
if depth == 0 {
return serde_json::from_str(&text[start..=i]).ok();
}
}
_ => {}
}
}
None
}
/// Fold a spec already on disk into a freshly-seeded one, so step 0 is **re-runnable**.
///
/// The model-free seed rediscovers only what the corpus states structurally. Semantic facets an `--llm`
/// pass introduced, and anything hand-authored, are therefore absent from it — so adopting the existing
/// relations without their endpoint facets leaves those relations dangling and [`VocabularySpace::validate`]
/// rejects the result. An existing entity facet is carried across when it is either a relation endpoint
/// (required for validity) or non-structural (authored vocabulary the seed cannot recover).
pub fn merge_existing(mut seed: VocabularySpace, existing: &VocabularySpace) -> VocabularySpace {
// nothing authored to preserve
if existing.relation_facets.is_empty() && !existing.entity_facets.iter().any(|f| f.parent.is_some()) {
return seed;
}
seed.relation_facets = existing.relation_facets.clone();
for f in &mut seed.entity_facets {
if let Some(prev) = existing.entity_facets.iter().find(|e| e.name == f.name) {
f.parent = prev.parent.clone();
}
}
let referenced: std::collections::HashSet<&str> = seed
.relation_facets
.iter()
.flat_map(|r| [r.head.as_str(), r.tail.as_str()])
.collect();
for f in &existing.entity_facets {
if seed.entity_facets.iter().any(|e| e.name == f.name) {
continue;
}
if referenced.contains(f.name.as_str()) || !f.structural {
seed.entity_facets.push(f.clone());
}
}
seed
}
/// Merge an agent-proposed taxonomy with a structurally-seeded field list. The proposal is authoritative
/// for hierarchy and relations (it is the only source of `parent`/`head`/`tail`); observed fields that no
/// proposed facet already covers are appended as flat facets so nothing in the corpus is unreachable.
pub fn merge(proposed: VocabularySpace, structural: &VocabularySpace) -> VocabularySpace {
let mut out = proposed;
for f in &structural.entity_facets {
let covered = out.entity_facets.iter().any(|e| e.name == f.name) || out.entity_facets.iter().any(|e| f.name.starts_with(&format!("{}-", e.name)));
if !covered {
out.entity_facets.push(f.clone());
}
}
out.metrics = Some(serde_json::json!({
"source": "merge(propose, seed_from_sample)",
"hierarchy": out.entity_facets.iter().any(|f| f.parent.is_some()),
"seed_facets": out.entity_facets.len() - structural.entity_facets.iter().filter(|f| out.entity_facets.iter().any(|e| e.name == f.name)).count(),
"relations": out.relation_facets.len(),
"structural_facets": structural.entity_facets.len(),
}));
out
}
/// The structured-output schema an LLM must fill to propose a spec — mirrors `design.py`'s `seed_facets`
/// tool schema (disjoint entity facets, directed relation facets, seed gazetteer).
/// The shape of a proposed ontology, used both as a tool's parameter schema and as a decoding grammar.
///
/// **Every array is bounded, and that is load-bearing.** Constraining a small model's decoding to this schema is
/// not on its own enough: given an unbounded `examples` array, `qwen2.5:0.5b` emitted forty-odd strings and ran
/// into the token limit, so the object was cut off mid-string and the reply — though grammatically constrained —
/// would not parse. With `maxItems` the grammar cannot ask for more than fits, and the same model returns valid
/// JSON. The bounds also match what the prompt asks for, so they cost nothing on a capable model.
pub fn proposal_schema() -> serde_json::Value {
serde_json::json!({
"type": "object",
"additionalProperties": false,
"properties": {
"entity_facets": { "type": "array", "minItems": 1, "maxItems": 6,
"items": { "type": "object", "additionalProperties": false, "properties": {
"name": {"type": "string"}, "parent": {"type": "string"},
"description": {"type": "string"},
"examples": {"type": "array", "maxItems": 6, "items": {"type": "string"}}
}, "required": ["name", "description"] } },
"relation_facets": { "type": "array", "maxItems": 8,
"items": { "type": "object", "additionalProperties": false, "properties": {
"name": {"type": "string"}, "head": {"type": "string"}, "tail": {"type": "string"}
}, "required": ["name", "head", "tail"] } },
"gazetteer": { "type": "array", "maxItems": 16,
"items": { "type": "object", "additionalProperties": false, "properties": {
"surface": {"type": "string"}, "token": {"type": "string"}
}, "required": ["surface", "token"] } }
},
"required": ["entity_facets", "relation_facets"]
})
}
/// System prompt for facet discovery — the disjointness requirement is the load-bearing instruction
/// (each facet becomes a retrieval head).
pub const PROPOSAL_SYSTEM: &str = "You design the ontology for a document corpus. Propose 3-4 SEED entity facets \u{2014} the orthogonal TYPES worth projecting (e.g. actor/org, platform, method, constraint). Each entity facet becomes a retrieval head, so they must be DISJOINT and each a distinct KIND of thing. Also propose the directed relation facets (name, head-facet, tail-facet) and a small seed gazetteer mapping each multi-word surface form to a FACET-QUALIFIED token of the form 'facet/value' using one of your entity facet names (e.g. 'unmanned aerial vehicle' \u{2192} 'system/uav'). Keep it minimal; growth adds more later. Facet names are short lowercase slugs.";
/// System prompt for the GROW step (`design.py::propose_candidate`): one candidate that specialises an
/// existing facet, covering what current heads leave unexplained. `parent` appears here, not at seed.
pub const CANDIDATE_SYSTEM: &str = "You grow an ontology. Given the existing facets and a corpus sample, propose ONE candidate facet covering what the existing facets leave unexplained. It must specialise exactly one existing facet (set 'parent') and be a distinct KIND of thing, disjoint from its siblings. Set 'examples' to 3-6 surface forms copied VERBATIM from the corpus sample (exact substrings), so the facet is detectable in the text. Set worth_adding=false if the existing facets already cover the corpus. Facet names are short lowercase slugs.";
/// Schema for a growth candidate — mirrors `design.py::propose_candidate`.
pub fn candidate_schema() -> serde_json::Value {
serde_json::json!({
"type": "object",
"properties": {
"name": {"type": "string"}, "parent": {"type": "string"}, "description": {"type": "string"},
"examples": {"type": "array", "items": {"type": "string"}},
"worth_adding": {"type": "boolean"}
},
"required": ["name", "parent", "description", "examples", "worth_adding"]
})
}
/// Parse a model's structured proposal into a spec (tolerant of a missing/blank `parent`).
pub fn spec_from_proposal(corpus: &str, v: &serde_json::Value, source: &str) -> VocabularySpace {
let ents = v
.get("entity_facets")
.and_then(|x| x.as_array())
.map(|a| {
a.iter()
.filter_map(|e| {
let name = crate::projector::slug(e.get("name")?.as_str()?);
if name.is_empty() {
return None;
}
let parent = e.get("parent").and_then(|p| p.as_str()).map(crate::projector::slug).filter(|p| !p.is_empty() && *p != name);
Some(EntityFacet {
structural: false,
name,
parent,
description: e.get("description").and_then(|d| d.as_str()).unwrap_or("").to_string(),
examples: e
.get("examples")
.and_then(|x| x.as_array())
.map(|a| a.iter().filter_map(|s| s.as_str().map(String::from)).collect())
.unwrap_or_default(),
})
})
.collect::<Vec<_>>()
})
.unwrap_or_default();
let rels = v
.get("relation_facets")
.and_then(|x| x.as_array())
.map(|a| {
a.iter()
.filter_map(|r| {
Some(RelationFacet {
name: crate::projector::slug(r.get("name")?.as_str()?),
head: crate::projector::slug(r.get("head")?.as_str()?),
tail: crate::projector::slug(r.get("tail")?.as_str()?),
})
})
.collect::<Vec<_>>()
})
.unwrap_or_default();
let gaz = v
.get("gazetteer")
.and_then(|x| x.as_array())
.map(|a| {
a.iter()
.filter_map(|g| Some(GazEntry { surface: g.get("surface")?.as_str()?.to_string(), token: g.get("token")?.as_str()?.to_string() }))
.collect::<Vec<_>>()
})
.unwrap_or_default();
let mut spec = VocabularySpace {
version: 1,
corpus: corpus.to_string(),
entity_facets: ents,
relation_facets: rels,
gazetteer: gaz,
metrics: None,
};
spec.qualify_gazetteer();
let hierarchical = spec.entity_facets.iter().any(|f| f.parent.is_some());
spec.metrics = Some(serde_json::json!({ "source": source, "hierarchy": hierarchical }));
spec
}
#[cfg(test)]
mod tests {
use super::*;
fn spec() -> VocabularySpace {
VocabularySpace {
version: 1,
corpus: "t".into(),
entity_facets: vec![
EntityFacet { name: "artifact".into(), parent: None, description: "physical part".into(), examples: vec![], structural: false },
EntityFacet { name: "battery".into(), parent: Some("artifact".into()), description: "cells".into(), examples: vec![], structural: false },
EntityFacet { name: "org".into(), parent: None, description: "company".into(), examples: vec![], structural: false },
],
relation_facets: vec![RelationFacet { name: "supplies".into(), head: "org".into(), tail: "artifact".into() }],
gazetteer: vec![],
metrics: None,
}
}
#[test]
fn hierarchy_makes_wildcards_expandable() {
let s = spec();
assert_eq!(s.facet_path("battery"), "artifact/battery");
assert_eq!(s.facet_path("org"), "org");
assert_eq!(s.entity_uri("battery", "Cell 18650"), "artifact/battery/cell-18650");
// the URI is reachable from every ancestor prefix — the whole point of the hierarchy
let u = s.entity_uri("battery", "cell");
assert!(u.starts_with("artifact/"));
assert!(u.starts_with("artifact/battery/"));
assert!(s.valid_prefixes().contains(&"artifact/battery".to_string()));
assert!(s.valid_prefixes().contains(&"rel/supplies".to_string()));
}
#[test]
fn validate_catches_bad_specs() {
assert!(spec().validate().is_ok());
let mut dup = spec();
dup.entity_facets.push(EntityFacet { name: "org".into(), parent: None, description: String::new(), examples: vec![], structural: false });
assert!(dup.validate().unwrap_err().contains("disjoint"));
let mut bad_rel = spec();
bad_rel.relation_facets.push(RelationFacet { name: "ships".into(), head: "org".into(), tail: "nonexistent".into() });
assert!(bad_rel.validate().unwrap_err().contains("not a declared entity facet"));
}
#[test]
fn roundtrip_and_vocab_seed() {
let dir = std::env::temp_dir().join(format!("steeldb-spec-{}", std::process::id()));
std::fs::create_dir_all(&dir).unwrap();
let p = spec_path(&dir);
spec().save(&p).unwrap();
let back = VocabularySpace::load(&p).unwrap();
assert_eq!(back.facet_path("battery"), "artifact/battery");
let seeded = VocabularySpace::seed_from_vocab("c", ["org/toyota".into(), "qty/temp/celsius/under_30".into(), "src/f.txt".into()]);
assert_eq!(seeded.entity_names(), vec!["org", "qty"]); // src dropped
let _ = std::fs::remove_dir_all(&dir);
}
#[test]
fn extract_json_handles_prose_and_fences() {
let v = extract_json("sure, here:\n```json\n{\"entity_facets\":[{\"name\":\"org\",\"description\":\"x\"}]}\n```").unwrap();
assert_eq!(v["entity_facets"][0]["name"], "org");
assert!(extract_json("no json here").is_none());
// nested braces + braces inside strings must not terminate early
let v2 = extract_json(r#"{"a":{"b":1},"c":"}{"}"#).unwrap();
assert_eq!(v2["a"]["b"], 1);
}
#[test]
fn merge_keeps_hierarchy_and_adds_uncovered_fields() {
let proposed = spec();
let structural = VocabularySpace::seed_from_vocab("c", ["agency/x".into(), "org/y".into()]);
let m = super::merge(proposed, &structural);
assert_eq!(m.facet_path("battery"), "artifact/battery"); // hierarchy preserved
assert!(m.has_entity_facet("agency")); // uncovered structural field added
assert_eq!(m.entity_facets.iter().filter(|f| f.name == "org").count(), 1); // no dup
}
#[test]
fn proposal_parsing_is_tolerant() {
let v = serde_json::json!({
"entity_facets": [
{"name": "Org", "description": "companies"},
{"name": "battery", "parent": "artifact", "description": "cells", "examples": ["18650"]},
{"name": "", "description": "junk"}
],
"relation_facets": [{"name": "supplies", "head": "org", "tail": "battery"}]
});
let s = spec_from_proposal("c", &v, "test");
assert_eq!(s.entity_facets.len(), 2); // blank dropped
assert_eq!(s.entity_facets[0].name, "org"); // slugged
assert_eq!(s.facet_path("battery"), "artifact/battery");
assert_eq!(s.relation_facets[0].head, "org");
}
#[test]
fn merge_existing_keeps_relation_endpoints_the_seed_cannot_rediscover() {
// The bug this guards: the model-free seed only finds structural field names, so adopting the
// existing relations without their endpoint facets left them dangling and step 0 stopped being
// re-runnable ("relation 'located_in' head facet 'location' is not a declared entity facet").
let existing = VocabularySpace {
version: 1,
corpus: "c".into(),
entity_facets: vec![
EntityFacet { name: "location".into(), parent: None, description: "places".into(), examples: vec![], structural: false },
EntityFacet { name: "venue".into(), parent: None, description: "field".into(), examples: vec![], structural: true },
],
relation_facets: vec![RelationFacet { name: "located_in".into(), head: "location".into(), tail: "location".into() }],
gazetteer: vec![],
metrics: None,
};
// a fresh structural seed: field names only, no semantic "location"
let seed = VocabularySpace {
version: 1,
corpus: "c".into(),
entity_facets: vec![EntityFacet { name: "venue".into(), parent: None, description: "observed field".into(), examples: vec![], structural: true }],
relation_facets: vec![],
gazetteer: vec![],
metrics: None,
};
let merged = merge_existing(seed, &existing);
assert!(merged.validate().is_ok(), "merge must validate: {:?}", merged.validate().err());
assert!(merged.entity_facets.iter().any(|f| f.name == "location"), "relation endpoint must be carried across");
assert_eq!(merged.relation_facets.len(), 1);
// and it must be stable: merging the result again changes nothing
let again = merge_existing(merged.clone(), &existing);
assert_eq!(again.entity_facets.len(), merged.entity_facets.len());
assert!(again.validate().is_ok());
}
#[test]
fn merge_existing_is_a_no_op_without_authored_content() {
let empty = VocabularySpace { version: 1, corpus: "c".into(), entity_facets: vec![], relation_facets: vec![], gazetteer: vec![], metrics: None };
let seed = spec();
let n = seed.entity_facets.len();
assert_eq!(merge_existing(seed, &empty).entity_facets.len(), n);
}
#[test]
fn field_spans_index_the_original_text() {
let doc = "# Entry\n\n- **Species:** Snorlax\n- **Type:** normal\n\nSnorlax is heavy.\n";
let spans = field_spans(doc);
let found: Vec<(&str, &str)> = spans
.iter()
.map(|s| (s.facet.as_str(), &doc[s.start..s.end]))
.collect();
// the slice must equal the recorded value: offsets index the ORIGINAL document
for s in &spans {
assert_eq!(&doc[s.start..s.end], s.value, "offset drift on {}", s.facet);
}
assert!(found.contains(&("species", "Snorlax")), "{found:?}");
assert!(found.contains(&("type", "normal")), "{found:?}");
// prose lines are not fields
assert!(!found.iter().any(|(_, v)| v.contains("heavy")), "{found:?}");
}
#[test]
fn tfidf_names_a_cluster_by_what_distinguishes_it() {
let corpus: Vec<String> = [
"The trainer used Registeel in the battle at the venue",
"The trainer used Gengar in the battle at the venue",
"The survey recorded elevation and rainfall at the habitat",
"The survey recorded elevation and temperature at the habitat",
].iter().map(|s| s.to_string()).collect();
// the survey cluster must be named by survey vocabulary, not by words common to everything
let cluster = vec![corpus[2].clone(), corpus[3].clone()];
let top: Vec<String> = tfidf_terms(&cluster, &corpus, 4).into_iter().map(|(t, _)| t).collect();
assert!(top.contains(&"survey".to_string()) || top.contains(&"elevation".to_string()), "{top:?}");
// "trainer"/"battle" belong to the other cluster and must not label this one
assert!(!top.contains(&"trainer".to_string()), "{top:?}");
assert!(!top.contains(&"battle".to_string()), "{top:?}");
// a term present in every document carries no signal
assert!(!top.contains(&"the".to_string()), "stopword leaked: {top:?}");
}
#[test]
fn pruning_keeps_good_facets_when_a_relation_is_malformed() {
// `qwen3.5:0.8b` emitted usable entity facets alongside a relation whose tail was `2025`. Validation
// rejected the whole proposal over it, and `learn::propose_categories` never reads relations at all.
let mut spec = VocabularySpace {
version: 1,
corpus: "t".into(),
entity_facets: vec![
EntityFacet { name: "battle".into(), parent: None, description: String::new(), examples: vec!["defeated".into()], structural: false },
EntityFacet { name: "survey".into(), parent: None, description: String::new(), examples: vec!["elevation".into()], structural: false },
],
relation_facets: vec![
RelationFacet { name: "fought_at".into(), head: "battle".into(), tail: "survey".into() },
RelationFacet { name: "held_in".into(), head: "battle".into(), tail: "2025".into() },
RelationFacet { name: "nowhere".into(), head: "ghost".into(), tail: "battle".into() },
],
gazetteer: Vec::new(),
metrics: None,
};
assert!(spec.validate().is_err(), "the undeclared tail should fail strict validation");
let dropped = spec.prune_invalid_relations();
assert_eq!(dropped.len(), 2, "both bad relations should go: {dropped:?}");
assert!(dropped.iter().any(|d| d.contains("held_in")), "{dropped:?}");
assert!(dropped.iter().any(|d| d.contains("nowhere")), "{dropped:?}");
// the good work survives
assert_eq!(spec.entity_facets.len(), 2, "entity facets must not be touched");
assert_eq!(spec.relation_facets.len(), 1);
assert_eq!(spec.relation_facets[0].name, "fought_at");
spec.validate().expect("pruned spec must validate");
}
#[test]
fn pruning_a_clean_spec_changes_nothing() {
let mut spec = VocabularySpace {
version: 1,
corpus: "t".into(),
entity_facets: vec![EntityFacet {
name: "battle".into(),
parent: None,
description: String::new(),
examples: vec![],
structural: false,
}],
relation_facets: vec![RelationFacet {
name: "self_rel".into(),
head: "battle".into(),
tail: "battle".into(),
}],
gazetteer: Vec::new(),
metrics: None,
};
assert!(spec.prune_invalid_relations().is_empty());
assert_eq!(spec.relation_facets.len(), 1);
}
}