#![allow(clippy::print_stdout, clippy::print_stderr)]
use std::io::IsTerminal;
use std::path::{Path, PathBuf};
use std::process::ExitCode;
use clap::{Parser, Subcommand};
use mif_problem::{OutputFormat, ToProblem};
const DEFAULT_CATALOG: &str = ".claude/enabled-packs.json";
const DEFAULT_CONFIG: &str = "harness.config.json";
const DEFAULT_REPORTS_DIR: &str = "reports";
#[derive(Parser)]
#[command(
name = "mif-rh-cli",
version,
about = "CLI for the mif-rh research-harness ontology engine"
)]
struct Cli {
#[arg(long, global = true, value_parser = ["pretty", "json"])]
format: Option<String>,
#[command(subcommand)]
command: Command,
}
#[derive(Subcommand)]
enum Command {
Resolve {
finding: PathBuf,
#[arg(long)]
topic: Option<String>,
#[arg(long)]
catalog: Option<PathBuf>,
#[arg(long)]
config: Option<PathBuf>,
#[arg(long)]
map: Option<PathBuf>,
#[arg(long)]
root: Option<PathBuf>,
},
Review {
#[arg(long)]
topic: Vec<String>,
#[arg(long)]
strict: bool,
#[arg(long)]
reports_dir: Option<PathBuf>,
#[arg(long)]
config: Option<PathBuf>,
#[arg(long)]
catalog: Option<PathBuf>,
#[arg(long)]
followup: Option<PathBuf>,
#[arg(long)]
root: Option<PathBuf>,
#[arg(long)]
relationship_script: Option<PathBuf>,
#[arg(long)]
build_index: bool,
#[arg(long)]
index: Option<PathBuf>,
#[arg(long)]
suggest: bool,
#[arg(long, requires = "suggest")]
calibration: Option<PathBuf>,
},
SuggestType {
#[arg(required_unless_present = "finding", conflicts_with = "finding")]
text: Option<String>,
#[arg(long)]
finding: Option<PathBuf>,
#[arg(long, required_unless_present = "finding")]
topic: Option<String>,
#[arg(long)]
catalog: Option<PathBuf>,
#[arg(long)]
config: Option<PathBuf>,
#[arg(long)]
root: Option<PathBuf>,
#[arg(long, default_value_t = 10)]
limit: usize,
#[arg(long)]
calibration: Option<PathBuf>,
#[arg(long, requires = "finding")]
record: bool,
#[arg(long)]
index: Option<PathBuf>,
},
Calibrate {
#[arg(long)]
reports_dir: Option<PathBuf>,
#[arg(long)]
config: Option<PathBuf>,
#[arg(long)]
catalog: Option<PathBuf>,
#[arg(long)]
root: Option<PathBuf>,
#[arg(long, default_value_t = 0.95)]
target_precision: f32,
#[arg(long, default_value_t = 0.5)]
tier2_target: f32,
#[arg(long)]
sample: Option<usize>,
#[arg(long, default_value_t = 0)]
seed: u64,
#[arg(long)]
out: Option<PathBuf>,
#[arg(long)]
confusions: Option<PathBuf>,
},
ExpansionCandidates {
#[arg(long)]
index: Option<PathBuf>,
#[arg(long)]
calibration: Option<PathBuf>,
#[arg(long)]
out: Option<PathBuf>,
},
}
type CliError = mif_rh::MifRhError;
struct Outcome {
message: String,
exit_code: u8,
}
fn main() -> ExitCode {
let cli = Cli::parse();
let format = OutputFormat::select(cli.format.as_deref(), std::io::stderr().is_terminal());
match run(&cli.command) {
Ok(outcome) => {
println!("{}", outcome.message);
ExitCode::from(outcome.exit_code)
},
Err(error) => {
eprintln!("{}", error.render(format));
ExitCode::from(error.to_problem().exit_code.unwrap_or(1))
},
}
}
fn run(command: &Command) -> Result<Outcome, CliError> {
match command {
Command::Resolve {
finding,
topic,
catalog,
config,
map,
root,
} => resolve(
finding,
topic.as_deref(),
catalog.as_deref(),
config.as_deref(),
map.as_deref(),
root.as_deref(),
),
Command::Review {
topic,
strict,
reports_dir,
config,
catalog,
followup,
root,
relationship_script,
build_index,
index,
suggest,
calibration,
} => review(&ReviewArgs {
topics: topic,
strict: *strict,
reports_dir: reports_dir.as_deref(),
config: config.as_deref(),
catalog: catalog.as_deref(),
followup: followup.as_deref(),
root: root.as_deref(),
relationship_script: relationship_script.as_deref(),
build_index: *build_index,
index: index.as_deref(),
suggest: *suggest,
calibration: calibration.as_deref(),
}),
Command::SuggestType {
text,
finding,
topic,
catalog,
config,
root,
limit,
calibration,
record,
index,
} => suggest_type_cmd(&SuggestTypeArgs {
text: text.as_deref(),
finding: finding.as_deref(),
topic: topic.as_deref(),
catalog: catalog.as_deref(),
config: config.as_deref(),
root: root.as_deref(),
limit: *limit,
calibration: calibration.as_deref(),
record: *record,
index: index.as_deref(),
}),
Command::Calibrate {
reports_dir,
config,
catalog,
root,
target_precision,
tier2_target,
sample,
seed,
out,
confusions,
} => calibrate_cmd(&CalibrateArgs {
reports_dir: reports_dir.as_deref(),
config: config.as_deref(),
catalog: catalog.as_deref(),
root: root.as_deref(),
target_precision: *target_precision,
tier2_target: *tier2_target,
sample: *sample,
seed: *seed,
out: out.as_deref(),
confusions: confusions.as_deref(),
}),
Command::ExpansionCandidates {
index,
calibration,
out,
} => expansion_candidates_cmd(index.as_deref(), calibration.as_deref(), out.as_deref()),
}
}
fn effective_path(given: Option<&Path>, default: &str) -> PathBuf {
given.map_or_else(|| PathBuf::from(default), Path::to_path_buf)
}
fn topic_from_path(finding: &Path) -> Option<String> {
let components: Vec<&str> = finding
.components()
.filter_map(|c| c.as_os_str().to_str())
.collect();
let index = components.iter().position(|c| *c == "reports")?;
components.get(index + 1).map(|s| (*s).to_string())
}
fn resolve(
finding_path: &Path,
topic: Option<&str>,
catalog: Option<&Path>,
config: Option<&Path>,
map: Option<&Path>,
root: Option<&Path>,
) -> Result<Outcome, CliError> {
let catalog_path = effective_path(catalog, DEFAULT_CATALOG);
let config_path = effective_path(config, DEFAULT_CONFIG);
let root = effective_path(root, ".");
let topic = topic
.map(str::to_string)
.or_else(|| topic_from_path(finding_path))
.unwrap_or_default();
let finding = mif_rh::Finding::load(finding_path)?;
let catalog = mif_rh::Catalog::load(&catalog_path)?;
let config = mif_rh::HarnessConfig::load(&config_path)?;
let ontology_packs = mif_rh::ontology_pack::load_packs_via_catalog(&catalog, &root)?;
let ctx = mif_rh::ResolveContext {
topic: &topic,
catalog: &catalog,
config: &config,
ontology_packs: &ontology_packs,
};
let record = mif_rh::resolve_finding(&finding, &ctx)?;
let map_path = map.map(Path::to_path_buf).or_else(|| {
let topic_dir = PathBuf::from("reports").join(&topic);
topic_dir
.is_dir()
.then(|| topic_dir.join("ontology-map.json"))
});
if let Some(map_path) = &map_path {
upsert_map_record(map_path, &record)?;
}
let message = format!(
"{}: {} -> {} (valid={})",
finding.id,
record.basis.label(),
record.resolved_ontology.as_deref().unwrap_or("-"),
record.valid
);
Ok(Outcome {
message,
exit_code: u8::from(!record.valid),
})
}
fn upsert_map_record(
map_path: &Path,
record: &mif_rh::MapRecord,
) -> Result<(), mif_rh::MifRhError> {
let mut records: Vec<mif_rh::MapRecord> = std::fs::read_to_string(map_path)
.ok()
.and_then(|contents| serde_json::from_str(&contents).ok())
.unwrap_or_default();
records.retain(|r| r.finding_id != record.finding_id);
records.push(record.clone());
records.sort_by(|a, b| a.finding_id.cmp(&b.finding_id));
mif_rh::write_json_atomic(map_path, &records)
}
struct SuggestTypeArgs<'a> {
text: Option<&'a str>,
finding: Option<&'a Path>,
topic: Option<&'a str>,
catalog: Option<&'a Path>,
config: Option<&'a Path>,
root: Option<&'a Path>,
limit: usize,
calibration: Option<&'a Path>,
record: bool,
index: Option<&'a Path>,
}
const DEFAULT_CALIBRATION: &str = "reports/_meta/confidence-calibration.json";
const DEFAULT_INDEX: &str = "reports/_meta/search-index.sqlite";
fn run_id() -> String {
let epoch_secs = std::time::SystemTime::now()
.duration_since(std::time::UNIX_EPOCH)
.map_or(0, |d| d.as_secs());
format!("{epoch_secs}-{}", std::process::id())
}
fn is_expansion_miss(suggestions: &[mif_rh::TypeSuggestion]) -> bool {
suggestions
.first()
.is_none_or(|top| top.tier == mif_ontology::ConfidenceTier::TriggerExpansion)
}
fn suggest_type_cmd(args: &SuggestTypeArgs<'_>) -> Result<Outcome, CliError> {
let catalog_path = effective_path(args.catalog, DEFAULT_CATALOG);
let config_path = effective_path(args.config, DEFAULT_CONFIG);
let root = effective_path(args.root, ".");
let calibration_path = effective_path(args.calibration, DEFAULT_CALIBRATION);
let (query, topic, finding_id) = if let Some(finding_path) = args.finding {
let finding = mif_rh::Finding::load(finding_path)?;
let topic = args
.topic
.map(str::to_string)
.or_else(|| topic_from_path(finding_path))
.unwrap_or_default();
(mif_rh::index_text(&finding), topic, Some(finding.id))
} else {
(
args.text.unwrap_or_default().to_string(),
args.topic.unwrap_or_default().to_string(),
None,
)
};
let catalog = mif_rh::Catalog::load(&catalog_path)?;
let config = mif_rh::HarnessConfig::load(&config_path)?;
let ontology_packs = mif_rh::ontology_pack::load_packs_via_catalog(&catalog, &root)?;
let ctx = mif_rh::ResolveContext {
topic: &topic,
catalog: &catalog,
config: &config,
ontology_packs: &ontology_packs,
};
let cal = mif_ontology::CalibrationConfig::load_or_default(&calibration_path)
.map_err(mif_rh::MifRhError::from)?;
let embedder = mif_embed::Embedder::load()?;
let candidates = mif_rh::suggest::build_candidates(&ctx, &embedder, &cal)?;
let query_vector = embedder.embed(&query)?;
let suggestions =
mif_rh::suggest::suggest_from_candidates(&query_vector, &candidates, &cal, args.limit);
if args.record && is_expansion_miss(&suggestions) {
if let Some(finding_id) = finding_id {
let index_path = effective_path(args.index, DEFAULT_INDEX);
if let Some(parent) = index_path.parent() {
std::fs::create_dir_all(parent).map_err(|source| mif_rh::MifRhError::Io {
path: parent.display().to_string(),
source,
})?;
}
let index = mif_rh::FindingIndex::open(&index_path)?;
index.record_miss(&mif_rh::Miss {
finding_id,
topic: topic.clone(),
content: query,
vector: query_vector,
run_id: run_id(),
model: mif_embed::MODEL_ID.to_string(),
})?;
}
}
let message =
serde_json::to_string_pretty(&suggestions).map_err(|source| CliError::JsonSerialize {
path: "<stdout>".to_string(),
source,
})?;
Ok(Outcome {
message,
exit_code: 0,
})
}
struct CalibrateArgs<'a> {
reports_dir: Option<&'a Path>,
config: Option<&'a Path>,
catalog: Option<&'a Path>,
root: Option<&'a Path>,
target_precision: f32,
tier2_target: f32,
sample: Option<usize>,
seed: u64,
out: Option<&'a Path>,
confusions: Option<&'a Path>,
}
fn calibrate_cmd(args: &CalibrateArgs<'_>) -> Result<Outcome, CliError> {
let reports_dir = effective_path(args.reports_dir, DEFAULT_REPORTS_DIR);
let config_path = effective_path(args.config, DEFAULT_CONFIG);
let catalog_path = effective_path(args.catalog, DEFAULT_CATALOG);
let root = effective_path(args.root, ".");
let out = args.out.map_or_else(
|| reports_dir.join("_meta/confidence-calibration.json"),
Path::to_path_buf,
);
let catalog = mif_rh::Catalog::load(&catalog_path)?;
let config = mif_rh::HarnessConfig::load(&config_path)?;
let ontology_packs = mif_rh::ontology_pack::load_packs_via_catalog(&catalog, &root)?;
let embedder = mif_embed::Embedder::load()?;
let opts = mif_rh::CalibrateOptions {
target_precision: args.target_precision,
tier2_target: args.tier2_target,
sample: args.sample,
seed: args.seed,
};
let mut samples = Vec::new();
for topic in &config.topics {
let ctx = mif_rh::ResolveContext {
topic: &topic.id,
catalog: &catalog,
config: &config,
ontology_packs: &ontology_packs,
};
samples.extend(mif_rh::collect_topic_samples(
&reports_dir,
&ctx,
&embedder,
)?);
}
let samples = mif_rh::subsample(samples, &opts);
let confusions_note = if let Some(confusions_path) = args.confusions {
let report = mif_rh::confusions(&samples);
if let Some(parent) = confusions_path.parent()
&& !parent.as_os_str().is_empty()
{
std::fs::create_dir_all(parent).map_err(|source| mif_rh::MifRhError::Io {
path: parent.display().to_string(),
source,
})?;
}
mif_rh::write_json_atomic(confusions_path, &report)?;
format!(
", {} confusion pair(s) written to {}",
report.pairs.len(),
confusions_path.display()
)
} else {
String::new()
};
let mut cal = mif_rh::sweep(&samples, &opts, &out)?;
let scored_topics: std::collections::BTreeSet<&str> =
samples.iter().map(|s| s.topic.as_str()).collect();
let mut negatives_active = false;
for topic in &config.topics {
if !scored_topics.contains(topic.id.as_str()) {
continue;
}
let ctx = mif_rh::ResolveContext {
topic: &topic.id,
catalog: &catalog,
config: &config,
ontology_packs: &ontology_packs,
};
if mif_rh::packs_carry_negatives(mif_rh::build_allowed(&ctx)?) {
negatives_active = true;
break;
}
}
cal.negatives_active = negatives_active;
if let Some(parent) = out.parent() {
std::fs::create_dir_all(parent).map_err(|source| mif_rh::MifRhError::Io {
path: parent.display().to_string(),
source,
})?;
}
mif_rh::write_json_atomic(&out, &cal)?;
let message = format!(
"calibrate: {} sample(s) -> tier1_floor={:.2} tier1_margin={:.2} tier2_floor={:.2} \
(method {}, written to {}{confusions_note})",
samples.len(),
cal.tier1_floor,
cal.tier1_margin,
cal.tier2_floor,
cal.method.as_deref().unwrap_or("-"),
out.display()
);
Ok(Outcome {
message,
exit_code: 0,
})
}
fn expansion_candidates_cmd(
index: Option<&Path>,
calibration: Option<&Path>,
out: Option<&Path>,
) -> Result<Outcome, CliError> {
let index_path = effective_path(index, DEFAULT_INDEX);
let calibration_path = effective_path(calibration, DEFAULT_CALIBRATION);
let cal = mif_ontology::CalibrationConfig::load_or_default(&calibration_path)
.map_err(mif_rh::MifRhError::from)?;
let misses = if index_path.exists() {
let idx = mif_rh::FindingIndex::open(&index_path)?;
idx.misses()?
} else {
Vec::new()
};
let misses: Vec<mif_rh::Miss> = misses
.into_iter()
.filter(|m| m.model == mif_embed::MODEL_ID)
.collect();
let candidates = mif_rh::expansion_candidates(&misses, &cal.expansion);
let payload = serde_json::json!({
"clusters": candidates,
"misses_considered": misses.len(),
"expansion": cal.expansion,
});
if let Some(out_path) = out {
mif_rh::write_json_atomic(out_path, &payload)?;
return Ok(Outcome {
message: format!(
"expansion-candidates: {} cluster(s) from {} miss(es) written to {}",
candidates.len(),
misses.len(),
out_path.display()
),
exit_code: 0,
});
}
let message =
serde_json::to_string_pretty(&payload).map_err(|source| CliError::JsonSerialize {
path: "<stdout>".to_string(),
source,
})?;
Ok(Outcome {
message,
exit_code: 0,
})
}
struct ReviewArgs<'a> {
topics: &'a [String],
strict: bool,
reports_dir: Option<&'a Path>,
config: Option<&'a Path>,
catalog: Option<&'a Path>,
followup: Option<&'a Path>,
root: Option<&'a Path>,
relationship_script: Option<&'a Path>,
build_index: bool,
index: Option<&'a Path>,
suggest: bool,
calibration: Option<&'a Path>,
}
struct SuggestPassInputs<'a> {
backlog: &'a mif_rh::FollowupBacklog,
catalog: &'a mif_rh::Catalog,
config: &'a mif_rh::HarnessConfig,
ontology_packs: &'a std::collections::HashMap<String, mif_rh::OntologyPack>,
meta_dir: &'a Path,
index: Option<&'a Path>,
calibration: Option<&'a Path>,
}
fn run_suggest_pass(inputs: &SuggestPassInputs<'_>) -> Result<String, CliError> {
let calibration_path = inputs.calibration.map_or_else(
|| inputs.meta_dir.join("confidence-calibration.json"),
Path::to_path_buf,
);
let cal = mif_ontology::CalibrationConfig::load_or_default(&calibration_path)
.map_err(mif_rh::MifRhError::from)?;
let embedder = mif_embed::Embedder::load()?;
let index_path = inputs.index.map_or_else(
|| inputs.meta_dir.join("search-index.sqlite"),
Path::to_path_buf,
);
let index = mif_rh::FindingIndex::open(&index_path)?;
let run = run_id();
let mut topic_ids: Vec<&String> = inputs.backlog.topics.keys().collect();
topic_ids.sort();
let (mut total_entries, mut topics_written, mut misses_recorded) = (0_usize, 0_usize, 0_usize);
for topic_id in topic_ids {
let ctx = mif_rh::ResolveContext {
topic: topic_id,
catalog: inputs.catalog,
config: inputs.config,
ontology_packs: inputs.ontology_packs,
};
let candidates = mif_rh::suggest::build_candidates(&ctx, &embedder, &cal)?;
let mut fresh = Vec::new();
for followup_entry in &inputs.backlog.topics[topic_id] {
let Some(file) = &followup_entry.file else {
continue;
};
let Ok(finding) = mif_rh::Finding::load(Path::new(file)) else {
continue;
};
let query = mif_rh::index_text(&finding);
if query.is_empty() {
continue;
}
let query_vector = embedder.embed(&query)?;
let suggestions = mif_rh::suggest::suggest_from_candidates(
&query_vector,
&candidates,
&cal,
mif_rh::suggest::SUGGESTION_DEPTH,
);
if is_expansion_miss(&suggestions) {
index.record_miss(&mif_rh::Miss {
finding_id: finding.id.clone(),
topic: topic_id.clone(),
vector: query_vector,
content: query,
run_id: run.clone(),
model: mif_embed::MODEL_ID.to_string(),
})?;
misses_recorded += 1;
}
fresh.push(mif_rh::SuggestionEntry {
finding_id: finding.id,
file: followup_entry.file.clone(),
basis: followup_entry.basis.clone(),
run_id: run.clone(),
candidates: suggestions,
status: mif_rh::queue::STATUS_PENDING.to_string(),
});
}
if fresh.is_empty() {
continue;
}
total_entries += fresh.len();
topics_written += 1;
let queue_path = inputs
.meta_dir
.join("suggestions")
.join(format!("{topic_id}.json"));
mif_rh::upsert_suggestions(&queue_path, topic_id, fresh)?;
}
if total_entries == 0 {
return Ok(format!(
"ontology-review: no findings needed suggestions; {misses_recorded} miss(es) recorded"
));
}
Ok(format!(
"ontology-review: suggestions written to {} ({} finding(s) across {} topic(s); {} \
miss(es) recorded)",
inputs.meta_dir.join("suggestions").display(),
total_entries,
topics_written,
misses_recorded,
))
}
fn format_topic_table(report: &mif_rh::ReviewReport) -> String {
use std::fmt::Write as _;
let mut out = format!(
"{:<28} {:<22} {:>6} {:>8} {:>10} {:>8} {:>9}\n",
"TOPIC", "BOUND", "FIND", "STAMPED", "DISCOVERY", "UNTYPED", "INVALID"
);
for topic in &report.topics {
let bound = if topic.bound.is_empty() {
"(core-only)".to_string()
} else {
topic.bound.join(",")
};
let bound: String = bound.chars().take(22).collect();
let _ = writeln!(
out,
"{:<28} {:<22} {:>6} {:>8} {:>10} {:>8} {:>9}",
topic.topic,
bound,
topic.total,
topic.stamped,
topic.discovery,
topic.untyped,
topic.bad
);
}
out.push_str("---");
out
}
fn relationship_script_path(given: Option<&Path>, root: &Path) -> Option<PathBuf> {
given.map(Path::to_path_buf).or_else(|| {
let candidate = root.join("scripts/check-relationship-targets.sh");
candidate.is_file().then_some(candidate)
})
}
fn review(args: &ReviewArgs<'_>) -> Result<Outcome, CliError> {
let reports_dir = effective_path(args.reports_dir, DEFAULT_REPORTS_DIR);
let config_path = effective_path(args.config, DEFAULT_CONFIG);
let catalog_path = effective_path(args.catalog, DEFAULT_CATALOG);
let root = effective_path(args.root, ".");
let meta_dir = reports_dir.join("_meta");
std::fs::create_dir_all(&meta_dir).map_err(|source| mif_rh::MifRhError::Io {
path: meta_dir.display().to_string(),
source,
})?;
let _lock = mif_rh::ReviewLock::acquire(&meta_dir.join(".review.lock"))?;
let catalog = mif_rh::Catalog::load(&catalog_path)?;
let config = mif_rh::HarnessConfig::load(&config_path)?;
let ontology_packs = mif_rh::ontology_pack::load_packs_via_catalog(&catalog, &root)?;
let relationship_script = relationship_script_path(args.relationship_script, &root);
let topic_ids: Option<Vec<String>> = (!args.topics.is_empty()).then(|| args.topics.to_vec());
let opts = mif_rh::ReviewOptions {
topics: topic_ids.as_deref(),
reports_dir: &reports_dir,
ontology_packs: &ontology_packs,
catalog: &catalog,
config: &config,
check_relationship_targets_script: relationship_script.as_deref(),
};
let (report, backlog) = mif_rh::review(&opts)?;
let mut message = format_topic_table(&report);
if let Some(followup_path) = args.followup {
use std::fmt::Write as _;
mif_rh::write_followup(followup_path, &backlog)?;
message.push('\n');
let _ = write!(
message,
"ontology-review: followup backlog written to {} ({} finding(s) across {} topic(s))",
followup_path.display(),
backlog.total_needs_followup,
backlog.topics.len(),
);
}
if args.suggest {
message.push('\n');
message.push_str(&run_suggest_pass(&SuggestPassInputs {
backlog: &backlog,
catalog: &catalog,
config: &config,
ontology_packs: &ontology_packs,
meta_dir: &meta_dir,
index: args.index,
calibration: args.calibration,
})?);
}
message.push('\n');
message.push_str(&report.summary_line());
if args.build_index {
let all_topic_ids: Vec<String> = config.topics.iter().map(|t| t.id.clone()).collect();
let index_path = args
.index
.map_or_else(|| meta_dir.join("search-index.sqlite"), Path::to_path_buf);
let mut index = mif_rh::FindingIndex::open(&index_path)?;
mif_rh::build_search_index(&reports_dir, &all_topic_ids, &mut index)?;
}
let exit_ok = if args.strict {
!report.strict_should_fail()
} else {
true
};
Ok(Outcome {
message,
exit_code: u8::from(!exit_ok),
})
}
#[cfg(test)]
mod tests {
use std::fs;
use super::{
CalibrateArgs, ReviewArgs, SuggestTypeArgs, calibrate_cmd, expansion_candidates_cmd,
resolve, review, suggest_type_cmd, topic_from_path,
};
fn write_fixture(dir: &std::path::Path) {
fs::create_dir_all(dir.join(".claude")).unwrap();
fs::create_dir_all(dir.join("packs")).unwrap();
fs::create_dir_all(dir.join("reports/edu/findings")).unwrap();
fs::write(
dir.join("harness.config.json"),
r#"{"topics":[{"id":"edu","ontologies":["edu-fixture"]}]}"#,
)
.unwrap();
fs::write(
dir.join(".claude/enabled-packs.json"),
r#"{"ontologies":[{"id":"edu-fixture","version":"0.1.0","source":"packs/edu-fixture.yaml","core":false}]}"#,
)
.unwrap();
fs::write(
dir.join("packs/edu-fixture.yaml"),
"ontology:\n id: edu-fixture\n version: \"0.1.0\"\nentity_types:\n - name: title\n schema:\n required: [name]\n properties: {name: {type: string}}\n",
)
.unwrap();
fs::write(
dir.join("reports/edu/findings/good.json"),
r#"{"@id":"f-good","entity":{"name":"Algebra I","entity_type":"title"}}"#,
)
.unwrap();
fs::write(
dir.join("reports/edu/findings/invalid.json"),
r#"{"@id":"f-invalid","entity":{"entity_type":"title"}}"#,
)
.unwrap();
}
#[test]
fn resolve_a_valid_finding_exits_zero_and_writes_the_map() {
let dir = tempfile::tempdir().unwrap();
write_fixture(dir.path());
let map_path = dir.path().join("map.json");
let outcome = resolve(
&dir.path().join("reports/edu/findings/good.json"),
Some("edu"),
Some(&dir.path().join(".claude/enabled-packs.json")),
Some(&dir.path().join("harness.config.json")),
Some(&map_path),
Some(dir.path()),
)
.unwrap();
assert_eq!(outcome.exit_code, 0);
assert!(outcome.message.contains("resolved"));
assert!(map_path.exists());
}
#[test]
fn resolve_an_invalid_finding_exits_nonzero() {
let dir = tempfile::tempdir().unwrap();
write_fixture(dir.path());
let outcome = resolve(
&dir.path().join("reports/edu/findings/invalid.json"),
Some("edu"),
Some(&dir.path().join(".claude/enabled-packs.json")),
Some(&dir.path().join("harness.config.json")),
None,
Some(dir.path()),
)
.unwrap();
assert_eq!(outcome.exit_code, 1);
assert!(outcome.message.contains("valid=false"));
}
#[test]
fn suggest_type_prints_tier_annotated_json_and_exits_zero() {
if mif_embed::Embedder::load().is_err() {
eprintln!("skipping: embedding model unavailable in this environment");
return;
}
let dir = tempfile::tempdir().unwrap();
write_fixture(dir.path());
let outcome = suggest_type_cmd(&SuggestTypeArgs {
text: Some("A textbook titled Algebra I"),
finding: None,
topic: Some("edu"),
catalog: Some(&dir.path().join(".claude/enabled-packs.json")),
config: Some(&dir.path().join("harness.config.json")),
root: Some(dir.path()),
limit: 10,
calibration: Some(&dir.path().join("absent-calibration.json")),
record: false,
index: None,
})
.unwrap();
assert_eq!(outcome.exit_code, 0);
let suggestions: serde_json::Value = serde_json::from_str(&outcome.message).unwrap();
let list = suggestions.as_array().unwrap();
assert!(list.is_empty());
}
#[test]
fn expansion_candidates_on_a_fresh_index_emit_an_empty_cluster_list() {
let dir = tempfile::tempdir().unwrap();
let index_path = dir.path().join("index.sqlite");
let outcome = expansion_candidates_cmd(
Some(&index_path),
Some(&dir.path().join("absent-calibration.json")),
None,
)
.unwrap();
assert_eq!(outcome.exit_code, 0);
let payload: serde_json::Value = serde_json::from_str(&outcome.message).unwrap();
assert!(payload["clusters"].as_array().unwrap().is_empty());
assert_eq!(payload["misses_considered"], 0);
}
fn enrich_fixture_pack(dir: &std::path::Path) {
fs::write(
dir.join("packs/edu-fixture.yaml"),
"ontology:\n id: edu-fixture\n version: \"0.1.0\"\nentity_types:\n - name: title\n description: A published educational title\n aliases: [textbook]\n schema:\n required: [name]\n properties: {name: {type: string}}\n",
)
.unwrap();
}
#[test]
fn calibrate_derives_a_wellformed_artifact_from_stamped_findings() {
if mif_embed::Embedder::load().is_err() {
eprintln!("skipping: embedding model unavailable in this environment");
return;
}
let dir = tempfile::tempdir().unwrap();
write_fixture(dir.path());
enrich_fixture_pack(dir.path());
resolve(
&dir.path().join("reports/edu/findings/good.json"),
Some("edu"),
Some(&dir.path().join(".claude/enabled-packs.json")),
Some(&dir.path().join("harness.config.json")),
Some(&dir.path().join("reports/edu/ontology-map.json")),
Some(dir.path()),
)
.unwrap();
let out = dir.path().join("reports/_meta/confidence-calibration.json");
let confusions_path = dir.path().join("reports/_meta/confusions.json");
let outcome = calibrate_cmd(&CalibrateArgs {
reports_dir: Some(&dir.path().join("reports")),
config: Some(&dir.path().join("harness.config.json")),
catalog: Some(&dir.path().join(".claude/enabled-packs.json")),
root: Some(dir.path()),
target_precision: 1.0,
tier2_target: 0.5,
sample: None,
seed: 0,
out: Some(&out),
confusions: Some(&confusions_path),
})
.unwrap();
assert_eq!(outcome.exit_code, 0);
let cal = mif_ontology::CalibrationConfig::load_or_default(&out).unwrap();
assert!(cal.calibrated);
assert_eq!(cal.method.as_deref(), Some("stamped-quantile-v1"));
assert_eq!(cal.sample_size, Some(1));
assert!(cal.tier2_floor <= cal.tier1_floor);
assert!(!cal.negatives_active);
assert!(outcome.message.contains("confusion pair(s)"));
let report: serde_json::Value =
serde_json::from_str(&fs::read_to_string(&confusions_path).unwrap()).unwrap();
assert_eq!(report["version"], "confusions-v1");
assert_eq!(report["sample_count"], 1);
assert!(report["pairs"].as_array().unwrap().is_empty());
}
#[test]
fn calibrate_records_negatives_participation_in_the_artifact() {
if mif_embed::Embedder::load().is_err() {
eprintln!("skipping: embedding model unavailable in this environment");
return;
}
let dir = tempfile::tempdir().unwrap();
write_fixture(dir.path());
fs::write(
dir.path().join("packs/edu-fixture.yaml"),
"ontology:\n id: edu-fixture\n version: \"0.1.0\"\nentity_types:\n - name: title\n description: A published educational title\n aliases: [textbook]\n negative_examples:\n - A lesson plan for teachers\n schema:\n required: [name]\n properties: {name: {type: string}}\n",
)
.unwrap();
resolve(
&dir.path().join("reports/edu/findings/good.json"),
Some("edu"),
Some(&dir.path().join(".claude/enabled-packs.json")),
Some(&dir.path().join("harness.config.json")),
Some(&dir.path().join("reports/edu/ontology-map.json")),
Some(dir.path()),
)
.unwrap();
let out = dir.path().join("reports/_meta/confidence-calibration.json");
let outcome = calibrate_cmd(&CalibrateArgs {
reports_dir: Some(&dir.path().join("reports")),
config: Some(&dir.path().join("harness.config.json")),
catalog: Some(&dir.path().join(".claude/enabled-packs.json")),
root: Some(dir.path()),
target_precision: 1.0,
tier2_target: 0.5,
sample: None,
seed: 0,
out: Some(&out),
confusions: None,
})
.unwrap();
assert_eq!(outcome.exit_code, 0);
let cal = mif_ontology::CalibrationConfig::load_or_default(&out).unwrap();
assert!(cal.negatives_active);
}
#[test]
fn an_unbound_negatives_pack_never_claims_participation() {
if mif_embed::Embedder::load().is_err() {
eprintln!("skipping: embedding model unavailable in this environment");
return;
}
let dir = tempfile::tempdir().unwrap();
write_fixture(dir.path());
enrich_fixture_pack(dir.path());
fs::write(
dir.path().join("packs/unbound-fixture.yaml"),
"ontology:\n id: unbound-fixture\n version: \"0.1.0\"\nentity_types:\n - name: lesson\n description: A lesson plan\n negative_examples:\n - A published textbook\n",
)
.unwrap();
fs::write(
dir.path().join(".claude/enabled-packs.json"),
r#"{"ontologies":[{"id":"edu-fixture","version":"0.1.0","source":"packs/edu-fixture.yaml","core":false},{"id":"unbound-fixture","version":"0.1.0","source":"packs/unbound-fixture.yaml","core":false}]}"#,
)
.unwrap();
resolve(
&dir.path().join("reports/edu/findings/good.json"),
Some("edu"),
Some(&dir.path().join(".claude/enabled-packs.json")),
Some(&dir.path().join("harness.config.json")),
Some(&dir.path().join("reports/edu/ontology-map.json")),
Some(dir.path()),
)
.unwrap();
let out = dir.path().join("reports/_meta/confidence-calibration.json");
let outcome = calibrate_cmd(&CalibrateArgs {
reports_dir: Some(&dir.path().join("reports")),
config: Some(&dir.path().join("harness.config.json")),
catalog: Some(&dir.path().join(".claude/enabled-packs.json")),
root: Some(dir.path()),
target_precision: 1.0,
tier2_target: 0.5,
sample: None,
seed: 0,
out: Some(&out),
confusions: None,
})
.unwrap();
assert_eq!(outcome.exit_code, 0);
let cal = mif_ontology::CalibrationConfig::load_or_default(&out).unwrap();
assert!(!cal.negatives_active);
}
#[test]
fn calibrate_writes_the_confusion_export_even_when_the_sweep_fails() {
if mif_embed::Embedder::load().is_err() {
eprintln!("skipping: embedding model unavailable in this environment");
return;
}
let dir = tempfile::tempdir().unwrap();
write_fixture(dir.path());
enrich_fixture_pack(dir.path());
let confusions_path = dir.path().join("reports/_meta/confusions.json");
let result = calibrate_cmd(&CalibrateArgs {
reports_dir: Some(&dir.path().join("reports")),
config: Some(&dir.path().join("harness.config.json")),
catalog: Some(&dir.path().join(".claude/enabled-packs.json")),
root: Some(dir.path()),
target_precision: 1.0,
tier2_target: 0.5,
sample: None,
seed: 0,
out: Some(&dir.path().join("reports/_meta/confidence-calibration.json")),
confusions: Some(&confusions_path),
});
let error = result.map(|_| ()).unwrap_err();
assert!(error.to_string().contains("no stamped"));
let report: serde_json::Value =
serde_json::from_str(&fs::read_to_string(&confusions_path).unwrap()).unwrap();
assert_eq!(report["version"], "confusions-v1");
assert_eq!(report["sample_count"], 0);
assert!(report["pairs"].as_array().unwrap().is_empty());
}
#[test]
fn review_suggest_writes_a_topic_queue_for_followup_findings() {
if mif_embed::Embedder::load().is_err() {
eprintln!("skipping: embedding model unavailable in this environment");
return;
}
let dir = tempfile::tempdir().unwrap();
write_fixture(dir.path());
enrich_fixture_pack(dir.path());
fs::write(
dir.path().join("reports/edu/findings/untyped.json"),
r#"{"@id":"f-untyped","content":"A fascinating textbook about geometry"}"#,
)
.unwrap();
let outcome = review(&ReviewArgs {
topics: &[],
strict: false,
reports_dir: Some(&dir.path().join("reports")),
config: Some(&dir.path().join("harness.config.json")),
catalog: Some(&dir.path().join(".claude/enabled-packs.json")),
followup: None,
root: Some(dir.path()),
relationship_script: None,
build_index: false,
index: None,
suggest: true,
calibration: None,
})
.unwrap();
let lines: Vec<&str> = outcome.message.lines().collect();
let confirmation = lines
.iter()
.position(|l| l.starts_with("ontology-review: suggestions written"))
.expect("suggestions confirmation line present");
assert!(confirmation < lines.len() - 1);
let queue_path = dir.path().join("reports/_meta/suggestions/edu.json");
let queue: serde_json::Value =
serde_json::from_str(&fs::read_to_string(&queue_path).unwrap()).unwrap();
let entries = queue["entries"].as_array().unwrap();
assert!(
entries
.iter()
.any(|e| e["finding_id"] == "f-untyped" && e["status"] == "pending")
);
}
#[test]
fn suggest_type_ranks_a_described_type_from_a_finding_query() {
if mif_embed::Embedder::load().is_err() {
eprintln!("skipping: embedding model unavailable in this environment");
return;
}
let dir = tempfile::tempdir().unwrap();
write_fixture(dir.path());
fs::write(
dir.path().join("packs/edu-fixture.yaml"),
"ontology:\n id: edu-fixture\n version: \"0.1.0\"\nentity_types:\n - name: title\n description: A published educational title\n aliases: [textbook]\n schema:\n required: [name]\n properties: {name: {type: string}}\n",
)
.unwrap();
let outcome = suggest_type_cmd(&SuggestTypeArgs {
text: None,
finding: Some(&dir.path().join("reports/edu/findings/good.json")),
topic: None, catalog: Some(&dir.path().join(".claude/enabled-packs.json")),
config: Some(&dir.path().join("harness.config.json")),
root: Some(dir.path()),
limit: 10,
calibration: Some(&dir.path().join("absent-calibration.json")),
record: false,
index: None,
})
.unwrap();
assert_eq!(outcome.exit_code, 0);
let suggestions: serde_json::Value = serde_json::from_str(&outcome.message).unwrap();
let list = suggestions.as_array().unwrap();
assert_eq!(list.len(), 1);
assert_eq!(list[0]["entity_type"], "title");
assert!(list[0]["tier"].is_string());
assert_eq!(list[0]["calibrated"], false);
}
#[test]
fn review_strict_fails_closed_on_an_invalid_finding_but_succeeds_without_strict() {
let dir = tempfile::tempdir().unwrap();
write_fixture(dir.path());
let strict_outcome = review(&ReviewArgs {
topics: &[],
strict: true,
reports_dir: Some(&dir.path().join("reports")),
config: Some(&dir.path().join("harness.config.json")),
catalog: Some(&dir.path().join(".claude/enabled-packs.json")),
followup: None,
root: Some(dir.path()),
relationship_script: None,
build_index: false,
index: None,
suggest: false,
calibration: None,
})
.unwrap();
assert_eq!(strict_outcome.exit_code, 1);
assert!(strict_outcome.message.contains("1 invalid/unresolved"));
let lenient_outcome = review(&ReviewArgs {
topics: &[],
strict: false,
reports_dir: Some(&dir.path().join("reports")),
config: Some(&dir.path().join("harness.config.json")),
catalog: Some(&dir.path().join(".claude/enabled-packs.json")),
followup: None,
root: Some(dir.path()),
relationship_script: None,
build_index: false,
index: None,
suggest: false,
calibration: None,
})
.unwrap();
assert_eq!(lenient_outcome.exit_code, 0);
}
#[test]
fn review_acquires_and_releases_the_lock_and_prints_the_topic_table() {
let dir = tempfile::tempdir().unwrap();
write_fixture(dir.path());
let outcome = review(&ReviewArgs {
topics: &[],
strict: false,
reports_dir: Some(&dir.path().join("reports")),
config: Some(&dir.path().join("harness.config.json")),
catalog: Some(&dir.path().join(".claude/enabled-packs.json")),
followup: None,
root: Some(dir.path()),
relationship_script: None,
build_index: false,
index: None,
suggest: false,
calibration: None,
})
.unwrap();
assert!(outcome.message.contains("TOPIC"), "{}", outcome.message);
assert!(outcome.message.contains("edu"), "{}", outcome.message);
assert!(
!dir.path().join("reports/_meta/.review.lock").exists(),
"lock file must be released after review() returns"
);
}
#[test]
fn review_with_followup_prints_the_summary_line_last() {
let dir = tempfile::tempdir().unwrap();
write_fixture(dir.path());
let followup_path = dir.path().join("followup.json");
let outcome = review(&ReviewArgs {
topics: &[],
strict: false,
reports_dir: Some(&dir.path().join("reports")),
config: Some(&dir.path().join("harness.config.json")),
catalog: Some(&dir.path().join(".claude/enabled-packs.json")),
followup: Some(&followup_path),
root: Some(dir.path()),
relationship_script: None,
build_index: false,
index: None,
suggest: false,
calibration: None,
})
.unwrap();
let last_line = outcome.message.lines().next_back().unwrap();
assert!(
last_line.starts_with("1 topic(s);"),
"last line should be the summary, got: {last_line}"
);
}
#[test]
fn review_with_build_index_populates_the_default_index_path() {
if mif_embed::Embedder::load().is_err() {
eprintln!("skipping: could not load embedding model");
return;
}
let dir = tempfile::tempdir().unwrap();
write_fixture(dir.path());
review(&ReviewArgs {
topics: &[],
strict: false,
reports_dir: Some(&dir.path().join("reports")),
config: Some(&dir.path().join("harness.config.json")),
catalog: Some(&dir.path().join(".claude/enabled-packs.json")),
followup: None,
root: Some(dir.path()),
relationship_script: None,
build_index: true,
index: None,
suggest: false,
calibration: None,
})
.unwrap();
assert!(
dir.path()
.join("reports/_meta/search-index.sqlite")
.exists()
);
}
#[test]
fn topic_from_path_derives_the_topic_from_a_reports_relative_finding_path() {
assert_eq!(
topic_from_path(std::path::Path::new("reports/edu/findings/f.json")),
Some("edu".to_string())
);
assert_eq!(
topic_from_path(std::path::Path::new("/some/other/path.json")),
None
);
}
}