oxibrain_cli/cmd/
eval.rs

1//! `oxibrain eval` — extraction evaluation suite (DESIGN §14.2).
2//!
3//! `fast` replays fixture responses through FakeLlmPort — no network, deterministic.
4//! `full` requires a live provider (nightly only).
5
6use oxibrain::Brain;
7use oxibrain_core::eval::{
8    ExtractedTriple, compute_metrics_with_fabrication, measure_fabrication_rate,
9};
10use oxibrain_core::extraction::{ExtractMechanism, ExtractorConfig};
11use oxibrain_core::{SourceRef, TrustTier};
12use oxibrain_ports::{FakeClock, FakeLlmPort, LlmResponse, Timestamp};
13
14struct GoldenFixture {
15    content: &'static str,
16    canned_response: &'static str,
17    expected_triples: Vec<ExtractedTriple>,
18}
19
20fn fixtures() -> Vec<GoldenFixture> {
21    vec![
22        GoldenFixture {
23            content: "Alice works on ProjectX at Acme Corp",
24            canned_response: r#"{"claims":[
25                {"predicate":"works_on","subject":{"surface":"Alice","entity_type":"Person","span":[0,5]},"object":{"kind":"entity","mention":{"surface":"ProjectX","entity_type":"Project","span":[15,23]}},"polarity":"affirm","confidence":0.95},
26                {"predicate":"employed_by","subject":{"surface":"Alice","entity_type":"Person","span":[0,5]},"object":{"kind":"entity","mention":{"surface":"Acme Corp","entity_type":"Organization","span":[27,36]}},"polarity":"affirm","confidence":0.9}
27            ]}"#,
28            expected_triples: vec![
29                triple("works_on", "Alice", "ProjectX"),
30                triple("employed_by", "Alice", "Acme Corp"),
31            ],
32        },
33        GoldenFixture {
34            content: "Bob knows Carol. Bob was born in Seoul.",
35            canned_response: r#"{"claims":[
36                {"predicate":"knows","subject":{"surface":"Bob","entity_type":"Person","span":[0,3]},"object":{"kind":"entity","mention":{"surface":"Carol","entity_type":"Person","span":[10,15]}},"polarity":"affirm","confidence":0.9},
37                {"predicate":"born_in","subject":{"surface":"Bob","entity_type":"Person","span":[17,20]},"object":{"kind":"entity","mention":{"surface":"Seoul","entity_type":"Place","span":[33,38]}},"polarity":"affirm","confidence":0.95}
38            ]}"#,
39            expected_triples: vec![
40                triple("knows", "Bob", "Carol"),
41                triple("born_in", "Bob", "Seoul"),
42            ],
43        },
44        GoldenFixture {
45            content: "Alice full name is Alice Smith.",
46            canned_response: r#"{"claims":[
47                {"predicate":"full_name","subject":{"surface":"Alice","entity_type":"Person","span":[0,5]},"object":{"kind":"literal","literal_type":"text","value":"Alice Smith","span":[18,30]},"polarity":"affirm","confidence":0.95}
48            ]}"#,
49            expected_triples: vec![triple("full_name", "Alice", "Alice Smith")],
50        },
51    ]
52}
53
54fn triple(p: &str, s: &str, o: &str) -> ExtractedTriple {
55    ExtractedTriple {
56        predicate: p.into(),
57        subject_surface: s.into(),
58        object_surface: o.into(),
59    }
60}
61
62fn test_extractor() -> ExtractorConfig {
63    ExtractorConfig {
64        model_id: "test-model".into(),
65        prompt_version: 2, // v2: quote-based mentions (ADR-006); fixtures replay legacy spans via the compat ladder
66        registry_major: 1,
67        mechanism: ExtractMechanism::JsonSchema,
68        max_tokens: 4096,
69        model_digest: None,
70        provider_profile_id: None,
71    }
72}
73
74pub async fn run(suite: &str) -> anyhow::Result<()> {
75    match suite {
76        "fast" => run_fast().await,
77        "full" => {
78            anyhow::bail!("full suite requires a live provider — run via CI nightly, not locally")
79        }
80        "gate" => super::gate::run_with_dir(suite, None).await,
81        other => anyhow::bail!("unknown suite '{other}': use 'fast', 'full', or 'gate'"),
82    }
83}
84
85async fn run_fast() -> anyhow::Result<()> {
86    let fixtures = fixtures();
87    let extractor = test_extractor();
88    let mut all_extracted = Vec::new();
89    let mut all_expected = Vec::new();
90    let mut all_entity_surfaces: Vec<String> = Vec::new();
91
92    for fixture in &fixtures {
93        let dir = tempfile::TempDir::new()?;
94        let config = oxibrain::BrainConfig::at(dir.path());
95
96        let clock = std::sync::Arc::new(FakeClock::new(Timestamp::from_millis(10000)));
97        let llm = std::sync::Arc::new(FakeLlmPort::new());
98        llm.respond_to(
99            &fixture.content[..20.min(fixture.content.len())],
100            LlmResponse {
101                text: fixture.canned_response.into(),
102                raw: serde_json::Value::Null,
103            },
104        );
105
106        let brain = Brain::with_llm(config, clock, llm).await?;
107        let space = brain.ensure_space("eval").await?;
108
109        let ep_id = brain
110            .ingest(
111                &space,
112                fixture.content.into(),
113                SourceRef::Note {
114                    path: "fixture.md".into(),
115                },
116                TrustTier::Trusted,
117                &extractor.id(),
118            )
119            .await?;
120
121        let summary = brain.extract_one(&space, &ep_id, &extractor).await?;
122
123        if summary.extracted == 0 {
124            anyhow::bail!("fixture extracted 0 claims: {}", fixture.content);
125        }
126
127        let extracted = brain.debug_triples(&space).await?;
128        // Collect entity surfaces for fabrication measurement (§17.3, 10.7).
129        for t in &extracted {
130            all_entity_surfaces.push(t.subject_surface.clone());
131            all_entity_surfaces.push(t.object_surface.clone());
132        }
133        all_extracted.extend(extracted);
134        all_expected.extend(fixture.expected_triples.clone());
135    }
136
137    // Fabricated entity rate measured from source text (§17.3, 10.7).
138    // Each entity surface must appear verbatim in some fixture's content; the
139    // validator enforces this and measure_fabrication_rate proves it. No
140    // hardcoded 0.0.
141    let combined_source: String = fixtures
142        .iter()
143        .map(|f| f.content)
144        .collect::<Vec<_>>()
145        .join(" ");
146    let global_rate = measure_fabrication_rate(&all_entity_surfaces, &combined_source);
147
148    let metrics = compute_metrics_with_fabrication(&all_extracted, &all_expected, global_rate);
149    println!();
150    println!(
151        "Fabricated entity rate: {:.3}  (gate: 0.000)",
152        metrics.fabricated_entity_rate
153    );
154    println!(
155        "Statement precision:    {:.3}  (gate: ≥ 0.90)",
156        metrics.statement_precision
157    );
158    println!(
159        "Statement recall:       {:.3}  (gate: ≥ 0.70)",
160        metrics.statement_recall
161    );
162    println!();
163    println!("Extracted: {all_extracted:?}");
164    println!("Expected:  {all_expected:?}");
165
166    match metrics.check_gates() {
167        Ok(()) => {
168            println!();
169            println!("✅ All §14.2 quality gates passed.");
170            Ok(())
171        }
172        Err(e) => {
173            println!();
174            println!("❌ Quality gates failed: {e}");
175            std::process::exit(1);
176        }
177    }
178}
oxibrain_cli/cmd/eval.rs

oxibrain_cli/cmd/
eval.rs
