eidos_kernel/
eval.rs

1//! Retrieval eval — score `ground` against a golden set so graph changes are MEASURED,
2//! not asserted. Mirrors the discipline of the Python `test_retrieval_eval`, but runs over
3//! whatever graph it's handed (a hermetic fixture in tests; the real vault from the CLI).
4//!
5//! Metrics (over the CLEAN cases): `p_at_1` (top hit is an expected id), `hit_at_k` (an
6//! expected id appears within the returned hits), `mrr` (mean reciprocal rank of the first
7//! expected hit). Over the GARBAGE cases: `garbage_rejected` (the engine did NOT answer with
8//! false confidence — top hit is weak/fallback, or there were no hits at all).
9
10use serde::{Deserialize, Serialize};
11
12use crate::graph_index::GraphIndex;
13use crate::retrieval::RelationMatch;
14mod combo;
15mod context;
16mod dogfood;
17mod route;
18mod workflow;
19
20pub use combo::{ComboKind, ComboSpec, ComboSpecError, ComboValidationError, classify_combo_case};
21pub use context::{ContextCaseResult, ContextReport, evaluate_context};
22pub use dogfood::{
23    DogfoodCaseResult, DogfoodContextJudgment, DogfoodDifficultyStat, DogfoodProtocolComparison,
24    DogfoodProtocolInput, DogfoodProtocolRun, DogfoodProtocolSummary, DogfoodReport,
25    advance_dogfood_context_step, compare_protocol_economics, dogfood_case_difficulty,
26    dogfood_operator_label, dogfood_task_context_is_useful, evaluate_dogfood, is_dogfood_code_kind,
27    is_dogfood_docs_kind, judge_dogfood_context, score_dogfood_checks, summarize_dogfood,
28};
29pub use route::{CaseResult, ClassStat, EvalReport, evaluate, evaluate_with_config};
30pub use workflow::{
31    WorkflowCaseResult, WorkflowInspectionJudgment, WorkflowReport, evaluate_workflow,
32    has_required_operator_chain, judge_required_checks, judge_risk_hints,
33    judge_workflow_inspections, summarize_workflow, workflow_initial_ready_context_only,
34};
35
36/// One golden judgment. `expect` is the set of acceptable node ids (any one counts as a
37/// correct top hit). `garbage` marks a nonsense query that MUST be rejected.
38#[derive(Deserialize, Serialize, Default, Clone, Debug)]
39pub struct GoldenCase {
40    pub query: String,
41    #[serde(default)]
42    pub expect: Vec<String>,
43    #[serde(default)]
44    pub garbage: bool,
45    /// Node ids that SHOULD appear in the activated context for this query (support recall).
46    #[serde(default)]
47    pub context_must: Vec<String>,
48    /// Node ids that should NOT appear in the context (noise — e.g. a spurious mention leak).
49    #[serde(default)]
50    pub context_must_not: Vec<String>,
51    /// Directed relation edges that SHOULD appear in the focused context.
52    #[serde(default)]
53    pub context_edges_must: Vec<ContextEdgeExpectation>,
54    /// Structured relation evidence that SHOULD justify the top route hit.
55    #[serde(default)]
56    pub route_relation_must: Vec<RelationMatchExpectation>,
57    /// Optional focused-route node id expected after graph-aware route selection.
58    #[serde(default)]
59    pub expected_focus_route: Option<String>,
60    /// Optional set of focused-route node ids accepted for graph-aware route selection. Use when a
61    /// type and the helper that materializes it are both valid anchors for the same agent task.
62    #[serde(default)]
63    pub expected_focus_routes: Vec<String>,
64    /// Optional compound primary node id expected for multi-anchor/faceted focus queries.
65    #[serde(default)]
66    pub expected_compound_primary: Option<String>,
67    /// Vocabulary-gap terms expected from compound focus.
68    #[serde(default)]
69    pub expected_vocabulary_gap_terms: Vec<String>,
70    /// Whether compound focus is expected to produce no vocabulary gap.
71    #[serde(default)]
72    pub expected_no_vocabulary_gap: bool,
73    /// Editable vocabulary-repair targets expected from compound focus.
74    #[serde(default)]
75    pub expected_vocab_editable_targets: Vec<String>,
76    /// Non-editable vocabulary-repair targets expected from compound focus.
77    #[serde(default)]
78    pub expected_vocab_non_editable_targets: Vec<String>,
79    /// Context candidates that should be reported as considered but omitted from focus.
80    #[serde(default)]
81    pub expected_omitted_context: Vec<ContextOmissionExpectation>,
82    /// Compound anchors that should be reported as considered but omitted from the selected
83    /// multi-anchor interpretation.
84    #[serde(default)]
85    pub expected_omitted_compound_anchors: Vec<ContextOmissionExpectation>,
86    /// Optional minimum confidence band required for the top route hit.
87    #[serde(default)]
88    pub min_confidence: Option<String>,
89    /// Optional exact confidence band expected for the top route hit.
90    #[serde(default)]
91    pub expected_confidence: Option<String>,
92    /// Optional generated EKF partition expected for the top/focused route node.
93    #[serde(default)]
94    pub expected_partition: Option<String>,
95    /// Optional package combo evaluator id, e.g. `ground_gate` or `verify_receipt`.
96    #[serde(default)]
97    pub combo: Option<String>,
98    /// Optional tool id used by combo evaluators that exercise a concrete MCP/engine tool.
99    #[serde(default)]
100    pub tool: Option<String>,
101    /// Optional target node for combo evaluators that attach an operation to an existing node.
102    #[serde(default)]
103    pub target_node: Option<String>,
104    /// Expected boolean result for combo evaluators with pass/fail semantics.
105    #[serde(default)]
106    pub expected_pass: Option<bool>,
107    /// Receipt file for `verify_receipt` combo cases. Relative paths resolve from package root.
108    #[serde(default)]
109    pub file: Option<String>,
110    /// 1-indexed receipt start line for `verify_receipt`.
111    #[serde(default)]
112    pub line_start: Option<usize>,
113    /// Optional 1-indexed receipt end line for `verify_receipt`.
114    #[serde(default)]
115    pub line_end: Option<usize>,
116    /// Expected quote for `verify_receipt`.
117    #[serde(default)]
118    pub quote: Option<String>,
119    /// Expected `verify_receipt` result.
120    #[serde(default)]
121    pub expected_holds: Option<bool>,
122    /// Optional intent-class tag (capability, exact-lookup, troubleshoot, ambiguous, garbage-*, …)
123    /// — aggregated per class in the report so a large stratified set shows where it's weak.
124    #[serde(default)]
125    pub class: Option<String>,
126    /// Optional dogfood difficulty bucket: easy, intermediate, or complex.
127    #[serde(default)]
128    pub difficulty: Option<String>,
129    /// Whether the task should pull at least one code node into focused context.
130    #[serde(default)]
131    pub requires_code: bool,
132    /// Whether the task should pull at least one docs/skill/section node into focused context.
133    #[serde(default)]
134    pub requires_docs: bool,
135    /// Operators that should appear in the brief decomposition, serialized as snake_case names.
136    #[serde(default)]
137    pub expected_operators: Vec<String>,
138    /// Required checks that should be present in the selected brief/decomposition.
139    #[serde(default)]
140    pub expected_required_checks: Vec<String>,
141    /// Risk/review hint titles that should be present in the selected brief/decomposition.
142    #[serde(default)]
143    pub expected_risk_hints: Vec<String>,
144    /// EKF workflow profile id expected for package-level workflow planning.
145    #[serde(default)]
146    pub expected_profile: Option<String>,
147}
148
149#[derive(Deserialize, Serialize, Default, Clone, Debug, PartialEq, Eq)]
150pub struct ContextEdgeExpectation {
151    pub from: String,
152    pub to: String,
153    pub relation: String,
154}
155
156#[derive(Deserialize, Serialize, Default, Clone, Debug, PartialEq, Eq)]
157pub struct ContextOmissionExpectation {
158    pub node_id: String,
159    pub reason: String,
160}
161
162#[derive(Deserialize, Serialize, Default, Clone, Debug, PartialEq, Eq)]
163pub struct RelationMatchExpectation {
164    pub relation: String,
165    #[serde(default)]
166    pub direction: Option<String>,
167    #[serde(default)]
168    pub endpoint_id: Option<String>,
169    #[serde(default)]
170    pub endpoint_title: Option<String>,
171    #[serde(default)]
172    pub phrase: Option<String>,
173}
174
175pub fn missing_relation_matches(
176    expected_matches: &[RelationMatchExpectation],
177    relation_matches: &[RelationMatch],
178) -> Vec<RelationMatchExpectation> {
179    expected_matches
180        .iter()
181        .filter(|expected| !relation_match_present(expected, relation_matches))
182        .cloned()
183        .collect()
184}
185
186fn relation_match_present(
187    expected: &RelationMatchExpectation,
188    relation_matches: &[RelationMatch],
189) -> bool {
190    relation_matches.iter().any(|actual| {
191        actual.relation == expected.relation
192            && expected
193                .direction
194                .as_deref()
195                .is_none_or(|direction| actual.direction == direction)
196            && expected
197                .endpoint_id
198                .as_deref()
199                .is_none_or(|endpoint_id| actual.endpoint_id == endpoint_id)
200            && expected
201                .endpoint_title
202                .as_deref()
203                .is_none_or(|endpoint_title| actual.endpoint_title == endpoint_title)
204            && expected
205                .phrase
206                .as_deref()
207                .is_none_or(|phrase| actual.phrase == phrase)
208    })
209}
210
211/// A golden set file: `{ "cases": [ {query, expect|garbage}, ... ] }`.
212#[derive(Deserialize, Serialize, Default, Clone, Debug)]
213pub struct GoldenSet {
214    #[serde(default)]
215    pub cases: Vec<GoldenCase>,
216}
217
218fn kind_label(kind: crate::schema::Kind) -> String {
219    serde_json::to_value(kind)
220        .ok()
221        .and_then(|value| value.as_str().map(ToOwned::to_owned))
222        .unwrap_or_else(|| "unknown".to_string())
223}
224
225fn receipt_coverage(index: &GraphIndex<'_>, context_order: &[String]) -> f64 {
226    if context_order.is_empty() {
227        return 0.0;
228    }
229    let receipted = context_order
230        .iter()
231        .filter(|id| {
232            index
233                .node(id)
234                .is_some_and(|n| n.span.is_some() || !n.source_files.is_empty())
235        })
236        .count();
237    receipted as f64 / context_order.len() as f64
238}
239
240#[cfg(test)]
241mod tests {
242    use super::*;
243    use crate::schema::{Edge, EdgeBasis, Graph, Kind, Node};
244
245    #[test]
246    fn garbage_rejection_is_unmeasured_without_garbage_cases() {
247        let report = evaluate(
248            &Graph::default(),
249            &[GoldenCase {
250                query: "missing clean target".to_string(),
251                expect: vec!["doc.missing".to_string()],
252                ..Default::default()
253            }],
254            5,
255        );
256
257        assert_eq!(report.clean, 1);
258        assert_eq!(report.garbage, 0);
259        assert_eq!(report.garbage_rejected, None);
260    }
261
262    #[test]
263    fn garbage_rejection_is_measured_for_garbage_cases() {
264        let report = evaluate(
265            &Graph::default(),
266            &[GoldenCase {
267                query: "absent infrastructure scheduler".to_string(),
268                garbage: true,
269                ..Default::default()
270            }],
271            5,
272        );
273
274        assert_eq!(report.clean, 0);
275        assert_eq!(report.garbage, 1);
276        assert_eq!(report.garbage_rejected, Some(1.0));
277    }
278
279    #[test]
280    fn answer_precision_scores_exact_clean_answers() {
281        let graph = graph_with_doc("doc.answer", "answer");
282        let report = evaluate(
283            &graph,
284            &[GoldenCase {
285                query: "answer".to_string(),
286                expect: vec!["doc.answer".to_string()],
287                ..Default::default()
288            }],
289            5,
290        );
291
292        assert_eq!(report.answer_count, 1);
293        assert_eq!(report.answer_precision, Some(1.0));
294    }
295
296    #[test]
297    fn answer_precision_penalizes_confident_garbage_answers() {
298        let graph = graph_with_doc("doc.answer", "answer");
299        let report = evaluate(
300            &graph,
301            &[GoldenCase {
302                query: "answer".to_string(),
303                garbage: true,
304                ..Default::default()
305            }],
306            5,
307        );
308
309        assert_eq!(report.answer_count, 1);
310        assert_eq!(report.answer_precision, Some(0.0));
311    }
312
313    #[test]
314    fn route_eval_can_require_structured_relation_evidence() {
315        let graph = relation_graph();
316        let report = evaluate(
317            &graph,
318            &[GoldenCase {
319                query: "depends on auth contract".to_string(),
320                expect: vec!["doc.router".to_string()],
321                route_relation_must: vec![RelationMatchExpectation {
322                    relation: "depends_on".to_string(),
323                    direction: Some("forward".to_string()),
324                    endpoint_id: Some("doc.auth-contract".to_string()),
325                    phrase: Some("depends on".to_string()),
326                    ..Default::default()
327                }],
328                ..Default::default()
329            }],
330            5,
331        );
332
333        assert_eq!(report.p_at_1, 1.0);
334        assert_eq!(report.relation_evidence_recall, Some(1.0));
335        assert_eq!(report.relation_evidence_expected, 1);
336        assert_eq!(report.relation_evidence_missing, 0);
337        assert!(report.cases[0].ok);
338        assert_eq!(report.cases[0].relation_matches.len(), 1);
339        assert!(report.cases[0].missing_relation_matches.is_empty());
340    }
341
342    #[test]
343    fn route_eval_fails_when_required_relation_evidence_is_missing() {
344        let graph = relation_graph();
345        let report = evaluate(
346            &graph,
347            &[GoldenCase {
348                query: "depends on auth contract".to_string(),
349                expect: vec!["doc.router".to_string()],
350                route_relation_must: vec![RelationMatchExpectation {
351                    relation: "verifies".to_string(),
352                    endpoint_id: Some("doc.auth-contract".to_string()),
353                    ..Default::default()
354                }],
355                ..Default::default()
356            }],
357            5,
358        );
359
360        assert_eq!(report.cases[0].rank, Some(1));
361        assert_eq!(report.p_at_1, 0.0);
362        assert_eq!(report.relation_evidence_recall, Some(0.0));
363        assert_eq!(report.relation_evidence_expected, 1);
364        assert_eq!(report.relation_evidence_missing, 1);
365        assert!(!report.cases[0].ok);
366        assert_eq!(report.cases[0].missing_relation_matches.len(), 1);
367    }
368
369    #[test]
370    fn route_eval_relation_recall_is_unmeasured_without_relation_judgments() {
371        let graph = graph_with_doc("doc.answer", "answer");
372        let report = evaluate(
373            &graph,
374            &[GoldenCase {
375                query: "answer".to_string(),
376                expect: vec!["doc.answer".to_string()],
377                ..Default::default()
378            }],
379            5,
380        );
381
382        assert_eq!(report.relation_evidence_recall, None);
383        assert_eq!(report.relation_evidence_expected, 0);
384        assert_eq!(report.relation_evidence_missing, 0);
385    }
386
387    #[test]
388    fn route_eval_can_require_confidence_bands() {
389        let graph = graph_with_doc("doc.answer", "answer");
390        let report = evaluate(
391            &graph,
392            &[
393                GoldenCase {
394                    query: "answer".to_string(),
395                    expect: vec!["doc.answer".to_string()],
396                    min_confidence: Some("ambiguous".to_string()),
397                    ..Default::default()
398                },
399                GoldenCase {
400                    query: "answer".to_string(),
401                    expect: vec!["doc.answer".to_string()],
402                    expected_confidence: Some("fallback".to_string()),
403                    ..Default::default()
404                },
405            ],
406            5,
407        );
408
409        assert_eq!(report.confidence_expectation_rate, Some(0.5));
410        assert_eq!(report.confidence_expected, 2);
411        assert_eq!(report.confidence_missing, 1);
412        assert_eq!(report.p_at_1, 0.5);
413        assert_eq!(report.cases[0].confidence_ok, Some(true));
414        assert_eq!(report.cases[1].confidence_ok, Some(false));
415    }
416
417    fn graph_with_doc(id: &str, title: &str) -> Graph {
418        Graph {
419            nodes: vec![Node {
420                id: id.to_string(),
421                kind: Kind::Doc,
422                subkind: None,
423                title: title.to_string(),
424                summary: String::new(),
425                aliases: Vec::new(),
426                tags: Vec::new(),
427                query_examples: Vec::new(),
428                source_files: vec![format!("fixtures/{title}.md")],
429                span: None,
430                partition: None,
431            }],
432            edges: Vec::new(),
433            ..Default::default()
434        }
435    }
436
437    fn relation_graph() -> Graph {
438        Graph {
439            nodes: vec![
440                Node {
441                    id: "doc.router".to_string(),
442                    kind: Kind::Doc,
443                    subkind: None,
444                    title: "Router".to_string(),
445                    summary: "Request routing policy.".to_string(),
446                    aliases: Vec::new(),
447                    tags: Vec::new(),
448                    query_examples: Vec::new(),
449                    source_files: vec!["router.md".to_string()],
450                    span: None,
451                    partition: None,
452                },
453                Node {
454                    id: "doc.auth-contract".to_string(),
455                    kind: Kind::Doc,
456                    subkind: None,
457                    title: "Auth Contract".to_string(),
458                    summary: "Authentication requirements.".to_string(),
459                    aliases: Vec::new(),
460                    tags: Vec::new(),
461                    query_examples: Vec::new(),
462                    source_files: vec!["auth.md".to_string()],
463                    span: None,
464                    partition: None,
465                },
466            ],
467            edges: vec![Edge {
468                from: "doc.router".to_string(),
469                to: "doc.auth-contract".to_string(),
470                relation: "depends_on".to_string(),
471                evidence: "test".to_string(),
472                basis: EdgeBasis::Resolved,
473                ..Default::default()
474            }],
475            ..Default::default()
476        }
477    }
478}
eidos_kernel/eval.rs

eidos_kernel/
eval.rs
