eidos-kernel 0.1.0

//! Retrieval eval — score `ground` against a golden set so graph changes are MEASURED,
//! not asserted. Mirrors the discipline of the Python `test_retrieval_eval`, but runs over
//! whatever graph it's handed (a hermetic fixture in tests; the real vault from the CLI).
//!
//! Metrics (over the CLEAN cases): `p_at_1` (top hit is an expected id), `hit_at_k` (an
//! expected id appears within the returned hits), `mrr` (mean reciprocal rank of the first
//! expected hit). Over the GARBAGE cases: `garbage_rejected` (the engine did NOT answer with
//! false confidence — top hit is weak/fallback, or there were no hits at all).

use serde::{Deserialize, Serialize};

use crate::graph_index::GraphIndex;
use crate::retrieval::RelationMatch;
mod combo;
mod context;
mod dogfood;
mod route;
mod workflow;

pub use combo::{ComboKind, ComboSpec, ComboSpecError, ComboValidationError, classify_combo_case};
pub use context::{ContextCaseResult, ContextReport, evaluate_context};
pub use dogfood::{
    DogfoodCaseResult, DogfoodContextJudgment, DogfoodDifficultyStat, DogfoodProtocolComparison,
    DogfoodProtocolInput, DogfoodProtocolRun, DogfoodProtocolSummary, DogfoodReport,
    advance_dogfood_context_step, compare_protocol_economics, dogfood_case_difficulty,
    dogfood_operator_label, dogfood_task_context_is_useful, evaluate_dogfood, is_dogfood_code_kind,
    is_dogfood_docs_kind, judge_dogfood_context, score_dogfood_checks, summarize_dogfood,
};
pub use route::{CaseResult, ClassStat, EvalReport, evaluate, evaluate_with_config};
pub use workflow::{
    WorkflowCaseResult, WorkflowInspectionJudgment, WorkflowReport, evaluate_workflow,
    has_required_operator_chain, judge_required_checks, judge_risk_hints,
    judge_workflow_inspections, summarize_workflow, workflow_initial_ready_context_only,
};

/// One golden judgment. `expect` is the set of acceptable node ids (any one counts as a
/// correct top hit). `garbage` marks a nonsense query that MUST be rejected.
#[derive(Deserialize, Serialize, Default, Clone, Debug)]
pub struct GoldenCase {
    pub query: String,
    #[serde(default)]
    pub expect: Vec<String>,
    #[serde(default)]
    pub garbage: bool,
    /// Node ids that SHOULD appear in the activated context for this query (support recall).
    #[serde(default)]
    pub context_must: Vec<String>,
    /// Node ids that should NOT appear in the context (noise — e.g. a spurious mention leak).
    #[serde(default)]
    pub context_must_not: Vec<String>,
    /// Directed relation edges that SHOULD appear in the focused context.
    #[serde(default)]
    pub context_edges_must: Vec<ContextEdgeExpectation>,
    /// Structured relation evidence that SHOULD justify the top route hit.
    #[serde(default)]
    pub route_relation_must: Vec<RelationMatchExpectation>,
    /// Optional focused-route node id expected after graph-aware route selection.
    #[serde(default)]
    pub expected_focus_route: Option<String>,
    /// Optional set of focused-route node ids accepted for graph-aware route selection. Use when a
    /// type and the helper that materializes it are both valid anchors for the same agent task.
    #[serde(default)]
    pub expected_focus_routes: Vec<String>,
    /// Optional compound primary node id expected for multi-anchor/faceted focus queries.
    #[serde(default)]
    pub expected_compound_primary: Option<String>,
    /// Vocabulary-gap terms expected from compound focus.
    #[serde(default)]
    pub expected_vocabulary_gap_terms: Vec<String>,
    /// Whether compound focus is expected to produce no vocabulary gap.
    #[serde(default)]
    pub expected_no_vocabulary_gap: bool,
    /// Editable vocabulary-repair targets expected from compound focus.
    #[serde(default)]
    pub expected_vocab_editable_targets: Vec<String>,
    /// Non-editable vocabulary-repair targets expected from compound focus.
    #[serde(default)]
    pub expected_vocab_non_editable_targets: Vec<String>,
    /// Context candidates that should be reported as considered but omitted from focus.
    #[serde(default)]
    pub expected_omitted_context: Vec<ContextOmissionExpectation>,
    /// Compound anchors that should be reported as considered but omitted from the selected
    /// multi-anchor interpretation.
    #[serde(default)]
    pub expected_omitted_compound_anchors: Vec<ContextOmissionExpectation>,
    /// Optional minimum confidence band required for the top route hit.
    #[serde(default)]
    pub min_confidence: Option<String>,
    /// Optional exact confidence band expected for the top route hit.
    #[serde(default)]
    pub expected_confidence: Option<String>,
    /// Optional generated EKF partition expected for the top/focused route node.
    #[serde(default)]
    pub expected_partition: Option<String>,
    /// Optional package combo evaluator id, e.g. `ground_gate` or `verify_receipt`.
    #[serde(default)]
    pub combo: Option<String>,
    /// Optional tool id used by combo evaluators that exercise a concrete MCP/engine tool.
    #[serde(default)]
    pub tool: Option<String>,
    /// Optional target node for combo evaluators that attach an operation to an existing node.
    #[serde(default)]
    pub target_node: Option<String>,
    /// Expected boolean result for combo evaluators with pass/fail semantics.
    #[serde(default)]
    pub expected_pass: Option<bool>,
    /// Receipt file for `verify_receipt` combo cases. Relative paths resolve from package root.
    #[serde(default)]
    pub file: Option<String>,
    /// 1-indexed receipt start line for `verify_receipt`.
    #[serde(default)]
    pub line_start: Option<usize>,
    /// Optional 1-indexed receipt end line for `verify_receipt`.
    #[serde(default)]
    pub line_end: Option<usize>,
    /// Expected quote for `verify_receipt`.
    #[serde(default)]
    pub quote: Option<String>,
    /// Expected `verify_receipt` result.
    #[serde(default)]
    pub expected_holds: Option<bool>,
    /// Optional intent-class tag (capability, exact-lookup, troubleshoot, ambiguous, garbage-*, …)
    /// — aggregated per class in the report so a large stratified set shows where it's weak.
    #[serde(default)]
    pub class: Option<String>,
    /// Optional dogfood difficulty bucket: easy, intermediate, or complex.
    #[serde(default)]
    pub difficulty: Option<String>,
    /// Whether the task should pull at least one code node into focused context.
    #[serde(default)]
    pub requires_code: bool,
    /// Whether the task should pull at least one docs/skill/section node into focused context.
    #[serde(default)]
    pub requires_docs: bool,
    /// Operators that should appear in the brief decomposition, serialized as snake_case names.
    #[serde(default)]
    pub expected_operators: Vec<String>,
    /// Required checks that should be present in the selected brief/decomposition.
    #[serde(default)]
    pub expected_required_checks: Vec<String>,
    /// Risk/review hint titles that should be present in the selected brief/decomposition.
    #[serde(default)]
    pub expected_risk_hints: Vec<String>,
    /// EKF workflow profile id expected for package-level workflow planning.
    #[serde(default)]
    pub expected_profile: Option<String>,
}

#[derive(Deserialize, Serialize, Default, Clone, Debug, PartialEq, Eq)]
pub struct ContextEdgeExpectation {
    pub from: String,
    pub to: String,
    pub relation: String,
}

#[derive(Deserialize, Serialize, Default, Clone, Debug, PartialEq, Eq)]
pub struct ContextOmissionExpectation {
    pub node_id: String,
    pub reason: String,
}

#[derive(Deserialize, Serialize, Default, Clone, Debug, PartialEq, Eq)]
pub struct RelationMatchExpectation {
    pub relation: String,
    #[serde(default)]
    pub direction: Option<String>,
    #[serde(default)]
    pub endpoint_id: Option<String>,
    #[serde(default)]
    pub endpoint_title: Option<String>,
    #[serde(default)]
    pub phrase: Option<String>,
}

pub fn missing_relation_matches(
    expected_matches: &[RelationMatchExpectation],
    relation_matches: &[RelationMatch],
) -> Vec<RelationMatchExpectation> {
    expected_matches
        .iter()
        .filter(|expected| !relation_match_present(expected, relation_matches))
        .cloned()
        .collect()
}

fn relation_match_present(
    expected: &RelationMatchExpectation,
    relation_matches: &[RelationMatch],
) -> bool {
    relation_matches.iter().any(|actual| {
        actual.relation == expected.relation
            && expected
                .direction
                .as_deref()
                .is_none_or(|direction| actual.direction == direction)
            && expected
                .endpoint_id
                .as_deref()
                .is_none_or(|endpoint_id| actual.endpoint_id == endpoint_id)
            && expected
                .endpoint_title
                .as_deref()
                .is_none_or(|endpoint_title| actual.endpoint_title == endpoint_title)
            && expected
                .phrase
                .as_deref()
                .is_none_or(|phrase| actual.phrase == phrase)
    })
}

/// A golden set file: `{ "cases": [ {query, expect|garbage}, ... ] }`.
#[derive(Deserialize, Serialize, Default, Clone, Debug)]
pub struct GoldenSet {
    #[serde(default)]
    pub cases: Vec<GoldenCase>,
}

fn kind_label(kind: crate::schema::Kind) -> String {
    serde_json::to_value(kind)
        .ok()
        .and_then(|value| value.as_str().map(ToOwned::to_owned))
        .unwrap_or_else(|| "unknown".to_string())
}

fn receipt_coverage(index: &GraphIndex<'_>, context_order: &[String]) -> f64 {
    if context_order.is_empty() {
        return 0.0;
    }
    let receipted = context_order
        .iter()
        .filter(|id| {
            index
                .node(id)
                .is_some_and(|n| n.span.is_some() || !n.source_files.is_empty())
        })
        .count();
    receipted as f64 / context_order.len() as f64
}

#[cfg(test)]
mod tests {
    use super::*;
    use crate::schema::{Edge, EdgeBasis, Graph, Kind, Node};

    #[test]
    fn garbage_rejection_is_unmeasured_without_garbage_cases() {
        let report = evaluate(
            &Graph::default(),
            &[GoldenCase {
                query: "missing clean target".to_string(),
                expect: vec!["doc.missing".to_string()],
                ..Default::default()
            }],
            5,
        );

        assert_eq!(report.clean, 1);
        assert_eq!(report.garbage, 0);
        assert_eq!(report.garbage_rejected, None);
    }

    #[test]
    fn garbage_rejection_is_measured_for_garbage_cases() {
        let report = evaluate(
            &Graph::default(),
            &[GoldenCase {
                query: "absent infrastructure scheduler".to_string(),
                garbage: true,
                ..Default::default()
            }],
            5,
        );

        assert_eq!(report.clean, 0);
        assert_eq!(report.garbage, 1);
        assert_eq!(report.garbage_rejected, Some(1.0));
    }

    #[test]
    fn answer_precision_scores_exact_clean_answers() {
        let graph = graph_with_doc("doc.answer", "answer");
        let report = evaluate(
            &graph,
            &[GoldenCase {
                query: "answer".to_string(),
                expect: vec!["doc.answer".to_string()],
                ..Default::default()
            }],
            5,
        );

        assert_eq!(report.answer_count, 1);
        assert_eq!(report.answer_precision, Some(1.0));
    }

    #[test]
    fn answer_precision_penalizes_confident_garbage_answers() {
        let graph = graph_with_doc("doc.answer", "answer");
        let report = evaluate(
            &graph,
            &[GoldenCase {
                query: "answer".to_string(),
                garbage: true,
                ..Default::default()
            }],
            5,
        );

        assert_eq!(report.answer_count, 1);
        assert_eq!(report.answer_precision, Some(0.0));
    }

    #[test]
    fn route_eval_can_require_structured_relation_evidence() {
        let graph = relation_graph();
        let report = evaluate(
            &graph,
            &[GoldenCase {
                query: "depends on auth contract".to_string(),
                expect: vec!["doc.router".to_string()],
                route_relation_must: vec![RelationMatchExpectation {
                    relation: "depends_on".to_string(),
                    direction: Some("forward".to_string()),
                    endpoint_id: Some("doc.auth-contract".to_string()),
                    phrase: Some("depends on".to_string()),
                    ..Default::default()
                }],
                ..Default::default()
            }],
            5,
        );

        assert_eq!(report.p_at_1, 1.0);
        assert_eq!(report.relation_evidence_recall, Some(1.0));
        assert_eq!(report.relation_evidence_expected, 1);
        assert_eq!(report.relation_evidence_missing, 0);
        assert!(report.cases[0].ok);
        assert_eq!(report.cases[0].relation_matches.len(), 1);
        assert!(report.cases[0].missing_relation_matches.is_empty());
    }

    #[test]
    fn route_eval_fails_when_required_relation_evidence_is_missing() {
        let graph = relation_graph();
        let report = evaluate(
            &graph,
            &[GoldenCase {
                query: "depends on auth contract".to_string(),
                expect: vec!["doc.router".to_string()],
                route_relation_must: vec![RelationMatchExpectation {
                    relation: "verifies".to_string(),
                    endpoint_id: Some("doc.auth-contract".to_string()),
                    ..Default::default()
                }],
                ..Default::default()
            }],
            5,
        );

        assert_eq!(report.cases[0].rank, Some(1));
        assert_eq!(report.p_at_1, 0.0);
        assert_eq!(report.relation_evidence_recall, Some(0.0));
        assert_eq!(report.relation_evidence_expected, 1);
        assert_eq!(report.relation_evidence_missing, 1);
        assert!(!report.cases[0].ok);
        assert_eq!(report.cases[0].missing_relation_matches.len(), 1);
    }

    #[test]
    fn route_eval_relation_recall_is_unmeasured_without_relation_judgments() {
        let graph = graph_with_doc("doc.answer", "answer");
        let report = evaluate(
            &graph,
            &[GoldenCase {
                query: "answer".to_string(),
                expect: vec!["doc.answer".to_string()],
                ..Default::default()
            }],
            5,
        );

        assert_eq!(report.relation_evidence_recall, None);
        assert_eq!(report.relation_evidence_expected, 0);
        assert_eq!(report.relation_evidence_missing, 0);
    }

    #[test]
    fn route_eval_can_require_confidence_bands() {
        let graph = graph_with_doc("doc.answer", "answer");
        let report = evaluate(
            &graph,
            &[
                GoldenCase {
                    query: "answer".to_string(),
                    expect: vec!["doc.answer".to_string()],
                    min_confidence: Some("ambiguous".to_string()),
                    ..Default::default()
                },
                GoldenCase {
                    query: "answer".to_string(),
                    expect: vec!["doc.answer".to_string()],
                    expected_confidence: Some("fallback".to_string()),
                    ..Default::default()
                },
            ],
            5,
        );

        assert_eq!(report.confidence_expectation_rate, Some(0.5));
        assert_eq!(report.confidence_expected, 2);
        assert_eq!(report.confidence_missing, 1);
        assert_eq!(report.p_at_1, 0.5);
        assert_eq!(report.cases[0].confidence_ok, Some(true));
        assert_eq!(report.cases[1].confidence_ok, Some(false));
    }

    fn graph_with_doc(id: &str, title: &str) -> Graph {
        Graph {
            nodes: vec![Node {
                id: id.to_string(),
                kind: Kind::Doc,
                subkind: None,
                title: title.to_string(),
                summary: String::new(),
                aliases: Vec::new(),
                tags: Vec::new(),
                query_examples: Vec::new(),
                source_files: vec![format!("fixtures/{title}.md")],
                span: None,
                partition: None,
            }],
            edges: Vec::new(),
            ..Default::default()
        }
    }

    fn relation_graph() -> Graph {
        Graph {
            nodes: vec![
                Node {
                    id: "doc.router".to_string(),
                    kind: Kind::Doc,
                    subkind: None,
                    title: "Router".to_string(),
                    summary: "Request routing policy.".to_string(),
                    aliases: Vec::new(),
                    tags: Vec::new(),
                    query_examples: Vec::new(),
                    source_files: vec!["router.md".to_string()],
                    span: None,
                    partition: None,
                },
                Node {
                    id: "doc.auth-contract".to_string(),
                    kind: Kind::Doc,
                    subkind: None,
                    title: "Auth Contract".to_string(),
                    summary: "Authentication requirements.".to_string(),
                    aliases: Vec::new(),
                    tags: Vec::new(),
                    query_examples: Vec::new(),
                    source_files: vec!["auth.md".to_string()],
                    span: None,
                    partition: None,
                },
            ],
            edges: vec![Edge {
                from: "doc.router".to_string(),
                to: "doc.auth-contract".to_string(),
                relation: "depends_on".to_string(),
                evidence: "test".to_string(),
                basis: EdgeBasis::Resolved,
                ..Default::default()
            }],
            ..Default::default()
        }
    }
}