eidos_kernel/eval/

route.rs

use std::collections::BTreeMap;

use serde::Serialize;

use crate::graph_index::GraphIndex;
use crate::retrieval::{Confidence, GroundIndex, RelationMatch, ground_with};
use crate::schema::Graph;

use super::{GoldenCase, RelationMatchExpectation, missing_relation_matches};

/// Per-case outcome, for a readable report.
#[derive(Serialize, Clone, Debug)]
pub struct CaseResult {
    pub query: String,
    pub garbage: bool,
    /// The top-ranked hit id, if any.
    pub top: Option<String>,
    #[serde(skip_serializing_if = "Option::is_none")]
    pub expected_partition: Option<String>,
    #[serde(skip_serializing_if = "Option::is_none")]
    pub top_partition: Option<String>,
    #[serde(skip_serializing_if = "Option::is_none")]
    pub partition_ok: Option<bool>,
    pub top_confidence: Option<Confidence>,
    #[serde(skip_serializing_if = "Option::is_none")]
    pub min_confidence: Option<String>,
    #[serde(skip_serializing_if = "Option::is_none")]
    pub expected_confidence: Option<String>,
    #[serde(skip_serializing_if = "Option::is_none")]
    pub confidence_ok: Option<bool>,
    /// Structured relation reasons attached to the top hit.
    #[serde(skip_serializing_if = "Vec::is_empty")]
    pub relation_matches: Vec<RelationMatch>,
    /// Required relation reasons not found on the top hit.
    pub missing_relation_matches: Vec<RelationMatchExpectation>,
    /// 1-based rank of the first expected id within the returned hits (clean cases only).
    pub rank: Option<usize>,
    /// Clean: top hit was expected. Garbage: query was rejected.
    pub ok: bool,
}

/// Ok/total for one intent class (ok = p@1 for clean, rejected for garbage).
#[derive(Serialize, Clone, Debug, Default)]
pub struct ClassStat {
    pub ok: usize,
    pub total: usize,
}

/// Aggregate metrics. Fractions are in `[0.0, 1.0]`; nullable fractions were not measured.
#[derive(Serialize, Clone, Debug)]
pub struct EvalReport {
    pub clean: usize,
    pub p_at_1: f64,
    pub hit_at_k: f64,
    pub mrr: f64,
    /// Fraction of declared `route_relation_must` expectations present on the top route hit.
    /// `None` means the suite did not judge route relation evidence.
    pub relation_evidence_recall: Option<f64>,
    pub relation_evidence_expected: usize,
    pub relation_evidence_missing: usize,
    /// Fraction of clean cases with a declared confidence expectation whose top confidence met it.
    /// `None` means the suite did not judge confidence bands.
    pub confidence_expectation_rate: Option<f64>,
    pub confidence_expected: usize,
    pub confidence_missing: usize,
    pub garbage: usize,
    pub garbage_rejected: Option<f64>,
    /// Precision of answer-grade top hits (`exact`/`strong`) across clean and garbage cases.
    /// Clean cases count correct only at rank 1; garbage cases are correct only if not answer-grade.
    pub answer_precision: Option<f64>,
    pub answer_count: usize,
    /// Per-`class` ok/total (sorted), so a large stratified set shows which class is weak.
    pub by_class: BTreeMap<String, ClassStat>,
    /// Per top-hit confidence band ok/total over CLEAN cases — does "strong" predict correct?
    /// The safety property for a trusted brain: strong should stay high-accuracy.
    pub calibration: BTreeMap<String, ClassStat>,
    pub cases: Vec<CaseResult>,
}

/// Run every golden case through `ground` and aggregate the metrics, using the FROZEN ranker
/// calibration.
pub fn evaluate(graph: &Graph, cases: &[GoldenCase], limit: usize) -> EvalReport {
    evaluate_with_config(
        graph,
        cases,
        limit,
        crate::retrieval::RankerConfig::default(),
    )
}

/// Like [`evaluate`], but with an explicit [`RankerConfig`] — the injection seam for the dev-only
/// parameter sweep (`eidos-eval-runner --k1/--b/--weights`). The kernel never reads the environment.
pub fn evaluate_with_config(
    graph: &Graph,
    cases: &[GoldenCase],
    limit: usize,
    config: crate::retrieval::RankerConfig,
) -> EvalReport {
    let index = GroundIndex::build_with_config(graph, config);
    let graph_index = GraphIndex::build(graph);
    let mut results = Vec::with_capacity(cases.len());
    let (mut clean, mut garbage) = (0usize, 0usize);
    let (mut p1, mut hitk, mut rr_sum, mut rejected) = (0usize, 0usize, 0.0f64, 0usize);
    let (mut answer_ok, mut answer_count) = (0usize, 0usize);
    let (mut relation_expected, mut relation_missing) = (0usize, 0usize);
    let (mut confidence_expected, mut confidence_missing) = (0usize, 0usize);
    let mut by_class: BTreeMap<String, ClassStat> = BTreeMap::new();
    let mut calibration: BTreeMap<String, ClassStat> = BTreeMap::new();

    for c in cases {
        let hits = ground_with(graph, &index, &c.query, limit);
        let top = hits.first().map(|h| h.id.clone());
        let top_partition = top
            .as_deref()
            .and_then(|id| graph_index.node(id))
            .and_then(|node| node.partition.clone());
        let partition_ok = c
            .expected_partition
            .as_ref()
            .map(|expected| top_partition.as_deref() == Some(expected.as_str()));
        let top_confidence = hits.first().map(|h| h.confidence);
        let confidence_ok = confidence_expectation_ok(c, top_confidence);
        if let Some(ok) = confidence_ok {
            confidence_expected += 1;
            confidence_missing += usize::from(!ok);
        }
        let relation_matches = hits
            .first()
            .map(|h| h.relation_matches.clone())
            .unwrap_or_default();
        let missing_relation_matches =
            missing_relation_matches(&c.route_relation_must, &relation_matches);
        relation_expected += c.route_relation_must.len();
        relation_missing += missing_relation_matches.len();
        let class = c.class.clone().unwrap_or_else(|| "unclassified".into());

        if c.garbage {
            garbage += 1;
            let ok = is_rejected(top_confidence);
            if ok {
                rejected += 1;
            }
            if is_answer_confidence(top_confidence) {
                answer_count += 1;
            }
            let s = by_class.entry(class).or_default();
            s.total += 1;
            s.ok += ok as usize;
            results.push(CaseResult {
                query: c.query.clone(),
                garbage: true,
                top,
                expected_partition: c.expected_partition.clone(),
                top_partition,
                partition_ok,
                top_confidence,
                min_confidence: c.min_confidence.clone(),
                expected_confidence: c.expected_confidence.clone(),
                confidence_ok,
                relation_matches,
                missing_relation_matches,
                rank: None,
                ok,
            });
            continue;
        }

        clean += 1;
        let rank = hits
            .iter()
            .position(|h| c.expect.iter().any(|e| e == &h.id))
            .map(|i| i + 1);
        let relation_ok = missing_relation_matches.is_empty();
        let at_1 = rank == Some(1)
            && relation_ok
            && partition_ok.unwrap_or(true)
            && confidence_ok.unwrap_or(true);
        if at_1 {
            p1 += 1;
        }
        if is_answer_confidence(top_confidence) {
            answer_count += 1;
            answer_ok += at_1 as usize;
        }
        if let Some(r) = rank {
            hitk += 1;
            rr_sum += 1.0 / r as f64;
        }
        let s = by_class.entry(class).or_default();
        s.total += 1;
        s.ok += at_1 as usize;
        let cal = calibration.entry(band_str(top_confidence)).or_default();
        cal.total += 1;
        cal.ok += at_1 as usize;
        results.push(CaseResult {
            query: c.query.clone(),
            garbage: false,
            top,
            expected_partition: c.expected_partition.clone(),
            top_partition,
            partition_ok,
            top_confidence,
            min_confidence: c.min_confidence.clone(),
            expected_confidence: c.expected_confidence.clone(),
            confidence_ok,
            relation_matches,
            missing_relation_matches,
            rank,
            ok: at_1,
        });
    }

    let frac = |n: usize, d: usize| if d == 0 { 0.0 } else { n as f64 / d as f64 };
    let measured_frac = |n: usize, d: usize| (d > 0).then(|| n as f64 / d as f64);
    EvalReport {
        clean,
        p_at_1: frac(p1, clean),
        hit_at_k: frac(hitk, clean),
        mrr: if clean == 0 {
            0.0
        } else {
            rr_sum / clean as f64
        },
        relation_evidence_recall: measured_frac(
            relation_expected - relation_missing,
            relation_expected,
        ),
        relation_evidence_expected: relation_expected,
        relation_evidence_missing: relation_missing,
        confidence_expectation_rate: measured_frac(
            confidence_expected - confidence_missing,
            confidence_expected,
        ),
        confidence_expected,
        confidence_missing,
        garbage,
        garbage_rejected: measured_frac(rejected, garbage),
        answer_precision: measured_frac(answer_ok, answer_count),
        answer_count,
        by_class,
        calibration,
        cases: results,
    }
}

fn band_str(top_confidence: Option<Confidence>) -> String {
    match top_confidence {
        Some(c) => serde_json::to_value(c)
            .ok()
            .and_then(|v| v.as_str().map(String::from))
            .unwrap_or_else(|| "none".into()),
        None => "none".into(),
    }
}

fn is_rejected(top_confidence: Option<Confidence>) -> bool {
    matches!(
        top_confidence,
        None | Some(Confidence::Weak) | Some(Confidence::Fallback)
    )
}

pub(super) fn is_answer_confidence(confidence: Option<Confidence>) -> bool {
    matches!(
        confidence,
        Some(Confidence::Exact) | Some(Confidence::Strong)
    )
}

fn confidence_expectation_ok(case: &GoldenCase, actual: Option<Confidence>) -> Option<bool> {
    let mut judged = false;
    let mut ok = true;
    if let Some(expected) = case.expected_confidence.as_deref() {
        judged = true;
        ok &= parse_confidence(expected).is_some_and(|expected| actual == Some(expected));
    }
    if let Some(minimum) = case.min_confidence.as_deref() {
        judged = true;
        ok &= parse_confidence(minimum).is_some_and(|minimum| {
            actual.is_some_and(|actual| confidence_rank(actual) >= confidence_rank(minimum))
        });
    }
    judged.then_some(ok)
}

fn parse_confidence(value: &str) -> Option<Confidence> {
    match value.trim().to_ascii_lowercase().as_str() {
        "exact" => Some(Confidence::Exact),
        "strong" => Some(Confidence::Strong),
        "ambiguous" => Some(Confidence::Ambiguous),
        "weak" => Some(Confidence::Weak),
        "fallback" => Some(Confidence::Fallback),
        _ => None,
    }
}

fn confidence_rank(confidence: Confidence) -> u8 {
    match confidence {
        Confidence::Exact => 4,
        Confidence::Strong => 3,
        Confidence::Ambiguous => 2,
        Confidence::Weak => 1,
        Confidence::Fallback => 0,
    }
}