klieo-eval 3.0.0

//! Deterministic, structured label-match eval for classifier agents.
//! Pure scoring — no I/O, no provider, no agent driver. A
//! classifier's output is a closed label set, so exact-label-match gates the
//! actual decision while tolerating the prose a real LLM wraps it in.

use serde::{Deserialize, Serialize};
use std::collections::HashMap;

/// A canonical label plus the surface forms that resolve to it.
#[derive(Debug, Clone)]
#[non_exhaustive]
pub struct LabelSpec {
    /// Matched case-insensitively as a whole token; this exact string is what
    /// [`extract_label`] hands back when the spec wins.
    pub canonical: String,
    /// Additional surface forms (besides `canonical`) that resolve here, e.g.
    /// canonical `"AIAct"` with aliases `["AI Act", "ai-act", "EU AI Act"]`.
    pub aliases: Vec<String>,
}

impl LabelSpec {
    /// `canonical` is always matched regardless of the alias list; `aliases` may
    /// be empty. Keep aliases distinct from every other spec's canonical form,
    /// or [`extract_label`] will treat an output mentioning both as ambiguous.
    pub fn new(canonical: impl Into<String>, aliases: &[&str]) -> Self {
        Self {
            canonical: canonical.into(),
            aliases: aliases.iter().map(|a| (*a).to_string()).collect(),
        }
    }
}

/// Resolve a raw agent output to a known label. Scans for each spec's canonical
/// form and aliases as whole tokens, case-insensitively, and returns
/// `Some(canonical)` iff EXACTLY ONE distinct known label appears. Zero matches
/// or an ambiguous multi-label output → `None` (a miss). This separates a real
/// misclassification from format noise (prose wrapping, alias rendering).
pub fn extract_label(output: &str, labels: &[LabelSpec]) -> Option<String> {
    let hay = output.to_lowercase();
    let mut found: Option<String> = None;
    for spec in labels {
        let present = std::iter::once(&spec.canonical)
            .chain(spec.aliases.iter())
            .any(|form| contains_token(&hay, &form.to_lowercase()));
        if present {
            match &found {
                Some(seen) if seen != &spec.canonical => return None,
                _ => found = Some(spec.canonical.clone()),
            }
        }
    }
    found
}

/// True when `needle` occurs in `haystack` bounded by non-alphanumeric chars (or
/// string ends) on both sides — so `"dora"` matches `"under dora"` but not
/// `"fedora"`. Both args are assumed already lowercased.
fn contains_token(haystack: &str, needle: &str) -> bool {
    if needle.is_empty() {
        return false;
    }
    haystack.match_indices(needle).any(|(i, _)| {
        let before = haystack[..i].chars().next_back();
        let after = haystack[i + needle.len()..].chars().next();
        before.is_none_or(|c| !c.is_alphanumeric()) && after.is_none_or(|c| !c.is_alphanumeric())
    })
}

/// One golden row. `expected` must be the canonical form of a [`LabelSpec`] the
/// scorer is given, or the case can never be counted correct.
#[derive(Debug, Clone, Deserialize)]
#[non_exhaustive]
pub struct ClassificationCase {
    /// Verbatim text handed to the classifier; never normalised before the run.
    pub input: String,
    /// The single label this input should resolve to under [`extract_label`].
    pub expected: String,
}

/// Per-label tally (named fields — a 3-tuple would be an opaque API surface).
#[derive(Debug, Clone, PartialEq, Serialize)]
#[non_exhaustive]
pub struct LabelStat {
    /// The expected label this row tallies.
    pub label: String,
    /// How many cases with this expected label the classifier got right.
    pub correct: usize,
    /// How many cases carried this expected label.
    pub total: usize,
}

/// One missed case: expected label vs what parsed out (`None` = output did not
/// resolve to exactly one known label).
#[derive(Debug, Clone, PartialEq, Serialize)]
#[non_exhaustive]
pub struct Miss {
    /// Copied from the case so a failing run is diagnosable without the golden set.
    pub input: String,
    /// The canonical label the case declared as correct.
    pub expected: String,
    /// What the output resolved to, or `None` when it matched no single known label.
    pub got: Option<String>,
}

/// Aggregate result over a golden set.
#[derive(Debug, Clone, PartialEq, Serialize)]
#[non_exhaustive]
pub struct ClassificationReport {
    /// Count of golden cases scored — equals the `cases` slice length.
    pub total: usize,
    /// How many cases resolved to their `expected` label.
    pub correct: usize,
    /// `correct / total`; `0.0` when `total == 0`.
    pub accuracy: f64,
    /// Per expected label, in first-seen order.
    pub per_label: Vec<LabelStat>,
    /// One entry per missed case.
    pub misses: Vec<Miss>,
}

#[derive(Default)]
struct LabelTally {
    correct: usize,
    total: usize,
}

/// Score predictions against the golden cases (aligned by index). A prediction
/// of `None`, or one not equal to the case's `expected`, is a miss. A prediction
/// missing for a case (short slice) counts as `None`; predictions beyond
/// `cases.len()` are ignored — the report is defined over the golden set.
pub fn score_classification(
    predicted: &[Option<String>],
    cases: &[ClassificationCase],
) -> ClassificationReport {
    let total = cases.len();
    let mut correct = 0usize;
    let mut misses = Vec::new();
    let mut order: Vec<String> = Vec::new();
    let mut tally: HashMap<String, LabelTally> = HashMap::new();

    for (i, case) in cases.iter().enumerate() {
        let got = predicted.get(i).cloned().flatten();
        if !tally.contains_key(&case.expected) {
            order.push(case.expected.clone());
        }
        let entry = tally.entry(case.expected.clone()).or_default();
        entry.total += 1;
        if got.as_deref() == Some(case.expected.as_str()) {
            correct += 1;
            entry.correct += 1;
        } else {
            misses.push(Miss {
                input: case.input.clone(),
                expected: case.expected.clone(),
                got,
            });
        }
    }

    let accuracy = if total == 0 {
        0.0
    } else {
        correct as f64 / total as f64
    };
    let per_label = order
        .into_iter()
        .map(|label| {
            let LabelTally { correct, total } = tally[&label];
            LabelStat {
                label,
                correct,
                total,
            }
        })
        .collect();

    ClassificationReport {
        total,
        correct,
        accuracy,
        per_label,
        misses,
    }
}

#[cfg(test)]
mod tests {
    use super::*;

    fn specs() -> Vec<LabelSpec> {
        vec![
            LabelSpec::new("DORA", &[]),
            LabelSpec::new("AIAct", &["AI Act", "ai-act", "EU AI Act"]),
            LabelSpec::new("GDPR", &[]),
            LabelSpec::new("Other", &[]),
        ]
    }

    #[test]
    fn bare_label_resolves() {
        assert_eq!(extract_label("DORA", &specs()), Some("DORA".into()));
    }

    #[test]
    fn prose_wrapped_single_label_resolves() {
        assert_eq!(
            extract_label("This concerns DORA.", &specs()),
            Some("DORA".into())
        );
    }

    #[test]
    fn alias_resolves_to_canonical() {
        assert_eq!(
            extract_label("Falls under the EU AI Act", &specs()),
            Some("AIAct".into())
        );
    }

    #[test]
    fn two_distinct_labels_are_ambiguous() {
        assert_eq!(extract_label("Both DORA and GDPR apply", &specs()), None);
    }

    #[test]
    fn no_known_label_is_none() {
        assert_eq!(extract_label("I'm not sure", &specs()), None);
    }

    #[test]
    fn substring_is_not_a_token_match() {
        // "dora" inside "fedora" must NOT match.
        assert_eq!(extract_label("running fedora linux", &specs()), None);
    }

    fn case(input: &str, expected: &str) -> ClassificationCase {
        ClassificationCase {
            input: input.into(),
            expected: expected.into(),
        }
    }

    #[test]
    fn all_correct_is_full_accuracy() {
        let cases = vec![case("a", "DORA"), case("b", "GDPR")];
        let preds = vec![Some("DORA".into()), Some("GDPR".into())];
        let r = score_classification(&preds, &cases);
        assert_eq!(r.total, 2);
        assert_eq!(r.correct, 2);
        assert!((r.accuracy - 1.0).abs() < 1e-9);
        assert!(r.misses.is_empty());
    }

    #[test]
    fn wrong_and_unparseable_are_misses() {
        let cases = vec![case("a", "DORA"), case("b", "GDPR"), case("c", "AIAct")];
        let preds = vec![Some("DORA".into()), Some("DORA".into()), None];
        let r = score_classification(&preds, &cases);
        assert_eq!(r.correct, 1);
        assert!((r.accuracy - 1.0 / 3.0).abs() < 1e-9);
        assert_eq!(r.misses.len(), 2);
        assert_eq!(
            r.misses[0],
            Miss {
                input: "b".into(),
                expected: "GDPR".into(),
                got: Some("DORA".into())
            }
        );
        assert_eq!(
            r.misses[1],
            Miss {
                input: "c".into(),
                expected: "AIAct".into(),
                got: None
            }
        );
    }

    #[test]
    fn per_label_counts_in_first_seen_order() {
        let cases = vec![case("a", "DORA"), case("b", "GDPR"), case("c", "DORA")];
        let preds = vec![Some("DORA".into()), None, Some("DORA".into())];
        let r = score_classification(&preds, &cases);
        assert_eq!(
            r.per_label,
            vec![
                LabelStat {
                    label: "DORA".into(),
                    correct: 2,
                    total: 2
                },
                LabelStat {
                    label: "GDPR".into(),
                    correct: 0,
                    total: 1
                },
            ]
        );
    }

    #[test]
    fn empty_cases_is_zero_accuracy() {
        let r = score_classification(&[], &[]);
        assert_eq!(r.total, 0);
        assert_eq!(r.accuracy, 0.0);
    }

    #[test]
    fn short_prediction_slice_counts_tail_as_none_misses() {
        let cases = vec![case("a", "DORA"), case("b", "GDPR"), case("c", "AIAct")];
        let preds = vec![Some("DORA".into())]; // only one prediction for three cases
        let r = score_classification(&preds, &cases);
        assert_eq!(r.correct, 1);
        assert_eq!(r.misses.len(), 2);
        assert!(r.misses.iter().all(|m| m.got.is_none()));
    }

    #[test]
    fn canonical_and_its_own_alias_coincide_resolve_once() {
        // "AI Act" (alias) and "AIAct" (canonical) both present → still AIAct, not ambiguous.
        assert_eq!(
            extract_label("The AI Act, i.e. AIAct, applies", &specs()),
            Some("AIAct".into())
        );
    }
}