klieo_eval/
classifier_eval.rs

1//! Deterministic, structured label-match eval for classifier agents.
2//! Pure scoring — no I/O, no provider, no agent driver. A
3//! classifier's output is a closed label set, so exact-label-match gates the
4//! actual decision while tolerating the prose a real LLM wraps it in.
5
6use serde::{Deserialize, Serialize};
7use std::collections::HashMap;
8
9/// A canonical label plus the surface forms that resolve to it.
10#[derive(Debug, Clone)]
11#[non_exhaustive]
12pub struct LabelSpec {
13    /// Matched case-insensitively as a whole token; this exact string is what
14    /// [`extract_label`] hands back when the spec wins.
15    pub canonical: String,
16    /// Additional surface forms (besides `canonical`) that resolve here, e.g.
17    /// canonical `"AIAct"` with aliases `["AI Act", "ai-act", "EU AI Act"]`.
18    pub aliases: Vec<String>,
19}
20
21impl LabelSpec {
22    /// `canonical` is always matched regardless of the alias list; `aliases` may
23    /// be empty. Keep aliases distinct from every other spec's canonical form,
24    /// or [`extract_label`] will treat an output mentioning both as ambiguous.
25    pub fn new(canonical: impl Into<String>, aliases: &[&str]) -> Self {
26        Self {
27            canonical: canonical.into(),
28            aliases: aliases.iter().map(|a| (*a).to_string()).collect(),
29        }
30    }
31}
32
33/// Resolve a raw agent output to a known label. Scans for each spec's canonical
34/// form and aliases as whole tokens, case-insensitively, and returns
35/// `Some(canonical)` iff EXACTLY ONE distinct known label appears. Zero matches
36/// or an ambiguous multi-label output → `None` (a miss). This separates a real
37/// misclassification from format noise (prose wrapping, alias rendering).
38pub fn extract_label(output: &str, labels: &[LabelSpec]) -> Option<String> {
39    let hay = output.to_lowercase();
40    let mut found: Option<String> = None;
41    for spec in labels {
42        let present = std::iter::once(&spec.canonical)
43            .chain(spec.aliases.iter())
44            .any(|form| contains_token(&hay, &form.to_lowercase()));
45        if present {
46            match &found {
47                Some(seen) if seen != &spec.canonical => return None,
48                _ => found = Some(spec.canonical.clone()),
49            }
50        }
51    }
52    found
53}
54
55/// True when `needle` occurs in `haystack` bounded by non-alphanumeric chars (or
56/// string ends) on both sides — so `"dora"` matches `"under dora"` but not
57/// `"fedora"`. Both args are assumed already lowercased.
58fn contains_token(haystack: &str, needle: &str) -> bool {
59    if needle.is_empty() {
60        return false;
61    }
62    haystack.match_indices(needle).any(|(i, _)| {
63        let before = haystack[..i].chars().next_back();
64        let after = haystack[i + needle.len()..].chars().next();
65        before.is_none_or(|c| !c.is_alphanumeric()) && after.is_none_or(|c| !c.is_alphanumeric())
66    })
67}
68
69/// One golden row. `expected` must be the canonical form of a [`LabelSpec`] the
70/// scorer is given, or the case can never be counted correct.
71#[derive(Debug, Clone, Deserialize)]
72#[non_exhaustive]
73pub struct ClassificationCase {
74    /// Verbatim text handed to the classifier; never normalised before the run.
75    pub input: String,
76    /// The single label this input should resolve to under [`extract_label`].
77    pub expected: String,
78}
79
80/// Per-label tally (named fields — a 3-tuple would be an opaque API surface).
81#[derive(Debug, Clone, PartialEq, Serialize)]
82#[non_exhaustive]
83pub struct LabelStat {
84    /// The expected label this row tallies.
85    pub label: String,
86    /// How many cases with this expected label the classifier got right.
87    pub correct: usize,
88    /// How many cases carried this expected label.
89    pub total: usize,
90}
91
92/// One missed case: expected label vs what parsed out (`None` = output did not
93/// resolve to exactly one known label).
94#[derive(Debug, Clone, PartialEq, Serialize)]
95#[non_exhaustive]
96pub struct Miss {
97    /// Copied from the case so a failing run is diagnosable without the golden set.
98    pub input: String,
99    /// The canonical label the case declared as correct.
100    pub expected: String,
101    /// What the output resolved to, or `None` when it matched no single known label.
102    pub got: Option<String>,
103}
104
105/// Aggregate result over a golden set.
106#[derive(Debug, Clone, PartialEq, Serialize)]
107#[non_exhaustive]
108pub struct ClassificationReport {
109    /// Count of golden cases scored — equals the `cases` slice length.
110    pub total: usize,
111    /// How many cases resolved to their `expected` label.
112    pub correct: usize,
113    /// `correct / total`; `0.0` when `total == 0`.
114    pub accuracy: f64,
115    /// Per expected label, in first-seen order.
116    pub per_label: Vec<LabelStat>,
117    /// One entry per missed case.
118    pub misses: Vec<Miss>,
119}
120
121#[derive(Default)]
122struct LabelTally {
123    correct: usize,
124    total: usize,
125}
126
127/// Score predictions against the golden cases (aligned by index). A prediction
128/// of `None`, or one not equal to the case's `expected`, is a miss. A prediction
129/// missing for a case (short slice) counts as `None`; predictions beyond
130/// `cases.len()` are ignored — the report is defined over the golden set.
131pub fn score_classification(
132    predicted: &[Option<String>],
133    cases: &[ClassificationCase],
134) -> ClassificationReport {
135    let total = cases.len();
136    let mut correct = 0usize;
137    let mut misses = Vec::new();
138    let mut order: Vec<String> = Vec::new();
139    let mut tally: HashMap<String, LabelTally> = HashMap::new();
140
141    for (i, case) in cases.iter().enumerate() {
142        let got = predicted.get(i).cloned().flatten();
143        if !tally.contains_key(&case.expected) {
144            order.push(case.expected.clone());
145        }
146        let entry = tally.entry(case.expected.clone()).or_default();
147        entry.total += 1;
148        if got.as_deref() == Some(case.expected.as_str()) {
149            correct += 1;
150            entry.correct += 1;
151        } else {
152            misses.push(Miss {
153                input: case.input.clone(),
154                expected: case.expected.clone(),
155                got,
156            });
157        }
158    }
159
160    let accuracy = if total == 0 {
161        0.0
162    } else {
163        correct as f64 / total as f64
164    };
165    let per_label = order
166        .into_iter()
167        .map(|label| {
168            let LabelTally { correct, total } = tally[&label];
169            LabelStat {
170                label,
171                correct,
172                total,
173            }
174        })
175        .collect();
176
177    ClassificationReport {
178        total,
179        correct,
180        accuracy,
181        per_label,
182        misses,
183    }
184}
185
186#[cfg(test)]
187mod tests {
188    use super::*;
189
190    fn specs() -> Vec<LabelSpec> {
191        vec![
192            LabelSpec::new("DORA", &[]),
193            LabelSpec::new("AIAct", &["AI Act", "ai-act", "EU AI Act"]),
194            LabelSpec::new("GDPR", &[]),
195            LabelSpec::new("Other", &[]),
196        ]
197    }
198
199    #[test]
200    fn bare_label_resolves() {
201        assert_eq!(extract_label("DORA", &specs()), Some("DORA".into()));
202    }
203
204    #[test]
205    fn prose_wrapped_single_label_resolves() {
206        assert_eq!(
207            extract_label("This concerns DORA.", &specs()),
208            Some("DORA".into())
209        );
210    }
211
212    #[test]
213    fn alias_resolves_to_canonical() {
214        assert_eq!(
215            extract_label("Falls under the EU AI Act", &specs()),
216            Some("AIAct".into())
217        );
218    }
219
220    #[test]
221    fn two_distinct_labels_are_ambiguous() {
222        assert_eq!(extract_label("Both DORA and GDPR apply", &specs()), None);
223    }
224
225    #[test]
226    fn no_known_label_is_none() {
227        assert_eq!(extract_label("I'm not sure", &specs()), None);
228    }
229
230    #[test]
231    fn substring_is_not_a_token_match() {
232        // "dora" inside "fedora" must NOT match.
233        assert_eq!(extract_label("running fedora linux", &specs()), None);
234    }
235
236    fn case(input: &str, expected: &str) -> ClassificationCase {
237        ClassificationCase {
238            input: input.into(),
239            expected: expected.into(),
240        }
241    }
242
243    #[test]
244    fn all_correct_is_full_accuracy() {
245        let cases = vec![case("a", "DORA"), case("b", "GDPR")];
246        let preds = vec![Some("DORA".into()), Some("GDPR".into())];
247        let r = score_classification(&preds, &cases);
248        assert_eq!(r.total, 2);
249        assert_eq!(r.correct, 2);
250        assert!((r.accuracy - 1.0).abs() < 1e-9);
251        assert!(r.misses.is_empty());
252    }
253
254    #[test]
255    fn wrong_and_unparseable_are_misses() {
256        let cases = vec![case("a", "DORA"), case("b", "GDPR"), case("c", "AIAct")];
257        let preds = vec![Some("DORA".into()), Some("DORA".into()), None];
258        let r = score_classification(&preds, &cases);
259        assert_eq!(r.correct, 1);
260        assert!((r.accuracy - 1.0 / 3.0).abs() < 1e-9);
261        assert_eq!(r.misses.len(), 2);
262        assert_eq!(
263            r.misses[0],
264            Miss {
265                input: "b".into(),
266                expected: "GDPR".into(),
267                got: Some("DORA".into())
268            }
269        );
270        assert_eq!(
271            r.misses[1],
272            Miss {
273                input: "c".into(),
274                expected: "AIAct".into(),
275                got: None
276            }
277        );
278    }
279
280    #[test]
281    fn per_label_counts_in_first_seen_order() {
282        let cases = vec![case("a", "DORA"), case("b", "GDPR"), case("c", "DORA")];
283        let preds = vec![Some("DORA".into()), None, Some("DORA".into())];
284        let r = score_classification(&preds, &cases);
285        assert_eq!(
286            r.per_label,
287            vec![
288                LabelStat {
289                    label: "DORA".into(),
290                    correct: 2,
291                    total: 2
292                },
293                LabelStat {
294                    label: "GDPR".into(),
295                    correct: 0,
296                    total: 1
297                },
298            ]
299        );
300    }
301
302    #[test]
303    fn empty_cases_is_zero_accuracy() {
304        let r = score_classification(&[], &[]);
305        assert_eq!(r.total, 0);
306        assert_eq!(r.accuracy, 0.0);
307    }
308
309    #[test]
310    fn short_prediction_slice_counts_tail_as_none_misses() {
311        let cases = vec![case("a", "DORA"), case("b", "GDPR"), case("c", "AIAct")];
312        let preds = vec![Some("DORA".into())]; // only one prediction for three cases
313        let r = score_classification(&preds, &cases);
314        assert_eq!(r.correct, 1);
315        assert_eq!(r.misses.len(), 2);
316        assert!(r.misses.iter().all(|m| m.got.is_none()));
317    }
318
319    #[test]
320    fn canonical_and_its_own_alias_coincide_resolve_once() {
321        // "AI Act" (alias) and "AIAct" (canonical) both present → still AIAct, not ambiguous.
322        assert_eq!(
323            extract_label("The AI Act, i.e. AIAct, applies", &specs()),
324            Some("AIAct".into())
325        );
326    }
327}
klieo_eval/classifier_eval.rs

klieo_eval/
classifier_eval.rs
