cratefield_core/
classifier_agreement.rs

1//! Does a cheap classifier answer like the expensive one, per kind of
2//! question? (issue #457). [`measure_agreement`] runs a labelled corpus
3//! through both; [`ShadowClassifier`](crate::ShadowClassifier) feeds the
4//! same [`AgreementLog`] from live traffic. The [`AgreementReport`] is the
5//! evidence [`RoutingClassifier`](crate::RoutingClassifier) routes on.
6
7use std::collections::BTreeMap;
8use std::fmt;
9use std::sync::Arc;
10
11use serde::{Deserialize, Serialize};
12
13use crate::cost::{Accounting, AdapterId, CallRole, QuestionKind, value_text};
14use crate::ports::{Answer, AnswerValue, Classifier, Question};
15
16/// Whether two answer values are the same answer.
17///
18/// - `Choice`: the same label.
19/// - `Noul`: the same verdict.
20/// - `Score`: the same whole level once rounded (`3.2` and `2.9` agree,
21///   `3.4` and `3.6` do not) — the `"1"`..`"5"` convention
22///   [`Answer::score`] documents. Rounding makes this an equivalence, so
23///   two disagreeing answers can never both match the ground truth. A
24///   non-finite score agrees with nothing.
25/// - Different shapes never agree.
26///
27/// Ground truth is matched by the same rule.
28#[must_use]
29pub fn values_agree(a: &AnswerValue, b: &AnswerValue) -> bool {
30    match (a, b) {
31        (AnswerValue::Choice(a), AnswerValue::Choice(b)) => a == b,
32        (AnswerValue::Noul(a), AnswerValue::Noul(b)) => a == b,
33        (AnswerValue::Score(a), AnswerValue::Score(b)) => {
34            a.is_finite() && b.is_finite() && (a.round() - b.round()).abs() < 0.5
35        }
36        _ => false,
37    }
38}
39
40/// How narrowly the answer won: the top probability minus the runner-up,
41/// never negative.
42///
43/// The runner-up is the larger of the second reported probability and the
44/// mass nobody reported (`1 - sum`, at least zero): a provider reporting
45/// only `{"true": 0.55}` has left 0.45 on the other side, so its margin is
46/// 0.10, not 0.55. With no probabilities at all the top is
47/// [`Answer::confidence`] and the rest of the mass is the runner-up, so the
48/// margin is `2 * confidence - 1`.
49///
50/// A NaN among the numbers it is computed from gives a NaN, which clears
51/// no threshold and is stored as absent, rather than a made-up zero.
52#[must_use]
53pub fn answer_margin(answer: &Answer) -> f32 {
54    let mut top = [None::<f32>; 2];
55    for &p in answer.probabilities.values() {
56        if top[0].is_none_or(|first| p > first) {
57            top = [Some(p), top[0]];
58        } else if top[1].is_none_or(|second| p > second) {
59            top[1] = Some(p);
60        }
61    }
62    let (first, second) = match top {
63        [Some(first), second] => (first, second.unwrap_or(0.0)),
64        [None, _] => (answer.confidence, 0.0),
65    };
66    let unreported = if answer.probabilities.is_empty() {
67        1.0 - answer.confidence
68    } else {
69        1.0 - answer.probabilities.values().sum::<f32>()
70    };
71    if [first, second, unreported].iter().any(|v| v.is_nan()) {
72        return f32::NAN;
73    }
74    (first - second.max(unreported).max(0.0)).max(0.0)
75}
76
77/// `value` if it is a number a report can carry: `serde_json` writes a NaN
78/// or an infinity as `null`, which would not load back as an `f32`.
79fn finite(value: f32) -> Option<f32> {
80    value.is_finite().then_some(value)
81}
82
83/// One labelled corpus item: a state, the questions asked of it in one
84/// batch, and the gold answer for whichever ids have one (a missing id is
85/// ungraded).
86#[derive(Debug, Clone, PartialEq)]
87pub struct CorpusItem {
88    pub state: String,
89    pub questions: BTreeMap<String, Question>,
90    pub expected: BTreeMap<String, AnswerValue>,
91}
92
93/// Which adapter the ground truth sided with when the two disagreed.
94#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
95#[serde(rename_all = "snake_case")]
96pub enum GroundTruth {
97    Ungraded,
98    Cheap,
99    Expensive,
100    Neither,
101}
102
103/// One question the two adapters answered differently. Values only, never
104/// the state: a report is safe to paste into a ticket.
105#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
106pub struct Disagreement {
107    /// Index into the corpus; `None` for live shadow traffic.
108    pub item: Option<usize>,
109    pub cheap: String,
110    pub expensive: String,
111    pub expected: Option<String>,
112    pub right: GroundTruth,
113}
114
115/// The cheap adapter's confidence and margin on one compared question,
116/// and how it went — the evidence a [`Thresholds`](crate::Thresholds) is
117/// checked against. Only the cheap side is kept: its confidence is what
118/// routing reads, and it is not comparable with the expensive one's.
119///
120/// A non-finite confidence or margin is kept as `None`, so a report always
121/// loads back from JSON, and `None` never clears a threshold.
122#[derive(Debug, Clone, Copy, PartialEq, Serialize, Deserialize)]
123pub struct CalibrationPoint {
124    pub confidence: Option<f32>,
125    pub margin: Option<f32>,
126    pub agreed: bool,
127    /// `None` when ungraded.
128    pub cheap_right: Option<bool>,
129}
130
131/// Agreement on one [`QuestionKind`].
132#[derive(Debug, Clone, Default, PartialEq, Serialize, Deserialize)]
133pub struct KindAgreement {
134    /// Questions both adapters answered.
135    pub questions: usize,
136    pub agreed: usize,
137    /// Questions with a gold answer, and how many each adapter got right.
138    pub graded: usize,
139    pub cheap_right: usize,
140    pub expensive_right: usize,
141    /// Questions that could not be compared: either adapter failed or left
142    /// the id out of its answers.
143    pub skipped: usize,
144    pub disagreements: Vec<Disagreement>,
145    pub calibration: Vec<CalibrationPoint>,
146}
147
148/// The slice of a kind's evidence the cheap adapter would have been
149/// trusted on at some thresholds.
150#[derive(Debug, Clone, Copy, PartialEq, Eq)]
151pub struct AgreementAt {
152    pub questions: usize,
153    pub agreed: usize,
154}
155
156impl AgreementAt {
157    /// Share of those questions answered differently; `None` for none.
158    #[must_use]
159    pub fn disagreement_rate(&self) -> Option<f64> {
160        rate(self.questions - self.agreed, self.questions)
161    }
162}
163
164impl KindAgreement {
165    /// Share of compared questions answered alike; `None` when nothing was
166    /// compared — unmeasured, which is not the same as zero.
167    #[must_use]
168    pub fn agreement_rate(&self) -> Option<f64> {
169        rate(self.agreed, self.questions)
170    }
171
172    /// The evidence restricted to questions the cheap adapter answered with
173    /// at least `min_confidence` and `min_margin` — what it would have
174    /// served under those thresholds. Written in the positive, and an
175    /// absent (non-finite) value never passes.
176    #[must_use]
177    pub fn agreement_at(&self, min_confidence: f32, min_margin: f32) -> AgreementAt {
178        let kept = self.calibration.iter().filter(|p| {
179            p.confidence.is_some_and(|c| c >= min_confidence)
180                && p.margin.is_some_and(|m| m >= min_margin)
181        });
182        let (questions, agreed) = kept.fold((0, 0), |(n, a), p| (n + 1, a + usize::from(p.agreed)));
183        AgreementAt { questions, agreed }
184    }
185}
186
187#[expect(
188    clippy::cast_precision_loss,
189    reason = "counts of classifier calls stay far below 2^52"
190)]
191fn rate(part: usize, whole: usize) -> Option<f64> {
192    (whole > 0).then(|| part as f64 / whole as f64)
193}
194
195/// Agreement between two named adapters, per question kind.
196#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
197pub struct AgreementReport {
198    pub cheap: AdapterId,
199    pub expensive: AdapterId,
200    pub kinds: BTreeMap<QuestionKind, KindAgreement>,
201}
202
203impl fmt::Display for AgreementReport {
204    /// One line per kind; `n/a` where nothing was compared or graded.
205    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
206        writeln!(f, "{} vs {}", self.cheap, self.expensive)?;
207        for (kind, k) in &self.kinds {
208            let agree = k
209                .agreement_rate()
210                .map_or_else(|| "n/a".to_owned(), |r| format!("{:.1}%", r * 100.0));
211            let graded = if k.graded == 0 {
212                "n/a".to_owned()
213            } else {
214                format!(
215                    "{}/{} vs {}/{}",
216                    k.cheap_right, k.graded, k.expensive_right, k.graded
217                )
218            };
219            writeln!(
220                f,
221                "{kind}: {agree} agree over {} (skipped {}); right: {graded}",
222                k.questions, k.skipped
223            )?;
224        }
225        Ok(())
226    }
227}
228
229/// Accumulates an [`AgreementReport`] one compared answer at a time.
230#[derive(Debug)]
231pub struct AgreementLog {
232    #[expect(
233        clippy::disallowed_types,
234        reason = "an explicitly wired measurement cell, not ambient request state (ADR 0007)"
235    )]
236    report: std::sync::Mutex<AgreementReport>,
237}
238
239impl AgreementLog {
240    /// An empty log comparing `cheap` against `expensive`.
241    #[must_use]
242    pub fn new(cheap: AdapterId, expensive: AdapterId) -> Self {
243        Self {
244            report: AgreementReport {
245                cheap,
246                expensive,
247                kinds: BTreeMap::new(),
248            }
249            .into(),
250        }
251    }
252
253    fn with_kind(&self, kind: &QuestionKind, f: impl FnOnce(&mut KindAgreement)) {
254        let mut report = self
255            .report
256            .lock()
257            .unwrap_or_else(std::sync::PoisonError::into_inner);
258        f(report.kinds.entry(kind.clone()).or_default());
259    }
260
261    /// Records one question both adapters answered, graded against
262    /// `expected` when there is one. `item` is the corpus index, if any.
263    pub fn observe(
264        &self,
265        kind: &QuestionKind,
266        cheap: &Answer,
267        expensive: &Answer,
268        expected: Option<&AnswerValue>,
269        item: Option<usize>,
270    ) {
271        let agreed = values_agree(&cheap.value, &expensive.value);
272        let cheap_right = expected.map(|gold| values_agree(&cheap.value, gold));
273        let expensive_right = expected.map(|gold| values_agree(&expensive.value, gold));
274        self.with_kind(kind, |k| {
275            k.questions += 1;
276            k.agreed += usize::from(agreed);
277            if let (Some(c), Some(e)) = (cheap_right, expensive_right) {
278                k.graded += 1;
279                k.cheap_right += usize::from(c);
280                k.expensive_right += usize::from(e);
281            }
282            k.calibration.push(CalibrationPoint {
283                confidence: finite(cheap.confidence),
284                margin: finite(answer_margin(cheap)),
285                agreed,
286                cheap_right,
287            });
288            if !agreed {
289                k.disagreements.push(Disagreement {
290                    item,
291                    cheap: value_text(&cheap.value),
292                    expensive: value_text(&expensive.value),
293                    expected: expected.map(value_text),
294                    right: match (cheap_right, expensive_right) {
295                        (Some(true), _) => GroundTruth::Cheap,
296                        (_, Some(true)) => GroundTruth::Expensive,
297                        (Some(false), _) => GroundTruth::Neither,
298                        (None, _) => GroundTruth::Ungraded,
299                    },
300                });
301            }
302        });
303    }
304
305    /// Records a question that could not be compared.
306    pub(crate) fn skip(&self, kind: &QuestionKind) {
307        self.with_kind(kind, |k| k.skipped += 1);
308    }
309
310    /// The evidence so far.
311    #[must_use]
312    pub fn report(&self) -> AgreementReport {
313        self.report
314            .lock()
315            .unwrap_or_else(std::sync::PoisonError::into_inner)
316            .clone()
317    }
318}
319
320/// Asks both adapters every corpus item — one batch per item, cheap first,
321/// expensive only when cheap answered — and reports agreement per kind.
322/// Every call is recorded to `accounting` as [`CallRole::Shadow`]. An
323/// adapter error skips the item's questions rather than aborting the run:
324/// a failure rate is evidence too.
325pub async fn measure_agreement(
326    cheap: &(AdapterId, Arc<dyn Classifier>),
327    expensive: &(AdapterId, Arc<dyn Classifier>),
328    corpus: &[CorpusItem],
329    accounting: &Accounting,
330) -> AgreementReport {
331    let log = AgreementLog::new(cheap.0.clone(), expensive.0.clone());
332    for (index, item) in corpus.iter().enumerate() {
333        let (state, questions) = (item.state.as_str(), &item.questions);
334        let cheap_answers = cheap.1.ask(state, questions).await;
335        accounting.record(
336            &cheap.0,
337            &cheap.1.profile(),
338            state,
339            questions,
340            cheap_answers.as_ref().ok(),
341            CallRole::Shadow,
342        );
343        let expensive_answers = match &cheap_answers {
344            Ok(_) => {
345                let answers = expensive.1.ask(state, questions).await;
346                accounting.record(
347                    &expensive.0,
348                    &expensive.1.profile(),
349                    state,
350                    questions,
351                    answers.as_ref().ok(),
352                    CallRole::Shadow,
353                );
354                answers.ok()
355            }
356            Err(_) => None,
357        };
358        compare(
359            &log,
360            questions,
361            cheap_answers.as_ref().ok(),
362            expensive_answers.as_ref(),
363            &item.expected,
364            Some(index),
365        );
366    }
367    log.report()
368}
369
370/// Observes every question both answer maps hold, and skips the rest (all
371/// of them when either call failed, `None`).
372pub(crate) fn compare(
373    log: &AgreementLog,
374    questions: &BTreeMap<String, Question>,
375    cheap: Option<&BTreeMap<String, Answer>>,
376    expensive: Option<&BTreeMap<String, Answer>>,
377    expected: &BTreeMap<String, AnswerValue>,
378    item: Option<usize>,
379) {
380    for id in questions.keys() {
381        let kind = QuestionKind::new(id.as_str());
382        match cheap
383            .and_then(|c| c.get(id))
384            .zip(expensive.and_then(|e| e.get(id)))
385        {
386            Some((c, e)) => log.observe(&kind, c, e, expected.get(id), item),
387            None => log.skip(&kind),
388        }
389    }
390}
391
392#[cfg(test)]
393mod tests {
394    use super::*;
395
396    fn probs(pairs: &[(&str, f32)]) -> BTreeMap<String, f32> {
397        pairs.iter().map(|(l, p)| ((*l).to_owned(), *p)).collect()
398    }
399
400    #[test]
401    fn agreement_is_defined_per_answer_shape() {
402        use AnswerValue::{Choice, Noul, Score};
403        assert!(values_agree(&Choice("a".into()), &Choice("a".into())));
404        assert!(!values_agree(&Choice("a".into()), &Choice("b".into())));
405        assert!(values_agree(&Noul(true), &Noul(true)));
406        assert!(!values_agree(&Noul(true), &Noul(false)));
407        assert!(values_agree(&Score(3.2), &Score(2.9)), "same whole level");
408        assert!(!values_agree(&Score(3.4), &Score(3.6)));
409        assert!(!values_agree(&Score(f32::NAN), &Score(f32::NAN)));
410        assert!(
411            !values_agree(&Choice("true".into()), &Noul(true)),
412            "shapes never agree"
413        );
414    }
415
416    #[test]
417    fn margin_counts_unreported_mass_as_the_runner_up() {
418        let close = |a: &Answer, want: f32| (answer_margin(a) - want).abs() < 1e-6;
419        let rising = probs(&[("a", 0.1), ("b", 0.3), ("c", 0.6)]);
420        assert!(close(&Answer::choice("c", rising), 0.3));
421        let falling = probs(&[("a", 0.6), ("b", 0.3), ("c", 0.1)]);
422        assert!(close(&Answer::choice("a", falling), 0.3));
423        let top_one = Answer::noul(true, probs(&[("true", 0.55)]));
424        assert!(close(&top_one, 0.1), "0.45 unreported is the runner-up");
425        let bare = Answer::new(AnswerValue::Noul(true), BTreeMap::new(), 0.8);
426        assert!(close(&bare, 0.6), "no probabilities: 2 * confidence - 1");
427        let unsure = Answer::new(AnswerValue::Noul(true), BTreeMap::new(), 0.3);
428        assert!(close(&unsure, 0.0), "never negative");
429    }
430
431    #[test]
432    fn a_non_finite_confidence_is_absent_and_the_report_round_trips() {
433        let log = AgreementLog::new(AdapterId::new("cheap"), AdapterId::new("dear"));
434        let kind = QuestionKind::new("lang");
435        let nan = Answer::new(AnswerValue::Choice("en".into()), BTreeMap::new(), f32::NAN);
436        log.observe(&kind, &nan, &nan, None, None);
437        let report = log.report();
438        let point = report.kinds[&kind].calibration[0];
439        assert_eq!((point.confidence, point.margin), (None, None));
440        assert_eq!(report.kinds[&kind].agreement_at(0.0, 0.0).questions, 0);
441        let json = serde_json::to_string(&report).expect("serialises");
442        let loaded: AgreementReport = serde_json::from_str(&json).expect("loads back");
443        assert_eq!(loaded, report);
444    }
445
446    #[test]
447    fn evidence_is_graded_sliced_by_thresholds_and_unmeasured_is_not_zero() {
448        let log = AgreementLog::new(AdapterId::new("cheap"), AdapterId::new("dear"));
449        let kind = QuestionKind::new("lang");
450        let sure = Answer::choice("en", probs(&[("en", 0.9), ("de", 0.1)]));
451        let unsure = Answer::choice("de", probs(&[("de", 0.5), ("en", 0.4)]));
452        let en = AnswerValue::Choice("en".into());
453        log.observe(&kind, &sure, &sure, Some(&en), Some(0));
454        log.observe(&kind, &unsure, &sure, Some(&en), Some(1));
455        log.skip(&kind);
456
457        let report = log.report();
458        let k = &report.kinds[&kind];
459        assert_eq!(
460            (
461                k.questions,
462                k.agreed,
463                k.graded,
464                k.cheap_right,
465                k.expensive_right,
466                k.skipped
467            ),
468            (2, 1, 2, 1, 2, 1)
469        );
470        assert_eq!(k.disagreements[0].right, GroundTruth::Expensive);
471        assert_eq!(
472            k.agreement_at(0.8, 0.0),
473            AgreementAt {
474                questions: 1,
475                agreed: 1
476            }
477        );
478        assert_eq!(k.agreement_at(0.8, 0.0).disagreement_rate(), Some(0.0));
479        assert_eq!(k.agreement_at(f32::NAN, 0.0).questions, 0);
480        assert_eq!(KindAgreement::default().agreement_rate(), None);
481        assert!(
482            report
483                .to_string()
484                .contains("lang: 50.0% agree over 2 (skipped 1); right: 1/2 vs 2/2")
485        );
486    }
487}
cratefield_core/classifier_agreement.rs

cratefield_core/
classifier_agreement.rs
