jev_bridge/
readout.rs

1//! The readout self-check: does this model actually answer with slot letters?
2//!
3//! The startup checks in [`crate::prompt`] prove the *mechanism*: the answer
4//! letters are single tokens, appending one cannot re-tokenize the prompt, and
5//! a transport can read the distribution. They say nothing about whether the
6//! model was trained to answer that way. A general instruct model that never
7//! saw single-token slot readout does not fail loudly — it degrades silently,
8//! typically by acquiring a probability floor on one option — and the bridge
9//! would return confident-looking numbers about nothing.
10//!
11//! So this module asks a few questions whose answers are obvious, through the
12//! exact readout path every real request uses, and reports what came back.
13//! It cannot prove the model is good at decisions (that needs a real,
14//! gold-labelled workload — see [`crate::evaluate`]); it catches the model
15//! that is not answering the question at all.
16
17use crate::wire::{Question, RowOption};
18use serde::Serialize;
19
20/// The state every probe is asked about.
21pub const PROBE_STATE: &str = "This is a self-check of the decision channel. Answer each question with the option that is obviously correct.";
22
23/// One probe: a question whose answer is obvious, and the option that names it.
24#[derive(Debug, Clone, Copy)]
25pub struct Probe {
26    /// Stable id for the report.
27    pub id: &'static str,
28    /// The criterion to apply to [`PROBE_STATE`].
29    pub question: &'static str,
30    /// `(option id, description)` pairs, in slot order.
31    pub options: &'static [(&'static str, &'static str)],
32    /// The option id an answer that understood the question would name.
33    pub expected: &'static str,
34}
35
36/// The probes, covering all three question shapes.
37///
38/// Deliberately trivial: a model that cannot name the right option here is not
39/// making decisions, whatever its probabilities look like.
40pub const PROBES: &[Probe] = &[
41    Probe {
42        id: "sky-is-blue",
43        question: "On a clear day, is the sky blue?",
44        options: &[("true", "Yes"), ("false", "No")],
45        expected: "true",
46    },
47    Probe {
48        id: "ice-not-hot",
49        question: "Is ice hotter than boiling water?",
50        options: &[("true", "Yes"), ("false", "No")],
51        expected: "false",
52    },
53    Probe {
54        id: "which-is-fruit",
55        question: "Which of these is a fruit?",
56        options: &[
57            ("apple", "An apple"),
58            ("granite", "A piece of granite"),
59            ("iron", "An iron bar"),
60        ],
61        expected: "apple",
62    },
63    Probe {
64        id: "summer-day",
65        question: "How hot is a summer day in the sun?",
66        options: &[
67            ("0", "Freezing"),
68            ("1", "Cold"),
69            ("2", "Warm"),
70            ("3", "Hot"),
71        ],
72        expected: "3",
73    },
74];
75
76/// The probes as scorer questions, all asked about [`PROBE_STATE`].
77pub fn questions() -> Vec<Question> {
78    PROBES
79        .iter()
80        .map(|probe| Question {
81            id: probe.id.to_string(),
82            question: probe.question.to_string(),
83            options: probe
84                .options
85                .iter()
86                .map(|(id, description)| RowOption {
87                    id: (*id).to_string(),
88                    description: (*description).to_string(),
89                })
90                .collect(),
91        })
92        .collect()
93}
94
95/// One probe's outcome.
96#[derive(Debug, Clone, Serialize)]
97pub struct ProbeResult {
98    /// Which probe this is.
99    pub id: String,
100    /// The option that is obviously right.
101    pub expected: String,
102    /// The option the readout named (first maximum, ties to the earlier slot).
103    pub argmax: String,
104    /// The probability the readout gave the expected option.
105    pub expected_probability: f64,
106    /// Whether the readout named the expected option.
107    pub passed: bool,
108}
109
110/// What the self-check found, as reported by `GET /health`.
111#[derive(Debug, Clone, Serialize)]
112pub struct ReadoutCheck {
113    /// How many probes ran.
114    pub probes: usize,
115    /// How many named the expected option.
116    pub passed: usize,
117    /// Every probe's outcome.
118    pub results: Vec<ProbeResult>,
119}
120
121impl ReadoutCheck {
122    /// Whether every probe named the expected option.
123    pub fn is_ok(&self) -> bool {
124        self.probes > 0 && self.passed == self.probes
125    }
126
127    /// Build the report from the per-probe outcomes.
128    pub fn from_results(results: Vec<ProbeResult>) -> Self {
129        let passed = results.iter().filter(|result| result.passed).count();
130        Self {
131            probes: results.len(),
132            passed,
133            results,
134        }
135    }
136}
137
138/// Score one probe's outcome from its answer, using the same first-maximum
139/// rule the wire layer reports its answers with.
140pub fn outcome(probe: &Probe, option_ids: &[String], probabilities: &[f64]) -> ProbeResult {
141    let argmax_index = probabilities
142        .iter()
143        .enumerate()
144        .fold(0usize, |best, (index, value)| {
145            if *value > probabilities[best] {
146                index
147            } else {
148                best
149            }
150        });
151    let argmax = option_ids.get(argmax_index).cloned().unwrap_or_default();
152    let expected_probability = option_ids
153        .iter()
154        .position(|id| *id == probe.expected)
155        .and_then(|index| probabilities.get(index))
156        .copied()
157        .unwrap_or(0.0);
158    ProbeResult {
159        id: probe.id.to_string(),
160        expected: probe.expected.to_string(),
161        passed: argmax == probe.expected,
162        argmax,
163        expected_probability,
164    }
165}
166
167#[cfg(test)]
168mod tests {
169    use super::*;
170
171    #[test]
172    fn every_probe_declares_its_expected_option() {
173        for probe in PROBES {
174            assert!(
175                probe.options.iter().any(|(id, _)| *id == probe.expected),
176                "probe {:?}: expected {:?} is not one of its options",
177                probe.id,
178                probe.expected
179            );
180            assert!(
181                probe.options.len() >= 2,
182                "probe {:?} needs at least two options",
183                probe.id
184            );
185        }
186        let ids: Vec<&str> = PROBES.iter().map(|probe| probe.id).collect();
187        let mut unique = ids.clone();
188        unique.sort_unstable();
189        unique.dedup();
190        assert_eq!(unique.len(), ids.len(), "probe ids must be unique");
191    }
192
193    #[test]
194    fn questions_carry_the_probes_verbatim() {
195        let questions = questions();
196        assert_eq!(questions.len(), PROBES.len());
197        assert_eq!(questions[0].id, "sky-is-blue");
198        assert_eq!(questions[0].options[0].id, "true");
199        assert_eq!(questions[0].options[1].description, "No");
200    }
201
202    #[test]
203    fn an_answer_that_names_the_expected_option_passes() {
204        let probe = &PROBES[0];
205        let ids = vec!["true".to_string(), "false".to_string()];
206        let result = outcome(probe, &ids, &[0.9, 0.1]);
207        assert!(result.passed);
208        assert_eq!(result.argmax, "true");
209        assert!((result.expected_probability - 0.9).abs() < 1e-12);
210    }
211
212    #[test]
213    fn an_answer_that_names_another_option_fails_and_reports_what_it_named() {
214        let probe = &PROBES[1];
215        let ids = vec!["true".to_string(), "false".to_string()];
216        let result = outcome(probe, &ids, &[0.8, 0.2]);
217        assert!(!result.passed);
218        assert_eq!(result.argmax, "true");
219        assert!((result.expected_probability - 0.2).abs() < 1e-12);
220    }
221
222    #[test]
223    fn ties_resolve_to_the_earlier_slot() {
224        let probe = &PROBES[0];
225        let ids = vec!["true".to_string(), "false".to_string()];
226        // A flat distribution resolves to the first declared option, exactly
227        // as the wire layer's first-max rule does.
228        assert_eq!(outcome(probe, &ids, &[0.5, 0.5]).argmax, "true");
229    }
230
231    #[test]
232    fn a_report_counts_its_passes() {
233        let ids = vec!["true".to_string(), "false".to_string()];
234        let results = vec![
235            outcome(&PROBES[0], &ids, &[0.9, 0.1]),
236            outcome(&PROBES[1], &ids, &[0.8, 0.2]),
237        ];
238        let check = ReadoutCheck::from_results(results);
239        assert_eq!(check.probes, 2);
240        assert_eq!(check.passed, 1);
241        assert!(!check.is_ok());
242
243        let all_pass = ReadoutCheck::from_results(vec![outcome(&PROBES[0], &ids, &[0.9, 0.1])]);
244        assert!(all_pass.is_ok());
245
246        // An empty check is not a pass: nothing was verified.
247        assert!(!ReadoutCheck::from_results(Vec::new()).is_ok());
248    }
249}
jev_bridge/readout.rs

jev_bridge/
readout.rs
