pictor_eval/
boolq.rs

1//! BoolQ yes/no question answering evaluation harness.
2//!
3//! BoolQ (Clark et al., 2019) tests reading comprehension via binary yes/no
4//! questions. Each item has a passage, a question, and a boolean answer.
5//!
6//! # Example
7//!
8//! ```rust
9//! use pictor_eval::boolq::{BoolQDataset, BoolQEvaluator, BoolQItem};
10//!
11//! let evaluator = BoolQEvaluator::new();
12//! let answer = BoolQEvaluator::extract_answer("Yes, that is correct.");
13//! assert_eq!(answer, Some(true));
14//! ```
15
16// ──────────────────────────────────────────────────────────────────────────────
17// BoolQItem
18// ──────────────────────────────────────────────────────────────────────────────
19
20/// A single BoolQ dataset item.
21#[derive(Debug, Clone, PartialEq)]
22pub struct BoolQItem {
23    /// The context passage from which the answer can be inferred.
24    pub passage: String,
25    /// The yes/no question about the passage.
26    pub question: String,
27    /// Gold answer: `true` = yes, `false` = no.
28    pub answer: bool,
29}
30
31// ──────────────────────────────────────────────────────────────────────────────
32// BoolQDataset
33// ──────────────────────────────────────────────────────────────────────────────
34
35/// A collection of [`BoolQItem`] instances.
36pub struct BoolQDataset {
37    /// All items in insertion order.
38    pub items: Vec<BoolQItem>,
39}
40
41impl BoolQDataset {
42    /// Create a dataset from a vector of items.
43    pub fn from_items(items: Vec<BoolQItem>) -> Self {
44        Self { items }
45    }
46
47    /// Return the number of items in the dataset.
48    pub fn len(&self) -> usize {
49        self.items.len()
50    }
51
52    /// Return `true` if the dataset contains no items.
53    pub fn is_empty(&self) -> bool {
54        self.items.is_empty()
55    }
56}
57
58// ──────────────────────────────────────────────────────────────────────────────
59// BoolQResult
60// ──────────────────────────────────────────────────────────────────────────────
61
62/// Aggregated result from a BoolQ evaluation pass.
63#[derive(Debug, Clone)]
64pub struct BoolQResult {
65    /// Fraction of items answered correctly (0.0–1.0).
66    pub accuracy: f32,
67    /// Accuracy as a percentage (0.0–100.0).
68    pub accuracy_pct: f32,
69    /// Number of correctly answered items.
70    pub correct: usize,
71    /// Total items evaluated (excludes items beyond the shorter of dataset/completions).
72    pub total: usize,
73    /// Number of items where the model predicted "yes" (true).
74    pub yes_predicted: usize,
75    /// Number of items where the model predicted "no" (false).
76    pub no_predicted: usize,
77}
78
79// ──────────────────────────────────────────────────────────────────────────────
80// BoolQEvaluator
81// ──────────────────────────────────────────────────────────────────────────────
82
83/// Evaluator for BoolQ yes/no question answering.
84///
85/// The evaluator is stateless; all scoring logic is encoded in its methods.
86/// It implements [`Default`] for use with frameworks that require that bound.
87pub struct BoolQEvaluator;
88
89impl BoolQEvaluator {
90    /// Create a new evaluator (equivalent to [`Default::default`]).
91    pub fn new() -> Self {
92        Self
93    }
94
95    /// Build a prompt string for a BoolQ item.
96    ///
97    /// Format: `"Passage: {passage}\nQuestion: {question}\nAnswer:"`
98    pub fn build_prompt(&self, item: &BoolQItem) -> String {
99        format!(
100            "Passage: {}\nQuestion: {}\nAnswer:",
101            item.passage, item.question
102        )
103    }
104
105    /// Extract a boolean answer from a model completion string.
106    ///
107    /// Algorithm:
108    /// 1. Strip leading ASCII whitespace.
109    /// 2. Take the first 3 characters and lowercase them.
110    /// 3. If the prefix is `"yes"`, return `Some(true)`.
111    /// 4. If the prefix starts with `"no"`, return `Some(false)`.
112    /// 5. Otherwise return `None`.
113    ///
114    /// A string shorter than 2 characters always returns `None`.
115    pub fn extract_answer(completion: &str) -> Option<bool> {
116        let trimmed = completion.trim_start();
117        // Need at least "no" (2 chars) to match anything.
118        if trimmed.len() < 2 {
119            return None;
120        }
121        // Collect up to 3 chars for case-insensitive prefix matching.
122        let prefix: String = trimmed.chars().take(3).collect::<String>().to_lowercase();
123        if prefix.starts_with("yes") {
124            Some(true)
125        } else if prefix.starts_with("no") {
126            Some(false)
127        } else {
128            None
129        }
130    }
131
132    /// Score a single completion against the gold boolean answer.
133    ///
134    /// Returns `true` iff [`extract_answer`](Self::extract_answer) produces
135    /// `Some(gold)`, `false` otherwise (including when extraction fails).
136    pub fn score(&self, completion: &str, gold: bool) -> bool {
137        Self::extract_answer(completion) == Some(gold)
138    }
139
140    /// Evaluate a list of string completions against the dataset.
141    ///
142    /// Only the first `min(dataset.len(), completions.len())` items are scored.
143    /// Completions where [`extract_answer`](Self::extract_answer) returns `None`
144    /// are counted as wrong but not added to `yes_predicted` or `no_predicted`.
145    pub fn evaluate_completions(
146        &self,
147        dataset: &BoolQDataset,
148        completions: &[String],
149    ) -> BoolQResult {
150        let n = dataset.items.len().min(completions.len());
151        let mut correct = 0usize;
152        let mut yes_predicted = 0usize;
153        let mut no_predicted = 0usize;
154
155        for (i, completion) in completions.iter().enumerate().take(n) {
156            let prediction = Self::extract_answer(completion);
157            match prediction {
158                Some(true) => yes_predicted += 1,
159                Some(false) => no_predicted += 1,
160                None => {}
161            }
162            if prediction == Some(dataset.items[i].answer) {
163                correct += 1;
164            }
165        }
166
167        let total = n;
168        let accuracy = if total == 0 {
169            0.0_f32
170        } else {
171            correct as f32 / total as f32
172        };
173        BoolQResult {
174            accuracy,
175            accuracy_pct: accuracy * 100.0,
176            correct,
177            total,
178            yes_predicted,
179            no_predicted,
180        }
181    }
182
183    /// Evaluate using logit pairs `[logit_yes, logit_no]` per item.
184    ///
185    /// Prediction: if `logit_pairs[i][0] > logit_pairs[i][1]` then `true` (yes),
186    /// otherwise `false` (no). The prediction is compared to `dataset.items[i].answer`.
187    ///
188    /// Only the first `min(dataset.len(), logit_pairs.len())` items are scored.
189    pub fn evaluate_logits(&self, dataset: &BoolQDataset, logit_pairs: &[[f32; 2]]) -> BoolQResult {
190        let n = dataset.items.len().min(logit_pairs.len());
191        let mut correct = 0usize;
192        let mut yes_predicted = 0usize;
193        let mut no_predicted = 0usize;
194
195        for (i, pair) in logit_pairs.iter().enumerate().take(n) {
196            let pred_yes = pair[0] > pair[1];
197            if pred_yes {
198                yes_predicted += 1;
199            } else {
200                no_predicted += 1;
201            }
202            if pred_yes == dataset.items[i].answer {
203                correct += 1;
204            }
205        }
206
207        let total = n;
208        let accuracy = if total == 0 {
209            0.0_f32
210        } else {
211            correct as f32 / total as f32
212        };
213        BoolQResult {
214            accuracy,
215            accuracy_pct: accuracy * 100.0,
216            correct,
217            total,
218            yes_predicted,
219            no_predicted,
220        }
221    }
222}
223
224impl Default for BoolQEvaluator {
225    fn default() -> Self {
226        Self::new()
227    }
228}
229
230// ──────────────────────────────────────────────────────────────────────────────
231// Unit tests
232// ──────────────────────────────────────────────────────────────────────────────
233
234#[cfg(test)]
235mod tests {
236    use super::*;
237
238    #[test]
239    fn extract_answer_yes() {
240        assert_eq!(
241            BoolQEvaluator::extract_answer("Yes, that is correct."),
242            Some(true)
243        );
244    }
245
246    #[test]
247    fn extract_answer_no() {
248        assert_eq!(
249            BoolQEvaluator::extract_answer("No, it is not."),
250            Some(false)
251        );
252    }
253
254    #[test]
255    fn extract_answer_case_insensitive() {
256        assert_eq!(BoolQEvaluator::extract_answer("YES"), Some(true));
257        assert_eq!(BoolQEvaluator::extract_answer("NO"), Some(false));
258        assert_eq!(BoolQEvaluator::extract_answer("yes."), Some(true));
259    }
260
261    #[test]
262    fn extract_answer_leading_whitespace() {
263        assert_eq!(BoolQEvaluator::extract_answer("  yes"), Some(true));
264        assert_eq!(BoolQEvaluator::extract_answer("\t\nno"), Some(false));
265    }
266
267    #[test]
268    fn extract_answer_none_for_garbage() {
269        assert_eq!(BoolQEvaluator::extract_answer("maybe"), None);
270        assert_eq!(BoolQEvaluator::extract_answer(""), None);
271        assert_eq!(BoolQEvaluator::extract_answer("I don't know"), None);
272    }
273
274    #[test]
275    fn extract_answer_short_string_no_panic() {
276        assert_eq!(BoolQEvaluator::extract_answer("y"), None);
277        assert_eq!(BoolQEvaluator::extract_answer("n"), None);
278    }
279}
pictor_eval/boolq.rs

pictor_eval/
boolq.rs
