pictor_eval/arc.rs
1//! ARC (AI2 Reasoning Challenge) evaluation harness.
2//!
3//! ARC-Easy and ARC-Challenge are four-to-five option multiple-choice benchmarks
4//! from Clark et al. (2018). Both splits use the same [`McDataset`] structure
5//! already in this crate; this module provides thin delegation wrappers that
6//! carry the split identity and produce a strongly-typed [`ArcResult`].
7//!
8//! # Example
9//!
10//! ```rust
11//! use pictor_eval::arc::{ArcEvaluator, ArcSplit};
12//! use pictor_eval::dataset::McDataset;
13//!
14//! let evaluator = ArcEvaluator::easy();
15//! assert_eq!(evaluator.split(), ArcSplit::Easy);
16//! ```
17
18use crate::accuracy::{AccuracyResult, McEvaluator, McLogitEvaluator};
19use crate::dataset::McDataset;
20
21// ──────────────────────────────────────────────────────────────────────────────
22// ArcSplit
23// ──────────────────────────────────────────────────────────────────────────────
24
25/// Which partition of the ARC benchmark this evaluator targets.
26#[derive(Debug, Clone, Copy, PartialEq, Eq)]
27pub enum ArcSplit {
28 /// ARC-Easy: questions that the majority of retrieval and word-co-occurrence
29 /// algorithms could answer correctly.
30 Easy,
31 /// ARC-Challenge: questions answered incorrectly by both a retrieval-based
32 /// algorithm and a word co-occurrence algorithm.
33 Challenge,
34}
35
36impl ArcSplit {
37 /// The canonical benchmark name string for this split.
38 pub fn name(&self) -> &'static str {
39 match self {
40 ArcSplit::Easy => "ARC-Easy",
41 ArcSplit::Challenge => "ARC-Challenge",
42 }
43 }
44}
45
46// ──────────────────────────────────────────────────────────────────────────────
47// ArcEvaluator
48// ──────────────────────────────────────────────────────────────────────────────
49
50/// Evaluator for ARC-Easy or ARC-Challenge benchmarks.
51///
52/// Delegates to [`McEvaluator`] (completion-based) and [`McLogitEvaluator`]
53/// (logit-based) — ARC is structurally identical to MMLU multiple-choice.
54///
55/// ARC questions may have 4 **or** 5 options (keyed A-E). The underlying
56/// evaluators already handle variable-length choice vectors gracefully.
57pub struct ArcEvaluator {
58 /// Which split this evaluator represents.
59 split: ArcSplit,
60 /// String-completion-based evaluator (delegates to [`McEvaluator`]).
61 mc: McEvaluator,
62 /// Logit-based evaluator (delegates to [`McLogitEvaluator`]).
63 mc_logit: McLogitEvaluator,
64}
65
66impl ArcEvaluator {
67 /// Construct an evaluator for the ARC-Easy split.
68 pub fn easy() -> Self {
69 Self::new(ArcSplit::Easy)
70 }
71
72 /// Construct an evaluator for the ARC-Challenge split.
73 pub fn challenge() -> Self {
74 Self::new(ArcSplit::Challenge)
75 }
76
77 /// Internal constructor shared by [`easy`](Self::easy) and
78 /// [`challenge`](Self::challenge).
79 fn new(split: ArcSplit) -> Self {
80 // ARC questions typically present 4 or 5 options labelled A-E.
81 // The template uses {a}..{d}; a fifth choice would be at index 4 but
82 // McEvaluator::extract_answer also handles 'E' if we extend the template.
83 // For now, use the same standard template as MMLU — callers that need
84 // five-option support can inject a custom template via McEvaluator::with_template.
85 let template = "{question}\nA) {a}\nB) {b}\nC) {c}\nD) {d}\nAnswer:".to_string();
86
87 Self {
88 split,
89 mc: McEvaluator {
90 prompt_template: template.clone(),
91 },
92 mc_logit: McLogitEvaluator {
93 prompt_template: template,
94 },
95 }
96 }
97
98 /// Which split this evaluator represents.
99 pub fn split(&self) -> ArcSplit {
100 self.split
101 }
102
103 /// Evaluate by comparing model completions to answer letter choices.
104 ///
105 /// `completions` must be the same length as `dataset.questions`; surplus or
106 /// missing entries are handled by zipping (shortest wins). Each completion is
107 /// expected to begin with the letter of the chosen answer (A, B, C, D, or E).
108 pub fn evaluate_completions(
109 &self,
110 dataset: &McDataset,
111 completions: &[String],
112 ) -> AccuracyResult {
113 self.mc.evaluate_dataset(dataset, completions)
114 }
115
116 /// Evaluate by selecting the choice with the highest per-choice log-probability.
117 ///
118 /// `per_choice_logits[i]` is a vector of log-probabilities — one per answer
119 /// option of `dataset.questions[i]`. The evaluator picks the argmax.
120 pub fn evaluate_logits(
121 &self,
122 dataset: &McDataset,
123 per_choice_logits: &[Vec<f32>],
124 ) -> AccuracyResult {
125 self.mc_logit.evaluate_dataset(dataset, per_choice_logits)
126 }
127}
128
129// ──────────────────────────────────────────────────────────────────────────────
130// ArcResult
131// ──────────────────────────────────────────────────────────────────────────────
132
133/// Strongly-typed result from an ARC evaluation run.
134///
135/// Mirrors the fields of [`AccuracyResult`] while attaching the ARC split
136/// identity so that results from different splits remain unambiguous.
137#[derive(Debug, Clone)]
138pub struct ArcResult {
139 /// Which split produced this result.
140 pub split: ArcSplit,
141 /// Accuracy as a fraction in \[0, 1\].
142 pub accuracy: f32,
143 /// Number of correctly answered questions.
144 pub correct: usize,
145 /// Total number of questions evaluated.
146 pub total: usize,
147}
148
149impl ArcResult {
150 /// Build an [`ArcResult`] from a generic [`AccuracyResult`] and the split identity.
151 pub fn from_accuracy_result(split: ArcSplit, result: AccuracyResult) -> Self {
152 Self {
153 split,
154 accuracy: result.accuracy,
155 correct: result.correct,
156 total: result.total,
157 }
158 }
159
160 /// Return accuracy as a percentage in \[0, 100\].
161 pub fn accuracy_pct(&self) -> f32 {
162 self.accuracy * 100.0
163 }
164
165 /// The canonical benchmark name of the split ("ARC-Easy" or "ARC-Challenge").
166 pub fn split_name(&self) -> &'static str {
167 self.split.name()
168 }
169}