zeph_bench/loaders/
frames.rs

1// SPDX-FileCopyrightText: 2026 Andrei G <bug-ops>
2// SPDX-License-Identifier: MIT OR Apache-2.0
3
4use std::{
5    io::{BufRead as _, BufReader},
6    path::Path,
7};
8
9use serde::Deserialize;
10
11use crate::{
12    error::BenchError,
13    scenario::{DatasetLoader, EvalResult, Evaluator, Scenario, exact_match},
14};
15
16#[derive(Debug, Deserialize)]
17struct FramesRecord {
18    #[serde(rename = "Prompt")]
19    prompt: String,
20    #[serde(rename = "Answer")]
21    answer: String,
22    reasoning_types: Option<serde_json::Value>,
23}
24
25/// Loads FRAMES benchmark scenarios from a JSONL file.
26///
27/// **Source**: [`google/frames-benchmark`](https://huggingface.co/datasets/google/frames-benchmark)
28/// on `HuggingFace`.
29///
30/// **Schema**: one JSON object per line:
31/// ```json
32/// {"Prompt": "...", "Answer": "...", "reasoning_types": [...], "wiki_links": [...]}
33/// ```
34///
35/// Each non-empty line becomes one [`Scenario`]:
36/// - `id` — `"frames_{line_number}"` (zero-based, counting from the first line of the file).
37/// - `prompt` — value of `"Prompt"`.
38/// - `expected` — value of `"Answer"`.
39/// - `metadata` — value of `"reasoning_types"` (array of strings, or `null`).
40///
41/// Empty lines are skipped. Unknown fields (e.g. `"wiki_links"`) are ignored.
42///
43/// # Examples
44///
45/// ```no_run
46/// use std::path::Path;
47/// use zeph_bench::loaders::FramesLoader;
48/// use zeph_bench::scenario::DatasetLoader;
49///
50/// let scenarios = FramesLoader.load(Path::new("/data/frames.jsonl")).unwrap();
51/// println!("loaded {} scenarios", scenarios.len());
52/// ```
53#[derive(Debug)]
54pub struct FramesLoader;
55
56impl DatasetLoader for FramesLoader {
57    fn name(&self) -> &'static str {
58        "frames"
59    }
60
61    /// # Errors
62    ///
63    /// Returns [`BenchError::Io`] when the file cannot be read and
64    /// [`BenchError::InvalidFormat`] when a JSONL line cannot be parsed.
65    fn load(&self, path: &Path) -> Result<Vec<Scenario>, BenchError> {
66        let file = std::fs::File::open(path)?;
67        let reader = BufReader::new(file);
68
69        let mut scenarios = Vec::new();
70        for (line_number, line) in reader.lines().enumerate() {
71            let line = line?;
72            let trimmed = line.trim();
73            if trimmed.is_empty() {
74                continue;
75            }
76            let record: FramesRecord = serde_json::from_str(trimmed)
77                .map_err(|e| BenchError::InvalidFormat(format!("line {line_number}: {e}")))?;
78
79            let metadata = record.reasoning_types.unwrap_or(serde_json::Value::Null);
80
81            scenarios.push(Scenario::single(
82                format!("frames_{line_number}"),
83                record.prompt,
84                record.answer,
85                metadata,
86            ));
87        }
88        Ok(scenarios)
89    }
90}
91
92/// Evaluates FRAMES responses using case-insensitive exact match.
93///
94/// Normalization (applied to both prediction and reference before comparison):
95/// 1. Keep only alphanumeric characters and whitespace.
96/// 2. Convert to lowercase.
97/// 3. Collapse runs of whitespace.
98///
99/// Score is `1.0` when the normalized strings match, `0.0` otherwise.
100///
101/// # Examples
102///
103/// ```
104/// use zeph_bench::{Scenario, loaders::FramesEvaluator};
105/// use zeph_bench::scenario::Evaluator;
106///
107/// let scenario = Scenario::single("frames_0", "Capital of France?", "Paris", serde_json::Value::Null);
108///
109/// // Case-insensitive and punctuation-stripped.
110/// assert!(FramesEvaluator.evaluate(&scenario, "paris").passed);
111/// assert!(FramesEvaluator.evaluate(&scenario, "Paris!").passed);
112/// assert!(!FramesEvaluator.evaluate(&scenario, "London").passed);
113/// ```
114#[derive(Debug)]
115pub struct FramesEvaluator;
116
117impl Evaluator for FramesEvaluator {
118    fn evaluate(&self, scenario: &Scenario, agent_response: &str) -> EvalResult {
119        let passed = exact_match(agent_response, &scenario.expected);
120        EvalResult {
121            scenario_id: scenario.id.clone(),
122            score: if passed { 1.0 } else { 0.0 },
123            passed,
124            details: format!("exact_match={}", if passed { "true" } else { "false" }),
125        }
126    }
127}
128
129#[cfg(test)]
130mod tests {
131    use super::*;
132
133    const FIXTURE: &str = r#"{"Prompt": "What is 2+2?", "Answer": "4", "reasoning_types": ["math"], "wiki_links": []}
134{"Prompt": "Capital of France?", "Answer": "Paris", "reasoning_types": ["geography"]}
135"#;
136
137    fn load_from_str(jsonl: &str) -> Vec<Scenario> {
138        let dir = tempfile::tempdir().unwrap();
139        let path = dir.path().join("frames.jsonl");
140        std::fs::write(&path, jsonl).unwrap();
141        FramesLoader.load(&path).unwrap()
142    }
143
144    #[test]
145    fn load_parses_scenario_count() {
146        let scenarios = load_from_str(FIXTURE);
147        assert_eq!(scenarios.len(), 2);
148    }
149
150    #[test]
151    fn load_builds_correct_ids() {
152        let scenarios = load_from_str(FIXTURE);
153        assert_eq!(scenarios[0].id, "frames_0");
154        assert_eq!(scenarios[1].id, "frames_1");
155    }
156
157    #[test]
158    fn load_maps_prompt_and_expected() {
159        let scenarios = load_from_str(FIXTURE);
160        assert_eq!(scenarios[0].primary_prompt().unwrap(), "What is 2+2?");
161        assert_eq!(scenarios[0].expected, "4");
162    }
163
164    #[test]
165    fn load_stores_reasoning_types_in_metadata() {
166        let scenarios = load_from_str(FIXTURE);
167        assert!(scenarios[0].metadata.is_array());
168    }
169
170    #[test]
171    fn evaluator_exact_match_passes() {
172        let scenarios = load_from_str(FIXTURE);
173        let result = FramesEvaluator.evaluate(&scenarios[0], "4");
174        assert!(result.passed);
175        assert!((result.score - 1.0).abs() < f64::EPSILON);
176    }
177
178    #[test]
179    fn evaluator_wrong_answer_fails() {
180        let scenarios = load_from_str(FIXTURE);
181        let result = FramesEvaluator.evaluate(&scenarios[0], "5");
182        assert!(!result.passed);
183        assert!(result.score < f64::EPSILON);
184    }
185
186    #[test]
187    fn evaluator_case_insensitive_match() {
188        let scenarios = load_from_str(FIXTURE);
189        let result = FramesEvaluator.evaluate(&scenarios[1], "paris");
190        assert!(result.passed);
191    }
192
193    #[test]
194    fn load_invalid_jsonl_returns_error() {
195        let dir = tempfile::tempdir().unwrap();
196        let path = dir.path().join("bad.jsonl");
197        std::fs::write(&path, "not json\n").unwrap();
198        assert!(FramesLoader.load(&path).is_err());
199    }
200}
zeph_bench/loaders/frames.rs

zeph_bench/loaders/
frames.rs
