rusty_eval/dataset.rs
1//! Versioned evaluation datasets.
2//!
3//! A dataset is a named, versioned set of [`EvalCase`]s stored as JSONL:
4//! line 1 is the header, every following non-blank line is one case.
5//!
6//! ```jsonl
7//! {"kind":"header","format_version":1,"name":"math-tools","version":"1.0.0"}
8//! {"kind":"case","id":"add","input":{"messages":[{"role":"user","content":"2+3?"}]},"expect":{"tool_trajectory":[{"name":"calculator","args":{"/op":"add"}}]},"tags":["smoke"]}
9//! ```
10//!
11//! Loading validates the declared `format_version` against
12//! [`DATASET_FORMAT_VERSION`] and refuses to guess across versions — a
13//! dataset from the future is an error, never a silent misread. Serialization
14//! is canonical (struct field order, sorted map keys, compact JSON), so
15//! `load → save` is byte-stable and datasets diff cleanly in version control.
16
17use std::collections::HashSet;
18use std::path::Path;
19
20use serde::{Deserialize, Serialize};
21use serde_json::{Map, Value};
22
23use crate::assertion::Assertion;
24use crate::error::{EvalError, Result};
25
26/// The dataset format version this build loads and writes.
27pub const DATASET_FORMAT_VERSION: u64 = 1;
28
29/// One line of the JSONL dataset file. The `kind` tag is written first, so
30/// headers and cases are greppable without a JSON parser.
31#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
32#[serde(tag = "kind", rename_all = "snake_case")]
33enum DatasetLine {
34 /// Line 1: dataset identity and format version.
35 Header {
36 /// Schema version of the file; must equal [`DATASET_FORMAT_VERSION`].
37 format_version: u64,
38 /// Dataset name (e.g. `math-tools`).
39 name: String,
40 /// Dataset version (semver string, bumped by the author on any case
41 /// or expectation change).
42 version: String,
43 },
44 /// One evaluation case.
45 Case(EvalCase),
46}
47
48/// One evaluation case: an input payload plus what a correct run looks like.
49#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
50pub struct EvalCase {
51 /// Stable case id, unique within the dataset. Reports and comparisons key
52 /// on it, so renaming an id retires the old case's history.
53 pub id: String,
54
55 /// The run input payload. By default the experiment runner merges this
56 /// into the run's initial [`rusty_agent_runtime::state::State`], so it
57 /// must be a JSON object whose keys are state channels (e.g. `messages`
58 /// for the prebuilt ReAct agent). Agent factories may interpret it
59 /// differently by returning an explicit initial state.
60 pub input: Value,
61
62 /// What a correct run looks like, converted into [`Assertion`]s at
63 /// evaluation time.
64 #[serde(default)]
65 pub expect: Expectation,
66
67 /// Free-form labels for slicing reports (`smoke`, `regression`, ...).
68 #[serde(default, skip_serializing_if = "Vec::is_empty")]
69 pub tags: Vec<String>,
70}
71
72/// The declarative expectations of an [`EvalCase`].
73///
74/// Expectations stay data (JSON-serializable, diffable); they become
75/// executable [`Assertion`]s via [`Expectation::assertions`]. Empty sections
76/// are omitted on the wire, so a case declares only what it checks.
77#[derive(Debug, Clone, Default, PartialEq, Serialize, Deserialize)]
78pub struct Expectation {
79 /// The tool calls a correct run makes, as an ordered subsequence: each
80 /// expected call must appear in order, but the run may make additional
81 /// calls between them. Empty means "no trajectory requirement" — use
82 /// `forbid_tools` to require tool-free runs.
83 #[serde(default, skip_serializing_if = "Vec::is_empty")]
84 pub tool_trajectory: Vec<ExpectedToolCall>,
85
86 /// Predicates over the run's final state.
87 #[serde(default, skip_serializing_if = "Vec::is_empty")]
88 pub state: Vec<StatePredicate>,
89
90 /// Tools a correct run never calls.
91 #[serde(default, skip_serializing_if = "Vec::is_empty")]
92 pub forbid_tools: Vec<String>,
93
94 /// Upper bound on the run's total journaled cost in USD.
95 #[serde(default, skip_serializing_if = "Option::is_none")]
96 pub max_cost_usd: Option<f64>,
97
98 /// Upper bound on the run's wall latency in milliseconds.
99 #[serde(default, skip_serializing_if = "Option::is_none")]
100 pub max_latency_ms: Option<u64>,
101}
102
103impl Expectation {
104 /// `true` when the case declares no expectations at all.
105 pub fn is_empty(&self) -> bool {
106 self.tool_trajectory.is_empty()
107 && self.state.is_empty()
108 && self.forbid_tools.is_empty()
109 && self.max_cost_usd.is_none()
110 && self.max_latency_ms.is_none()
111 }
112
113 /// Convert into executable assertions. Each expectation section maps to
114 /// at most one assertion (`state` maps to one per predicate), so a case
115 /// with a trajectory, two state predicates, and a latency bound yields
116 /// four assertions, each reported and aggregated separately.
117 pub fn assertions(&self) -> Vec<Assertion> {
118 let mut out = Vec::new();
119 if !self.tool_trajectory.is_empty() {
120 out.push(Assertion::ToolCallOrder {
121 expected: self.tool_trajectory.clone(),
122 });
123 }
124 for predicate in &self.state {
125 out.push(Assertion::StatePredicate {
126 pointer: predicate.pointer.clone(),
127 expected: predicate.expected.clone(),
128 });
129 }
130 if !self.forbid_tools.is_empty() {
131 out.push(Assertion::NoToolCall {
132 names: self.forbid_tools.clone(),
133 });
134 }
135 if let Some(usd) = self.max_cost_usd {
136 out.push(Assertion::MaxCost { usd });
137 }
138 if let Some(ms) = self.max_latency_ms {
139 out.push(Assertion::MaxLatency { ms });
140 }
141 out
142 }
143}
144
145/// One expected tool call in a trajectory: a name, plus optional matchers
146/// against its arguments.
147#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
148pub struct ExpectedToolCall {
149 /// The tool name, exactly as journaled (the `tool` field of the canonical
150 /// tool-call request payload).
151 pub name: String,
152
153 /// Argument matchers: JSON pointer (RFC 6901) into the call's arguments
154 /// mapped to the expected value. `{"op": "add"}`-style checks are written
155 /// `{"/op": "add"}`; the empty pointer `""` matches the whole arguments
156 /// value. An empty map matches any arguments.
157 #[serde(default, skip_serializing_if = "Map::is_empty")]
158 pub args: Map<String, Value>,
159}
160
161impl ExpectedToolCall {
162 /// Name-only expectation (any arguments).
163 pub fn named(name: impl Into<String>) -> Self {
164 Self {
165 name: name.into(),
166 args: Map::new(),
167 }
168 }
169
170 /// `true` when `arguments` satisfies every matcher.
171 pub fn matches_arguments(&self, arguments: &Value) -> bool {
172 self.args
173 .iter()
174 .all(|(pointer, expected)| arguments.pointer(pointer) == Some(expected))
175 }
176}
177
178/// One predicate over the run's final state: the value at a JSON pointer
179/// must equal `expected`.
180#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
181pub struct StatePredicate {
182 /// JSON pointer (RFC 6901) into the final state object, e.g.
183 /// `/messages/3/content`.
184 pub pointer: String,
185
186 /// The expected value, compared with `==`.
187 pub expected: Value,
188}
189
190/// A named, versioned set of evaluation cases.
191#[derive(Debug, Clone, PartialEq)]
192pub struct Dataset {
193 name: String,
194 version: String,
195 cases: Vec<EvalCase>,
196}
197
198impl Dataset {
199 /// Build a dataset in memory. Case ids must be unique.
200 pub fn new(
201 name: impl Into<String>,
202 version: impl Into<String>,
203 cases: Vec<EvalCase>,
204 ) -> Result<Self> {
205 let dataset = Self {
206 name: name.into(),
207 version: version.into(),
208 cases,
209 };
210 dataset.validate_ids()?;
211 Ok(dataset)
212 }
213
214 /// The dataset name.
215 pub fn name(&self) -> &str {
216 &self.name
217 }
218
219 /// The dataset version.
220 pub fn version(&self) -> &str {
221 &self.version
222 }
223
224 /// The cases, in file order.
225 pub fn cases(&self) -> &[EvalCase] {
226 &self.cases
227 }
228
229 /// Parse a dataset from JSONL text. Blank lines are ignored; the first
230 /// non-blank line must be the header.
231 pub fn from_jsonl(text: &str) -> Result<Self> {
232 let mut header: Option<(String, String)> = None;
233 let mut cases = Vec::new();
234
235 for (index, line) in text.lines().enumerate() {
236 let line = line.trim();
237 if line.is_empty() {
238 continue;
239 }
240 let line_no = index + 1;
241 let parsed: DatasetLine = serde_json::from_str(line)
242 .map_err(|e| EvalError::Dataset(format!("line {line_no}: malformed JSON: {e}")))?;
243 match parsed {
244 DatasetLine::Header {
245 format_version,
246 name,
247 version,
248 } => {
249 if header.is_some() || !cases.is_empty() {
250 return Err(EvalError::Dataset(format!(
251 "line {line_no}: header must be the first non-blank line"
252 )));
253 }
254 if format_version != DATASET_FORMAT_VERSION {
255 return Err(EvalError::UnsupportedVersion {
256 found: format_version,
257 supported: DATASET_FORMAT_VERSION,
258 });
259 }
260 header = Some((name, version));
261 }
262 DatasetLine::Case(case) => {
263 if header.is_none() {
264 return Err(EvalError::Dataset(format!(
265 "line {line_no}: case before the header"
266 )));
267 }
268 cases.push(case);
269 }
270 }
271 }
272
273 let (name, version) =
274 header.ok_or_else(|| EvalError::Dataset("missing header line".to_owned()))?;
275 Self::new(name, version, cases)
276 }
277
278 /// Load a dataset from a JSONL file.
279 pub fn load(path: impl AsRef<Path>) -> Result<Self> {
280 let text = std::fs::read_to_string(path.as_ref())?;
281 Self::from_jsonl(&text)
282 .map_err(|e| EvalError::Dataset(format!("{}: {e}", path.as_ref().display())))
283 }
284
285 /// Serialize to canonical JSONL (trailing newline on every line).
286 pub fn to_jsonl(&self) -> String {
287 let mut out = String::new();
288 let header = DatasetLine::Header {
289 format_version: DATASET_FORMAT_VERSION,
290 name: self.name.clone(),
291 version: self.version.clone(),
292 };
293 // Serialization of a struct we fully control cannot fail.
294 out.push_str(&serde_json::to_string(&header).expect("dataset header serializes"));
295 out.push('\n');
296 for case in &self.cases {
297 out.push_str(
298 &serde_json::to_string(&DatasetLine::Case(case.clone()))
299 .expect("dataset case serializes"),
300 );
301 out.push('\n');
302 }
303 out
304 }
305
306 /// Save the dataset as canonical JSONL.
307 pub fn save(&self, path: impl AsRef<Path>) -> Result<()> {
308 std::fs::write(path.as_ref(), self.to_jsonl())?;
309 Ok(())
310 }
311
312 fn validate_ids(&self) -> Result<()> {
313 let mut seen = HashSet::with_capacity(self.cases.len());
314 for case in &self.cases {
315 if case.id.is_empty() {
316 return Err(EvalError::Dataset("case id must not be empty".to_owned()));
317 }
318 if !seen.insert(case.id.as_str()) {
319 return Err(EvalError::Dataset(format!(
320 "duplicate case id `{}`",
321 case.id
322 )));
323 }
324 }
325 Ok(())
326 }
327}