Skip to main content

rusty_eval/
dataset.rs

1//! Versioned evaluation datasets.
2//!
3//! A dataset is a named, versioned set of [`EvalCase`]s stored as JSONL:
4//! line 1 is the header, every following non-blank line is one case.
5//!
6//! ```jsonl
7//! {"kind":"header","format_version":1,"name":"math-tools","version":"1.0.0"}
8//! {"kind":"case","id":"add","input":{"messages":[{"role":"user","content":"2+3?"}]},"expect":{"tool_trajectory":[{"name":"calculator","args":{"/op":"add"}}]},"tags":["smoke"]}
9//! ```
10//!
11//! Loading validates the declared `format_version` against
12//! [`DATASET_FORMAT_VERSION`] and refuses to guess across versions — a
13//! dataset from the future is an error, never a silent misread. Serialization
14//! is canonical (struct field order, sorted map keys, compact JSON), so
15//! `load → save` is byte-stable and datasets diff cleanly in version control.
16
17use std::collections::HashSet;
18use std::path::Path;
19
20use serde::{Deserialize, Serialize};
21use serde_json::{Map, Value};
22
23use crate::assertion::Assertion;
24use crate::error::{EvalError, Result};
25
26/// The dataset format version this build loads and writes.
27pub const DATASET_FORMAT_VERSION: u64 = 1;
28
29/// One line of the JSONL dataset file. The `kind` tag is written first, so
30/// headers and cases are greppable without a JSON parser.
31#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
32#[serde(tag = "kind", rename_all = "snake_case")]
33enum DatasetLine {
34    /// Line 1: dataset identity and format version.
35    Header {
36        /// Schema version of the file; must equal [`DATASET_FORMAT_VERSION`].
37        format_version: u64,
38        /// Dataset name (e.g. `math-tools`).
39        name: String,
40        /// Dataset version (semver string, bumped by the author on any case
41        /// or expectation change).
42        version: String,
43    },
44    /// One evaluation case.
45    Case(EvalCase),
46}
47
48/// One evaluation case: an input payload plus what a correct run looks like.
49#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
50pub struct EvalCase {
51    /// Stable case id, unique within the dataset. Reports and comparisons key
52    /// on it, so renaming an id retires the old case's history.
53    pub id: String,
54
55    /// The run input payload. By default the experiment runner merges this
56    /// into the run's initial [`rusty_agent_runtime::state::State`], so it
57    /// must be a JSON object whose keys are state channels (e.g. `messages`
58    /// for the prebuilt ReAct agent). Agent factories may interpret it
59    /// differently by returning an explicit initial state.
60    pub input: Value,
61
62    /// What a correct run looks like, converted into [`Assertion`]s at
63    /// evaluation time.
64    #[serde(default)]
65    pub expect: Expectation,
66
67    /// Free-form labels for slicing reports (`smoke`, `regression`, ...).
68    #[serde(default, skip_serializing_if = "Vec::is_empty")]
69    pub tags: Vec<String>,
70}
71
72/// The declarative expectations of an [`EvalCase`].
73///
74/// Expectations stay data (JSON-serializable, diffable); they become
75/// executable [`Assertion`]s via [`Expectation::assertions`]. Empty sections
76/// are omitted on the wire, so a case declares only what it checks.
77#[derive(Debug, Clone, Default, PartialEq, Serialize, Deserialize)]
78pub struct Expectation {
79    /// The tool calls a correct run makes, as an ordered subsequence: each
80    /// expected call must appear in order, but the run may make additional
81    /// calls between them. Empty means "no trajectory requirement" — use
82    /// `forbid_tools` to require tool-free runs.
83    #[serde(default, skip_serializing_if = "Vec::is_empty")]
84    pub tool_trajectory: Vec<ExpectedToolCall>,
85
86    /// Predicates over the run's final state.
87    #[serde(default, skip_serializing_if = "Vec::is_empty")]
88    pub state: Vec<StatePredicate>,
89
90    /// Tools a correct run never calls.
91    #[serde(default, skip_serializing_if = "Vec::is_empty")]
92    pub forbid_tools: Vec<String>,
93
94    /// Upper bound on the run's total journaled cost in USD.
95    #[serde(default, skip_serializing_if = "Option::is_none")]
96    pub max_cost_usd: Option<f64>,
97
98    /// Upper bound on the run's wall latency in milliseconds.
99    #[serde(default, skip_serializing_if = "Option::is_none")]
100    pub max_latency_ms: Option<u64>,
101}
102
103impl Expectation {
104    /// `true` when the case declares no expectations at all.
105    pub fn is_empty(&self) -> bool {
106        self.tool_trajectory.is_empty()
107            && self.state.is_empty()
108            && self.forbid_tools.is_empty()
109            && self.max_cost_usd.is_none()
110            && self.max_latency_ms.is_none()
111    }
112
113    /// Convert into executable assertions. Each expectation section maps to
114    /// at most one assertion (`state` maps to one per predicate), so a case
115    /// with a trajectory, two state predicates, and a latency bound yields
116    /// four assertions, each reported and aggregated separately.
117    pub fn assertions(&self) -> Vec<Assertion> {
118        let mut out = Vec::new();
119        if !self.tool_trajectory.is_empty() {
120            out.push(Assertion::ToolCallOrder {
121                expected: self.tool_trajectory.clone(),
122            });
123        }
124        for predicate in &self.state {
125            out.push(Assertion::StatePredicate {
126                pointer: predicate.pointer.clone(),
127                expected: predicate.expected.clone(),
128            });
129        }
130        if !self.forbid_tools.is_empty() {
131            out.push(Assertion::NoToolCall {
132                names: self.forbid_tools.clone(),
133            });
134        }
135        if let Some(usd) = self.max_cost_usd {
136            out.push(Assertion::MaxCost { usd });
137        }
138        if let Some(ms) = self.max_latency_ms {
139            out.push(Assertion::MaxLatency { ms });
140        }
141        out
142    }
143}
144
145/// One expected tool call in a trajectory: a name, plus optional matchers
146/// against its arguments.
147#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
148pub struct ExpectedToolCall {
149    /// The tool name, exactly as journaled (the `tool` field of the canonical
150    /// tool-call request payload).
151    pub name: String,
152
153    /// Argument matchers: JSON pointer (RFC 6901) into the call's arguments
154    /// mapped to the expected value. `{"op": "add"}`-style checks are written
155    /// `{"/op": "add"}`; the empty pointer `""` matches the whole arguments
156    /// value. An empty map matches any arguments.
157    #[serde(default, skip_serializing_if = "Map::is_empty")]
158    pub args: Map<String, Value>,
159}
160
161impl ExpectedToolCall {
162    /// Name-only expectation (any arguments).
163    pub fn named(name: impl Into<String>) -> Self {
164        Self {
165            name: name.into(),
166            args: Map::new(),
167        }
168    }
169
170    /// `true` when `arguments` satisfies every matcher.
171    pub fn matches_arguments(&self, arguments: &Value) -> bool {
172        self.args
173            .iter()
174            .all(|(pointer, expected)| arguments.pointer(pointer) == Some(expected))
175    }
176}
177
178/// One predicate over the run's final state: the value at a JSON pointer
179/// must equal `expected`.
180#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
181pub struct StatePredicate {
182    /// JSON pointer (RFC 6901) into the final state object, e.g.
183    /// `/messages/3/content`.
184    pub pointer: String,
185
186    /// The expected value, compared with `==`.
187    pub expected: Value,
188}
189
190/// A named, versioned set of evaluation cases.
191#[derive(Debug, Clone, PartialEq)]
192pub struct Dataset {
193    name: String,
194    version: String,
195    cases: Vec<EvalCase>,
196}
197
198impl Dataset {
199    /// Build a dataset in memory. Case ids must be unique.
200    pub fn new(
201        name: impl Into<String>,
202        version: impl Into<String>,
203        cases: Vec<EvalCase>,
204    ) -> Result<Self> {
205        let dataset = Self {
206            name: name.into(),
207            version: version.into(),
208            cases,
209        };
210        dataset.validate_ids()?;
211        Ok(dataset)
212    }
213
214    /// The dataset name.
215    pub fn name(&self) -> &str {
216        &self.name
217    }
218
219    /// The dataset version.
220    pub fn version(&self) -> &str {
221        &self.version
222    }
223
224    /// The cases, in file order.
225    pub fn cases(&self) -> &[EvalCase] {
226        &self.cases
227    }
228
229    /// Parse a dataset from JSONL text. Blank lines are ignored; the first
230    /// non-blank line must be the header.
231    pub fn from_jsonl(text: &str) -> Result<Self> {
232        let mut header: Option<(String, String)> = None;
233        let mut cases = Vec::new();
234
235        for (index, line) in text.lines().enumerate() {
236            let line = line.trim();
237            if line.is_empty() {
238                continue;
239            }
240            let line_no = index + 1;
241            let parsed: DatasetLine = serde_json::from_str(line)
242                .map_err(|e| EvalError::Dataset(format!("line {line_no}: malformed JSON: {e}")))?;
243            match parsed {
244                DatasetLine::Header {
245                    format_version,
246                    name,
247                    version,
248                } => {
249                    if header.is_some() || !cases.is_empty() {
250                        return Err(EvalError::Dataset(format!(
251                            "line {line_no}: header must be the first non-blank line"
252                        )));
253                    }
254                    if format_version != DATASET_FORMAT_VERSION {
255                        return Err(EvalError::UnsupportedVersion {
256                            found: format_version,
257                            supported: DATASET_FORMAT_VERSION,
258                        });
259                    }
260                    header = Some((name, version));
261                }
262                DatasetLine::Case(case) => {
263                    if header.is_none() {
264                        return Err(EvalError::Dataset(format!(
265                            "line {line_no}: case before the header"
266                        )));
267                    }
268                    cases.push(case);
269                }
270            }
271        }
272
273        let (name, version) =
274            header.ok_or_else(|| EvalError::Dataset("missing header line".to_owned()))?;
275        Self::new(name, version, cases)
276    }
277
278    /// Load a dataset from a JSONL file.
279    pub fn load(path: impl AsRef<Path>) -> Result<Self> {
280        let text = std::fs::read_to_string(path.as_ref())?;
281        Self::from_jsonl(&text)
282            .map_err(|e| EvalError::Dataset(format!("{}: {e}", path.as_ref().display())))
283    }
284
285    /// Serialize to canonical JSONL (trailing newline on every line).
286    pub fn to_jsonl(&self) -> String {
287        let mut out = String::new();
288        let header = DatasetLine::Header {
289            format_version: DATASET_FORMAT_VERSION,
290            name: self.name.clone(),
291            version: self.version.clone(),
292        };
293        // Serialization of a struct we fully control cannot fail.
294        out.push_str(&serde_json::to_string(&header).expect("dataset header serializes"));
295        out.push('\n');
296        for case in &self.cases {
297            out.push_str(
298                &serde_json::to_string(&DatasetLine::Case(case.clone()))
299                    .expect("dataset case serializes"),
300            );
301            out.push('\n');
302        }
303        out
304    }
305
306    /// Save the dataset as canonical JSONL.
307    pub fn save(&self, path: impl AsRef<Path>) -> Result<()> {
308        std::fs::write(path.as_ref(), self.to_jsonl())?;
309        Ok(())
310    }
311
312    fn validate_ids(&self) -> Result<()> {
313        let mut seen = HashSet::with_capacity(self.cases.len());
314        for case in &self.cases {
315            if case.id.is_empty() {
316                return Err(EvalError::Dataset("case id must not be empty".to_owned()));
317            }
318            if !seen.insert(case.id.as_str()) {
319                return Err(EvalError::Dataset(format!(
320                    "duplicate case id `{}`",
321                    case.id
322                )));
323            }
324        }
325        Ok(())
326    }
327}