use std::collections::HashSet;
use std::path::Path;
use serde::{Deserialize, Serialize};
use serde_json::{Map, Value};
use crate::assertion::Assertion;
use crate::error::{EvalError, Result};
pub const DATASET_FORMAT_VERSION: u64 = 1;
#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
#[serde(tag = "kind", rename_all = "snake_case")]
enum DatasetLine {
Header {
format_version: u64,
name: String,
version: String,
},
Case(EvalCase),
}
#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
pub struct EvalCase {
pub id: String,
pub input: Value,
#[serde(default)]
pub expect: Expectation,
#[serde(default, skip_serializing_if = "Vec::is_empty")]
pub tags: Vec<String>,
}
#[derive(Debug, Clone, Default, PartialEq, Serialize, Deserialize)]
pub struct Expectation {
#[serde(default, skip_serializing_if = "Vec::is_empty")]
pub tool_trajectory: Vec<ExpectedToolCall>,
#[serde(default, skip_serializing_if = "Vec::is_empty")]
pub state: Vec<StatePredicate>,
#[serde(default, skip_serializing_if = "Vec::is_empty")]
pub forbid_tools: Vec<String>,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub max_cost_usd: Option<f64>,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub max_latency_ms: Option<u64>,
}
impl Expectation {
pub fn is_empty(&self) -> bool {
self.tool_trajectory.is_empty()
&& self.state.is_empty()
&& self.forbid_tools.is_empty()
&& self.max_cost_usd.is_none()
&& self.max_latency_ms.is_none()
}
pub fn assertions(&self) -> Vec<Assertion> {
let mut out = Vec::new();
if !self.tool_trajectory.is_empty() {
out.push(Assertion::ToolCallOrder {
expected: self.tool_trajectory.clone(),
});
}
for predicate in &self.state {
out.push(Assertion::StatePredicate {
pointer: predicate.pointer.clone(),
expected: predicate.expected.clone(),
});
}
if !self.forbid_tools.is_empty() {
out.push(Assertion::NoToolCall {
names: self.forbid_tools.clone(),
});
}
if let Some(usd) = self.max_cost_usd {
out.push(Assertion::MaxCost { usd });
}
if let Some(ms) = self.max_latency_ms {
out.push(Assertion::MaxLatency { ms });
}
out
}
}
#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
pub struct ExpectedToolCall {
pub name: String,
#[serde(default, skip_serializing_if = "Map::is_empty")]
pub args: Map<String, Value>,
}
impl ExpectedToolCall {
pub fn named(name: impl Into<String>) -> Self {
Self {
name: name.into(),
args: Map::new(),
}
}
pub fn matches_arguments(&self, arguments: &Value) -> bool {
self.args
.iter()
.all(|(pointer, expected)| arguments.pointer(pointer) == Some(expected))
}
}
#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
pub struct StatePredicate {
pub pointer: String,
pub expected: Value,
}
#[derive(Debug, Clone, PartialEq)]
pub struct Dataset {
name: String,
version: String,
cases: Vec<EvalCase>,
}
impl Dataset {
pub fn new(
name: impl Into<String>,
version: impl Into<String>,
cases: Vec<EvalCase>,
) -> Result<Self> {
let dataset = Self {
name: name.into(),
version: version.into(),
cases,
};
dataset.validate_ids()?;
Ok(dataset)
}
pub fn name(&self) -> &str {
&self.name
}
pub fn version(&self) -> &str {
&self.version
}
pub fn cases(&self) -> &[EvalCase] {
&self.cases
}
pub fn from_jsonl(text: &str) -> Result<Self> {
let mut header: Option<(String, String)> = None;
let mut cases = Vec::new();
for (index, line) in text.lines().enumerate() {
let line = line.trim();
if line.is_empty() {
continue;
}
let line_no = index + 1;
let parsed: DatasetLine = serde_json::from_str(line)
.map_err(|e| EvalError::Dataset(format!("line {line_no}: malformed JSON: {e}")))?;
match parsed {
DatasetLine::Header {
format_version,
name,
version,
} => {
if header.is_some() || !cases.is_empty() {
return Err(EvalError::Dataset(format!(
"line {line_no}: header must be the first non-blank line"
)));
}
if format_version != DATASET_FORMAT_VERSION {
return Err(EvalError::UnsupportedVersion {
found: format_version,
supported: DATASET_FORMAT_VERSION,
});
}
header = Some((name, version));
}
DatasetLine::Case(case) => {
if header.is_none() {
return Err(EvalError::Dataset(format!(
"line {line_no}: case before the header"
)));
}
cases.push(case);
}
}
}
let (name, version) =
header.ok_or_else(|| EvalError::Dataset("missing header line".to_owned()))?;
Self::new(name, version, cases)
}
pub fn load(path: impl AsRef<Path>) -> Result<Self> {
let text = std::fs::read_to_string(path.as_ref())?;
Self::from_jsonl(&text)
.map_err(|e| EvalError::Dataset(format!("{}: {e}", path.as_ref().display())))
}
pub fn to_jsonl(&self) -> String {
let mut out = String::new();
let header = DatasetLine::Header {
format_version: DATASET_FORMAT_VERSION,
name: self.name.clone(),
version: self.version.clone(),
};
out.push_str(&serde_json::to_string(&header).expect("dataset header serializes"));
out.push('\n');
for case in &self.cases {
out.push_str(
&serde_json::to_string(&DatasetLine::Case(case.clone()))
.expect("dataset case serializes"),
);
out.push('\n');
}
out
}
pub fn save(&self, path: impl AsRef<Path>) -> Result<()> {
std::fs::write(path.as_ref(), self.to_jsonl())?;
Ok(())
}
fn validate_ids(&self) -> Result<()> {
let mut seen = HashSet::with_capacity(self.cases.len());
for case in &self.cases {
if case.id.is_empty() {
return Err(EvalError::Dataset("case id must not be empty".to_owned()));
}
if !seen.insert(case.id.as_str()) {
return Err(EvalError::Dataset(format!(
"duplicate case id `{}`",
case.id
)));
}
}
Ok(())
}
}