synaptic_eval/
dataset.rs

1use synaptic_core::SynapticError;
2
3use crate::evaluator::Evaluator;
4use crate::EvalReport;
5
6/// A single item in an evaluation dataset.
7#[derive(Debug, Clone)]
8pub struct DatasetItem {
9    pub input: String,
10    pub reference: String,
11}
12
13/// A collection of input-reference pairs for evaluation.
14#[derive(Debug, Clone)]
15pub struct Dataset {
16    pub items: Vec<DatasetItem>,
17}
18
19impl Dataset {
20    /// Create a new dataset from items.
21    pub fn new(items: Vec<DatasetItem>) -> Self {
22        Self { items }
23    }
24
25    /// Create a dataset from (input, reference) string pairs.
26    pub fn from_pairs(pairs: Vec<(&str, &str)>) -> Self {
27        Self {
28            items: pairs
29                .into_iter()
30                .map(|(i, r)| DatasetItem {
31                    input: i.to_string(),
32                    reference: r.to_string(),
33                })
34                .collect(),
35        }
36    }
37}
38
39/// Evaluate predictions against a dataset using an evaluator.
40///
41/// Each prediction is evaluated against the corresponding dataset item.
42/// The number of predictions must match the number of dataset items.
43pub async fn evaluate(
44    evaluator: &dyn Evaluator,
45    dataset: &Dataset,
46    predictions: &[String],
47) -> Result<EvalReport, SynapticError> {
48    if predictions.len() != dataset.items.len() {
49        return Err(SynapticError::Validation(format!(
50            "Number of predictions ({}) does not match dataset size ({})",
51            predictions.len(),
52            dataset.items.len()
53        )));
54    }
55
56    let mut results = Vec::with_capacity(dataset.items.len());
57
58    for (prediction, item) in predictions.iter().zip(dataset.items.iter()) {
59        let result = evaluator
60            .evaluate(prediction, &item.reference, &item.input)
61            .await?;
62        results.push(result);
63    }
64
65    Ok(EvalReport::from_results(results))
66}
synaptic_eval/dataset.rs

synaptic_eval/
dataset.rs
