recallbench 0.1.0

A universal benchmark harness for AI memory systems
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126

use anyhow::{Context, Result};

use crate::traits::BenchmarkDataset;
use crate::types::BenchmarkQuestion;

/// A user-defined dataset loaded from a JSON file.
pub struct CustomDataset {
    dataset_name: String,
    questions: Vec<BenchmarkQuestion>,
}

impl CustomDataset {
    /// Load a custom dataset from a JSON file.
    pub fn load(path: &std::path::Path) -> Result<Self> {
        let content = std::fs::read_to_string(path)
            .context("Failed to read custom dataset file")?;
        Self::from_json(
            path.file_stem()
                .and_then(|s| s.to_str())
                .unwrap_or("custom"),
            &content,
        )
    }

    /// Parse from a JSON string.
    pub fn from_json(name: &str, json: &str) -> Result<Self> {
        let questions: Vec<BenchmarkQuestion> = serde_json::from_str(json)
            .context("Failed to parse custom dataset JSON")?;
        Ok(Self {
            dataset_name: name.to_string(),
            questions,
        })
    }

    /// Validate the dataset structure. Returns a list of errors (empty = valid).
    pub fn validate(&self) -> Vec<String> {
        let mut errors = Vec::new();

        if self.questions.is_empty() {
            errors.push("Dataset has no questions".to_string());
        }

        for (i, q) in self.questions.iter().enumerate() {
            if q.id.is_empty() {
                errors.push(format!("Question {i}: missing id"));
            }
            if q.question_type.is_empty() {
                errors.push(format!("Question {} (index {i}): missing question_type", q.id));
            }
            if q.question.is_empty() {
                errors.push(format!("Question {}: missing question text", q.id));
            }
            if q.ground_truth.is_empty() {
                errors.push(format!("Question {}: missing ground_truth", q.id));
            }
        }

        errors
    }
}

impl BenchmarkDataset for CustomDataset {
    fn name(&self) -> &str {
        &self.dataset_name
    }

    fn variant(&self) -> &str {
        "default"
    }

    fn description(&self) -> &str {
        "User-defined custom dataset"
    }

    fn questions(&self) -> &[BenchmarkQuestion] {
        &self.questions
    }

    fn question_types(&self) -> Vec<String> {
        let mut types: Vec<String> = self.questions.iter()
            .map(|q| q.question_type.clone())
            .collect::<std::collections::HashSet<_>>()
            .into_iter()
            .collect();
        types.sort();
        types
    }
}

#[cfg(test)]
mod tests {
    use super::*;

    const VALID_JSON: &str = r#"[
        {
            "id": "q001",
            "question_type": "factual",
            "question": "What color is the sky?",
            "ground_truth": ["blue"],
            "sessions": []
        }
    ]"#;

    #[test]
    fn parse_valid_custom() {
        let ds = CustomDataset::from_json("test", VALID_JSON).unwrap();
        assert_eq!(ds.questions().len(), 1);
        assert_eq!(ds.name(), "test");
        assert!(ds.validate().is_empty());
    }

    #[test]
    fn validate_missing_fields() {
        let json = r#"[{"id":"","question_type":"","question":"","ground_truth":[],"sessions":[]}]"#;
        let ds = CustomDataset::from_json("test", json).unwrap();
        let errors = ds.validate();
        assert!(!errors.is_empty());
    }

    #[test]
    fn validate_empty_dataset() {
        let ds = CustomDataset::from_json("test", "[]").unwrap();
        let errors = ds.validate();
        assert!(errors.iter().any(|e| e.contains("no questions")));
    }
}