runifold_testkit/
evaluation_store.rs

1use std::{
2    fs::{self, OpenOptions},
3    io::{self, Write},
4    path::{Path, PathBuf},
5};
6
7use runifold_core::RunId;
8use thiserror::Error;
9
10use crate::{EvaluationDataset, EvaluationError, EvaluationReport};
11
12/// Evaluation artifact repository failure.
13#[derive(Debug, Error)]
14#[non_exhaustive]
15pub enum EvaluationStoreError {
16    /// Filesystem operation failed.
17    #[error("evaluation store {operation} failed for {path}")]
18    Io {
19        /// Stable operation name.
20        operation: &'static str,
21        /// Affected path.
22        path: PathBuf,
23        /// Original filesystem error.
24        #[source]
25        source: io::Error,
26    },
27    /// JSON encoding or decoding failed.
28    #[error("evaluation artifact JSON is invalid")]
29    Json(#[from] serde_json::Error),
30    /// Loaded domain data violates evaluation invariants.
31    #[error(transparent)]
32    InvalidArtifact(#[from] EvaluationError),
33    /// An immutable artifact key already contains different content.
34    #[error("evaluation artifact conflict at {path}")]
35    Conflict {
36        /// Conflicting artifact path.
37        path: PathBuf,
38    },
39}
40
41/// Versioned dataset and candidate-report repository.
42pub trait EvaluationRepository: Send + Sync {
43    /// Saves one immutable dataset version.
44    ///
45    /// # Errors
46    ///
47    /// Returns a typed storage, serialization, validation, or conflict error.
48    fn save_dataset(&self, dataset: &EvaluationDataset) -> Result<(), EvaluationStoreError>;
49
50    /// Loads one dataset version.
51    ///
52    /// # Errors
53    ///
54    /// Returns a typed storage, deserialization, or validation error.
55    fn load_dataset(
56        &self,
57        name: &str,
58        version: &str,
59    ) -> Result<EvaluationDataset, EvaluationStoreError>;
60
61    /// Saves one immutable candidate report.
62    ///
63    /// # Errors
64    ///
65    /// Returns a typed storage, serialization, validation, or conflict error.
66    fn save_report(&self, report: &EvaluationReport) -> Result<(), EvaluationStoreError>;
67
68    /// Loads one candidate report.
69    ///
70    /// # Errors
71    ///
72    /// Returns a typed storage, deserialization, or validation error.
73    fn load_report(
74        &self,
75        dataset_name: &str,
76        dataset_version: &str,
77        candidate_version: &str,
78    ) -> Result<EvaluationReport, EvaluationStoreError>;
79}
80
81/// Filesystem-backed immutable JSON evaluation repository.
82///
83/// Identity components are hex encoded before path construction. Writes use a
84/// same-directory temporary file and atomic no-replace link. Re-saving identical bytes
85/// is idempotent; replacing an existing version with different bytes fails.
86#[derive(Clone, Debug)]
87pub struct FileEvaluationRepository {
88    root: PathBuf,
89}
90
91impl FileEvaluationRepository {
92    /// Creates a repository handle without touching the filesystem.
93    pub fn new(root: impl AsRef<Path>) -> Self {
94        Self {
95            root: root.as_ref().to_path_buf(),
96        }
97    }
98
99    fn dataset_path(&self, name: &str, version: &str) -> PathBuf {
100        self.root
101            .join("datasets")
102            .join(encode_component(name))
103            .join(format!("{}.json", encode_component(version)))
104    }
105
106    fn report_path(
107        &self,
108        dataset_name: &str,
109        dataset_version: &str,
110        candidate_version: &str,
111    ) -> PathBuf {
112        self.root
113            .join("reports")
114            .join(encode_component(dataset_name))
115            .join(encode_component(dataset_version))
116            .join(format!("{}.json", encode_component(candidate_version)))
117    }
118}
119
120impl EvaluationRepository for FileEvaluationRepository {
121    fn save_dataset(&self, dataset: &EvaluationDataset) -> Result<(), EvaluationStoreError> {
122        dataset.validate()?;
123        let bytes = serde_json::to_vec_pretty(dataset)?;
124        write_immutable(
125            &self.dataset_path(dataset.name(), dataset.version()),
126            &bytes,
127        )
128    }
129
130    fn load_dataset(
131        &self,
132        name: &str,
133        version: &str,
134    ) -> Result<EvaluationDataset, EvaluationStoreError> {
135        let path = self.dataset_path(name, version);
136        let bytes = read(&path)?;
137        let dataset = serde_json::from_slice::<EvaluationDataset>(&bytes)?;
138        dataset.validate()?;
139        Ok(dataset)
140    }
141
142    fn save_report(&self, report: &EvaluationReport) -> Result<(), EvaluationStoreError> {
143        report.validate()?;
144        let bytes = serde_json::to_vec_pretty(report)?;
145        write_immutable(
146            &self.report_path(
147                &report.dataset_name,
148                &report.dataset_version,
149                &report.candidate_version,
150            ),
151            &bytes,
152        )
153    }
154
155    fn load_report(
156        &self,
157        dataset_name: &str,
158        dataset_version: &str,
159        candidate_version: &str,
160    ) -> Result<EvaluationReport, EvaluationStoreError> {
161        let path = self.report_path(dataset_name, dataset_version, candidate_version);
162        let bytes = read(&path)?;
163        let report = serde_json::from_slice::<EvaluationReport>(&bytes)?;
164        report.validate()?;
165        Ok(report)
166    }
167}
168
169fn read(path: &Path) -> Result<Vec<u8>, EvaluationStoreError> {
170    fs::read(path).map_err(|source| EvaluationStoreError::Io {
171        operation: "read",
172        path: path.to_path_buf(),
173        source,
174    })
175}
176
177fn write_immutable(path: &Path, bytes: &[u8]) -> Result<(), EvaluationStoreError> {
178    if let Some(existing) = read_if_present(path)? {
179        return if existing == bytes {
180            Ok(())
181        } else {
182            Err(EvaluationStoreError::Conflict {
183                path: path.to_path_buf(),
184            })
185        };
186    }
187    let parent = path.parent().ok_or_else(|| EvaluationStoreError::Io {
188        operation: "resolve parent",
189        path: path.to_path_buf(),
190        source: io::Error::new(io::ErrorKind::InvalidInput, "artifact path has no parent"),
191    })?;
192    fs::create_dir_all(parent).map_err(|source| EvaluationStoreError::Io {
193        operation: "create directory",
194        path: parent.to_path_buf(),
195        source,
196    })?;
197    let temporary = parent.join(format!(".{}.tmp", RunId::new()));
198    let mut file = OpenOptions::new()
199        .write(true)
200        .create_new(true)
201        .open(&temporary)
202        .map_err(|source| EvaluationStoreError::Io {
203            operation: "create temporary artifact",
204            path: temporary.clone(),
205            source,
206        })?;
207    if let Err(source) = file.write_all(bytes).and_then(|()| file.sync_all()) {
208        drop(file);
209        let _ = fs::remove_file(&temporary);
210        return Err(EvaluationStoreError::Io {
211            operation: "persist temporary artifact",
212            path: temporary,
213            source,
214        });
215    }
216    match fs::hard_link(&temporary, path) {
217        Ok(()) => fs::remove_file(&temporary).map_err(|source| EvaluationStoreError::Io {
218            operation: "remove temporary artifact",
219            path: temporary,
220            source,
221        }),
222        Err(_source) if path.exists() => {
223            let _ = fs::remove_file(&temporary);
224            if read(path)? == bytes {
225                Ok(())
226            } else {
227                Err(EvaluationStoreError::Conflict {
228                    path: path.to_path_buf(),
229                })
230            }
231        }
232        Err(source) => {
233            let _ = fs::remove_file(&temporary);
234            Err(EvaluationStoreError::Io {
235                operation: "commit artifact",
236                path: path.to_path_buf(),
237                source,
238            })
239        }
240    }
241}
242
243fn read_if_present(path: &Path) -> Result<Option<Vec<u8>>, EvaluationStoreError> {
244    match fs::read(path) {
245        Ok(bytes) => Ok(Some(bytes)),
246        Err(error) if error.kind() == io::ErrorKind::NotFound => Ok(None),
247        Err(source) => Err(EvaluationStoreError::Io {
248            operation: "read existing artifact",
249            path: path.to_path_buf(),
250            source,
251        }),
252    }
253}
254
255fn encode_component(value: &str) -> String {
256    const HEX: &[u8; 16] = b"0123456789abcdef";
257    let mut encoded = String::with_capacity(value.len() * 2);
258    for byte in value.as_bytes() {
259        encoded.push(char::from(HEX[usize::from(byte >> 4)]));
260        encoded.push(char::from(HEX[usize::from(byte & 0x0f)]));
261    }
262    encoded
263}
264
265#[cfg(test)]
266mod tests {
267    use std::fs;
268
269    use crate::{
270        EvaluationCase, EvaluationDataset, EvaluationRepository, EvaluationStoreError,
271        FileEvaluationRepository,
272    };
273
274    #[test]
275    fn file_repository_round_trips_and_rejects_version_replacement() {
276        let root =
277            std::env::temp_dir().join(format!("runifold-eval-{}", runifold_core::RunId::new()));
278        let repository = FileEvaluationRepository::new(&root);
279        let dataset = EvaluationDataset::new(
280            "../unsafe name",
281            "v1",
282            vec![EvaluationCase::new("one", serde_json::json!("private")).unwrap()],
283        )
284        .unwrap();
285
286        repository.save_dataset(&dataset).unwrap();
287        repository.save_dataset(&dataset).unwrap();
288        let loaded = repository.load_dataset("../unsafe name", "v1").unwrap();
289        assert_eq!(loaded.name(), "../unsafe name");
290        assert_eq!(loaded.cases()[0].input(), &serde_json::json!("private"));
291
292        let changed = EvaluationDataset::new(
293            "../unsafe name",
294            "v1",
295            vec![EvaluationCase::new("two", serde_json::json!("changed")).unwrap()],
296        )
297        .unwrap();
298        assert!(matches!(
299            repository.save_dataset(&changed),
300            Err(EvaluationStoreError::Conflict { .. })
301        ));
302        fs::remove_dir_all(root).unwrap();
303    }
304}
runifold_testkit/evaluation_store.rs

runifold_testkit/
evaluation_store.rs
