eval_core/persist.rs
1//! Automatic persistence of an eval run: write it as a JSON [`RunRecord`] into a results directory
2//! and (re)generate the self-contained `report.html` over every run saved there.
3//!
4//! This is the engine behind [`RunMeta::persist_to`](crate::RunMeta::persist_to): when a run carries a
5//! [`Persist`] target, [`run_eval_with_meta`](crate::run_eval_with_meta) calls [`save_and_report`] once
6//! the [`EvalReport`] is assembled, so a host gets the saved JSON + the HTML report for free — no manual
7//! wiring. The filename pattern (`{slug(model)}_{timestamp_file}.json`), the pretty-printed JSON, and
8//! the "warn, don't fail" posture all match the convention hosts used before this was built in, so
9//! existing `results/*.json` and `report.html` stay shape-compatible.
10
11use std::path::{Path, PathBuf};
12
13use crate::report::{EvalReport, RunRecord};
14use crate::report_html::generate_report;
15
16/// Where and how to persist a run, attached to a [`RunMeta`](crate::RunMeta) via
17/// [`RunMeta::persist_to`](crate::RunMeta::persist_to). When present on the meta passed to a run, the
18/// runner writes `{slug(model)}_{timestamp_file}.json` into `results_dir` and regenerates
19/// `results_dir/report.html`.
20///
21/// `#[non_exhaustive]`: build it through the `RunMeta` builder (`persist_to` + optional `backend_kind` /
22/// `cases_dir`), never a struct literal, so new fields stay non-breaking.
23#[derive(Debug, Clone)]
24#[non_exhaustive]
25pub struct Persist {
26 /// Directory the per-run JSON and `report.html` are written to (created if missing).
27 pub results_dir: PathBuf,
28 /// The model / grouping key recorded on the run ([`RunRecord::model`](crate::report::RunRecord)),
29 /// also used (slugged) in the JSON filename.
30 pub model: String,
31 /// The backend KIND shown in the report's Backend column
32 /// ([`RunRecord::backend`](crate::report::RunRecord)), e.g. `"local"` / `"remote"`. When empty it
33 /// falls back to the run's descriptive backend label ([`EvalReport::backend`]).
34 pub backend: String,
35 /// The case directory recorded on the run ([`RunRecord::cases_dir`](crate::report::RunRecord)),
36 /// shown in the report. May be empty.
37 pub cases_dir: String,
38}
39
40impl Persist {
41 /// A persist target writing into `results_dir` and recording `model` as the grouping key. The
42 /// backend kind and cases dir default to empty (set them via the `RunMeta` builder).
43 pub(crate) fn new(results_dir: PathBuf, model: String) -> Self {
44 Self {
45 results_dir,
46 model,
47 backend: String::new(),
48 cases_dir: String::new(),
49 }
50 }
51}
52
53/// Build a [`RunRecord`] from `report` + the [`Persist`] target, write it as
54/// `{slug(model)}_{timestamp_file}.json` under `persist.results_dir`, then (re)generate `report.html`
55/// over every run in that directory. Returns the written JSON path.
56///
57/// The record's `backend` is `persist.backend` when set, else the report's descriptive
58/// [`backend`](EvalReport::backend) label; its `system_prompt` is copied from the report so a consumer
59/// reading just the record sees it without descending into the nested report.
60pub fn save_and_report(persist: &Persist, report: &EvalReport) -> anyhow::Result<PathBuf> {
61 let (timestamp_display, timestamp_file) = timestamps();
62 let backend = if persist.backend.is_empty() {
63 report.backend.clone()
64 } else {
65 persist.backend.clone()
66 };
67 let record = RunRecord {
68 model: persist.model.clone(),
69 timestamp_display,
70 timestamp_file,
71 backend,
72 cases_dir: persist.cases_dir.clone(),
73 system_prompt: report.system_prompt.clone(),
74 report: report.clone(),
75 };
76 let path = save_record(&persist.results_dir, &record)?;
77 generate_report(&persist.results_dir)?;
78 Ok(path)
79}
80
81/// Write `record` as pretty JSON to `{results_dir}/{slug(model)}_{timestamp_file}.json`, creating the
82/// directory if needed, and return the written path. Public so a host can persist a hand-built
83/// [`RunRecord`] without driving a full run through the runner.
84pub fn save_record(results_dir: &Path, record: &RunRecord) -> anyhow::Result<PathBuf> {
85 std::fs::create_dir_all(results_dir)
86 .map_err(|e| anyhow::anyhow!("creating results dir {}: {e}", results_dir.display()))?;
87 let file = results_dir.join(format!(
88 "{}_{}.json",
89 slug(&record.model),
90 record.timestamp_file
91 ));
92 let json = serde_json::to_string_pretty(record)
93 .map_err(|e| anyhow::anyhow!("serializing run record: {e}"))?;
94 std::fs::write(&file, json).map_err(|e| anyhow::anyhow!("writing {}: {e}", file.display()))?;
95 Ok(file)
96}
97
98/// `(display, file)` timestamps for *now* in LOCAL time, e.g.
99/// `("2026-06-18 14:03:21", "20260618-140321")` — the display form labels the run in the report, the
100/// file form is the filesystem-safe sortable filename/sort key.
101fn timestamps() -> (String, String) {
102 let now = chrono::Local::now();
103 (
104 now.format("%Y-%m-%d %H:%M:%S").to_string(),
105 now.format("%Y%m%d-%H%M%S").to_string(),
106 )
107}
108
109/// Sanitize a model label into a filesystem-safe slug for the results filename: keep `[A-Za-z0-9._-]`,
110/// replace every other char with `-`, collapse runs of `-`, trim leading/trailing `-`, and fall back to
111/// `"model"` if nothing is left.
112pub fn slug(model: &str) -> String {
113 let mut out = String::with_capacity(model.len());
114 let mut last_dash = false;
115 for c in model.chars() {
116 if c.is_ascii_alphanumeric() || c == '.' || c == '_' || c == '-' {
117 out.push(c);
118 last_dash = false;
119 } else if !last_dash {
120 out.push('-');
121 last_dash = true;
122 }
123 }
124 let trimmed = out.trim_matches('-').to_owned();
125 if trimmed.is_empty() {
126 "model".to_owned()
127 } else {
128 trimmed
129 }
130}
131
132#[cfg(test)]
133mod tests {
134 use super::*;
135
136 #[test]
137 fn slug_is_filesystem_safe_and_collapses() {
138 assert_eq!(slug("Qwen/Qwen3:7b.gguf"), "Qwen-Qwen3-7b.gguf");
139 assert_eq!(slug("a b"), "a-b");
140 assert_eq!(slug("***"), "model");
141 assert_eq!(slug("ok_name-1.2"), "ok_name-1.2");
142 }
143
144 #[test]
145 fn save_record_writes_pretty_json_with_slugged_name() {
146 let dir = tempfile::tempdir().expect("tempdir");
147 let report = EvalReport::new(Vec::new(), 0.0, "local: m".into(), "sys".into());
148 let record = RunRecord {
149 model: "my/model".into(),
150 timestamp_display: "2026-06-18 14:03:21".into(),
151 timestamp_file: "20260618-140321".into(),
152 backend: "local".into(),
153 cases_dir: "cases".into(),
154 system_prompt: "sys".into(),
155 report,
156 };
157 let path = save_record(dir.path(), &record).expect("save");
158 assert_eq!(
159 path.file_name().unwrap().to_string_lossy(),
160 "my-model_20260618-140321.json"
161 );
162 let text = std::fs::read_to_string(&path).expect("read");
163 assert!(
164 text.contains("\n \"model\": \"my/model\""),
165 "pretty-printed"
166 );
167 }
168}