eval_core/persist.rs
1//! Automatic persistence of an eval run: write it as a JSON [`RunRecord`] into a results directory
2//! and (re)generate the self-contained `report.html` over every run saved there.
3//!
4//! This is the engine behind [`RunMeta::persist_to`](crate::RunMeta::persist_to): when a run carries a
5//! [`Persist`] target, [`run_eval_with_meta`](crate::run_eval_with_meta) assembles a [`RunRecord`] via
6//! [`build_record`] and writes it + the report via [`write_record_and_report`] once the [`EvalReport`]
7//! is ready (the one-shot [`save_and_report`] does both), so a host gets the saved JSON + the HTML
8//! report for free — no manual wiring. The same record is reused for the optional EvalForge upload, so
9//! the saved file and the uploaded record share one timestamp / dedup key. The filename pattern
10//! (`{slug(model)}_{timestamp_file}.json`), the pretty-printed JSON, and the "warn, don't fail" posture
11//! all match the convention hosts used before this was built in, so existing `results/*.json` and
12//! `report.html` stay shape-compatible.
13
14use std::path::{Path, PathBuf};
15
16use crate::report::{EvalReport, RunRecord};
17use crate::report_html::generate_report;
18
19/// Where and how to persist a run, attached to a [`RunMeta`](crate::RunMeta) via
20/// [`RunMeta::persist_to`](crate::RunMeta::persist_to). When present on the meta passed to a run, the
21/// runner writes `{slug(model)}_{timestamp_file}.json` into `results_dir` and regenerates
22/// `results_dir/report.html`.
23///
24/// `#[non_exhaustive]`: build it through the `RunMeta` builder (`persist_to` + optional `backend_kind` /
25/// `cases_dir`), never a struct literal, so new fields stay non-breaking.
26#[derive(Debug, Clone)]
27#[non_exhaustive]
28pub struct Persist {
29 /// Directory the per-run JSON and `report.html` are written to (created if missing).
30 pub results_dir: PathBuf,
31 /// The model / grouping key recorded on the run ([`RunRecord::model`](crate::report::RunRecord)),
32 /// also used (slugged) in the JSON filename.
33 pub model: String,
34 /// The backend KIND shown in the report's Backend column
35 /// ([`RunRecord::backend`](crate::report::RunRecord)), e.g. `"local"` / `"remote"`. When empty it
36 /// falls back to the run's descriptive backend label ([`EvalReport::backend`]).
37 pub backend: String,
38 /// The case directory recorded on the run ([`RunRecord::cases_dir`](crate::report::RunRecord)),
39 /// shown in the report. May be empty.
40 pub cases_dir: String,
41}
42
43impl Persist {
44 /// A persist target writing into `results_dir` and recording `model` as the grouping key. The
45 /// backend kind and cases dir default to empty (set them via the `RunMeta` builder).
46 pub(crate) fn new(results_dir: PathBuf, model: String) -> Self {
47 Self {
48 results_dir,
49 model,
50 backend: String::new(),
51 cases_dir: String::new(),
52 }
53 }
54}
55
56/// Build a [`RunRecord`] from `report` + the [`Persist`] target, write it as
57/// `{slug(model)}_{timestamp_file}.json` under `persist.results_dir`, then (re)generate `report.html`
58/// over every run in that directory. Returns the written JSON path.
59///
60/// The record's `backend` is `persist.backend` when set, else the report's descriptive
61/// [`backend`](EvalReport::backend) label; its `system_prompt` is copied from the report so a consumer
62/// reading just the record sees it without descending into the nested report.
63pub fn save_and_report(persist: &Persist, report: &EvalReport) -> anyhow::Result<PathBuf> {
64 let record = build_record(
65 persist.model.clone(),
66 persist.backend.clone(),
67 persist.cases_dir.clone(),
68 report,
69 );
70 write_record_and_report(&persist.results_dir, &record)
71}
72
73/// Assemble a [`RunRecord`] from a finished `report` plus the run identity, stamping it with a single
74/// `now` timestamp (so a record built once can be written to disk AND uploaded sharing the same
75/// timestamp / dedup key).
76///
77/// `backend_kind` is the report's Backend column: when empty it falls back to the report's descriptive
78/// [`backend`](EvalReport::backend) label; otherwise it overrides it. `system_prompt` is copied from the
79/// report so a consumer reading just the record sees it without descending into the nested report.
80/// Public so a host can build a record without driving a full persist.
81pub fn build_record(
82 model: String,
83 backend_kind: String,
84 cases_dir: String,
85 report: &EvalReport,
86) -> RunRecord {
87 let (timestamp_display, timestamp_file) = timestamps();
88 let backend = if backend_kind.is_empty() {
89 report.backend.clone()
90 } else {
91 backend_kind
92 };
93 RunRecord {
94 model,
95 timestamp_display,
96 timestamp_file,
97 backend,
98 cases_dir,
99 system_prompt: report.system_prompt.clone(),
100 report: report.clone(),
101 }
102}
103
104/// Write `record` into `results_dir` (via [`save_record`]) and (re)generate `report.html` over every run
105/// in that directory, returning the written JSON path. Public so a host can persist a pre-built
106/// [`RunRecord`] (e.g. one from [`build_record`]) without re-assembling it.
107pub fn write_record_and_report(results_dir: &Path, record: &RunRecord) -> anyhow::Result<PathBuf> {
108 let path = save_record(results_dir, record)?;
109 generate_report(results_dir)?;
110 Ok(path)
111}
112
113/// Write `record` as pretty JSON to `{results_dir}/{slug(model)}_{timestamp_file}.json`, creating the
114/// directory if needed, and return the written path. Public so a host can persist a hand-built
115/// [`RunRecord`] without driving a full run through the runner.
116pub fn save_record(results_dir: &Path, record: &RunRecord) -> anyhow::Result<PathBuf> {
117 std::fs::create_dir_all(results_dir)
118 .map_err(|e| anyhow::anyhow!("creating results dir {}: {e}", results_dir.display()))?;
119 let file = results_dir.join(format!(
120 "{}_{}.json",
121 slug(&record.model),
122 record.timestamp_file
123 ));
124 let json = serde_json::to_string_pretty(record)
125 .map_err(|e| anyhow::anyhow!("serializing run record: {e}"))?;
126 std::fs::write(&file, json).map_err(|e| anyhow::anyhow!("writing {}: {e}", file.display()))?;
127 Ok(file)
128}
129
130/// `(display, file)` timestamps for *now* in LOCAL time, e.g.
131/// `("2026-06-18 14:03:21", "20260618-140321")` — the display form labels the run in the report, the
132/// file form is the filesystem-safe sortable filename/sort key.
133fn timestamps() -> (String, String) {
134 let now = chrono::Local::now();
135 (
136 now.format("%Y-%m-%d %H:%M:%S").to_string(),
137 now.format("%Y%m%d-%H%M%S").to_string(),
138 )
139}
140
141/// Sanitize a model label into a filesystem-safe slug for the results filename: keep `[A-Za-z0-9._-]`,
142/// replace every other char with `-`, collapse runs of `-`, trim leading/trailing `-`, and fall back to
143/// `"model"` if nothing is left.
144pub fn slug(model: &str) -> String {
145 let mut out = String::with_capacity(model.len());
146 let mut last_dash = false;
147 for c in model.chars() {
148 if c.is_ascii_alphanumeric() || c == '.' || c == '_' || c == '-' {
149 out.push(c);
150 last_dash = false;
151 } else if !last_dash {
152 out.push('-');
153 last_dash = true;
154 }
155 }
156 let trimmed = out.trim_matches('-').to_owned();
157 if trimmed.is_empty() {
158 "model".to_owned()
159 } else {
160 trimmed
161 }
162}
163
164#[cfg(test)]
165mod tests {
166 use super::*;
167
168 #[test]
169 fn slug_is_filesystem_safe_and_collapses() {
170 assert_eq!(slug("Qwen/Qwen3:7b.gguf"), "Qwen-Qwen3-7b.gguf");
171 assert_eq!(slug("a b"), "a-b");
172 assert_eq!(slug("***"), "model");
173 assert_eq!(slug("ok_name-1.2"), "ok_name-1.2");
174 }
175
176 #[test]
177 fn save_record_writes_pretty_json_with_slugged_name() {
178 let dir = tempfile::tempdir().expect("tempdir");
179 let report = EvalReport::new(Vec::new(), 0.0, "local: m".into(), "sys".into());
180 let record = RunRecord {
181 model: "my/model".into(),
182 timestamp_display: "2026-06-18 14:03:21".into(),
183 timestamp_file: "20260618-140321".into(),
184 backend: "local".into(),
185 cases_dir: "cases".into(),
186 system_prompt: "sys".into(),
187 report,
188 };
189 let path = save_record(dir.path(), &record).expect("save");
190 assert_eq!(
191 path.file_name().unwrap().to_string_lossy(),
192 "my-model_20260618-140321.json"
193 );
194 let text = std::fs::read_to_string(&path).expect("read");
195 assert!(
196 text.contains("\n \"model\": \"my/model\""),
197 "pretty-printed"
198 );
199 }
200}