eval_core/report.rs
1//! The eval **report** types: one [`CaseOutcome`] per case and the aggregate [`EvalReport`] with a
2//! readable [`Display`](std::fmt::Display) summary table.
3
4use serde::{Deserialize, Serialize};
5use serde_json::Value;
6use std::time::Duration;
7
8/// The result of running one eval case.
9///
10/// `Serialize`/`Deserialize` let a host persist a run to `results/*.json` and the HTML report load it
11/// back. `Duration` round-trips through serde's built-in `{secs, nanos}` form; the `predicates` tuples
12/// serialize as `[label, passed]` pairs.
13///
14/// `#[non_exhaustive]`: likely to gain diagnostic fields. The runner builds it via [`CaseOutcome::new`]
15/// (the canonical constructor) rather than a struct literal, so a new field is a one-line change there;
16/// external crates that need to construct one should use [`CaseOutcome::new`] too.
17#[derive(Debug, Clone, Serialize, Deserialize)]
18#[non_exhaustive]
19pub struct CaseOutcome {
20 /// The case name (the host's case identifier).
21 pub name: String,
22 /// Whether EVERY expectation held (the case passed).
23 pub passed: bool,
24 /// Per-predicate `(label, passed)` in the case's `expect` order — for pinpointing WHICH predicate
25 /// failed.
26 pub predicates: Vec<(String, bool)>,
27 /// Wall-clock time for the whole case run (player-context prefetch + every model turn + tool
28 /// application).
29 pub latency: Duration,
30 /// Completion tokens summed across the case's model turns, when the backend reported `usage`;
31 /// `None` when no turn carried a usage object (e.g. a `llama-server` route that omits it).
32 pub tokens: Option<u32>,
33 /// The tool calls the model made, as `"name(compact-args)"` strings (for debugging a failure).
34 /// Excludes the harness's automatic up-front `get_player_context` prefetch.
35 pub tool_calls: Vec<String>,
36 /// The model's final free-text reply (the summary line), if any.
37 pub final_text: Option<String>,
38 /// An error from the run itself (backend failure, etc.). When `Some`, `passed` is `false` and the
39 /// predicates were scored against whatever world state existed when the run aborted.
40 pub error: Option<String>,
41 /// The full per-case conversation transcript (every `user`/`assistant`/`tool` message), with the
42 /// leading `role:"system"` message(s) STRIPPED — the system prompt is stored once at run level (see
43 /// [`RunRecord::system_prompt`]). Rendered by the HTML report's per-case expander.
44 ///
45 /// `#[serde(default)]` is REQUIRED for backward compatibility: existing `results/*.json` were
46 /// written before this field existed; without the default, the report loader
47 /// ([`crate::report_html::generate_report`]) would fail to deserialize them and silently drop those
48 /// runs from the report.
49 #[serde(default)]
50 pub transcript: Vec<Value>,
51}
52
53impl CaseOutcome {
54 /// The canonical constructor, taking every field in declaration order. Used by the runner (and any
55 /// external crate) so the `#[non_exhaustive]` struct can be built without a struct literal; adding a
56 /// field updates this one signature.
57 #[allow(clippy::too_many_arguments)]
58 pub fn new(
59 name: String,
60 passed: bool,
61 predicates: Vec<(String, bool)>,
62 latency: Duration,
63 tokens: Option<u32>,
64 tool_calls: Vec<String>,
65 final_text: Option<String>,
66 error: Option<String>,
67 transcript: Vec<Value>,
68 ) -> Self {
69 Self {
70 name,
71 passed,
72 predicates,
73 latency,
74 tokens,
75 tool_calls,
76 final_text,
77 error,
78 transcript,
79 }
80 }
81}
82
83/// The aggregate report over a set of cases, plus latency/token statistics.
84#[derive(Debug, Clone, Serialize, Deserialize)]
85pub struct EvalReport {
86 /// One outcome per case, in run order.
87 pub outcomes: Vec<CaseOutcome>,
88 /// The temperature the eval ran at (noted in the summary; default 0 for determinism).
89 pub temperature: f32,
90 /// A short description of the backend the eval ran against (from `ChatBackend::describe`).
91 pub backend: String,
92 /// The exact system prompt every case in this run used (the loop's compact-vs-full selection). Stored
93 /// ONCE here rather than on each [`CaseOutcome`] (it is identical across cases), and shown at the top
94 /// of the run's expander in the HTML report. `#[serde(default)]` keeps old result JSON loadable.
95 #[serde(default)]
96 pub system_prompt: String,
97}
98
99/// One persisted eval run: an [`EvalReport`] plus the run metadata the HTML report needs to label,
100/// sort, and group runs across models and over time. Serialized to
101/// `results/{model}_{timestamp_file}.json`.
102#[derive(Debug, Clone, Serialize, Deserialize)]
103pub struct RunRecord {
104 /// Human-friendly model label for this run (an alias like `qwen2.5-7b`, the GGUF filename for a
105 /// local file, or the remote model id). Used as the comparison/grouping key.
106 pub model: String,
107 /// Wall-clock start of the run for display, e.g. `2026-06-18 14:03:21` (local time).
108 pub timestamp_display: String,
109 /// The same instant in a filesystem-safe sortable form, e.g. `20260618-140321` — used in the
110 /// results filename and as a stable per-run sort key in the report.
111 pub timestamp_file: String,
112 /// `"local"` (a llama-server we spawned) or `"remote"` (the configured provider).
113 pub backend: String,
114 /// The case directory this run scored against (so a report can flag runs over different sets).
115 pub cases_dir: String,
116 /// The system prompt this run used, copied from [`EvalReport::system_prompt`] so the persisted
117 /// record carries it at the top level too. `#[serde(default)]` keeps old result JSON loadable.
118 #[serde(default)]
119 pub system_prompt: String,
120 /// The full eval report (per-case outcomes + aggregate stats).
121 pub report: EvalReport,
122}
123
124impl EvalReport {
125 /// Build a report from the per-case outcomes, recording the run temperature + backend label + the
126 /// shared system prompt for the summary and the HTML report's per-run expander.
127 pub fn new(
128 outcomes: Vec<CaseOutcome>,
129 temperature: f32,
130 backend: String,
131 system_prompt: String,
132 ) -> Self {
133 Self {
134 outcomes,
135 temperature,
136 backend,
137 system_prompt,
138 }
139 }
140
141 /// Number of cases run.
142 pub fn total(&self) -> usize {
143 self.outcomes.len()
144 }
145
146 /// Number of cases that fully passed.
147 pub fn passed(&self) -> usize {
148 self.outcomes.iter().filter(|o| o.passed).count()
149 }
150
151 /// Accuracy = fraction of cases that fully passed, in `0.0..=1.0`. `0.0` for an empty report.
152 pub fn accuracy(&self) -> f64 {
153 if self.outcomes.is_empty() {
154 0.0
155 } else {
156 self.passed() as f64 / self.total() as f64
157 }
158 }
159
160 /// Mean per-case latency. `Duration::ZERO` for an empty report.
161 pub fn mean_latency(&self) -> Duration {
162 if self.outcomes.is_empty() {
163 return Duration::ZERO;
164 }
165 let total: Duration = self.outcomes.iter().map(|o| o.latency).sum();
166 total / self.outcomes.len() as u32
167 }
168
169 /// The `p`-quantile of per-case latency (`p` in `0.0..=1.0`), via the nearest-rank method on the
170 /// sorted latencies. `Duration::ZERO` for an empty report.
171 pub fn latency_percentile(&self, p: f64) -> Duration {
172 if self.outcomes.is_empty() {
173 return Duration::ZERO;
174 }
175 let mut lats: Vec<Duration> = self.outcomes.iter().map(|o| o.latency).collect();
176 lats.sort_unstable();
177 // Nearest-rank: rank = ceil(p * n), clamped to 1..=n, then 0-indexed.
178 let n = lats.len();
179 let rank = (p * n as f64).ceil().max(1.0) as usize;
180 lats[rank.min(n) - 1]
181 }
182
183 /// Median (p50) per-case latency.
184 pub fn p50_latency(&self) -> Duration {
185 self.latency_percentile(0.50)
186 }
187
188 /// p95 per-case latency.
189 pub fn p95_latency(&self) -> Duration {
190 self.latency_percentile(0.95)
191 }
192
193 /// Total completion tokens across cases that reported a token count. `None` when NO case reported
194 /// any (so the summary can say "tokens unavailable" rather than print a misleading 0).
195 pub fn total_tokens(&self) -> Option<u32> {
196 let reported: Vec<u32> = self.outcomes.iter().filter_map(|o| o.tokens).collect();
197 if reported.is_empty() {
198 None
199 } else {
200 Some(reported.iter().sum())
201 }
202 }
203
204 /// Mean completion tokens over the cases that reported a count; `None` when none reported.
205 pub fn mean_tokens(&self) -> Option<f64> {
206 let reported: Vec<u32> = self.outcomes.iter().filter_map(|o| o.tokens).collect();
207 if reported.is_empty() {
208 None
209 } else {
210 Some(reported.iter().map(|&t| t as f64).sum::<f64>() / reported.len() as f64)
211 }
212 }
213}
214
215/// Milliseconds with one decimal, for the table (durations are short).
216fn ms(d: Duration) -> String {
217 format!("{:.1}ms", d.as_secs_f64() * 1000.0)
218}
219
220impl std::fmt::Display for EvalReport {
221 fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
222 writeln!(f, "AI eval report")?;
223 writeln!(f, " backend: {}", self.backend)?;
224 writeln!(f, " temperature: {} (0 = deterministic)", self.temperature)?;
225 writeln!(
226 f,
227 " accuracy: {}/{} cases passed ({:.0}%)",
228 self.passed(),
229 self.total(),
230 self.accuracy() * 100.0
231 )?;
232 writeln!(
233 f,
234 " latency: mean {} p50 {} p95 {}",
235 ms(self.mean_latency()),
236 ms(self.p50_latency()),
237 ms(self.p95_latency())
238 )?;
239 match (self.total_tokens(), self.mean_tokens()) {
240 (Some(total), Some(mean)) => writeln!(
241 f,
242 " tokens: {total} completion total, {mean:.0} mean/case (cases reporting usage)"
243 )?,
244 _ => writeln!(
245 f,
246 " tokens: unavailable (no turn reported a `usage` object)"
247 )?,
248 }
249
250 writeln!(f, " cases:")?;
251 // Column widths: name padded to the longest name for an aligned table.
252 let name_w = self
253 .outcomes
254 .iter()
255 .map(|o| o.name.len())
256 .max()
257 .unwrap_or(4)
258 .max(4);
259 for o in &self.outcomes {
260 let mark = if o.passed { "PASS" } else { "FAIL" };
261 let passed_preds = o.predicates.iter().filter(|(_, p)| *p).count();
262 write!(
263 f,
264 " [{mark}] {:<name_w$} {} preds {}/{} calls {}",
265 o.name,
266 ms(o.latency),
267 passed_preds,
268 o.predicates.len(),
269 o.tool_calls.len(),
270 )?;
271 if let Some(t) = o.tokens {
272 write!(f, " tok {t}")?;
273 }
274 writeln!(f)?;
275 // On a failure, show which predicates failed + any run error, for fast diagnosis.
276 if !o.passed {
277 for (label, passed) in &o.predicates {
278 if !passed {
279 writeln!(f, " - FAILED: {label}")?;
280 }
281 }
282 if let Some(err) = &o.error {
283 writeln!(f, " - run error: {err}")?;
284 }
285 }
286 }
287 Ok(())
288 }
289}