1use std::collections::BTreeMap;
19use std::sync::Arc;
20use std::time::Instant;
21
22use futures::{stream::FuturesUnordered, StreamExt};
23use serde::{Deserialize, Serialize};
24
25use rusty_agent_runtime::error::Result as RuntimeResult;
26use rusty_agent_runtime::executor::{ExecutionOutcome, Executor, RunConfig};
27use rusty_agent_runtime::graph::Graph;
28use rusty_agent_runtime::journal::{Clock, Journal};
29use rusty_agent_runtime::state::{State, StateSpec};
30use serde_json::Value;
31
32use crate::assertion::AssertionResult;
33use crate::dataset::{Dataset, EvalCase};
34use crate::error::{EvalError, Result};
35use crate::evidence::{RunEvidence, RunStatus};
36use crate::judge::{JudgeModel, JudgeRequest, JudgeVerdict};
37
38pub const REPORT_FORMAT_VERSION: u64 = 1;
40
41pub struct PreparedRun {
49 pub graph: Graph,
51
52 pub spec: StateSpec,
54
55 pub initial_state: Option<State>,
58}
59
60impl PreparedRun {
61 pub fn new(graph: Graph, spec: StateSpec) -> Self {
63 Self {
64 graph,
65 spec,
66 initial_state: None,
67 }
68 }
69
70 pub fn with_initial_state(mut self, state: State) -> Self {
72 self.initial_state = Some(state);
73 self
74 }
75}
76
77pub struct ExperimentConfig {
79 runs_per_case: usize,
80 max_concurrency: usize,
81 judge: Option<Arc<dyn JudgeModel>>,
82}
83
84impl Default for ExperimentConfig {
85 fn default() -> Self {
86 Self::new()
87 }
88}
89
90impl ExperimentConfig {
91 pub fn new() -> Self {
93 Self {
94 runs_per_case: 1,
95 max_concurrency: 1,
96 judge: None,
97 }
98 }
99
100 pub fn with_runs_per_case(mut self, n: usize) -> Self {
103 self.runs_per_case = n.max(1);
104 self
105 }
106
107 pub fn with_max_concurrency(mut self, n: usize) -> Self {
115 self.max_concurrency = n.max(1);
116 self
117 }
118
119 pub fn with_judge(mut self, judge: Arc<dyn JudgeModel>) -> Self {
122 self.judge = Some(judge);
123 self
124 }
125}
126
127impl std::fmt::Debug for ExperimentConfig {
128 fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
129 f.debug_struct("ExperimentConfig")
130 .field("runs_per_case", &self.runs_per_case)
131 .field("max_concurrency", &self.max_concurrency)
132 .field("judge", &self.judge.as_ref().map(|_| "<judge>"))
133 .finish()
134 }
135}
136
137pub struct ExperimentRunner {
139 config: ExperimentConfig,
140}
141
142impl ExperimentRunner {
143 pub fn new(config: ExperimentConfig) -> Self {
145 Self { config }
146 }
147
148 pub async fn run<F>(&self, dataset: &Dataset, prepare: F) -> Result<ExperimentReport>
159 where
160 F: Fn(&EvalCase, &Journal) -> RuntimeResult<PreparedRun>,
161 {
162 if self.config.max_concurrency == 1 {
163 let mut case_reports = Vec::with_capacity(dataset.cases().len());
164 for case in dataset.cases() {
165 let mut runs = Vec::with_capacity(self.config.runs_per_case);
166 for repetition in 0..self.config.runs_per_case {
167 runs.push(
168 self.run_case_once(dataset, case, repetition, &prepare)
169 .await?,
170 );
171 }
172 case_reports.push(case_report(case, runs));
173 }
174 return Ok(self.finish_report(dataset, case_reports));
175 }
176
177 let mut jobs = dataset
178 .cases()
179 .iter()
180 .enumerate()
181 .flat_map(|(case_index, case)| {
182 (0..self.config.runs_per_case).map(move |repetition| (case_index, case, repetition))
183 });
184 let prepare = &prepare;
185 let run_job = |(case_index, case, repetition)| async move {
186 (
187 case_index,
188 repetition,
189 self.run_case_once(dataset, case, repetition, prepare).await,
190 )
191 };
192 let mut active = FuturesUnordered::new();
193 for _ in 0..self.config.max_concurrency {
194 let Some(job) = jobs.next() else {
195 break;
196 };
197 active.push(run_job(job));
198 }
199
200 let mut completed = Vec::new();
201 let mut admission_stopped = false;
202 while let Some(result) = active.next().await {
203 admission_stopped |= result.2.is_err();
204 completed.push(result);
205 if !admission_stopped {
206 if let Some(job) = jobs.next() {
207 active.push(run_job(job));
208 }
209 }
210 }
211
212 completed.sort_unstable_by_key(|(case_index, repetition, _)| (*case_index, *repetition));
213 let mut runs_by_case: Vec<Vec<CaseRunReport>> = (0..dataset.cases().len())
214 .map(|_| Vec::with_capacity(self.config.runs_per_case))
215 .collect();
216 for (case_index, _, result) in completed {
217 runs_by_case[case_index].push(result?);
218 }
219
220 let mut case_reports = Vec::with_capacity(dataset.cases().len());
221 for (case, runs) in dataset.cases().iter().zip(runs_by_case) {
222 case_reports.push(case_report(case, runs));
223 }
224
225 Ok(self.finish_report(dataset, case_reports))
226 }
227
228 fn finish_report(&self, dataset: &Dataset, case_reports: Vec<CaseReport>) -> ExperimentReport {
229 let summary = ReportSummary::compute(&case_reports);
230 tracing::info!(
231 dataset = %dataset.name(),
232 version = %dataset.version(),
233 max_concurrency = self.config.max_concurrency,
234 cases = summary.cases,
235 runs = summary.runs,
236 run_pass_rate = summary.run_pass_rate,
237 "experiment complete"
238 );
239 ExperimentReport {
240 format_version: REPORT_FORMAT_VERSION,
241 name: format!("{}@{}", dataset.name(), dataset.version()),
242 dataset_name: dataset.name().to_owned(),
243 dataset_version: dataset.version().to_owned(),
244 runs_per_case: self.config.runs_per_case,
245 max_concurrency: self.config.max_concurrency,
246 cases: case_reports,
247 summary,
248 }
249 }
250
251 async fn run_case_once<F>(
253 &self,
254 dataset: &Dataset,
255 case: &EvalCase,
256 repetition: usize,
257 prepare: &F,
258 ) -> Result<CaseRunReport>
259 where
260 F: Fn(&EvalCase, &Journal) -> RuntimeResult<PreparedRun>,
261 {
262 let run_id = format!("eval:{}:{}:{repetition}", dataset.name(), case.id);
263 let journal = Journal::new(run_id.clone(), run_id.clone(), Clock::default());
264 let prepared = prepare(case, &journal)?;
265
266 let initial_state = match prepared.initial_state {
267 Some(state) => state,
268 None => State::from_value(case.input.clone()).map_err(|e| {
269 EvalError::AgentBuild(format!(
270 "case `{}`: input must be a JSON object of channel values: {e}",
271 case.id
272 ))
273 })?,
274 };
275
276 let executor = Executor::new();
277 let started = Instant::now();
278 let outcome = executor
279 .run(
280 &prepared.graph,
281 &prepared.spec,
282 initial_state,
283 RunConfig::new(&run_id).with_journal(journal.clone()),
284 )
285 .await;
286 let latency_ms = started.elapsed().as_millis() as u64;
287
288 let (status, final_state) = match outcome {
289 Ok(ExecutionOutcome::Done(state)) => (RunStatus::Done, state.to_value()),
290 Ok(ExecutionOutcome::Interrupted { state, .. }) => {
291 (RunStatus::Interrupted, state.to_value())
292 }
293 Err(error) => (
294 RunStatus::Failed {
295 error: error.to_string(),
296 },
297 Value::Null,
298 ),
299 };
300
301 let evidence = RunEvidence::from_journal(&journal, status, final_state, latency_ms);
302 let assertions: Vec<AssertionResult> = case
303 .expect
304 .assertions()
305 .iter()
306 .map(|assertion| assertion.evaluate(&evidence))
307 .collect();
308
309 let judge = match &self.config.judge {
310 Some(judge) => {
311 let request = JudgeRequest {
312 case_id: case.id.clone(),
313 input: case.input.clone(),
314 expectations: case.expect.clone(),
315 evidence: evidence.clone(),
316 };
317 Some(
318 judge
319 .judge(&request)
320 .await
321 .map_err(|e| EvalError::Judge(format!("case `{}`: {e}", case.id)))?,
322 )
323 }
324 None => None,
325 };
326
327 let passed = evidence.status.is_done()
328 && assertions.iter().all(|result| result.passed)
329 && judge.as_ref().map(|verdict| verdict.passed).unwrap_or(true);
330
331 Ok(CaseRunReport {
332 repetition,
333 status: evidence.status,
334 passed,
335 assertions,
336 judge,
337 tool_calls: evidence.tool_calls.len(),
338 latency_ms: evidence.latency_ms,
339 cost_usd: evidence.cost_usd,
340 total_tokens: evidence.total_tokens,
341 })
342 }
343}
344
345fn case_report(case: &EvalCase, runs: Vec<CaseRunReport>) -> CaseReport {
346 let pass_rate = runs.iter().filter(|run| run.passed).count() as f64 / runs.len() as f64;
347 CaseReport {
348 case_id: case.id.clone(),
349 tags: case.tags.clone(),
350 pass_rate,
351 runs,
352 }
353}
354
355#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
357pub struct CaseReport {
358 pub case_id: String,
360
361 #[serde(default, skip_serializing_if = "Vec::is_empty")]
363 pub tags: Vec<String>,
364
365 pub pass_rate: f64,
368
369 pub runs: Vec<CaseRunReport>,
371}
372
373#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
375pub struct CaseRunReport {
376 pub repetition: usize,
378
379 pub status: RunStatus,
381
382 pub passed: bool,
385
386 pub assertions: Vec<AssertionResult>,
388
389 #[serde(default, skip_serializing_if = "Option::is_none")]
391 pub judge: Option<JudgeVerdict>,
392
393 pub tool_calls: usize,
395
396 pub latency_ms: u64,
398
399 pub cost_usd: f64,
401
402 pub total_tokens: u64,
404}
405
406#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
408pub struct AssertionPassRate {
409 pub assertion: String,
411
412 pub passed: usize,
414
415 pub total: usize,
417
418 pub rate: f64,
420}
421
422#[derive(Debug, Clone, Copy, PartialEq, Serialize, Deserialize)]
424pub struct LatencyStats {
425 pub min: u64,
427 pub p50: u64,
429 pub p95: u64,
431 pub max: u64,
433 pub mean: f64,
435}
436
437#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
439pub struct ReportSummary {
440 pub cases: usize,
442
443 pub runs: usize,
445
446 pub runs_passed: usize,
448
449 pub run_pass_rate: f64,
451
452 pub case_pass_rate: f64,
454
455 pub assertions: Vec<AssertionPassRate>,
457
458 pub latency_ms: LatencyStats,
460
461 pub total_cost_usd: f64,
463
464 pub total_tokens: u64,
466}
467
468impl ReportSummary {
469 fn compute(cases: &[CaseReport]) -> Self {
470 let runs: Vec<&CaseRunReport> = cases.iter().flat_map(|case| &case.runs).collect();
471 let runs_passed = runs.iter().filter(|run| run.passed).count();
472
473 let mut latencies: Vec<u64> = runs.iter().map(|run| run.latency_ms).collect();
474 latencies.sort_unstable();
475 let latency_ms = LatencyStats {
476 min: latencies.first().copied().unwrap_or(0),
477 p50: percentile(&latencies, 50.0),
478 p95: percentile(&latencies, 95.0),
479 max: latencies.last().copied().unwrap_or(0),
480 mean: if latencies.is_empty() {
481 0.0
482 } else {
483 latencies.iter().sum::<u64>() as f64 / latencies.len() as f64
484 },
485 };
486
487 let mut by_assertion: BTreeMap<String, (usize, usize)> = BTreeMap::new();
490 for run in &runs {
491 for result in &run.assertions {
492 let entry = by_assertion.entry(result.assertion.clone()).or_default();
493 entry.1 += 1;
494 entry.0 += usize::from(result.passed);
495 }
496 }
497 let assertions = by_assertion
498 .into_iter()
499 .map(|(assertion, (passed, total))| AssertionPassRate {
500 assertion,
501 passed,
502 total,
503 rate: if total == 0 {
504 0.0
505 } else {
506 passed as f64 / total as f64
507 },
508 })
509 .collect();
510
511 Self {
512 cases: cases.len(),
513 runs: runs.len(),
514 runs_passed,
515 run_pass_rate: if runs.is_empty() {
516 0.0
517 } else {
518 runs_passed as f64 / runs.len() as f64
519 },
520 case_pass_rate: if cases.is_empty() {
521 0.0
522 } else {
523 cases.iter().map(|case| case.pass_rate).sum::<f64>() / cases.len() as f64
524 },
525 assertions,
526 latency_ms,
527 total_cost_usd: runs.iter().map(|run| run.cost_usd).sum(),
528 total_tokens: runs.iter().map(|run| run.total_tokens).sum(),
529 }
530 }
531}
532
533#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
535pub struct ExperimentReport {
536 pub format_version: u64,
538
539 pub name: String,
541
542 pub dataset_name: String,
544
545 pub dataset_version: String,
547
548 pub runs_per_case: usize,
550
551 #[serde(default = "default_max_concurrency")]
553 pub max_concurrency: usize,
554
555 pub cases: Vec<CaseReport>,
557
558 pub summary: ReportSummary,
560}
561
562fn default_max_concurrency() -> usize {
563 1
564}
565
566impl ExperimentReport {
567 pub fn to_json(&self) -> Result<String> {
569 Ok(serde_json::to_string_pretty(self)?)
570 }
571
572 pub fn from_json(text: &str) -> Result<Self> {
575 let report: ExperimentReport = serde_json::from_str(text)?;
576 if report.format_version != REPORT_FORMAT_VERSION {
577 return Err(EvalError::UnsupportedVersion {
578 found: report.format_version,
579 supported: REPORT_FORMAT_VERSION,
580 });
581 }
582 Ok(report)
583 }
584}
585
586fn percentile(sorted: &[u64], p: f64) -> u64 {
590 if sorted.is_empty() {
591 return 0;
592 }
593 let rank = (p / 100.0 * sorted.len() as f64).ceil() as usize;
594 sorted[rank.clamp(1, sorted.len()) - 1]
595}
596
597#[cfg(test)]
598mod tests {
599 use super::*;
600
601 #[test]
602 fn percentile_nearest_rank() {
603 let values: Vec<u64> = (1..=100).collect();
604 assert_eq!(percentile(&values, 50.0), 50);
605 assert_eq!(percentile(&values, 95.0), 95);
606 assert_eq!(percentile(&values, 100.0), 100);
607 assert_eq!(percentile(&values, 0.0), 1);
608 }
609
610 #[test]
611 fn percentile_small_samples() {
612 assert_eq!(percentile(&[7], 95.0), 7);
613 assert_eq!(percentile(&[10, 20], 50.0), 10);
614 assert_eq!(percentile(&[10, 20], 95.0), 20);
615 assert_eq!(percentile(&[], 50.0), 0);
616 }
617}