Skip to main content

rusty_eval/
experiment.rs

1//! The experiment runner: dataset in, evidence-graded report out.
2//!
3//! [`ExperimentRunner`] drives one agent over a [`Dataset`], N times per
4//! case, through the real `rusty-agent-runtime` executor — no simulation
5//! harness. Each run records into its own Flight Recorder journal; the
6//! journal plus the run outcome become [`RunEvidence`], which the case's
7//! assertions grade. Runs are sequential by default or explicitly
8//! bounded-parallel; each gets a fresh journal, so repetition *i*'s evidence
9//! can never bleed into repetition *i+1*'s.
10//!
11//! The output is a serializable [`ExperimentReport`]: per-case-run detail
12//! (assertion verdicts with evidence, status, latency, cost), pass rates per
13//! assertion, latency percentiles, and totals — everything
14//! [`crate::compare()`] needs to judge a candidate against a baseline.
15//! Parallel completion never controls artifact ordering or which
16//! infrastructure error is returned.
17
18use std::collections::BTreeMap;
19use std::sync::Arc;
20use std::time::Instant;
21
22use futures::{stream::FuturesUnordered, StreamExt};
23use serde::{Deserialize, Serialize};
24
25use rusty_agent_runtime::error::Result as RuntimeResult;
26use rusty_agent_runtime::executor::{ExecutionOutcome, Executor, RunConfig};
27use rusty_agent_runtime::graph::Graph;
28use rusty_agent_runtime::journal::{Clock, Journal};
29use rusty_agent_runtime::state::{State, StateSpec};
30use serde_json::Value;
31
32use crate::assertion::AssertionResult;
33use crate::dataset::{Dataset, EvalCase};
34use crate::error::{EvalError, Result};
35use crate::evidence::{RunEvidence, RunStatus};
36use crate::judge::{JudgeModel, JudgeRequest, JudgeVerdict};
37
38/// The report format version this build writes and reads.
39pub const REPORT_FORMAT_VERSION: u64 = 1;
40
41/// Everything the runner needs to execute one case repetition.
42///
43/// Built by the agent factory per run: graphs are cheap to compile, and a
44/// fresh build per repetition guarantees no state leaks between runs. The
45/// factory receives the run's [`Journal`] so recording-wired graphs (e.g.
46/// `create_react_agent_with_recording`) journal their model and tool calls
47/// into the run's evidence.
48pub struct PreparedRun {
49    /// The compiled graph under test.
50    pub graph: Graph,
51
52    /// The state schema (channels + reducers).
53    pub spec: StateSpec,
54
55    /// Explicit initial state. `None` derives it from the case's `input`
56    /// payload (which must then be a JSON object of channel values).
57    pub initial_state: Option<State>,
58}
59
60impl PreparedRun {
61    /// A prepared run whose initial state derives from the case input.
62    pub fn new(graph: Graph, spec: StateSpec) -> Self {
63        Self {
64            graph,
65            spec,
66            initial_state: None,
67        }
68    }
69
70    /// Override the initial state (bypasses derivation from the case input).
71    pub fn with_initial_state(mut self, state: State) -> Self {
72        self.initial_state = Some(state);
73        self
74    }
75}
76
77/// Experiment configuration.
78pub struct ExperimentConfig {
79    runs_per_case: usize,
80    max_concurrency: usize,
81    judge: Option<Arc<dyn JudgeModel>>,
82}
83
84impl Default for ExperimentConfig {
85    fn default() -> Self {
86        Self::new()
87    }
88}
89
90impl ExperimentConfig {
91    /// One run per case, no judge.
92    pub fn new() -> Self {
93        Self {
94            runs_per_case: 1,
95            max_concurrency: 1,
96            judge: None,
97        }
98    }
99
100    /// Run each case `n` times. Repetitions exist to expose flaky behavior;
101    /// pass rates are computed over all repetitions.
102    pub fn with_runs_per_case(mut self, n: usize) -> Self {
103        self.runs_per_case = n.max(1);
104        self
105    }
106
107    /// Allow at most `n` case runs to execute concurrently.
108    ///
109    /// The default is `1`, preserving sequential execution and uncontended
110    /// latency measurement. Zero is normalized to one, matching
111    /// [`Self::with_runs_per_case`]. Parallel runs still receive isolated
112    /// graphs and journals, and the report remains ordered by dataset case
113    /// then repetition regardless of completion order.
114    pub fn with_max_concurrency(mut self, n: usize) -> Self {
115        self.max_concurrency = n.max(1);
116        self
117    }
118
119    /// Attach a judge: every case run is additionally scored through it and
120    /// the verdict recorded in the report.
121    pub fn with_judge(mut self, judge: Arc<dyn JudgeModel>) -> Self {
122        self.judge = Some(judge);
123        self
124    }
125}
126
127impl std::fmt::Debug for ExperimentConfig {
128    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
129        f.debug_struct("ExperimentConfig")
130            .field("runs_per_case", &self.runs_per_case)
131            .field("max_concurrency", &self.max_concurrency)
132            .field("judge", &self.judge.as_ref().map(|_| "<judge>"))
133            .finish()
134    }
135}
136
137/// The experiment runner. Stateless across experiments.
138pub struct ExperimentRunner {
139    config: ExperimentConfig,
140}
141
142impl ExperimentRunner {
143    /// A runner with `config`.
144    pub fn new(config: ExperimentConfig) -> Self {
145        Self { config }
146    }
147
148    /// Run `prepare`'s agent over every case of `dataset`,
149    /// `runs_per_case` times each, and grade the evidence.
150    ///
151    /// `prepare` builds the graph under test for one case run and receives
152    /// the run's journal for evidence wiring. At most the configured number
153    /// of runs are in flight. The default sequential path retains fail-fast
154    /// behavior. In parallel mode, observing an infrastructure error stops
155    /// admission of new runs; the already-active window settles, and the
156    /// earliest dataset-case/repetition error from that window wins
157    /// deterministically.
158    pub async fn run<F>(&self, dataset: &Dataset, prepare: F) -> Result<ExperimentReport>
159    where
160        F: Fn(&EvalCase, &Journal) -> RuntimeResult<PreparedRun>,
161    {
162        if self.config.max_concurrency == 1 {
163            let mut case_reports = Vec::with_capacity(dataset.cases().len());
164            for case in dataset.cases() {
165                let mut runs = Vec::with_capacity(self.config.runs_per_case);
166                for repetition in 0..self.config.runs_per_case {
167                    runs.push(
168                        self.run_case_once(dataset, case, repetition, &prepare)
169                            .await?,
170                    );
171                }
172                case_reports.push(case_report(case, runs));
173            }
174            return Ok(self.finish_report(dataset, case_reports));
175        }
176
177        let mut jobs = dataset
178            .cases()
179            .iter()
180            .enumerate()
181            .flat_map(|(case_index, case)| {
182                (0..self.config.runs_per_case).map(move |repetition| (case_index, case, repetition))
183            });
184        let prepare = &prepare;
185        let run_job = |(case_index, case, repetition)| async move {
186            (
187                case_index,
188                repetition,
189                self.run_case_once(dataset, case, repetition, prepare).await,
190            )
191        };
192        let mut active = FuturesUnordered::new();
193        for _ in 0..self.config.max_concurrency {
194            let Some(job) = jobs.next() else {
195                break;
196            };
197            active.push(run_job(job));
198        }
199
200        let mut completed = Vec::new();
201        let mut admission_stopped = false;
202        while let Some(result) = active.next().await {
203            admission_stopped |= result.2.is_err();
204            completed.push(result);
205            if !admission_stopped {
206                if let Some(job) = jobs.next() {
207                    active.push(run_job(job));
208                }
209            }
210        }
211
212        completed.sort_unstable_by_key(|(case_index, repetition, _)| (*case_index, *repetition));
213        let mut runs_by_case: Vec<Vec<CaseRunReport>> = (0..dataset.cases().len())
214            .map(|_| Vec::with_capacity(self.config.runs_per_case))
215            .collect();
216        for (case_index, _, result) in completed {
217            runs_by_case[case_index].push(result?);
218        }
219
220        let mut case_reports = Vec::with_capacity(dataset.cases().len());
221        for (case, runs) in dataset.cases().iter().zip(runs_by_case) {
222            case_reports.push(case_report(case, runs));
223        }
224
225        Ok(self.finish_report(dataset, case_reports))
226    }
227
228    fn finish_report(&self, dataset: &Dataset, case_reports: Vec<CaseReport>) -> ExperimentReport {
229        let summary = ReportSummary::compute(&case_reports);
230        tracing::info!(
231            dataset = %dataset.name(),
232            version = %dataset.version(),
233            max_concurrency = self.config.max_concurrency,
234            cases = summary.cases,
235            runs = summary.runs,
236            run_pass_rate = summary.run_pass_rate,
237            "experiment complete"
238        );
239        ExperimentReport {
240            format_version: REPORT_FORMAT_VERSION,
241            name: format!("{}@{}", dataset.name(), dataset.version()),
242            dataset_name: dataset.name().to_owned(),
243            dataset_version: dataset.version().to_owned(),
244            runs_per_case: self.config.runs_per_case,
245            max_concurrency: self.config.max_concurrency,
246            cases: case_reports,
247            summary,
248        }
249    }
250
251    /// One case repetition: build, run, distill evidence, grade.
252    async fn run_case_once<F>(
253        &self,
254        dataset: &Dataset,
255        case: &EvalCase,
256        repetition: usize,
257        prepare: &F,
258    ) -> Result<CaseRunReport>
259    where
260        F: Fn(&EvalCase, &Journal) -> RuntimeResult<PreparedRun>,
261    {
262        let run_id = format!("eval:{}:{}:{repetition}", dataset.name(), case.id);
263        let journal = Journal::new(run_id.clone(), run_id.clone(), Clock::default());
264        let prepared = prepare(case, &journal)?;
265
266        let initial_state = match prepared.initial_state {
267            Some(state) => state,
268            None => State::from_value(case.input.clone()).map_err(|e| {
269                EvalError::AgentBuild(format!(
270                    "case `{}`: input must be a JSON object of channel values: {e}",
271                    case.id
272                ))
273            })?,
274        };
275
276        let executor = Executor::new();
277        let started = Instant::now();
278        let outcome = executor
279            .run(
280                &prepared.graph,
281                &prepared.spec,
282                initial_state,
283                RunConfig::new(&run_id).with_journal(journal.clone()),
284            )
285            .await;
286        let latency_ms = started.elapsed().as_millis() as u64;
287
288        let (status, final_state) = match outcome {
289            Ok(ExecutionOutcome::Done(state)) => (RunStatus::Done, state.to_value()),
290            Ok(ExecutionOutcome::Interrupted { state, .. }) => {
291                (RunStatus::Interrupted, state.to_value())
292            }
293            Err(error) => (
294                RunStatus::Failed {
295                    error: error.to_string(),
296                },
297                Value::Null,
298            ),
299        };
300
301        let evidence = RunEvidence::from_journal(&journal, status, final_state, latency_ms);
302        let assertions: Vec<AssertionResult> = case
303            .expect
304            .assertions()
305            .iter()
306            .map(|assertion| assertion.evaluate(&evidence))
307            .collect();
308
309        let judge = match &self.config.judge {
310            Some(judge) => {
311                let request = JudgeRequest {
312                    case_id: case.id.clone(),
313                    input: case.input.clone(),
314                    expectations: case.expect.clone(),
315                    evidence: evidence.clone(),
316                };
317                Some(
318                    judge
319                        .judge(&request)
320                        .await
321                        .map_err(|e| EvalError::Judge(format!("case `{}`: {e}", case.id)))?,
322                )
323            }
324            None => None,
325        };
326
327        let passed = evidence.status.is_done()
328            && assertions.iter().all(|result| result.passed)
329            && judge.as_ref().map(|verdict| verdict.passed).unwrap_or(true);
330
331        Ok(CaseRunReport {
332            repetition,
333            status: evidence.status,
334            passed,
335            assertions,
336            judge,
337            tool_calls: evidence.tool_calls.len(),
338            latency_ms: evidence.latency_ms,
339            cost_usd: evidence.cost_usd,
340            total_tokens: evidence.total_tokens,
341        })
342    }
343}
344
345fn case_report(case: &EvalCase, runs: Vec<CaseRunReport>) -> CaseReport {
346    let pass_rate = runs.iter().filter(|run| run.passed).count() as f64 / runs.len() as f64;
347    CaseReport {
348        case_id: case.id.clone(),
349        tags: case.tags.clone(),
350        pass_rate,
351        runs,
352    }
353}
354
355/// One case's runs within an experiment.
356#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
357pub struct CaseReport {
358    /// The case id (dataset key).
359    pub case_id: String,
360
361    /// The case's tags, carried for report slicing.
362    #[serde(default, skip_serializing_if = "Vec::is_empty")]
363    pub tags: Vec<String>,
364
365    /// Fraction of this case's runs that passed (completed run, all
366    /// assertions green, judge — when present — passing).
367    pub pass_rate: f64,
368
369    /// Per-repetition detail.
370    pub runs: Vec<CaseRunReport>,
371}
372
373/// One case repetition, fully graded.
374#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
375pub struct CaseRunReport {
376    /// The 0-based repetition index.
377    pub repetition: usize,
378
379    /// How the run terminated.
380    pub status: RunStatus,
381
382    /// The case-run verdict: run completed, every assertion passed, and the
383    /// judge (when configured) passed.
384    pub passed: bool,
385
386    /// Every assertion verdict, with evidence.
387    pub assertions: Vec<AssertionResult>,
388
389    /// The judge's verdict, when a judge is configured.
390    #[serde(default, skip_serializing_if = "Option::is_none")]
391    pub judge: Option<JudgeVerdict>,
392
393    /// Number of journaled tool calls.
394    pub tool_calls: usize,
395
396    /// Wall latency of the run in milliseconds.
397    pub latency_ms: u64,
398
399    /// Total journaled cost of the run in USD.
400    pub cost_usd: f64,
401
402    /// Total tokens reported by the run's model calls.
403    pub total_tokens: u64,
404}
405
406/// Pass-rate aggregate for one assertion key across all runs.
407#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
408pub struct AssertionPassRate {
409    /// The assertion's report key ([`crate::assertion::Assertion::name`]).
410    pub assertion: String,
411
412    /// Runs where this assertion passed.
413    pub passed: usize,
414
415    /// Runs where this assertion was evaluated.
416    pub total: usize,
417
418    /// `passed / total` (0.0 when never evaluated).
419    pub rate: f64,
420}
421
422/// Latency distribution over all case runs, nearest-rank percentiles.
423#[derive(Debug, Clone, Copy, PartialEq, Serialize, Deserialize)]
424pub struct LatencyStats {
425    /// Fastest run (ms).
426    pub min: u64,
427    /// Median run (ms).
428    pub p50: u64,
429    /// 95th-percentile run (ms).
430    pub p95: u64,
431    /// Slowest run (ms).
432    pub max: u64,
433    /// Mean latency (ms).
434    pub mean: f64,
435}
436
437/// Experiment-wide aggregates.
438#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
439pub struct ReportSummary {
440    /// Number of cases in the dataset.
441    pub cases: usize,
442
443    /// Total case runs executed.
444    pub runs: usize,
445
446    /// Case runs that passed.
447    pub runs_passed: usize,
448
449    /// `runs_passed / runs` (0.0 for an empty dataset).
450    pub run_pass_rate: f64,
451
452    /// Mean of per-case pass rates.
453    pub case_pass_rate: f64,
454
455    /// Pass rate per assertion key, aggregated over all runs.
456    pub assertions: Vec<AssertionPassRate>,
457
458    /// Latency distribution over all runs.
459    pub latency_ms: LatencyStats,
460
461    /// Total journaled cost across all runs (USD).
462    pub total_cost_usd: f64,
463
464    /// Total tokens across all runs.
465    pub total_tokens: u64,
466}
467
468impl ReportSummary {
469    fn compute(cases: &[CaseReport]) -> Self {
470        let runs: Vec<&CaseRunReport> = cases.iter().flat_map(|case| &case.runs).collect();
471        let runs_passed = runs.iter().filter(|run| run.passed).count();
472
473        let mut latencies: Vec<u64> = runs.iter().map(|run| run.latency_ms).collect();
474        latencies.sort_unstable();
475        let latency_ms = LatencyStats {
476            min: latencies.first().copied().unwrap_or(0),
477            p50: percentile(&latencies, 50.0),
478            p95: percentile(&latencies, 95.0),
479            max: latencies.last().copied().unwrap_or(0),
480            mean: if latencies.is_empty() {
481                0.0
482            } else {
483                latencies.iter().sum::<u64>() as f64 / latencies.len() as f64
484            },
485        };
486
487        // BTreeMap: assertion keys aggregate in sorted order, so reports
488        // serialize deterministically.
489        let mut by_assertion: BTreeMap<String, (usize, usize)> = BTreeMap::new();
490        for run in &runs {
491            for result in &run.assertions {
492                let entry = by_assertion.entry(result.assertion.clone()).or_default();
493                entry.1 += 1;
494                entry.0 += usize::from(result.passed);
495            }
496        }
497        let assertions = by_assertion
498            .into_iter()
499            .map(|(assertion, (passed, total))| AssertionPassRate {
500                assertion,
501                passed,
502                total,
503                rate: if total == 0 {
504                    0.0
505                } else {
506                    passed as f64 / total as f64
507                },
508            })
509            .collect();
510
511        Self {
512            cases: cases.len(),
513            runs: runs.len(),
514            runs_passed,
515            run_pass_rate: if runs.is_empty() {
516                0.0
517            } else {
518                runs_passed as f64 / runs.len() as f64
519            },
520            case_pass_rate: if cases.is_empty() {
521                0.0
522            } else {
523                cases.iter().map(|case| case.pass_rate).sum::<f64>() / cases.len() as f64
524            },
525            assertions,
526            latency_ms,
527            total_cost_usd: runs.iter().map(|run| run.cost_usd).sum(),
528            total_tokens: runs.iter().map(|run| run.total_tokens).sum(),
529        }
530    }
531}
532
533/// The graded output of one experiment.
534#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
535pub struct ExperimentReport {
536    /// Report schema version ([`REPORT_FORMAT_VERSION`]).
537    pub format_version: u64,
538
539    /// Experiment identity: `{dataset_name}@{dataset_version}`.
540    pub name: String,
541
542    /// The dataset that was run.
543    pub dataset_name: String,
544
545    /// The dataset version that was run.
546    pub dataset_version: String,
547
548    /// Repetitions per case.
549    pub runs_per_case: usize,
550
551    /// Maximum case runs allowed in flight when this report was produced.
552    #[serde(default = "default_max_concurrency")]
553    pub max_concurrency: usize,
554
555    /// Per-case detail.
556    pub cases: Vec<CaseReport>,
557
558    /// Experiment-wide aggregates.
559    pub summary: ReportSummary,
560}
561
562fn default_max_concurrency() -> usize {
563    1
564}
565
566impl ExperimentReport {
567    /// Serialize as pretty JSON (for `baseline.json`-style artifacts).
568    pub fn to_json(&self) -> Result<String> {
569        Ok(serde_json::to_string_pretty(self)?)
570    }
571
572    /// Parse a report written by [`ExperimentReport::to_json`]. Refuses
573    /// reports from an incompatible format version.
574    pub fn from_json(text: &str) -> Result<Self> {
575        let report: ExperimentReport = serde_json::from_str(text)?;
576        if report.format_version != REPORT_FORMAT_VERSION {
577            return Err(EvalError::UnsupportedVersion {
578                found: report.format_version,
579                supported: REPORT_FORMAT_VERSION,
580            });
581        }
582        Ok(report)
583    }
584}
585
586/// Nearest-rank percentile over a sorted slice. Empty input yields 0 — an
587/// experiment with no runs has no distribution, and zero is the honest
588/// placeholder rather than a panic.
589fn percentile(sorted: &[u64], p: f64) -> u64 {
590    if sorted.is_empty() {
591        return 0;
592    }
593    let rank = (p / 100.0 * sorted.len() as f64).ceil() as usize;
594    sorted[rank.clamp(1, sorted.len()) - 1]
595}
596
597#[cfg(test)]
598mod tests {
599    use super::*;
600
601    #[test]
602    fn percentile_nearest_rank() {
603        let values: Vec<u64> = (1..=100).collect();
604        assert_eq!(percentile(&values, 50.0), 50);
605        assert_eq!(percentile(&values, 95.0), 95);
606        assert_eq!(percentile(&values, 100.0), 100);
607        assert_eq!(percentile(&values, 0.0), 1);
608    }
609
610    #[test]
611    fn percentile_small_samples() {
612        assert_eq!(percentile(&[7], 95.0), 7);
613        assert_eq!(percentile(&[10, 20], 50.0), 10);
614        assert_eq!(percentile(&[10, 20], 95.0), 20);
615        assert_eq!(percentile(&[], 50.0), 0);
616    }
617}