use std::collections::BTreeMap;
use std::sync::Arc;
use std::time::Instant;
use futures::{stream::FuturesUnordered, StreamExt};
use serde::{Deserialize, Serialize};
use rusty_agent_runtime::error::Result as RuntimeResult;
use rusty_agent_runtime::executor::{ExecutionOutcome, Executor, RunConfig};
use rusty_agent_runtime::graph::Graph;
use rusty_agent_runtime::journal::{Clock, Journal};
use rusty_agent_runtime::state::{State, StateSpec};
use serde_json::Value;
use crate::assertion::AssertionResult;
use crate::dataset::{Dataset, EvalCase};
use crate::error::{EvalError, Result};
use crate::evidence::{RunEvidence, RunStatus};
use crate::judge::{JudgeModel, JudgeRequest, JudgeVerdict};
pub const REPORT_FORMAT_VERSION: u64 = 1;
pub struct PreparedRun {
pub graph: Graph,
pub spec: StateSpec,
pub initial_state: Option<State>,
}
impl PreparedRun {
pub fn new(graph: Graph, spec: StateSpec) -> Self {
Self {
graph,
spec,
initial_state: None,
}
}
pub fn with_initial_state(mut self, state: State) -> Self {
self.initial_state = Some(state);
self
}
}
pub struct ExperimentConfig {
runs_per_case: usize,
max_concurrency: usize,
judge: Option<Arc<dyn JudgeModel>>,
}
impl Default for ExperimentConfig {
fn default() -> Self {
Self::new()
}
}
impl ExperimentConfig {
pub fn new() -> Self {
Self {
runs_per_case: 1,
max_concurrency: 1,
judge: None,
}
}
pub fn with_runs_per_case(mut self, n: usize) -> Self {
self.runs_per_case = n.max(1);
self
}
pub fn with_max_concurrency(mut self, n: usize) -> Self {
self.max_concurrency = n.max(1);
self
}
pub fn with_judge(mut self, judge: Arc<dyn JudgeModel>) -> Self {
self.judge = Some(judge);
self
}
}
impl std::fmt::Debug for ExperimentConfig {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
f.debug_struct("ExperimentConfig")
.field("runs_per_case", &self.runs_per_case)
.field("max_concurrency", &self.max_concurrency)
.field("judge", &self.judge.as_ref().map(|_| "<judge>"))
.finish()
}
}
pub struct ExperimentRunner {
config: ExperimentConfig,
}
impl ExperimentRunner {
pub fn new(config: ExperimentConfig) -> Self {
Self { config }
}
pub async fn run<F>(&self, dataset: &Dataset, prepare: F) -> Result<ExperimentReport>
where
F: Fn(&EvalCase, &Journal) -> RuntimeResult<PreparedRun>,
{
if self.config.max_concurrency == 1 {
let mut case_reports = Vec::with_capacity(dataset.cases().len());
for case in dataset.cases() {
let mut runs = Vec::with_capacity(self.config.runs_per_case);
for repetition in 0..self.config.runs_per_case {
runs.push(
self.run_case_once(dataset, case, repetition, &prepare)
.await?,
);
}
case_reports.push(case_report(case, runs));
}
return Ok(self.finish_report(dataset, case_reports));
}
let mut jobs = dataset
.cases()
.iter()
.enumerate()
.flat_map(|(case_index, case)| {
(0..self.config.runs_per_case).map(move |repetition| (case_index, case, repetition))
});
let prepare = &prepare;
let run_job = |(case_index, case, repetition)| async move {
(
case_index,
repetition,
self.run_case_once(dataset, case, repetition, prepare).await,
)
};
let mut active = FuturesUnordered::new();
for _ in 0..self.config.max_concurrency {
let Some(job) = jobs.next() else {
break;
};
active.push(run_job(job));
}
let mut completed = Vec::new();
let mut admission_stopped = false;
while let Some(result) = active.next().await {
admission_stopped |= result.2.is_err();
completed.push(result);
if !admission_stopped {
if let Some(job) = jobs.next() {
active.push(run_job(job));
}
}
}
completed.sort_unstable_by_key(|(case_index, repetition, _)| (*case_index, *repetition));
let mut runs_by_case: Vec<Vec<CaseRunReport>> = (0..dataset.cases().len())
.map(|_| Vec::with_capacity(self.config.runs_per_case))
.collect();
for (case_index, _, result) in completed {
runs_by_case[case_index].push(result?);
}
let mut case_reports = Vec::with_capacity(dataset.cases().len());
for (case, runs) in dataset.cases().iter().zip(runs_by_case) {
case_reports.push(case_report(case, runs));
}
Ok(self.finish_report(dataset, case_reports))
}
fn finish_report(&self, dataset: &Dataset, case_reports: Vec<CaseReport>) -> ExperimentReport {
let summary = ReportSummary::compute(&case_reports);
tracing::info!(
dataset = %dataset.name(),
version = %dataset.version(),
max_concurrency = self.config.max_concurrency,
cases = summary.cases,
runs = summary.runs,
run_pass_rate = summary.run_pass_rate,
"experiment complete"
);
ExperimentReport {
format_version: REPORT_FORMAT_VERSION,
name: format!("{}@{}", dataset.name(), dataset.version()),
dataset_name: dataset.name().to_owned(),
dataset_version: dataset.version().to_owned(),
runs_per_case: self.config.runs_per_case,
max_concurrency: self.config.max_concurrency,
cases: case_reports,
summary,
}
}
async fn run_case_once<F>(
&self,
dataset: &Dataset,
case: &EvalCase,
repetition: usize,
prepare: &F,
) -> Result<CaseRunReport>
where
F: Fn(&EvalCase, &Journal) -> RuntimeResult<PreparedRun>,
{
let run_id = format!("eval:{}:{}:{repetition}", dataset.name(), case.id);
let journal = Journal::new(run_id.clone(), run_id.clone(), Clock::default());
let prepared = prepare(case, &journal)?;
let initial_state = match prepared.initial_state {
Some(state) => state,
None => State::from_value(case.input.clone()).map_err(|e| {
EvalError::AgentBuild(format!(
"case `{}`: input must be a JSON object of channel values: {e}",
case.id
))
})?,
};
let executor = Executor::new();
let started = Instant::now();
let outcome = executor
.run(
&prepared.graph,
&prepared.spec,
initial_state,
RunConfig::new(&run_id).with_journal(journal.clone()),
)
.await;
let latency_ms = started.elapsed().as_millis() as u64;
let (status, final_state) = match outcome {
Ok(ExecutionOutcome::Done(state)) => (RunStatus::Done, state.to_value()),
Ok(ExecutionOutcome::Interrupted { state, .. }) => {
(RunStatus::Interrupted, state.to_value())
}
Err(error) => (
RunStatus::Failed {
error: error.to_string(),
},
Value::Null,
),
};
let evidence = RunEvidence::from_journal(&journal, status, final_state, latency_ms);
let assertions: Vec<AssertionResult> = case
.expect
.assertions()
.iter()
.map(|assertion| assertion.evaluate(&evidence))
.collect();
let judge = match &self.config.judge {
Some(judge) => {
let request = JudgeRequest {
case_id: case.id.clone(),
input: case.input.clone(),
expectations: case.expect.clone(),
evidence: evidence.clone(),
};
Some(
judge
.judge(&request)
.await
.map_err(|e| EvalError::Judge(format!("case `{}`: {e}", case.id)))?,
)
}
None => None,
};
let passed = evidence.status.is_done()
&& assertions.iter().all(|result| result.passed)
&& judge.as_ref().map(|verdict| verdict.passed).unwrap_or(true);
Ok(CaseRunReport {
repetition,
status: evidence.status,
passed,
assertions,
judge,
tool_calls: evidence.tool_calls.len(),
latency_ms: evidence.latency_ms,
cost_usd: evidence.cost_usd,
total_tokens: evidence.total_tokens,
})
}
}
fn case_report(case: &EvalCase, runs: Vec<CaseRunReport>) -> CaseReport {
let pass_rate = runs.iter().filter(|run| run.passed).count() as f64 / runs.len() as f64;
CaseReport {
case_id: case.id.clone(),
tags: case.tags.clone(),
pass_rate,
runs,
}
}
#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
pub struct CaseReport {
pub case_id: String,
#[serde(default, skip_serializing_if = "Vec::is_empty")]
pub tags: Vec<String>,
pub pass_rate: f64,
pub runs: Vec<CaseRunReport>,
}
#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
pub struct CaseRunReport {
pub repetition: usize,
pub status: RunStatus,
pub passed: bool,
pub assertions: Vec<AssertionResult>,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub judge: Option<JudgeVerdict>,
pub tool_calls: usize,
pub latency_ms: u64,
pub cost_usd: f64,
pub total_tokens: u64,
}
#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
pub struct AssertionPassRate {
pub assertion: String,
pub passed: usize,
pub total: usize,
pub rate: f64,
}
#[derive(Debug, Clone, Copy, PartialEq, Serialize, Deserialize)]
pub struct LatencyStats {
pub min: u64,
pub p50: u64,
pub p95: u64,
pub max: u64,
pub mean: f64,
}
#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
pub struct ReportSummary {
pub cases: usize,
pub runs: usize,
pub runs_passed: usize,
pub run_pass_rate: f64,
pub case_pass_rate: f64,
pub assertions: Vec<AssertionPassRate>,
pub latency_ms: LatencyStats,
pub total_cost_usd: f64,
pub total_tokens: u64,
}
impl ReportSummary {
fn compute(cases: &[CaseReport]) -> Self {
let runs: Vec<&CaseRunReport> = cases.iter().flat_map(|case| &case.runs).collect();
let runs_passed = runs.iter().filter(|run| run.passed).count();
let mut latencies: Vec<u64> = runs.iter().map(|run| run.latency_ms).collect();
latencies.sort_unstable();
let latency_ms = LatencyStats {
min: latencies.first().copied().unwrap_or(0),
p50: percentile(&latencies, 50.0),
p95: percentile(&latencies, 95.0),
max: latencies.last().copied().unwrap_or(0),
mean: if latencies.is_empty() {
0.0
} else {
latencies.iter().sum::<u64>() as f64 / latencies.len() as f64
},
};
let mut by_assertion: BTreeMap<String, (usize, usize)> = BTreeMap::new();
for run in &runs {
for result in &run.assertions {
let entry = by_assertion.entry(result.assertion.clone()).or_default();
entry.1 += 1;
entry.0 += usize::from(result.passed);
}
}
let assertions = by_assertion
.into_iter()
.map(|(assertion, (passed, total))| AssertionPassRate {
assertion,
passed,
total,
rate: if total == 0 {
0.0
} else {
passed as f64 / total as f64
},
})
.collect();
Self {
cases: cases.len(),
runs: runs.len(),
runs_passed,
run_pass_rate: if runs.is_empty() {
0.0
} else {
runs_passed as f64 / runs.len() as f64
},
case_pass_rate: if cases.is_empty() {
0.0
} else {
cases.iter().map(|case| case.pass_rate).sum::<f64>() / cases.len() as f64
},
assertions,
latency_ms,
total_cost_usd: runs.iter().map(|run| run.cost_usd).sum(),
total_tokens: runs.iter().map(|run| run.total_tokens).sum(),
}
}
}
#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
pub struct ExperimentReport {
pub format_version: u64,
pub name: String,
pub dataset_name: String,
pub dataset_version: String,
pub runs_per_case: usize,
#[serde(default = "default_max_concurrency")]
pub max_concurrency: usize,
pub cases: Vec<CaseReport>,
pub summary: ReportSummary,
}
fn default_max_concurrency() -> usize {
1
}
impl ExperimentReport {
pub fn to_json(&self) -> Result<String> {
Ok(serde_json::to_string_pretty(self)?)
}
pub fn from_json(text: &str) -> Result<Self> {
let report: ExperimentReport = serde_json::from_str(text)?;
if report.format_version != REPORT_FORMAT_VERSION {
return Err(EvalError::UnsupportedVersion {
found: report.format_version,
supported: REPORT_FORMAT_VERSION,
});
}
Ok(report)
}
}
fn percentile(sorted: &[u64], p: f64) -> u64 {
if sorted.is_empty() {
return 0;
}
let rank = (p / 100.0 * sorted.len() as f64).ceil() as usize;
sorted[rank.clamp(1, sorted.len()) - 1]
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn percentile_nearest_rank() {
let values: Vec<u64> = (1..=100).collect();
assert_eq!(percentile(&values, 50.0), 50);
assert_eq!(percentile(&values, 95.0), 95);
assert_eq!(percentile(&values, 100.0), 100);
assert_eq!(percentile(&values, 0.0), 1);
}
#[test]
fn percentile_small_samples() {
assert_eq!(percentile(&[7], 95.0), 7);
assert_eq!(percentile(&[10, 20], 50.0), 10);
assert_eq!(percentile(&[10, 20], 95.0), 20);
assert_eq!(percentile(&[], 50.0), 0);
}
}