use super::evaluation::{
EvaluationHandler, EvaluationOutcome, SampleFailureContext, failed_sample_context,
};
use super::options::RunOptions;
use crate::compare::RunComparator;
use crate::dataset_loader::DatasetLoader;
use crate::evaluator::Evaluator;
use crate::execution::BenchExecutionPlan;
use crate::metric::Suite;
use crate::report_gen::Reporter;
use crate::result::{BenchRunResult, BenchSampleResult, BranchResult, DatasetInfo, MetricResult};
use crate::run_id::generate_run_id;
use crate::run_storage::RunStorage;
use crate::schema;
use crate::types::ScoredSample;
use futures::stream::{FuturesUnordered, StreamExt};
use rskit_errors::{AppError, AppResult, ErrorCode};
use rskit_util::time::{SharedClock, elapsed_millis, format_rfc3339, system_clock};
use rskit_worker::Pool;
use std::collections::HashMap;
use std::sync::Arc;
use tracing::Instrument;
struct Branch<L> {
name: String,
evaluator: Arc<dyn Evaluator<L>>,
tier: i32,
}
pub struct BenchRunner<L> {
branches: Vec<Branch<L>>,
metrics: Option<Suite<L>>,
storage: Option<Box<dyn RunStorage>>,
reporters: Vec<Box<dyn Reporter>>,
comparator: Option<RunComparator>,
clock: SharedClock,
}
impl<L> Default for BenchRunner<L>
where
L: Clone + Send + Sync + std::fmt::Display + 'static,
{
fn default() -> Self {
Self::new()
}
}
impl<L> BenchRunner<L>
where
L: Clone + Send + Sync + std::fmt::Display + 'static,
{
pub fn new() -> Self {
Self {
branches: Vec::new(),
metrics: None,
storage: None,
reporters: Vec::new(),
comparator: None,
clock: system_clock(),
}
}
pub fn register(
mut self,
name: impl Into<String>,
evaluator: Box<dyn Evaluator<L>>,
tier: i32,
) -> Self {
self.branches.push(Branch {
name: name.into(),
evaluator: Arc::from(evaluator),
tier,
});
self
}
#[must_use]
pub fn with_metrics(mut self, suite: Suite<L>) -> Self {
self.metrics = Some(suite);
self
}
#[must_use]
pub fn with_storage(mut self, storage: Box<dyn RunStorage>) -> Self {
self.storage = Some(storage);
self
}
#[must_use]
pub fn with_reporter(mut self, reporter: Box<dyn Reporter>) -> Self {
self.reporters.push(reporter);
self
}
#[must_use]
pub fn with_comparator(mut self, comparator: RunComparator) -> Self {
self.comparator = Some(comparator);
self
}
#[must_use]
pub fn with_clock(mut self, clock: SharedClock) -> Self {
self.clock = clock;
self
}
pub async fn run(
&self,
loader: &DatasetLoader<L>,
opts: RunOptions,
) -> AppResult<BenchRunResult> {
let execution = BenchExecutionPlan::new(&opts.tag, opts.concurrency);
let span = execution.operation.start_span("bench.run");
span.in_scope(|| {
tracing::info!(
tag = %opts.tag,
concurrency = opts.concurrency,
"benchmark run started"
);
});
let result = self
.run_with_execution(loader, opts, &execution)
.instrument(span)
.await;
match &result {
Ok(_) => execution.operation.end_operation("ok", None),
Err(error) => execution.operation.end_operation("error", Some(error)),
}
result
}
async fn run_with_execution(
&self,
loader: &DatasetLoader<L>,
opts: RunOptions,
execution: &BenchExecutionPlan,
) -> AppResult<BenchRunResult> {
let start = self.clock.monotonic_millis();
let samples = loader.all()?;
let dataset_info = {
let mut label_distribution: HashMap<String, usize> = HashMap::new();
for s in &samples {
*label_distribution.entry(s.label.to_string()).or_insert(0) += 1;
}
DatasetInfo {
name: opts.tag.clone(),
version: String::new(),
sample_count: samples.len(),
label_distribution,
}
};
let mut branch_results = HashMap::new();
let mut all_scored: Vec<ScoredSample<L>> = Vec::new();
let mut sample_results: Vec<BenchSampleResult> = Vec::new();
for branch in &self.branches {
let branch_start = self.clock.monotonic_millis();
let handler = Arc::new(EvaluationHandler {
evaluator: Arc::clone(&branch.evaluator),
branch_name: branch.name.clone(),
timeout_secs: opts.timeout_secs,
clock: Arc::clone(&self.clock),
});
let pool = Pool::new(handler, execution.pool_config_for(&branch.name));
let mut branch_metrics: HashMap<String, f64> = HashMap::new();
let mut total_score_pos = 0.0_f64;
let mut total_score_neg = 0.0_f64;
let mut count_pos = 0usize;
let mut count_neg = 0usize;
let mut errors = 0usize;
let mut pending = FuturesUnordered::new();
let mut sample_iter = samples.iter();
let concurrency = opts.concurrency.max(1);
loop {
while pending.len() < concurrency {
let Some(sample) = sample_iter.next() else {
break;
};
let context = SampleFailureContext::from_sample(sample);
let handle = pool.submit(sample.clone()).await?;
let submitted_at = self.clock.monotonic_millis();
pending.push(async move { (context, submitted_at, handle.result().await) });
}
let Some((submitted_sample, submitted_at, result)) = pending.next().await else {
break;
};
match result {
Ok(EvaluationOutcome::Success {
sample,
prediction: pred,
duration_ms,
}) => {
let correct = pred.label.to_string() == sample.label.to_string();
if correct {
total_score_pos += pred.score;
count_pos += 1;
} else {
total_score_neg += pred.score;
count_neg += 1;
}
let mut branch_scores = HashMap::new();
branch_scores.insert(branch.name.clone(), pred.score);
sample_results.push(BenchSampleResult {
id: sample.id.clone(),
label: sample.label.to_string(),
predicted: pred.label.to_string(),
score: pred.score,
correct,
branch_scores,
duration_ms,
error: String::new(),
});
all_scored.push(ScoredSample {
sample: sample.clone(),
prediction: pred,
});
}
Ok(EvaluationOutcome::Failure {
sample,
duration_ms,
error,
}) => {
tracing::warn!(
sample_id = %sample.id,
branch = %branch.name,
error = %error,
"Evaluation failed"
);
errors += 1;
sample_results.push(BenchSampleResult {
id: sample.id.clone(),
label: sample.label.to_string(),
predicted: String::new(),
score: 0.0,
correct: false,
branch_scores: HashMap::new(),
duration_ms,
error,
});
}
Err(error) => {
errors += 1;
tracing::warn!(
sample_id = %submitted_sample.id,
branch = %branch.name,
error = %error,
"worker evaluation failed"
);
sample_results.push(failed_sample_context(
&submitted_sample,
elapsed_millis(submitted_at, self.clock.monotonic_millis()),
format!("worker evaluation failed: {error}"),
));
}
}
}
pool.shutdown().await?;
let total = count_pos + count_neg + errors;
if total > 0 {
branch_metrics.insert("accuracy".to_string(), count_pos as f64 / total as f64);
}
let avg_score_positive = if count_pos > 0 {
total_score_pos / count_pos as f64
} else {
0.0
};
let avg_score_negative = if count_neg > 0 {
total_score_neg / count_neg as f64
} else {
0.0
};
branch_results.insert(
branch.name.clone(),
BranchResult {
name: branch.name.clone(),
tier: branch.tier,
metrics: branch_metrics,
avg_score_positive,
avg_score_negative,
duration_ms: elapsed_millis(branch_start, self.clock.monotonic_millis()),
errors,
},
);
}
let metric_results: Vec<MetricResult> = if let Some(ref suite) = self.metrics {
suite.compute(&all_scored)
} else {
Vec::new()
};
let duration_ms = elapsed_millis(start, self.clock.monotonic_millis());
let epoch_seconds = self.clock.epoch_seconds();
let run_id = generate_run_id(&opts.tag, epoch_seconds);
let timestamp = i64::try_from(epoch_seconds)
.ok()
.and_then(format_rfc3339)
.unwrap_or_else(|| "1970-01-01T00:00:00Z".to_string());
let result = BenchRunResult {
id: run_id,
schema: schema::schema_url(),
version: schema::version(),
timestamp,
tag: opts.tag.clone(),
duration_ms,
dataset: dataset_info,
metrics: metric_results,
branches: branch_results,
samples: sample_results,
curves: HashMap::new(),
};
if let Some(ref storage) = self.storage {
storage.save(&result)?;
}
for reporter in &self.reporters {
let mut buf = Vec::new();
if let Err(e) = reporter.generate(&mut buf, &result) {
tracing::warn!(
reporter = reporter.name(),
error = %e,
"Report generation failed"
);
}
}
if let (Some(comparator), Some(storage)) = (&self.comparator, &self.storage)
&& let Ok(prev) = storage.latest()
{
let diff = comparator.compare(&prev, &result);
if opts.fail_on_regression && diff.has_regression() {
let error = AppError::new(
ErrorCode::Internal,
format!("Regression detected:\n{}", diff.summary()),
);
return Err(error);
}
tracing::info!("{}", diff.summary());
}
Ok(result)
}
}