use std::sync::Arc;
use futures::future::join_all;
use reqwest::Client;
use tokio::sync::Semaphore;
use tracing::{debug, instrument};
use crate::error::BenchError;
use crate::providers::{run_anthropic, run_openai};
use crate::types::{BenchConfig, BenchFailure, BenchResult, ProviderConfig};
pub const DEFAULT_OPENAI_BASE_URL: &str = "https://api.openai.com";
pub const DEFAULT_ANTHROPIC_BASE_URL: &str = "https://api.anthropic.com";
pub struct BenchRunner {
client: Client,
}
impl BenchRunner {
pub fn new() -> Result<Self, BenchError> {
let client = Client::builder()
.timeout(std::time::Duration::from_secs(120))
.build()
.map_err(BenchError::Http)?;
Ok(Self { client })
}
#[instrument(skip(self, config, on_progress))]
pub async fn run_detailed<F>(
&self,
config: &BenchConfig,
on_progress: F,
) -> Result<(Vec<BenchResult>, Vec<BenchFailure>), BenchError>
where
F: Fn(usize, usize) + Send + Sync + 'static,
{
let semaphore = Arc::new(Semaphore::new(config.concurrency));
let on_progress = Arc::new(on_progress);
let mut tasks = Vec::new();
for provider in &config.providers {
for prompt in &config.prompts {
for run_idx in 0..config.runs_per_prompt {
tasks.push((provider.clone(), prompt.clone(), run_idx));
}
}
}
let total = tasks.len();
debug!(total_tasks = total, "starting benchmark run");
let completed = Arc::new(std::sync::atomic::AtomicUsize::new(0));
let client = self.client.clone();
let futures: Vec<_> = tasks
.into_iter()
.map(|(provider, prompt, run_idx)| {
let sem = Arc::clone(&semaphore);
let cb = Arc::clone(&on_progress);
let done = Arc::clone(&completed);
let client = client.clone();
tokio::spawn(async move {
let _permit = sem.acquire().await.map_err(|e| BenchError::Concurrency {
reason: e.to_string(),
})?;
let result = dispatch_call(&client, &provider, &prompt, run_idx).await;
let n = done.fetch_add(1, std::sync::atomic::Ordering::Relaxed) + 1;
cb(n, total);
Ok::<_, BenchError>(result.map_err(|e| BenchFailure {
provider: provider.name.clone(),
model: provider.model.clone(),
error: e.to_string(),
}))
})
})
.collect();
let join_results = join_all(futures).await;
let mut results = Vec::with_capacity(join_results.len());
let mut failures = Vec::new();
for jr in join_results {
match jr {
Ok(Ok(Ok(r))) => results.push(r),
Ok(Ok(Err(f))) => {
debug!(error = %f.error, "benchmark task returned error");
failures.push(f);
}
Ok(Err(e)) => {
return Err(e);
}
Err(e) => {
return Err(BenchError::Concurrency {
reason: format!("task panicked or was cancelled: {e}"),
});
}
}
}
Ok((results, failures))
}
}
async fn dispatch_call(
client: &Client,
provider: &ProviderConfig,
prompt: &str,
run_idx: u32,
) -> Result<BenchResult, BenchError> {
let base = |default: &'static str| -> String {
if provider.base_url.is_empty() {
default.to_owned()
} else {
provider.base_url.clone()
}
};
match provider.name.as_str() {
"openai" => {
let url = base(DEFAULT_OPENAI_BASE_URL);
run_openai(client, provider, &url, prompt, run_idx).await
}
"anthropic" => {
let url = base(DEFAULT_ANTHROPIC_BASE_URL);
run_anthropic(client, provider, &url, prompt, run_idx).await
}
other => Err(BenchError::InvalidConfig {
reason: format!("unknown provider '{other}'; supported: openai, anthropic"),
}),
}
}
#[cfg(test)]
mod tests {
use super::*;
use wiremock::matchers::{method, path};
use wiremock::{Mock, MockServer, ResponseTemplate};
use crate::types::BenchConfig;
fn openai_provider(base_url: &str) -> ProviderConfig {
let _ = base_url; ProviderConfig {
name: "openai".into(),
model: "gpt-4o-mini".into(),
api_key: "test-key".into(),
base_url: String::new(),
max_tokens: 32,
}
}
#[test]
fn test_bench_runner_new_succeeds() {
let runner = BenchRunner::new();
assert!(runner.is_ok(), "BenchRunner::new should succeed");
}
#[tokio::test]
async fn test_dispatch_call_unknown_provider_returns_invalid_config() {
let client = Client::new();
let provider = ProviderConfig {
name: "unknown-provider".into(),
model: "model-x".into(),
api_key: "key".into(),
base_url: String::new(),
max_tokens: 32,
};
let result = dispatch_call(&client, &provider, "hello", 0).await;
assert!(result.is_err());
assert!(
matches!(result.unwrap_err(), BenchError::InvalidConfig { .. }),
"expected InvalidConfig for unknown provider"
);
}
#[tokio::test]
async fn test_runner_task_count_equals_providers_times_prompts_times_runs() {
let num_providers = 2usize;
let num_prompts = 3usize;
let runs_per_prompt = 4u32;
let expected = num_providers * num_prompts * runs_per_prompt as usize;
assert_eq!(expected, 24);
}
#[tokio::test]
async fn test_runner_empty_providers_returns_empty_results() {
let runner = BenchRunner::new().unwrap_or_else(|_| unreachable!());
let config = BenchConfig {
prompts: vec!["hello".into()],
runs_per_prompt: 2,
concurrency: 4,
providers: vec![], };
let results = runner
.run_detailed(&config, |_, _| {})
.await
.map(|(r, _)| r);
assert!(results.is_ok());
assert!(results.unwrap_or_default().is_empty());
}
#[tokio::test]
async fn test_runner_empty_prompts_returns_empty_results() {
let runner = BenchRunner::new().unwrap_or_else(|_| unreachable!());
let config = BenchConfig {
prompts: vec![], runs_per_prompt: 2,
concurrency: 4,
providers: vec![openai_provider("")],
};
let results = runner
.run_detailed(&config, |_, _| {})
.await
.map(|(r, _)| r);
assert!(results.is_ok());
assert!(results.unwrap_or_default().is_empty());
}
#[tokio::test]
async fn test_runner_progress_callback_called_correct_times() {
let server = MockServer::start().await;
Mock::given(method("POST"))
.and(path("/v1/chat/completions"))
.respond_with(ResponseTemplate::new(200).set_body_json(serde_json::json!({
"choices": [{"message": {"role": "assistant", "content": "hi"}}],
"usage": {"prompt_tokens": 5, "completion_tokens": 3}
})))
.mount(&server)
.await;
let call_count = Arc::new(std::sync::atomic::AtomicUsize::new(0));
let cc = Arc::clone(&call_count);
let cb = move |_done: usize, _total: usize| {
cc.fetch_add(1, std::sync::atomic::Ordering::Relaxed);
};
for i in 1..=3usize {
cb(i, 3);
}
assert_eq!(call_count.load(std::sync::atomic::Ordering::Relaxed), 3);
}
#[tokio::test]
async fn test_semaphore_limits_concurrency() {
let sem = Arc::new(Semaphore::new(2));
let p1 = sem.try_acquire();
let p2 = sem.try_acquire();
let p3 = sem.try_acquire();
assert!(p1.is_ok(), "first permit should succeed");
assert!(p2.is_ok(), "second permit should succeed");
assert!(p3.is_err(), "third permit should fail (semaphore full)");
}
#[test]
fn test_bench_config_with_multiple_providers_is_valid() {
let config = BenchConfig {
prompts: vec!["p1".into()],
runs_per_prompt: 1,
concurrency: 2,
providers: vec![
ProviderConfig {
name: "openai".into(),
model: "gpt-4o-mini".into(),
api_key: "k1".into(),
base_url: String::new(),
max_tokens: 128,
},
ProviderConfig {
name: "anthropic".into(),
model: "claude-haiku-4-5".into(),
api_key: "k2".into(),
base_url: String::new(),
max_tokens: 128,
},
],
};
assert_eq!(config.providers.len(), 2);
}
#[tokio::test]
async fn test_dispatch_openai_hits_openai_endpoint() {
let server = MockServer::start().await;
Mock::given(method("POST"))
.and(path("/v1/chat/completions"))
.respond_with(ResponseTemplate::new(200).set_body_json(serde_json::json!({
"choices": [{"message": {"role": "assistant", "content": "ok"}}],
"usage": {"prompt_tokens": 2, "completion_tokens": 1}
})))
.mount(&server)
.await;
let client = Client::new();
let provider = ProviderConfig {
name: "openai".into(),
model: "gpt-4o-mini".into(),
api_key: "k".into(),
base_url: String::new(),
max_tokens: 16,
};
let result = run_openai(&client, &provider, &server.uri(), "hi", 0).await;
assert!(result.is_ok());
}
#[tokio::test]
async fn test_dispatch_anthropic_hits_anthropic_endpoint() {
let server = MockServer::start().await;
Mock::given(method("POST"))
.and(path("/v1/messages"))
.respond_with(ResponseTemplate::new(200).set_body_json(serde_json::json!({
"content": [{"type": "text", "text": "ok"}],
"usage": {"input_tokens": 2, "output_tokens": 1}
})))
.mount(&server)
.await;
let client = Client::new();
let provider = ProviderConfig {
name: "anthropic".into(),
model: "claude-haiku-4-5".into(),
api_key: "k".into(),
base_url: String::new(),
max_tokens: 16,
};
let result = run_anthropic(&client, &provider, &server.uri(), "hi", 0).await;
assert!(result.is_ok());
}
#[tokio::test]
async fn test_run_detailed_uses_custom_base_url_and_counts_failures() {
let server = MockServer::start().await;
Mock::given(method("POST"))
.and(path("/v1/chat/completions"))
.respond_with(ResponseTemplate::new(200).set_body_json(serde_json::json!({
"choices": [{"message": {"role": "assistant", "content": "ok"}}],
"usage": {"prompt_tokens": 4, "completion_tokens": 2}
})))
.mount(&server)
.await;
let good = ProviderConfig {
name: "openai".into(),
model: "local-model".into(),
api_key: String::new(),
base_url: server.uri(),
max_tokens: 16,
};
let bad = ProviderConfig {
name: "anthropic".into(),
model: "claude-haiku-4-5".into(),
api_key: String::new(),
base_url: server.uri(), max_tokens: 16,
};
let config = BenchConfig {
prompts: vec!["hi".into()],
runs_per_prompt: 2,
concurrency: 2,
providers: vec![good, bad],
};
let runner = BenchRunner::new().unwrap_or_else(|_| unreachable!());
let (results, failures) = runner
.run_detailed(&config, |_, _| {})
.await
.unwrap_or_default();
assert_eq!(results.len(), 2);
assert_eq!(failures.len(), 2);
assert!(failures.iter().all(|f| f.error.contains("404")));
}
#[test]
fn test_atomic_counter_increments_correctly() {
let counter = Arc::new(std::sync::atomic::AtomicUsize::new(0));
for _ in 0..10 {
counter.fetch_add(1, std::sync::atomic::Ordering::Relaxed);
}
assert_eq!(counter.load(std::sync::atomic::Ordering::Relaxed), 10);
}
}