use std::sync::atomic::{AtomicU32, AtomicUsize, Ordering};
use std::sync::Arc;
use std::time::Duration;
use backon::{ExponentialBuilder, Retryable};
use futures::future::join_all;
use reqwest::Client;
use tokio::sync::Semaphore;
use tracing::{debug, instrument};
use crate::error::BenchError;
use crate::providers::{run_anthropic, run_openai};
use crate::types::{BenchConfig, BenchFailure, BenchResult, ProviderConfig};
pub const DEFAULT_OPENAI_BASE_URL: &str = "https://api.openai.com";
pub const DEFAULT_ANTHROPIC_BASE_URL: &str = "https://api.anthropic.com";
pub const DEFAULT_OLLAMA_BASE_URL: &str = "http://localhost:11434";
pub const DEFAULT_OPENROUTER_BASE_URL: &str = "https://openrouter.ai/api/v1";
pub const DEFAULT_GEMINI_BASE_URL: &str =
"https://generativelanguage.googleapis.com/v1beta/openai";
const MAX_RETRY_AFTER: Duration = Duration::from_secs(60);
pub struct BenchRunner {
client: Client,
}
impl BenchRunner {
pub fn new() -> Result<Self, BenchError> {
let client = Client::builder()
.timeout(Duration::from_secs(120))
.build()
.map_err(BenchError::Http)?;
Ok(Self { client })
}
#[instrument(skip(self, config, on_progress))]
pub async fn run_detailed<F>(
&self,
config: &BenchConfig,
on_progress: F,
) -> Result<(Vec<BenchResult>, Vec<BenchFailure>), BenchError>
where
F: Fn(usize, usize) + Send + Sync + 'static,
{
let semaphore = Arc::new(Semaphore::new(config.concurrency.max(1)));
let on_progress = Arc::new(on_progress);
let mut tasks = Vec::new();
for provider in &config.providers {
for prompt in &config.prompts {
for run_idx in 0..config.runs_per_prompt {
tasks.push((provider.clone(), prompt.clone(), run_idx));
}
}
}
for provider in &config.providers {
for _ in 0..config.warmup {
let prompt = config.prompts.first().map(String::as_str).unwrap_or("");
let r = dispatch_call(&self.client, provider, prompt, 0, config.stream).await;
debug!(model = %provider.model, ok = r.is_ok(), "warm-up request");
}
}
let total = tasks.len();
debug!(total_tasks = total, "starting benchmark run");
let completed = Arc::new(AtomicUsize::new(0));
let (stream, retries) = (config.stream, config.retries);
let futures: Vec<_> = tasks
.into_iter()
.map(|(provider, prompt, run_idx)| {
let sem = Arc::clone(&semaphore);
let cb = Arc::clone(&on_progress);
let done = Arc::clone(&completed);
let client = self.client.clone();
tokio::spawn(async move {
let _permit = sem.acquire().await.map_err(|e| BenchError::Concurrency {
reason: e.to_string(),
})?;
let result =
call_with_retries(&client, &provider, &prompt, run_idx, stream, retries)
.await;
let n = done.fetch_add(1, Ordering::Relaxed) + 1;
cb(n, total);
Ok::<_, BenchError>(result.map_err(|e| BenchFailure {
provider: provider.name.clone(),
model: provider.model.clone(),
error: e.to_string(),
}))
})
})
.collect();
let join_results = join_all(futures).await;
let mut results = Vec::with_capacity(join_results.len());
let mut failures = Vec::new();
for jr in join_results {
match jr {
Ok(Ok(Ok(r))) => results.push(r),
Ok(Ok(Err(f))) => {
debug!(error = %f.error, "benchmark task returned error");
failures.push(f);
}
Ok(Err(e)) => return Err(e),
Err(e) => {
return Err(BenchError::Concurrency {
reason: format!("task panicked or was cancelled: {e}"),
});
}
}
}
Ok((results, failures))
}
}
pub fn is_retryable(e: &BenchError) -> bool {
match e {
BenchError::RateLimited { .. } => true,
BenchError::ApiError { status, .. } => {
matches!(status, 408 | 409 | 500 | 502 | 503 | 504 | 529)
}
BenchError::Http(e) => e.is_timeout() || e.is_connect(),
_ => false,
}
}
async fn call_with_retries(
client: &Client,
provider: &ProviderConfig,
prompt: &str,
run_idx: u32,
stream: bool,
retries: u32,
) -> Result<BenchResult, BenchError> {
let attempts = AtomicU32::new(0);
let backoff = ExponentialBuilder::default()
.with_min_delay(Duration::from_millis(500))
.with_max_delay(Duration::from_secs(30))
.with_max_times(retries as usize)
.with_jitter();
let mut result = (|| {
attempts.fetch_add(1, Ordering::Relaxed);
dispatch_call(client, provider, prompt, run_idx, stream)
})
.retry(backoff)
.when(is_retryable)
.adjust(|e: &BenchError, dur| match (dur, e) {
(None, _) => None,
(
Some(_),
BenchError::RateLimited {
retry_after_secs: Some(s),
..
},
) => Some(Duration::from_secs(*s).min(MAX_RETRY_AFTER)),
(d, _) => d,
})
.await;
if let Ok(r) = result.as_mut() {
r.retries = attempts.load(Ordering::Relaxed).saturating_sub(1);
}
result
}
pub fn default_base_url(provider: &str) -> Option<&'static str> {
Some(match provider {
"openai" => DEFAULT_OPENAI_BASE_URL,
"anthropic" => DEFAULT_ANTHROPIC_BASE_URL,
"ollama" => DEFAULT_OLLAMA_BASE_URL,
"openrouter" => DEFAULT_OPENROUTER_BASE_URL,
"gemini" => DEFAULT_GEMINI_BASE_URL,
_ => return None,
})
}
async fn dispatch_call(
client: &Client,
provider: &ProviderConfig,
prompt: &str,
run_idx: u32,
stream: bool,
) -> Result<BenchResult, BenchError> {
let Some(default) = default_base_url(&provider.name) else {
return Err(BenchError::InvalidConfig {
reason: format!(
"unknown provider '{}'; supported: openai, anthropic, ollama, openrouter, gemini",
provider.name
),
});
};
let url = if provider.base_url.is_empty() {
default.to_owned()
} else {
provider.base_url.clone()
};
if provider.name == "anthropic" {
run_anthropic(client, provider, &url, prompt, run_idx, stream).await
} else {
run_openai(client, provider, &url, prompt, run_idx, stream).await
}
}
#[cfg(test)]
mod tests {
use super::*;
use wiremock::matchers::{method, path};
use wiremock::{Mock, MockServer, ResponseTemplate};
use crate::types::BenchConfig;
fn openai_provider(base_url: &str) -> ProviderConfig {
let _ = base_url; ProviderConfig {
name: "openai".into(),
model: "gpt-4o-mini".into(),
api_key: "test-key".into(),
base_url: String::new(),
max_tokens: 32,
}
}
#[test]
fn test_bench_runner_new_succeeds() {
let runner = BenchRunner::new();
assert!(runner.is_ok(), "BenchRunner::new should succeed");
}
#[tokio::test]
async fn test_dispatch_call_unknown_provider_returns_invalid_config() {
let client = Client::new();
let provider = ProviderConfig {
name: "unknown-provider".into(),
model: "model-x".into(),
api_key: "key".into(),
base_url: String::new(),
max_tokens: 32,
};
let result = dispatch_call(&client, &provider, "hello", 0, false).await;
assert!(result.is_err());
assert!(
matches!(result.unwrap_err(), BenchError::InvalidConfig { .. }),
"expected InvalidConfig for unknown provider"
);
}
#[tokio::test]
async fn test_runner_task_count_equals_providers_times_prompts_times_runs() {
let num_providers = 2usize;
let num_prompts = 3usize;
let runs_per_prompt = 4u32;
let expected = num_providers * num_prompts * runs_per_prompt as usize;
assert_eq!(expected, 24);
}
#[tokio::test]
async fn test_runner_empty_providers_returns_empty_results() {
let runner = BenchRunner::new().unwrap_or_else(|_| unreachable!());
let config = BenchConfig {
prompts: vec!["hello".into()],
runs_per_prompt: 2,
concurrency: 4,
providers: vec![], stream: false,
retries: 0,
warmup: 0,
};
let results = runner
.run_detailed(&config, |_, _| {})
.await
.map(|(r, _)| r);
assert!(results.is_ok());
assert!(results.unwrap_or_default().is_empty());
}
#[tokio::test]
async fn test_runner_empty_prompts_returns_empty_results() {
let runner = BenchRunner::new().unwrap_or_else(|_| unreachable!());
let config = BenchConfig {
prompts: vec![], runs_per_prompt: 2,
concurrency: 4,
providers: vec![openai_provider("")],
stream: false,
retries: 0,
warmup: 0,
};
let results = runner
.run_detailed(&config, |_, _| {})
.await
.map(|(r, _)| r);
assert!(results.is_ok());
assert!(results.unwrap_or_default().is_empty());
}
#[tokio::test]
async fn test_runner_progress_callback_called_correct_times() {
let server = MockServer::start().await;
Mock::given(method("POST"))
.and(path("/v1/chat/completions"))
.respond_with(ResponseTemplate::new(200).set_body_json(serde_json::json!({
"choices": [{"message": {"role": "assistant", "content": "hi"}}],
"usage": {"prompt_tokens": 5, "completion_tokens": 3}
})))
.mount(&server)
.await;
let call_count = Arc::new(std::sync::atomic::AtomicUsize::new(0));
let cc = Arc::clone(&call_count);
let cb = move |_done: usize, _total: usize| {
cc.fetch_add(1, std::sync::atomic::Ordering::Relaxed);
};
for i in 1..=3usize {
cb(i, 3);
}
assert_eq!(call_count.load(std::sync::atomic::Ordering::Relaxed), 3);
}
#[tokio::test]
async fn test_semaphore_limits_concurrency() {
let sem = Arc::new(Semaphore::new(2));
let p1 = sem.try_acquire();
let p2 = sem.try_acquire();
let p3 = sem.try_acquire();
assert!(p1.is_ok(), "first permit should succeed");
assert!(p2.is_ok(), "second permit should succeed");
assert!(p3.is_err(), "third permit should fail (semaphore full)");
}
#[test]
fn test_bench_config_with_multiple_providers_is_valid() {
let config = BenchConfig {
prompts: vec!["p1".into()],
runs_per_prompt: 1,
concurrency: 2,
providers: vec![
ProviderConfig {
name: "openai".into(),
model: "gpt-4o-mini".into(),
api_key: "k1".into(),
base_url: String::new(),
max_tokens: 128,
},
ProviderConfig {
name: "anthropic".into(),
model: "claude-haiku-4-5".into(),
api_key: "k2".into(),
base_url: String::new(),
max_tokens: 128,
},
],
stream: false,
retries: 0,
warmup: 0,
};
assert_eq!(config.providers.len(), 2);
}
#[tokio::test]
async fn test_dispatch_openai_hits_openai_endpoint() {
let server = MockServer::start().await;
Mock::given(method("POST"))
.and(path("/v1/chat/completions"))
.respond_with(ResponseTemplate::new(200).set_body_json(serde_json::json!({
"choices": [{"message": {"role": "assistant", "content": "ok"}}],
"usage": {"prompt_tokens": 2, "completion_tokens": 1}
})))
.mount(&server)
.await;
let client = Client::new();
let provider = ProviderConfig {
name: "openai".into(),
model: "gpt-4o-mini".into(),
api_key: "k".into(),
base_url: String::new(),
max_tokens: 16,
};
let result = run_openai(&client, &provider, &server.uri(), "hi", 0, false).await;
assert!(result.is_ok());
}
#[tokio::test]
async fn test_dispatch_anthropic_hits_anthropic_endpoint() {
let server = MockServer::start().await;
Mock::given(method("POST"))
.and(path("/v1/messages"))
.respond_with(ResponseTemplate::new(200).set_body_json(serde_json::json!({
"content": [{"type": "text", "text": "ok"}],
"usage": {"input_tokens": 2, "output_tokens": 1}
})))
.mount(&server)
.await;
let client = Client::new();
let provider = ProviderConfig {
name: "anthropic".into(),
model: "claude-haiku-4-5".into(),
api_key: "k".into(),
base_url: String::new(),
max_tokens: 16,
};
let result = run_anthropic(&client, &provider, &server.uri(), "hi", 0, false).await;
assert!(result.is_ok());
}
#[tokio::test]
async fn test_run_detailed_uses_custom_base_url_and_counts_failures() {
let server = MockServer::start().await;
Mock::given(method("POST"))
.and(path("/v1/chat/completions"))
.respond_with(ResponseTemplate::new(200).set_body_json(serde_json::json!({
"choices": [{"message": {"role": "assistant", "content": "ok"}}],
"usage": {"prompt_tokens": 4, "completion_tokens": 2}
})))
.mount(&server)
.await;
let good = ProviderConfig {
name: "openai".into(),
model: "local-model".into(),
api_key: String::new(),
base_url: server.uri(),
max_tokens: 16,
};
let bad = ProviderConfig {
name: "anthropic".into(),
model: "claude-haiku-4-5".into(),
api_key: String::new(),
base_url: server.uri(), max_tokens: 16,
};
let config = BenchConfig {
prompts: vec!["hi".into()],
runs_per_prompt: 2,
concurrency: 2,
providers: vec![good, bad],
stream: false,
retries: 0,
warmup: 0,
};
let runner = BenchRunner::new().unwrap_or_else(|_| unreachable!());
let (results, failures) = runner
.run_detailed(&config, |_, _| {})
.await
.unwrap_or_default();
assert_eq!(results.len(), 2);
assert_eq!(failures.len(), 2);
assert!(failures.iter().all(|f| f.error.contains("404")));
}
#[test]
fn test_atomic_counter_increments_correctly() {
let counter = Arc::new(std::sync::atomic::AtomicUsize::new(0));
for _ in 0..10 {
counter.fetch_add(1, std::sync::atomic::Ordering::Relaxed);
}
assert_eq!(counter.load(std::sync::atomic::Ordering::Relaxed), 10);
}
fn local_openai(server: &MockServer) -> ProviderConfig {
ProviderConfig {
name: "openai".into(),
model: "local".into(),
api_key: String::new(),
base_url: server.uri(),
max_tokens: 8,
}
}
fn ok_body() -> ResponseTemplate {
ResponseTemplate::new(200).set_body_json(serde_json::json!({
"choices": [{"message": {"role": "assistant", "content": "ok"}}],
"usage": {"prompt_tokens": 1, "completion_tokens": 1}
}))
}
fn one_request(provider: ProviderConfig, retries: u32) -> BenchConfig {
BenchConfig {
prompts: vec!["hi".into()],
runs_per_prompt: 1,
concurrency: 1,
providers: vec![provider],
stream: false,
retries,
warmup: 0,
}
}
#[tokio::test]
async fn test_rate_limit_is_retried_after_retry_after() {
let server = MockServer::start().await;
Mock::given(method("POST"))
.and(path("/v1/chat/completions"))
.respond_with(ResponseTemplate::new(429).insert_header("retry-after", "1"))
.up_to_n_times(1)
.expect(1)
.mount(&server)
.await;
Mock::given(method("POST"))
.and(path("/v1/chat/completions"))
.respond_with(ok_body())
.expect(1)
.mount(&server)
.await;
let start = std::time::Instant::now();
let (results, failures) = BenchRunner::new()
.unwrap()
.run_detailed(&one_request(local_openai(&server), 2), |_, _| {})
.await
.unwrap();
assert_eq!((results.len(), failures.len()), (1, 0));
assert_eq!(results[0].retries, 1);
assert!(start.elapsed() >= Duration::from_millis(950), "{:?}", start.elapsed());
}
#[tokio::test]
async fn test_without_retries_a_429_is_a_failure() {
let server = MockServer::start().await;
Mock::given(method("POST"))
.and(path("/v1/chat/completions"))
.respond_with(ResponseTemplate::new(429))
.expect(1)
.mount(&server)
.await;
let (results, failures) = BenchRunner::new()
.unwrap()
.run_detailed(&one_request(local_openai(&server), 0), |_, _| {})
.await
.unwrap();
assert_eq!((results.len(), failures.len()), (0, 1));
}
#[tokio::test]
async fn test_server_errors_retry_but_bad_requests_do_not() {
let server = MockServer::start().await;
Mock::given(method("POST"))
.and(path("/v1/chat/completions"))
.respond_with(ResponseTemplate::new(503))
.expect(3)
.mount(&server)
.await;
let (_, failures) = BenchRunner::new()
.unwrap()
.run_detailed(&one_request(local_openai(&server), 2), |_, _| {})
.await
.unwrap();
assert_eq!(failures.len(), 1);
assert!(failures[0].error.contains("503"), "{}", failures[0].error);
let server = MockServer::start().await;
Mock::given(method("POST"))
.and(path("/v1/chat/completions"))
.respond_with(ResponseTemplate::new(404))
.expect(1)
.mount(&server)
.await;
let (_, failures) = BenchRunner::new()
.unwrap()
.run_detailed(&one_request(local_openai(&server), 3), |_, _| {})
.await
.unwrap();
assert_eq!(failures.len(), 1);
}
#[tokio::test]
async fn test_transport_errors_say_why() {
let mut p = local_openai_unreachable();
p.base_url = "http://127.0.0.1:9".into();
let (_, failures) = BenchRunner::new()
.unwrap()
.run_detailed(&one_request(p, 0), |_, _| {})
.await
.unwrap();
let msg = &failures[0].error;
assert!(msg.contains("error sending request"), "{msg}");
assert!(msg.matches(": ").count() >= 2, "{msg}");
}
fn local_openai_unreachable() -> ProviderConfig {
ProviderConfig {
name: "openai".into(),
model: "local".into(),
api_key: String::new(),
base_url: String::new(),
max_tokens: 8,
}
}
#[test]
fn test_default_base_urls() {
assert_eq!(default_base_url("ollama"), Some(DEFAULT_OLLAMA_BASE_URL));
assert_eq!(default_base_url("gemini"), Some(DEFAULT_GEMINI_BASE_URL));
assert_eq!(default_base_url("nope"), None);
}
}