use serde::{Deserialize, Serialize};
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct BenchResult {
pub provider: String,
pub model: String,
pub prompt: String,
pub latency_ms: u64,
pub total_ms: u64,
pub prompt_tokens: u32,
pub completion_tokens: u32,
pub cost_usd: f64,
pub tokens_per_second: f64,
pub response_text: String,
pub run_index: u32,
}
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct BenchFailure {
pub provider: String,
pub model: String,
pub error: String,
}
#[derive(Debug, Clone)]
pub struct BenchConfig {
pub prompts: Vec<String>,
pub runs_per_prompt: u32,
pub concurrency: usize,
pub providers: Vec<ProviderConfig>,
}
#[derive(Debug, Clone)]
pub struct ProviderConfig {
pub name: String,
pub model: String,
pub api_key: String,
pub base_url: String,
pub max_tokens: u32,
}
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct BenchSummary {
pub provider: String,
pub model: String,
pub p50_latency_ms: u64,
pub p99_latency_ms: u64,
pub avg_tokens_per_sec: f64,
pub avg_cost_usd: f64,
pub total_cost_usd: f64,
pub success_rate: f64,
}
#[cfg(test)]
mod tests {
use super::*;
fn sample_result(provider: &str, model: &str, latency: u64, cost: f64) -> BenchResult {
BenchResult {
provider: provider.into(),
model: model.into(),
prompt: "hello world".into(),
latency_ms: latency,
total_ms: latency + 50,
prompt_tokens: 10,
completion_tokens: 20,
cost_usd: cost,
tokens_per_second: 40.0,
response_text: "Hi!".into(),
run_index: 0,
}
}
#[test]
fn test_bench_result_fields_are_accessible() {
let r = sample_result("openai", "gpt-4o-mini", 120, 0.0001);
assert_eq!(r.provider, "openai");
assert_eq!(r.model, "gpt-4o-mini");
assert_eq!(r.latency_ms, 120);
assert_eq!(r.total_ms, 170);
assert_eq!(r.prompt_tokens, 10);
assert_eq!(r.completion_tokens, 20);
assert!((r.cost_usd - 0.0001).abs() < 1e-10);
assert!((r.tokens_per_second - 40.0).abs() < 1e-10);
}
#[test]
fn test_bench_result_serialises_and_deserialises() {
let r = sample_result("anthropic", "claude-haiku-4-5", 200, 0.0002);
let json = serde_json::to_string(&r).unwrap_or_default();
assert!(!json.is_empty(), "serialised JSON should not be empty");
let restored: Result<BenchResult, _> = serde_json::from_str(&json);
assert!(restored.is_ok(), "should deserialise cleanly");
let restored = restored.unwrap_or_else(|_| sample_result("x", "y", 0, 0.0));
assert_eq!(restored.provider, r.provider);
assert_eq!(restored.model, r.model);
assert_eq!(restored.latency_ms, r.latency_ms);
}
#[test]
fn test_bench_result_is_clone() {
let r = sample_result("openai", "gpt-4o", 300, 0.001);
let cloned = r.clone();
assert_eq!(cloned.provider, r.provider);
assert_eq!(cloned.latency_ms, r.latency_ms);
}
#[test]
fn test_bench_result_run_index_preserved() {
let mut r = sample_result("openai", "gpt-4o", 100, 0.0);
r.run_index = 7;
assert_eq!(r.run_index, 7);
}
#[test]
fn test_bench_result_tokens_per_second_zero_edge_case() {
let mut r = sample_result("openai", "gpt-4o", 0, 0.0);
r.tokens_per_second = 0.0;
assert_eq!(r.tokens_per_second, 0.0);
}
fn sample_config() -> BenchConfig {
BenchConfig {
prompts: vec!["Say hi".into(), "Explain Rust".into()],
runs_per_prompt: 3,
concurrency: 4,
providers: vec![ProviderConfig {
name: "openai".into(),
model: "gpt-4o-mini".into(),
api_key: "sk-test".into(),
base_url: String::new(),
max_tokens: 512,
}],
}
}
#[test]
fn test_bench_config_prompts_accessible() {
let c = sample_config();
assert_eq!(c.prompts.len(), 2);
assert_eq!(c.prompts[0], "Say hi");
}
#[test]
fn test_bench_config_runs_per_prompt() {
let c = sample_config();
assert_eq!(c.runs_per_prompt, 3);
}
#[test]
fn test_bench_config_concurrency() {
let c = sample_config();
assert_eq!(c.concurrency, 4);
}
#[test]
fn test_bench_config_providers_accessible() {
let c = sample_config();
assert_eq!(c.providers.len(), 1);
assert_eq!(c.providers[0].name, "openai");
assert_eq!(c.providers[0].model, "gpt-4o-mini");
assert_eq!(c.providers[0].max_tokens, 512);
}
#[test]
fn test_bench_config_is_clone() {
let c = sample_config();
let cloned = c.clone();
assert_eq!(cloned.prompts.len(), c.prompts.len());
assert_eq!(cloned.runs_per_prompt, c.runs_per_prompt);
}
#[test]
fn test_provider_config_fields() {
let p = ProviderConfig {
name: "anthropic".into(),
model: "claude-sonnet-5".into(),
api_key: "sk-ant-test".into(),
base_url: String::new(),
max_tokens: 1024,
};
assert_eq!(p.name, "anthropic");
assert_eq!(p.model, "claude-sonnet-5");
assert_eq!(p.api_key, "sk-ant-test");
assert_eq!(p.max_tokens, 1024);
}
fn sample_summary() -> BenchSummary {
BenchSummary {
provider: "openai".into(),
model: "gpt-4o-mini".into(),
p50_latency_ms: 150,
p99_latency_ms: 400,
avg_tokens_per_sec: 55.0,
avg_cost_usd: 0.0003,
total_cost_usd: 0.0009,
success_rate: 1.0,
}
}
#[test]
fn test_bench_summary_p50_less_than_p99() {
let s = sample_summary();
assert!(
s.p50_latency_ms <= s.p99_latency_ms,
"p50 should not exceed p99"
);
}
#[test]
fn test_bench_summary_success_rate_in_range() {
let s = sample_summary();
assert!(
(0.0..=1.0).contains(&s.success_rate),
"success rate must be in [0,1]"
);
}
#[test]
fn test_bench_summary_total_cost_gte_avg() {
let s = sample_summary();
assert!(
s.total_cost_usd >= s.avg_cost_usd,
"total cost should be >= average cost"
);
}
#[test]
fn test_bench_summary_serialises() {
let s = sample_summary();
let json = serde_json::to_string(&s).unwrap_or_default();
assert!(json.contains("openai"), "JSON should contain provider");
assert!(
json.contains("p50_latency_ms"),
"JSON should contain field name"
);
}
#[test]
fn test_bench_summary_is_clone() {
let s = sample_summary();
let cloned = s.clone();
assert_eq!(cloned.model, s.model);
assert_eq!(cloned.p50_latency_ms, s.p50_latency_ms);
}
#[test]
fn test_bench_summary_zero_success_rate() {
let mut s = sample_summary();
s.success_rate = 0.0;
assert_eq!(s.success_rate, 0.0);
}
#[test]
fn test_bench_summary_avg_tokens_per_sec_positive() {
let s = sample_summary();
assert!(s.avg_tokens_per_sec > 0.0, "avg tps should be positive");
}
#[test]
fn test_bench_summary_provider_non_empty() {
let s = sample_summary();
assert!(!s.provider.is_empty());
}
#[test]
fn test_bench_summary_model_non_empty() {
let s = sample_summary();
assert!(!s.model.is_empty());
}
#[test]
fn test_bench_result_prompt_stored() {
let r = sample_result("openai", "gpt-4o", 200, 0.001);
assert_eq!(r.prompt, "hello world");
}
#[test]
fn test_bench_result_total_ms_gte_latency() {
let r = sample_result("openai", "gpt-4o", 100, 0.001);
assert!(r.total_ms >= r.latency_ms);
}
#[test]
fn test_bench_config_empty_prompts_allowed() {
let c = BenchConfig {
prompts: vec![],
runs_per_prompt: 1,
concurrency: 1,
providers: vec![],
};
assert!(c.prompts.is_empty());
}
#[test]
fn test_bench_config_zero_runs_allowed() {
let c = BenchConfig {
prompts: vec!["hi".into()],
runs_per_prompt: 0,
concurrency: 1,
providers: vec![],
};
assert_eq!(c.runs_per_prompt, 0);
}
#[test]
fn test_bench_result_deserialise_missing_field_fails() {
let bad_json = r#"{"provider":"openai"}"#;
let result: Result<BenchResult, _> = serde_json::from_str(bad_json);
assert!(
result.is_err(),
"incomplete JSON should fail to deserialise"
);
}
#[test]
fn test_bench_summary_avg_cost_non_negative() {
let s = sample_summary();
assert!(s.avg_cost_usd >= 0.0);
}
#[test]
fn test_bench_summary_total_cost_non_negative() {
let s = sample_summary();
assert!(s.total_cost_usd >= 0.0);
}
}