use std::collections::BTreeMap;
use serde::{Deserialize, Serialize};
use crate::types::{BenchFailure, BenchResult};
#[derive(Debug, Clone, Copy, PartialEq, Serialize, Deserialize)]
#[non_exhaustive]
pub struct Stats {
pub count: usize,
pub mean: f64,
pub stddev: f64,
pub min: f64,
pub p50: f64,
pub p90: f64,
pub p95: f64,
pub p99: f64,
pub max: f64,
}
impl Stats {
pub fn of(samples: impl IntoIterator<Item = f64>) -> Option<Stats> {
let mut v: Vec<f64> = samples.into_iter().filter(|x| x.is_finite()).collect();
if v.is_empty() {
return None;
}
v.sort_by(f64::total_cmp);
let n = v.len();
let mean = v.iter().sum::<f64>() / n as f64;
let stddev = if n > 1 {
(v.iter().map(|x| (x - mean).powi(2)).sum::<f64>() / (n - 1) as f64).sqrt()
} else {
0.0
};
let rank = |p: usize| v[(p * n).div_ceil(100).saturating_sub(1).min(n - 1)];
Some(Stats {
count: n,
mean,
stddev,
min: v[0],
p50: rank(50),
p90: rank(90),
p95: rank(95),
p99: rank(99),
max: v[n - 1],
})
}
}
#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
#[non_exhaustive]
pub struct ModelReport {
pub provider: String,
pub model: String,
pub succeeded: usize,
pub failed: usize,
pub latency_ms: Option<Stats>,
pub total_ms: Option<Stats>,
pub ttft_ms: Option<Stats>,
pub tpot_ms: Option<Stats>,
pub tokens_per_second: Option<Stats>,
pub total_cost_usd: f64,
pub retries: u32,
#[serde(default)]
pub error_rate: f64,
#[serde(default)]
pub cost_per_request_usd: Option<f64>,
}
pub fn model_reports(results: &[BenchResult], failures: &[BenchFailure]) -> Vec<ModelReport> {
let mut groups: BTreeMap<(String, String), (Vec<&BenchResult>, usize)> = BTreeMap::new();
for r in results {
groups
.entry((r.provider.clone(), r.model.clone()))
.or_default()
.0
.push(r);
}
for f in failures {
groups
.entry((f.provider.clone(), f.model.clone()))
.or_default()
.1 += 1;
}
groups
.into_iter()
.map(|((provider, model), (rs, failed))| ModelReport {
error_rate: if rs.len() + failed > 0 {
failed as f64 / (rs.len() + failed) as f64
} else {
0.0
},
cost_per_request_usd: {
let priced: Vec<f64> = rs
.iter()
.filter(|r| r.cost_source.is_some())
.map(|r| r.cost_usd)
.collect();
(!priced.is_empty()).then(|| priced.iter().sum::<f64>() / priced.len() as f64)
},
provider,
model,
succeeded: rs.len(),
failed,
latency_ms: Stats::of(rs.iter().map(|r| r.latency_ms as f64)),
total_ms: Stats::of(rs.iter().map(|r| r.total_ms as f64)),
ttft_ms: Stats::of(rs.iter().filter_map(|r| r.ttft_ms.map(|t| t as f64))),
tpot_ms: Stats::of(rs.iter().filter_map(|r| r.tpot_ms)),
tokens_per_second: Stats::of(
rs.iter()
.filter(|r| r.completion_tokens > 0)
.map(|r| r.tokens_per_second),
),
total_cost_usd: rs.iter().map(|r| r.cost_usd).sum(),
retries: rs.iter().map(|r| r.retries).sum(),
})
.collect()
}
#[cfg(test)]
mod tests {
use super::*;
use proptest::prelude::*;
#[test]
fn known_values() {
let s = Stats::of((1..=100).map(f64::from)).unwrap();
assert_eq!((s.min, s.p50, s.p90, s.p95, s.p99, s.max), (1.0, 50.0, 90.0, 95.0, 99.0, 100.0));
assert!((s.mean - 50.5).abs() < 1e-12);
assert!((s.stddev - 29.011_491_975_882_016).abs() < 1e-9);
let one = Stats::of([7.0]).unwrap();
assert_eq!((one.p50, one.p99, one.stddev), (7.0, 7.0, 0.0));
assert!(Stats::of([]).is_none());
assert!(Stats::of([f64::NAN]).is_none());
}
#[test]
fn error_rate_and_cost_per_request() {
use crate::types::CostSource;
let ok = |cost: f64, source: Option<CostSource>| BenchResult {
provider: "p".into(),
model: "m".into(),
prompt: String::new(),
latency_ms: 10,
total_ms: 10,
prompt_tokens: 1,
completion_tokens: 1,
cost_usd: cost,
tokens_per_second: 100.0,
response_text: String::new(),
run_index: 0,
ttft_ms: None,
retries: 0,
tpot_ms: None,
cached_prompt_tokens: 0,
cache_write_prompt_tokens: 0,
cost_source: source,
};
let fail = BenchFailure {
provider: "p".into(),
model: "m".into(),
error: "HTTP 500".into(),
};
let priced = [ok(0.002, Some(CostSource::LlmPrices)), ok(0.004, Some(CostSource::LlmPrices)), ok(0.0, None)];
let r = &model_reports(&priced, std::slice::from_ref(&fail))[0];
assert!((r.error_rate - 0.25).abs() < 1e-12);
assert!((r.cost_per_request_usd.unwrap() - 0.003).abs() < 1e-12);
let unpriced = [ok(0.0, None)];
let r = &model_reports(&unpriced, &[])[0];
assert_eq!((r.error_rate, r.cost_per_request_usd), (0.0, None));
let r = &model_reports(&[], &[fail])[0];
assert_eq!((r.error_rate, r.cost_per_request_usd), (1.0, None));
}
proptest! {
#[test]
fn percentiles_are_ordered_samples(v in prop::collection::vec(0.0f64..1e7, 1..300)) {
let s = Stats::of(v.clone()).unwrap();
prop_assert!(s.min <= s.p50 && s.p50 <= s.p90 && s.p90 <= s.p95 && s.p95 <= s.p99 && s.p99 <= s.max);
for p in [s.p50, s.p90, s.p95, s.p99] {
prop_assert!(v.contains(&p), "percentile {} is not a sample", p);
}
prop_assert!(s.mean >= s.min - 1e-6 && s.mean <= s.max + 1e-6);
prop_assert!(s.stddev >= 0.0);
}
}
}