runtime/benchmark/
metrics.rs

1//! Benchmark metrics and results structures
2
3use std::path::PathBuf;
4
5/// Configuration for benchmark runs
6#[derive(Debug, Clone)]
7pub struct BenchmarkConfig {
8    /// Path to the model file (GGUF)
9    pub model_path: PathBuf,
10    /// Prompts to use for benchmarking
11    pub prompts: Vec<String>,
12    /// Maximum new tokens to generate per prompt
13    pub max_new_tokens: usize,
14    /// Number of warmup iterations before measurement
15    pub warmup_iterations: usize,
16    /// Number of measurement iterations
17    pub measurement_iterations: usize,
18    /// Random seed for reproducibility
19    pub seed: u64,
20}
21
22impl Default for BenchmarkConfig {
23    fn default() -> Self {
24        Self {
25            model_path: PathBuf::new(),
26            prompts: vec![
27                "The capital of France is".to_string(),
28                "In machine learning, a neural network is".to_string(),
29                "The quick brown fox".to_string(),
30            ],
31            max_new_tokens: 50,
32            warmup_iterations: 3,
33            measurement_iterations: 10,
34            seed: 42,
35        }
36    }
37}
38
39/// Metrics collected per inference run
40#[derive(Debug, Clone)]
41pub struct InferenceMetrics {
42    /// Model load time in milliseconds
43    pub load_time_ms: f64,
44    /// Time to first token in milliseconds
45    pub time_to_first_token_ms: f64,
46    /// Total inference time in milliseconds
47    pub total_inference_time_ms: f64,
48    /// Number of tokens generated
49    pub tokens_generated: usize,
50    /// Tokens per second
51    pub tokens_per_second: f64,
52    /// Number of prompt tokens
53    pub prompt_tokens: usize,
54    /// Memory before model load (bytes)
55    pub memory_before_load_bytes: u64,
56    /// Memory after model load (bytes)
57    pub memory_after_load_bytes: u64,
58    /// Peak memory during inference (bytes)
59    pub peak_memory_bytes: u64,
60}
61
62/// Result of a single generation
63#[derive(Debug, Clone)]
64pub struct GenerationResult {
65    /// Generated text
66    pub output_text: String,
67    /// Number of tokens generated
68    pub tokens_generated: usize,
69    /// Number of prompt tokens
70    pub prompt_tokens: usize,
71    /// Time to first token in milliseconds
72    pub time_to_first_token_ms: f64,
73    /// Total time in milliseconds
74    pub total_time_ms: f64,
75}
76
77/// Aggregated benchmark results for a backend
78#[derive(Debug, Clone)]
79pub struct BenchmarkResults {
80    /// Backend name
81    pub backend_name: String,
82    /// All individual run metrics
83    pub runs: Vec<InferenceMetrics>,
84    /// Model load time in milliseconds
85    pub load_time_ms: f64,
86    /// Average time to first token in milliseconds
87    pub avg_ttft_ms: f64,
88    /// Average tokens per second
89    pub avg_tokens_per_sec: f64,
90    /// Median (p50) tokens per second
91    pub p50_tokens_per_sec: f64,
92    /// 99th percentile tokens per second
93    pub p99_tokens_per_sec: f64,
94    /// Average memory usage in MB
95    pub avg_memory_mb: f64,
96    /// Peak memory usage in MB
97    pub peak_memory_mb: f64,
98}
99
100impl BenchmarkResults {
101    /// Aggregate metrics from individual runs
102    pub fn aggregate(
103        backend_name: &str,
104        runs: Vec<InferenceMetrics>,
105        load_time_ms: f64,
106        peak_memory_bytes: u64,
107    ) -> Self {
108        let n = runs.len() as f64;
109
110        // Calculate averages
111        let avg_ttft_ms = runs.iter().map(|r| r.time_to_first_token_ms).sum::<f64>() / n;
112        let avg_tokens_per_sec = runs.iter().map(|r| r.tokens_per_second).sum::<f64>() / n;
113        let avg_memory_bytes =
114            runs.iter().map(|r| r.memory_after_load_bytes).sum::<u64>() as f64 / n;
115
116        // Calculate percentiles for tokens/sec
117        let mut tps_values: Vec<f64> = runs.iter().map(|r| r.tokens_per_second).collect();
118        tps_values.sort_by(|a, b| a.partial_cmp(b).unwrap_or(std::cmp::Ordering::Equal));
119
120        let p50_idx = (tps_values.len() as f64 * 0.5) as usize;
121        let p99_idx = (tps_values.len() as f64 * 0.99) as usize;
122
123        let p50_tokens_per_sec = tps_values.get(p50_idx).copied().unwrap_or(0.0);
124        let p99_tokens_per_sec = tps_values
125            .get(p99_idx.min(tps_values.len() - 1))
126            .copied()
127            .unwrap_or(0.0);
128
129        Self {
130            backend_name: backend_name.to_string(),
131            runs,
132            load_time_ms,
133            avg_ttft_ms,
134            avg_tokens_per_sec,
135            p50_tokens_per_sec,
136            p99_tokens_per_sec,
137            avg_memory_mb: avg_memory_bytes / (1024.0 * 1024.0),
138            peak_memory_mb: peak_memory_bytes as f64 / (1024.0 * 1024.0),
139        }
140    }
141}
142
143/// Comparison between two backends
144#[derive(Debug)]
145pub struct BenchmarkComparison {
146    pub llama_cpp: BenchmarkResults,
147    pub unillm: BenchmarkResults,
148}
149
150impl BenchmarkComparison {
151    /// Calculate percentage difference (positive means UniLLM is slower/uses more)
152    pub fn diff_percent(baseline: f64, comparison: f64) -> f64 {
153        if baseline == 0.0 {
154            return 0.0;
155        }
156        ((comparison - baseline) / baseline) * 100.0
157    }
158
159    /// Format difference as string with sign
160    pub fn format_diff(baseline: f64, comparison: f64) -> String {
161        let diff = Self::diff_percent(baseline, comparison);
162        if diff >= 0.0 {
163            format!("+{:.0}%", diff)
164        } else {
165            format!("{:.0}%", diff)
166        }
167    }
168}
runtime/benchmark/metrics.rs

runtime/benchmark/
metrics.rs
