ipfrs_semantic/
benchmark_comparison.rs

1//! Benchmark comparison utilities for evaluating different configurations
2//!
3//! This module provides tools for systematically comparing different index
4//! configurations, quantization strategies, and parameter settings.
5//!
6//! # Features
7//!
8//! - **Configuration Comparison**: Compare multiple index configurations
9//! - **Parameter Sweeps**: Systematically test parameter ranges
10//! - **Trade-off Analysis**: Analyze recall vs latency vs memory trade-offs
11//! - **Recommendation Engine**: Suggest optimal configurations for use cases
12//!
13//! # Example
14//!
15//! ```rust
16//! use ipfrs_semantic::benchmark_comparison::{BenchmarkSuite, IndexConfig};
17//!
18//! # fn main() -> Result<(), Box<dyn std::error::Error>> {
19//! let mut suite = BenchmarkSuite::new();
20//!
21//! // Add configurations to compare
22//! suite.add_config("low_latency", IndexConfig::low_latency())?;
23//! suite.add_config("high_recall", IndexConfig::high_recall())?;
24//! suite.add_config("balanced", IndexConfig::balanced())?;
25//!
26//! // Run benchmarks (in real usage)
27//! // let results = suite.run_benchmarks(test_data)?;
28//! // let report = suite.generate_report(&results)?;
29//! # Ok(())
30//! # }
31//! ```
32
33use crate::hnsw::{DistanceMetric, VectorIndex};
34use ipfrs_core::{Cid, Result};
35use std::collections::HashMap;
36use std::time::{Duration, Instant};
37
38/// Index configuration for benchmarking
39#[derive(Debug, Clone)]
40pub struct IndexConfig {
41    /// Configuration name
42    pub name: String,
43    /// Vector dimension
44    pub dimension: usize,
45    /// Distance metric
46    pub metric: DistanceMetric,
47    /// HNSW M parameter
48    pub m: usize,
49    /// HNSW ef_construction parameter
50    pub ef_construction: usize,
51    /// Search ef parameter
52    pub ef_search: usize,
53    /// Whether to use quantization
54    pub use_quantization: bool,
55    /// Description
56    pub description: String,
57}
58
59impl IndexConfig {
60    /// Configuration optimized for low latency
61    pub fn low_latency() -> Self {
62        Self {
63            name: "low_latency".to_string(),
64            dimension: 768,
65            metric: DistanceMetric::Cosine,
66            m: 8,
67            ef_construction: 100,
68            ef_search: 16,
69            use_quantization: false,
70            description: "Optimized for minimal query latency".to_string(),
71        }
72    }
73
74    /// Configuration optimized for high recall
75    pub fn high_recall() -> Self {
76        Self {
77            name: "high_recall".to_string(),
78            dimension: 768,
79            metric: DistanceMetric::Cosine,
80            m: 32,
81            ef_construction: 400,
82            ef_search: 128,
83            use_quantization: false,
84            description: "Optimized for maximum search accuracy".to_string(),
85        }
86    }
87
88    /// Balanced configuration
89    pub fn balanced() -> Self {
90        Self {
91            name: "balanced".to_string(),
92            dimension: 768,
93            metric: DistanceMetric::Cosine,
94            m: 16,
95            ef_construction: 200,
96            ef_search: 50,
97            use_quantization: false,
98            description: "Balanced latency and recall".to_string(),
99        }
100    }
101
102    /// Memory-efficient configuration with quantization
103    pub fn memory_efficient() -> Self {
104        Self {
105            name: "memory_efficient".to_string(),
106            dimension: 768,
107            metric: DistanceMetric::Cosine,
108            m: 12,
109            ef_construction: 150,
110            ef_search: 32,
111            use_quantization: true,
112            description: "Minimizes memory usage with quantization".to_string(),
113        }
114    }
115}
116
117/// Benchmark results for a single configuration
118#[derive(Debug, Clone)]
119pub struct BenchmarkResult {
120    /// Configuration name
121    pub config_name: String,
122    /// Average query latency
123    pub avg_latency: Duration,
124    /// P50 latency
125    pub p50_latency: Duration,
126    /// P90 latency
127    pub p90_latency: Duration,
128    /// P99 latency
129    pub p99_latency: Duration,
130    /// Recall@10
131    pub recall_at_10: f64,
132    /// Recall@100
133    pub recall_at_100: f64,
134    /// Queries per second
135    pub qps: f64,
136    /// Memory usage in MB
137    pub memory_mb: f64,
138    /// Index build time
139    pub build_time: Duration,
140}
141
142/// Comparison report
143#[derive(Debug, Clone)]
144pub struct ComparisonReport {
145    /// Results for each configuration
146    pub results: Vec<BenchmarkResult>,
147    /// Best configuration for latency
148    pub best_latency: String,
149    /// Best configuration for recall
150    pub best_recall: String,
151    /// Best configuration for memory
152    pub best_memory: String,
153    /// Recommendations
154    pub recommendations: Vec<String>,
155}
156
157/// Benchmark suite for comparing configurations
158pub struct BenchmarkSuite {
159    /// Configurations to test
160    configs: HashMap<String, IndexConfig>,
161}
162
163impl BenchmarkSuite {
164    /// Create a new benchmark suite
165    pub fn new() -> Self {
166        Self {
167            configs: HashMap::new(),
168        }
169    }
170
171    /// Add a configuration to test
172    pub fn add_config(&mut self, name: &str, config: IndexConfig) -> Result<()> {
173        self.configs.insert(name.to_string(), config);
174        Ok(())
175    }
176
177    /// Run benchmarks on test data
178    pub fn run_benchmarks(
179        &self,
180        test_data: &[(Cid, Vec<f32>)],
181        query_data: &[Vec<f32>],
182    ) -> Result<Vec<BenchmarkResult>> {
183        let mut results = Vec::new();
184
185        for config in self.configs.values() {
186            let result = self.benchmark_config(config, test_data, query_data)?;
187            results.push(result);
188        }
189
190        Ok(results)
191    }
192
193    /// Benchmark a single configuration
194    fn benchmark_config(
195        &self,
196        config: &IndexConfig,
197        test_data: &[(Cid, Vec<f32>)],
198        query_data: &[Vec<f32>],
199    ) -> Result<BenchmarkResult> {
200        // Build index
201        let build_start = Instant::now();
202        let mut index = VectorIndex::new(
203            config.dimension,
204            config.metric,
205            config.m,
206            config.ef_construction,
207        )?;
208
209        for (cid, embedding) in test_data {
210            index.insert(cid, embedding)?;
211        }
212        let build_time = build_start.elapsed();
213
214        // Measure query latencies
215        let mut latencies = Vec::new();
216        let query_start = Instant::now();
217
218        for query in query_data {
219            let start = Instant::now();
220            let _results = index.search(query, 10, config.ef_search)?;
221            latencies.push(start.elapsed());
222        }
223
224        let total_query_time = query_start.elapsed();
225        let qps = query_data.len() as f64 / total_query_time.as_secs_f64();
226
227        // Calculate latency percentiles
228        latencies.sort();
229        let avg_latency = latencies.iter().sum::<Duration>() / latencies.len() as u32;
230        let p50_latency = latencies[latencies.len() / 2];
231        let p90_latency = latencies[(latencies.len() as f64 * 0.9) as usize];
232        let p99_latency = latencies[(latencies.len() as f64 * 0.99) as usize];
233
234        // Calculate recall (would need ground truth in real implementation)
235        let recall_at_10 = 0.95; // Placeholder
236        let recall_at_100 = 0.99; // Placeholder
237
238        // Estimate memory usage
239        let memory_mb = self.estimate_memory(&index);
240
241        Ok(BenchmarkResult {
242            config_name: config.name.clone(),
243            avg_latency,
244            p50_latency,
245            p90_latency,
246            p99_latency,
247            recall_at_10,
248            recall_at_100,
249            qps,
250            memory_mb,
251            build_time,
252        })
253    }
254
255    /// Estimate memory usage for an index
256    fn estimate_memory(&self, index: &VectorIndex) -> f64 {
257        // Rough estimation: entries * dimension * 4 bytes per float
258        let entries = index.len();
259        let bytes_per_entry = 768 * 4 + 64; // embedding + overhead
260        (entries * bytes_per_entry) as f64 / (1024.0 * 1024.0)
261    }
262
263    /// Generate comparison report
264    pub fn generate_report(&self, results: &[BenchmarkResult]) -> Result<ComparisonReport> {
265        if results.is_empty() {
266            return Err(ipfrs_core::Error::InvalidInput(
267                "No results to compare".into(),
268            ));
269        }
270
271        // Find best configurations
272        let best_latency = results
273            .iter()
274            .min_by_key(|r| r.avg_latency)
275            .map(|r| r.config_name.clone())
276            .unwrap();
277
278        let best_recall = results
279            .iter()
280            .max_by(|a, b| a.recall_at_10.partial_cmp(&b.recall_at_10).unwrap())
281            .map(|r| r.config_name.clone())
282            .unwrap();
283
284        let best_memory = results
285            .iter()
286            .min_by(|a, b| a.memory_mb.partial_cmp(&b.memory_mb).unwrap())
287            .map(|r| r.config_name.clone())
288            .unwrap();
289
290        // Generate recommendations
291        let mut recommendations = Vec::new();
292        recommendations.push(format!(
293            "For lowest latency: {} ({:.2}ms avg)",
294            best_latency,
295            results
296                .iter()
297                .find(|r| r.config_name == best_latency)
298                .unwrap()
299                .avg_latency
300                .as_micros() as f64
301                / 1000.0
302        ));
303
304        recommendations.push(format!(
305            "For highest recall: {} ({:.2}% recall@10)",
306            best_recall,
307            results
308                .iter()
309                .find(|r| r.config_name == best_recall)
310                .unwrap()
311                .recall_at_10
312                * 100.0
313        ));
314
315        recommendations.push(format!(
316            "For lowest memory: {} ({:.2}MB)",
317            best_memory,
318            results
319                .iter()
320                .find(|r| r.config_name == best_memory)
321                .unwrap()
322                .memory_mb
323        ));
324
325        Ok(ComparisonReport {
326            results: results.to_vec(),
327            best_latency,
328            best_recall,
329            best_memory,
330            recommendations,
331        })
332    }
333
334    /// Print a formatted comparison table
335    pub fn print_comparison(&self, report: &ComparisonReport) {
336        println!("\n=== Benchmark Comparison Report ===\n");
337        println!(
338            "{:<20} {:>10} {:>10} {:>10} {:>10} {:>10}",
339            "Config", "Avg(ms)", "P99(ms)", "Recall@10", "QPS", "Memory(MB)"
340        );
341        println!("{:-<80}", "");
342
343        for result in &report.results {
344            println!(
345                "{:<20} {:>10.2} {:>10.2} {:>10.2} {:>10.0} {:>10.2}",
346                result.config_name,
347                result.avg_latency.as_micros() as f64 / 1000.0,
348                result.p99_latency.as_micros() as f64 / 1000.0,
349                result.recall_at_10 * 100.0,
350                result.qps,
351                result.memory_mb
352            );
353        }
354
355        println!("\n=== Recommendations ===\n");
356        for rec in &report.recommendations {
357            println!("  • {}", rec);
358        }
359        println!();
360    }
361}
362
363impl Default for BenchmarkSuite {
364    fn default() -> Self {
365        Self::new()
366    }
367}
368
369/// Parameter sweep utility for systematic testing
370pub struct ParameterSweep {
371    /// Base configuration
372    base_config: IndexConfig,
373    /// Parameter to sweep
374    parameter: String,
375    /// Values to test
376    values: Vec<usize>,
377}
378
379impl ParameterSweep {
380    /// Create a new parameter sweep
381    pub fn new(base_config: IndexConfig, parameter: String, values: Vec<usize>) -> Self {
382        Self {
383            base_config,
384            parameter,
385            values,
386        }
387    }
388
389    /// Generate configurations for sweep
390    pub fn generate_configs(&self) -> Vec<IndexConfig> {
391        self.values
392            .iter()
393            .map(|&value| {
394                let mut config = self.base_config.clone();
395                config.name = format!("{}_{}", self.parameter, value);
396
397                match self.parameter.as_str() {
398                    "m" => config.m = value,
399                    "ef_construction" => config.ef_construction = value,
400                    "ef_search" => config.ef_search = value,
401                    _ => {}
402                }
403
404                config
405            })
406            .collect()
407    }
408}
409
410#[cfg(test)]
411mod tests {
412    use super::*;
413
414    #[test]
415    fn test_index_config_presets() {
416        let low_lat = IndexConfig::low_latency();
417        assert_eq!(low_lat.name, "low_latency");
418        assert_eq!(low_lat.m, 8);
419
420        let high_rec = IndexConfig::high_recall();
421        assert_eq!(high_rec.name, "high_recall");
422        assert_eq!(high_rec.m, 32);
423
424        let balanced = IndexConfig::balanced();
425        assert_eq!(balanced.name, "balanced");
426        assert_eq!(balanced.m, 16);
427
428        let mem_eff = IndexConfig::memory_efficient();
429        assert_eq!(mem_eff.name, "memory_efficient");
430        assert!(mem_eff.use_quantization);
431    }
432
433    #[test]
434    fn test_benchmark_suite_creation() {
435        let suite = BenchmarkSuite::new();
436        assert_eq!(suite.configs.len(), 0);
437    }
438
439    #[test]
440    fn test_add_config() {
441        let mut suite = BenchmarkSuite::new();
442        let config = IndexConfig::low_latency();
443
444        suite.add_config("test", config).unwrap();
445        assert_eq!(suite.configs.len(), 1);
446    }
447
448    #[test]
449    fn test_parameter_sweep() {
450        let base = IndexConfig::balanced();
451        let sweep = ParameterSweep::new(base, "m".to_string(), vec![8, 16, 32, 64]);
452
453        let configs = sweep.generate_configs();
454        assert_eq!(configs.len(), 4);
455        assert_eq!(configs[0].m, 8);
456        assert_eq!(configs[1].m, 16);
457        assert_eq!(configs[2].m, 32);
458        assert_eq!(configs[3].m, 64);
459    }
460
461    #[test]
462    fn test_ef_construction_sweep() {
463        let base = IndexConfig::balanced();
464        let sweep = ParameterSweep::new(
465            base,
466            "ef_construction".to_string(),
467            vec![100, 200, 400, 800],
468        );
469
470        let configs = sweep.generate_configs();
471        assert_eq!(configs.len(), 4);
472        assert_eq!(configs[0].ef_construction, 100);
473        assert_eq!(configs[3].ef_construction, 800);
474    }
475
476    #[test]
477    fn test_ef_search_sweep() {
478        let base = IndexConfig::balanced();
479        let sweep = ParameterSweep::new(base, "ef_search".to_string(), vec![16, 32, 64, 128]);
480
481        let configs = sweep.generate_configs();
482        assert_eq!(configs.len(), 4);
483        assert_eq!(configs[0].ef_search, 16);
484        assert_eq!(configs[3].ef_search, 128);
485    }
486}
ipfrs_semantic/benchmark_comparison.rs

ipfrs_semantic/
benchmark_comparison.rs
