oxirs_vec/
bench_metrics.rs

1//! Benchmark metric types and data structures for the advanced benchmarking framework.
2
3use crate::benchmarking::{BenchmarkConfig, BenchmarkDataset, BenchmarkOutputFormat};
4use crate::VectorIndex;
5use serde::{Deserialize, Serialize};
6use std::collections::HashMap;
7use std::time::Duration;
8
9/// Advanced benchmarking configuration with comprehensive options
10#[derive(Debug, Clone, Serialize, Deserialize)]
11pub struct AdvancedBenchmarkConfig {
12    /// Base configuration
13    pub base_config: BenchmarkConfig,
14    /// Statistical confidence level (e.g., 0.95 for 95%)
15    pub confidence_level: f64,
16    /// Minimum number of runs for statistical significance
17    pub min_runs: usize,
18    /// Maximum coefficient of variation allowed
19    pub max_cv: f64,
20    /// Enable memory profiling
21    pub memory_profiling: bool,
22    /// Enable latency distribution analysis
23    pub latency_distribution: bool,
24    /// Enable throughput testing
25    pub throughput_testing: bool,
26    /// Enable quality degradation analysis
27    pub quality_degradation: bool,
28    /// Enable hyperparameter optimization
29    pub hyperparameter_optimization: bool,
30    /// Enable comparative analysis
31    pub comparative_analysis: bool,
32    /// ANN-Benchmarks compatibility mode
33    pub ann_benchmarks_mode: bool,
34    /// Export detailed traces
35    pub export_traces: bool,
36    /// Parallel execution configuration
37    pub parallel_config: ParallelBenchmarkConfig,
38}
39
40/// Configuration for parallel benchmark execution
41#[derive(Debug, Clone, Serialize, Deserialize)]
42pub struct ParallelBenchmarkConfig {
43    /// Number of concurrent threads
44    pub num_threads: usize,
45    /// Enable NUMA-aware allocation
46    pub numa_aware: bool,
47    /// Thread affinity settings
48    pub thread_affinity: bool,
49    /// Memory bandwidth testing
50    pub memory_bandwidth_test: bool,
51}
52
53/// Enhanced dataset with comprehensive metadata and analysis
54#[derive(Debug, Clone, Serialize, Deserialize)]
55pub struct EnhancedBenchmarkDataset {
56    /// Base dataset information
57    pub base_dataset: BenchmarkDataset,
58    /// Dataset statistics
59    pub statistics: DatasetStatistics,
60    /// Quality metrics
61    pub quality_metrics: DatasetQualityMetrics,
62    /// Intrinsic dimensionality
63    pub intrinsic_dimensionality: f32,
64    /// Clustering coefficient
65    pub clustering_coefficient: f32,
66    /// Hubness score
67    pub hubness_score: f32,
68    /// Local intrinsic dimensionality
69    pub local_id: Vec<f32>,
70}
71
72/// Comprehensive dataset statistics
73#[derive(Debug, Clone, Serialize, Deserialize)]
74pub struct DatasetStatistics {
75    /// Number of vectors
76    pub vector_count: usize,
77    /// Dimensionality
78    pub dimensions: usize,
79    /// Mean vector magnitude
80    pub mean_magnitude: f32,
81    /// Standard deviation of magnitudes
82    pub std_magnitude: f32,
83    /// Inter-vector distance statistics
84    pub distance_stats: DistanceStatistics,
85    /// Nearest neighbor distribution
86    pub nn_distribution: Vec<f32>,
87    /// Sparsity ratio (if applicable)
88    pub sparsity_ratio: Option<f32>,
89}
90
91/// Distance statistics for dataset analysis
92#[derive(Debug, Clone, Serialize, Deserialize)]
93pub struct DistanceStatistics {
94    /// Mean pairwise distance
95    pub mean_distance: f32,
96    /// Standard deviation of distances
97    pub std_distance: f32,
98    /// Minimum distance
99    pub min_distance: f32,
100    /// Maximum distance
101    pub max_distance: f32,
102    /// Distance distribution percentiles
103    pub percentiles: Vec<(f32, f32)>, // (percentile, value)
104}
105
106/// Quality metrics for dataset characterization
107#[derive(Debug, Clone, Serialize, Deserialize)]
108pub struct DatasetQualityMetrics {
109    /// Effective dimensionality
110    pub effective_dimensionality: f32,
111    /// Concentration measure
112    pub concentration_measure: f32,
113    /// Outlier ratio
114    pub outlier_ratio: f32,
115    /// Cluster quality (silhouette score)
116    pub cluster_quality: f32,
117    /// Manifold quality
118    pub manifold_quality: f32,
119}
120
121/// Algorithm wrapper for benchmarking
122pub struct BenchmarkAlgorithm {
123    pub name: String,
124    pub description: String,
125    pub index: Box<dyn VectorIndex>,
126    pub parameters: AlgorithmParameters,
127    pub build_time: Option<Duration>,
128    pub memory_usage: Option<usize>,
129}
130
131/// Algorithm-specific parameters
132#[derive(Debug, Clone, Serialize, Deserialize)]
133pub struct AlgorithmParameters {
134    /// Parameter map
135    pub params: HashMap<String, ParameterValue>,
136    /// Search parameters
137    pub search_params: HashMap<String, ParameterValue>,
138    /// Build parameters
139    pub build_params: HashMap<String, ParameterValue>,
140}
141
142/// Parameter values with type information
143#[derive(Debug, Clone, Serialize, Deserialize)]
144pub enum ParameterValue {
145    Integer(i64),
146    Float(f64),
147    String(String),
148    Boolean(bool),
149    IntegerRange(i64, i64, i64), // min, max, step
150    FloatRange(f64, f64, f64),   // min, max, step
151}
152
153/// Advanced benchmark result with comprehensive metrics
154#[derive(Debug, Clone, Serialize, Deserialize)]
155pub struct AdvancedBenchmarkResult {
156    /// Basic result information
157    pub algorithm_name: String,
158    pub dataset_name: String,
159    pub timestamp: std::time::SystemTime,
160
161    /// Performance metrics
162    pub performance: PerformanceMetrics,
163    /// Quality metrics
164    pub quality: QualityMetrics,
165    /// Scalability metrics
166    pub scalability: ScalabilityMetrics,
167    /// Memory metrics
168    pub memory: MemoryMetrics,
169    /// Statistical analysis
170    pub statistics: StatisticalMetrics,
171
172    /// Detailed traces
173    pub traces: Option<BenchmarkTraces>,
174    /// Error information
175    pub errors: Vec<String>,
176}
177
178impl Default for AdvancedBenchmarkResult {
179    fn default() -> Self {
180        Self {
181            algorithm_name: String::new(),
182            dataset_name: String::new(),
183            timestamp: std::time::SystemTime::now(),
184            performance: PerformanceMetrics {
185                latency: LatencyMetrics {
186                    mean_ms: 0.0,
187                    std_ms: 0.0,
188                    percentiles: std::collections::HashMap::new(),
189                    distribution: Vec::new(),
190                    max_ms: 0.0,
191                    min_ms: 0.0,
192                },
193                throughput: ThroughputMetrics {
194                    qps: 0.0,
195                    batch_qps: std::collections::HashMap::new(),
196                    concurrent_qps: std::collections::HashMap::new(),
197                    saturation_qps: 0.0,
198                },
199                build_time: BuildTimeMetrics {
200                    total_seconds: 0.0,
201                    per_vector_ms: 0.0,
202                    allocation_seconds: 0.0,
203                    construction_seconds: 0.0,
204                    optimization_seconds: 0.0,
205                },
206                index_size: IndexSizeMetrics {
207                    total_bytes: 0,
208                    per_vector_bytes: 0.0,
209                    overhead_ratio: 0.0,
210                    compression_ratio: 0.0,
211                    serialized_bytes: 0,
212                },
213            },
214            quality: QualityMetrics {
215                recall_at_k: std::collections::HashMap::new(),
216                precision_at_k: std::collections::HashMap::new(),
217                mean_average_precision: 0.0,
218                ndcg_at_k: std::collections::HashMap::new(),
219                f1_at_k: std::collections::HashMap::new(),
220                mean_reciprocal_rank: 0.0,
221                quality_degradation: QualityDegradation {
222                    recall_latency_tradeoff: Vec::new(),
223                    quality_size_tradeoff: Vec::new(),
224                    quality_buildtime_tradeoff: Vec::new(),
225                },
226            },
227            scalability: ScalabilityMetrics {
228                latency_scaling: Vec::new(),
229                memory_scaling: Vec::new(),
230                buildtime_scaling: Vec::new(),
231                throughput_scaling: Vec::new(),
232                scaling_efficiency: 0.0,
233            },
234            memory: MemoryMetrics {
235                peak_memory_mb: 0.0,
236                average_memory_mb: 0.0,
237                allocation_patterns: Vec::new(),
238                fragmentation_ratio: 0.0,
239                cache_metrics: CacheMetrics {
240                    l1_hit_ratio: 0.0,
241                    l2_hit_ratio: 0.0,
242                    l3_hit_ratio: 0.0,
243                    memory_bandwidth_util: 0.0,
244                },
245            },
246            statistics: StatisticalMetrics {
247                sample_size: 0,
248                confidence_intervals: std::collections::HashMap::new(),
249                significance_tests: std::collections::HashMap::new(),
250                effect_sizes: std::collections::HashMap::new(),
251                power_analysis: PowerAnalysis {
252                    power: 0.0,
253                    effect_size: 0.0,
254                    required_sample_size: 0,
255                },
256            },
257            traces: None,
258            errors: Vec::new(),
259        }
260    }
261}
262
263/// Comprehensive performance metrics
264#[derive(Debug, Clone, Serialize, Deserialize)]
265pub struct PerformanceMetrics {
266    /// Query latency statistics
267    pub latency: LatencyMetrics,
268    /// Throughput measurements
269    pub throughput: ThroughputMetrics,
270    /// Build time metrics
271    pub build_time: BuildTimeMetrics,
272    /// Index size metrics
273    pub index_size: IndexSizeMetrics,
274}
275
276/// Detailed latency analysis
277#[derive(Debug, Clone, Serialize, Deserialize)]
278pub struct LatencyMetrics {
279    /// Mean latency
280    pub mean_ms: f64,
281    /// Standard deviation
282    pub std_ms: f64,
283    /// Percentile latencies
284    pub percentiles: HashMap<String, f64>, // P50, P95, P99, P99.9
285    /// Latency distribution
286    pub distribution: Vec<f64>,
287    /// Worst-case latency
288    pub max_ms: f64,
289    /// Best-case latency
290    pub min_ms: f64,
291}
292
293/// Throughput analysis
294#[derive(Debug, Clone, Serialize, Deserialize)]
295pub struct ThroughputMetrics {
296    /// Queries per second
297    pub qps: f64,
298    /// Batched QPS
299    pub batch_qps: HashMap<usize, f64>, // batch_size -> qps
300    /// Concurrent QPS
301    pub concurrent_qps: HashMap<usize, f64>, // thread_count -> qps
302    /// Saturation point
303    pub saturation_qps: f64,
304}
305
306/// Build time analysis
307#[derive(Debug, Clone, Serialize, Deserialize)]
308pub struct BuildTimeMetrics {
309    /// Total build time
310    pub total_seconds: f64,
311    /// Build time per vector
312    pub per_vector_ms: f64,
313    /// Memory allocation time
314    pub allocation_seconds: f64,
315    /// Index construction time
316    pub construction_seconds: f64,
317    /// Optimization time
318    pub optimization_seconds: f64,
319}
320
321/// Index size analysis
322#[derive(Debug, Clone, Serialize, Deserialize)]
323pub struct IndexSizeMetrics {
324    /// Total memory usage (bytes)
325    pub total_bytes: usize,
326    /// Memory per vector (bytes)
327    pub per_vector_bytes: f64,
328    /// Overhead ratio
329    pub overhead_ratio: f64,
330    /// Compression ratio
331    pub compression_ratio: f64,
332    /// Serialized size
333    pub serialized_bytes: usize,
334}
335
336/// Quality metrics for search accuracy
337#[derive(Debug, Clone, Serialize, Deserialize)]
338pub struct QualityMetrics {
339    /// Recall at different k values
340    pub recall_at_k: HashMap<usize, f64>,
341    /// Precision at different k values
342    pub precision_at_k: HashMap<usize, f64>,
343    /// Mean Average Precision
344    pub mean_average_precision: f64,
345    /// Normalized Discounted Cumulative Gain
346    pub ndcg_at_k: HashMap<usize, f64>,
347    /// F1 scores
348    pub f1_at_k: HashMap<usize, f64>,
349    /// Mean Reciprocal Rank
350    pub mean_reciprocal_rank: f64,
351    /// Quality degradation analysis
352    pub quality_degradation: QualityDegradation,
353}
354
355/// Quality degradation under different conditions
356#[derive(Debug, Clone, Serialize, Deserialize)]
357pub struct QualityDegradation {
358    /// Recall vs latency trade-off
359    pub recall_latency_tradeoff: Vec<(f64, f64)>, // (recall, latency_ms)
360    /// Quality vs index size trade-off
361    pub quality_size_tradeoff: Vec<(f64, usize)>, // (recall, size_bytes)
362    /// Quality vs build time trade-off
363    pub quality_buildtime_tradeoff: Vec<(f64, f64)>, // (recall, build_seconds)
364}
365
366/// Scalability analysis
367#[derive(Debug, Clone, Serialize, Deserialize)]
368pub struct ScalabilityMetrics {
369    /// Latency vs dataset size
370    pub latency_scaling: Vec<(usize, f64)>, // (dataset_size, latency_ms)
371    /// Memory vs dataset size
372    pub memory_scaling: Vec<(usize, usize)>, // (dataset_size, memory_bytes)
373    /// Build time vs dataset size
374    pub buildtime_scaling: Vec<(usize, f64)>, // (dataset_size, build_seconds)
375    /// Throughput vs concurrent users
376    pub throughput_scaling: Vec<(usize, f64)>, // (concurrent_users, qps)
377    /// Scaling efficiency
378    pub scaling_efficiency: f64,
379}
380
381/// Memory usage analysis
382#[derive(Debug, Clone, Serialize, Deserialize)]
383pub struct MemoryMetrics {
384    /// Peak memory usage
385    pub peak_memory_mb: f64,
386    /// Average memory usage
387    pub average_memory_mb: f64,
388    /// Memory allocation patterns
389    pub allocation_patterns: Vec<MemoryAllocation>,
390    /// Memory fragmentation
391    pub fragmentation_ratio: f64,
392    /// Cache hit ratios
393    pub cache_metrics: CacheMetrics,
394}
395
396/// Memory allocation tracking
397#[derive(Debug, Clone, Serialize, Deserialize)]
398pub struct MemoryAllocation {
399    /// Timestamp
400    pub timestamp_ms: u64,
401    /// Allocated bytes
402    pub allocated_bytes: usize,
403    /// Allocation type
404    pub allocation_type: String,
405}
406
407/// Cache performance metrics
408#[derive(Debug, Clone, Serialize, Deserialize)]
409pub struct CacheMetrics {
410    /// L1 cache hit ratio
411    pub l1_hit_ratio: f64,
412    /// L2 cache hit ratio
413    pub l2_hit_ratio: f64,
414    /// L3 cache hit ratio
415    pub l3_hit_ratio: f64,
416    /// Memory bandwidth utilization
417    pub memory_bandwidth_util: f64,
418}
419
420/// Statistical analysis of benchmark results
421#[derive(Debug, Clone, Serialize, Deserialize)]
422pub struct StatisticalMetrics {
423    /// Sample size
424    pub sample_size: usize,
425    /// Confidence intervals
426    pub confidence_intervals: HashMap<String, (f64, f64)>, // metric -> (lower, upper)
427    /// Statistical significance tests
428    pub significance_tests: HashMap<String, StatisticalTest>,
429    /// Effect sizes
430    pub effect_sizes: HashMap<String, f64>,
431    /// Power analysis
432    pub power_analysis: PowerAnalysis,
433}
434
435/// Statistical test results
436#[derive(Debug, Clone, Serialize, Deserialize)]
437pub struct StatisticalTest {
438    /// Test type (t-test, Mann-Whitney U, etc.)
439    pub test_type: String,
440    /// P-value
441    pub p_value: f64,
442    /// Test statistic
443    pub test_statistic: f64,
444    /// Significant at alpha=0.05
445    pub is_significant: bool,
446}
447
448/// Power analysis for statistical tests
449#[derive(Debug, Clone, Serialize, Deserialize)]
450pub struct PowerAnalysis {
451    /// Statistical power
452    pub power: f64,
453    /// Effect size
454    pub effect_size: f64,
455    /// Required sample size
456    pub required_sample_size: usize,
457}
458
459/// Detailed benchmark traces
460#[derive(Debug, Clone, Serialize, Deserialize)]
461pub struct BenchmarkTraces {
462    /// Query-level traces
463    pub query_traces: Vec<QueryTrace>,
464    /// System-level traces
465    pub system_traces: Vec<SystemTrace>,
466    /// Memory traces
467    pub memory_traces: Vec<MemoryTrace>,
468}
469
470/// Individual query trace
471#[derive(Debug, Clone, Serialize, Deserialize)]
472pub struct QueryTrace {
473    /// Query ID
474    pub query_id: usize,
475    /// Start timestamp
476    pub start_time: u64,
477    /// End timestamp
478    pub end_time: u64,
479    /// Results returned
480    pub results_count: usize,
481    /// Distance computations
482    pub distance_computations: usize,
483    /// Cache hits
484    pub cache_hits: usize,
485    /// Memory allocations
486    pub memory_allocations: usize,
487}
488
489/// System-level trace
490#[derive(Debug, Clone, Serialize, Deserialize)]
491pub struct SystemTrace {
492    /// Timestamp
493    pub timestamp: u64,
494    /// CPU usage
495    pub cpu_usage: f64,
496    /// Memory usage
497    pub memory_usage: usize,
498    /// IO operations
499    pub io_operations: usize,
500    /// Context switches
501    pub context_switches: usize,
502}
503
504/// Memory trace
505#[derive(Debug, Clone, Serialize, Deserialize)]
506pub struct MemoryTrace {
507    /// Timestamp
508    pub timestamp: u64,
509    /// Heap usage
510    pub heap_usage: usize,
511    /// Stack usage
512    pub stack_usage: usize,
513    /// Page faults
514    pub page_faults: usize,
515    /// Memory bandwidth
516    pub memory_bandwidth: f64,
517}
518
519/// Optimization strategies for hyperparameter tuning
520#[derive(Debug, Clone, Serialize, Deserialize)]
521pub enum OptimizationStrategy {
522    GridSearch,
523    RandomSearch,
524    BayesianOptimization,
525    EvolutionaryOptimization,
526    MultiObjective,
527}
528
529/// Parameter search space definition
530#[derive(Debug, Clone, Serialize, Deserialize)]
531pub struct ParameterSpace {
532    pub parameter_type: ParameterType,
533    pub constraints: Vec<ParameterConstraint>,
534}
535
536#[derive(Debug, Clone, Serialize, Deserialize)]
537pub enum ParameterType {
538    Categorical(Vec<String>),
539    Continuous { min: f64, max: f64 },
540    Integer { min: i64, max: i64 },
541    Boolean,
542}
543
544#[derive(Debug, Clone, Serialize, Deserialize)]
545pub enum ParameterConstraint {
546    GreaterThan(f64),
547    LessThan(f64),
548    Conditional {
549        if_param: String,
550        if_value: String,
551        then_constraint: Box<ParameterConstraint>,
552    },
553}
554
555/// Objective function for optimization
556#[derive(Debug, Clone, Serialize, Deserialize)]
557pub enum ObjectiveFunction {
558    Recall { k: usize, weight: f64 },
559    Latency { percentile: f64, weight: f64 },
560    Throughput { weight: f64 },
561    MemoryUsage { weight: f64 },
562    Composite { objectives: Vec<ObjectiveFunction> },
563    Pareto { objectives: Vec<ObjectiveFunction> },
564}
565
566impl Default for AdvancedBenchmarkConfig {
567    fn default() -> Self {
568        Self::new()
569    }
570}
571
572impl AdvancedBenchmarkConfig {
573    pub fn new() -> Self {
574        Self {
575            base_config: BenchmarkConfig::default(),
576            confidence_level: 0.95,
577            min_runs: 10,
578            max_cv: 0.05, // 5% coefficient of variation
579            memory_profiling: true,
580            latency_distribution: true,
581            throughput_testing: true,
582            quality_degradation: true,
583            hyperparameter_optimization: false,
584            comparative_analysis: true,
585            ann_benchmarks_mode: false,
586            export_traces: false,
587            parallel_config: ParallelBenchmarkConfig {
588                num_threads: num_cpus::get(),
589                numa_aware: false,
590                thread_affinity: false,
591                memory_bandwidth_test: false,
592            },
593        }
594    }
595
596    pub fn ann_benchmarks_compatible() -> Self {
597        let mut config = Self::new();
598        config.ann_benchmarks_mode = true;
599        config.base_config.output_format = BenchmarkOutputFormat::AnnBenchmarks;
600        config.base_config.quality_metrics = true;
601        config.comparative_analysis = false;
602        config
603    }
604}
oxirs_vec/bench_metrics.rs

oxirs_vec/
bench_metrics.rs
