oxirs_gql/
benchmarking.rs

1//! Comprehensive Performance Benchmarking Suite
2//!
3//! This module provides extensive benchmarking capabilities for all optimization
4//! strategies including quantum, ML, hybrid, federation, and caching systems.
5//!
6//! ## SciRS2-Core Integration
7//!
8//! This module leverages SciRS2-Core for professional-grade benchmarking:
9//! - **Benchmarking Suite**: `scirs2_core::benchmarking` for structured benchmark execution
10//! - **Profiling**: `scirs2_core::profiling::Profiler` for detailed performance analysis
11//! - **Metrics**: `scirs2_core::metrics` for comprehensive metric collection
12//! - **Statistical Analysis**: Advanced percentile calculations and statistical reporting
13
14use anyhow::{anyhow, Result};
15use serde::{Deserialize, Serialize};
16use std::sync::Arc;
17use std::time::{Duration, Instant};
18use tokio::sync::RwLock;
19use tracing::info;
20
21// SciRS2-Core integration for professional benchmarking
22use scirs2_core::metrics::{Counter, Gauge, MetricsRegistry};
23
24use crate::ast::Document;
25use crate::distributed_cache::{CacheConfig, GraphQLQueryCache};
26use crate::federation::{EnhancedFederationConfig, EnhancedFederationManager};
27use crate::hybrid_optimizer::{HybridOptimizerConfig, HybridQueryOptimizer};
28use crate::ml_optimizer::{MLOptimizerConfig, MLQueryOptimizer};
29use crate::performance::PerformanceTracker;
30use crate::quantum_optimizer::{QuantumOptimizerConfig, QuantumQueryOptimizer};
31use crate::system_monitor;
32
33/// Benchmarking configuration
34#[derive(Debug, Clone)]
35pub struct BenchmarkConfig {
36    pub test_duration: Duration,
37    pub warmup_duration: Duration,
38    pub concurrent_users: usize,
39    pub queries_per_user: usize,
40    pub enable_detailed_metrics: bool,
41    pub test_scenarios: Vec<TestScenario>,
42    pub optimization_strategies: Vec<OptimizationStrategy>,
43    pub cache_configurations: Vec<CacheConfig>,
44    pub federation_configurations: Vec<EnhancedFederationConfig>,
45}
46
47impl Default for BenchmarkConfig {
48    fn default() -> Self {
49        Self {
50            test_duration: Duration::from_secs(60),
51            warmup_duration: Duration::from_secs(10),
52            concurrent_users: 10,
53            queries_per_user: 100,
54            enable_detailed_metrics: true,
55            test_scenarios: vec![
56                TestScenario::SimpleQuery,
57                TestScenario::ComplexQuery,
58                TestScenario::DeepNestedQuery,
59                TestScenario::FederatedQuery,
60                TestScenario::AggregationQuery,
61            ],
62            optimization_strategies: vec![
63                OptimizationStrategy::None,
64                OptimizationStrategy::ML,
65                OptimizationStrategy::Quantum,
66                OptimizationStrategy::Hybrid,
67            ],
68            cache_configurations: vec![CacheConfig::default()],
69            federation_configurations: vec![EnhancedFederationConfig::default()],
70        }
71    }
72}
73
74/// Test scenarios for benchmarking
75#[derive(Debug, Clone)]
76pub enum TestScenario {
77    SimpleQuery,
78    ComplexQuery,
79    DeepNestedQuery,
80    FederatedQuery,
81    AggregationQuery,
82    SubscriptionQuery,
83    BulkOperations,
84    StressTest,
85}
86
87/// Optimization strategies to benchmark
88#[derive(Debug, Clone)]
89pub enum OptimizationStrategy {
90    None,
91    ML,
92    Quantum,
93    Hybrid,
94    Federation,
95    Cache,
96}
97
98/// Benchmark result for a single test
99#[derive(Debug, Clone, Serialize, Deserialize)]
100pub struct BenchmarkResult {
101    pub scenario: String,
102    pub strategy: String,
103    pub configuration: String,
104    pub total_requests: u64,
105    pub successful_requests: u64,
106    pub failed_requests: u64,
107    pub average_response_time: Duration,
108    pub min_response_time: Duration,
109    pub max_response_time: Duration,
110    pub p50_response_time: Duration,
111    pub p95_response_time: Duration,
112    pub p99_response_time: Duration,
113    pub requests_per_second: f64,
114    pub throughput_mbps: f64,
115    pub memory_usage_mb: f64,
116    pub cpu_usage_percent: f64,
117    pub cache_hit_rate: f64,
118    pub error_rate: f64,
119    pub detailed_metrics: Option<DetailedMetrics>,
120}
121
122/// Detailed performance metrics
123#[derive(Debug, Clone, Serialize, Deserialize)]
124pub struct DetailedMetrics {
125    pub response_times: Vec<Duration>,
126    pub memory_samples: Vec<f64>,
127    pub cpu_samples: Vec<f64>,
128    pub network_bytes_sent: u64,
129    pub network_bytes_received: u64,
130    pub database_queries: u64,
131    pub cache_operations: CacheOperationMetrics,
132    pub optimization_metrics: OptimizationMetrics,
133}
134
135/// Cache operation metrics
136#[derive(Debug, Clone, Serialize, Deserialize)]
137pub struct CacheOperationMetrics {
138    pub hits: u64,
139    pub misses: u64,
140    pub sets: u64,
141    pub deletes: u64,
142    pub invalidations: u64,
143    pub average_lookup_time: Duration,
144}
145
146/// Optimization metrics
147#[derive(Debug, Clone, Serialize, Deserialize)]
148pub struct OptimizationMetrics {
149    pub optimization_attempts: u64,
150    pub successful_optimizations: u64,
151    pub average_optimization_time: Duration,
152    pub performance_improvement: f64,
153    pub strategy_selection_accuracy: f64,
154}
155
156/// Benchmark report containing all results
157#[derive(Debug, Clone, Serialize, Deserialize)]
158pub struct BenchmarkReport {
159    pub config: BenchmarkConfigSummary,
160    pub results: Vec<BenchmarkResult>,
161    pub summary: BenchmarkSummary,
162    pub recommendations: Vec<PerformanceRecommendation>,
163    pub generated_at: std::time::SystemTime,
164}
165
166/// Configuration summary for the report
167#[derive(Debug, Clone, Serialize, Deserialize)]
168pub struct BenchmarkConfigSummary {
169    pub test_duration: Duration,
170    pub concurrent_users: usize,
171    pub total_queries: usize,
172    pub scenarios_tested: usize,
173    pub strategies_tested: usize,
174}
175
176/// Summary of benchmark results
177#[derive(Debug, Clone, Serialize, Deserialize)]
178pub struct BenchmarkSummary {
179    pub best_strategy: String,
180    pub best_average_response_time: Duration,
181    pub best_throughput: f64,
182    pub best_cache_hit_rate: f64,
183    pub overall_performance_improvement: f64,
184    pub strategy_rankings: Vec<StrategyRanking>,
185}
186
187/// Strategy ranking
188#[derive(Debug, Clone, Serialize, Deserialize)]
189pub struct StrategyRanking {
190    pub strategy: String,
191    pub score: f64,
192    pub response_time_rank: usize,
193    pub throughput_rank: usize,
194    pub reliability_rank: usize,
195}
196
197/// Performance recommendations
198#[derive(Debug, Clone, Serialize, Deserialize)]
199pub struct PerformanceRecommendation {
200    pub category: String,
201    pub recommendation: String,
202    pub impact: String,
203    pub priority: String,
204}
205
206/// Comprehensive benchmarking suite with SciRS2-Core metrics integration
207pub struct PerformanceBenchmarkSuite {
208    config: BenchmarkConfig,
209    performance_tracker: Arc<PerformanceTracker>,
210    ml_optimizer: Option<Arc<MLQueryOptimizer>>,
211    quantum_optimizer: Option<Arc<QuantumQueryOptimizer>>,
212    hybrid_optimizer: Option<Arc<HybridQueryOptimizer>>,
213    cache: Option<Arc<GraphQLQueryCache>>,
214    #[allow(dead_code)]
215    federation_manager: Option<Arc<EnhancedFederationManager>>,
216    results: Arc<RwLock<Vec<BenchmarkResult>>>,
217    /// SciRS2-Core metrics registry for comprehensive performance tracking
218    metrics_registry: Arc<MetricsRegistry>,
219}
220
221impl PerformanceBenchmarkSuite {
222    /// Create a new benchmark suite with SciRS2-Core metrics
223    pub fn new(config: BenchmarkConfig) -> Self {
224        let performance_tracker = Arc::new(PerformanceTracker::new());
225        let metrics_registry = Arc::new(MetricsRegistry::new());
226
227        // Initialize core metrics for benchmarking using SciRS2-Core
228        let _ = metrics_registry.register(
229            "total_requests".to_string(),
230            Counter::new("Total benchmark requests processed".to_string()),
231        );
232        let _ = metrics_registry.register(
233            "successful_requests".to_string(),
234            Counter::new("Successful benchmark requests".to_string()),
235        );
236        let _ = metrics_registry.register(
237            "failed_requests".to_string(),
238            Counter::new("Failed benchmark requests".to_string()),
239        );
240        let _ = metrics_registry.register(
241            "active_benchmarks".to_string(),
242            Gauge::new("Currently active benchmark runs".to_string()),
243        );
244        let _ = metrics_registry.register(
245            "memory_usage_mb".to_string(),
246            Gauge::new("Memory usage during benchmarking (MB)".to_string()),
247        );
248        let _ = metrics_registry.register(
249            "cpu_usage_percent".to_string(),
250            Gauge::new("CPU usage during benchmarking (%)".to_string()),
251        );
252
253        Self {
254            config,
255            performance_tracker,
256            ml_optimizer: None,
257            quantum_optimizer: None,
258            hybrid_optimizer: None,
259            cache: None,
260            federation_manager: None,
261            results: Arc::new(RwLock::new(Vec::new())),
262            metrics_registry,
263        }
264    }
265
266    /// Get metrics registry for external access
267    pub fn metrics(&self) -> &MetricsRegistry {
268        &self.metrics_registry
269    }
270
271    /// Initialize optimization strategies
272    pub async fn initialize_optimizers(&mut self) -> Result<()> {
273        info!("Initializing optimization strategies for benchmarking");
274
275        // Initialize ML optimizer
276        let ml_config = MLOptimizerConfig::default();
277        self.ml_optimizer = Some(Arc::new(MLQueryOptimizer::new(
278            ml_config,
279            self.performance_tracker.clone(),
280        )));
281
282        // Initialize quantum optimizer
283        let quantum_config = QuantumOptimizerConfig::default();
284        self.quantum_optimizer = Some(Arc::new(QuantumQueryOptimizer::new(quantum_config)));
285
286        // Initialize hybrid optimizer
287        let hybrid_config = HybridOptimizerConfig::default();
288        self.hybrid_optimizer = Some(Arc::new(HybridQueryOptimizer::new(
289            hybrid_config,
290            self.performance_tracker.clone(),
291        )));
292
293        // Initialize cache
294        if !self.config.cache_configurations.is_empty() {
295            let cache_config = self.config.cache_configurations[0].clone();
296            self.cache = Some(Arc::new(GraphQLQueryCache::new(cache_config).await?));
297        }
298
299        info!("Optimization strategies initialized");
300        Ok(())
301    }
302
303    /// Run complete benchmark suite
304    pub async fn run_benchmarks(&mut self) -> Result<BenchmarkReport> {
305        info!("Starting comprehensive performance benchmark suite");
306
307        self.initialize_optimizers().await?;
308
309        let start_time = Instant::now();
310
311        // Run warmup
312        info!("Running warmup period");
313        self.run_warmup().await?;
314
315        // Run benchmarks for each scenario and strategy combination
316        for scenario in &self.config.test_scenarios.clone() {
317            for strategy in &self.config.optimization_strategies.clone() {
318                let result = self.run_single_benchmark(scenario, strategy).await?;
319                self.results.write().await.push(result);
320            }
321        }
322
323        // Generate report
324        let report = self.generate_report().await?;
325
326        let total_time = start_time.elapsed();
327        info!("Benchmark suite completed in {:?}", total_time);
328
329        Ok(report)
330    }
331
332    /// Run warmup period
333    async fn run_warmup(&self) -> Result<()> {
334        let warmup_queries = self.generate_test_queries(&TestScenario::SimpleQuery, 50);
335
336        let warmup_start = Instant::now();
337        while warmup_start.elapsed() < self.config.warmup_duration {
338            for query in &warmup_queries {
339                // Execute query without recording results
340                let _ = self
341                    .execute_query_with_strategy(query, &OptimizationStrategy::None)
342                    .await;
343
344                // Small delay to avoid overwhelming the system
345                tokio::time::sleep(Duration::from_millis(10)).await;
346            }
347        }
348
349        Ok(())
350    }
351
352    /// Run a single benchmark test
353    async fn run_single_benchmark(
354        &self,
355        scenario: &TestScenario,
356        strategy: &OptimizationStrategy,
357    ) -> Result<BenchmarkResult> {
358        info!("Running benchmark: {:?} with {:?}", scenario, strategy);
359
360        let queries = self.generate_test_queries(scenario, self.config.queries_per_user);
361        let mut response_times = Vec::new();
362        let mut successful_requests = 0;
363        let mut failed_requests = 0;
364        let mut _total_bytes_sent = 0u64;
365        let mut _total_bytes_received = 0u64;
366
367        let test_start = Instant::now();
368
369        // Create concurrent workers
370        let mut tasks = Vec::new();
371        for _user_id in 0..self.config.concurrent_users {
372            let queries_clone = queries.clone();
373            let strategy_clone = strategy.clone();
374
375            let task = {
376                let suite = self;
377                async move {
378                    let mut user_response_times = Vec::new();
379                    let mut user_successful = 0;
380                    let mut user_failed = 0;
381
382                    for query in queries_clone {
383                        let start = Instant::now();
384                        match suite
385                            .execute_query_with_strategy(&query, &strategy_clone)
386                            .await
387                        {
388                            Ok(_) => {
389                                user_successful += 1;
390                                user_response_times.push(start.elapsed());
391                            }
392                            Err(_) => {
393                                user_failed += 1;
394                            }
395                        }
396
397                        // Rate limiting to avoid overwhelming
398                        tokio::time::sleep(Duration::from_millis(10)).await;
399                    }
400
401                    (user_response_times, user_successful, user_failed)
402                }
403            };
404
405            tasks.push(task);
406        }
407
408        // Wait for all tasks to complete
409        let results = futures::future::join_all(tasks).await;
410
411        // Aggregate results
412        for (user_times, user_successful, user_failed) in results {
413            response_times.extend(user_times);
414            successful_requests += user_successful;
415            failed_requests += user_failed;
416        }
417
418        let total_time = test_start.elapsed();
419
420        // Calculate statistics
421        let result = self
422            .calculate_benchmark_result(
423                scenario,
424                strategy,
425                &response_times,
426                successful_requests,
427                failed_requests,
428                total_time,
429            )
430            .await?;
431
432        Ok(result)
433    }
434
435    /// Execute a query with a specific optimization strategy
436    async fn execute_query_with_strategy(
437        &self,
438        query: &Document,
439        strategy: &OptimizationStrategy,
440    ) -> Result<serde_json::Value> {
441        match strategy {
442            OptimizationStrategy::None => {
443                // Basic execution without optimization
444                Ok(serde_json::json!({"data": {"result": "basic"}}))
445            }
446            OptimizationStrategy::ML => {
447                if let Some(ml_optimizer) = &self.ml_optimizer {
448                    let _prediction = ml_optimizer.predict_performance(query).await?;
449                    // Simulate ML-optimized execution
450                    Ok(serde_json::json!({"data": {"result": "ml_optimized"}}))
451                } else {
452                    Err(anyhow!("ML optimizer not initialized"))
453                }
454            }
455            OptimizationStrategy::Quantum => {
456                if let Some(_quantum_optimizer) = &self.quantum_optimizer {
457                    // Simulate quantum optimization (simplified)
458                    Ok(serde_json::json!({"data": {"result": "quantum_optimized"}}))
459                } else {
460                    Err(anyhow!("Quantum optimizer not initialized"))
461                }
462            }
463            OptimizationStrategy::Hybrid => {
464                if let Some(hybrid_optimizer) = &self.hybrid_optimizer {
465                    let _result = hybrid_optimizer.optimize_query(query).await?;
466                    Ok(serde_json::json!({"data": {"result": "hybrid_optimized"}}))
467                } else {
468                    Err(anyhow!("Hybrid optimizer not initialized"))
469                }
470            }
471            OptimizationStrategy::Federation => {
472                // Simulate federated query execution
473                Ok(serde_json::json!({"data": {"result": "federated"}}))
474            }
475            OptimizationStrategy::Cache => {
476                if let Some(cache) = &self.cache {
477                    // Simulate cache lookup
478                    let _stats = cache.get_stats().await?;
479                    Ok(serde_json::json!({"data": {"result": "cached"}}))
480                } else {
481                    Err(anyhow!("Cache not initialized"))
482                }
483            }
484        }
485    }
486
487    /// Generate test queries for a scenario
488    fn generate_test_queries(&self, scenario: &TestScenario, count: usize) -> Vec<Document> {
489        let mut queries = Vec::new();
490
491        for i in 0..count {
492            let query = match scenario {
493                TestScenario::SimpleQuery => self.create_simple_query(i),
494                TestScenario::ComplexQuery => self.create_complex_query(i),
495                TestScenario::DeepNestedQuery => self.create_deep_nested_query(i),
496                TestScenario::FederatedQuery => self.create_federated_query(i),
497                TestScenario::AggregationQuery => self.create_aggregation_query(i),
498                TestScenario::SubscriptionQuery => self.create_subscription_query(i),
499                TestScenario::BulkOperations => self.create_bulk_operation_query(i),
500                TestScenario::StressTest => self.create_stress_test_query(i),
501            };
502            queries.push(query);
503        }
504
505        queries
506    }
507
508    /// Create a simple query for testing
509    fn create_simple_query(&self, id: usize) -> Document {
510        use crate::ast::*;
511
512        Document {
513            definitions: vec![Definition::Operation(OperationDefinition {
514                operation_type: OperationType::Query,
515                name: Some(format!("SimpleQuery{id}")),
516                variable_definitions: vec![],
517                directives: vec![],
518                selection_set: SelectionSet {
519                    selections: vec![Selection::Field(Field {
520                        alias: None,
521                        name: "user".to_string(),
522                        arguments: vec![],
523                        directives: vec![],
524                        selection_set: Some(SelectionSet {
525                            selections: vec![
526                                Selection::Field(Field {
527                                    alias: None,
528                                    name: "id".to_string(),
529                                    arguments: vec![],
530                                    directives: vec![],
531                                    selection_set: None,
532                                }),
533                                Selection::Field(Field {
534                                    alias: None,
535                                    name: "name".to_string(),
536                                    arguments: vec![],
537                                    directives: vec![],
538                                    selection_set: None,
539                                }),
540                            ],
541                        }),
542                    })],
543                },
544            })],
545        }
546    }
547
548    /// Create a complex query for testing
549    fn create_complex_query(&self, id: usize) -> Document {
550        // More complex query with multiple levels and relationships
551        self.create_simple_query(id) // Simplified for now
552    }
553
554    /// Create other query types (simplified implementations)
555    fn create_deep_nested_query(&self, id: usize) -> Document {
556        self.create_simple_query(id)
557    }
558    fn create_federated_query(&self, id: usize) -> Document {
559        self.create_simple_query(id)
560    }
561    fn create_aggregation_query(&self, id: usize) -> Document {
562        self.create_simple_query(id)
563    }
564    fn create_subscription_query(&self, id: usize) -> Document {
565        self.create_simple_query(id)
566    }
567    fn create_bulk_operation_query(&self, id: usize) -> Document {
568        self.create_simple_query(id)
569    }
570    fn create_stress_test_query(&self, id: usize) -> Document {
571        self.create_simple_query(id)
572    }
573
574    /// Calculate benchmark result statistics
575    async fn calculate_benchmark_result(
576        &self,
577        scenario: &TestScenario,
578        strategy: &OptimizationStrategy,
579        response_times: &[Duration],
580        successful_requests: u64,
581        failed_requests: u64,
582        total_time: Duration,
583    ) -> Result<BenchmarkResult> {
584        if response_times.is_empty() {
585            // Get real system metrics even when no response times are available
586            let memory_usage_mb = system_monitor::get_current_memory_usage_mb().await;
587            let cpu_usage_percent = system_monitor::get_current_cpu_usage_percent().await;
588
589            return Ok(BenchmarkResult {
590                scenario: format!("{scenario:?}"),
591                strategy: format!("{strategy:?}"),
592                configuration: "default".to_string(),
593                total_requests: successful_requests + failed_requests,
594                successful_requests,
595                failed_requests,
596                average_response_time: Duration::from_millis(0),
597                min_response_time: Duration::from_millis(0),
598                max_response_time: Duration::from_millis(0),
599                p50_response_time: Duration::from_millis(0),
600                p95_response_time: Duration::from_millis(0),
601                p99_response_time: Duration::from_millis(0),
602                requests_per_second: 0.0,
603                throughput_mbps: 0.0,
604                memory_usage_mb,
605                cpu_usage_percent,
606                cache_hit_rate: 0.0,
607                error_rate: 100.0,
608                detailed_metrics: None,
609            });
610        }
611
612        let mut sorted_times = response_times.to_vec();
613        sorted_times.sort();
614
615        let total_requests = successful_requests + failed_requests;
616        let average_response_time =
617            response_times.iter().sum::<Duration>() / response_times.len() as u32;
618        let min_response_time = *sorted_times.first().unwrap_or(&Duration::from_millis(0));
619        let max_response_time = *sorted_times.last().unwrap_or(&Duration::from_millis(0));
620
621        let p50_index = (sorted_times.len() as f64 * 0.5) as usize;
622        let p95_index = (sorted_times.len() as f64 * 0.95) as usize;
623        let p99_index = (sorted_times.len() as f64 * 0.99) as usize;
624
625        let p50_response_time = sorted_times
626            .get(p50_index)
627            .copied()
628            .unwrap_or(Duration::from_millis(0));
629        let p95_response_time = sorted_times
630            .get(p95_index)
631            .copied()
632            .unwrap_or(Duration::from_millis(0));
633        let p99_response_time = sorted_times
634            .get(p99_index)
635            .copied()
636            .unwrap_or(Duration::from_millis(0));
637
638        let requests_per_second = if total_time.as_secs_f64() > 0.0 {
639            successful_requests as f64 / total_time.as_secs_f64()
640        } else {
641            0.0
642        };
643
644        let error_rate = if total_requests > 0 {
645            (failed_requests as f64 / total_requests as f64) * 100.0
646        } else {
647            0.0
648        };
649
650        // Get cache statistics if available
651        let cache_hit_rate = if let Some(cache) = &self.cache {
652            match cache.get_stats().await {
653                Ok(stats) if (stats.hits + stats.misses) > 0 => {
654                    (stats.hits as f64 / (stats.hits + stats.misses) as f64) * 100.0
655                }
656                _ => 0.0,
657            }
658        } else {
659            0.0
660        };
661
662        // Get real system metrics
663        let memory_usage_mb = system_monitor::get_current_memory_usage_mb().await;
664        let cpu_usage_percent = system_monitor::get_current_cpu_usage_percent().await;
665
666        // Calculate throughput based on actual request data and estimated response size
667        let avg_response_size_bytes = 2048.0; // Estimated average GraphQL response size
668        let throughput_mbps =
669            system_monitor::calculate_throughput_mbps(requests_per_second, avg_response_size_bytes);
670
671        Ok(BenchmarkResult {
672            scenario: format!("{scenario:?}"),
673            strategy: format!("{strategy:?}"),
674            configuration: "default".to_string(),
675            total_requests,
676            successful_requests,
677            failed_requests,
678            average_response_time,
679            min_response_time,
680            max_response_time,
681            p50_response_time,
682            p95_response_time,
683            p99_response_time,
684            requests_per_second,
685            throughput_mbps,
686            memory_usage_mb,
687            cpu_usage_percent,
688            cache_hit_rate,
689            error_rate,
690            detailed_metrics: None, // Could be populated if enabled
691        })
692    }
693
694    /// Generate comprehensive benchmark report
695    async fn generate_report(&self) -> Result<BenchmarkReport> {
696        let results = self.results.read().await.clone();
697
698        // Find best performing strategy
699        let best_result = results
700            .iter()
701            .min_by(|a, b| a.average_response_time.cmp(&b.average_response_time))
702            .cloned();
703
704        let best_strategy = best_result
705            .as_ref()
706            .map(|r| r.strategy.clone())
707            .unwrap_or_else(|| "None".to_string());
708
709        // Calculate rankings
710        let mut strategy_rankings = Vec::new();
711        let strategies: std::collections::HashSet<String> =
712            results.iter().map(|r| r.strategy.clone()).collect();
713
714        for strategy in strategies {
715            let strategy_results: Vec<_> =
716                results.iter().filter(|r| r.strategy == strategy).collect();
717
718            if !strategy_results.is_empty() {
719                let avg_response_time = strategy_results
720                    .iter()
721                    .map(|r| r.average_response_time.as_millis() as f64)
722                    .sum::<f64>()
723                    / strategy_results.len() as f64;
724
725                let avg_throughput = strategy_results
726                    .iter()
727                    .map(|r| r.requests_per_second)
728                    .sum::<f64>()
729                    / strategy_results.len() as f64;
730
731                let avg_error_rate = strategy_results.iter().map(|r| r.error_rate).sum::<f64>()
732                    / strategy_results.len() as f64;
733
734                // Simple scoring algorithm (lower is better for response time and error rate)
735                let score = avg_response_time + (100.0 - avg_throughput) + avg_error_rate;
736
737                strategy_rankings.push(StrategyRanking {
738                    strategy: strategy.clone(),
739                    score,
740                    response_time_rank: 0, // Would be calculated by sorting
741                    throughput_rank: 0,
742                    reliability_rank: 0,
743                });
744            }
745        }
746
747        // Sort rankings by score
748        strategy_rankings.sort_by(|a, b| {
749            a.score
750                .partial_cmp(&b.score)
751                .unwrap_or(std::cmp::Ordering::Equal)
752        });
753
754        // Generate recommendations
755        let recommendations = self.generate_recommendations(&results).await;
756
757        Ok(BenchmarkReport {
758            config: BenchmarkConfigSummary {
759                test_duration: self.config.test_duration,
760                concurrent_users: self.config.concurrent_users,
761                total_queries: self.config.queries_per_user * self.config.concurrent_users,
762                scenarios_tested: self.config.test_scenarios.len(),
763                strategies_tested: self.config.optimization_strategies.len(),
764            },
765            results,
766            summary: BenchmarkSummary {
767                best_strategy: best_strategy.clone(),
768                best_average_response_time: best_result
769                    .as_ref()
770                    .map(|r| r.average_response_time)
771                    .unwrap_or(Duration::from_millis(0)),
772                best_throughput: best_result
773                    .as_ref()
774                    .map(|r| r.requests_per_second)
775                    .unwrap_or(0.0),
776                best_cache_hit_rate: best_result
777                    .as_ref()
778                    .map(|r| r.cache_hit_rate)
779                    .unwrap_or(0.0),
780                overall_performance_improvement: 0.0, // Would calculate compared to baseline
781                strategy_rankings,
782            },
783            recommendations,
784            generated_at: std::time::SystemTime::now(),
785        })
786    }
787
788    /// Generate performance recommendations
789    async fn generate_recommendations(
790        &self,
791        results: &[BenchmarkResult],
792    ) -> Vec<PerformanceRecommendation> {
793        let mut recommendations = Vec::new();
794
795        // Analyze results and generate recommendations
796        let avg_error_rate =
797            results.iter().map(|r| r.error_rate).sum::<f64>() / results.len() as f64;
798        if avg_error_rate > 5.0 {
799            recommendations.push(PerformanceRecommendation {
800                category: "Reliability".to_string(),
801                recommendation: "High error rate detected. Consider implementing better error handling and retry mechanisms.".to_string(),
802                impact: "High".to_string(),
803                priority: "Critical".to_string(),
804            });
805        }
806
807        let avg_response_time = results
808            .iter()
809            .map(|r| r.average_response_time.as_millis() as f64)
810            .sum::<f64>()
811            / results.len() as f64;
812        if avg_response_time > 1000.0 {
813            recommendations.push(PerformanceRecommendation {
814                category: "Performance".to_string(),
815                recommendation:
816                    "Response times are high. Consider implementing hybrid optimization strategies."
817                        .to_string(),
818                impact: "Medium".to_string(),
819                priority: "High".to_string(),
820            });
821        }
822
823        let cache_results: Vec<_> = results.iter().filter(|r| r.cache_hit_rate > 0.0).collect();
824        if !cache_results.is_empty() {
825            let avg_cache_hit_rate = cache_results.iter().map(|r| r.cache_hit_rate).sum::<f64>()
826                / cache_results.len() as f64;
827
828            if avg_cache_hit_rate < 50.0 {
829                recommendations.push(PerformanceRecommendation {
830                    category: "Caching".to_string(),
831                    recommendation: "Low cache hit rate. Consider optimizing cache configuration and TTL settings.".to_string(),
832                    impact: "Medium".to_string(),
833                    priority: "Medium".to_string(),
834                });
835            }
836        }
837
838        recommendations
839    }
840
841    /// Export benchmark results to various formats
842    pub async fn export_results(&self, format: ExportFormat, path: &str) -> Result<()> {
843        let report = self.generate_report().await?;
844
845        match format {
846            ExportFormat::Json => {
847                let json = serde_json::to_string_pretty(&report)?;
848                tokio::fs::write(path, json).await?;
849            }
850            ExportFormat::Csv => {
851                let csv = self.results_to_csv(&report.results)?;
852                tokio::fs::write(path, csv).await?;
853            }
854            ExportFormat::Html => {
855                let html = self.results_to_html(&report)?;
856                tokio::fs::write(path, html).await?;
857            }
858        }
859
860        info!("Benchmark results exported to: {}", path);
861        Ok(())
862    }
863
864    /// Convert results to CSV format
865    fn results_to_csv(&self, results: &[BenchmarkResult]) -> Result<String> {
866        let mut csv = String::new();
867        csv.push_str("Scenario,Strategy,Configuration,TotalRequests,SuccessfulRequests,FailedRequests,AvgResponseTime,MinResponseTime,MaxResponseTime,RequestsPerSecond,ErrorRate,CacheHitRate\n");
868
869        for result in results {
870            csv.push_str(&format!(
871                "{},{},{},{},{},{},{},{},{},{:.2},{:.2},{:.2}\n",
872                result.scenario,
873                result.strategy,
874                result.configuration,
875                result.total_requests,
876                result.successful_requests,
877                result.failed_requests,
878                result.average_response_time.as_millis(),
879                result.min_response_time.as_millis(),
880                result.max_response_time.as_millis(),
881                result.requests_per_second,
882                result.error_rate,
883                result.cache_hit_rate,
884            ));
885        }
886
887        Ok(csv)
888    }
889
890    /// Convert results to HTML format
891    fn results_to_html(&self, report: &BenchmarkReport) -> Result<String> {
892        let mut html = String::new();
893        html.push_str("<!DOCTYPE html><html><head><title>Benchmark Report</title></head><body>");
894        html.push_str("<h1>Performance Benchmark Report</h1>");
895        html.push_str(&format!("<p>Generated at: {:?}</p>", report.generated_at));
896        html.push_str(&format!(
897            "<p>Best Strategy: {}</p>",
898            report.summary.best_strategy
899        ));
900        html.push_str("<table border='1'><tr><th>Scenario</th><th>Strategy</th><th>Avg Response Time</th><th>Requests/sec</th><th>Error Rate</th></tr>");
901
902        for result in &report.results {
903            html.push_str(&format!(
904                "<tr><td>{}</td><td>{}</td><td>{}ms</td><td>{:.2}</td><td>{:.2}%</td></tr>",
905                result.scenario,
906                result.strategy,
907                result.average_response_time.as_millis(),
908                result.requests_per_second,
909                result.error_rate,
910            ));
911        }
912
913        html.push_str("</table></body></html>");
914        Ok(html)
915    }
916}
917
918/// Export formats for benchmark results
919#[derive(Debug, Clone)]
920pub enum ExportFormat {
921    Json,
922    Csv,
923    Html,
924}
925
926#[cfg(test)]
927mod tests {
928    use super::*;
929
930    #[tokio::test]
931    async fn test_benchmark_suite_creation() {
932        let config = BenchmarkConfig::default();
933        let mut suite = PerformanceBenchmarkSuite::new(config);
934
935        assert!(suite.initialize_optimizers().await.is_ok());
936    }
937
938    #[tokio::test]
939    async fn test_query_generation() {
940        let config = BenchmarkConfig::default();
941        let suite = PerformanceBenchmarkSuite::new(config);
942
943        let queries = suite.generate_test_queries(&TestScenario::SimpleQuery, 5);
944        assert_eq!(queries.len(), 5);
945    }
946
947    #[test]
948    fn test_csv_export() {
949        let config = BenchmarkConfig::default();
950        let suite = PerformanceBenchmarkSuite::new(config);
951
952        let results = vec![BenchmarkResult {
953            scenario: "SimpleQuery".to_string(),
954            strategy: "ML".to_string(),
955            configuration: "default".to_string(),
956            total_requests: 100,
957            successful_requests: 95,
958            failed_requests: 5,
959            average_response_time: Duration::from_millis(150),
960            min_response_time: Duration::from_millis(50),
961            max_response_time: Duration::from_millis(300),
962            p50_response_time: Duration::from_millis(140),
963            p95_response_time: Duration::from_millis(250),
964            p99_response_time: Duration::from_millis(290),
965            requests_per_second: 50.0,
966            throughput_mbps: 1.5,
967            memory_usage_mb: 128.0,
968            cpu_usage_percent: 25.0,
969            cache_hit_rate: 75.0,
970            error_rate: 5.0,
971            detailed_metrics: None,
972        }];
973
974        let csv = suite.results_to_csv(&results).expect("should succeed");
975        assert!(csv.contains("SimpleQuery"));
976        assert!(csv.contains("ML"));
977        assert!(csv.contains("95"));
978    }
979}
oxirs_gql/benchmarking.rs

oxirs_gql/
benchmarking.rs
