1use crate::error::{StatsError, StatsResult};
16use crate::numerical_stability_analyzer::{
17 NumericalStabilityAnalyzer, StabilityAnalysisResult, StabilityConfig,
18};
19use crate::property_based_validation::{
20 ComprehensivePropertyTestSuite, PropertyTestConfig, PropertyTestResult,
21};
22use crate::scipy_benchmark_framework::{BenchmarkConfig, BenchmarkResult, ScipyBenchmarkFramework};
23use scirs2_core::ndarray::{Array1, ArrayView1};
24use serde::{Deserialize, Serialize};
25use std::collections::HashMap;
26use std::time::Instant;
27
28#[derive(Debug)]
30pub struct ComprehensiveValidationSuite {
31 benchmark_framework: ScipyBenchmarkFramework,
33 property_test_suite: ComprehensivePropertyTestSuite,
35 stability_analyzer: NumericalStabilityAnalyzer,
37 config: ValidationSuiteConfig,
39 cached_results: HashMap<String, ComprehensiveValidationResult>,
41}
42
43#[derive(Debug, Clone)]
45pub struct ValidationSuiteConfig {
46 pub benchmark_config: BenchmarkConfig,
48 pub property_config: PropertyTestConfig,
50 pub stability_config: StabilityConfig,
52 pub enable_cross_validation: bool,
54 pub enable_regression_detection: bool,
56 pub production_readiness_threshold: f64,
58}
59
60#[derive(Debug, Clone, Serialize, Deserialize)]
62pub struct ComprehensiveValidationResult {
63 pub function_name: String,
65 pub benchmark_results: Vec<BenchmarkResult>,
67 pub property_results: Vec<PropertyTestResult>,
69 pub stability_result: StabilityAnalysisResult,
71 pub overall_status: ValidationStatus,
73 pub production_readiness: ProductionReadinessAssessment,
75 pub cross_validation: CrossValidationAnalysis,
77 pub validation_time: std::time::Duration,
79 pub validated_at: chrono::DateTime<chrono::Utc>,
81}
82
83#[derive(Debug, Clone, Copy, Serialize, Deserialize, PartialEq, Eq)]
85pub enum ValidationStatus {
86 FullyValidated,
88 MostlyValidated,
90 PartiallyValidated,
92 PoorlyValidated,
94 ValidationFailed,
96}
97
98#[derive(Debug, Clone, Serialize, Deserialize)]
100pub struct ProductionReadinessAssessment {
101 pub is_production_ready: bool,
103 pub readiness_score: f64,
105 pub readiness_criteria: ReadinessCriteria,
107 pub production_blockers: Vec<ProductionBlocker>,
109 pub recommendations: Vec<ProductionRecommendation>,
111}
112
113#[derive(Debug, Clone, Serialize, Deserialize)]
115pub struct ReadinessCriteria {
116 pub accuracy_ready: bool,
118 pub performance_ready: bool,
120 pub stability_ready: bool,
122 pub error_handling_ready: bool,
124 pub documentation_ready: bool,
126}
127
128#[derive(Debug, Clone, Serialize, Deserialize)]
130pub struct ProductionBlocker {
131 pub blocker_type: BlockerType,
133 pub description: String,
135 pub severity: BlockerSeverity,
137 pub resolution_effort: ResolutionEffort,
139}
140
141#[derive(Debug, Clone, Copy, Serialize, Deserialize, PartialEq, Eq)]
143pub enum BlockerType {
144 Accuracy,
146 Performance,
148 Stability,
150 API,
152 ErrorHandling,
154 Documentation,
156}
157
158#[derive(Debug, Clone, Copy, Serialize, Deserialize, PartialEq, Eq, PartialOrd, Ord)]
160pub enum BlockerSeverity {
161 Critical,
163 High,
165 Medium,
167 Low,
169}
170
171#[derive(Debug, Clone, Copy, Serialize, Deserialize, PartialEq, Eq)]
173pub enum ResolutionEffort {
174 Minimal,
176 Low,
178 Medium,
180 High,
182 VeryHigh,
184}
185
186#[derive(Debug, Clone, Serialize, Deserialize)]
188pub struct ProductionRecommendation {
189 pub area: RecommendationArea,
191 pub recommendation: String,
193 pub priority: RecommendationPriority,
195 pub expected_impact: f64,
197}
198
199#[derive(Debug, Clone, Copy, Serialize, Deserialize, PartialEq, Eq)]
201pub enum RecommendationArea {
202 Algorithm,
204 Performance,
206 ErrorHandling,
208 Testing,
210 Documentation,
212}
213
214#[derive(Debug, Clone, Copy, Serialize, Deserialize, PartialEq, Eq, PartialOrd, Ord)]
216pub enum RecommendationPriority {
217 Critical,
219 High,
221 Medium,
223 Low,
225}
226
227#[derive(Debug, Clone, Serialize, Deserialize)]
229pub struct CrossValidationAnalysis {
230 pub benchmark_stability_correlation: f64,
232 pub property_benchmark_correlation: f64,
234 pub property_stability_correlation: f64,
236 pub framework_agreement: f64,
238 pub validation_confidence: f64,
240}
241
242#[derive(Debug, Clone, Serialize, Deserialize)]
244pub struct ComprehensiveValidationReport {
245 pub total_functions: usize,
247 pub production_ready_functions: usize,
249 pub functions_needing_improvement: usize,
251 pub validation_summary: ValidationSummary,
253 pub function_results: Vec<ComprehensiveValidationResult>,
255 pub framework_analysis: FrameworkAnalysis,
257 pub overall_production_readiness: OverallProductionReadiness,
259 pub generated_at: chrono::DateTime<chrono::Utc>,
261}
262
263#[derive(Debug, Clone, Serialize, Deserialize)]
265pub struct ValidationSummary {
266 pub average_benchmark_score: f64,
268 pub average_property_pass_rate: f64,
270 pub average_stability_score: f64,
272 pub overall_validation_score: f64,
274}
275
276#[derive(Debug, Clone, Serialize, Deserialize)]
278pub struct FrameworkAnalysis {
279 pub benchmark_reliability: f64,
281 pub property_test_reliability: f64,
283 pub stability_reliability: f64,
285 pub inter_framework_agreement: f64,
287}
288
289#[derive(Debug, Clone, Serialize, Deserialize)]
291pub struct OverallProductionReadiness {
292 pub is_production_ready: bool,
294 pub production_ready_percentage: f64,
296 pub critical_blockers: Vec<ProductionBlocker>,
298 pub top_recommendations: Vec<ProductionRecommendation>,
300}
301
302impl Default for ValidationSuiteConfig {
303 fn default() -> Self {
304 Self {
305 benchmark_config: BenchmarkConfig::default(),
306 property_config: PropertyTestConfig::default(),
307 stability_config: StabilityConfig::default(),
308 enable_cross_validation: true,
309 enable_regression_detection: true,
310 production_readiness_threshold: 0.85,
311 }
312 }
313}
314
315impl ComprehensiveValidationSuite {
316 pub fn new(config: ValidationSuiteConfig) -> Self {
318 Self {
319 benchmark_framework: ScipyBenchmarkFramework::new(config.benchmark_config.clone()),
320 property_test_suite: ComprehensivePropertyTestSuite::new(
321 config.property_config.clone(),
322 ),
323 stability_analyzer: NumericalStabilityAnalyzer::new(config.stability_config.clone()),
324 config,
325 cached_results: HashMap::new(),
326 }
327 }
328
329 pub fn default() -> Self {
331 Self::new(ValidationSuiteConfig::default())
332 }
333
334 pub fn validate_function<F, G>(
336 &mut self,
337 function_name: &str,
338 scirs2_impl: F,
339 scipy_reference: Option<G>,
340 ) -> StatsResult<ComprehensiveValidationResult>
341 where
342 F: Fn(&ArrayView1<f64>) -> StatsResult<f64> + Clone,
343 G: Fn(&ArrayView1<f64>) -> f64,
344 {
345 let start_time = Instant::now();
346
347 let benchmark_results = if let Some(scipy_func) = scipy_reference {
349 self.benchmark_framework.benchmark_function(
350 function_name,
351 scirs2_impl.clone(),
352 scipy_func,
353 )?
354 } else {
355 Vec::new()
356 };
357
358 let property_results = self.property_test_suite.test_function(function_name)?;
360
361 let testdata = self.generate_testdata(1000)?;
363 let stability_result = self.stability_analyzer.analyze_function(
364 function_name,
365 scirs2_impl,
366 &testdata.view(),
367 )?;
368
369 let cross_validation = if self.config.enable_cross_validation {
371 self.perform_cross_validation(&benchmark_results, &property_results, &stability_result)
372 } else {
373 CrossValidationAnalysis {
374 benchmark_stability_correlation: 0.0,
375 property_benchmark_correlation: 0.0,
376 property_stability_correlation: 0.0,
377 framework_agreement: 0.0,
378 validation_confidence: 0.5,
379 }
380 };
381
382 let overall_status =
384 self.determine_overall_status(&benchmark_results, &property_results, &stability_result);
385
386 let production_readiness = self.assess_production_readiness(
388 &benchmark_results,
389 &property_results,
390 &stability_result,
391 &cross_validation,
392 );
393
394 let validation_time = start_time.elapsed();
395
396 let result = ComprehensiveValidationResult {
397 function_name: function_name.to_string(),
398 benchmark_results,
399 property_results,
400 stability_result,
401 overall_status,
402 production_readiness,
403 cross_validation,
404 validation_time,
405 validated_at: chrono::Utc::now(),
406 };
407
408 self.cached_results
409 .insert(function_name.to_string(), result.clone());
410 Ok(result)
411 }
412
413 fn generate_testdata(&self, size: usize) -> StatsResult<Array1<f64>> {
415 use scirs2_core::random::prelude::*;
416 use scirs2_core::random::{Distribution, Normal};
417
418 let mut rng = StdRng::seed_from_u64(self.config.property_config.seed);
419 let normal = Normal::new(0.0, 1.0)
420 .map_err(|e| StatsError::InvalidInput(format!("Distribution error: {}", e)))?;
421
422 let mut data = Array1::zeros(size);
423 for val in data.iter_mut() {
424 *val = normal.sample(&mut rng);
425 }
426
427 Ok(data)
428 }
429
430 fn perform_cross_validation(
432 &self,
433 benchmark_results: &[BenchmarkResult],
434 property_results: &[PropertyTestResult],
435 stability_result: &StabilityAnalysisResult,
436 ) -> CrossValidationAnalysis {
437 let benchmark_score = if !benchmark_results.is_empty() {
439 benchmark_results
440 .iter()
441 .map(|r| match r.status {
442 crate::scipy_benchmark_framework::BenchmarkStatus::Pass => 1.0,
443 crate::scipy_benchmark_framework::BenchmarkStatus::AccuracyPass => 0.7,
444 crate::scipy_benchmark_framework::BenchmarkStatus::PerformancePass => 0.7,
445 crate::scipy_benchmark_framework::BenchmarkStatus::Fail => 0.0,
446 crate::scipy_benchmark_framework::BenchmarkStatus::Error => 0.0,
447 })
448 .sum::<f64>()
449 / benchmark_results.len() as f64
450 } else {
451 0.5
452 };
453
454 let property_score = if !property_results.is_empty() {
455 property_results
456 .iter()
457 .map(|r| r.test_cases_passed as f64 / r.test_cases_run.max(1) as f64)
458 .sum::<f64>()
459 / property_results.len() as f64
460 } else {
461 0.5
462 };
463
464 let stability_score = stability_result.stability_score / 100.0;
465
466 let benchmark_stability_correlation = 1.0 - (benchmark_score - stability_score).abs();
468 let property_benchmark_correlation = 1.0 - (property_score - benchmark_score).abs();
469 let property_stability_correlation = 1.0 - (property_score - stability_score).abs();
470
471 let framework_agreement = (benchmark_stability_correlation
472 + property_benchmark_correlation
473 + property_stability_correlation)
474 / 3.0;
475
476 let validation_confidence = framework_agreement;
477
478 CrossValidationAnalysis {
479 benchmark_stability_correlation,
480 property_benchmark_correlation,
481 property_stability_correlation,
482 framework_agreement,
483 validation_confidence,
484 }
485 }
486
487 fn determine_overall_status(
489 &self,
490 benchmark_results: &[BenchmarkResult],
491 property_results: &[PropertyTestResult],
492 stability_result: &StabilityAnalysisResult,
493 ) -> ValidationStatus {
494 let mut validation_scores = Vec::new();
495
496 if !benchmark_results.is_empty() {
507 let benchmark_pass_rate = benchmark_results
508 .iter()
509 .map(|r| {
510 use crate::scipy_benchmark_framework::BenchmarkStatus;
511 match r.status {
512 BenchmarkStatus::Pass => 1.0,
513 BenchmarkStatus::AccuracyPass | BenchmarkStatus::PerformancePass => 0.7,
514 BenchmarkStatus::Fail | BenchmarkStatus::Error => 0.0,
515 }
516 })
517 .sum::<f64>()
518 / benchmark_results.len() as f64;
519 validation_scores.push(benchmark_pass_rate);
520 }
521
522 if !property_results.is_empty() {
524 let property_pass_rate = property_results
525 .iter()
526 .map(|r| r.test_cases_passed as f64 / r.test_cases_run.max(1) as f64)
527 .sum::<f64>()
528 / property_results.len() as f64;
529 validation_scores.push(property_pass_rate);
530 }
531
532 validation_scores.push(stability_result.stability_score / 100.0);
534
535 let average_score = validation_scores.iter().sum::<f64>() / validation_scores.len() as f64;
536
537 if average_score >= 0.9 {
538 ValidationStatus::FullyValidated
539 } else if average_score >= 0.75 {
540 ValidationStatus::MostlyValidated
541 } else if average_score >= 0.5 {
542 ValidationStatus::PartiallyValidated
543 } else if average_score >= 0.25 {
544 ValidationStatus::PoorlyValidated
545 } else {
546 ValidationStatus::ValidationFailed
547 }
548 }
549
550 fn assess_production_readiness(
552 &self,
553 benchmark_results: &[BenchmarkResult],
554 property_results: &[PropertyTestResult],
555 stability_result: &StabilityAnalysisResult,
556 cross_validation: &CrossValidationAnalysis,
557 ) -> ProductionReadinessAssessment {
558 let mut readiness_score = 0.0;
559 let mut production_blockers = Vec::new();
560 let recommendations = Vec::new();
561
562 let accuracy_ready = benchmark_results
564 .iter()
565 .all(|r| r.accuracy.passes_tolerance);
566 if accuracy_ready {
567 readiness_score += 25.0;
568 } else {
569 production_blockers.push(ProductionBlocker {
570 blocker_type: BlockerType::Accuracy,
571 description: "Accuracy does not meet tolerance requirements".to_string(),
572 severity: BlockerSeverity::Critical,
573 resolution_effort: ResolutionEffort::Medium,
574 });
575 }
576
577 let performance_ready = benchmark_results.iter().all(|r| {
579 matches!(
580 r.performance.performance_grade,
581 crate::scipy_benchmark_framework::PerformanceGrade::A
582 | crate::scipy_benchmark_framework::PerformanceGrade::B
583 | crate::scipy_benchmark_framework::PerformanceGrade::C
584 )
585 });
586 if performance_ready {
587 readiness_score += 20.0;
588 }
589
590 let stability_ready = matches!(
592 stability_result.stability_grade,
593 crate::numerical_stability_analyzer::StabilityGrade::Excellent
594 | crate::numerical_stability_analyzer::StabilityGrade::Good
595 );
596 if stability_ready {
597 readiness_score += 25.0;
598 }
599
600 let property_ready = property_results.iter().all(|r| {
602 matches!(
603 r.status,
604 crate::property_based_validation::PropertyTestStatus::Pass
605 )
606 });
607 if property_ready {
608 readiness_score += 20.0;
609 }
610
611 if cross_validation.validation_confidence > 0.8 {
613 readiness_score += 10.0;
614 }
615
616 let is_production_ready =
617 readiness_score >= (self.config.production_readiness_threshold * 100.0);
618
619 let readiness_criteria = ReadinessCriteria {
620 accuracy_ready,
621 performance_ready,
622 stability_ready,
623 error_handling_ready: true, documentation_ready: true, };
626
627 ProductionReadinessAssessment {
628 is_production_ready,
629 readiness_score,
630 readiness_criteria,
631 production_blockers,
632 recommendations,
633 }
634 }
635
636 pub fn generate_comprehensive_report(&self) -> ComprehensiveValidationReport {
638 let function_results: Vec<_> = self.cached_results.values().cloned().collect();
639
640 let total_functions = function_results.len();
641 let production_ready_functions = function_results
642 .iter()
643 .filter(|r| r.production_readiness.is_production_ready)
644 .count();
645 let functions_needing_improvement = total_functions - production_ready_functions;
646
647 let average_benchmark_score = if total_functions > 0 {
649 function_results
650 .iter()
651 .map(|r| {
652 r.benchmark_results
653 .iter()
654 .map(|b| {
655 if matches!(
656 b.status,
657 crate::scipy_benchmark_framework::BenchmarkStatus::Pass
658 ) {
659 100.0
660 } else {
661 0.0
662 }
663 })
664 .sum::<f64>()
665 / r.benchmark_results.len().max(1) as f64
666 })
667 .sum::<f64>()
668 / total_functions as f64
669 } else {
670 0.0
671 };
672
673 let average_stability_score = if total_functions > 0 {
674 function_results
675 .iter()
676 .map(|r| r.stability_result.stability_score)
677 .sum::<f64>()
678 / total_functions as f64
679 } else {
680 0.0
681 };
682
683 let validation_summary = ValidationSummary {
684 average_benchmark_score,
685 average_property_pass_rate: 0.0, average_stability_score,
687 overall_validation_score: (average_benchmark_score + average_stability_score) / 2.0,
688 };
689
690 let framework_analysis = FrameworkAnalysis {
691 benchmark_reliability: 0.9, property_test_reliability: 0.85,
693 stability_reliability: 0.8,
694 inter_framework_agreement: function_results
695 .iter()
696 .map(|r| r.cross_validation.framework_agreement)
697 .sum::<f64>()
698 / total_functions.max(1) as f64,
699 };
700
701 let overall_production_readiness = OverallProductionReadiness {
702 is_production_ready: production_ready_functions as f64 / total_functions.max(1) as f64
703 > self.config.production_readiness_threshold,
704 production_ready_percentage: production_ready_functions as f64
705 / total_functions.max(1) as f64
706 * 100.0,
707 critical_blockers: Vec::new(), top_recommendations: Vec::new(), };
710
711 ComprehensiveValidationReport {
712 total_functions,
713 production_ready_functions,
714 functions_needing_improvement,
715 validation_summary,
716 function_results,
717 framework_analysis,
718 overall_production_readiness,
719 generated_at: chrono::Utc::now(),
720 }
721 }
722}
723
724#[cfg(test)]
725mod tests {
726 use super::*;
727 use crate::descriptive::mean;
728
729 #[test]
730 fn test_comprehensive_validation_suite_creation() {
731 let suite = ComprehensiveValidationSuite::default();
732 assert_eq!(suite.config.production_readiness_threshold, 0.85);
733 }
734
735 #[test]
736 fn test_validation_status_determination() {
737 let suite = ComprehensiveValidationSuite::default();
738
739 let benchmark_results = vec![];
741 let property_results = vec![];
742 let stability_result = crate::numerical_stability_analyzer::StabilityAnalysisResult {
743 function_name: "test".to_string(),
744 stability_grade: crate::numerical_stability_analyzer::StabilityGrade::Excellent,
745 condition_analysis: crate::numerical_stability_analyzer::ConditionNumberAnalysis {
746 condition_number: 1.0,
747 conditioning_class:
748 crate::numerical_stability_analyzer::ConditioningClass::WellConditioned,
749 accuracy_loss_digits: 0.0,
750 input_sensitivity: 0.0,
751 },
752 error_propagation: crate::numerical_stability_analyzer::ErrorPropagationAnalysis {
753 forward_error_bound: 0.0,
754 backward_error_bound: 0.0,
755 error_amplification: 1.0,
756 rounding_error_stability: 1.0,
757 },
758 edge_case_robustness: crate::numerical_stability_analyzer::EdgeCaseRobustness {
759 handles_infinity: true,
760 handles_nan: true,
761 handles_zero: true,
762 handles_large_values: true,
763 handles_small_values: true,
764 edge_case_success_rate: 1.0,
765 },
766 precision_analysis: crate::numerical_stability_analyzer::PrecisionAnalysis {
767 precision_loss_bits: 0.0,
768 relative_precision: 1.0,
769 cancellation_errors: vec![],
770 overflow_underflow_risk: crate::numerical_stability_analyzer::OverflowRisk::None,
771 },
772 recommendations: vec![],
773 stability_score: 95.0,
774 };
775
776 let status = suite.determine_overall_status(
777 &benchmark_results,
778 &property_results,
779 &stability_result,
780 );
781 assert_eq!(status, ValidationStatus::FullyValidated);
782 }
783
784 #[test]
785 fn test_mean_comprehensive_validation() {
786 let mut suite = ComprehensiveValidationSuite::new(ValidationSuiteConfig {
787 benchmark_config: BenchmarkConfig {
788 testsizes: vec![100],
789 performance_iterations: 5,
790 warmup_iterations: 1,
791 ..Default::default()
792 },
793 property_config: PropertyTestConfig {
794 test_cases_per_property: 10,
795 ..Default::default()
796 },
797 ..Default::default()
798 });
799
800 let scipy_mean = |data: &ArrayView1<f64>| -> f64 { data.sum() / data.len() as f64 };
802
803 let result = suite
804 .validate_function("mean", |data| mean(data), Some(scipy_mean))
805 .expect("Operation failed");
806
807 assert_eq!(result.function_name, "mean");
808 assert!(matches!(
809 result.overall_status,
810 ValidationStatus::FullyValidated | ValidationStatus::MostlyValidated
811 ));
812 assert!(result.validation_time.as_secs() < 60); }
814}