use crate ::measurement ::BenchmarkResult;
use std ::collections ::HashMap;
use std ::time ::SystemTime;
type Result< T > = std ::result ::Result< T, Box< dyn std ::error ::Error > >;
#[ derive( Debug, Clone ) ]
pub struct HistoricalResults
{
baseline_data: HashMap< String, BenchmarkResult >,
historical_runs: Vec< TimestampedResults >,
}
#[ derive( Debug, Clone ) ]
pub struct TimestampedResults
{
timestamp: SystemTime,
results: HashMap< String, BenchmarkResult >,
}
impl TimestampedResults
{
#[ must_use ]
pub fn new( timestamp: SystemTime, results: HashMap< String, BenchmarkResult > ) -> Self
{
Self { timestamp, results }
}
#[ must_use ]
pub fn timestamp( &self ) -> SystemTime
{
self.timestamp
}
#[ must_use ]
pub fn results( &self ) -> &HashMap< String, BenchmarkResult >
{
&self.results
}
}
impl HistoricalResults
{
#[ must_use ]
pub fn new() -> Self
{
Self
{
baseline_data: HashMap ::new(),
historical_runs: Vec ::new(),
}
}
#[ must_use ]
pub fn with_baseline( mut self, baseline: HashMap< String, BenchmarkResult > ) -> Self
{
self.baseline_data = baseline;
self
}
#[ must_use ]
pub fn with_historical_run( mut self, timestamp: SystemTime, results: HashMap< String, BenchmarkResult > ) -> Self
{
self.historical_runs.push( TimestampedResults ::new( timestamp, results ) );
self
}
#[ must_use ]
pub fn with_historical_runs( mut self, runs: Vec< TimestampedResults > ) -> Self
{
self.historical_runs = runs;
self
}
#[ must_use ]
pub fn with_previous_run( mut self, run: TimestampedResults ) -> Self
{
self.historical_runs = vec![ run ];
self
}
#[ must_use ]
pub fn baseline_data( &self ) -> &HashMap< String, BenchmarkResult >
{
&self.baseline_data
}
#[ must_use ]
pub fn historical_runs( &self ) -> &Vec< TimestampedResults >
{
&self.historical_runs
}
}
impl Default for HistoricalResults
{
fn default() -> Self
{
Self ::new()
}
}
#[ derive( Debug, Clone, PartialEq ) ]
pub enum BaselineStrategy
{
FixedBaseline,
RollingAverage,
PreviousRun,
}
#[ derive( Debug, Clone, PartialEq ) ]
pub enum PerformanceTrend
{
Improving,
Degrading,
Stable,
}
#[ derive( Debug, Clone ) ]
pub struct RegressionAnalyzer
{
significance_threshold: f64,
trend_window: usize,
baseline_strategy: BaselineStrategy,
}
impl RegressionAnalyzer
{
#[ must_use ]
pub fn new() -> Self
{
Self
{
significance_threshold: 0.05,
trend_window: 5,
baseline_strategy: BaselineStrategy ::FixedBaseline,
}
}
#[ must_use ]
pub fn with_baseline_strategy( mut self, strategy: BaselineStrategy ) -> Self
{
self.baseline_strategy = strategy;
self
}
#[ must_use ]
pub fn with_significance_threshold( mut self, threshold: f64 ) -> Self
{
self.significance_threshold = threshold;
self
}
#[ must_use ]
pub fn with_trend_window( mut self, window: usize ) -> Self
{
self.trend_window = window;
self
}
#[ must_use ]
pub fn analyze( &self, results: &HashMap< String, BenchmarkResult >, historical: &HistoricalResults ) -> RegressionReport
{
let mut report = RegressionReport ::new();
for ( operation_name, current_result ) in results
{
let analysis = self.analyze_single_operation( operation_name, current_result, historical );
report.add_operation_analysis( operation_name.clone(), analysis );
}
report
}
fn analyze_single_operation( &self, operation_name: &str, current_result: &BenchmarkResult, historical: &HistoricalResults ) -> OperationAnalysis
{
match self.baseline_strategy
{
BaselineStrategy ::FixedBaseline => self.analyze_against_fixed_baseline( operation_name, current_result, historical ),
BaselineStrategy ::RollingAverage => self.analyze_against_rolling_average( operation_name, current_result, historical ),
BaselineStrategy ::PreviousRun => self.analyze_against_previous_run( operation_name, current_result, historical ),
}
}
fn analyze_against_fixed_baseline( &self, operation_name: &str, current_result: &BenchmarkResult, historical: &HistoricalResults ) -> OperationAnalysis
{
if let Some( baseline_result ) = historical.baseline_data().get( operation_name )
{
let current_time = current_result.mean_time().as_secs_f64();
let baseline_time = baseline_result.mean_time().as_secs_f64();
let improvement_ratio = baseline_time / current_time;
let trend = if improvement_ratio > 1.0 + self.significance_threshold
{
PerformanceTrend ::Improving
}
else if improvement_ratio < 1.0 - self.significance_threshold
{
PerformanceTrend ::Degrading
}
else
{
PerformanceTrend ::Stable
};
let is_significant = ( improvement_ratio - 1.0 ).abs() > self.significance_threshold;
OperationAnalysis
{
trend,
improvement_ratio,
is_statistically_significant: is_significant,
baseline_time: Some( baseline_time ),
has_historical_data: true,
}
}
else
{
OperationAnalysis ::no_data()
}
}
fn analyze_against_rolling_average( &self, operation_name: &str, current_result: &BenchmarkResult, historical: &HistoricalResults ) -> OperationAnalysis
{
let historical_runs = historical.historical_runs();
if historical_runs.is_empty()
{
return OperationAnalysis ::no_data();
}
let recent_runs: Vec< _ > = historical_runs
.iter()
.rev() .take( self.trend_window )
.filter_map( | run | run.results().get( operation_name ) )
.collect();
if recent_runs.is_empty()
{
return OperationAnalysis ::no_data();
}
let avg_time = recent_runs.iter()
.map( | result | result.mean_time().as_secs_f64() )
.sum :: < f64 >() / recent_runs.len() as f64;
let current_time = current_result.mean_time().as_secs_f64();
let improvement_ratio = avg_time / current_time;
let trend = if improvement_ratio > 1.0 + self.significance_threshold
{
PerformanceTrend ::Improving
}
else if improvement_ratio < 1.0 - self.significance_threshold
{
PerformanceTrend ::Degrading
}
else
{
PerformanceTrend ::Stable
};
let is_significant = ( improvement_ratio - 1.0 ).abs() > self.significance_threshold;
OperationAnalysis
{
trend,
improvement_ratio,
is_statistically_significant: is_significant,
baseline_time: Some( avg_time ),
has_historical_data: true,
}
}
fn analyze_against_previous_run( &self, operation_name: &str, current_result: &BenchmarkResult, historical: &HistoricalResults ) -> OperationAnalysis
{
let historical_runs = historical.historical_runs();
if let Some( previous_run ) = historical_runs.last()
{
if let Some( previous_result ) = previous_run.results().get( operation_name )
{
let current_time = current_result.mean_time().as_secs_f64();
let previous_time = previous_result.mean_time().as_secs_f64();
let improvement_ratio = previous_time / current_time;
let trend = if improvement_ratio > 1.0 + self.significance_threshold
{
PerformanceTrend ::Improving
}
else if improvement_ratio < 1.0 - self.significance_threshold
{
PerformanceTrend ::Degrading
}
else
{
PerformanceTrend ::Stable
};
let is_significant = ( improvement_ratio - 1.0 ).abs() > self.significance_threshold;
OperationAnalysis
{
trend,
improvement_ratio,
is_statistically_significant: is_significant,
baseline_time: Some( previous_time ),
has_historical_data: true,
}
}
else
{
OperationAnalysis ::no_data()
}
}
else
{
OperationAnalysis ::no_data()
}
}
}
impl Default for RegressionAnalyzer
{
fn default() -> Self
{
Self ::new()
}
}
#[ derive( Debug, Clone ) ]
pub struct OperationAnalysis
{
trend: PerformanceTrend,
improvement_ratio: f64,
is_statistically_significant: bool,
baseline_time: Option< f64 >,
has_historical_data: bool,
}
impl OperationAnalysis
{
#[ must_use ]
fn no_data() -> Self
{
Self
{
trend: PerformanceTrend ::Stable,
improvement_ratio: 1.0,
is_statistically_significant: false,
baseline_time: None,
has_historical_data: false,
}
}
}
#[ derive( Debug, Clone ) ]
pub struct RegressionReport
{
operations: HashMap< String, OperationAnalysis >,
}
impl RegressionReport
{
#[ must_use ]
fn new() -> Self
{
Self
{
operations: HashMap ::new(),
}
}
fn add_operation_analysis( &mut self, operation: String, analysis: OperationAnalysis )
{
self.operations.insert( operation, analysis );
}
#[ must_use ]
pub fn has_significant_changes( &self ) -> bool
{
self.operations.values().any( | analysis | analysis.is_statistically_significant )
}
#[ must_use ]
pub fn get_trend_for( &self, operation: &str ) -> Option< PerformanceTrend >
{
self.operations.get( operation ).map( | analysis | analysis.trend.clone() )
}
#[ must_use ]
pub fn is_statistically_significant( &self, operation: &str ) -> bool
{
self.operations.get( operation )
.is_some_and( | analysis | analysis.is_statistically_significant )
}
#[ must_use ]
pub fn has_historical_data( &self, operation: &str ) -> bool
{
self.operations.get( operation )
.is_some_and( | analysis | analysis.has_historical_data )
}
#[ must_use ]
pub fn has_previous_run_data( &self ) -> bool
{
self.operations.values().any( | analysis | analysis.has_historical_data )
}
#[ must_use ]
pub fn format_markdown( &self ) -> String
{
let mut output = String ::new();
output.push_str( "### Performance Comparison Against Baseline\n\n" );
for ( operation_name, analysis ) in &self.operations
{
if !analysis.has_historical_data
{
output.push_str( &format!(
"**{}** : ℹ️ **New operation** - no baseline data available for comparison\n\n",
operation_name
) );
continue;
}
if let Some( _baseline_time ) = analysis.baseline_time
{
let improvement_percent = ( analysis.improvement_ratio - 1.0 ) * 100.0;
match analysis.trend
{
PerformanceTrend ::Improving =>
{
output.push_str( &format!(
"**{}** : 🎉 **Performance improvement detected** - {:.1}% faster than baseline\n\n",
operation_name,
improvement_percent
) );
},
PerformanceTrend ::Degrading =>
{
output.push_str( &format!(
"**{}** : ⚠️ **Performance regression detected** - {:.1}% slower than baseline\n\n",
operation_name,
improvement_percent.abs()
) );
},
PerformanceTrend ::Stable =>
{
output.push_str( &format!(
"**{}** : ✅ **Performance stable** - within normal variation of baseline\n\n",
operation_name
) );
},
}
}
}
output.push_str( "### Analysis Summary & Recommendations\n\n" );
output.push_str( "Regression analysis complete. See individual operation results above for detailed findings.\n\n" );
output
}
}
pub trait ReportTemplate
{
fn generate( &self, results: &HashMap< String, BenchmarkResult > ) -> Result< String >;
}
#[ derive( Debug, Clone ) ]
pub struct PerformanceReport
{
title: String,
context: Option< String >,
include_statistical_analysis: bool,
include_regression_analysis: bool,
custom_sections: Vec< CustomSection >,
historical_data: Option< HistoricalResults >,
}
impl PerformanceReport
{
#[ must_use ]
pub fn new() -> Self
{
Self
{
title: "Performance Analysis".to_string(),
context: None,
include_statistical_analysis: true,
include_regression_analysis: false,
custom_sections: Vec ::new(),
historical_data: None,
}
}
#[ must_use ]
pub fn title( mut self, title: impl Into< String > ) -> Self
{
self.title = title.into();
self
}
#[ must_use ]
pub fn add_context( mut self, context: impl Into< String > ) -> Self
{
self.context = Some( context.into() );
self
}
#[ must_use ]
pub fn include_statistical_analysis( mut self, include: bool ) -> Self
{
self.include_statistical_analysis = include;
self
}
#[ must_use ]
pub fn include_regression_analysis( mut self, include: bool ) -> Self
{
self.include_regression_analysis = include;
self
}
#[ must_use ]
pub fn add_custom_section( mut self, section: CustomSection ) -> Self
{
self.custom_sections.push( section );
self
}
#[ must_use ]
pub fn with_historical_data( mut self, historical: HistoricalResults ) -> Self
{
self.historical_data = Some( historical );
self
}
}
impl Default for PerformanceReport
{
fn default() -> Self
{
Self ::new()
}
}
impl ReportTemplate for PerformanceReport
{
fn generate( &self, results: &HashMap< String, BenchmarkResult > ) -> Result< String >
{
let mut output = String ::new();
output.push_str( &format!( "# {}\n\n", self.title ) );
if let Some( ref context ) = self.context
{
output.push_str( &format!( "*{}*\n\n", context ) );
}
if results.is_empty()
{
output.push_str( "No benchmark results available.\n" );
return Ok( output );
}
output.push_str( "## Executive Summary\n\n" );
self.add_executive_summary( &mut output, results );
output.push_str( "## Performance Results\n\n" );
self.add_performance_table( &mut output, results );
if self.include_statistical_analysis
{
output.push_str( "## Statistical Analysis\n\n" );
self.add_statistical_analysis( &mut output, results );
}
if self.include_regression_analysis
{
output.push_str( "## Regression Analysis\n\n" );
self.add_regression_analysis( &mut output, results );
}
for section in &self.custom_sections
{
output.push_str( &format!( "## {}\n\n", section.title ) );
output.push_str( §ion.content );
output.push_str( "\n\n" );
}
output.push_str( "## Methodology\n\n" );
self.add_methodology_note( &mut output );
Ok( output )
}
}
impl PerformanceReport
{
fn add_executive_summary( &self, output: &mut String, results: &HashMap< String, BenchmarkResult > )
{
let total_tests = results.len();
let reliable_tests = results.values().filter( | r | r.is_reliable() ).count();
let reliability_rate = ( reliable_tests as f64 / total_tests as f64 ) * 100.0;
output.push_str( &format!( "- **Total operations benchmarked** : {}\n", total_tests ) );
output.push_str( &format!( "- **Statistically reliable results** : {}/{} ({:.1}%)\n",
reliable_tests, total_tests, reliability_rate ) );
if let Some( ( fastest_name, fastest_result ) ) = self.find_fastest( results )
{
output.push_str( &format!( "- **Best performing operation** : {} ({:.2?})\n",
fastest_name, fastest_result.mean_time() ) );
}
if results.len() > 1
{
if let Some( ( slowest_name, slowest_result ) ) = self.find_slowest( results )
{
if let Some( ( fastest_name_inner, fastest_result ) ) = self.find_fastest( results )
{
let ratio = slowest_result.mean_time().as_secs_f64() / fastest_result.mean_time().as_secs_f64();
output.push_str( &format!( "- **Performance range** : {:.1}x difference ({} vs {})\n",
ratio, fastest_name_inner, slowest_name ) );
}
}
}
output.push_str( "\n" );
}
fn add_performance_table( &self, output: &mut String, results: &HashMap< String, BenchmarkResult > )
{
output.push_str( "| Operation | Mean Time | 95% CI | Ops/sec | CV | Reliability | Samples |\n" );
output.push_str( "|-----------|-----------|--------|---------|----|-----------|---------|\n" );
let mut sorted_results: Vec< _ > = results.iter().collect();
sorted_results.sort_by( | a, b | a.1.mean_time().cmp( &b.1.mean_time() ) );
for ( name, result ) in sorted_results
{
let ( ci_lower, ci_upper ) = result.confidence_interval_95();
let cv = result.coefficient_of_variation();
let reliability = if result.is_reliable() { "✅" } else { "⚠️" };
output.push_str( &format!(
"| {} | {:.2?} | [{:.2?} - {:.2?}] | {:.0} | {:.1}% | {} | {} |\n",
name,
result.mean_time(),
ci_lower,
ci_upper,
result.operations_per_second(),
cv * 100.0,
reliability,
result.times.len()
) );
}
output.push_str( "\n" );
}
fn add_statistical_analysis( &self, output: &mut String, results: &HashMap< String, BenchmarkResult > )
{
let mut high_quality = Vec ::new();
let mut needs_improvement = Vec ::new();
for ( name, result ) in results
{
if result.is_reliable()
{
high_quality.push( name );
}
else
{
let cv = result.coefficient_of_variation();
let sample_size = result.times.len();
let mut issues = Vec ::new();
if sample_size < 10
{
issues.push( "insufficient samples" );
}
if cv > 0.1
{
issues.push( "high variability" );
}
needs_improvement.push( ( name, issues ) );
}
}
if !high_quality.is_empty()
{
output.push_str( "### ✅ Reliable Results\n" );
output.push_str( "*These measurements meet research-grade statistical standards*\n\n" );
for name in high_quality
{
let result = &results[ name ];
output.push_str( &format!( "- **{}** : {} samples, CV={:.1}%\n",
name,
result.times.len(),
result.coefficient_of_variation() * 100.0 ) );
}
output.push_str( "\n" );
}
if !needs_improvement.is_empty()
{
output.push_str( "### ⚠️ Measurements Needing Attention\n" );
output.push_str( "*Consider additional measurements for more reliable conclusions*\n\n" );
for ( name, issues ) in needs_improvement
{
output.push_str( &format!( "- **{}** : {}\n", name, issues.join( ", " ) ) );
}
output.push_str( "\n" );
}
}
fn add_regression_analysis( &self, output: &mut String, results: &HashMap< String, BenchmarkResult > )
{
if let Some( ref historical ) = self.historical_data
{
let analyzer = RegressionAnalyzer ::new()
.with_baseline_strategy( BaselineStrategy ::FixedBaseline )
.with_significance_threshold( 0.05 );
let regression_report = analyzer.analyze( results, historical );
let markdown_output = regression_report.format_markdown();
output.push_str( &markdown_output );
self.add_enhanced_recommendations( output, ®ression_report, results );
}
else
{
output.push_str( "**Regression Analysis** : Not yet implemented. Historical baseline data required.\n\n" );
output.push_str( "**📖 Setup Guide** : See [`usage.md`](usage.md) for mandatory standards and requirements on: \n" );
output.push_str( "- Historical data collection and baseline management\n" );
output.push_str( "- Statistical analysis requirements and validation criteria\n" );
output.push_str( "- Integration with CI/CD pipelines for automated regression detection\n" );
output.push_str( "- Documentation automation best practices\n\n" );
}
}
fn add_enhanced_recommendations( &self, output: &mut String, regression_report: &RegressionReport, results: &HashMap< String, BenchmarkResult > )
{
let mut improving_ops = Vec ::new();
let mut degrading_ops = Vec ::new();
let mut stable_ops = Vec ::new();
let mut new_ops = Vec ::new();
for operation_name in results.keys()
{
match regression_report.get_trend_for( operation_name )
{
Some( PerformanceTrend ::Improving ) =>
{
if regression_report.is_statistically_significant( operation_name )
{
improving_ops.push( operation_name );
}
},
Some( PerformanceTrend ::Degrading ) =>
{
if regression_report.is_statistically_significant( operation_name )
{
degrading_ops.push( operation_name );
}
},
Some( PerformanceTrend ::Stable ) =>
{
stable_ops.push( operation_name );
},
None =>
{
if !regression_report.has_historical_data( operation_name )
{
new_ops.push( operation_name );
}
},
}
}
if !improving_ops.is_empty() || !degrading_ops.is_empty() || regression_report.has_significant_changes()
{
output.push_str( "### 📊 **Statistical Analysis Summary**\n\n" );
if regression_report.has_significant_changes()
{
output.push_str( "**Statistically Significant Changes Detected** : This analysis identified performance changes that exceed normal measurement variance.\n\n" );
}
else
{
output.push_str( "**No Statistically Significant Changes** : All performance variations are within expected measurement noise.\n\n" );
}
}
if !improving_ops.is_empty()
{
output.push_str( "### 🎯 **Performance Optimization Insights**\n\n" );
output.push_str( "The following operations show statistically significant improvements: \n" );
for op in &improving_ops
{
output.push_str( &format!( "- **{}** : Consider documenting optimization techniques for knowledge sharing\n", op ) );
}
output.push_str( "\n**Next Steps** : Update performance baselines and validate improvements under production conditions.\n\n" );
}
if !degrading_ops.is_empty()
{
output.push_str( "### ⚠️ **Regression Investigation Required**\n\n" );
output.push_str( "**Critical** : The following operations show statistically significant performance degradation: \n" );
for op in °rading_ops
{
output.push_str( &format!( "- **{}** : Requires immediate investigation\n", op ) );
}
output.push_str( "\n**Recommended Actions** : \n" );
output.push_str( "1. **Profile regressed operations** to identify bottlenecks\n" );
output.push_str( "2. **Review recent code changes** affecting these operations\n" );
output.push_str( "3. **Run additional validation** with increased sample sizes\n" );
output.push_str( "4. **Consider deployment hold** until regressions are resolved\n\n" );
}
output.push_str( "### 🔗 **Integration Resources**\n\n" );
output.push_str( "For enhanced regression analysis capabilities: \n" );
output.push_str( "- **Configure baseline strategies** : Use `RegressionAnalyzer ::with_baseline_strategy()` for rolling averages or previous-run comparisons\n" );
output.push_str( "- **Adjust significance thresholds** : Use `with_significance_threshold()` for domain-specific sensitivity\n" );
output.push_str( "- **Historical data management** : Implement `TimestampedResults` for comprehensive trend analysis\n" );
output.push_str( "- **Automated monitoring** : Integrate with CI/CD pipelines for continuous performance validation\n\n" );
}
fn add_methodology_note( &self, output: &mut String )
{
output.push_str( "**Statistical Reliability Criteria** : \n" );
output.push_str( "- Sample size ≥ 10 measurements\n" );
output.push_str( "- Coefficient of variation ≤ 10%\n" );
output.push_str( "- Maximum/minimum time ratio < 3.0x\n\n" );
output.push_str( "**Confidence Intervals** : 95% CI calculated using t-distribution\n" );
output.push_str( "**CV** : Coefficient of Variation (relative standard deviation)\n\n" );
output.push_str( "---\n" );
output.push_str( "*Generated by benchkit - Professional benchmarking toolkit*\n" );
}
fn find_fastest< 'a >( &self, results: &'a HashMap< String, BenchmarkResult > ) -> Option< ( &'a String, &'a BenchmarkResult ) >
{
results.iter().min_by( | a, b | a.1.mean_time().cmp( &b.1.mean_time() ) )
}
fn find_slowest< 'a >( &self, results: &'a HashMap< String, BenchmarkResult > ) -> Option< ( &'a String, &'a BenchmarkResult ) >
{
results.iter().max_by( | a, b | a.1.mean_time().cmp( &b.1.mean_time() ) )
}
}
#[ derive( Debug, Clone ) ]
pub struct ComparisonReport
{
title: String,
baseline: String,
candidate: String,
significance_threshold: f64,
practical_significance_threshold: f64,
}
impl ComparisonReport
{
#[ must_use ]
pub fn new() -> Self
{
Self
{
title: "Performance Comparison".to_string(),
baseline: "Baseline".to_string(),
candidate: "Candidate".to_string(),
significance_threshold: 0.05,
practical_significance_threshold: 0.10,
}
}
#[ must_use ]
pub fn title( mut self, title: impl Into< String > ) -> Self
{
self.title = title.into();
self
}
#[ must_use ]
pub fn baseline( mut self, baseline: impl Into< String > ) -> Self
{
self.baseline = baseline.into();
self
}
#[ must_use ]
pub fn candidate( mut self, candidate: impl Into< String > ) -> Self
{
self.candidate = candidate.into();
self
}
#[ must_use ]
pub fn significance_threshold( mut self, threshold: f64 ) -> Self
{
self.significance_threshold = threshold;
self
}
#[ must_use ]
pub fn practical_significance_threshold( mut self, threshold: f64 ) -> Self
{
self.practical_significance_threshold = threshold;
self
}
}
impl Default for ComparisonReport
{
fn default() -> Self
{
Self ::new()
}
}
impl ComparisonReport
{
#[ must_use ]
pub fn baseline_name( &self ) -> &str
{
&self.baseline
}
#[ must_use ]
pub fn candidate_name( &self ) -> &str
{
&self.candidate
}
#[ must_use ]
pub fn significance_threshold_value( &self ) -> f64
{
self.significance_threshold
}
#[ must_use ]
pub fn practical_significance_threshold_value( &self ) -> f64
{
self.practical_significance_threshold
}
}
impl ReportTemplate for ComparisonReport
{
fn generate( &self, results: &HashMap< String, BenchmarkResult > ) -> Result< String >
{
let mut output = String ::new();
output.push_str( &format!( "# {}\n\n", self.title ) );
let baseline_result = results.get( &self.baseline )
.ok_or_else( || -> Box< dyn std ::error ::Error > { format!( "Baseline result '{}' not found", self.baseline ).into() } )?;
let candidate_result = results.get( &self.candidate )
.ok_or_else( || -> Box< dyn std ::error ::Error > { format!( "Candidate result '{}' not found", self.candidate ).into() } )?;
let baseline_time = baseline_result.mean_time().as_secs_f64();
let candidate_time = candidate_result.mean_time().as_secs_f64();
let improvement_ratio = baseline_time / candidate_time;
let improvement_percent = ( improvement_ratio - 1.0 ) * 100.0;
output.push_str( "## Comparison Summary\n\n" );
if improvement_ratio > 1.0 + self.practical_significance_threshold
{
output.push_str( &format!( "✅ **{} is {:.1}% faster** than {}\n\n",
self.candidate, improvement_percent, self.baseline ) );
}
else if improvement_ratio < 1.0 - self.practical_significance_threshold
{
let regression_percent = ( 1.0 - improvement_ratio ) * 100.0;
output.push_str( &format!( "🚨 **{} is {:.1}% slower** than {}\n\n",
self.candidate, regression_percent, self.baseline ) );
}
else
{
output.push_str( &format!( "⚖️ **No significant difference** between {} and {}\n\n",
self.baseline, self.candidate ) );
}
output.push_str( "## Detailed Comparison\n\n" );
output.push_str( "| Algorithm | Mean Time | 95% CI | Ops/sec | CV | Samples | Reliability |\n" );
output.push_str( "|-----------|-----------|--------|---------|----|---------|-----------|\n" );
for ( name, result ) in [ ( &self.baseline, baseline_result ), ( &self.candidate, candidate_result ) ]
{
let ( ci_lower, ci_upper ) = result.confidence_interval_95();
let cv = result.coefficient_of_variation();
let reliability = if result.is_reliable() { "✅" } else { "⚠️" };
output.push_str( &format!(
"| {} | {:.2?} | [{:.2?} - {:.2?}] | {:.0} | {:.1}% | {} | {} |\n",
name,
result.mean_time(),
ci_lower,
ci_upper,
result.operations_per_second(),
cv * 100.0,
result.times.len(),
reliability
) );
}
output.push_str( "\n" );
output.push_str( "## Statistical Analysis\n\n" );
output.push_str( &format!( "- **Performance ratio** : {:.3}x\n", improvement_ratio ) );
output.push_str( &format!( "- **Improvement** : {:.1}%\n", improvement_percent ) );
let baseline_ci = baseline_result.confidence_interval_95();
let candidate_ci = candidate_result.confidence_interval_95();
let ci_overlap = baseline_ci.1 >= candidate_ci.0 && candidate_ci.1 >= baseline_ci.0;
if ci_overlap
{
output.push_str( "- **Statistical significance** : ⚠️ Confidence intervals overlap - difference may not be statistically significant\n" );
}
else
{
output.push_str( "- **Statistical significance** : ✅ No confidence interval overlap - difference is likely statistically significant\n" );
}
if improvement_percent.abs() >= self.practical_significance_threshold * 100.0
{
output.push_str( &format!( "- **Practical significance** : ✅ Difference exceeds {:.1}% threshold\n",
self.practical_significance_threshold * 100.0 ) );
}
else
{
output.push_str( &format!( "- **Practical significance** : ⚠️ Difference below {:.1}% threshold\n",
self.practical_significance_threshold * 100.0 ) );
}
output.push_str( "\n" );
output.push_str( "## Reliability Assessment\n\n" );
if baseline_result.is_reliable() && candidate_result.is_reliable()
{
output.push_str( "✅ **Both measurements are statistically reliable** - conclusions can be drawn with confidence.\n\n" );
}
else
{
output.push_str( "⚠️ **One or both measurements have reliability concerns** - consider additional sampling.\n\n" );
if !baseline_result.is_reliable()
{
output.push_str( &format!( "- **{}** : {} samples, CV={:.1}%\n",
self.baseline,
baseline_result.times.len(),
baseline_result.coefficient_of_variation() * 100.0 ) );
}
if !candidate_result.is_reliable()
{
output.push_str( &format!( "- **{}** : {} samples, CV={:.1}%\n",
self.candidate,
candidate_result.times.len(),
candidate_result.coefficient_of_variation() * 100.0 ) );
}
output.push_str( "\n" );
}
output.push_str( "## Methodology\n\n" );
output.push_str( &format!( "**Significance Thresholds** : Statistical p < {}, Practical > {:.1}%\n",
self.significance_threshold,
self.practical_significance_threshold * 100.0 ) );
output.push_str( "**Confidence Intervals** : 95% CI using t-distribution\n" );
output.push_str( "**Reliability Criteria** : ≥10 samples, CV ≤10%, max/min ratio <3x\n\n" );
output.push_str( "---\n" );
output.push_str( "*Generated by benchkit - Professional benchmarking toolkit*\n" );
Ok( output )
}
}
#[ derive( Debug, Clone ) ]
pub struct CustomSection
{
pub title: String,
pub content: String,
}
impl CustomSection
{
#[ must_use ]
pub fn new( title: impl Into< String >, content: impl Into< String > ) -> Self
{
Self
{
title: title.into(),
content: content.into(),
}
}
}