//! Utilities module for common debugging operations and helper functions
use anyhow::Result;
use ndarray::{Array, ArrayD, IxDyn};
use serde::{Deserialize, Serialize};
use std::collections::HashMap;
use std::path::Path;
use std::time::{Duration, Instant};
use crate::{DebugConfig, DebugSession, QuickDebugLevel, SimplifiedDebugResult};
/// Common debugging utilities and helper functions
pub struct DebugUtils;
impl DebugUtils {
/// Quick model health check with automatic issue detection
pub async fn quick_health_check<T>(model: &T) -> Result<HealthCheckResult> {
let result = crate::quick_debug(model, QuickDebugLevel::Light).await?;
let health_score = match &result {
SimplifiedDebugResult::Light(health) => health.score,
SimplifiedDebugResult::Standard { health, .. } => health.score,
SimplifiedDebugResult::Deep(report) => {
let summary = report.summary();
100.0 - (summary.critical_issues as f64 * 20.0 + summary.total_issues as f64 * 5.0)
},
SimplifiedDebugResult::Production(anomaly) => {
100.0 - (anomaly.anomaly_count as f64 * 10.0)
}
};
Ok(HealthCheckResult {
overall_score: health_score,
status: Self::score_to_status(health_score),
issues: result.recommendations(),
critical_issues: result.has_critical_issues(),
timestamp: chrono::Utc::now(),
})
}
/// Batch tensor analysis with statistical insights
pub fn analyze_tensors_batch(tensors: &[ArrayD<f32>]) -> Result<BatchTensorAnalysis> {
let mut results = Vec::new();
let mut overall_stats = TensorStatistics::default();
for (i, tensor) in tensors.iter().enumerate() {
let stats = Self::compute_tensor_statistics(tensor)?;
results.push(TensorAnalysisResult {
tensor_index: i,
shape: tensor.shape().to_vec(),
statistics: stats.clone(),
anomalies: Self::detect_tensor_anomalies(&stats),
});
// Accumulate overall statistics
overall_stats.accumulate(&stats);
}
overall_stats.finalize(tensors.len());
Ok(BatchTensorAnalysis {
individual_results: results,
overall_statistics: overall_stats,
batch_size: tensors.len(),
analysis_timestamp: chrono::Utc::now(),
})
}
/// Compute comprehensive tensor statistics
pub fn compute_tensor_statistics(tensor: &ArrayD<f32>) -> Result<TensorStatistics> {
let values: Vec<f32> = tensor.iter().cloned().collect();
let n = values.len() as f64;
if n == 0.0 {
return Ok(TensorStatistics::default());
}
let sum: f64 = values.iter().map(|&x| x as f64).sum::<f64>();
let mean = sum / n;
let variance: f64 = values.iter()
.map(|&x| (x as f64 - mean).powi(2))
.sum::<f64>() / n;
let std_dev = variance.sqrt();
let mut sorted_values = values.clone();
sorted_values.sort_by(|a, b| a.partial_cmp(b).unwrap_or(std::cmp::Ordering::Equal));
let min = *sorted_values.first().unwrap_or(&0.0);
let max = *sorted_values.last().unwrap_or(&0.0);
let median = if sorted_values.len() % 2 == 0 {
let mid = sorted_values.len() / 2;
(sorted_values[mid - 1] + sorted_values[mid]) / 2.0
} else {
sorted_values[sorted_values.len() / 2]
};
let p25 = Self::percentile(&sorted_values, 0.25);
let p75 = Self::percentile(&sorted_values, 0.75);
let nan_count = values.iter().filter(|&&x| x.is_nan()).count();
let inf_count = values.iter().filter(|&&x| x.is_infinite()).count();
let zero_count = values.iter().filter(|&&x| x == 0.0).count();
let skewness = Self::compute_skewness(&values, mean, std_dev);
let kurtosis = Self::compute_kurtosis(&values, mean, std_dev);
Ok(TensorStatistics {
count: n as usize,
mean: mean as f32,
std_dev: std_dev as f32,
min,
max,
median,
p25,
p75,
nan_count,
inf_count,
zero_count,
skewness: skewness as f32,
kurtosis: kurtosis as f32,
})
}
/// Detect anomalies in tensor statistics
pub fn detect_tensor_anomalies(stats: &TensorStatistics) -> Vec<TensorAnomaly> {
let mut anomalies = Vec::new();
// Check for NaN/Inf values
if stats.nan_count > 0 {
anomalies.push(TensorAnomaly {
anomaly_type: AnomalyType::NanValues,
severity: AnomalySeverity::Critical,
description: format!("Found {} NaN values", stats.nan_count),
suggested_fix: "Check input data and model initialization".to_string(),
});
}
if stats.inf_count > 0 {
anomalies.push(TensorAnomaly {
anomaly_type: AnomalyType::InfiniteValues,
severity: AnomalySeverity::Critical,
description: format!("Found {} infinite values", stats.inf_count),
suggested_fix: "Reduce learning rate or add gradient clipping".to_string(),
});
}
// Check for extreme skewness
if stats.skewness.abs() > 3.0 {
anomalies.push(TensorAnomaly {
anomaly_type: AnomalyType::ExtremeSkewness,
severity: AnomalySeverity::Medium,
description: format!("Extreme skewness: {:.2}", stats.skewness),
suggested_fix: "Consider data normalization or different initialization".to_string(),
});
}
// Check for extreme kurtosis
if stats.kurtosis.abs() > 7.0 {
anomalies.push(TensorAnomaly {
anomaly_type: AnomalyType::ExtremeKurtosis,
severity: AnomalySeverity::Medium,
description: format!("Extreme kurtosis: {:.2}", stats.kurtosis),
suggested_fix: "Check for outliers or consider robust normalization".to_string(),
});
}
// Check for too many zeros (dead neurons indicator)
let zero_ratio = stats.zero_count as f32 / stats.count as f32;
if zero_ratio > 0.5 {
anomalies.push(TensorAnomaly {
anomaly_type: AnomalyType::DeadNeurons,
severity: AnomalySeverity::High,
description: format!("High zero ratio: {:.1}%", zero_ratio * 100.0),
suggested_fix: "Check activation functions and learning rate".to_string(),
});
}
// Check for extremely small or large values
if stats.max > 1e6 || stats.min < -1e6 {
anomalies.push(TensorAnomaly {
anomaly_type: AnomalyType::ExtremeValues,
severity: AnomalySeverity::High,
description: format!("Extreme values: min={:.2e}, max={:.2e}", stats.min, stats.max),
suggested_fix: "Consider gradient clipping or normalization".to_string(),
});
}
anomalies
}
/// Generate debug report summary
pub fn generate_debug_summary(config: &DebugConfig, results: &[SimplifiedDebugResult]) -> DebugSummary {
let mut total_issues = 0;
let mut critical_issues = 0;
let mut all_recommendations = Vec::new();
for result in results {
if result.has_critical_issues() {
critical_issues += 1;
}
total_issues += 1;
all_recommendations.extend(result.recommendations());
}
// Deduplicate recommendations
all_recommendations.sort();
all_recommendations.dedup();
DebugSummary {
config_hash: Self::hash_config(config),
total_debug_runs: results.len(),
total_issues,
critical_issues,
recommendations: all_recommendations,
timestamp: chrono::Utc::now(),
}
}
/// Export debug session data to various formats
pub async fn export_debug_data(
session: &DebugSession,
format: ExportFormat,
output_path: &str
) -> Result<String> {
let report = session.generate_snapshot().await?;
match format {
ExportFormat::Json => {
let json_data = serde_json::to_string_pretty(&report)?;
tokio::fs::write(output_path, json_data).await?;
},
ExportFormat::Csv => {
let csv_data = Self::report_to_csv(&report)?;
tokio::fs::write(output_path, csv_data).await?;
},
ExportFormat::Html => {
let html_data = Self::report_to_html(&report)?;
tokio::fs::write(output_path, html_data).await?;
},
}
Ok(output_path.to_string())
}
/// Create a debug session template for common use cases
pub fn create_debug_template(template_type: DebugTemplate) -> DebugConfig {
match template_type {
DebugTemplate::Development => DebugConfig {
enable_tensor_inspection: true,
enable_gradient_debugging: true,
enable_model_diagnostics: true,
enable_visualization: true,
enable_memory_profiling: true,
enable_computation_graph_analysis: true,
max_tracked_tensors: 1000,
max_gradient_history: 200,
sampling_rate: 1.0,
..Default::default()
},
DebugTemplate::Production => DebugConfig {
enable_tensor_inspection: false,
enable_gradient_debugging: false,
enable_model_diagnostics: false,
enable_visualization: false,
enable_memory_profiling: true,
enable_computation_graph_analysis: false,
max_tracked_tensors: 50,
max_gradient_history: 10,
sampling_rate: 0.1,
..Default::default()
},
DebugTemplate::Training => DebugConfig {
enable_tensor_inspection: true,
enable_gradient_debugging: true,
enable_model_diagnostics: true,
enable_visualization: false,
enable_memory_profiling: true,
enable_computation_graph_analysis: true,
max_tracked_tensors: 500,
max_gradient_history: 100,
sampling_rate: 0.5,
..Default::default()
},
DebugTemplate::Research => DebugConfig {
enable_tensor_inspection: true,
enable_gradient_debugging: true,
enable_model_diagnostics: true,
enable_visualization: true,
enable_memory_profiling: true,
enable_computation_graph_analysis: true,
max_tracked_tensors: 2000,
max_gradient_history: 500,
sampling_rate: 1.0,
..Default::default()
},
}
}
// Helper functions
fn score_to_status(score: f64) -> String {
match score {
s if s >= 90.0 => "Excellent".to_string(),
s if s >= 80.0 => "Good".to_string(),
s if s >= 70.0 => "Fair".to_string(),
s if s >= 60.0 => "Poor".to_string(),
_ => "Critical".to_string(),
}
}
fn percentile(sorted_values: &[f32], p: f64) -> f32 {
let index = (p * (sorted_values.len() - 1) as f64) as usize;
sorted_values.get(index).copied().unwrap_or(0.0)
}
fn compute_skewness(values: &[f32], mean: f64, std_dev: f64) -> f64 {
if std_dev == 0.0 {
return 0.0;
}
let n = values.len() as f64;
let sum: f64 = values.iter()
.map(|&x| ((x as f64 - mean) / std_dev).powi(3))
.sum::<f64>();
sum / n
}
fn compute_kurtosis(values: &[f32], mean: f64, std_dev: f64) -> f64 {
if std_dev == 0.0 {
return 0.0;
}
let n = values.len() as f64;
let sum: f64 = values.iter()
.map(|&x| ((x as f64 - mean) / std_dev).powi(4))
.sum::<f64>();
(sum / n) - 3.0 // Subtract 3 for excess kurtosis
}
fn hash_config(config: &DebugConfig) -> String {
// Simple hash implementation for config tracking
format!("{:x}",
config.enable_tensor_inspection as u8 |
(config.enable_gradient_debugging as u8) << 1 |
(config.enable_model_diagnostics as u8) << 2 |
(config.enable_visualization as u8) << 3 |
(config.enable_memory_profiling as u8) << 4
)
}
fn report_to_csv(report: &crate::DebugReport) -> Result<String> {
let mut csv = String::new();
csv.push_str("session_id,component,metric,value\n");
csv.push_str(&format!("{},session,id,{}\n", report.session_id, report.session_id));
if let Some(ref tensor_report) = report.tensor_report {
csv.push_str(&format!("{},tensor,total_tensors,{}\n",
report.session_id, tensor_report.total_tensors));
csv.push_str(&format!("{},tensor,tensors_with_issues,{}\n",
report.session_id, tensor_report.tensors_with_issues));
}
Ok(csv)
}
fn report_to_html(report: &crate::DebugReport) -> Result<String> {
let mut html = String::new();
html.push_str("<!DOCTYPE html><html><head><title>Debug Report</title></head><body>");
html.push_str(&format!("<h1>Debug Report - Session {}</h1>", report.session_id));
if let Some(ref tensor_report) = report.tensor_report {
html.push_str("<h2>Tensor Analysis</h2>");
html.push_str(&format!("<p>Total tensors: {}</p>", tensor_report.total_tensors));
html.push_str(&format!("<p>Tensors with issues: {}</p>", tensor_report.tensors_with_issues));
}
html.push_str("</body></html>");
Ok(html)
}
}
/// Health check result structure
#[derive(Debug, Serialize, Deserialize)]
pub struct HealthCheckResult {
pub overall_score: f64,
pub status: String,
pub issues: Vec<String>,
pub critical_issues: bool,
pub timestamp: chrono::DateTime<chrono::Utc>,
}
/// Batch tensor analysis result
#[derive(Debug, Serialize, Deserialize)]
pub struct BatchTensorAnalysis {
pub individual_results: Vec<TensorAnalysisResult>,
pub overall_statistics: TensorStatistics,
pub batch_size: usize,
pub analysis_timestamp: chrono::DateTime<chrono::Utc>,
}
/// Individual tensor analysis result
#[derive(Debug, Serialize, Deserialize)]
pub struct TensorAnalysisResult {
pub tensor_index: usize,
pub shape: Vec<usize>,
pub statistics: TensorStatistics,
pub anomalies: Vec<TensorAnomaly>,
}
/// Comprehensive tensor statistics
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct TensorStatistics {
pub count: usize,
pub mean: f32,
pub std_dev: f32,
pub min: f32,
pub max: f32,
pub median: f32,
pub p25: f32,
pub p75: f32,
pub nan_count: usize,
pub inf_count: usize,
pub zero_count: usize,
pub skewness: f32,
pub kurtosis: f32,
}
impl Default for TensorStatistics {
fn default() -> Self {
Self {
count: 0,
mean: 0.0,
std_dev: 0.0,
min: 0.0,
max: 0.0,
median: 0.0,
p25: 0.0,
p75: 0.0,
nan_count: 0,
inf_count: 0,
zero_count: 0,
skewness: 0.0,
kurtosis: 0.0,
}
}
}
impl TensorStatistics {
fn accumulate(&mut self, other: &TensorStatistics) {
self.count += other.count;
self.mean += other.mean;
self.std_dev += other.std_dev;
self.min = self.min.min(other.min);
self.max = self.max.max(other.max);
self.nan_count += other.nan_count;
self.inf_count += other.inf_count;
self.zero_count += other.zero_count;
}
fn finalize(&mut self, batch_size: usize) {
if batch_size > 0 {
self.mean /= batch_size as f32;
self.std_dev /= batch_size as f32;
}
}
}
/// Tensor anomaly detection result
#[derive(Debug, Serialize, Deserialize)]
pub struct TensorAnomaly {
pub anomaly_type: AnomalyType,
pub severity: AnomalySeverity,
pub description: String,
pub suggested_fix: String,
}
/// Types of tensor anomalies
#[derive(Debug, Serialize, Deserialize)]
pub enum AnomalyType {
NanValues,
InfiniteValues,
ExtremeSkewness,
ExtremeKurtosis,
DeadNeurons,
ExtremeValues,
Saturation,
Outliers,
}
/// Severity levels for anomalies
#[derive(Debug, Serialize, Deserialize)]
pub enum AnomalySeverity {
Low,
Medium,
High,
Critical,
}
/// Debug summary structure
#[derive(Debug, Serialize, Deserialize)]
pub struct DebugSummary {
pub config_hash: String,
pub total_debug_runs: usize,
pub total_issues: usize,
pub critical_issues: usize,
pub recommendations: Vec<String>,
pub timestamp: chrono::DateTime<chrono::Utc>,
}
/// Export format options
#[derive(Debug, Clone)]
pub enum ExportFormat {
Json,
Csv,
Html,
}
/// Debug template types
#[derive(Debug, Clone)]
pub enum DebugTemplate {
Development,
Production,
Training,
Research,
}
/// Debugging convenience macros
#[macro_export]
macro_rules! debug_tensor {
($session:expr, $tensor:expr, $name:expr) => {
$session.tensor_inspector_mut().inspect_tensor(
$tensor,
$name,
None,
Some("debug_macro")
)
};
($session:expr, $tensor:expr, $name:expr, $layer:expr) => {
$session.tensor_inspector_mut().inspect_tensor(
$tensor,
$name,
Some($layer),
Some("debug_macro")
)
};
}
#[macro_export]
macro_rules! debug_gradient {
($session:expr, $layer:expr, $gradients:expr) => {
$session.gradient_debugger_mut().record_gradient_flow($layer, $gradients)
};
}
#[macro_export]
macro_rules! quick_debug_check {
($model:expr) => {
trustformers_debug::debug($model).await
};
($model:expr, $level:expr) => {
trustformers_debug::quick_debug($model, $level).await
};
}
/// Performance monitoring utilities
pub struct PerformanceMonitor {
start_time: Instant,
checkpoints: HashMap<String, Instant>,
durations: HashMap<String, Duration>,
}
impl PerformanceMonitor {
pub fn new() -> Self {
Self {
start_time: Instant::now(),
checkpoints: HashMap::new(),
durations: HashMap::new(),
}
}
pub fn checkpoint(&mut self, name: &str) {
self.checkpoints.insert(name.to_string(), Instant::now());
}
pub fn end_checkpoint(&mut self, name: &str) -> Option<Duration> {
if let Some(start) = self.checkpoints.remove(name) {
let duration = start.elapsed();
self.durations.insert(name.to_string(), duration);
Some(duration)
} else {
None
}
}
pub fn total_elapsed(&self) -> Duration {
self.start_time.elapsed()
}
pub fn get_durations(&self) -> &HashMap<String, Duration> {
&self.durations
}
pub fn performance_report(&self) -> String {
let mut report = format!("Performance Report - Total: {:.2}ms\n",
self.total_elapsed().as_millis());
for (name, duration) in &self.durations {
report.push_str(&format!(" {}: {:.2}ms\n", name, duration.as_millis()));
}
report
}
}
impl Default for PerformanceMonitor {
fn default() -> Self {
Self::new()
}
}
/// Advanced tensor analysis with ML-specific insights
pub struct AdvancedTensorAnalyzer;
impl AdvancedTensorAnalyzer {
/// Detect gradient explosion patterns
pub fn detect_gradient_explosion(gradients: &[ArrayD<f32>], threshold: f32) -> GradientExplosionAnalysis {
let mut exploding_layers = Vec::new();
let mut max_gradient_norm = 0.0f32;
let mut gradient_norms = Vec::new();
for (layer_idx, gradient) in gradients.iter().enumerate() {
let l2_norm = Self::compute_l2_norm(gradient);
gradient_norms.push(l2_norm);
if l2_norm > max_gradient_norm {
max_gradient_norm = l2_norm;
}
if l2_norm > threshold {
exploding_layers.push(ExplodingLayer {
layer_index: layer_idx,
gradient_norm: l2_norm,
severity: Self::classify_explosion_severity(l2_norm, &gradient_norms),
recommended_action: Self::recommend_explosion_mitigation(l2_norm),
});
}
}
let mean_norm = gradient_norms.iter().sum::<f32>() / gradient_norms.len() as f32;
let std_norm = {
let variance: f32 = gradient_norms.iter()
.map(|&x| (x - mean_norm).powi(2))
.sum::<f32>() / gradient_norms.len() as f32;
variance.sqrt()
};
let explosion_ratio = exploding_layers.len() as f32 / gradients.len() as f32;
GradientExplosionAnalysis {
has_explosion: !exploding_layers.is_empty(),
exploding_layers,
max_gradient_norm,
mean_gradient_norm: mean_norm,
std_gradient_norm: std_norm,
explosion_ratio,
recommended_clip_value: max_gradient_norm * 0.5,
}
}
/// Analyze model weight distribution patterns
pub fn analyze_weight_distribution(weights: &[ArrayD<f32>]) -> Result<WeightDistributionAnalysis> {
let mut layer_analyses = Vec::new();
let mut all_weights = Vec::new();
let mut overall_stats = WeightStatistics::default();
for (layer_idx, weight) in weights.iter().enumerate() {
let values: Vec<f32> = weight.iter().cloned().collect();
all_weights.extend_from_slice(&values);
let stats = Self::compute_advanced_weight_stats(weight)?;
let health = Self::assess_weight_health(&stats);
let layer_analysis = LayerWeightAnalysis {
layer_index: layer_idx,
statistics: stats.clone(),
health_score: health.score,
issues: health.issues,
recommendations: health.recommendations,
};
overall_stats.accumulate(&stats);
layer_analyses.push(layer_analysis);
}
// Overall analysis
let overall_mean = all_weights.iter().sum::<f32>() / all_weights.len() as f32;
let overall_std = {
let variance = all_weights.iter()
.map(|&x| (x - overall_mean).powi(2))
.sum::<f32>() / all_weights.len() as f32;
variance.sqrt()
};
overall_stats.finalize(weights.len());
Ok(WeightDistributionAnalysis {
layer_analyses,
overall_statistics: overall_stats.clone(),
distribution_health: Self::assess_overall_distribution_health(&overall_stats),
outlier_detection: Self::detect_weight_outliers(weights)?,
})
}
/// Compare model states for debugging
pub fn compare_model_states(
state_a: &ModelDebugState,
state_b: &ModelDebugState,
) -> ModelStateComparison {
let weight_diff = Self::compute_weight_differences(&state_a.weights, &state_b.weights);
let gradient_diff = Self::compute_gradient_differences(&state_a.gradients, &state_b.gradients);
let significant_changes = weight_diff.iter()
.enumerate()
.filter(|(_, &diff)| diff > 0.1)
.map(|(idx, &diff)| LayerChange {
layer_index: idx,
change_type: ChangeType::WeightUpdate,
magnitude: diff,
description: format!("Weight change: {:.3}", diff),
})
.collect();
let overall_change_magnitude = weight_diff.iter().sum::<f32>() / weight_diff.len() as f32;
ModelStateComparison {
weight_differences: weight_diff,
gradient_differences: gradient_diff,
significant_changes,
overall_change_magnitude,
regression_detected: Self::detect_regression(&state_a.metrics, &state_b.metrics),
}
}
fn compute_l2_norm(tensor: &ArrayD<f32>) -> f32 {
tensor.iter().map(|&x| x * x).sum::<f32>().sqrt()
}
fn assess_distribution_health(weights: &[f32]) -> DistributionHealth {
let mean = weights.iter().sum::<f32>() / weights.len() as f32;
let std_dev = {
let variance = weights.iter()
.map(|&x| (x - mean).powi(2))
.sum::<f32>() / weights.len() as f32;
variance.sqrt()
};
let dead_ratio = weights.iter().filter(|&&x| x.abs() < 1e-8).count() as f32 / weights.len() as f32;
let extreme_ratio = weights.iter().filter(|&&x| x.abs() > 3.0).count() as f32 / weights.len() as f32;
let score = if dead_ratio > 0.5 || extreme_ratio > 0.1 {
20.0
} else if dead_ratio > 0.2 || extreme_ratio > 0.05 {
40.0
} else if std_dev > 0.1 && std_dev < 2.0 && dead_ratio < 0.1 {
80.0
} else {
100.0
};
let status = match score {
s if s >= 90.0 => DistributionHealthStatus::Excellent,
s if s >= 70.0 => DistributionHealthStatus::Good,
s if s >= 50.0 => DistributionHealthStatus::Fair,
s if s >= 30.0 => DistributionHealthStatus::Poor,
_ => DistributionHealthStatus::Critical,
};
DistributionHealth { score, status }
}
fn compute_gradient_differences(grads_a: &[ArrayD<f32>], grads_b: &[ArrayD<f32>]) -> Vec<f32> {
grads_a.iter()
.zip(grads_b.iter())
.map(|(a, b)| {
let diff: f32 = a.iter().zip(b.iter())
.map(|(&x, &y)| (x - y).abs())
.sum();
diff / a.len() as f32
})
.collect()
}
fn detect_regression(metrics_a: &ModelMetrics, metrics_b: &ModelMetrics) -> bool {
metrics_b.loss > metrics_a.loss * 1.1 ||
metrics_b.accuracy < metrics_a.accuracy * 0.95
}
}
/// Model debug state for comparison
#[derive(Debug, Clone)]
pub struct ModelDebugState {
pub weights: Vec<ArrayD<f32>>,
pub gradients: Vec<ArrayD<f32>>,
pub metrics: ModelMetrics,
pub timestamp: chrono::DateTime<chrono::Utc>,
}
/// Model metrics for tracking
#[derive(Debug, Clone)]
pub struct ModelMetrics {
pub loss: f32,
pub accuracy: f32,
pub learning_rate: f32,
pub epoch: u32,
}
/// Gradient explosion analysis results
#[derive(Debug, Serialize, Deserialize)]
pub struct GradientExplosionAnalysis {
pub has_explosion: bool,
pub exploding_layers: Vec<ExplodingLayer>,
pub max_gradient_norm: f32,
pub mean_gradient_norm: f32,
pub std_gradient_norm: f32,
pub explosion_ratio: f32,
pub recommended_clip_value: f32,
}
impl AdvancedTensorAnalyzer {
/// Perform advanced gradient explosion detection with sophisticated algorithms
pub fn detect_gradient_explosion_advanced(
gradients: &[ArrayD<f32>],
threshold_multiplier: f32
) -> Result<GradientExplosionAnalysis> {
let mut gradient_norms = Vec::new();
let mut exploding_layers = Vec::new();
// Calculate gradient norms for each layer
for (layer_idx, grad) in gradients.iter().enumerate() {
let norm = Self::compute_gradient_norm(grad);
gradient_norms.push(norm);
// Detect exploding gradients using adaptive thresholding
if Self::is_gradient_exploding(norm, &gradient_norms, threshold_multiplier) {
exploding_layers.push(ExplodingLayer {
layer_index: layer_idx,
gradient_norm: norm,
severity: Self::classify_explosion_severity(norm, &gradient_norms),
recommended_action: Self::recommend_explosion_mitigation(norm),
});
}
}
let max_norm = gradient_norms.iter().copied().fold(0.0, f32::max);
let mean_norm = gradient_norms.iter().sum::<f32>() / gradient_norms.len() as f32;
let variance: f32 = gradient_norms.iter()
.map(|&x| (x - mean_norm).powi(2))
.sum::<f32>() / gradient_norms.len() as f32;
let std_norm = variance.sqrt();
// Calculate explosion ratio using statistical analysis
let explosion_ratio = if mean_norm > 0.0 {
max_norm / mean_norm
} else {
0.0
};
// Intelligent gradient clipping recommendation
let recommended_clip_value = Self::compute_optimal_clip_value(&gradient_norms);
Ok(GradientExplosionAnalysis {
has_explosion: !exploding_layers.is_empty(),
exploding_layers,
max_gradient_norm: max_norm,
mean_gradient_norm: mean_norm,
std_gradient_norm: std_norm,
explosion_ratio,
recommended_clip_value,
})
}
/// Analyze weight distribution with advanced statistical methods
pub fn analyze_weight_distribution_advanced(
weights: &[ArrayD<f32>]
) -> Result<WeightDistributionAnalysis> {
let mut layer_analyses = Vec::new();
let mut overall_stats = WeightStatistics::default();
for (layer_idx, weight) in weights.iter().enumerate() {
let stats = Self::compute_advanced_weight_stats(weight)?;
let health = Self::assess_weight_health(&stats);
layer_analyses.push(LayerWeightAnalysis {
layer_index: layer_idx,
statistics: stats.clone(),
health_score: health.score,
issues: health.issues,
recommendations: health.recommendations,
});
overall_stats.accumulate(&stats);
}
overall_stats.finalize(weights.len());
Ok(WeightDistributionAnalysis {
layer_analyses,
overall_statistics: overall_stats.clone(),
distribution_health: Self::assess_overall_distribution_health(&overall_stats),
outlier_detection: Self::detect_weight_outliers(weights)?,
})
}
/// Compare model states with advanced regression detection
pub fn compare_model_states_advanced(
state_a: &ModelDebugState,
state_b: &ModelDebugState
) -> Result<AdvancedModelComparison> {
// Weight comparison with statistical significance testing
let weight_differences = Self::compute_weight_differences(&state_a.weights, &state_b.weights);
let weight_drift = Self::calculate_weight_drift(&weight_differences);
// Gradient comparison with flow analysis
let gradient_differences = Self::compute_gradient_differences(&state_a.gradients, &state_b.gradients);
let gradient_coherence = Self::analyze_gradient_coherence(&gradient_differences);
// Performance regression analysis
let performance_change = Self::analyze_performance_change(&state_a.metrics, &state_b.metrics);
let regression_probability = Self::calculate_regression_probability(&performance_change);
// Advanced change magnitude assessment
let change_magnitude = Self::assess_change_magnitude(&weight_drift, &gradient_coherence);
Ok(AdvancedModelComparison {
weight_drift_analysis: weight_drift.clone(),
gradient_coherence_analysis: gradient_coherence,
performance_change_analysis: performance_change.clone(),
regression_probability,
change_magnitude,
recommendations: Self::generate_comparison_recommendations(
&weight_drift, &performance_change, regression_probability
),
})
}
// Helper methods for advanced analysis
fn compute_gradient_norm(gradient: &ArrayD<f32>) -> f32 {
gradient.iter().map(|&x| x * x).sum::<f32>().sqrt()
}
fn is_gradient_exploding(norm: f32, all_norms: &[f32], threshold_multiplier: f32) -> bool {
if all_norms.is_empty() {
return false;
}
let mean = all_norms.iter().sum::<f32>() / all_norms.len() as f32;
let variance: f32 = all_norms.iter()
.map(|&x| (x - mean).powi(2))
.sum::<f32>() / all_norms.len() as f32;
let std_dev = variance.sqrt();
// Use statistical threshold: mean + threshold_multiplier * std_dev
norm > mean + threshold_multiplier * std_dev && norm > 1.0
}
fn classify_explosion_severity(norm: f32, all_norms: &[f32]) -> ExplosionSeverity {
let mean = all_norms.iter().sum::<f32>() / all_norms.len() as f32;
let ratio = if mean > 0.0 { norm / mean } else { 0.0 };
match ratio {
r if r > 10.0 => ExplosionSeverity::Critical,
r if r > 5.0 => ExplosionSeverity::High,
r if r > 2.0 => ExplosionSeverity::Medium,
_ => ExplosionSeverity::Low,
}
}
fn recommend_explosion_mitigation(norm: f32) -> String {
match norm {
n if n > 10.0 => "Apply aggressive gradient clipping (0.1-0.5) and reduce learning rate by 50%".to_string(),
n if n > 5.0 => "Apply moderate gradient clipping (0.5-1.0) and reduce learning rate by 25%".to_string(),
n if n > 2.0 => "Apply light gradient clipping (1.0-2.0) and monitor closely".to_string(),
_ => "Monitor gradient norms for trends".to_string(),
}
}
fn compute_optimal_clip_value(gradient_norms: &[f32]) -> f32 {
if gradient_norms.is_empty() {
return 1.0;
}
let mut sorted_norms = gradient_norms.to_vec();
sorted_norms.sort_by(|a, b| a.partial_cmp(b).unwrap_or(std::cmp::Ordering::Equal));
// Use 95th percentile as clipping value for robustness
let percentile_95_idx = (sorted_norms.len() as f32 * 0.95) as usize;
sorted_norms.get(percentile_95_idx).copied().unwrap_or(1.0)
}
fn compute_advanced_weight_stats(weights: &ArrayD<f32>) -> Result<WeightStatistics> {
let values: Vec<f32> = weights.iter().cloned().collect();
if values.is_empty() {
return Ok(WeightStatistics::default());
}
let n = values.len() as f64;
let sum: f64 = values.iter().map(|&x| x as f64).sum();
let mean = sum / n;
let variance: f64 = values.iter()
.map(|&x| (x as f64 - mean).powi(2))
.sum::<f64>() / n;
let std_dev = variance.sqrt();
// Advanced statistics
let skewness = Self::compute_skewness(&values, mean, std_dev);
let kurtosis = Self::compute_kurtosis(&values, mean, std_dev);
let entropy = Self::compute_entropy(&values);
Ok(WeightStatistics {
mean: mean as f32,
std_dev: std_dev as f32,
skewness,
kurtosis,
entropy,
min: values.iter().copied().fold(f32::INFINITY, f32::min),
max: values.iter().copied().fold(f32::NEG_INFINITY, f32::max),
zero_fraction: values.iter().filter(|&&x| x.abs() < 1e-8).count() as f32 / values.len() as f32,
})
}
fn compute_skewness(values: &[f32], mean: f64, std_dev: f64) -> f32 {
if std_dev == 0.0 {
return 0.0;
}
let n = values.len() as f64;
let skew: f64 = values.iter()
.map(|&x| ((x as f64 - mean) / std_dev).powi(3))
.sum::<f64>() / n;
skew as f32
}
fn compute_kurtosis(values: &[f32], mean: f64, std_dev: f64) -> f32 {
if std_dev == 0.0 {
return 0.0;
}
let n = values.len() as f64;
let kurt: f64 = values.iter()
.map(|&x| ((x as f64 - mean) / std_dev).powi(4))
.sum::<f64>() / n - 3.0; // Excess kurtosis
kurt as f32
}
fn compute_entropy(values: &[f32]) -> f32 {
// Simplified entropy calculation using binning
let mut bins = vec![0; 50]; // Use 50 bins
let min_val = values.iter().copied().fold(f32::INFINITY, f32::min);
let max_val = values.iter().copied().fold(f32::NEG_INFINITY, f32::max);
if max_val <= min_val {
return 0.0;
}
let bin_width = (max_val - min_val) / 50.0;
for &value in values {
let bin_idx = ((value - min_val) / bin_width).floor() as usize;
let bin_idx = bin_idx.min(49); // Ensure we don't exceed bounds
bins[bin_idx] += 1;
}
let n = values.len() as f32;
let entropy: f32 = bins.iter()
.filter(|&&count| count > 0)
.map(|&count| {
let p = count as f32 / n;
-p * p.log2()
})
.sum();
entropy
}
fn assess_weight_health(stats: &WeightStatistics) -> WeightHealth {
let mut issues = Vec::new();
let mut recommendations = Vec::new();
let mut score: f32 = 100.0;
// Check for common weight issues
if stats.zero_fraction > 0.5 {
issues.push("High sparsity detected".to_string());
recommendations.push("Consider reducing regularization or adjusting initialization".to_string());
score -= 20.0;
}
if stats.std_dev < 0.01 {
issues.push("Low weight variance - potential underfitting".to_string());
recommendations.push("Increase model capacity or reduce regularization".to_string());
score -= 15.0;
}
if stats.std_dev > 1.0 {
issues.push("High weight variance - potential instability".to_string());
recommendations.push("Add weight regularization or reduce learning rate".to_string());
score -= 15.0;
}
if stats.skewness.abs() > 2.0 {
issues.push("Highly skewed weight distribution".to_string());
recommendations.push("Check for gradient flow issues or adjust initialization".to_string());
score -= 10.0;
}
if stats.kurtosis > 5.0 {
issues.push("Heavy-tailed weight distribution".to_string());
recommendations.push("Monitor for outliers and consider gradient clipping".to_string());
score -= 10.0;
}
WeightHealth {
score: score.max(0.0),
issues,
recommendations,
}
}
fn assess_overall_distribution_health(stats: &WeightStatistics) -> DistributionHealth {
let score = 100.0
- (stats.zero_fraction * 30.0) // Penalize high sparsity
- (stats.skewness.abs() * 10.0) // Penalize skewness
- ((stats.kurtosis - 3.0).abs() * 5.0); // Penalize non-normal kurtosis
let status = match score {
s if s >= 80.0 => DistributionHealthStatus::Excellent,
s if s >= 60.0 => DistributionHealthStatus::Good,
s if s >= 40.0 => DistributionHealthStatus::Fair,
s if s >= 20.0 => DistributionHealthStatus::Poor,
_ => DistributionHealthStatus::Critical,
};
DistributionHealth {
score: score.max(0.0),
status,
}
}
fn detect_weight_outliers(weights: &[ArrayD<f32>]) -> Result<Vec<WeightOutlier>> {
let mut outliers = Vec::new();
for (layer_idx, weight) in weights.iter().enumerate() {
let values: Vec<f32> = weight.iter().cloned().collect();
let mean = values.iter().sum::<f32>() / values.len() as f32;
let variance = values.iter()
.map(|&x| (x - mean).powi(2))
.sum::<f32>() / values.len() as f32;
let std_dev = variance.sqrt();
// Detect outliers using 3-sigma rule
let threshold = 3.0 * std_dev;
for (idx, &value) in values.iter().enumerate() {
if (value - mean).abs() > threshold {
outliers.push(WeightOutlier {
layer_index: layer_idx,
weight_index: idx,
value,
z_score: (value - mean) / std_dev,
severity: if (value - mean).abs() > 4.0 * std_dev {
OutlierSeverity::High
} else {
OutlierSeverity::Medium
},
});
}
}
}
Ok(outliers)
}
fn compute_weight_differences(weights_a: &[ArrayD<f32>], weights_b: &[ArrayD<f32>]) -> Vec<f32> {
weights_a.iter()
.zip(weights_b.iter())
.map(|(a, b)| {
let diff: f32 = a.iter().zip(b.iter())
.map(|(&x, &y)| (x - y).abs())
.sum();
diff / a.len() as f32
})
.collect()
}
fn calculate_weight_drift(differences: &[f32]) -> WeightDriftAnalysis {
let mean_drift = differences.iter().sum::<f32>() / differences.len() as f32;
let max_drift = differences.iter().copied().fold(0.0, f32::max);
let drift_severity = match mean_drift {
d if d > 0.1 => DriftSeverity::High,
d if d > 0.05 => DriftSeverity::Medium,
d if d > 0.01 => DriftSeverity::Low,
_ => DriftSeverity::Minimal,
};
WeightDriftAnalysis {
mean_drift,
max_drift,
severity: drift_severity,
affected_layers: differences.iter()
.enumerate()
.filter(|(_, &diff)| diff > mean_drift * 2.0)
.map(|(idx, _)| idx)
.collect(),
}
}
fn analyze_gradient_coherence(differences: &[f32]) -> GradientCoherenceAnalysis {
let coherence_score = 1.0 - (differences.iter().sum::<f32>() / differences.len() as f32);
GradientCoherenceAnalysis {
coherence_score,
inconsistent_layers: differences.iter()
.enumerate()
.filter(|(_, &diff)| diff > 0.1)
.map(|(idx, _)| idx)
.collect(),
}
}
fn analyze_performance_change(metrics_a: &ModelMetrics, metrics_b: &ModelMetrics) -> PerformanceChangeAnalysis {
let loss_change = ((metrics_b.loss - metrics_a.loss) / metrics_a.loss) * 100.0;
let accuracy_change = ((metrics_b.accuracy - metrics_a.accuracy) / metrics_a.accuracy) * 100.0;
PerformanceChangeAnalysis {
loss_change_percent: loss_change,
accuracy_change_percent: accuracy_change,
is_improvement: loss_change < 0.0 && accuracy_change > 0.0,
is_regression: loss_change > 5.0 || accuracy_change < -2.0,
}
}
fn calculate_regression_probability(performance_change: &PerformanceChangeAnalysis) -> f32 {
let mut probability: f32 = 0.0;
if performance_change.loss_change_percent > 10.0 {
probability += 0.4;
} else if performance_change.loss_change_percent > 5.0 {
probability += 0.2;
}
if performance_change.accuracy_change_percent < -5.0 {
probability += 0.4;
} else if performance_change.accuracy_change_percent < -2.0 {
probability += 0.2;
}
probability.min(1.0)
}
fn assess_change_magnitude(
weight_drift: &WeightDriftAnalysis,
gradient_coherence: &GradientCoherenceAnalysis
) -> ChangeMagnitude {
let weight_score = match weight_drift.severity {
DriftSeverity::High => 3,
DriftSeverity::Medium => 2,
DriftSeverity::Low => 1,
DriftSeverity::Minimal => 0,
};
let coherence_score = if gradient_coherence.coherence_score < 0.7 { 2 } else { 0 };
match weight_score + coherence_score {
5.. => ChangeMagnitude::Major,
3..=4 => ChangeMagnitude::Moderate,
1..=2 => ChangeMagnitude::Minor,
_ => ChangeMagnitude::Negligible,
}
}
fn generate_comparison_recommendations(
weight_drift: &WeightDriftAnalysis,
performance_change: &PerformanceChangeAnalysis,
regression_probability: f32
) -> Vec<String> {
let mut recommendations = Vec::new();
if regression_probability > 0.5 {
recommendations.push("High regression probability detected - consider reverting changes".to_string());
}
match weight_drift.severity {
DriftSeverity::High => {
recommendations.push("Significant weight drift detected - investigate learning rate and optimization settings".to_string());
},
DriftSeverity::Medium => {
recommendations.push("Moderate weight changes observed - monitor training stability".to_string());
},
_ => {}
}
if performance_change.loss_change_percent > 10.0 {
recommendations.push("Large loss increase - check for overfitting or data distribution changes".to_string());
}
if performance_change.accuracy_change_percent < -5.0 {
recommendations.push("Significant accuracy drop - validate model architecture and training data".to_string());
}
recommendations
}
}
#[derive(Debug, Serialize, Deserialize)]
pub struct ExplodingLayer {
pub layer_index: usize,
pub gradient_norm: f32,
pub severity: ExplosionSeverity,
pub recommended_action: String,
}
#[derive(Debug, Serialize, Deserialize)]
pub enum ExplosionSeverity {
Low,
Medium,
High,
Critical,
}
/// Weight distribution analysis
#[derive(Debug, Serialize, Deserialize)]
pub struct WeightDistributionAnalysis {
pub layer_analyses: Vec<LayerWeightAnalysis>,
pub overall_statistics: WeightStatistics,
pub distribution_health: DistributionHealth,
pub outlier_detection: Vec<WeightOutlier>,
}
#[derive(Debug, Serialize, Deserialize)]
pub struct LayerWeightAnalysis {
pub layer_index: usize,
pub statistics: WeightStatistics,
pub health_score: f32,
pub issues: Vec<String>,
pub recommendations: Vec<String>,
}
#[derive(Debug, Clone, Serialize, Deserialize, Default)]
pub struct WeightStatistics {
pub mean: f32,
pub std_dev: f32,
pub skewness: f32,
pub kurtosis: f32,
pub entropy: f32,
pub min: f32,
pub max: f32,
pub zero_fraction: f32,
}
impl WeightStatistics {
pub fn accumulate(&mut self, other: &WeightStatistics) {
// Simple accumulation for overall statistics
self.mean += other.mean;
self.std_dev += other.std_dev;
self.skewness += other.skewness;
self.kurtosis += other.kurtosis;
self.entropy += other.entropy;
self.min = self.min.min(other.min);
self.max = self.max.max(other.max);
self.zero_fraction += other.zero_fraction;
}
pub fn finalize(&mut self, count: usize) {
if count > 0 {
let count_f32 = count as f32;
self.mean /= count_f32;
self.std_dev /= count_f32;
self.skewness /= count_f32;
self.kurtosis /= count_f32;
self.entropy /= count_f32;
self.zero_fraction /= count_f32;
}
}
}
#[derive(Debug, Serialize, Deserialize)]
pub struct WeightHealth {
pub score: f32,
pub issues: Vec<String>,
pub recommendations: Vec<String>,
}
#[derive(Debug, Serialize, Deserialize)]
pub struct DistributionHealth {
pub score: f32,
pub status: DistributionHealthStatus,
}
#[derive(Debug, Serialize, Deserialize)]
pub enum DistributionHealthStatus {
Excellent,
Good,
Fair,
Poor,
Critical,
}
#[derive(Debug, Serialize, Deserialize)]
pub struct WeightOutlier {
pub layer_index: usize,
pub weight_index: usize,
pub value: f32,
pub z_score: f32,
pub severity: OutlierSeverity,
}
#[derive(Debug, Serialize, Deserialize)]
pub enum OutlierSeverity {
Medium,
High,
}
#[derive(Debug, Serialize, Deserialize)]
pub struct AdvancedModelComparison {
pub weight_drift_analysis: WeightDriftAnalysis,
pub gradient_coherence_analysis: GradientCoherenceAnalysis,
pub performance_change_analysis: PerformanceChangeAnalysis,
pub regression_probability: f32,
pub change_magnitude: ChangeMagnitude,
pub recommendations: Vec<String>,
}
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct WeightDriftAnalysis {
pub mean_drift: f32,
pub max_drift: f32,
pub severity: DriftSeverity,
pub affected_layers: Vec<usize>,
}
#[derive(Debug, Clone, Serialize, Deserialize)]
pub enum DriftSeverity {
Minimal,
Low,
Medium,
High,
}
#[derive(Debug, Serialize, Deserialize)]
pub struct GradientCoherenceAnalysis {
pub coherence_score: f32,
pub inconsistent_layers: Vec<usize>,
}
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct PerformanceChangeAnalysis {
pub loss_change_percent: f32,
pub accuracy_change_percent: f32,
pub is_improvement: bool,
pub is_regression: bool,
}
#[derive(Debug, Serialize, Deserialize)]
pub enum ChangeMagnitude {
Negligible,
Minor,
Moderate,
Major,
}
/// ML-specific metrics analyzer for learning dynamics and attention analysis
pub struct MLMetricsAnalyzer;
impl MLMetricsAnalyzer {
/// Analyze learning dynamics and convergence patterns
pub fn analyze_learning_dynamics(training_history: &[TrainingStep]) -> Result<LearningDynamicsAnalysis> {
if training_history.is_empty() {
return Ok(LearningDynamicsAnalysis::default());
}
let losses: Vec<f32> = training_history.iter().map(|step| step.loss).collect();
let accuracies: Vec<f32> = training_history.iter().map(|step| step.accuracy).collect();
let loss_trend = Self::compute_trend(&losses);
let accuracy_trend = Self::compute_trend(&accuracies);
// Convergence analysis
let convergence_probability = Self::calculate_convergence_probability(&losses, &accuracies);
// Overfitting detection
let overfitting_risk = Self::detect_overfitting_risk(&losses, &accuracies);
// Plateau detection
let plateau_detected = Self::detect_plateau(&losses);
Ok(LearningDynamicsAnalysis {
loss_trend,
accuracy_trend,
convergence_probability,
overfitting_risk,
plateau_detected,
learning_rate_recommendations: Self::generate_lr_recommendations(&losses),
})
}
/// Analyze attention patterns in transformer models
pub fn analyze_attention_patterns(
attention_weights: &[ArrayD<f32>], // [batch, heads, seq_len, seq_len]
) -> Result<AttentionAnalysis> {
let mut head_analyses = Vec::new();
let num_heads = attention_weights.len();
for (head_idx, attention) in attention_weights.iter().enumerate() {
let head_analysis = Self::analyze_single_attention_head(attention)?;
head_analyses.push(AttentionHeadAnalysis {
head_index: head_idx,
specialization_type: Self::classify_head_specialization(&head_analysis),
attention_entropy: head_analysis.entropy,
pattern_strength: head_analysis.pattern_strength,
locality_score: head_analysis.locality_score,
});
}
let attention_health = Self::assess_attention_health(&head_analyses);
let redundancy_score = Self::calculate_head_redundancy(&head_analyses);
let optimization_suggestions = Self::generate_attention_optimizations(&head_analyses);
Ok(AttentionAnalysis {
head_analyses,
attention_health,
redundancy_score,
optimization_suggestions,
})
}
/// Analyze model efficiency across multiple dimensions
pub fn analyze_model_efficiency(
model_stats: &ModelStats,
performance_metrics: &PerformanceMetrics,
) -> Result<ModelEfficiencyAnalysis> {
let parameter_efficiency = Self::calculate_parameter_efficiency(
model_stats.total_parameters,
performance_metrics.accuracy,
);
let computational_efficiency = Self::calculate_computational_efficiency(
model_stats.flops,
performance_metrics.accuracy,
performance_metrics.inference_time,
);
let memory_efficiency = Self::calculate_memory_efficiency(
model_stats.memory_usage,
performance_metrics.accuracy,
);
let bottlenecks = Self::identify_efficiency_bottlenecks(
parameter_efficiency,
computational_efficiency,
memory_efficiency,
);
let optimization_recommendations = Self::generate_efficiency_recommendations(&bottlenecks);
Ok(ModelEfficiencyAnalysis {
parameter_efficiency,
computational_efficiency,
memory_efficiency,
overall_efficiency_score: (parameter_efficiency + computational_efficiency + memory_efficiency) / 3.0,
bottlenecks,
optimization_recommendations,
})
}
// Helper methods for learning dynamics analysis
fn compute_trend(values: &[f32]) -> TrendDirection {
if values.len() < 2 {
return TrendDirection::Stable;
}
let start = values[0];
let end = values[values.len() - 1];
let change = (end - start) / start;
match change {
c if c > 0.05 => TrendDirection::Increasing,
c if c < -0.05 => TrendDirection::Decreasing,
_ => TrendDirection::Stable,
}
}
fn calculate_convergence_probability(losses: &[f32], accuracies: &[f32]) -> f32 {
let loss_variance = Self::calculate_variance(losses);
let accuracy_variance = Self::calculate_variance(accuracies);
// Lower variance indicates higher convergence probability
let loss_convergence = (1.0 - loss_variance.min(1.0)).max(0.0);
let accuracy_convergence = (1.0 - accuracy_variance.min(1.0)).max(0.0);
(loss_convergence + accuracy_convergence) / 2.0
}
fn detect_overfitting_risk(losses: &[f32], accuracies: &[f32]) -> OverfittingRisk {
// Simplified overfitting detection based on trends
let loss_trend = Self::compute_trend(losses);
let accuracy_trend = Self::compute_trend(accuracies);
match (loss_trend, accuracy_trend) {
(TrendDirection::Increasing, TrendDirection::Decreasing) => OverfittingRisk::High,
(TrendDirection::Increasing, TrendDirection::Stable) => OverfittingRisk::Medium,
(TrendDirection::Stable, TrendDirection::Decreasing) => OverfittingRisk::Medium,
_ => OverfittingRisk::Low,
}
}
fn detect_plateau(losses: &[f32]) -> bool {
if losses.len() < 10 {
return false;
}
let recent_losses = &losses[losses.len() - 10..];
let variance = Self::calculate_variance(recent_losses);
variance < 0.001 // Very low variance indicates plateau
}
fn generate_lr_recommendations(losses: &[f32]) -> Vec<String> {
let mut recommendations = Vec::new();
if losses.len() < 2 {
return recommendations;
}
let trend = Self::compute_trend(losses);
match trend {
TrendDirection::Increasing => {
recommendations.push("Loss is increasing - consider reducing learning rate".to_string());
recommendations.push("Check for gradient explosion or data issues".to_string());
},
TrendDirection::Stable => {
recommendations.push("Loss has plateaued - consider learning rate scheduling".to_string());
recommendations.push("Try warmup restarts or cyclical learning rates".to_string());
},
TrendDirection::Decreasing => {
recommendations.push("Good convergence - maintain current learning rate".to_string());
},
}
recommendations
}
fn calculate_variance(values: &[f32]) -> f32 {
if values.is_empty() {
return 0.0;
}
let mean = values.iter().sum::<f32>() / values.len() as f32;
let variance = values.iter()
.map(|&x| (x - mean).powi(2))
.sum::<f32>() / values.len() as f32;
variance
}
// Helper methods for attention analysis
fn analyze_single_attention_head(attention: &ArrayD<f32>) -> Result<SingleHeadAnalysis> {
let values: Vec<f32> = attention.iter().cloned().collect();
let entropy = Self::calculate_attention_entropy(&values);
let pattern_strength = Self::calculate_pattern_strength(&values);
let locality_score = Self::calculate_locality_score(&values);
Ok(SingleHeadAnalysis {
entropy,
pattern_strength,
locality_score,
})
}
fn calculate_attention_entropy(values: &[f32]) -> f32 {
// Simplified entropy calculation
let sum: f32 = values.iter().sum();
if sum == 0.0 {
return 0.0;
}
let entropy: f32 = values.iter()
.filter(|&&x| x > 0.0)
.map(|&x| {
let p = x / sum;
-p * p.log2()
})
.sum();
entropy
}
fn calculate_pattern_strength(values: &[f32]) -> f32 {
// Measure how structured the attention pattern is
let max_val = values.iter().copied().fold(0.0, f32::max);
let mean_val = values.iter().sum::<f32>() / values.len() as f32;
if mean_val > 0.0 {
max_val / mean_val
} else {
0.0
}
}
fn calculate_locality_score(values: &[f32]) -> f32 {
// Simplified locality score - measures how local the attention is
// Higher score = more local attention
let seq_len = (values.len() as f32).sqrt() as usize;
if seq_len == 0 {
return 0.0;
}
let mut local_sum = 0.0;
let mut total_sum = 0.0;
for i in 0..seq_len {
for j in 0..seq_len {
let idx = i * seq_len + j;
if idx < values.len() {
let value = values[idx];
total_sum += value;
// Consider attention as local if within distance of 3
if (i as i32 - j as i32).abs() <= 3 {
local_sum += value;
}
}
}
}
if total_sum > 0.0 {
local_sum / total_sum
} else {
0.0
}
}
fn classify_head_specialization(analysis: &SingleHeadAnalysis) -> HeadSpecializationType {
match (analysis.entropy, analysis.locality_score, analysis.pattern_strength) {
(e, l, _) if e < 2.0 && l > 0.7 => HeadSpecializationType::Local,
(e, l, _) if e > 4.0 && l < 0.3 => HeadSpecializationType::Global,
(_, _, p) if p > 5.0 => HeadSpecializationType::Structured,
_ => HeadSpecializationType::General,
}
}
fn assess_attention_health(head_analyses: &[AttentionHeadAnalysis]) -> AttentionHealth {
let avg_entropy = head_analyses.iter()
.map(|h| h.attention_entropy)
.sum::<f32>() / head_analyses.len() as f32;
let specialization_diversity = Self::calculate_specialization_diversity(head_analyses);
let health_score = (avg_entropy / 5.0).min(1.0) * 50.0 + specialization_diversity * 50.0;
AttentionHealth {
overall_score: health_score,
entropy_score: avg_entropy,
diversity_score: specialization_diversity,
}
}
fn calculate_specialization_diversity(head_analyses: &[AttentionHeadAnalysis]) -> f32 {
use std::collections::HashSet;
let unique_types: HashSet<_> = head_analyses.iter()
.map(|h| std::mem::discriminant(&h.specialization_type))
.collect();
unique_types.len() as f32 / 4.0 // 4 possible specialization types
}
fn calculate_head_redundancy(head_analyses: &[AttentionHeadAnalysis]) -> f32 {
// Simplified redundancy calculation
let mut redundancy_score = 0.0;
let num_heads = head_analyses.len();
if num_heads < 2 {
return 0.0;
}
for i in 0..num_heads {
for j in (i + 1)..num_heads {
let similarity = Self::calculate_head_similarity(&head_analyses[i], &head_analyses[j]);
if similarity > 0.8 {
redundancy_score += 1.0;
}
}
}
redundancy_score / ((num_heads * (num_heads - 1)) / 2) as f32
}
fn calculate_head_similarity(head1: &AttentionHeadAnalysis, head2: &AttentionHeadAnalysis) -> f32 {
let entropy_similarity = 1.0 - (head1.attention_entropy - head2.attention_entropy).abs() / 5.0;
let locality_similarity = 1.0 - (head1.locality_score - head2.locality_score).abs();
let pattern_similarity = 1.0 - (head1.pattern_strength - head2.pattern_strength).abs() / 10.0;
(entropy_similarity + locality_similarity + pattern_similarity) / 3.0
}
fn generate_attention_optimizations(head_analyses: &[AttentionHeadAnalysis]) -> Vec<String> {
let mut suggestions = Vec::new();
let redundancy_score = Self::calculate_head_redundancy(head_analyses);
if redundancy_score > 0.5 {
suggestions.push("High head redundancy detected - consider head pruning".to_string());
}
let local_heads = head_analyses.iter()
.filter(|h| matches!(h.specialization_type, HeadSpecializationType::Local))
.count();
if local_heads == 0 {
suggestions.push("No local attention heads found - consider adding positional encodings".to_string());
}
suggestions
}
// Helper methods for efficiency analysis
fn calculate_parameter_efficiency(total_params: usize, accuracy: f32) -> f32 {
if total_params == 0 {
return 0.0;
}
// Parameters per accuracy point (lower is better, so invert)
let params_per_accuracy = total_params as f32 / accuracy;
1.0 / (1.0 + params_per_accuracy / 1000000.0) // Normalize to 0-1 scale
}
fn calculate_computational_efficiency(flops: u64, accuracy: f32, inference_time: f32) -> f32 {
if flops == 0 || inference_time == 0.0 {
return 0.0;
}
let flops_per_accuracy = flops as f32 / accuracy;
let time_per_accuracy = inference_time / accuracy;
// Combine FLOPS and time efficiency
let flop_efficiency = 1.0 / (1.0 + flops_per_accuracy / 1e9);
let time_efficiency = 1.0 / (1.0 + time_per_accuracy);
(flop_efficiency + time_efficiency) / 2.0
}
fn calculate_memory_efficiency(memory_usage: usize, accuracy: f32) -> f32 {
if memory_usage == 0 {
return 0.0;
}
let memory_per_accuracy = memory_usage as f32 / accuracy;
1.0 / (1.0 + memory_per_accuracy / 1000000.0) // Normalize to 0-1 scale
}
fn identify_efficiency_bottlenecks(
param_eff: f32,
comp_eff: f32,
mem_eff: f32,
) -> Vec<EfficiencyBottleneck> {
let mut bottlenecks = Vec::new();
if param_eff < 0.3 {
bottlenecks.push(EfficiencyBottleneck::Parameters);
}
if comp_eff < 0.3 {
bottlenecks.push(EfficiencyBottleneck::Computation);
}
if mem_eff < 0.3 {
bottlenecks.push(EfficiencyBottleneck::Memory);
}
bottlenecks
}
fn generate_efficiency_recommendations(bottlenecks: &[EfficiencyBottleneck]) -> Vec<String> {
let mut recommendations = Vec::new();
for bottleneck in bottlenecks {
match bottleneck {
EfficiencyBottleneck::Parameters => {
recommendations.push("Consider parameter pruning or knowledge distillation".to_string());
recommendations.push("Evaluate if model architecture can be simplified".to_string());
},
EfficiencyBottleneck::Computation => {
recommendations.push("Consider operation fusion or quantization".to_string());
recommendations.push("Optimize attention mechanisms or use linear attention".to_string());
},
EfficiencyBottleneck::Memory => {
recommendations.push("Consider gradient checkpointing or activation compression".to_string());
recommendations.push("Evaluate memory-efficient attention implementations".to_string());
},
}
}
recommendations
}
}
// Additional data structures for ML metrics analysis
#[derive(Debug, Serialize, Deserialize)]
pub struct TrainingStep {
pub epoch: u32,
pub step: u32,
pub loss: f32,
pub accuracy: f32,
pub learning_rate: f32,
}
#[derive(Debug, Default, Serialize, Deserialize)]
pub struct LearningDynamicsAnalysis {
pub loss_trend: TrendDirection,
pub accuracy_trend: TrendDirection,
pub convergence_probability: f32,
pub overfitting_risk: OverfittingRisk,
pub plateau_detected: bool,
pub learning_rate_recommendations: Vec<String>,
}
#[derive(Debug, Serialize, Deserialize)]
pub enum TrendDirection {
Increasing,
Decreasing,
Stable,
}
impl Default for TrendDirection {
fn default() -> Self {
TrendDirection::Stable
}
}
#[derive(Debug, Serialize, Deserialize)]
pub enum OverfittingRisk {
Low,
Medium,
High,
}
impl Default for OverfittingRisk {
fn default() -> Self {
OverfittingRisk::Low
}
}
#[derive(Debug, Serialize, Deserialize)]
pub struct AttentionAnalysis {
pub head_analyses: Vec<AttentionHeadAnalysis>,
pub attention_health: AttentionHealth,
pub redundancy_score: f32,
pub optimization_suggestions: Vec<String>,
}
#[derive(Debug, Serialize, Deserialize)]
pub struct AttentionHeadAnalysis {
pub head_index: usize,
pub specialization_type: HeadSpecializationType,
pub attention_entropy: f32,
pub pattern_strength: f32,
pub locality_score: f32,
}
#[derive(Debug, Serialize, Deserialize)]
pub enum HeadSpecializationType {
Local,
Global,
Structured,
General,
}
#[derive(Debug, Serialize, Deserialize)]
pub struct AttentionHealth {
pub overall_score: f32,
pub entropy_score: f32,
pub diversity_score: f32,
}
#[derive(Debug, Serialize, Deserialize)]
pub struct SingleHeadAnalysis {
pub entropy: f32,
pub pattern_strength: f32,
pub locality_score: f32,
}
#[derive(Debug, Serialize, Deserialize)]
pub struct ModelStats {
pub total_parameters: usize,
pub flops: u64,
pub memory_usage: usize,
}
#[derive(Debug, Serialize, Deserialize)]
pub struct PerformanceMetrics {
pub accuracy: f32,
pub inference_time: f32,
}
#[derive(Debug, Serialize, Deserialize)]
pub struct ModelEfficiencyAnalysis {
pub parameter_efficiency: f32,
pub computational_efficiency: f32,
pub memory_efficiency: f32,
pub overall_efficiency_score: f32,
pub bottlenecks: Vec<EfficiencyBottleneck>,
pub optimization_recommendations: Vec<String>,
}
#[derive(Debug, Serialize, Deserialize)]
pub enum EfficiencyBottleneck {
Parameters,
Computation,
Memory,
}
/// Model state comparison
#[derive(Debug, Serialize, Deserialize)]
pub struct ModelStateComparison {
pub weight_differences: Vec<f32>,
pub gradient_differences: Vec<f32>,
pub significant_changes: Vec<LayerChange>,
pub overall_change_magnitude: f32,
pub regression_detected: bool,
}
#[derive(Debug, Serialize, Deserialize)]
pub struct LayerChange {
pub layer_index: usize,
pub change_type: ChangeType,
pub magnitude: f32,
pub description: String,
}
#[derive(Debug, Serialize, Deserialize)]
pub enum ChangeType {
WeightUpdate,
GradientChange,
ActivationShift,
}
/// Advanced spectral analysis utilities for deep mathematical insights into model behavior
pub struct SpectralAnalyzer;
impl SpectralAnalyzer {
/// Perform comprehensive spectral analysis of weight matrices
pub fn analyze_weight_spectrum(weights: &[ArrayD<f32>]) -> Result<SpectralAnalysisResult> {
let mut layer_results = Vec::new();
for (layer_idx, weight_matrix) in weights.iter().enumerate() {
let layer_analysis = Self::analyze_single_matrix_spectrum(weight_matrix)?;
layer_results.push(LayerSpectralAnalysis {
layer_index: layer_idx,
eigenvalue_analysis: layer_analysis.eigenvalue_analysis,
singular_value_analysis: layer_analysis.singular_value_analysis,
condition_number: layer_analysis.condition_number,
effective_rank: layer_analysis.effective_rank,
spectral_norm: layer_analysis.spectral_norm,
nuclear_norm: layer_analysis.nuclear_norm,
stability_analysis: layer_analysis.stability_analysis,
});
}
let recommendations = Self::generate_spectral_recommendations(&layer_results);
let global_analysis = Self::compute_global_spectral_properties(&layer_results);
Ok(SpectralAnalysisResult {
layer_analyses: layer_results,
global_properties: global_analysis,
recommendations,
})
}
/// Analyze eigenvalue spectrum of a single weight matrix
fn analyze_single_matrix_spectrum(matrix: &ArrayD<f32>) -> Result<SingleMatrixSpectralAnalysis> {
let shape = matrix.shape();
if shape.len() != 2 {
return Err(anyhow::anyhow!("Matrix must be 2D for spectral analysis"));
}
let (rows, cols) = (shape[0], shape[1]);
let min_dim = rows.min(cols);
// For computational efficiency, limit analysis to reasonable matrix sizes
if min_dim > 1000 {
return Self::analyze_large_matrix_approximate(matrix);
}
// Convert to square matrix for eigenvalue analysis (A^T A)
let gram_matrix = Self::compute_gram_matrix(matrix)?;
let eigenvalues = Self::compute_eigenvalues(&gram_matrix)?;
// Singular value analysis
let singular_values = Self::compute_singular_values(matrix)?;
// Compute derived metrics
let condition_number = Self::compute_condition_number(&singular_values);
let effective_rank = Self::compute_effective_rank(&singular_values);
let spectral_norm = singular_values.first().copied().unwrap_or(0.0);
let nuclear_norm = singular_values.iter().sum::<f32>();
// Stability analysis
let stability_analysis = Self::analyze_numerical_stability(&eigenvalues, &singular_values);
Ok(SingleMatrixSpectralAnalysis {
eigenvalue_analysis: EigenvalueAnalysis {
eigenvalues: eigenvalues.clone(),
dominant_eigenvalue: eigenvalues.first().copied().unwrap_or(0.0),
eigenvalue_spread: Self::compute_eigenvalue_spread(&eigenvalues),
spectral_radius: eigenvalues.iter().map(|&x| x.abs()).fold(0.0, f32::max),
},
singular_value_analysis: SingularValueAnalysis {
singular_values: singular_values.clone(),
rank_estimate: Self::estimate_numerical_rank(&singular_values),
decay_rate: Self::compute_decay_rate(&singular_values),
},
condition_number,
effective_rank,
spectral_norm,
nuclear_norm,
stability_analysis,
})
}
/// Approximate analysis for large matrices using random sampling
fn analyze_large_matrix_approximate(matrix: &ArrayD<f32>) -> Result<SingleMatrixSpectralAnalysis> {
// For large matrices, use approximate methods
let approx_singular_values = Self::approximate_singular_values(matrix, 50)?;
let condition_number = Self::compute_condition_number(&approx_singular_values);
let effective_rank = Self::compute_effective_rank(&approx_singular_values);
Ok(SingleMatrixSpectralAnalysis {
eigenvalue_analysis: EigenvalueAnalysis {
eigenvalues: approx_singular_values.iter().map(|&x| x * x).collect(),
dominant_eigenvalue: approx_singular_values.first().copied().unwrap_or(0.0).powi(2),
eigenvalue_spread: Self::compute_eigenvalue_spread(&approx_singular_values.iter().map(|&x| x * x).collect::<Vec<_>>()),
spectral_radius: approx_singular_values.first().copied().unwrap_or(0.0),
},
singular_value_analysis: SingularValueAnalysis {
singular_values: approx_singular_values.clone(),
rank_estimate: Self::estimate_numerical_rank(&approx_singular_values),
decay_rate: Self::compute_decay_rate(&approx_singular_values),
},
condition_number,
effective_rank,
spectral_norm: approx_singular_values.first().copied().unwrap_or(0.0),
nuclear_norm: approx_singular_values.iter().sum::<f32>(),
stability_analysis: Self::analyze_numerical_stability(
&approx_singular_values.iter().map(|&x| x * x).collect::<Vec<_>>(),
&approx_singular_values
),
})
}
/// Compute Gram matrix A^T A for eigenvalue analysis
fn compute_gram_matrix(matrix: &ArrayD<f32>) -> Result<Vec<Vec<f32>>> {
let shape = matrix.shape();
let (rows, cols) = (shape[0], shape[1]);
let mut gram = vec![vec![0.0; cols]; cols];
for i in 0..cols {
for j in 0..cols {
let mut sum = 0.0;
for k in 0..rows {
sum += matrix[[k, i]] * matrix[[k, j]];
}
gram[i][j] = sum;
}
}
Ok(gram)
}
/// Simplified eigenvalue computation using power iteration for dominant eigenvalue
fn compute_eigenvalues(matrix: &[Vec<f32>]) -> Result<Vec<f32>> {
let n = matrix.len();
if n == 0 {
return Ok(Vec::new());
}
// For simplicity, compute only the dominant eigenvalue using power iteration
let mut eigenvalues = Vec::new();
let dominant = Self::power_iteration(matrix)?;
eigenvalues.push(dominant);
// Estimate other eigenvalues using Gershgorin circles (approximate)
for i in 0..n.min(10) { // Limit to first 10 for efficiency
let mut center = matrix[i][i];
let mut radius = 0.0;
for j in 0..n {
if i != j {
radius += matrix[i][j].abs();
}
}
// Add estimated eigenvalue from Gershgorin circle
if eigenvalues.len() < 10 {
eigenvalues.push(center + radius * 0.5); // Simplified estimate
}
}
eigenvalues.sort_by(|a, b| b.abs().partial_cmp(&a.abs()).unwrap_or(std::cmp::Ordering::Equal));
Ok(eigenvalues)
}
/// Power iteration to find dominant eigenvalue
fn power_iteration(matrix: &[Vec<f32>]) -> Result<f32> {
let n = matrix.len();
if n == 0 {
return Ok(0.0);
}
let mut x = vec![1.0; n];
let max_iterations = 100;
let tolerance = 1e-6;
for _ in 0..max_iterations {
let mut x_new = vec![0.0; n];
// Matrix-vector multiplication
for i in 0..n {
for j in 0..n {
x_new[i] += matrix[i][j] * x[j];
}
}
// Normalize
let norm = x_new.iter().map(|&x| x * x).sum::<f32>().sqrt();
if norm < tolerance {
return Ok(0.0);
}
for val in &mut x_new {
*val /= norm;
}
// Check convergence
let diff: f32 = x.iter().zip(&x_new).map(|(&a, &b)| (a - b).abs()).sum();
if diff < tolerance {
break;
}
x = x_new;
}
// Compute Rayleigh quotient for eigenvalue estimate
let mut numerator = 0.0;
let mut denominator = 0.0;
for i in 0..n {
let mut ax_i = 0.0;
for j in 0..n {
ax_i += matrix[i][j] * x[j];
}
numerator += x[i] * ax_i;
denominator += x[i] * x[i];
}
Ok(if denominator > tolerance { numerator / denominator } else { 0.0 })
}
/// Approximate singular values using random projection
fn approximate_singular_values(matrix: &ArrayD<f32>, num_singular_values: usize) -> Result<Vec<f32>> {
let shape = matrix.shape();
let (rows, cols) = (shape[0], shape[1]);
let min_dim = rows.min(cols);
let k = num_singular_values.min(min_dim);
// Simplified approximation - use norm of random projections
let mut singular_values = Vec::new();
for _ in 0..k {
// Random vector
let mut random_vec = vec![0.0; cols];
for val in &mut random_vec {
*val = rand::random::<f32>() - 0.5; // Simple random initialization
}
// Matrix-vector multiplication
let mut result = vec![0.0; rows];
for i in 0..rows {
for j in 0..cols {
result[i] += matrix[[i, j]] * random_vec[j];
}
}
// Compute norm
let norm = result.iter().map(|&x| x * x).sum::<f32>().sqrt();
singular_values.push(norm);
}
singular_values.sort_by(|a, b| b.partial_cmp(a).unwrap_or(std::cmp::Ordering::Equal));
Ok(singular_values)
}
/// Compute singular values using simplified SVD approximation
fn compute_singular_values(matrix: &ArrayD<f32>) -> Result<Vec<f32>> {
// For efficiency, use approximate method for larger matrices
let shape = matrix.shape();
let (rows, cols) = (shape[0], shape[1]);
if rows.min(cols) > 100 {
return Self::approximate_singular_values(matrix, 50);
}
// For smaller matrices, compute more accurately
let mut values = Vec::new();
let min_dim = rows.min(cols);
// Simple approximation using matrix norms
for i in 0..min_dim.min(20) {
let mut row_norm = 0.0;
let mut col_norm = 0.0;
if i < rows {
for j in 0..cols {
row_norm += matrix[[i, j]].abs();
}
}
if i < cols {
for j in 0..rows {
col_norm += matrix[[j, i]].abs();
}
}
values.push((row_norm * col_norm).sqrt());
}
values.sort_by(|a, b| b.partial_cmp(a).unwrap_or(std::cmp::Ordering::Equal));
Ok(values)
}
/// Compute condition number
fn compute_condition_number(singular_values: &[f32]) -> f32 {
if singular_values.is_empty() {
return f32::INFINITY;
}
let max_sv = singular_values.first().copied().unwrap_or(0.0);
let min_sv = singular_values.last().copied().unwrap_or(0.0);
if min_sv > 1e-10 {
max_sv / min_sv
} else {
f32::INFINITY
}
}
/// Compute effective rank based on singular value decay
fn compute_effective_rank(singular_values: &[f32]) -> f32 {
if singular_values.is_empty() {
return 0.0;
}
let max_sv = singular_values.first().copied().unwrap_or(0.0);
if max_sv == 0.0 {
return 0.0;
}
let threshold = max_sv * 0.01; // 1% threshold
singular_values.iter().take_while(|&&sv| sv > threshold).count() as f32
}
/// Compute eigenvalue spread
fn compute_eigenvalue_spread(eigenvalues: &[f32]) -> f32 {
if eigenvalues.len() < 2 {
return 0.0;
}
let max_ev = eigenvalues.iter().copied().fold(f32::NEG_INFINITY, f32::max);
let min_ev = eigenvalues.iter().copied().fold(f32::INFINITY, f32::min);
max_ev - min_ev
}
/// Estimate numerical rank
fn estimate_numerical_rank(singular_values: &[f32]) -> usize {
if singular_values.is_empty() {
return 0;
}
let max_sv = singular_values.first().copied().unwrap_or(0.0);
let threshold = max_sv * 1e-10; // Numerical tolerance
singular_values.iter().take_while(|&&sv| sv > threshold).count()
}
/// Compute decay rate of singular values
fn compute_decay_rate(singular_values: &[f32]) -> f32 {
if singular_values.len() < 2 {
return 0.0;
}
let mut decay_sum = 0.0;
let mut count = 0;
for i in 0..(singular_values.len() - 1) {
if singular_values[i] > 0.0 && singular_values[i + 1] > 0.0 {
decay_sum += (singular_values[i + 1] / singular_values[i]).ln().abs();
count += 1;
}
}
if count > 0 {
decay_sum / count as f32
} else {
0.0
}
}
/// Analyze numerical stability
fn analyze_numerical_stability(eigenvalues: &[f32], singular_values: &[f32]) -> StabilityAnalysis {
let condition_number = Self::compute_condition_number(singular_values);
let stability_score = match condition_number {
x if x.is_infinite() => 0.0,
x if x > 1e12 => 0.1,
x if x > 1e8 => 0.3,
x if x > 1e4 => 0.6,
_ => 1.0,
};
let issues = Self::identify_stability_issues(eigenvalues, singular_values, condition_number);
let recommendations = Self::generate_stability_recommendations(&issues);
StabilityAnalysis {
stability_score,
condition_number,
issues,
recommendations,
}
}
/// Identify stability issues
fn identify_stability_issues(
eigenvalues: &[f32],
singular_values: &[f32],
condition_number: f32,
) -> Vec<StabilityIssue> {
let mut issues = Vec::new();
if condition_number > 1e8 {
issues.push(StabilityIssue::IllConditioned);
}
if singular_values.iter().any(|&sv| sv < 1e-10) {
issues.push(StabilityIssue::NearSingular);
}
if eigenvalues.iter().any(|&ev| ev.abs() > 1e6) {
issues.push(StabilityIssue::LargeEigenvalues);
}
let effective_rank = Self::compute_effective_rank(singular_values);
if effective_rank < singular_values.len() as f32 * 0.1 {
issues.push(StabilityIssue::LowRank);
}
issues
}
/// Generate stability recommendations
fn generate_stability_recommendations(issues: &[StabilityIssue]) -> Vec<String> {
let mut recommendations = Vec::new();
for issue in issues {
match issue {
StabilityIssue::IllConditioned => {
recommendations.push("Matrix is ill-conditioned - consider regularization or preconditioning".to_string());
},
StabilityIssue::NearSingular => {
recommendations.push("Matrix is near-singular - check for redundant parameters or add regularization".to_string());
},
StabilityIssue::LargeEigenvalues => {
recommendations.push("Large eigenvalues detected - consider spectral normalization or gradient clipping".to_string());
},
StabilityIssue::LowRank => {
recommendations.push("Low effective rank detected - model may be under-parameterized".to_string());
},
}
}
recommendations
}
/// Compute global spectral properties across all layers
fn compute_global_spectral_properties(layer_analyses: &[LayerSpectralAnalysis]) -> GlobalSpectralProperties {
if layer_analyses.is_empty() {
return GlobalSpectralProperties::default();
}
let avg_condition_number = layer_analyses.iter()
.map(|a| a.condition_number)
.filter(|&x| !x.is_infinite())
.sum::<f32>() / layer_analyses.len() as f32;
let avg_effective_rank = layer_analyses.iter()
.map(|a| a.effective_rank)
.sum::<f32>() / layer_analyses.len() as f32;
let max_spectral_norm = layer_analyses.iter()
.map(|a| a.spectral_norm)
.fold(0.0, f32::max);
let stability_distribution = Self::compute_stability_distribution(layer_analyses);
GlobalSpectralProperties {
average_condition_number: avg_condition_number,
average_effective_rank: avg_effective_rank,
max_spectral_norm,
stability_distribution,
global_stability_score: Self::compute_global_stability_score(layer_analyses),
}
}
/// Compute stability distribution across layers
fn compute_stability_distribution(layer_analyses: &[LayerSpectralAnalysis]) -> StabilityDistribution {
let mut stable_count = 0;
let mut unstable_count = 0;
let mut critical_count = 0;
for analysis in layer_analyses {
match analysis.stability_analysis.stability_score {
score if score > 0.8 => stable_count += 1,
score if score > 0.3 => unstable_count += 1,
_ => critical_count += 1,
}
}
StabilityDistribution {
stable_layers: stable_count,
unstable_layers: unstable_count,
critical_layers: critical_count,
}
}
/// Compute global stability score
fn compute_global_stability_score(layer_analyses: &[LayerSpectralAnalysis]) -> f32 {
if layer_analyses.is_empty() {
return 0.0;
}
layer_analyses.iter()
.map(|a| a.stability_analysis.stability_score)
.sum::<f32>() / layer_analyses.len() as f32
}
/// Generate spectral analysis recommendations
fn generate_spectral_recommendations(layer_analyses: &[LayerSpectralAnalysis]) -> Vec<String> {
let mut recommendations = Vec::new();
let avg_condition = layer_analyses.iter()
.map(|a| a.condition_number)
.filter(|&x| !x.is_infinite())
.sum::<f32>() / layer_analyses.len() as f32;
if avg_condition > 1e6 {
recommendations.push("High average condition number - consider spectral normalization across layers".to_string());
}
let low_rank_layers = layer_analyses.iter()
.filter(|a| a.effective_rank < 10.0)
.count();
if low_rank_layers > layer_analyses.len() / 2 {
recommendations.push("Many layers have low effective rank - consider increasing model capacity".to_string());
}
let unstable_layers = layer_analyses.iter()
.filter(|a| a.stability_analysis.stability_score < 0.5)
.count();
if unstable_layers > layer_analyses.len() / 4 {
recommendations.push("Multiple layers show numerical instability - review initialization and normalization".to_string());
}
recommendations
}
}
/// Information-theoretic analysis utilities for understanding information flow in neural networks
pub struct InformationTheoreticAnalyzer;
impl InformationTheoreticAnalyzer {
/// Analyze information flow between layers
pub fn analyze_information_flow(
layer_activations: &[ArrayD<f32>],
layer_weights: &[ArrayD<f32>],
) -> Result<InformationFlowAnalysis> {
let mut layer_info_analyses = Vec::new();
for (layer_idx, (activation, weight)) in layer_activations.iter().zip(layer_weights.iter()).enumerate() {
let layer_analysis = Self::analyze_single_layer_information(activation, weight)?;
layer_info_analyses.push(LayerInformationAnalysis {
layer_index: layer_idx,
entropy: layer_analysis.entropy,
mutual_information: layer_analysis.mutual_information,
information_bottleneck_score: layer_analysis.information_bottleneck_score,
compression_ratio: layer_analysis.compression_ratio,
});
}
let bottleneck_layers = Self::identify_information_bottlenecks(&layer_info_analyses);
let flow_recommendations = Self::generate_information_flow_recommendations(&layer_info_analyses);
let global_information_flow = Self::compute_global_information_properties(&layer_info_analyses);
Ok(InformationFlowAnalysis {
layer_analyses: layer_info_analyses,
global_flow: global_information_flow,
bottleneck_layers,
flow_recommendations,
})
}
/// Analyze information properties of a single layer
fn analyze_single_layer_information(
activation: &ArrayD<f32>,
weight: &ArrayD<f32>,
) -> Result<SingleLayerInformation> {
let entropy = Self::compute_differential_entropy(activation)?;
let mutual_information = Self::estimate_mutual_information(activation, weight)?;
let information_bottleneck_score = Self::compute_information_bottleneck_score(activation)?;
let compression_ratio = Self::compute_compression_ratio(activation, weight)?;
Ok(SingleLayerInformation {
entropy,
mutual_information,
information_bottleneck_score,
compression_ratio,
})
}
/// Compute differential entropy of activations
fn compute_differential_entropy(activations: &ArrayD<f32>) -> Result<f32> {
let values: Vec<f32> = activations.iter().cloned().collect();
if values.is_empty() {
return Ok(0.0);
}
// Use histogram-based entropy estimation
let num_bins = (values.len() as f32).sqrt().ceil() as usize;
let num_bins = num_bins.max(10).min(100); // Reasonable bounds
let min_val = values.iter().copied().fold(f32::INFINITY, f32::min);
let max_val = values.iter().copied().fold(f32::NEG_INFINITY, f32::max);
if max_val <= min_val {
return Ok(0.0);
}
let bin_width = (max_val - min_val) / num_bins as f32;
let mut histogram = vec![0; num_bins];
for &value in &values {
let bin_idx = ((value - min_val) / bin_width).floor() as usize;
let bin_idx = bin_idx.min(num_bins - 1);
histogram[bin_idx] += 1;
}
let n = values.len() as f32;
let entropy: f32 = histogram.iter()
.filter(|&&count| count > 0)
.map(|&count| {
let p = count as f32 / n;
-p * p.log2()
})
.sum();
// Differential entropy approximation
Ok(entropy + (bin_width.abs() + 1e-10).log2())
}
/// Estimate mutual information between activations and weights
fn estimate_mutual_information(
activations: &ArrayD<f32>,
weights: &ArrayD<f32>,
) -> Result<f32> {
// Simplified mutual information estimation using correlation
let act_values: Vec<f32> = activations.iter().cloned().collect();
let weight_values: Vec<f32> = weights.iter().cloned().collect();
if act_values.is_empty() || weight_values.is_empty() {
return Ok(0.0);
}
// For efficiency, sample from both if they're different sizes
let sample_size = act_values.len().min(weight_values.len()).min(10000);
let mut act_sample = Vec::new();
let mut weight_sample = Vec::new();
for i in 0..sample_size {
let act_idx = (i * act_values.len()) / sample_size;
let weight_idx = (i * weight_values.len()) / sample_size;
act_sample.push(act_values[act_idx]);
weight_sample.push(weight_values[weight_idx]);
}
// Compute correlation-based mutual information estimate
let correlation = Self::compute_correlation(&act_sample, &weight_sample);
let mutual_info = -0.5 * (1.0 - correlation.abs()).ln().max(0.0);
Ok(mutual_info)
}
/// Compute information bottleneck score
fn compute_information_bottleneck_score(activations: &ArrayD<f32>) -> Result<f32> {
let values: Vec<f32> = activations.iter().cloned().collect();
if values.is_empty() {
return Ok(0.0);
}
// Information bottleneck score based on activation compression
let entropy = Self::compute_differential_entropy(activations)?;
let capacity = (values.len() as f32).log2();
// Score represents how much information is preserved vs compressed
let bottleneck_score = if capacity > 0.0 {
(entropy / capacity).min(1.0)
} else {
0.0
};
Ok(bottleneck_score)
}
/// Compute compression ratio between input and output
fn compute_compression_ratio(
activations: &ArrayD<f32>,
weights: &ArrayD<f32>,
) -> Result<f32> {
let input_entropy = Self::compute_differential_entropy(weights)?;
let output_entropy = Self::compute_differential_entropy(activations)?;
if input_entropy > 0.0 {
Ok(output_entropy / input_entropy)
} else {
Ok(1.0)
}
}
/// Compute correlation between two vectors
fn compute_correlation(x: &[f32], y: &[f32]) -> f32 {
if x.len() != y.len() || x.is_empty() {
return 0.0;
}
let n = x.len() as f32;
let mean_x = x.iter().sum::<f32>() / n;
let mean_y = y.iter().sum::<f32>() / n;
let mut numerator = 0.0;
let mut sum_sq_x = 0.0;
let mut sum_sq_y = 0.0;
for (&xi, &yi) in x.iter().zip(y.iter()) {
let dx = xi - mean_x;
let dy = yi - mean_y;
numerator += dx * dy;
sum_sq_x += dx * dx;
sum_sq_y += dy * dy;
}
let denominator = (sum_sq_x * sum_sq_y).sqrt();
if denominator > 1e-10 {
numerator / denominator
} else {
0.0
}
}
/// Compute global information flow properties
fn compute_global_information_properties(
layer_analyses: &[LayerInformationAnalysis],
) -> GlobalInformationFlow {
if layer_analyses.is_empty() {
return GlobalInformationFlow::default();
}
let total_entropy = layer_analyses.iter()
.map(|a| a.entropy)
.sum::<f32>();
let avg_mutual_information = layer_analyses.iter()
.map(|a| a.mutual_information)
.sum::<f32>() / layer_analyses.len() as f32;
let information_flow_efficiency = Self::compute_information_flow_efficiency(layer_analyses);
GlobalInformationFlow {
total_entropy,
average_mutual_information: avg_mutual_information,
flow_efficiency: information_flow_efficiency,
entropy_trend: Self::compute_entropy_trend(layer_analyses),
}
}
/// Compute information flow efficiency
fn compute_information_flow_efficiency(layer_analyses: &[LayerInformationAnalysis]) -> f32 {
if layer_analyses.len() < 2 {
return 1.0;
}
let mut efficiency_sum = 0.0;
for i in 0..(layer_analyses.len() - 1) {
let current_entropy = layer_analyses[i].entropy;
let next_entropy = layer_analyses[i + 1].entropy;
if current_entropy > 0.0 {
let efficiency = (next_entropy / current_entropy).min(1.0);
efficiency_sum += efficiency;
}
}
efficiency_sum / (layer_analyses.len() - 1) as f32
}
/// Compute entropy trend across layers
fn compute_entropy_trend(layer_analyses: &[LayerInformationAnalysis]) -> EntropyTrend {
if layer_analyses.len() < 2 {
return EntropyTrend::Stable;
}
let first_entropy = layer_analyses.first().unwrap().entropy;
let last_entropy = layer_analyses.last().unwrap().entropy;
let change_ratio = if first_entropy > 0.0 {
(last_entropy - first_entropy) / first_entropy
} else {
0.0
};
match change_ratio {
x if x > 0.1 => EntropyTrend::Increasing,
x if x < -0.1 => EntropyTrend::Decreasing,
_ => EntropyTrend::Stable,
}
}
/// Identify information bottleneck layers
fn identify_information_bottlenecks(
layer_analyses: &[LayerInformationAnalysis],
) -> Vec<usize> {
layer_analyses.iter()
.enumerate()
.filter(|(_, analysis)| analysis.information_bottleneck_score < 0.3)
.map(|(idx, _)| idx)
.collect()
}
/// Generate information flow recommendations
fn generate_information_flow_recommendations(
layer_analyses: &[LayerInformationAnalysis],
) -> Vec<String> {
let mut recommendations = Vec::new();
let bottleneck_count = Self::identify_information_bottlenecks(layer_analyses).len();
if bottleneck_count > layer_analyses.len() / 3 {
recommendations.push("Multiple information bottlenecks detected - consider increasing layer capacity".to_string());
}
let low_mi_layers = layer_analyses.iter()
.filter(|a| a.mutual_information < 0.1)
.count();
if low_mi_layers > layer_analyses.len() / 2 {
recommendations.push("Low mutual information across layers - review weight initialization and activations".to_string());
}
let avg_compression = layer_analyses.iter()
.map(|a| a.compression_ratio)
.sum::<f32>() / layer_analyses.len() as f32;
if avg_compression > 2.0 {
recommendations.push("High compression ratios detected - model may be over-compressing information".to_string());
} else if avg_compression < 0.5 {
recommendations.push("Low compression ratios detected - model may not be learning efficient representations".to_string());
}
recommendations
}
}
// Data structures for spectral analysis
#[derive(Debug, Serialize, Deserialize)]
pub struct SpectralAnalysisResult {
pub layer_analyses: Vec<LayerSpectralAnalysis>,
pub global_properties: GlobalSpectralProperties,
pub recommendations: Vec<String>,
}
#[derive(Debug, Serialize, Deserialize)]
pub struct LayerSpectralAnalysis {
pub layer_index: usize,
pub eigenvalue_analysis: EigenvalueAnalysis,
pub singular_value_analysis: SingularValueAnalysis,
pub condition_number: f32,
pub effective_rank: f32,
pub spectral_norm: f32,
pub nuclear_norm: f32,
pub stability_analysis: StabilityAnalysis,
}
#[derive(Debug, Serialize, Deserialize)]
pub struct SingleMatrixSpectralAnalysis {
pub eigenvalue_analysis: EigenvalueAnalysis,
pub singular_value_analysis: SingularValueAnalysis,
pub condition_number: f32,
pub effective_rank: f32,
pub spectral_norm: f32,
pub nuclear_norm: f32,
pub stability_analysis: StabilityAnalysis,
}
#[derive(Debug, Serialize, Deserialize)]
pub struct EigenvalueAnalysis {
pub eigenvalues: Vec<f32>,
pub dominant_eigenvalue: f32,
pub eigenvalue_spread: f32,
pub spectral_radius: f32,
}
#[derive(Debug, Serialize, Deserialize)]
pub struct SingularValueAnalysis {
pub singular_values: Vec<f32>,
pub rank_estimate: usize,
pub decay_rate: f32,
}
#[derive(Debug, Serialize, Deserialize)]
pub struct StabilityAnalysis {
pub stability_score: f32,
pub condition_number: f32,
pub issues: Vec<StabilityIssue>,
pub recommendations: Vec<String>,
}
#[derive(Debug, Serialize, Deserialize)]
pub enum StabilityIssue {
IllConditioned,
NearSingular,
LargeEigenvalues,
LowRank,
}
#[derive(Debug, Default, Serialize, Deserialize)]
pub struct GlobalSpectralProperties {
pub average_condition_number: f32,
pub average_effective_rank: f32,
pub max_spectral_norm: f32,
pub stability_distribution: StabilityDistribution,
pub global_stability_score: f32,
}
#[derive(Debug, Default, Serialize, Deserialize)]
pub struct StabilityDistribution {
pub stable_layers: usize,
pub unstable_layers: usize,
pub critical_layers: usize,
}
// Data structures for information-theoretic analysis
#[derive(Debug, Serialize, Deserialize)]
pub struct InformationFlowAnalysis {
pub layer_analyses: Vec<LayerInformationAnalysis>,
pub global_flow: GlobalInformationFlow,
pub bottleneck_layers: Vec<usize>,
pub flow_recommendations: Vec<String>,
}
#[derive(Debug, Serialize, Deserialize)]
pub struct LayerInformationAnalysis {
pub layer_index: usize,
pub entropy: f32,
pub mutual_information: f32,
pub information_bottleneck_score: f32,
pub compression_ratio: f32,
}
#[derive(Debug, Serialize, Deserialize)]
pub struct SingleLayerInformation {
pub entropy: f32,
pub mutual_information: f32,
pub information_bottleneck_score: f32,
pub compression_ratio: f32,
}
#[derive(Debug, Default, Serialize, Deserialize)]
pub struct GlobalInformationFlow {
pub total_entropy: f32,
pub average_mutual_information: f32,
pub flow_efficiency: f32,
pub entropy_trend: EntropyTrend,
}
#[derive(Debug, Serialize, Deserialize)]
pub enum EntropyTrend {
Increasing,
Decreasing,
Stable,
}
impl Default for EntropyTrend {
fn default() -> Self {
EntropyTrend::Stable
}
}
/// Model complexity analysis utilities for comprehensive model assessment
pub struct ModelComplexityAnalyzer;
impl ModelComplexityAnalyzer {
/// Comprehensive model complexity analysis
pub fn analyze_model_complexity(
weights: &[ArrayD<f32>],
architecture_info: &ModelArchitectureInfo,
) -> Result<ModelComplexityAnalysis> {
let mut layer_complexities = Vec::new();
let mut total_parameters = 0;
let mut total_flops = 0;
for (layer_idx, weight) in weights.iter().enumerate() {
let layer_complexity = Self::analyze_layer_complexity(weight, layer_idx)?;
total_parameters += layer_complexity.parameter_count;
total_flops += layer_complexity.computational_complexity;
layer_complexities.push(layer_complexity);
}
let effective_model_rank = Self::compute_model_effective_rank(&layer_complexities);
let model_capacity = Self::compute_model_capacity(&layer_complexities, total_parameters);
let complexity_distribution = Self::analyze_complexity_distribution(&layer_complexities);
let efficiency_metrics = Self::compute_efficiency_metrics(&layer_complexities, architecture_info);
let bottleneck_analysis = Self::identify_complexity_bottlenecks(&layer_complexities);
let optimization_recommendations = Self::generate_complexity_recommendations(&layer_complexities, &efficiency_metrics);
Ok(ModelComplexityAnalysis {
layer_complexities,
total_parameters,
total_flops,
effective_model_rank,
model_capacity,
complexity_distribution,
efficiency_metrics,
bottleneck_analysis,
optimization_recommendations,
})
}
/// Analyze complexity of a single layer
fn analyze_layer_complexity(weight: &ArrayD<f32>, layer_idx: usize) -> Result<LayerComplexityAnalysis> {
let shape = weight.shape();
let parameter_count = weight.len();
// Compute effective rank and condition number
let effective_rank = Self::compute_layer_effective_rank(weight)?;
let condition_number = Self::compute_layer_condition_number(weight)?;
// Computational complexity estimation
let computational_complexity = Self::estimate_computational_complexity(shape, layer_idx);
// Memory complexity
let memory_complexity = Self::compute_memory_complexity(weight);
// Complexity ratio analysis
let complexity_ratio = Self::compute_complexity_ratio(effective_rank, parameter_count as f32);
// Redundancy analysis
let redundancy_score = Self::compute_redundancy_score(effective_rank, parameter_count);
// Expressiveness analysis
let expressiveness_score = Self::compute_expressiveness_score(weight)?;
Ok(LayerComplexityAnalysis {
layer_index: layer_idx,
parameter_count,
effective_rank,
condition_number,
computational_complexity,
memory_complexity,
complexity_ratio,
redundancy_score,
expressiveness_score,
complexity_class: Self::classify_layer_complexity(effective_rank, parameter_count, condition_number),
})
}
/// Compute effective rank for a single layer
fn compute_layer_effective_rank(weight: &ArrayD<f32>) -> Result<f32> {
let shape = weight.shape();
if shape.len() != 2 {
return Ok(shape.iter().product::<usize>() as f32); // For non-matrix tensors
}
// Simplified effective rank computation using Frobenius norm approximation
let values: Vec<f32> = weight.iter().cloned().collect();
let frobenius_norm = values.iter().map(|&x| x * x).sum::<f32>().sqrt();
let spectral_norm = values.iter().map(|&x| x.abs()).fold(0.0, f32::max);
if spectral_norm > 1e-10 {
let ratio = frobenius_norm / spectral_norm;
Ok(ratio.min(shape[0].min(shape[1]) as f32))
} else {
Ok(0.0)
}
}
/// Compute condition number for a single layer
fn compute_layer_condition_number(weight: &ArrayD<f32>) -> Result<f32> {
let shape = weight.shape();
if shape.len() != 2 {
return Ok(1.0); // For non-matrix tensors, assume well-conditioned
}
// Simplified condition number estimation
let values: Vec<f32> = weight.iter().cloned().collect();
let max_val = values.iter().copied().fold(0.0, f32::max);
let min_val = values.iter().filter(|&&x| x.abs() > 1e-10).map(|&x| x.abs()).fold(f32::INFINITY, f32::min);
if min_val > 1e-10 {
Ok(max_val / min_val)
} else {
Ok(f32::INFINITY)
}
}
/// Estimate computational complexity (FLOPs) for the layer
fn estimate_computational_complexity(shape: &[usize], layer_idx: usize) -> u64 {
match shape.len() {
2 => {
// Dense layer: input_dim * output_dim
(shape[0] as u64) * (shape[1] as u64) * 2 // Multiply + Add
},
4 => {
// Convolutional layer: output_channels * input_channels * kernel_h * kernel_w * output_h * output_w
// Simplified estimation
let total_weights = shape.iter().product::<usize>() as u64;
total_weights * 100 // Rough estimate for conv operations
},
_ => {
// Other layer types
shape.iter().product::<usize>() as u64
}
}
}
/// Compute memory complexity
fn compute_memory_complexity(weight: &ArrayD<f32>) -> MemoryComplexity {
let parameter_count = weight.len();
let memory_bytes = parameter_count * std::mem::size_of::<f32>();
MemoryComplexity {
parameter_memory: memory_bytes,
activation_memory: memory_bytes * 2, // Rough estimate for forward + backward
gradient_memory: memory_bytes,
total_memory: memory_bytes * 4, // Parameters + activations + gradients + buffers
}
}
/// Compute complexity ratio (effective rank / total parameters)
fn compute_complexity_ratio(effective_rank: f32, total_params: f32) -> f32 {
if total_params > 0.0 {
effective_rank / total_params
} else {
0.0
}
}
/// Compute redundancy score
fn compute_redundancy_score(effective_rank: f32, parameter_count: usize) -> f32 {
let theoretical_rank = (parameter_count as f32).sqrt(); // Rough theoretical maximum
if theoretical_rank > 0.0 {
1.0 - (effective_rank / theoretical_rank).min(1.0)
} else {
0.0
}
}
/// Compute expressiveness score
fn compute_expressiveness_score(weight: &ArrayD<f32>) -> Result<f32> {
let values: Vec<f32> = weight.iter().cloned().collect();
if values.is_empty() {
return Ok(0.0);
}
// Compute entropy-based expressiveness
let mut histogram = vec![0; 50];
let min_val = values.iter().copied().fold(f32::INFINITY, f32::min);
let max_val = values.iter().copied().fold(f32::NEG_INFINITY, f32::max);
if max_val <= min_val {
return Ok(0.0);
}
let bin_width = (max_val - min_val) / 50.0;
for &value in &values {
let bin_idx = ((value - min_val) / bin_width).floor() as usize;
let bin_idx = bin_idx.min(49);
histogram[bin_idx] += 1;
}
let n = values.len() as f32;
let entropy: f32 = histogram.iter()
.filter(|&&count| count > 0)
.map(|&count| {
let p = count as f32 / n;
-p * p.log2()
})
.sum();
// Normalize entropy to 0-1 scale
Ok((entropy / 50.0_f32.log2()).min(1.0))
}
/// Classify layer complexity
fn classify_layer_complexity(effective_rank: f32, parameter_count: usize, condition_number: f32) -> ComplexityClass {
let complexity_ratio = effective_rank / parameter_count as f32;
match (complexity_ratio, condition_number) {
(r, c) if r > 0.8 && c < 100.0 => ComplexityClass::HighExpressiveWellConditioned,
(r, c) if r > 0.8 && c >= 100.0 => ComplexityClass::HighExpressiveIllConditioned,
(r, c) if r > 0.4 && c < 100.0 => ComplexityClass::ModerateExpressiveWellConditioned,
(r, c) if r > 0.4 && c >= 100.0 => ComplexityClass::ModerateExpressiveIllConditioned,
(r, c) if r > 0.1 && c < 100.0 => ComplexityClass::LowExpressiveWellConditioned,
(r, c) if r > 0.1 && c >= 100.0 => ComplexityClass::LowExpressiveIllConditioned,
_ => ComplexityClass::Degenerate,
}
}
/// Compute model-wide effective rank
fn compute_model_effective_rank(layer_complexities: &[LayerComplexityAnalysis]) -> f32 {
if layer_complexities.is_empty() {
return 0.0;
}
// Weighted average of layer effective ranks
let total_params: usize = layer_complexities.iter().map(|l| l.parameter_count).sum();
if total_params == 0 {
return 0.0;
}
layer_complexities.iter()
.map(|l| l.effective_rank * (l.parameter_count as f32 / total_params as f32))
.sum()
}
/// Compute model capacity
fn compute_model_capacity(layer_complexities: &[LayerComplexityAnalysis], total_parameters: usize) -> ModelCapacity {
let effective_parameters: f32 = layer_complexities.iter()
.map(|l| l.effective_rank * l.parameter_count as f32 / l.parameter_count as f32)
.sum();
let utilization_ratio = if total_parameters > 0 {
effective_parameters / total_parameters as f32
} else {
0.0
};
let avg_expressiveness = layer_complexities.iter()
.map(|l| l.expressiveness_score)
.sum::<f32>() / layer_complexities.len() as f32;
ModelCapacity {
theoretical_capacity: total_parameters as f32,
effective_capacity: effective_parameters,
utilization_ratio,
expressiveness_score: avg_expressiveness,
}
}
/// Analyze complexity distribution across layers
fn analyze_complexity_distribution(layer_complexities: &[LayerComplexityAnalysis]) -> ComplexityDistribution {
let total_layers = layer_complexities.len();
let mut high_complexity = 0;
let mut medium_complexity = 0;
let mut low_complexity = 0;
let mut degenerate = 0;
for layer in layer_complexities {
match layer.complexity_class {
ComplexityClass::HighExpressiveWellConditioned |
ComplexityClass::HighExpressiveIllConditioned => high_complexity += 1,
ComplexityClass::ModerateExpressiveWellConditioned |
ComplexityClass::ModerateExpressiveIllConditioned => medium_complexity += 1,
ComplexityClass::LowExpressiveWellConditioned |
ComplexityClass::LowExpressiveIllConditioned => low_complexity += 1,
ComplexityClass::Degenerate => degenerate += 1,
}
}
ComplexityDistribution {
high_complexity_layers: high_complexity,
medium_complexity_layers: medium_complexity,
low_complexity_layers: low_complexity,
degenerate_layers: degenerate,
distribution_balance: Self::compute_distribution_balance(high_complexity, medium_complexity, low_complexity, degenerate),
}
}
/// Compute distribution balance score
fn compute_distribution_balance(high: usize, medium: usize, low: usize, degenerate: usize) -> f32 {
let total = high + medium + low + degenerate;
if total == 0 {
return 0.0;
}
// Ideal distribution is more high and medium complexity layers
let ideal_score = (high as f32 * 1.0 + medium as f32 * 0.8 + low as f32 * 0.4 + degenerate as f32 * 0.0) / total as f32;
ideal_score
}
/// Compute efficiency metrics
fn compute_efficiency_metrics(
layer_complexities: &[LayerComplexityAnalysis],
architecture_info: &ModelArchitectureInfo,
) -> EfficiencyMetrics {
let total_params: usize = layer_complexities.iter().map(|l| l.parameter_count).sum();
let total_flops: u64 = layer_complexities.iter().map(|l| l.computational_complexity).sum();
let parameter_efficiency = if architecture_info.target_accuracy > 0.0 {
architecture_info.target_accuracy / (total_params as f32 / 1_000_000.0) // Accuracy per million parameters
} else {
0.0
};
let computational_efficiency = if architecture_info.target_accuracy > 0.0 {
architecture_info.target_accuracy / (total_flops as f32 / 1_000_000_000.0) // Accuracy per billion FLOPs
} else {
0.0
};
let memory_efficiency = if architecture_info.memory_budget > 0 {
let total_memory: usize = layer_complexities.iter()
.map(|l| l.memory_complexity.total_memory)
.sum();
architecture_info.memory_budget as f32 / total_memory as f32
} else {
1.0
};
EfficiencyMetrics {
parameter_efficiency,
computational_efficiency,
memory_efficiency,
overall_efficiency: (parameter_efficiency + computational_efficiency + memory_efficiency) / 3.0,
}
}
/// Identify complexity bottlenecks
fn identify_complexity_bottlenecks(layer_complexities: &[LayerComplexityAnalysis]) -> BottleneckAnalysis {
let mut parameter_bottlenecks = Vec::new();
let mut computational_bottlenecks = Vec::new();
let mut memory_bottlenecks = Vec::new();
let mut conditioning_bottlenecks = Vec::new();
// Find layers with disproportionate resource usage
let total_params: usize = layer_complexities.iter().map(|l| l.parameter_count).sum();
let total_flops: u64 = layer_complexities.iter().map(|l| l.computational_complexity).sum();
let total_memory: usize = layer_complexities.iter()
.map(|l| l.memory_complexity.total_memory)
.sum();
for layer in layer_complexities {
// Parameter bottlenecks (layers using >20% of total parameters)
if layer.parameter_count as f32 / total_params as f32 > 0.2 {
parameter_bottlenecks.push(layer.layer_index);
}
// Computational bottlenecks (layers using >20% of total FLOPs)
if layer.computational_complexity as f32 / total_flops as f32 > 0.2 {
computational_bottlenecks.push(layer.layer_index);
}
// Memory bottlenecks (layers using >20% of total memory)
if layer.memory_complexity.total_memory as f32 / total_memory as f32 > 0.2 {
memory_bottlenecks.push(layer.layer_index);
}
// Conditioning bottlenecks (ill-conditioned layers)
if layer.condition_number > 1000.0 {
conditioning_bottlenecks.push(layer.layer_index);
}
}
let bottleneck_severity = Self::compute_bottleneck_severity(¶meter_bottlenecks, &computational_bottlenecks, &memory_bottlenecks, &conditioning_bottlenecks);
BottleneckAnalysis {
parameter_bottlenecks,
computational_bottlenecks,
memory_bottlenecks,
conditioning_bottlenecks,
bottleneck_severity,
}
}
/// Compute bottleneck severity
fn compute_bottleneck_severity(
param_bottlenecks: &[usize],
comp_bottlenecks: &[usize],
mem_bottlenecks: &[usize],
cond_bottlenecks: &[usize],
) -> BottleneckSeverity {
let total_bottlenecks = param_bottlenecks.len() + comp_bottlenecks.len() + mem_bottlenecks.len() + cond_bottlenecks.len();
match total_bottlenecks {
0 => BottleneckSeverity::None,
1..=2 => BottleneckSeverity::Low,
3..=5 => BottleneckSeverity::Medium,
6..=10 => BottleneckSeverity::High,
_ => BottleneckSeverity::Critical,
}
}
/// Generate complexity optimization recommendations
fn generate_complexity_recommendations(
layer_complexities: &[LayerComplexityAnalysis],
efficiency_metrics: &EfficiencyMetrics,
) -> Vec<ComplexityRecommendation> {
let mut recommendations = Vec::new();
// Check for low effective rank layers
for layer in layer_complexities {
if layer.redundancy_score > 0.7 {
recommendations.push(ComplexityRecommendation {
layer_index: Some(layer.layer_index),
recommendation_type: ComplexityRecommendationType::ReduceRedundancy,
description: format!("Layer {} has high redundancy ({}). Consider rank reduction or pruning.",
layer.layer_index, layer.redundancy_score),
expected_impact: if layer.redundancy_score > 0.9 {
RecommendationImpact::High
} else {
RecommendationImpact::Medium
},
implementation_difficulty: RecommendationDifficulty::Medium,
});
}
if layer.condition_number > 1000.0 {
recommendations.push(ComplexityRecommendation {
layer_index: Some(layer.layer_index),
recommendation_type: ComplexityRecommendationType::ImproveConditioning,
description: format!("Layer {} is ill-conditioned (condition number: {}). Consider regularization or normalization.",
layer.layer_index, layer.condition_number),
expected_impact: RecommendationImpact::High,
implementation_difficulty: RecommendationDifficulty::Low,
});
}
if layer.expressiveness_score < 0.3 {
recommendations.push(ComplexityRecommendation {
layer_index: Some(layer.layer_index),
recommendation_type: ComplexityRecommendationType::IncreaseExpressiveness,
description: format!("Layer {} has low expressiveness ({}). Consider different initialization or activation functions.",
layer.layer_index, layer.expressiveness_score),
expected_impact: RecommendationImpact::Medium,
implementation_difficulty: RecommendationDifficulty::Low,
});
}
}
// Global recommendations
if efficiency_metrics.parameter_efficiency < 0.5 {
recommendations.push(ComplexityRecommendation {
layer_index: None,
recommendation_type: ComplexityRecommendationType::OverallOptimization,
description: "Low parameter efficiency detected. Consider knowledge distillation or architectural optimization.".to_string(),
expected_impact: RecommendationImpact::High,
implementation_difficulty: RecommendationDifficulty::High,
});
}
if efficiency_metrics.computational_efficiency < 0.5 {
recommendations.push(ComplexityRecommendation {
layer_index: None,
recommendation_type: ComplexityRecommendationType::OverallOptimization,
description: "Low computational efficiency detected. Consider operation fusion or quantization.".to_string(),
expected_impact: RecommendationImpact::High,
implementation_difficulty: RecommendationDifficulty::Medium,
});
}
recommendations
}
}
// Data structures for model complexity analysis
#[derive(Debug, Serialize, Deserialize)]
pub struct ModelComplexityAnalysis {
pub layer_complexities: Vec<LayerComplexityAnalysis>,
pub total_parameters: usize,
pub total_flops: u64,
pub effective_model_rank: f32,
pub model_capacity: ModelCapacity,
pub complexity_distribution: ComplexityDistribution,
pub efficiency_metrics: EfficiencyMetrics,
pub bottleneck_analysis: BottleneckAnalysis,
pub optimization_recommendations: Vec<ComplexityRecommendation>,
}
#[derive(Debug, Serialize, Deserialize)]
pub struct LayerComplexityAnalysis {
pub layer_index: usize,
pub parameter_count: usize,
pub effective_rank: f32,
pub condition_number: f32,
pub computational_complexity: u64,
pub memory_complexity: MemoryComplexity,
pub complexity_ratio: f32,
pub redundancy_score: f32,
pub expressiveness_score: f32,
pub complexity_class: ComplexityClass,
}
#[derive(Debug, Serialize, Deserialize)]
pub struct ModelArchitectureInfo {
pub target_accuracy: f32,
pub memory_budget: usize,
pub latency_budget: f32,
pub model_type: String,
}
#[derive(Debug, Serialize, Deserialize)]
pub struct MemoryComplexity {
pub parameter_memory: usize,
pub activation_memory: usize,
pub gradient_memory: usize,
pub total_memory: usize,
}
#[derive(Debug, Serialize, Deserialize)]
pub enum ComplexityClass {
HighExpressiveWellConditioned,
HighExpressiveIllConditioned,
ModerateExpressiveWellConditioned,
ModerateExpressiveIllConditioned,
LowExpressiveWellConditioned,
LowExpressiveIllConditioned,
Degenerate,
}
#[derive(Debug, Serialize, Deserialize)]
pub struct ModelCapacity {
pub theoretical_capacity: f32,
pub effective_capacity: f32,
pub utilization_ratio: f32,
pub expressiveness_score: f32,
}
#[derive(Debug, Serialize, Deserialize)]
pub struct ComplexityDistribution {
pub high_complexity_layers: usize,
pub medium_complexity_layers: usize,
pub low_complexity_layers: usize,
pub degenerate_layers: usize,
pub distribution_balance: f32,
}
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct EfficiencyMetrics {
pub parameter_efficiency: f32,
pub computational_efficiency: f32,
pub memory_efficiency: f32,
pub overall_efficiency: f32,
}
#[derive(Debug, Serialize, Deserialize)]
pub struct BottleneckAnalysis {
pub parameter_bottlenecks: Vec<usize>,
pub computational_bottlenecks: Vec<usize>,
pub memory_bottlenecks: Vec<usize>,
pub conditioning_bottlenecks: Vec<usize>,
pub bottleneck_severity: BottleneckSeverity,
}
#[derive(Debug, Serialize, Deserialize)]
pub enum BottleneckSeverity {
None,
Low,
Medium,
High,
Critical,
}
#[derive(Debug, Serialize, Deserialize)]
pub struct ComplexityRecommendation {
pub layer_index: Option<usize>,
pub recommendation_type: ComplexityRecommendationType,
pub description: String,
pub expected_impact: RecommendationImpact,
pub implementation_difficulty: RecommendationDifficulty,
}
#[derive(Debug, Serialize, Deserialize)]
pub enum ComplexityRecommendationType {
ReduceRedundancy,
ImproveConditioning,
IncreaseExpressiveness,
OptimizeMemory,
OptimizeComputation,
OverallOptimization,
}
#[derive(Debug, Serialize, Deserialize)]
pub enum RecommendationImpact {
Low,
Medium,
High,
}
#[derive(Debug, Serialize, Deserialize)]
pub enum RecommendationDifficulty {
Low,
Medium,
High,
}
/// Advanced gradient flow analysis utilities for sophisticated flow pattern detection
pub struct AdvancedGradientFlowAnalyzer;
impl AdvancedGradientFlowAnalyzer {
/// Comprehensive gradient flow analysis with directional derivatives and flow patterns
pub fn analyze_gradient_flow(
gradients: &[ArrayD<f32>],
model_structure: &ModelStructureInfo,
) -> Result<AdvancedGradientFlowAnalysis> {
let mut layer_flow_analyses = Vec::new();
for (layer_idx, gradient) in gradients.iter().enumerate() {
let layer_analysis = Self::analyze_layer_gradient_flow(gradient, layer_idx)?;
layer_flow_analyses.push(layer_analysis);
}
let directional_derivatives = Self::compute_directional_derivatives(&gradients)?;
let flow_patterns = Self::analyze_flow_patterns(&gradients)?;
let gradient_topology = Self::analyze_gradient_topology(&gradients)?;
let flow_dynamics = Self::analyze_flow_dynamics(&gradients)?;
let convergence_analysis = Self::analyze_flow_convergence(&layer_flow_analyses);
// Compute recommendations before moving values
let flow_recommendations = Self::generate_flow_recommendations(&layer_flow_analyses, &flow_patterns);
Ok(AdvancedGradientFlowAnalysis {
layer_analyses: layer_flow_analyses,
directional_derivatives,
flow_patterns,
gradient_topology,
flow_dynamics,
convergence_analysis,
flow_recommendations,
})
}
/// Analyze gradient flow for a single layer
fn analyze_layer_gradient_flow(gradient: &ArrayD<f32>, layer_idx: usize) -> Result<LayerGradientFlowAnalysis> {
let gradient_magnitude = Self::compute_gradient_magnitude(gradient);
let flow_direction = Self::compute_flow_direction(gradient)?;
let flow_divergence = Self::compute_flow_divergence(gradient)?;
let flow_curl = Self::compute_flow_curl(gradient)?;
let flow_coherence = Self::compute_flow_coherence(gradient)?;
let flow_stability = Self::compute_flow_stability(gradient)?;
Ok(LayerGradientFlowAnalysis {
layer_index: layer_idx,
gradient_magnitude,
flow_direction,
flow_divergence,
flow_curl,
flow_coherence,
flow_stability,
flow_classification: Self::classify_flow_type(flow_divergence, flow_curl, flow_coherence),
})
}
/// Compute gradient magnitude
fn compute_gradient_magnitude(gradient: &ArrayD<f32>) -> f32 {
gradient.iter().map(|&x| x * x).sum::<f32>().sqrt()
}
/// Compute flow direction (normalized gradient)
fn compute_flow_direction(gradient: &ArrayD<f32>) -> Result<FlowDirection> {
let values: Vec<f32> = gradient.iter().cloned().collect();
if values.is_empty() {
return Ok(FlowDirection::default());
}
let magnitude = values.iter().map(|&x| x * x).sum::<f32>().sqrt();
if magnitude < 1e-10 {
return Ok(FlowDirection::default());
}
// Compute principal direction using SVD approximation
let dominant_direction = Self::compute_dominant_direction(&values)?;
let direction_strength = magnitude / values.len() as f32;
let direction_consistency = Self::compute_direction_consistency(&values, &dominant_direction);
Ok(FlowDirection {
dominant_direction,
direction_strength,
direction_consistency,
magnitude,
})
}
/// Compute dominant direction using simplified PCA
fn compute_dominant_direction(values: &[f32]) -> Result<Vec<f32>> {
if values.is_empty() {
return Ok(Vec::new());
}
// For simplicity, use the normalized gradient as the dominant direction
let magnitude = values.iter().map(|&x| x * x).sum::<f32>().sqrt();
if magnitude < 1e-10 {
return Ok(vec![0.0; values.len().min(3)]); // Return zero vector of max size 3
}
let normalized: Vec<f32> = values.iter()
.take(3) // Limit to 3 dimensions for visualization
.map(|&x| x / magnitude)
.collect();
Ok(normalized)
}
/// Compute direction consistency
fn compute_direction_consistency(values: &[f32], dominant_direction: &[f32]) -> f32 {
if values.is_empty() || dominant_direction.is_empty() {
return 0.0;
}
let magnitude = values.iter().map(|&x| x * x).sum::<f32>().sqrt();
if magnitude < 1e-10 {
return 0.0;
}
// Compute dot product with dominant direction
let dot_product: f32 = values.iter()
.zip(dominant_direction.iter())
.map(|(&v, &d)| v * d / magnitude)
.sum();
dot_product.abs()
}
/// Compute flow divergence (gradient of gradient magnitude)
fn compute_flow_divergence(gradient: &ArrayD<f32>) -> Result<f32> {
let values: Vec<f32> = gradient.iter().cloned().collect();
if values.len() < 2 {
return Ok(0.0);
}
// Simplified divergence computation using finite differences
let mut divergence = 0.0;
let step_size = 1.0 / values.len() as f32;
for i in 0..(values.len() - 1) {
let gradient_diff = values[i + 1] - values[i];
divergence += gradient_diff / step_size;
}
Ok(divergence / (values.len() - 1) as f32)
}
/// Compute flow curl (rotational component)
fn compute_flow_curl(gradient: &ArrayD<f32>) -> Result<f32> {
let values: Vec<f32> = gradient.iter().cloned().collect();
if values.len() < 3 {
return Ok(0.0);
}
// Simplified 2D curl computation
let mut curl = 0.0;
let step_size = 1.0 / values.len() as f32;
for i in 0..(values.len() - 2) {
// Approximate curl using finite differences
let dy_dx = (values[i + 1] - values[i]) / step_size;
let dx_dy = (values[i + 2] - values[i + 1]) / step_size;
curl += dx_dy - dy_dx;
}
Ok(curl / (values.len() - 2) as f32)
}
/// Compute flow coherence (uniformity of flow)
fn compute_flow_coherence(gradient: &ArrayD<f32>) -> Result<f32> {
let values: Vec<f32> = gradient.iter().cloned().collect();
if values.len() < 2 {
return Ok(1.0);
}
let mean = values.iter().sum::<f32>() / values.len() as f32;
let variance = values.iter()
.map(|&x| (x - mean).powi(2))
.sum::<f32>() / values.len() as f32;
let std_dev = variance.sqrt();
let mean_abs = mean.abs();
// Coherence as signal-to-noise ratio
if std_dev > 1e-10 {
Ok((mean_abs / std_dev).min(1.0))
} else {
Ok(1.0)
}
}
/// Compute flow stability
fn compute_flow_stability(gradient: &ArrayD<f32>) -> Result<FlowStability> {
let values: Vec<f32> = gradient.iter().cloned().collect();
if values.len() < 3 {
return Ok(FlowStability::default());
}
// Compute local stability indicators
let mut local_variations = Vec::new();
for i in 1..(values.len() - 1) {
let variation = (values[i + 1] - 2.0 * values[i] + values[i - 1]).abs();
local_variations.push(variation);
}
let stability_score = if !local_variations.is_empty() {
let max_variation = local_variations.iter().copied().fold(0.0, f32::max);
let mean_variation = local_variations.iter().sum::<f32>() / local_variations.len() as f32;
// Stability inversely related to variation
1.0 / (1.0 + mean_variation)
} else {
1.0
};
let stability_classification = match stability_score {
s if s > 0.8 => StabilityClass::Stable,
s if s > 0.6 => StabilityClass::ModeratelyStable,
s if s > 0.4 => StabilityClass::Unstable,
_ => StabilityClass::HighlyUnstable,
};
Ok(FlowStability {
stability_score,
local_variations,
stability_classification,
})
}
/// Classify flow type based on flow characteristics
fn classify_flow_type(divergence: f32, curl: f32, coherence: f32) -> FlowType {
match (divergence.abs(), curl.abs(), coherence) {
(d, c, coh) if d > 0.5 && c < 0.1 && coh > 0.7 => FlowType::Divergent,
(d, c, coh) if d < 0.1 && c > 0.5 && coh > 0.7 => FlowType::Rotational,
(d, c, coh) if d < 0.1 && c < 0.1 && coh > 0.8 => FlowType::Laminar,
(d, c, coh) if coh < 0.3 => FlowType::Turbulent,
_ => FlowType::Mixed,
}
}
/// Compute directional derivatives
fn compute_directional_derivatives(gradients: &[ArrayD<f32>]) -> Result<DirectionalDerivatives> {
if gradients.is_empty() {
return Ok(DirectionalDerivatives::default());
}
let mut layer_derivatives = Vec::new();
for (i, gradient) in gradients.iter().enumerate() {
let values: Vec<f32> = gradient.iter().cloned().collect();
if values.len() < 2 {
continue;
}
// Compute directional derivative in primary gradient direction
let primary_derivative = Self::compute_primary_directional_derivative(&values)?;
let secondary_derivative = Self::compute_secondary_directional_derivative(&values)?;
let cross_derivative = Self::compute_cross_directional_derivative(&values)?;
layer_derivatives.push(LayerDirectionalDerivatives {
layer_index: i,
primary_derivative,
secondary_derivative,
cross_derivative,
derivative_magnitude: (primary_derivative.powi(2) + secondary_derivative.powi(2)).sqrt(),
});
}
let flow_acceleration = Self::compute_flow_acceleration(&layer_derivatives);
let flow_jerk = Self::compute_flow_jerk(&layer_derivatives);
// Compute patterns before moving values
let derivative_patterns = Self::analyze_derivative_patterns(&layer_derivatives);
Ok(DirectionalDerivatives {
layer_derivatives,
flow_acceleration,
flow_jerk,
derivative_patterns,
})
}
/// Compute primary directional derivative
fn compute_primary_directional_derivative(values: &[f32]) -> Result<f32> {
if values.len() < 2 {
return Ok(0.0);
}
// First-order finite difference
let mut derivatives = Vec::new();
for i in 0..(values.len() - 1) {
derivatives.push(values[i + 1] - values[i]);
}
// Return average derivative
Ok(derivatives.iter().sum::<f32>() / derivatives.len() as f32)
}
/// Compute secondary directional derivative
fn compute_secondary_directional_derivative(values: &[f32]) -> Result<f32> {
if values.len() < 3 {
return Ok(0.0);
}
// Second-order finite difference
let mut second_derivatives = Vec::new();
for i in 0..(values.len() - 2) {
let second_deriv = values[i + 2] - 2.0 * values[i + 1] + values[i];
second_derivatives.push(second_deriv);
}
Ok(second_derivatives.iter().sum::<f32>() / second_derivatives.len() as f32)
}
/// Compute cross directional derivative
fn compute_cross_directional_derivative(values: &[f32]) -> Result<f32> {
if values.len() < 4 {
return Ok(0.0);
}
// Mixed partial derivative approximation
let mut cross_derivatives = Vec::new();
for i in 0..(values.len() - 3) {
let cross_deriv = values[i + 3] - values[i + 2] - values[i + 1] + values[i];
cross_derivatives.push(cross_deriv);
}
Ok(cross_derivatives.iter().sum::<f32>() / cross_derivatives.len() as f32)
}
/// Compute flow acceleration (rate of change of gradient flow)
fn compute_flow_acceleration(layer_derivatives: &[LayerDirectionalDerivatives]) -> f32 {
if layer_derivatives.len() < 2 {
return 0.0;
}
let mut accelerations = Vec::new();
for i in 0..(layer_derivatives.len() - 1) {
let accel = layer_derivatives[i + 1].derivative_magnitude - layer_derivatives[i].derivative_magnitude;
accelerations.push(accel);
}
accelerations.iter().sum::<f32>() / accelerations.len() as f32
}
/// Compute flow jerk (rate of change of acceleration)
fn compute_flow_jerk(layer_derivatives: &[LayerDirectionalDerivatives]) -> f32 {
if layer_derivatives.len() < 3 {
return 0.0;
}
let mut jerks = Vec::new();
for i in 0..(layer_derivatives.len() - 2) {
let jerk = layer_derivatives[i + 2].derivative_magnitude
- 2.0 * layer_derivatives[i + 1].derivative_magnitude
+ layer_derivatives[i].derivative_magnitude;
jerks.push(jerk);
}
jerks.iter().sum::<f32>() / jerks.len() as f32
}
/// Analyze derivative patterns
fn analyze_derivative_patterns(layer_derivatives: &[LayerDirectionalDerivatives]) -> DerivativePatterns {
if layer_derivatives.is_empty() {
return DerivativePatterns::default();
}
let primary_trend = Self::compute_derivative_trend(&layer_derivatives.iter().map(|d| d.primary_derivative).collect::<Vec<_>>());
let secondary_trend = Self::compute_derivative_trend(&layer_derivatives.iter().map(|d| d.secondary_derivative).collect::<Vec<_>>());
let oscillation_frequency = Self::compute_oscillation_frequency(layer_derivatives);
let dominant_frequency = Self::compute_dominant_frequency(layer_derivatives);
DerivativePatterns {
primary_trend,
secondary_trend,
oscillation_frequency,
dominant_frequency,
pattern_stability: Self::compute_pattern_stability(layer_derivatives),
}
}
/// Compute derivative trend
fn compute_derivative_trend(derivatives: &[f32]) -> DerivativeTrend {
if derivatives.len() < 2 {
return DerivativeTrend::Stable;
}
let start = derivatives[0];
let end = derivatives[derivatives.len() - 1];
let change = (end - start) / start.abs().max(1e-10);
match change {
c if c > 0.1 => DerivativeTrend::Increasing,
c if c < -0.1 => DerivativeTrend::Decreasing,
_ => DerivativeTrend::Stable,
}
}
/// Compute oscillation frequency
fn compute_oscillation_frequency(layer_derivatives: &[LayerDirectionalDerivatives]) -> f32 {
if layer_derivatives.len() < 3 {
return 0.0;
}
let mut zero_crossings = 0;
let derivatives: Vec<f32> = layer_derivatives.iter().map(|d| d.primary_derivative).collect();
for i in 0..(derivatives.len() - 1) {
if derivatives[i] * derivatives[i + 1] < 0.0 {
zero_crossings += 1;
}
}
// Frequency as zero crossings per layer
zero_crossings as f32 / (derivatives.len() - 1) as f32
}
/// Compute dominant frequency
fn compute_dominant_frequency(layer_derivatives: &[LayerDirectionalDerivatives]) -> f32 {
// Simplified frequency analysis using peak counting
if layer_derivatives.len() < 3 {
return 0.0;
}
let magnitudes: Vec<f32> = layer_derivatives.iter().map(|d| d.derivative_magnitude).collect();
let mut peaks = 0;
for i in 1..(magnitudes.len() - 1) {
if magnitudes[i] > magnitudes[i - 1] && magnitudes[i] > magnitudes[i + 1] {
peaks += 1;
}
}
peaks as f32 / (magnitudes.len() - 2) as f32
}
/// Compute pattern stability
fn compute_pattern_stability(layer_derivatives: &[LayerDirectionalDerivatives]) -> f32 {
if layer_derivatives.is_empty() {
return 0.0;
}
let magnitudes: Vec<f32> = layer_derivatives.iter().map(|d| d.derivative_magnitude).collect();
let mean = magnitudes.iter().sum::<f32>() / magnitudes.len() as f32;
let variance = magnitudes.iter()
.map(|&x| (x - mean).powi(2))
.sum::<f32>() / magnitudes.len() as f32;
// Stability inversely related to variance
1.0 / (1.0 + variance)
}
/// Analyze flow patterns
fn analyze_flow_patterns(gradients: &[ArrayD<f32>]) -> Result<FlowPatterns> {
let convergence_zones = Self::identify_convergence_zones(gradients)?;
let divergence_zones = Self::identify_divergence_zones(gradients)?;
let vortex_structures = Self::identify_vortex_structures(gradients)?;
let flow_boundaries = Self::identify_flow_boundaries(gradients)?;
Ok(FlowPatterns {
convergence_zones,
divergence_zones,
vortex_structures,
flow_boundaries,
pattern_complexity: Self::compute_pattern_complexity(gradients)?,
})
}
/// Identify convergence zones
fn identify_convergence_zones(gradients: &[ArrayD<f32>]) -> Result<Vec<ConvergenceZone>> {
let mut zones = Vec::new();
for (i, gradient) in gradients.iter().enumerate() {
let divergence = Self::compute_flow_divergence(gradient)?;
if divergence < -0.1 { // Negative divergence indicates convergence
zones.push(ConvergenceZone {
layer_index: i,
convergence_strength: -divergence,
zone_stability: Self::compute_zone_stability(gradient)?,
});
}
}
Ok(zones)
}
/// Identify divergence zones
fn identify_divergence_zones(gradients: &[ArrayD<f32>]) -> Result<Vec<DivergenceZone>> {
let mut zones = Vec::new();
for (i, gradient) in gradients.iter().enumerate() {
let divergence = Self::compute_flow_divergence(gradient)?;
if divergence > 0.1 { // Positive divergence
zones.push(DivergenceZone {
layer_index: i,
divergence_strength: divergence,
expansion_rate: Self::compute_expansion_rate(gradient)?,
});
}
}
Ok(zones)
}
/// Identify vortex structures
fn identify_vortex_structures(gradients: &[ArrayD<f32>]) -> Result<Vec<VortexStructure>> {
let mut vortices = Vec::new();
for (i, gradient) in gradients.iter().enumerate() {
let curl = Self::compute_flow_curl(gradient)?;
if curl.abs() > 0.1 { // Significant rotational component
vortices.push(VortexStructure {
layer_index: i,
vorticity_strength: curl.abs(),
rotation_direction: if curl > 0.0 {
RotationDirection::Counterclockwise
} else {
RotationDirection::Clockwise
},
core_stability: Self::compute_vortex_stability(gradient)?,
});
}
}
Ok(vortices)
}
/// Identify flow boundaries
fn identify_flow_boundaries(gradients: &[ArrayD<f32>]) -> Result<Vec<FlowBoundary>> {
let mut boundaries = Vec::new();
for i in 0..(gradients.len() - 1) {
let boundary_strength = Self::compute_boundary_strength(&gradients[i], &gradients[i + 1])?;
if boundary_strength > 0.5 {
boundaries.push(FlowBoundary {
between_layers: (i, i + 1),
boundary_strength,
boundary_type: Self::classify_boundary_type(&gradients[i], &gradients[i + 1])?,
});
}
}
Ok(boundaries)
}
/// Compute zone stability
fn compute_zone_stability(gradient: &ArrayD<f32>) -> Result<f32> {
let coherence = Self::compute_flow_coherence(gradient)?;
let stability = Self::compute_flow_stability(gradient)?;
Ok((coherence + stability.stability_score) / 2.0)
}
/// Compute expansion rate
fn compute_expansion_rate(gradient: &ArrayD<f32>) -> Result<f32> {
let values: Vec<f32> = gradient.iter().cloned().collect();
if values.len() < 2 {
return Ok(0.0);
}
// Rate of change of magnitude
let mut rates = Vec::new();
for i in 0..(values.len() - 1) {
rates.push((values[i + 1].abs() - values[i].abs()).abs());
}
Ok(rates.iter().sum::<f32>() / rates.len() as f32)
}
/// Compute vortex stability
fn compute_vortex_stability(gradient: &ArrayD<f32>) -> Result<f32> {
let curl = Self::compute_flow_curl(gradient)?;
let coherence = Self::compute_flow_coherence(gradient)?;
// Vortex stability based on curl consistency and coherence
Ok(coherence * (1.0 - (curl.abs() - 0.5).abs()).max(0.0))
}
/// Compute boundary strength between two layers
fn compute_boundary_strength(grad1: &ArrayD<f32>, grad2: &ArrayD<f32>) -> Result<f32> {
let values1: Vec<f32> = grad1.iter().cloned().collect();
let values2: Vec<f32> = grad2.iter().cloned().collect();
let min_len = values1.len().min(values2.len());
if min_len == 0 {
return Ok(0.0);
}
let mut differences = Vec::new();
for i in 0..min_len {
differences.push((values1[i] - values2[i]).abs());
}
let max_diff = differences.iter().copied().fold(0.0, f32::max);
let mean_diff = differences.iter().sum::<f32>() / differences.len() as f32;
// Normalized boundary strength
Ok(mean_diff / (max_diff + 1e-10))
}
/// Classify boundary type
fn classify_boundary_type(grad1: &ArrayD<f32>, grad2: &ArrayD<f32>) -> Result<BoundaryType> {
let mag1 = Self::compute_gradient_magnitude(grad1);
let mag2 = Self::compute_gradient_magnitude(grad2);
let ratio = if mag1 > 1e-10 { mag2 / mag1 } else { 1.0 };
Ok(match ratio {
r if r > 2.0 => BoundaryType::Amplification,
r if r < 0.5 => BoundaryType::Attenuation,
_ => BoundaryType::Transition,
})
}
/// Compute pattern complexity
fn compute_pattern_complexity(gradients: &[ArrayD<f32>]) -> Result<f32> {
if gradients.is_empty() {
return Ok(0.0);
}
let mut complexity_measures = Vec::new();
for gradient in gradients {
let curl = Self::compute_flow_curl(gradient)?;
let divergence = Self::compute_flow_divergence(gradient)?;
let coherence = Self::compute_flow_coherence(gradient)?;
// Complexity based on flow characteristics
let local_complexity = curl.abs() + divergence.abs() + (1.0 - coherence);
complexity_measures.push(local_complexity);
}
Ok(complexity_measures.iter().sum::<f32>() / complexity_measures.len() as f32)
}
/// Analyze gradient topology
fn analyze_gradient_topology(gradients: &[ArrayD<f32>]) -> Result<GradientTopology> {
let critical_points = Self::identify_critical_points(gradients)?;
let saddle_points = Self::identify_saddle_points(gradients)?;
let flow_separatrices = Self::identify_flow_separatrices(gradients)?;
let topology_classification = Self::classify_topology(gradients)?;
Ok(GradientTopology {
critical_points,
saddle_points,
flow_separatrices,
topology_classification,
topological_invariants: Self::compute_topological_invariants(gradients)?,
})
}
/// Identify critical points (where gradient is zero or near zero)
fn identify_critical_points(gradients: &[ArrayD<f32>]) -> Result<Vec<CriticalPoint>> {
let mut points = Vec::new();
for (i, gradient) in gradients.iter().enumerate() {
let magnitude = Self::compute_gradient_magnitude(gradient);
if magnitude < 0.01 { // Near-zero gradient
let point_type = Self::classify_critical_point(gradient)?;
points.push(CriticalPoint {
layer_index: i,
point_type,
stability_index: Self::compute_critical_point_stability(gradient)?,
});
}
}
Ok(points)
}
/// Classify critical point type
fn classify_critical_point(gradient: &ArrayD<f32>) -> Result<CriticalPointType> {
let divergence = Self::compute_flow_divergence(gradient)?;
let curl = Self::compute_flow_curl(gradient)?;
Ok(match (divergence, curl) {
(d, c) if d > 0.1 && c.abs() < 0.05 => CriticalPointType::Source,
(d, c) if d < -0.1 && c.abs() < 0.05 => CriticalPointType::Sink,
(d, c) if d.abs() < 0.05 && c.abs() > 0.1 => CriticalPointType::Vortex,
_ => CriticalPointType::Saddle,
})
}
/// Compute critical point stability
fn compute_critical_point_stability(gradient: &ArrayD<f32>) -> Result<f32> {
let stability = Self::compute_flow_stability(gradient)?;
Ok(stability.stability_score)
}
/// Identify saddle points
fn identify_saddle_points(gradients: &[ArrayD<f32>]) -> Result<Vec<SaddlePoint>> {
let mut saddles = Vec::new();
for (i, gradient) in gradients.iter().enumerate() {
let point_type = Self::classify_critical_point(gradient)?;
if matches!(point_type, CriticalPointType::Saddle) {
saddles.push(SaddlePoint {
layer_index: i,
saddle_strength: Self::compute_saddle_strength(gradient)?,
manifold_dimensions: Self::estimate_manifold_dimensions(gradient)?,
});
}
}
Ok(saddles)
}
/// Compute saddle strength
fn compute_saddle_strength(gradient: &ArrayD<f32>) -> Result<f32> {
let divergence = Self::compute_flow_divergence(gradient)?;
let curl = Self::compute_flow_curl(gradient)?;
// Saddle strength based on mixed characteristics
Ok((divergence.abs() + curl.abs()) / 2.0)
}
/// Estimate manifold dimensions
fn estimate_manifold_dimensions(gradient: &ArrayD<f32>) -> Result<usize> {
let values: Vec<f32> = gradient.iter().cloned().collect();
// Simplified dimension estimation based on effective rank
let effective_rank = values.len().min(10); // Cap at 10 for simplicity
Ok(effective_rank)
}
/// Identify flow separatrices
fn identify_flow_separatrices(gradients: &[ArrayD<f32>]) -> Result<Vec<FlowSeparatrix>> {
let mut separatrices = Vec::new();
for i in 0..(gradients.len() - 1) {
let boundary_strength = Self::compute_boundary_strength(&gradients[i], &gradients[i + 1])?;
if boundary_strength > 0.7 { // Strong flow separation
separatrices.push(FlowSeparatrix {
between_layers: (i, i + 1),
separation_strength: boundary_strength,
separatrix_type: Self::classify_separatrix_type(&gradients[i], &gradients[i + 1])?,
});
}
}
Ok(separatrices)
}
/// Classify separatrix type
fn classify_separatrix_type(grad1: &ArrayD<f32>, grad2: &ArrayD<f32>) -> Result<SeparatrixType> {
let divergence1 = Self::compute_flow_divergence(grad1)?;
let divergence2 = Self::compute_flow_divergence(grad2)?;
Ok(match (divergence1, divergence2) {
(d1, d2) if d1 * d2 < 0.0 => SeparatrixType::Heteroclinic,
(d1, d2) if (d1 - d2).abs() > 0.5 => SeparatrixType::Shock,
_ => SeparatrixType::Simple,
})
}
/// Classify overall topology
fn classify_topology(gradients: &[ArrayD<f32>]) -> Result<TopologyClass> {
if gradients.is_empty() {
return Ok(TopologyClass::Simple);
}
let critical_points = Self::identify_critical_points(gradients)?;
let saddle_points = Self::identify_saddle_points(gradients)?;
let complexity = critical_points.len() + saddle_points.len() * 2;
Ok(match complexity {
0..=2 => TopologyClass::Simple,
3..=5 => TopologyClass::Moderate,
6..=10 => TopologyClass::Complex,
_ => TopologyClass::Chaotic,
})
}
/// Compute topological invariants
fn compute_topological_invariants(gradients: &[ArrayD<f32>]) -> Result<TopologicalInvariants> {
let euler_characteristic = Self::compute_euler_characteristic(gradients)?;
let genus = Self::estimate_genus(gradients)?;
let betti_numbers = Self::compute_betti_numbers(gradients)?;
Ok(TopologicalInvariants {
euler_characteristic,
genus,
betti_numbers,
})
}
/// Compute Euler characteristic (simplified)
fn compute_euler_characteristic(gradients: &[ArrayD<f32>]) -> Result<i32> {
let critical_points = Self::identify_critical_points(gradients)?;
// Simplified Euler characteristic based on critical points
Ok(critical_points.len() as i32)
}
/// Estimate genus (simplified)
fn estimate_genus(gradients: &[ArrayD<f32>]) -> Result<usize> {
let saddle_points = Self::identify_saddle_points(gradients)?;
// Simplified genus estimation
Ok(saddle_points.len().saturating_sub(1))
}
/// Compute Betti numbers (simplified)
fn compute_betti_numbers(gradients: &[ArrayD<f32>]) -> Result<Vec<usize>> {
// Simplified Betti number computation
let connected_components = 1; // Assume single connected component
let loops = Self::identify_saddle_points(gradients)?.len();
let voids = 0; // Assume no voids in 2D/3D analysis
Ok(vec![connected_components, loops, voids])
}
/// Analyze flow dynamics
fn analyze_flow_dynamics(gradients: &[ArrayD<f32>]) -> Result<FlowDynamics> {
let temporal_evolution = Self::analyze_temporal_evolution(gradients)?;
let phase_space_analysis = Self::analyze_phase_space(gradients)?;
let lyapunov_exponents = Self::estimate_lyapunov_exponents(gradients)?;
let attractor_analysis = Self::analyze_attractors(gradients)?;
Ok(FlowDynamics {
temporal_evolution,
phase_space_analysis,
lyapunov_exponents,
attractor_analysis,
dynamics_classification: Self::classify_dynamics(gradients)?,
})
}
/// Analyze temporal evolution
fn analyze_temporal_evolution(gradients: &[ArrayD<f32>]) -> Result<TemporalEvolution> {
let magnitudes: Vec<f32> = gradients.iter().map(|g| Self::compute_gradient_magnitude(g)).collect();
let evolution_trend = Self::compute_derivative_trend(&magnitudes);
let evolution_rate = Self::compute_evolution_rate(&magnitudes);
let stability_over_time = Self::compute_temporal_stability(&magnitudes);
Ok(TemporalEvolution {
evolution_trend,
evolution_rate,
stability_over_time,
periodic_components: Self::detect_periodic_components(&magnitudes),
})
}
/// Compute evolution rate
fn compute_evolution_rate(magnitudes: &[f32]) -> f32 {
if magnitudes.len() < 2 {
return 0.0;
}
let mut rates = Vec::new();
for i in 0..(magnitudes.len() - 1) {
rates.push((magnitudes[i + 1] - magnitudes[i]).abs());
}
rates.iter().sum::<f32>() / rates.len() as f32
}
/// Compute temporal stability
fn compute_temporal_stability(magnitudes: &[f32]) -> f32 {
if magnitudes.is_empty() {
return 0.0;
}
let mean = magnitudes.iter().sum::<f32>() / magnitudes.len() as f32;
let variance = magnitudes.iter()
.map(|&x| (x - mean).powi(2))
.sum::<f32>() / magnitudes.len() as f32;
// Stability inversely related to variance
1.0 / (1.0 + variance)
}
/// Detect periodic components
fn detect_periodic_components(magnitudes: &[f32]) -> Vec<f32> {
// Simplified period detection using autocorrelation
let mut periods = Vec::new();
for lag in 2..magnitudes.len().min(10) {
let correlation = Self::compute_autocorrelation(magnitudes, lag);
if correlation > 0.7 {
periods.push(lag as f32);
}
}
periods
}
/// Compute autocorrelation
fn compute_autocorrelation(data: &[f32], lag: usize) -> f32 {
if data.len() <= lag {
return 0.0;
}
let n = data.len() - lag;
let mean = data.iter().sum::<f32>() / data.len() as f32;
let mut numerator = 0.0;
let mut denominator = 0.0;
for i in 0..n {
let x = data[i] - mean;
let y = data[i + lag] - mean;
numerator += x * y;
denominator += x * x;
}
if denominator > 1e-10 {
numerator / denominator
} else {
0.0
}
}
/// Analyze phase space
fn analyze_phase_space(gradients: &[ArrayD<f32>]) -> Result<PhaseSpaceAnalysis> {
let magnitudes: Vec<f32> = gradients.iter().map(|g| Self::compute_gradient_magnitude(g)).collect();
let phase_portrait = Self::construct_phase_portrait(&magnitudes);
let attractor_dimension = Self::estimate_attractor_dimension(&magnitudes);
let phase_space_volume = Self::compute_phase_space_volume(&magnitudes);
Ok(PhaseSpaceAnalysis {
phase_portrait,
attractor_dimension,
phase_space_volume,
embedding_dimension: Self::estimate_embedding_dimension(&magnitudes),
})
}
/// Construct phase portrait
fn construct_phase_portrait(magnitudes: &[f32]) -> Vec<(f32, f32)> {
let mut portrait = Vec::new();
for i in 0..(magnitudes.len() - 1) {
portrait.push((magnitudes[i], magnitudes[i + 1]));
}
portrait
}
/// Estimate attractor dimension
fn estimate_attractor_dimension(magnitudes: &[f32]) -> f32 {
// Simplified dimension estimation using correlation dimension
if magnitudes.len() < 3 {
return 1.0;
}
// Box-counting dimension approximation
let unique_values: std::collections::HashSet<_> = magnitudes.iter()
.map(|&x| (x * 100.0) as i32) // Discretize for counting
.collect();
(unique_values.len() as f32).log2()
}
/// Compute phase space volume
fn compute_phase_space_volume(magnitudes: &[f32]) -> f32 {
if magnitudes.is_empty() {
return 0.0;
}
let min_val = magnitudes.iter().copied().fold(f32::INFINITY, f32::min);
let max_val = magnitudes.iter().copied().fold(f32::NEG_INFINITY, f32::max);
(max_val - min_val).max(0.0)
}
/// Estimate embedding dimension
fn estimate_embedding_dimension(magnitudes: &[f32]) -> usize {
// Using Takens' theorem approximation
magnitudes.len().min(10)
}
/// Estimate Lyapunov exponents
fn estimate_lyapunov_exponents(gradients: &[ArrayD<f32>]) -> Result<Vec<f32>> {
let magnitudes: Vec<f32> = gradients.iter().map(|g| Self::compute_gradient_magnitude(g)).collect();
if magnitudes.len() < 3 {
return Ok(vec![0.0]);
}
// Simplified Lyapunov exponent estimation
let mut exponents = Vec::new();
for i in 1..(magnitudes.len() - 1) {
let divergence_rate = (magnitudes[i + 1] / magnitudes[i]).ln();
exponents.push(divergence_rate);
}
// Return the dominant (largest) exponent
Ok(vec![exponents.iter().copied().fold(f32::NEG_INFINITY, f32::max)])
}
/// Analyze attractors
fn analyze_attractors(gradients: &[ArrayD<f32>]) -> Result<AttractorAnalysis> {
let magnitudes: Vec<f32> = gradients.iter().map(|g| Self::compute_gradient_magnitude(g)).collect();
let attractor_type = Self::classify_attractor_type(&magnitudes);
let basin_of_attraction = Self::estimate_basin_size(&magnitudes);
let attractor_stability = Self::compute_attractor_stability(&magnitudes);
Ok(AttractorAnalysis {
attractor_type,
basin_of_attraction,
attractor_stability,
strange_attractor_indicators: Self::detect_strange_attractor(&magnitudes),
})
}
/// Classify attractor type
fn classify_attractor_type(magnitudes: &[f32]) -> AttractorType {
let stability = Self::compute_temporal_stability(magnitudes);
let evolution_rate = Self::compute_evolution_rate(magnitudes);
match (stability, evolution_rate) {
(s, r) if s > 0.9 && r < 0.01 => AttractorType::FixedPoint,
(s, r) if s > 0.7 && r < 0.1 => AttractorType::LimitCycle,
(s, r) if s > 0.5 => AttractorType::Torus,
_ => AttractorType::Strange,
}
}
/// Estimate basin of attraction size
fn estimate_basin_size(magnitudes: &[f32]) -> f32 {
if magnitudes.is_empty() {
return 0.0;
}
let min_val = magnitudes.iter().copied().fold(f32::INFINITY, f32::min);
let max_val = magnitudes.iter().copied().fold(f32::NEG_INFINITY, f32::max);
(max_val - min_val).max(0.0)
}
/// Compute attractor stability
fn compute_attractor_stability(magnitudes: &[f32]) -> f32 {
Self::compute_temporal_stability(magnitudes)
}
/// Detect strange attractor indicators
fn detect_strange_attractor(magnitudes: &[f32]) -> Vec<String> {
let mut indicators = Vec::new();
let stability = Self::compute_temporal_stability(magnitudes);
if stability < 0.3 {
indicators.push("Low temporal stability".to_string());
}
let evolution_rate = Self::compute_evolution_rate(magnitudes);
if evolution_rate > 0.5 {
indicators.push("High evolution rate".to_string());
}
// Check for sensitive dependence on initial conditions
let autocorr = Self::compute_autocorrelation(magnitudes, 1);
if autocorr < 0.1 {
indicators.push("Sensitive dependence on initial conditions".to_string());
}
indicators
}
/// Classify dynamics
fn classify_dynamics(gradients: &[ArrayD<f32>]) -> Result<DynamicsClass> {
let lyapunov = Self::estimate_lyapunov_exponents(gradients)?;
let attractor_analysis = Self::analyze_attractors(gradients)?;
Ok(match (lyapunov.first().copied().unwrap_or(0.0), attractor_analysis.attractor_type) {
(l, AttractorType::FixedPoint) if l < 0.0 => DynamicsClass::Stable,
(l, AttractorType::LimitCycle) if l.abs() < 0.1 => DynamicsClass::Periodic,
(l, AttractorType::Torus) => DynamicsClass::Quasiperiodic,
(l, _) if l > 0.1 => DynamicsClass::Chaotic,
_ => DynamicsClass::Complex,
})
}
/// Analyze flow convergence
fn analyze_flow_convergence(layer_analyses: &[LayerGradientFlowAnalysis]) -> FlowConvergenceAnalysis {
if layer_analyses.is_empty() {
return FlowConvergenceAnalysis::default();
}
let convergence_rate = Self::compute_convergence_rate(layer_analyses);
let convergence_quality = Self::compute_convergence_quality(layer_analyses);
let oscillation_damping = Self::compute_oscillation_damping(layer_analyses);
FlowConvergenceAnalysis {
convergence_rate,
convergence_quality,
oscillation_damping,
convergence_indicators: Self::compute_convergence_indicators(layer_analyses),
}
}
/// Compute convergence rate
fn compute_convergence_rate(layer_analyses: &[LayerGradientFlowAnalysis]) -> f32 {
if layer_analyses.len() < 2 {
return 0.0;
}
let magnitudes: Vec<f32> = layer_analyses.iter().map(|l| l.gradient_magnitude).collect();
// Exponential decay rate
let mut decay_rates = Vec::new();
for i in 0..(magnitudes.len() - 1) {
if magnitudes[i] > 1e-10 {
let rate = (magnitudes[i + 1] / magnitudes[i]).ln();
decay_rates.push(rate);
}
}
if decay_rates.is_empty() {
0.0
} else {
-decay_rates.iter().sum::<f32>() / decay_rates.len() as f32
}
}
/// Compute convergence quality
fn compute_convergence_quality(layer_analyses: &[LayerGradientFlowAnalysis]) -> f32 {
if layer_analyses.is_empty() {
return 0.0;
}
let coherences: Vec<f32> = layer_analyses.iter().map(|l| l.flow_coherence).collect();
let stabilities: Vec<f32> = layer_analyses.iter().map(|l| l.flow_stability.stability_score).collect();
let avg_coherence = coherences.iter().sum::<f32>() / coherences.len() as f32;
let avg_stability = stabilities.iter().sum::<f32>() / stabilities.len() as f32;
(avg_coherence + avg_stability) / 2.0
}
/// Compute oscillation damping
fn compute_oscillation_damping(layer_analyses: &[LayerGradientFlowAnalysis]) -> f32 {
if layer_analyses.len() < 3 {
return 1.0;
}
let magnitudes: Vec<f32> = layer_analyses.iter().map(|l| l.gradient_magnitude).collect();
// Measure reduction in oscillations
let mut oscillation_amplitude = Vec::new();
for i in 1..(magnitudes.len() - 1) {
let amplitude = (magnitudes[i] - (magnitudes[i - 1] + magnitudes[i + 1]) / 2.0).abs();
oscillation_amplitude.push(amplitude);
}
if oscillation_amplitude.len() < 2 {
return 1.0;
}
let first_half = &oscillation_amplitude[0..oscillation_amplitude.len() / 2];
let second_half = &oscillation_amplitude[oscillation_amplitude.len() / 2..];
let first_avg = first_half.iter().sum::<f32>() / first_half.len() as f32;
let second_avg = second_half.iter().sum::<f32>() / second_half.len() as f32;
if first_avg > 1e-10 {
1.0 - (second_avg / first_avg).min(1.0)
} else {
1.0
}
}
/// Compute convergence indicators
fn compute_convergence_indicators(layer_analyses: &[LayerGradientFlowAnalysis]) -> ConvergenceIndicators {
let magnitudes: Vec<f32> = layer_analyses.iter().map(|l| l.gradient_magnitude).collect();
let is_monotonic = Self::check_monotonic_decrease(&magnitudes);
let has_reached_plateau = Self::check_plateau_reached(&magnitudes);
let oscillation_frequency = Self::compute_oscillation_frequency_from_magnitudes(&magnitudes);
ConvergenceIndicators {
is_monotonic,
has_reached_plateau,
oscillation_frequency,
convergence_confidence: Self::compute_convergence_confidence(&magnitudes),
}
}
/// Check if magnitudes decrease monotonically
fn check_monotonic_decrease(magnitudes: &[f32]) -> bool {
magnitudes.windows(2).all(|w| w[1] <= w[0])
}
/// Check if plateau is reached
fn check_plateau_reached(magnitudes: &[f32]) -> bool {
if magnitudes.len() < 5 {
return false;
}
let last_5 = &magnitudes[magnitudes.len() - 5..];
let variance = Self::compute_temporal_stability(last_5);
variance > 0.95 // High stability indicates plateau
}
/// Compute oscillation frequency from magnitudes
fn compute_oscillation_frequency_from_magnitudes(magnitudes: &[f32]) -> f32 {
Self::compute_oscillation_frequency(&magnitudes.iter().enumerate().map(|(i, &mag)| LayerDirectionalDerivatives {
layer_index: i,
primary_derivative: mag,
secondary_derivative: 0.0,
cross_derivative: 0.0,
derivative_magnitude: mag,
}).collect::<Vec<_>>())
}
/// Compute convergence confidence
fn compute_convergence_confidence(magnitudes: &[f32]) -> f32 {
let stability = Self::compute_temporal_stability(magnitudes);
let trend_consistency = if magnitudes.len() >= 2 {
let last_val = magnitudes[magnitudes.len() - 1];
let first_val = magnitudes[0];
if first_val > 1e-10 {
1.0 - (last_val / first_val).min(1.0)
} else {
0.0
}
} else {
0.0
};
(stability + trend_consistency) / 2.0
}
/// Generate flow recommendations
fn generate_flow_recommendations(
layer_analyses: &[LayerGradientFlowAnalysis],
flow_patterns: &FlowPatterns,
) -> Vec<FlowRecommendation> {
let mut recommendations = Vec::new();
// Check for flow issues
for layer in layer_analyses {
if layer.flow_coherence < 0.3 {
recommendations.push(FlowRecommendation {
layer_index: Some(layer.layer_index),
recommendation_type: FlowRecommendationType::ImproveCoherence,
description: format!("Layer {} has low flow coherence ({}). Consider gradient normalization or learning rate adjustment.",
layer.layer_index, layer.flow_coherence),
urgency: RecommendationUrgency::High,
expected_benefit: "Improved training stability and convergence".to_string(),
});
}
if layer.flow_stability.stability_score < 0.4 {
recommendations.push(FlowRecommendation {
layer_index: Some(layer.layer_index),
recommendation_type: FlowRecommendationType::StabilizeFlow,
description: format!("Layer {} shows flow instability ({}). Consider regularization or architecture modifications.",
layer.layer_index, layer.flow_stability.stability_score),
urgency: RecommendationUrgency::High,
expected_benefit: "Reduced training variance and improved robustness".to_string(),
});
}
if matches!(layer.flow_classification, FlowType::Turbulent) {
recommendations.push(FlowRecommendation {
layer_index: Some(layer.layer_index),
recommendation_type: FlowRecommendationType::ReduceTurbulence,
description: format!("Layer {} exhibits turbulent flow. Consider batch normalization or skip connections.",
layer.layer_index),
urgency: RecommendationUrgency::Medium,
expected_benefit: "Smoother gradient flow and faster convergence".to_string(),
});
}
}
// Global pattern recommendations
if flow_patterns.pattern_complexity > 1.0 {
recommendations.push(FlowRecommendation {
layer_index: None,
recommendation_type: FlowRecommendationType::SimplifyFlow,
description: "High flow pattern complexity detected. Consider architectural simplification or regularization.".to_string(),
urgency: RecommendationUrgency::Medium,
expected_benefit: "More predictable training dynamics and improved interpretability".to_string(),
});
}
if flow_patterns.vortex_structures.len() > layer_analyses.len() / 3 {
recommendations.push(FlowRecommendation {
layer_index: None,
recommendation_type: FlowRecommendationType::ReduceVorticity,
description: "Multiple vortex structures detected. Consider residual connections or attention mechanisms.".to_string(),
urgency: RecommendationUrgency::Low,
expected_benefit: "Reduced gradient circulation and improved information flow".to_string(),
});
}
recommendations
}
}
// Data structures for advanced gradient flow analysis
#[derive(Debug, Serialize, Deserialize)]
pub struct AdvancedGradientFlowAnalysis {
pub layer_analyses: Vec<LayerGradientFlowAnalysis>,
pub directional_derivatives: DirectionalDerivatives,
pub flow_patterns: FlowPatterns,
pub gradient_topology: GradientTopology,
pub flow_dynamics: FlowDynamics,
pub convergence_analysis: FlowConvergenceAnalysis,
pub flow_recommendations: Vec<FlowRecommendation>,
}
#[derive(Debug, Serialize, Deserialize)]
pub struct ModelStructureInfo {
pub num_layers: usize,
pub layer_types: Vec<String>,
pub architecture_type: String,
}
#[derive(Debug, Serialize, Deserialize)]
pub struct LayerGradientFlowAnalysis {
pub layer_index: usize,
pub gradient_magnitude: f32,
pub flow_direction: FlowDirection,
pub flow_divergence: f32,
pub flow_curl: f32,
pub flow_coherence: f32,
pub flow_stability: FlowStability,
pub flow_classification: FlowType,
}
#[derive(Debug, Default, Serialize, Deserialize)]
pub struct FlowDirection {
pub dominant_direction: Vec<f32>,
pub direction_strength: f32,
pub direction_consistency: f32,
pub magnitude: f32,
}
#[derive(Debug, Default, Serialize, Deserialize)]
pub struct FlowStability {
pub stability_score: f32,
pub local_variations: Vec<f32>,
pub stability_classification: StabilityClass,
}
#[derive(Debug, Default, Serialize, Deserialize)]
pub enum StabilityClass {
#[default]
Stable,
ModeratelyStable,
Unstable,
HighlyUnstable,
}
#[derive(Debug, Serialize, Deserialize)]
pub enum FlowType {
Laminar,
Turbulent,
Divergent,
Rotational,
Mixed,
}
#[derive(Debug, Default, Serialize, Deserialize)]
pub struct DirectionalDerivatives {
pub layer_derivatives: Vec<LayerDirectionalDerivatives>,
pub flow_acceleration: f32,
pub flow_jerk: f32,
pub derivative_patterns: DerivativePatterns,
}
#[derive(Debug, Serialize, Deserialize)]
pub struct LayerDirectionalDerivatives {
pub layer_index: usize,
pub primary_derivative: f32,
pub secondary_derivative: f32,
pub cross_derivative: f32,
pub derivative_magnitude: f32,
}
#[derive(Debug, Default, Serialize, Deserialize)]
pub struct DerivativePatterns {
pub primary_trend: DerivativeTrend,
pub secondary_trend: DerivativeTrend,
pub oscillation_frequency: f32,
pub dominant_frequency: f32,
pub pattern_stability: f32,
}
#[derive(Debug, Default, Serialize, Deserialize)]
pub enum DerivativeTrend {
Increasing,
Decreasing,
#[default]
Stable,
}
#[derive(Debug, Serialize, Deserialize)]
pub struct FlowPatterns {
pub convergence_zones: Vec<ConvergenceZone>,
pub divergence_zones: Vec<DivergenceZone>,
pub vortex_structures: Vec<VortexStructure>,
pub flow_boundaries: Vec<FlowBoundary>,
pub pattern_complexity: f32,
}
#[derive(Debug, Serialize, Deserialize)]
pub struct ConvergenceZone {
pub layer_index: usize,
pub convergence_strength: f32,
pub zone_stability: f32,
}
#[derive(Debug, Serialize, Deserialize)]
pub struct DivergenceZone {
pub layer_index: usize,
pub divergence_strength: f32,
pub expansion_rate: f32,
}
#[derive(Debug, Serialize, Deserialize)]
pub struct VortexStructure {
pub layer_index: usize,
pub vorticity_strength: f32,
pub rotation_direction: RotationDirection,
pub core_stability: f32,
}
#[derive(Debug, Serialize, Deserialize)]
pub enum RotationDirection {
Clockwise,
Counterclockwise,
}
#[derive(Debug, Serialize, Deserialize)]
pub struct FlowBoundary {
pub between_layers: (usize, usize),
pub boundary_strength: f32,
pub boundary_type: BoundaryType,
}
#[derive(Debug, Serialize, Deserialize)]
pub enum BoundaryType {
Amplification,
Attenuation,
Transition,
}
#[derive(Debug, Serialize, Deserialize)]
pub struct GradientTopology {
pub critical_points: Vec<CriticalPoint>,
pub saddle_points: Vec<SaddlePoint>,
pub flow_separatrices: Vec<FlowSeparatrix>,
pub topology_classification: TopologyClass,
pub topological_invariants: TopologicalInvariants,
}
#[derive(Debug, Serialize, Deserialize)]
pub struct CriticalPoint {
pub layer_index: usize,
pub point_type: CriticalPointType,
pub stability_index: f32,
}
#[derive(Debug, Serialize, Deserialize)]
pub enum CriticalPointType {
Source,
Sink,
Vortex,
Saddle,
}
#[derive(Debug, Serialize, Deserialize)]
pub struct SaddlePoint {
pub layer_index: usize,
pub saddle_strength: f32,
pub manifold_dimensions: usize,
}
#[derive(Debug, Serialize, Deserialize)]
pub struct FlowSeparatrix {
pub between_layers: (usize, usize),
pub separation_strength: f32,
pub separatrix_type: SeparatrixType,
}
#[derive(Debug, Serialize, Deserialize)]
pub enum SeparatrixType {
Heteroclinic,
Shock,
Simple,
}
#[derive(Debug, Serialize, Deserialize)]
pub enum TopologyClass {
Simple,
Moderate,
Complex,
Chaotic,
}
#[derive(Debug, Serialize, Deserialize)]
pub struct TopologicalInvariants {
pub euler_characteristic: i32,
pub genus: usize,
pub betti_numbers: Vec<usize>,
}
#[derive(Debug, Serialize, Deserialize)]
pub struct FlowDynamics {
pub temporal_evolution: TemporalEvolution,
pub phase_space_analysis: PhaseSpaceAnalysis,
pub lyapunov_exponents: Vec<f32>,
pub attractor_analysis: AttractorAnalysis,
pub dynamics_classification: DynamicsClass,
}
#[derive(Debug, Serialize, Deserialize)]
pub struct TemporalEvolution {
pub evolution_trend: DerivativeTrend,
pub evolution_rate: f32,
pub stability_over_time: f32,
pub periodic_components: Vec<f32>,
}
#[derive(Debug, Serialize, Deserialize)]
pub struct PhaseSpaceAnalysis {
pub phase_portrait: Vec<(f32, f32)>,
pub attractor_dimension: f32,
pub phase_space_volume: f32,
pub embedding_dimension: usize,
}
#[derive(Debug, Serialize, Deserialize)]
pub struct AttractorAnalysis {
pub attractor_type: AttractorType,
pub basin_of_attraction: f32,
pub attractor_stability: f32,
pub strange_attractor_indicators: Vec<String>,
}
#[derive(Debug, Serialize, Deserialize)]
pub enum AttractorType {
FixedPoint,
LimitCycle,
Torus,
Strange,
}
#[derive(Debug, Serialize, Deserialize)]
pub enum DynamicsClass {
Stable,
Periodic,
Quasiperiodic,
Chaotic,
Complex,
}
#[derive(Debug, Default, Serialize, Deserialize)]
pub struct FlowConvergenceAnalysis {
pub convergence_rate: f32,
pub convergence_quality: f32,
pub oscillation_damping: f32,
pub convergence_indicators: ConvergenceIndicators,
}
#[derive(Debug, Default, Serialize, Deserialize)]
pub struct ConvergenceIndicators {
pub is_monotonic: bool,
pub has_reached_plateau: bool,
pub oscillation_frequency: f32,
pub convergence_confidence: f32,
}
#[derive(Debug, Serialize, Deserialize)]
pub struct FlowRecommendation {
pub layer_index: Option<usize>,
pub recommendation_type: FlowRecommendationType,
pub description: String,
pub urgency: RecommendationUrgency,
pub expected_benefit: String,
}
#[derive(Debug, Serialize, Deserialize)]
pub enum FlowRecommendationType {
ImproveCoherence,
StabilizeFlow,
ReduceTurbulence,
SimplifyFlow,
ReduceVorticity,
OptimizeConvergence,
}
#[derive(Debug, Serialize, Deserialize)]
pub enum RecommendationUrgency {
Low,
Medium,
High,
Critical,
}
/// Advanced Model Comparison Framework for debugging different model states
pub struct ModelComparisonAnalyzer;
impl ModelComparisonAnalyzer {
/// Compare two model states and identify key differences
pub fn compare_model_states<T>(
model_a: &T,
model_b: &T,
comparison_type: ComparisonType
) -> Result<ModelComparisonReport> {
let mut comparison = ModelComparisonReport {
comparison_id: uuid::Uuid::new_v4().to_string(),
comparison_type,
timestamp: chrono::Utc::now(),
overall_similarity: 0.0,
weight_differences: Vec::new(),
architecture_differences: Vec::new(),
performance_differences: Vec::new(),
recommendations: Vec::new(),
};
// Analyze architectural differences (placeholder for trait-based implementation)
comparison.architecture_differences = Self::analyze_architecture_differences();
// Calculate overall similarity score
comparison.overall_similarity = Self::calculate_overall_similarity(&comparison);
// Generate recommendations based on differences
comparison.recommendations = Self::generate_comparison_recommendations(&comparison);
Ok(comparison)
}
/// Analyze training convergence patterns across multiple epochs
pub fn analyze_training_convergence(
loss_history: &[f32],
accuracy_history: &[f32],
learning_rates: &[f32]
) -> Result<TrainingConvergenceAnalysis> {
let convergence_analysis = TrainingConvergenceAnalysis {
analysis_id: uuid::Uuid::new_v4().to_string(),
total_epochs: loss_history.len(),
convergence_status: Self::determine_convergence_status(loss_history),
loss_trend: Self::analyze_trend(loss_history),
accuracy_trend: Self::analyze_trend(accuracy_history),
learning_rate_impact: Self::analyze_lr_impact(learning_rates, loss_history),
plateau_detection: Self::detect_training_plateaus(loss_history, accuracy_history),
early_stopping_recommendation: Self::recommend_early_stopping(loss_history, accuracy_history),
optimization_suggestions: Self::generate_optimization_suggestions(loss_history, accuracy_history, learning_rates),
estimated_remaining_epochs: Self::estimate_remaining_epochs(loss_history, accuracy_history),
};
Ok(convergence_analysis)
}
/// Optimize memory usage during debugging sessions
pub fn optimize_debug_memory_usage(session_config: &DebugConfig) -> Result<MemoryOptimizationReport> {
let mut memory_optimizer = MemoryOptimizationReport {
optimization_id: uuid::Uuid::new_v4().to_string(),
current_memory_usage: Self::estimate_current_memory_usage(session_config),
optimized_memory_usage: 0,
memory_savings: 0,
optimization_strategies: Vec::new(),
performance_impact: PerformanceImpact::Minimal,
recommended_actions: Vec::new(),
};
// Analyze memory usage patterns
let strategies = Self::identify_memory_optimization_strategies(session_config);
memory_optimizer.optimization_strategies = strategies;
// Calculate potential savings
memory_optimizer.optimized_memory_usage = Self::calculate_optimized_memory(&memory_optimizer);
memory_optimizer.memory_savings = memory_optimizer.current_memory_usage - memory_optimizer.optimized_memory_usage;
// Generate recommendations
memory_optimizer.recommended_actions = Self::generate_memory_recommendations(&memory_optimizer);
Ok(memory_optimizer)
}
/// Create automated debugging workflows for common issues
pub fn create_automated_workflow(workflow_type: AutomatedWorkflowType) -> Result<DebuggingWorkflow> {
let workflow = match workflow_type {
AutomatedWorkflowType::GradientExplosion => Self::create_gradient_explosion_workflow(),
AutomatedWorkflowType::TrainingStagnation => Self::create_training_stagnation_workflow(),
AutomatedWorkflowType::MemoryLeakDetection => Self::create_memory_leak_workflow(),
AutomatedWorkflowType::PerformanceBottleneck => Self::create_performance_bottleneck_workflow(),
AutomatedWorkflowType::ComprehensiveHealthCheck => Self::create_comprehensive_health_workflow(),
};
Ok(workflow)
}
// Private helper methods for model comparison
fn analyze_architecture_differences() -> Vec<ArchitectureDifference> {
vec![
ArchitectureDifference {
component: "attention_heads".to_string(),
difference_type: DifferenceType::ParameterCount,
description: "Different number of attention heads detected".to_string(),
impact_level: ImpactLevel::Medium,
}
]
}
fn calculate_overall_similarity(comparison: &ModelComparisonReport) -> f32 {
// Sophisticated similarity calculation based on multiple factors
let architecture_similarity = 0.85; // Placeholder
let weight_similarity = 0.92; // Placeholder
let performance_similarity = 0.78; // Placeholder
(architecture_similarity + weight_similarity + performance_similarity) / 3.0
}
fn generate_comparison_recommendations(comparison: &ModelComparisonReport) -> Vec<ComparisonRecommendation> {
let mut recommendations = Vec::new();
if comparison.overall_similarity < 0.8 {
recommendations.push(ComparisonRecommendation {
recommendation_type: RecommendationType::Investigation,
description: "Significant differences detected between models - investigate training procedures".to_string(),
priority: RecommendationPriority::High,
estimated_impact: "May indicate model divergence or different training regimes".to_string(),
});
}
recommendations
}
// Training convergence analysis helpers
fn determine_convergence_status(loss_history: &[f32]) -> ConvergenceStatus {
if loss_history.len() < 10 {
return ConvergenceStatus::InsufficientData;
}
let recent_losses = &loss_history[loss_history.len()-10..];
let loss_variance = Self::calculate_variance(recent_losses);
if loss_variance < 0.01 {
ConvergenceStatus::Converged
} else if Self::is_decreasing_trend(recent_losses) {
ConvergenceStatus::Converging
} else {
ConvergenceStatus::Diverging
}
}
fn analyze_trend(values: &[f32]) -> TrendAnalysis {
if values.len() < 2 {
return TrendAnalysis {
trend_type: TrendType::Unknown,
trend_strength: 0.0,
confidence: 0.0,
slope: 0.0,
};
}
let slope = Self::calculate_slope(values);
let trend_type = if slope > 0.01 {
TrendType::Increasing
} else if slope < -0.01 {
TrendType::Decreasing
} else {
TrendType::Stable
};
TrendAnalysis {
trend_type,
trend_strength: slope.abs(),
confidence: Self::calculate_trend_confidence(values),
slope,
}
}
fn analyze_lr_impact(learning_rates: &[f32], loss_history: &[f32]) -> LearningRateImpact {
LearningRateImpact {
correlation_with_loss: Self::calculate_correlation(learning_rates, loss_history),
optimal_lr_estimate: Self::estimate_optimal_lr(learning_rates, loss_history),
lr_schedule_recommendation: LRScheduleRecommendation::ExponentialDecay,
sensitivity_analysis: Self::analyze_lr_sensitivity(learning_rates, loss_history),
}
}
fn detect_training_plateaus(loss_history: &[f32], accuracy_history: &[f32]) -> PlateauDetection {
PlateauDetection {
has_loss_plateau: Self::detect_plateau_in_series(loss_history),
has_accuracy_plateau: Self::detect_plateau_in_series(accuracy_history),
plateau_start_epoch: Self::find_plateau_start(loss_history),
plateau_duration: Self::calculate_plateau_duration(loss_history),
plateau_severity: PlateauSeverity::Moderate,
}
}
fn recommend_early_stopping(loss_history: &[f32], accuracy_history: &[f32]) -> EarlyStoppingRecommendation {
EarlyStoppingRecommendation {
should_stop: Self::should_recommend_early_stopping(loss_history, accuracy_history),
recommended_patience: 10,
confidence: Self::calculate_early_stopping_confidence(loss_history, accuracy_history),
reasoning: "Training appears to have plateaued based on loss and accuracy trends".to_string(),
}
}
fn generate_optimization_suggestions(
loss_history: &[f32],
accuracy_history: &[f32],
learning_rates: &[f32]
) -> Vec<OptimizationSuggestion> {
let mut suggestions = Vec::new();
if Self::detect_plateau_in_series(loss_history) {
suggestions.push(OptimizationSuggestion {
suggestion_type: OptimizationType::LearningRate,
description: "Consider reducing learning rate to escape plateau".to_string(),
expected_benefit: "May help model converge to better local minimum".to_string(),
implementation_difficulty: ImplementationDifficulty::Easy,
});
}
suggestions
}
fn estimate_remaining_epochs(loss_history: &[f32], accuracy_history: &[f32]) -> u32 {
// Simple heuristic based on convergence rate
if Self::is_decreasing_trend(loss_history) {
let convergence_rate = Self::estimate_convergence_rate(loss_history);
if convergence_rate > 0.0 {
return (0.01 / convergence_rate) as u32; // Estimate epochs to reach 1% of current loss
}
}
50 // Default estimate
}
// Memory optimization helpers
fn estimate_current_memory_usage(config: &DebugConfig) -> u64 {
// Estimate based on debug configuration
let base_usage = 100_000_000; // 100MB base
let tensor_usage = if config.enable_tensor_inspection { 500_000_000 } else { 0 }; // 500MB for tensors
let gradient_usage = if config.enable_gradient_debugging { 300_000_000 } else { 0 }; // 300MB for gradients
base_usage + tensor_usage + gradient_usage
}
fn identify_memory_optimization_strategies(config: &DebugConfig) -> Vec<MemoryOptimizationStrategy> {
let mut strategies = Vec::new();
if config.enable_tensor_inspection {
strategies.push(MemoryOptimizationStrategy {
strategy_type: OptimizationStrategyType::SelectiveTensorInspection,
description: "Only inspect critical tensors instead of all tensors".to_string(),
memory_savings_estimate: 200_000_000, // 200MB
performance_impact: PerformanceImpact::Minimal,
});
}
strategies
}
fn calculate_optimized_memory(optimizer: &MemoryOptimizationReport) -> u64 {
let total_savings: u64 = optimizer.optimization_strategies
.iter()
.map(|s| s.memory_savings_estimate)
.sum();
optimizer.current_memory_usage.saturating_sub(total_savings)
}
fn generate_memory_recommendations(optimizer: &MemoryOptimizationReport) -> Vec<String> {
let mut recommendations = Vec::new();
if optimizer.memory_savings > 100_000_000 { // 100MB savings possible
recommendations.push("Enable selective debugging to reduce memory footprint".to_string());
}
if optimizer.current_memory_usage > 1_000_000_000 { // 1GB current usage
recommendations.push("Consider using lazy evaluation for large tensors".to_string());
}
recommendations
}
// Automated workflow creation
fn create_gradient_explosion_workflow() -> DebuggingWorkflow {
DebuggingWorkflow {
workflow_id: uuid::Uuid::new_v4().to_string(),
workflow_type: AutomatedWorkflowType::GradientExplosion,
steps: vec![
WorkflowStep {
step_id: 1,
description: "Monitor gradient norms across all layers".to_string(),
action: WorkflowAction::EnableGradientMonitoring,
expected_duration: Duration::from_secs(5),
},
WorkflowStep {
step_id: 2,
description: "Check for gradient explosion patterns".to_string(),
action: WorkflowAction::AnalyzeGradients,
expected_duration: Duration::from_secs(10),
},
WorkflowStep {
step_id: 3,
description: "Generate gradient clipping recommendations".to_string(),
action: WorkflowAction::GenerateRecommendations,
expected_duration: Duration::from_secs(3),
},
],
estimated_total_duration: Duration::from_secs(18),
automation_level: AutomationLevel::FullyAutomated,
}
}
fn create_training_stagnation_workflow() -> DebuggingWorkflow {
DebuggingWorkflow {
workflow_id: uuid::Uuid::new_v4().to_string(),
workflow_type: AutomatedWorkflowType::TrainingStagnation,
steps: vec![
WorkflowStep {
step_id: 1,
description: "Analyze loss curve trends".to_string(),
action: WorkflowAction::AnalyzeLossHistory,
expected_duration: Duration::from_secs(5),
},
WorkflowStep {
step_id: 2,
description: "Check for training plateaus".to_string(),
action: WorkflowAction::DetectPlateaus,
expected_duration: Duration::from_secs(8),
},
WorkflowStep {
step_id: 3,
description: "Recommend optimization strategies".to_string(),
action: WorkflowAction::GenerateOptimizationSuggestions,
expected_duration: Duration::from_secs(5),
},
],
estimated_total_duration: Duration::from_secs(18),
automation_level: AutomationLevel::SemiAutomated,
}
}
fn create_memory_leak_workflow() -> DebuggingWorkflow {
DebuggingWorkflow {
workflow_id: uuid::Uuid::new_v4().to_string(),
workflow_type: AutomatedWorkflowType::MemoryLeakDetection,
steps: vec![
WorkflowStep {
step_id: 1,
description: "Monitor memory usage patterns".to_string(),
action: WorkflowAction::MonitorMemory,
expected_duration: Duration::from_secs(30),
},
WorkflowStep {
step_id: 2,
description: "Detect memory leak patterns".to_string(),
action: WorkflowAction::AnalyzeMemoryLeaks,
expected_duration: Duration::from_secs(15),
},
WorkflowStep {
step_id: 3,
description: "Generate cleanup recommendations".to_string(),
action: WorkflowAction::GenerateCleanupRecommendations,
expected_duration: Duration::from_secs(5),
},
],
estimated_total_duration: Duration::from_secs(50),
automation_level: AutomationLevel::FullyAutomated,
}
}
fn create_performance_bottleneck_workflow() -> DebuggingWorkflow {
DebuggingWorkflow {
workflow_id: uuid::Uuid::new_v4().to_string(),
workflow_type: AutomatedWorkflowType::PerformanceBottleneck,
steps: vec![
WorkflowStep {
step_id: 1,
description: "Profile model performance".to_string(),
action: WorkflowAction::ProfilePerformance,
expected_duration: Duration::from_secs(20),
},
WorkflowStep {
step_id: 2,
description: "Identify bottleneck layers".to_string(),
action: WorkflowAction::IdentifyBottlenecks,
expected_duration: Duration::from_secs(10),
},
WorkflowStep {
step_id: 3,
description: "Suggest performance optimizations".to_string(),
action: WorkflowAction::GeneratePerformanceOptimizations,
expected_duration: Duration::from_secs(8),
},
],
estimated_total_duration: Duration::from_secs(38),
automation_level: AutomationLevel::SemiAutomated,
}
}
fn create_comprehensive_health_workflow() -> DebuggingWorkflow {
DebuggingWorkflow {
workflow_id: uuid::Uuid::new_v4().to_string(),
workflow_type: AutomatedWorkflowType::ComprehensiveHealthCheck,
steps: vec![
WorkflowStep {
step_id: 1,
description: "Run complete model health assessment".to_string(),
action: WorkflowAction::ComprehensiveHealthCheck,
expected_duration: Duration::from_secs(45),
},
WorkflowStep {
step_id: 2,
description: "Analyze all subsystem health".to_string(),
action: WorkflowAction::AnalyzeSubsystemHealth,
expected_duration: Duration::from_secs(30),
},
WorkflowStep {
step_id: 3,
description: "Generate comprehensive report".to_string(),
action: WorkflowAction::GenerateComprehensiveReport,
expected_duration: Duration::from_secs(15),
},
],
estimated_total_duration: Duration::from_secs(90),
automation_level: AutomationLevel::FullyAutomated,
}
}
// Mathematical helper functions
fn calculate_variance(values: &[f32]) -> f32 {
if values.is_empty() {
return 0.0;
}
let mean = values.iter().sum::<f32>() / values.len() as f32;
let variance = values.iter()
.map(|&x| (x - mean).powi(2))
.sum::<f32>() / values.len() as f32;
variance
}
fn is_decreasing_trend(values: &[f32]) -> bool {
if values.len() < 2 {
return false;
}
let slope = Self::calculate_slope(values);
slope < -0.001 // Small negative threshold
}
fn calculate_slope(values: &[f32]) -> f32 {
if values.len() < 2 {
return 0.0;
}
let n = values.len() as f32;
let x_mean = (n - 1.0) / 2.0; // Since indices are 0, 1, 2, ..., n-1
let y_mean = values.iter().sum::<f32>() / n;
let numerator: f32 = values.iter().enumerate()
.map(|(i, &y)| (i as f32 - x_mean) * (y - y_mean))
.sum();
let denominator: f32 = (0..values.len())
.map(|i| (i as f32 - x_mean).powi(2))
.sum();
if denominator != 0.0 {
numerator / denominator
} else {
0.0
}
}
fn calculate_trend_confidence(values: &[f32]) -> f32 {
// Simple R-squared calculation for trend confidence
if values.len() < 3 {
return 0.0;
}
let slope = Self::calculate_slope(values);
let y_mean = values.iter().sum::<f32>() / values.len() as f32;
let ss_res: f32 = values.iter().enumerate()
.map(|(i, &y)| {
let y_pred = slope * i as f32 + (y_mean - slope * (values.len() - 1) as f32 / 2.0);
(y - y_pred).powi(2)
})
.sum();
let ss_tot: f32 = values.iter()
.map(|&y| (y - y_mean).powi(2))
.sum();
if ss_tot != 0.0 {
1.0 - (ss_res / ss_tot)
} else {
0.0
}
}
fn calculate_correlation(x_values: &[f32], y_values: &[f32]) -> f32 {
if x_values.len() != y_values.len() || x_values.is_empty() {
return 0.0;
}
let n = x_values.len() as f32;
let x_mean = x_values.iter().sum::<f32>() / n;
let y_mean = y_values.iter().sum::<f32>() / n;
let numerator: f32 = x_values.iter().zip(y_values.iter())
.map(|(&x, &y)| (x - x_mean) * (y - y_mean))
.sum();
let x_variance: f32 = x_values.iter()
.map(|&x| (x - x_mean).powi(2))
.sum();
let y_variance: f32 = y_values.iter()
.map(|&y| (y - y_mean).powi(2))
.sum();
let denominator = (x_variance * y_variance).sqrt();
if denominator != 0.0 {
numerator / denominator
} else {
0.0
}
}
fn estimate_optimal_lr(learning_rates: &[f32], loss_history: &[f32]) -> f32 {
// Simple heuristic: find LR with steepest loss decrease
if learning_rates.is_empty() || loss_history.len() < 2 {
return 0.001; // Default
}
// For simplicity, return the median learning rate
let mut sorted_lrs = learning_rates.to_vec();
sorted_lrs.sort_by(|a, b| a.partial_cmp(b).unwrap_or(std::cmp::Ordering::Equal));
sorted_lrs[sorted_lrs.len() / 2]
}
fn analyze_lr_sensitivity(learning_rates: &[f32], loss_history: &[f32]) -> f32 {
// Calculate how sensitive loss is to learning rate changes
Self::calculate_correlation(learning_rates, loss_history).abs()
}
fn detect_plateau_in_series(values: &[f32]) -> bool {
if values.len() < 10 {
return false;
}
let recent_values = &values[values.len()-10..];
let variance = Self::calculate_variance(recent_values);
variance < 0.001 // Very small variance indicates plateau
}
fn find_plateau_start(values: &[f32]) -> Option<usize> {
// Simple heuristic: find where variance becomes very small
const WINDOW_SIZE: usize = 10;
if values.len() < WINDOW_SIZE {
return None;
}
for i in 0..(values.len() - WINDOW_SIZE + 1) {
let window = &values[i..i + WINDOW_SIZE];
if Self::calculate_variance(window) < 0.001 {
return Some(i);
}
}
None
}
fn calculate_plateau_duration(values: &[f32]) -> u32 {
if let Some(start) = Self::find_plateau_start(values) {
(values.len() - start) as u32
} else {
0
}
}
fn should_recommend_early_stopping(loss_history: &[f32], accuracy_history: &[f32]) -> bool {
Self::detect_plateau_in_series(loss_history) && Self::detect_plateau_in_series(accuracy_history)
}
fn calculate_early_stopping_confidence(loss_history: &[f32], accuracy_history: &[f32]) -> f32 {
let loss_plateau_strength = if Self::detect_plateau_in_series(loss_history) { 0.5 } else { 0.0 };
let accuracy_plateau_strength = if Self::detect_plateau_in_series(accuracy_history) { 0.5 } else { 0.0 };
loss_plateau_strength + accuracy_plateau_strength
}
fn estimate_convergence_rate(loss_history: &[f32]) -> f32 {
if loss_history.len() < 2 {
return 0.0;
}
let recent_slope = Self::calculate_slope(loss_history).abs();
recent_slope
}
}
// Supporting data structures for the new capabilities
#[derive(Debug, Serialize, Deserialize)]
pub struct ModelComparisonReport {
pub comparison_id: String,
pub comparison_type: ComparisonType,
pub timestamp: chrono::DateTime<chrono::Utc>,
pub overall_similarity: f32,
pub weight_differences: Vec<WeightDifference>,
pub architecture_differences: Vec<ArchitectureDifference>,
pub performance_differences: Vec<PerformanceDifference>,
pub recommendations: Vec<ComparisonRecommendation>,
}
#[derive(Debug, Serialize, Deserialize)]
pub enum ComparisonType {
BeforeAfterTraining,
DifferentArchitectures,
CheckpointComparison,
AblationStudy,
}
#[derive(Debug, Serialize, Deserialize)]
pub struct WeightDifference {
pub layer_name: String,
pub difference_magnitude: f32,
pub difference_type: WeightDifferenceType,
}
#[derive(Debug, Serialize, Deserialize)]
pub enum WeightDifferenceType {
Drift,
Scale,
Distribution,
Sparsity,
}
#[derive(Debug, Serialize, Deserialize)]
pub struct ArchitectureDifference {
pub component: String,
pub difference_type: DifferenceType,
pub description: String,
pub impact_level: ImpactLevel,
}
#[derive(Debug, Serialize, Deserialize)]
pub enum DifferenceType {
ParameterCount,
LayerStructure,
ActivationFunction,
Normalization,
}
#[derive(Debug, Serialize, Deserialize)]
pub enum ImpactLevel {
Low,
Medium,
High,
Critical,
}
#[derive(Debug, Serialize, Deserialize)]
pub struct PerformanceDifference {
pub metric_name: String,
pub value_a: f32,
pub value_b: f32,
pub relative_difference: f32,
pub significance: SignificanceLevel,
}
#[derive(Debug, Serialize, Deserialize)]
pub enum SignificanceLevel {
NotSignificant,
Marginal,
Significant,
HighlySignificant,
}
#[derive(Debug, Serialize, Deserialize)]
pub struct ComparisonRecommendation {
pub recommendation_type: RecommendationType,
pub description: String,
pub priority: RecommendationPriority,
pub estimated_impact: String,
}
#[derive(Debug, Serialize, Deserialize)]
pub enum RecommendationType {
Investigation,
Optimization,
Validation,
Monitoring,
}
#[derive(Debug, Serialize, Deserialize)]
pub enum RecommendationPriority {
Low,
Medium,
High,
Critical,
}
#[derive(Debug, Serialize, Deserialize)]
pub struct TrainingConvergenceAnalysis {
pub analysis_id: String,
pub total_epochs: usize,
pub convergence_status: ConvergenceStatus,
pub loss_trend: TrendAnalysis,
pub accuracy_trend: TrendAnalysis,
pub learning_rate_impact: LearningRateImpact,
pub plateau_detection: PlateauDetection,
pub early_stopping_recommendation: EarlyStoppingRecommendation,
pub optimization_suggestions: Vec<OptimizationSuggestion>,
pub estimated_remaining_epochs: u32,
}
#[derive(Debug, Serialize, Deserialize)]
pub enum ConvergenceStatus {
Converged,
Converging,
Diverging,
Oscillating,
InsufficientData,
}
#[derive(Debug, Serialize, Deserialize)]
pub struct TrendAnalysis {
pub trend_type: TrendType,
pub trend_strength: f32,
pub confidence: f32,
pub slope: f32,
}
#[derive(Debug, Serialize, Deserialize)]
pub enum TrendType {
Increasing,
Decreasing,
Stable,
Oscillating,
Unknown,
}
#[derive(Debug, Serialize, Deserialize)]
pub struct LearningRateImpact {
pub correlation_with_loss: f32,
pub optimal_lr_estimate: f32,
pub lr_schedule_recommendation: LRScheduleRecommendation,
pub sensitivity_analysis: f32,
}
#[derive(Debug, Serialize, Deserialize)]
pub enum LRScheduleRecommendation {
Constant,
LinearDecay,
ExponentialDecay,
CosineAnnealing,
ReduceOnPlateau,
}
#[derive(Debug, Serialize, Deserialize)]
pub struct PlateauDetection {
pub has_loss_plateau: bool,
pub has_accuracy_plateau: bool,
pub plateau_start_epoch: Option<usize>,
pub plateau_duration: u32,
pub plateau_severity: PlateauSeverity,
}
#[derive(Debug, Serialize, Deserialize)]
pub enum PlateauSeverity {
Mild,
Moderate,
Severe,
}
#[derive(Debug, Serialize, Deserialize)]
pub struct EarlyStoppingRecommendation {
pub should_stop: bool,
pub recommended_patience: u32,
pub confidence: f32,
pub reasoning: String,
}
#[derive(Debug, Serialize, Deserialize)]
pub struct OptimizationSuggestion {
pub suggestion_type: OptimizationType,
pub description: String,
pub expected_benefit: String,
pub implementation_difficulty: ImplementationDifficulty,
}
#[derive(Debug, Serialize, Deserialize)]
pub enum OptimizationType {
LearningRate,
BatchSize,
Architecture,
Regularization,
DataAugmentation,
}
#[derive(Debug, Clone, Serialize, Deserialize)]
pub enum ImplementationDifficulty {
Easy,
Medium,
Hard,
}
#[derive(Debug, Serialize, Deserialize)]
pub struct MemoryOptimizationReport {
pub optimization_id: String,
pub current_memory_usage: u64,
pub optimized_memory_usage: u64,
pub memory_savings: u64,
pub optimization_strategies: Vec<MemoryOptimizationStrategy>,
pub performance_impact: PerformanceImpact,
pub recommended_actions: Vec<String>,
}
#[derive(Debug, Serialize, Deserialize)]
pub struct MemoryOptimizationStrategy {
pub strategy_type: OptimizationStrategyType,
pub description: String,
pub memory_savings_estimate: u64,
pub performance_impact: PerformanceImpact,
}
#[derive(Debug, Serialize, Deserialize)]
pub enum OptimizationStrategyType {
SelectiveTensorInspection,
LazyEvaluation,
MemoryPooling,
GradientCheckpointing,
BatchSizeReduction,
}
#[derive(Debug, Serialize, Deserialize)]
pub enum PerformanceImpact {
Minimal,
Low,
Medium,
High,
}
#[derive(Debug, Serialize, Deserialize)]
pub struct DebuggingWorkflow {
pub workflow_id: String,
pub workflow_type: AutomatedWorkflowType,
pub steps: Vec<WorkflowStep>,
pub estimated_total_duration: Duration,
pub automation_level: AutomationLevel,
}
#[derive(Debug, Serialize, Deserialize)]
pub enum AutomatedWorkflowType {
GradientExplosion,
TrainingStagnation,
MemoryLeakDetection,
PerformanceBottleneck,
ComprehensiveHealthCheck,
}
#[derive(Debug, Serialize, Deserialize)]
pub struct WorkflowStep {
pub step_id: u32,
pub description: String,
pub action: WorkflowAction,
pub expected_duration: Duration,
}
#[derive(Debug, Serialize, Deserialize)]
pub enum WorkflowAction {
EnableGradientMonitoring,
AnalyzeGradients,
GenerateRecommendations,
AnalyzeLossHistory,
DetectPlateaus,
GenerateOptimizationSuggestions,
MonitorMemory,
AnalyzeMemoryLeaks,
GenerateCleanupRecommendations,
ProfilePerformance,
IdentifyBottlenecks,
GeneratePerformanceOptimizations,
ComprehensiveHealthCheck,
AnalyzeSubsystemHealth,
GenerateComprehensiveReport,
}
#[derive(Debug, Serialize, Deserialize)]
pub enum AutomationLevel {
Manual,
SemiAutomated,
FullyAutomated,
}
// ================================================================================================
// REAL-TIME STREAMING ANALYTICS FOR LIVE TRAINING MONITORING
// ================================================================================================
/// Real-time streaming analytics framework for live training monitoring
/// Provides continuous monitoring and analysis of training metrics with minimal overhead
#[derive(Debug)]
pub struct RealTimeStreamingAnalytics {
pub config: StreamingConfig,
metrics_buffer: Vec<TrainingMetrics>,
alerts: Vec<StreamingAlert>,
analysis_history: Vec<StreamingAnalysis>,
last_update: std::time::Instant,
}
/// Configuration for real-time streaming analytics
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct StreamingConfig {
pub enable_streaming: bool,
pub buffer_size: usize,
pub analysis_interval: Duration,
pub alert_thresholds: AlertThresholds,
pub enable_anomaly_detection: bool,
pub enable_trend_analysis: bool,
}
impl Default for StreamingConfig {
fn default() -> Self {
Self {
enable_streaming: true,
buffer_size: 1000,
analysis_interval: Duration::from_secs(10),
alert_thresholds: AlertThresholds::default(),
enable_anomaly_detection: true,
enable_trend_analysis: true,
}
}
}
/// Real-time training metrics
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct TrainingMetrics {
pub timestamp: std::time::SystemTime,
pub epoch: u32,
pub step: u64,
pub loss: f32,
pub learning_rate: f32,
pub accuracy: Option<f32>,
pub gpu_memory_usage: f32,
pub cpu_utilization: f32,
pub gradient_norm: f32,
pub throughput: f32, // samples per second
}
/// Alert thresholds for streaming analytics
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct AlertThresholds {
pub loss_spike_threshold: f32,
pub gradient_explosion_threshold: f32,
pub memory_usage_threshold: f32,
pub cpu_utilization_threshold: f32,
pub throughput_drop_threshold: f32,
}
impl Default for AlertThresholds {
fn default() -> Self {
Self {
loss_spike_threshold: 2.0, // 2x increase in loss
gradient_explosion_threshold: 100.0,
memory_usage_threshold: 0.9, // 90% memory usage
cpu_utilization_threshold: 0.95, // 95% CPU usage
throughput_drop_threshold: 0.5, // 50% throughput drop
}
}
}
/// Real-time streaming alert
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct StreamingAlert {
pub alert_id: String,
pub timestamp: std::time::SystemTime,
pub severity: AlertSeverity,
pub alert_type: StreamingAlertType,
pub message: String,
pub metrics_snapshot: TrainingMetrics,
pub recommended_actions: Vec<String>,
}
#[derive(Debug, Clone, Serialize, Deserialize)]
pub enum AlertSeverity {
Info,
Warning,
Critical,
Emergency,
}
#[derive(Debug, Clone, Serialize, Deserialize)]
pub enum StreamingAlertType {
LossSpike,
GradientExplosion,
MemoryExhaustion,
CpuThrottling,
ThroughputDegradation,
TrainingStagnation,
NumericalInstability,
}
/// Real-time streaming analysis result
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct StreamingAnalysis {
pub analysis_id: String,
pub timestamp: std::time::SystemTime,
pub metrics_window: Duration,
pub trend_analysis: StreamingTrendAnalysis,
pub anomaly_score: f32,
pub performance_indicators: PerformanceIndicators,
pub training_health: TrainingHealthScore,
pub predictions: TrainingPredictions,
}
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct StreamingTrendAnalysis {
pub loss_trend: StreamingTrendDirection,
pub loss_volatility: f32,
pub learning_rate_effectiveness: f32,
pub gradient_stability: f32,
pub throughput_trend: StreamingTrendDirection,
}
#[derive(Debug, Clone, Serialize, Deserialize)]
pub enum StreamingTrendDirection {
Improving,
Stable,
Degrading,
Volatile,
}
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct PerformanceIndicators {
pub current_efficiency: f32,
pub resource_utilization_score: f32,
pub training_momentum: f32,
pub convergence_probability: f32,
}
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct TrainingHealthScore {
pub overall_health: f32, // 0.0 to 1.0
pub gradient_health: f32,
pub loss_health: f32,
pub resource_health: f32,
pub stability_health: f32,
}
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct TrainingPredictions {
pub estimated_convergence_time: Option<Duration>,
pub predicted_final_loss: Option<f32>,
pub recommended_early_stop: bool,
pub optimization_suggestions: Vec<String>,
}
impl RealTimeStreamingAnalytics {
/// Create a new real-time streaming analytics instance
pub fn new(config: StreamingConfig) -> Self {
Self {
config,
metrics_buffer: Vec::new(),
alerts: Vec::new(),
analysis_history: Vec::new(),
last_update: std::time::Instant::now(),
}
}
/// Add new training metrics to the streaming buffer
pub fn add_metrics(&mut self, metrics: TrainingMetrics) -> Result<()> {
self.metrics_buffer.push(metrics.clone());
// Maintain buffer size
if self.metrics_buffer.len() > self.config.buffer_size {
self.metrics_buffer.remove(0);
}
// Check for immediate alerts
self.check_immediate_alerts(&metrics)?;
// Perform analysis if interval has passed
if self.last_update.elapsed() >= self.config.analysis_interval {
self.perform_streaming_analysis()?;
self.last_update = std::time::Instant::now();
}
Ok(())
}
/// Check for immediate alerts based on current metrics
fn check_immediate_alerts(&mut self, metrics: &TrainingMetrics) -> Result<()> {
let thresholds = &self.config.alert_thresholds;
// Check for gradient explosion
if metrics.gradient_norm > thresholds.gradient_explosion_threshold {
let alert = StreamingAlert {
alert_id: format!("alert_{}", uuid::Uuid::new_v4()),
timestamp: std::time::SystemTime::now(),
severity: AlertSeverity::Critical,
alert_type: StreamingAlertType::GradientExplosion,
message: format!("Gradient explosion detected! Norm: {:.2}", metrics.gradient_norm),
metrics_snapshot: metrics.clone(),
recommended_actions: vec![
"Apply gradient clipping".to_string(),
"Reduce learning rate".to_string(),
"Check model architecture".to_string(),
],
};
self.alerts.push(alert);
}
// Check for memory exhaustion
if metrics.gpu_memory_usage > thresholds.memory_usage_threshold {
let alert = StreamingAlert {
alert_id: format!("alert_{}", uuid::Uuid::new_v4()),
timestamp: std::time::SystemTime::now(),
severity: AlertSeverity::Warning,
alert_type: StreamingAlertType::MemoryExhaustion,
message: format!("High GPU memory usage: {:.1}%", metrics.gpu_memory_usage * 100.0),
metrics_snapshot: metrics.clone(),
recommended_actions: vec![
"Reduce batch size".to_string(),
"Enable gradient checkpointing".to_string(),
"Consider model parallelism".to_string(),
],
};
self.alerts.push(alert);
}
// Check for loss spike
if let Some(previous_metrics) = self.metrics_buffer.last() {
let loss_ratio = metrics.loss / previous_metrics.loss;
if loss_ratio > thresholds.loss_spike_threshold {
let alert = StreamingAlert {
alert_id: format!("alert_{}", uuid::Uuid::new_v4()),
timestamp: std::time::SystemTime::now(),
severity: AlertSeverity::Warning,
alert_type: StreamingAlertType::LossSpike,
message: format!("Loss spike detected! Increase: {:.1}x", loss_ratio),
metrics_snapshot: metrics.clone(),
recommended_actions: vec![
"Check for data corruption".to_string(),
"Verify model stability".to_string(),
"Consider learning rate reduction".to_string(),
],
};
self.alerts.push(alert);
}
}
Ok(())
}
/// Perform comprehensive streaming analysis
fn perform_streaming_analysis(&mut self) -> Result<()> {
if self.metrics_buffer.len() < 2 {
return Ok(());
}
let analysis = StreamingAnalysis {
analysis_id: format!("analysis_{}", uuid::Uuid::new_v4()),
timestamp: std::time::SystemTime::now(),
metrics_window: self.config.analysis_interval,
trend_analysis: self.analyze_trends()?,
anomaly_score: self.calculate_anomaly_score()?,
performance_indicators: self.calculate_performance_indicators()?,
training_health: self.calculate_training_health()?,
predictions: self.generate_predictions()?,
};
self.analysis_history.push(analysis);
// Maintain analysis history size
if self.analysis_history.len() > 100 {
self.analysis_history.remove(0);
}
Ok(())
}
/// Analyze trends in the metrics buffer
fn analyze_trends(&self) -> Result<StreamingTrendAnalysis> {
if self.metrics_buffer.len() < 10 {
return Ok(StreamingTrendAnalysis {
loss_trend: StreamingTrendDirection::Stable,
loss_volatility: 0.0,
learning_rate_effectiveness: 0.5,
gradient_stability: 0.5,
throughput_trend: StreamingTrendDirection::Stable,
});
}
let recent_metrics = &self.metrics_buffer[self.metrics_buffer.len() - 10..];
// Analyze loss trend
let loss_values: Vec<f32> = recent_metrics.iter().map(|m| m.loss).collect();
let loss_trend = self.determine_trend_direction(&loss_values);
let loss_volatility = self.calculate_volatility(&loss_values);
// Analyze throughput trend
let throughput_values: Vec<f32> = recent_metrics.iter().map(|m| m.throughput).collect();
let throughput_trend = self.determine_trend_direction(&throughput_values);
// Calculate gradient stability
let gradient_values: Vec<f32> = recent_metrics.iter().map(|m| m.gradient_norm).collect();
let gradient_stability = 1.0 - self.calculate_volatility(&gradient_values);
// Estimate learning rate effectiveness
let lr_effectiveness = self.estimate_lr_effectiveness(recent_metrics);
Ok(StreamingTrendAnalysis {
loss_trend,
loss_volatility,
learning_rate_effectiveness: lr_effectiveness,
gradient_stability,
throughput_trend,
})
}
/// Determine trend direction from a series of values
fn determine_trend_direction(&self, values: &[f32]) -> StreamingTrendDirection {
if values.len() < 3 {
return StreamingTrendDirection::Stable;
}
let first_half = &values[..values.len() / 2];
let second_half = &values[values.len() / 2..];
let first_avg: f32 = first_half.iter().sum::<f32>() / first_half.len() as f32;
let second_avg: f32 = second_half.iter().sum::<f32>() / second_half.len() as f32;
let change_ratio = (second_avg - first_avg) / first_avg.abs();
if change_ratio.abs() < 0.05 {
StreamingTrendDirection::Stable
} else if change_ratio < -0.1 {
StreamingTrendDirection::Improving // For loss, decreasing is improving
} else if change_ratio > 0.1 {
StreamingTrendDirection::Degrading
} else {
StreamingTrendDirection::Volatile
}
}
/// Calculate volatility (coefficient of variation)
fn calculate_volatility(&self, values: &[f32]) -> f32 {
if values.len() < 2 {
return 0.0;
}
let mean: f32 = values.iter().sum::<f32>() / values.len() as f32;
let variance: f32 = values.iter()
.map(|&x| (x - mean).powi(2))
.sum::<f32>() / values.len() as f32;
let std_dev = variance.sqrt();
if mean.abs() < 1e-8 {
0.0
} else {
std_dev / mean.abs()
}
}
/// Estimate learning rate effectiveness
fn estimate_lr_effectiveness(&self, metrics: &[TrainingMetrics]) -> f32 {
if metrics.len() < 5 {
return 0.5;
}
let mut effectiveness_scores = Vec::new();
for window in metrics.windows(3) {
let loss_improvement = (window[0].loss - window[2].loss) / window[0].loss;
let lr_magnitude = window[1].learning_rate;
// Higher LR should lead to more improvement, but too high can cause instability
let score = if loss_improvement > 0.0 && lr_magnitude > 0.0 {
(loss_improvement * 10.0).min(1.0) // Cap at 1.0
} else {
0.0
};
effectiveness_scores.push(score);
}
effectiveness_scores.iter().sum::<f32>() / effectiveness_scores.len() as f32
}
/// Calculate anomaly score for current state
fn calculate_anomaly_score(&self) -> Result<f32> {
if self.metrics_buffer.len() < 10 {
return Ok(0.0);
}
let recent_metrics = self.metrics_buffer.last().unwrap();
let historical_metrics = &self.metrics_buffer[..self.metrics_buffer.len() - 1];
let mut anomaly_components = Vec::new();
// Loss anomaly
let historical_losses: Vec<f32> = historical_metrics.iter().map(|m| m.loss).collect();
let loss_mean: f32 = historical_losses.iter().sum::<f32>() / historical_losses.len() as f32;
let loss_std = self.calculate_standard_deviation(&historical_losses, loss_mean);
let loss_z_score = if loss_std > 0.0 {
(recent_metrics.loss - loss_mean) / loss_std
} else {
0.0
};
anomaly_components.push(loss_z_score.abs());
// Gradient norm anomaly
let historical_gradients: Vec<f32> = historical_metrics.iter().map(|m| m.gradient_norm).collect();
let grad_mean: f32 = historical_gradients.iter().sum::<f32>() / historical_gradients.len() as f32;
let grad_std = self.calculate_standard_deviation(&historical_gradients, grad_mean);
let grad_z_score = if grad_std > 0.0 {
(recent_metrics.gradient_norm - grad_mean) / grad_std
} else {
0.0
};
anomaly_components.push(grad_z_score.abs());
// Throughput anomaly
let historical_throughput: Vec<f32> = historical_metrics.iter().map(|m| m.throughput).collect();
let throughput_mean: f32 = historical_throughput.iter().sum::<f32>() / historical_throughput.len() as f32;
let throughput_std = self.calculate_standard_deviation(&historical_throughput, throughput_mean);
let throughput_z_score = if throughput_std > 0.0 {
(recent_metrics.throughput - throughput_mean) / throughput_std
} else {
0.0
};
anomaly_components.push(throughput_z_score.abs());
// Combine anomaly scores (max of 3.0 z-score indicates strong anomaly)
let total_anomaly: f32 = anomaly_components.iter().sum();
let normalized_anomaly = (total_anomaly / 9.0).min(1.0); // Normalize to 0-1
Ok(normalized_anomaly)
}
/// Calculate standard deviation
fn calculate_standard_deviation(&self, values: &[f32], mean: f32) -> f32 {
if values.len() < 2 {
return 0.0;
}
let variance: f32 = values.iter()
.map(|&x| (x - mean).powi(2))
.sum::<f32>() / values.len() as f32;
variance.sqrt()
}
/// Calculate performance indicators
fn calculate_performance_indicators(&self) -> Result<PerformanceIndicators> {
if self.metrics_buffer.is_empty() {
return Ok(PerformanceIndicators {
current_efficiency: 0.0,
resource_utilization_score: 0.0,
training_momentum: 0.0,
convergence_probability: 0.0,
});
}
let recent_metrics = self.metrics_buffer.last().unwrap();
// Current efficiency (samples per second per GPU memory usage)
let current_efficiency = if recent_metrics.gpu_memory_usage > 0.0 {
recent_metrics.throughput / recent_metrics.gpu_memory_usage
} else {
0.0
};
// Resource utilization score (balanced CPU and GPU usage)
let resource_utilization_score = (recent_metrics.cpu_utilization + recent_metrics.gpu_memory_usage) / 2.0;
// Training momentum (improvement rate over recent steps)
let training_momentum = if self.metrics_buffer.len() >= 5 {
let recent_five = &self.metrics_buffer[self.metrics_buffer.len() - 5..];
let initial_loss = recent_five[0].loss;
let final_loss = recent_five[4].loss;
if initial_loss > 0.0 {
((initial_loss - final_loss) / initial_loss).max(0.0)
} else {
0.0
}
} else {
0.0
};
// Convergence probability (based on loss stability and gradient magnitude)
let convergence_probability = if self.metrics_buffer.len() >= 10 {
let recent_losses: Vec<f32> = self.metrics_buffer.iter()
.rev()
.take(10)
.map(|m| m.loss)
.collect();
let loss_volatility = self.calculate_volatility(&recent_losses);
let avg_gradient = self.metrics_buffer.iter()
.rev()
.take(10)
.map(|m| m.gradient_norm)
.sum::<f32>() / 10.0;
// Higher probability if loss is stable and gradients are small
let stability_score = (1.0 - loss_volatility).max(0.0);
let gradient_score = (1.0 / (1.0 + avg_gradient / 10.0)).max(0.0);
(stability_score + gradient_score) / 2.0
} else {
0.0
};
Ok(PerformanceIndicators {
current_efficiency,
resource_utilization_score,
training_momentum,
convergence_probability,
})
}
/// Calculate training health score
fn calculate_training_health(&self) -> Result<TrainingHealthScore> {
if self.metrics_buffer.is_empty() {
return Ok(TrainingHealthScore {
overall_health: 0.0,
gradient_health: 0.0,
loss_health: 0.0,
resource_health: 0.0,
stability_health: 0.0,
});
}
let recent_metrics = self.metrics_buffer.last().unwrap();
// Gradient health (1.0 if gradients are in reasonable range)
let gradient_health = if recent_metrics.gradient_norm > 0.0 && recent_metrics.gradient_norm < 10.0 {
1.0 - (recent_metrics.gradient_norm - 1.0).abs() / 9.0
} else if recent_metrics.gradient_norm >= 10.0 {
0.0 // Exploding gradients
} else {
0.1 // Vanishing gradients
};
// Loss health (based on whether loss is decreasing and stable)
let loss_health = if self.metrics_buffer.len() >= 5 {
let recent_losses: Vec<f32> = self.metrics_buffer.iter()
.rev()
.take(5)
.map(|m| m.loss)
.collect();
let is_decreasing = recent_losses[0] < recent_losses[4];
let volatility = self.calculate_volatility(&recent_losses);
if is_decreasing && volatility < 0.1 {
1.0
} else if is_decreasing {
0.7
} else if volatility < 0.1 {
0.5
} else {
0.2
}
} else {
0.5
};
// Resource health (memory and CPU usage in reasonable ranges)
let memory_health = if recent_metrics.gpu_memory_usage < 0.8 {
1.0
} else if recent_metrics.gpu_memory_usage < 0.9 {
0.7
} else {
0.3
};
let cpu_health = if recent_metrics.cpu_utilization < 0.8 {
1.0
} else if recent_metrics.cpu_utilization < 0.95 {
0.7
} else {
0.3
};
let resource_health = (memory_health + cpu_health) / 2.0;
// Stability health (based on metrics volatility)
let stability_health = if self.metrics_buffer.len() >= 10 {
let recent_losses: Vec<f32> = self.metrics_buffer.iter()
.rev()
.take(10)
.map(|m| m.loss)
.collect();
let recent_gradients: Vec<f32> = self.metrics_buffer.iter()
.rev()
.take(10)
.map(|m| m.gradient_norm)
.collect();
let loss_volatility = self.calculate_volatility(&recent_losses);
let gradient_volatility = self.calculate_volatility(&recent_gradients);
let avg_volatility = (loss_volatility + gradient_volatility) / 2.0;
(1.0 - avg_volatility).max(0.0)
} else {
0.5
};
// Overall health (weighted average)
let overall_health = (gradient_health * 0.3 + loss_health * 0.3 +
resource_health * 0.2 + stability_health * 0.2);
Ok(TrainingHealthScore {
overall_health,
gradient_health,
loss_health,
resource_health,
stability_health,
})
}
/// Generate training predictions
fn generate_predictions(&self) -> Result<TrainingPredictions> {
if self.metrics_buffer.len() < 10 {
return Ok(TrainingPredictions {
estimated_convergence_time: None,
predicted_final_loss: None,
recommended_early_stop: false,
optimization_suggestions: vec!["Collect more data for predictions".to_string()],
});
}
let mut suggestions = Vec::new();
// Analyze recent trends for predictions
let recent_losses: Vec<f32> = self.metrics_buffer.iter()
.rev()
.take(20)
.map(|m| m.loss)
.collect();
let loss_improvement_rate = if recent_losses.len() >= 2 {
(recent_losses[19] - recent_losses[0]) / 19.0 // Change per step
} else {
0.0
};
// Estimate convergence time
let estimated_convergence_time = if loss_improvement_rate < -1e-6 {
let current_loss = recent_losses[0];
let target_loss = current_loss * 0.1; // Assume convergence at 10% of current loss
let steps_needed = ((current_loss - target_loss) / -loss_improvement_rate) as u64;
Some(Duration::from_secs(steps_needed * 10)) // Assume 10 seconds per step
} else {
None
};
// Predict final loss (extrapolate current trend)
let predicted_final_loss = if loss_improvement_rate < 0.0 {
let current_loss = recent_losses[0];
Some((current_loss + loss_improvement_rate * 1000.0).max(0.0)) // Extrapolate 1000 steps
} else {
None
};
// Recommend early stopping
let loss_volatility = self.calculate_volatility(&recent_losses);
let recommended_early_stop = loss_volatility < 0.01 && loss_improvement_rate.abs() < 1e-6;
// Generate optimization suggestions
let recent_metrics = self.metrics_buffer.last().unwrap();
if recent_metrics.gradient_norm > 5.0 {
suggestions.push("Apply gradient clipping to stabilize training".to_string());
}
if recent_metrics.gpu_memory_usage > 0.9 {
suggestions.push("Reduce batch size to prevent memory issues".to_string());
}
if loss_improvement_rate > -1e-6 {
suggestions.push("Consider reducing learning rate for better convergence".to_string());
}
if recent_metrics.throughput < 100.0 {
suggestions.push("Optimize data loading pipeline to improve throughput".to_string());
}
Ok(TrainingPredictions {
estimated_convergence_time,
predicted_final_loss,
recommended_early_stop,
optimization_suggestions: suggestions,
})
}
/// Get recent alerts
pub fn get_recent_alerts(&self, max_alerts: usize) -> Vec<&StreamingAlert> {
self.alerts.iter()
.rev()
.take(max_alerts)
.collect()
}
/// Get latest analysis
pub fn get_latest_analysis(&self) -> Option<&StreamingAnalysis> {
self.analysis_history.last()
}
/// Get streaming analytics summary
pub fn get_summary(&self) -> StreamingAnalyticsSummary {
let latest_analysis = self.get_latest_analysis();
let critical_alerts = self.alerts.iter()
.filter(|a| matches!(a.severity, AlertSeverity::Critical | AlertSeverity::Emergency))
.count();
StreamingAnalyticsSummary {
total_metrics_processed: self.metrics_buffer.len(),
total_alerts_generated: self.alerts.len(),
critical_alerts_count: critical_alerts,
latest_training_health: latest_analysis.map(|a| a.training_health.overall_health).unwrap_or(0.0),
anomaly_score: latest_analysis.map(|a| a.anomaly_score).unwrap_or(0.0),
convergence_probability: latest_analysis
.map(|a| a.performance_indicators.convergence_probability)
.unwrap_or(0.0),
}
}
}
/// Summary of streaming analytics
#[derive(Debug, Serialize, Deserialize)]
pub struct StreamingAnalyticsSummary {
pub total_metrics_processed: usize,
pub total_alerts_generated: usize,
pub critical_alerts_count: usize,
pub latest_training_health: f32,
pub anomaly_score: f32,
pub convergence_probability: f32,
}
// ================================================================================================
// ENHANCED GPU MEMORY DEBUGGING WITH CUDA-SPECIFIC OPTIMIZATIONS
// ================================================================================================
/// Enhanced GPU memory debugger with CUDA-specific optimizations
/// Provides detailed GPU memory analysis, leak detection, and optimization recommendations
#[derive(Debug)]
pub struct EnhancedGpuMemoryDebugger {
pub config: GpuMemoryDebugConfig,
memory_snapshots: Vec<GpuMemorySnapshot>,
allocation_tracker: HashMap<String, GpuAllocation>,
leak_detector: GpuLeakDetector,
optimization_engine: GpuOptimizationEngine,
}
/// Configuration for enhanced GPU memory debugging
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct GpuMemoryDebugConfig {
pub enable_cuda_profiling: bool,
pub enable_leak_detection: bool,
pub enable_fragmentation_analysis: bool,
pub enable_optimization_suggestions: bool,
pub snapshot_interval: Duration,
pub max_snapshots: usize,
pub leak_detection_threshold: usize, // MB
}
impl Default for GpuMemoryDebugConfig {
fn default() -> Self {
Self {
enable_cuda_profiling: true,
enable_leak_detection: true,
enable_fragmentation_analysis: true,
enable_optimization_suggestions: true,
snapshot_interval: Duration::from_secs(30),
max_snapshots: 100,
leak_detection_threshold: 100, // 100 MB
}
}
}
/// GPU memory snapshot with detailed allocation information
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct GpuMemorySnapshot {
pub timestamp: std::time::SystemTime,
pub device_id: u32,
pub total_memory: u64, // bytes
pub used_memory: u64, // bytes
pub free_memory: u64, // bytes
pub cached_memory: u64, // bytes
pub allocated_blocks: Vec<MemoryBlock>,
pub fragmentation_score: f32, // 0.0 to 1.0
pub cuda_context_info: Option<CudaContextInfo>,
}
/// Memory block information
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct MemoryBlock {
pub block_id: String,
pub address: u64,
pub size: u64,
pub allocation_type: AllocationType,
pub allocation_time: std::time::SystemTime,
pub stack_trace: Option<String>,
pub tensor_info: Option<TensorInfo>,
}
#[derive(Debug, Clone, Serialize, Deserialize)]
pub enum AllocationType {
Tensor,
Gradient,
Optimizer,
Cache,
Temporary,
Unknown,
}
/// Tensor information for memory blocks
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct TensorInfo {
pub shape: Vec<usize>,
pub dtype: String,
pub requires_grad: bool,
pub tensor_name: Option<String>,
}
/// CUDA context information
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct CudaContextInfo {
pub cuda_version: String,
pub driver_version: String,
pub compute_capability: String,
pub multiprocessor_count: u32,
pub max_threads_per_block: u32,
pub max_block_dimension: Vec<u32>,
pub max_grid_dimension: Vec<u32>,
pub memory_clock_rate: u32,
pub memory_bus_width: u32,
}
/// GPU allocation tracking
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct GpuAllocation {
pub allocation_id: String,
pub size: u64,
pub allocation_time: std::time::SystemTime,
pub last_access_time: std::time::SystemTime,
pub access_count: u64,
pub allocation_type: AllocationType,
pub stack_trace: Option<String>,
}
/// GPU memory leak detector
#[derive(Debug)]
pub struct GpuLeakDetector {
baseline_memory: u64,
suspicious_allocations: Vec<String>,
leak_patterns: Vec<LeakPattern>,
}
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct LeakPattern {
pub pattern_type: LeakPatternType,
pub description: String,
pub detection_confidence: f32,
pub suggested_fixes: Vec<String>,
}
#[derive(Debug, Clone, Serialize, Deserialize)]
pub enum LeakPatternType {
GradientAccumulation,
CacheGrowth,
TensorRetention,
OptimizerState,
GraphConstruction,
}
/// GPU optimization engine
#[derive(Debug)]
pub struct GpuOptimizationEngine {
optimization_history: Vec<GpuOptimization>,
current_recommendations: Vec<GpuOptimizationRecommendation>,
}
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct GpuOptimization {
pub optimization_id: String,
pub optimization_type: GpuOptimizationType,
pub description: String,
pub estimated_memory_savings: u64,
pub estimated_performance_impact: f32, // -1.0 to 1.0
pub implementation_complexity: ImplementationDifficulty,
pub implementation_steps: Vec<String>,
}
#[derive(Debug, Clone, Serialize, Deserialize)]
pub enum GpuOptimizationType {
MemoryDefragmentation,
AllocationPooling,
GradientCheckpointing,
ModelParallelism,
TensorFusion,
CacheOptimization,
BatchSizeReduction,
MixedPrecision,
}
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct GpuOptimizationRecommendation {
pub recommendation_id: String,
pub priority: OptimizationPriority,
pub optimization: GpuOptimization,
pub current_impact: f32,
pub confidence_score: f32,
}
#[derive(Debug, Clone, Serialize, Deserialize)]
pub enum OptimizationPriority {
Critical,
High,
Medium,
Low,
}
impl EnhancedGpuMemoryDebugger {
/// Create a new enhanced GPU memory debugger
pub fn new(config: GpuMemoryDebugConfig) -> Self {
Self {
config,
memory_snapshots: Vec::new(),
allocation_tracker: HashMap::new(),
leak_detector: GpuLeakDetector {
baseline_memory: 0,
suspicious_allocations: Vec::new(),
leak_patterns: Vec::new(),
},
optimization_engine: GpuOptimizationEngine {
optimization_history: Vec::new(),
current_recommendations: Vec::new(),
},
}
}
/// Take a memory snapshot and analyze GPU state
pub fn take_snapshot(&mut self, device_id: u32) -> Result<GpuMemorySnapshot> {
let snapshot = GpuMemorySnapshot {
timestamp: std::time::SystemTime::now(),
device_id,
total_memory: self.get_total_memory(device_id)?,
used_memory: self.get_used_memory(device_id)?,
free_memory: self.get_free_memory(device_id)?,
cached_memory: self.get_cached_memory(device_id)?,
allocated_blocks: self.get_allocated_blocks(device_id)?,
fragmentation_score: self.calculate_fragmentation_score(device_id)?,
cuda_context_info: self.get_cuda_context_info(device_id)?,
};
self.memory_snapshots.push(snapshot.clone());
// Maintain snapshot history
if self.memory_snapshots.len() > self.config.max_snapshots {
self.memory_snapshots.remove(0);
}
// Analyze for leaks and optimizations
if self.config.enable_leak_detection {
self.detect_memory_leaks(&snapshot)?;
}
if self.config.enable_optimization_suggestions {
self.generate_optimization_recommendations(&snapshot)?;
}
Ok(snapshot)
}
/// Simulate getting total GPU memory (in real implementation, use CUDA APIs)
fn get_total_memory(&self, _device_id: u32) -> Result<u64> {
// In real implementation, use cudaMemGetInfo or similar
Ok(8_589_934_592) // 8 GB
}
/// Simulate getting used GPU memory
fn get_used_memory(&self, _device_id: u32) -> Result<u64> {
// In real implementation, use CUDA memory APIs
Ok(3_221_225_472) // ~3 GB
}
/// Simulate getting free GPU memory
fn get_free_memory(&self, device_id: u32) -> Result<u64> {
let total = self.get_total_memory(device_id)?;
let used = self.get_used_memory(device_id)?;
Ok(total - used)
}
/// Simulate getting cached memory
fn get_cached_memory(&self, _device_id: u32) -> Result<u64> {
// In real implementation, use PyTorch cache APIs or similar
Ok(536_870_912) // 512 MB
}
/// Get allocated memory blocks
fn get_allocated_blocks(&self, _device_id: u32) -> Result<Vec<MemoryBlock>> {
// In real implementation, use memory profiling APIs
let mut blocks = Vec::new();
// Simulate some allocated blocks
for i in 0..10 {
blocks.push(MemoryBlock {
block_id: format!("block_{}", i),
address: 0x7f0000000000 + (i as u64 * 1024 * 1024),
size: 1024 * 1024 * (i as u64 + 1), // Variable sizes
allocation_type: match i % 3 {
0 => AllocationType::Tensor,
1 => AllocationType::Gradient,
_ => AllocationType::Optimizer,
},
allocation_time: std::time::SystemTime::now(),
stack_trace: Some(format!("stack_trace_{}", i)),
tensor_info: Some(TensorInfo {
shape: vec![32, 768, 768],
dtype: "float32".to_string(),
requires_grad: i % 2 == 0,
tensor_name: Some(format!("tensor_{}", i)),
}),
});
}
Ok(blocks)
}
/// Calculate memory fragmentation score
fn calculate_fragmentation_score(&self, device_id: u32) -> Result<f32> {
let free_memory = self.get_free_memory(device_id)?;
let total_memory = self.get_total_memory(device_id)?;
if total_memory == 0 {
return Ok(0.0);
}
// Simulate fragmentation calculation
// In real implementation, analyze free block sizes and distribution
let utilization = 1.0 - (free_memory as f32 / total_memory as f32);
// Higher utilization typically means higher fragmentation
let fragmentation = if utilization > 0.8 {
utilization * 0.7 // High utilization leads to fragmentation
} else {
utilization * 0.3 // Low utilization, low fragmentation
};
Ok(fragmentation.min(1.0))
}
/// Get CUDA context information
fn get_cuda_context_info(&self, _device_id: u32) -> Result<Option<CudaContextInfo>> {
// In real implementation, use CUDA runtime APIs
if !self.config.enable_cuda_profiling {
return Ok(None);
}
Ok(Some(CudaContextInfo {
cuda_version: "11.8".to_string(),
driver_version: "520.61.05".to_string(),
compute_capability: "8.6".to_string(),
multiprocessor_count: 84,
max_threads_per_block: 1024,
max_block_dimension: vec![1024, 1024, 64],
max_grid_dimension: vec![2147483647, 65535, 65535],
memory_clock_rate: 1215000,
memory_bus_width: 384,
}))
}
/// Detect memory leaks in the current snapshot
fn detect_memory_leaks(&mut self, snapshot: &GpuMemorySnapshot) -> Result<()> {
if self.leak_detector.baseline_memory == 0 {
self.leak_detector.baseline_memory = snapshot.used_memory;
return Ok(());
}
let memory_growth = snapshot.used_memory as i64 - self.leak_detector.baseline_memory as i64;
if memory_growth > (self.config.leak_detection_threshold as i64 * 1024 * 1024) {
// Analyze allocation patterns to identify potential leaks
self.analyze_leak_patterns(snapshot)?;
}
Ok(())
}
/// Analyze patterns that might indicate memory leaks
fn analyze_leak_patterns(&mut self, snapshot: &GpuMemorySnapshot) -> Result<()> {
let mut new_patterns = Vec::new();
// Check for gradient accumulation leaks
let gradient_blocks: Vec<&MemoryBlock> = snapshot.allocated_blocks.iter()
.filter(|b| matches!(b.allocation_type, AllocationType::Gradient))
.collect();
if gradient_blocks.len() > 100 {
new_patterns.push(LeakPattern {
pattern_type: LeakPatternType::GradientAccumulation,
description: "Excessive gradient tensors detected".to_string(),
detection_confidence: 0.8,
suggested_fixes: vec![
"Call optimizer.zero_grad() after each backward pass".to_string(),
"Use gradient checkpointing to reduce memory usage".to_string(),
"Consider gradient accumulation with smaller effective batch size".to_string(),
],
});
}
// Check for cache growth
if snapshot.cached_memory > 2_147_483_648 { // 2 GB
new_patterns.push(LeakPattern {
pattern_type: LeakPatternType::CacheGrowth,
description: "GPU cache memory growing beyond reasonable limits".to_string(),
detection_confidence: 0.9,
suggested_fixes: vec![
"Call torch.cuda.empty_cache() periodically".to_string(),
"Reduce model parallelism or batch size".to_string(),
"Use memory-mapped datasets to reduce caching".to_string(),
],
});
}
// Check for tensor retention
let old_tensors: Vec<&MemoryBlock> = snapshot.allocated_blocks.iter()
.filter(|b| {
matches!(b.allocation_type, AllocationType::Tensor) &&
b.allocation_time.elapsed().unwrap_or(Duration::from_secs(0)) > Duration::from_secs(300)
})
.collect();
if old_tensors.len() > 50 {
new_patterns.push(LeakPattern {
pattern_type: LeakPatternType::TensorRetention,
description: "Many long-lived tensors detected".to_string(),
detection_confidence: 0.7,
suggested_fixes: vec![
"Explicitly delete intermediate tensors".to_string(),
"Use context managers for temporary computations".to_string(),
"Avoid storing tensors in global variables".to_string(),
],
});
}
self.leak_detector.leak_patterns.extend(new_patterns);
Ok(())
}
/// Generate optimization recommendations based on current memory state
fn generate_optimization_recommendations(&mut self, snapshot: &GpuMemorySnapshot) -> Result<()> {
let mut recommendations = Vec::new();
// Memory usage is high
if snapshot.used_memory as f32 / snapshot.total_memory as f32 > 0.9 {
recommendations.push(GpuOptimizationRecommendation {
recommendation_id: format!("opt_{}", uuid::Uuid::new_v4()),
priority: OptimizationPriority::Critical,
optimization: GpuOptimization {
optimization_id: format!("opt_{}", uuid::Uuid::new_v4()),
optimization_type: GpuOptimizationType::MemoryDefragmentation,
description: "Memory usage is critically high - defragmentation needed".to_string(),
estimated_memory_savings: snapshot.total_memory / 10, // 10% savings
estimated_performance_impact: -0.1, // Slight performance cost
implementation_complexity: ImplementationDifficulty::Easy,
implementation_steps: vec![
"Call torch.cuda.empty_cache()".to_string(),
"Restart CUDA context if necessary".to_string(),
"Consider reducing batch size".to_string(),
],
},
current_impact: 0.9,
confidence_score: 0.95,
});
}
// High fragmentation
if snapshot.fragmentation_score > 0.6 {
recommendations.push(GpuOptimizationRecommendation {
recommendation_id: format!("opt_{}", uuid::Uuid::new_v4()),
priority: OptimizationPriority::High,
optimization: GpuOptimization {
optimization_id: format!("opt_{}", uuid::Uuid::new_v4()),
optimization_type: GpuOptimizationType::AllocationPooling,
description: "High memory fragmentation detected".to_string(),
estimated_memory_savings: snapshot.cached_memory / 2,
estimated_performance_impact: 0.2, // Performance improvement
implementation_complexity: ImplementationDifficulty::Medium,
implementation_steps: vec![
"Implement memory pooling for allocations".to_string(),
"Pre-allocate memory blocks of common sizes".to_string(),
"Use memory-efficient data structures".to_string(),
],
},
current_impact: snapshot.fragmentation_score,
confidence_score: 0.8,
});
}
// Many gradient allocations
let gradient_count = snapshot.allocated_blocks.iter()
.filter(|b| matches!(b.allocation_type, AllocationType::Gradient))
.count();
if gradient_count > 50 {
recommendations.push(GpuOptimizationRecommendation {
recommendation_id: format!("opt_{}", uuid::Uuid::new_v4()),
priority: OptimizationPriority::Medium,
optimization: GpuOptimization {
optimization_id: format!("opt_{}", uuid::Uuid::new_v4()),
optimization_type: GpuOptimizationType::GradientCheckpointing,
description: "Excessive gradient memory usage detected".to_string(),
estimated_memory_savings: snapshot.used_memory / 3, // ~33% savings
estimated_performance_impact: -0.2, // Some performance cost
implementation_complexity: ImplementationDifficulty::Hard,
implementation_steps: vec![
"Implement gradient checkpointing for transformer layers".to_string(),
"Use activation checkpointing for memory-intensive operations".to_string(),
"Configure checkpointing frequency based on memory constraints".to_string(),
],
},
current_impact: gradient_count as f32 / 100.0,
confidence_score: 0.75,
});
}
self.optimization_engine.current_recommendations = recommendations;
Ok(())
}
/// Get GPU memory analysis report
pub fn get_memory_analysis(&self) -> GpuMemoryAnalysis {
let latest_snapshot = self.memory_snapshots.last();
GpuMemoryAnalysis {
total_snapshots: self.memory_snapshots.len(),
latest_memory_usage: latest_snapshot.map(|s| s.used_memory).unwrap_or(0),
latest_fragmentation: latest_snapshot.map(|s| s.fragmentation_score).unwrap_or(0.0),
detected_leaks: self.leak_detector.leak_patterns.len(),
active_recommendations: self.optimization_engine.current_recommendations.len(),
memory_trend: self.calculate_memory_trend(),
optimization_potential: self.calculate_optimization_potential(),
}
}
/// Calculate memory usage trend
fn calculate_memory_trend(&self) -> MemoryTrend {
if self.memory_snapshots.len() < 5 {
return MemoryTrend::Stable;
}
let recent_snapshots = &self.memory_snapshots[self.memory_snapshots.len() - 5..];
let first_usage = recent_snapshots[0].used_memory as f32;
let last_usage = recent_snapshots[4].used_memory as f32;
let change_ratio = (last_usage - first_usage) / first_usage;
if change_ratio > 0.1 {
MemoryTrend::Increasing
} else if change_ratio < -0.1 {
MemoryTrend::Decreasing
} else {
MemoryTrend::Stable
}
}
/// Calculate optimization potential score
fn calculate_optimization_potential(&self) -> f32 {
let critical_recommendations = self.optimization_engine.current_recommendations.iter()
.filter(|r| matches!(r.priority, OptimizationPriority::Critical))
.count();
let high_recommendations = self.optimization_engine.current_recommendations.iter()
.filter(|r| matches!(r.priority, OptimizationPriority::High))
.count();
let total_estimated_savings: u64 = self.optimization_engine.current_recommendations.iter()
.map(|r| r.optimization.estimated_memory_savings)
.sum();
let latest_total_memory = self.memory_snapshots.last()
.map(|s| s.total_memory)
.unwrap_or(1);
let savings_ratio = total_estimated_savings as f32 / latest_total_memory as f32;
// Combine factors: urgency of recommendations and potential savings
let urgency_score = (critical_recommendations * 3 + high_recommendations * 2) as f32 / 10.0;
let savings_score = savings_ratio * 2.0; // Scale up savings importance
((urgency_score + savings_score) / 2.0).min(1.0)
}
/// Get optimization recommendations
pub fn get_recommendations(&self) -> &[GpuOptimizationRecommendation] {
&self.optimization_engine.current_recommendations
}
/// Get detected leak patterns
pub fn get_leak_patterns(&self) -> &[LeakPattern] {
&self.leak_detector.leak_patterns
}
}
/// GPU memory analysis summary
#[derive(Debug, Serialize, Deserialize)]
pub struct GpuMemoryAnalysis {
pub total_snapshots: usize,
pub latest_memory_usage: u64,
pub latest_fragmentation: f32,
pub detected_leaks: usize,
pub active_recommendations: usize,
pub memory_trend: MemoryTrend,
pub optimization_potential: f32,
}
#[derive(Debug, Clone, Serialize, Deserialize)]
pub enum MemoryTrend {
Increasing,
Stable,
Decreasing,
}