Skip to main content

trustformers_models/memory_profiling/
analytics.rs

1//! Advanced memory analytics components
2//!
3//! This module contains sophisticated analytics components for memory analysis
4//! including adaptive thresholds, leak detection, and memory prediction.
5
6use super::types::MemoryMetrics;
7use serde::{Deserialize, Serialize};
8use std::time::{Duration, SystemTime};
9
10/// Pre-allocated recommendations to reduce string allocations in hot paths
11#[derive(Debug, Clone)]
12pub struct AlertRecommendations {
13    pub high_memory: Vec<String>,
14    pub rapid_growth: Vec<String>,
15    pub fragmentation: Vec<String>,
16    pub memory_leak: Vec<String>,
17    pub gc_pressure: Vec<String>,
18}
19
20impl Default for AlertRecommendations {
21    fn default() -> Self {
22        Self::new()
23    }
24}
25
26impl AlertRecommendations {
27    pub fn new() -> Self {
28        Self {
29            high_memory: vec![
30                "Consider reducing batch size".to_string(),
31                "Enable gradient checkpointing".to_string(),
32                "Use mixed precision training".to_string(),
33                "Implement tensor sharding for large models".to_string(),
34            ],
35            rapid_growth: vec![
36                "Check for memory leaks".to_string(),
37                "Review tensor lifetime management".to_string(),
38                "Consider explicit garbage collection".to_string(),
39                "Monitor unclosed file handles and resources".to_string(),
40            ],
41            fragmentation: vec![
42                "Restart the process to defragment memory".to_string(),
43                "Use memory pools for frequent allocations".to_string(),
44                "Implement custom allocators for large tensors".to_string(),
45            ],
46            memory_leak: vec![
47                "Use weak references for circular dependencies".to_string(),
48                "Implement explicit cleanup for model components".to_string(),
49                "Review closure captures for tensor references".to_string(),
50                "Consider using RAII patterns for resource management".to_string(),
51            ],
52            gc_pressure: vec![
53                "Increase heap size if possible".to_string(),
54                "Reduce allocation frequency in hot paths".to_string(),
55                "Implement object pooling for frequent allocations".to_string(),
56                "Use pre-allocated buffers for intermediate results".to_string(),
57            ],
58        }
59    }
60}
61
62/// Adaptive threshold system that adjusts based on model characteristics
63#[derive(Debug, Clone, Serialize, Deserialize)]
64pub struct AdaptiveThresholds {
65    pub base_memory_threshold: f64,
66    pub growth_rate_threshold: f64,
67    pub fragmentation_threshold: f64,
68    pub adaptation_factor: f64,
69    pub last_updated: SystemTime,
70}
71
72impl Default for AdaptiveThresholds {
73    fn default() -> Self {
74        Self {
75            base_memory_threshold: 1024.0,
76            growth_rate_threshold: 50.0,
77            fragmentation_threshold: 0.3,
78            adaptation_factor: 0.1,
79            last_updated: SystemTime::now(),
80        }
81    }
82}
83
84impl AdaptiveThresholds {
85    /// Update thresholds based on current memory usage patterns
86    pub fn update_thresholds(&mut self, recent_metrics: &[MemoryMetrics]) {
87        if recent_metrics.is_empty() {
88            return;
89        }
90
91        let avg_memory = recent_metrics.iter().map(|m| m.total_memory_mb).sum::<f64>()
92            / recent_metrics.len() as f64;
93
94        let avg_growth =
95            recent_metrics.iter().map(|m| m.memory_growth_rate_mb_per_sec).sum::<f64>()
96                / recent_metrics.len() as f64;
97
98        let avg_fragmentation =
99            recent_metrics.iter().map(|m| m.memory_fragmentation_ratio).sum::<f64>()
100                / recent_metrics.len() as f64;
101
102        // Adaptive adjustment
103        self.base_memory_threshold = self.base_memory_threshold * (1.0 - self.adaptation_factor)
104            + avg_memory * 1.2 * self.adaptation_factor;
105        self.growth_rate_threshold = self.growth_rate_threshold * (1.0 - self.adaptation_factor)
106            + avg_growth * 2.0 * self.adaptation_factor;
107        self.fragmentation_threshold = self.fragmentation_threshold
108            * (1.0 - self.adaptation_factor)
109            + avg_fragmentation * 1.5 * self.adaptation_factor;
110
111        self.last_updated = SystemTime::now();
112    }
113}
114
115/// Memory leak detection heuristics
116#[derive(Debug, Clone)]
117pub struct LeakDetectionHeuristics {
118    pub sustained_growth_threshold: f64,
119    pub growth_duration_threshold: Duration,
120    pub allocation_pattern_threshold: usize,
121    pub false_positive_filter: f64,
122}
123
124impl Default for LeakDetectionHeuristics {
125    fn default() -> Self {
126        Self {
127            sustained_growth_threshold: 10.0,                    // MB/sec
128            growth_duration_threshold: Duration::from_secs(300), // 5 minutes
129            allocation_pattern_threshold: 1000,
130            false_positive_filter: 0.8, // Confidence threshold
131        }
132    }
133}
134
135impl LeakDetectionHeuristics {
136    /// Analyze memory patterns for potential leaks
137    pub fn detect_potential_leaks(&self, metrics: &[MemoryMetrics]) -> Vec<LeakAlert> {
138        let mut alerts = Vec::new();
139
140        if metrics.len() < 10 {
141            return alerts; // Not enough data
142        }
143
144        // Check for sustained growth
145        let recent_metrics = &metrics[metrics.len().saturating_sub(60)..];
146        let growth_rates: Vec<f64> =
147            recent_metrics.iter().map(|m| m.memory_growth_rate_mb_per_sec).collect();
148
149        let avg_growth = growth_rates.iter().sum::<f64>() / growth_rates.len() as f64;
150
151        if avg_growth > self.sustained_growth_threshold {
152            let confidence = self.calculate_leak_confidence(&growth_rates);
153            if confidence > self.false_positive_filter {
154                alerts.push(LeakAlert {
155                    alert_type: LeakAlertType::SustainedGrowth,
156                    confidence,
157                    description: format!(
158                        "Sustained memory growth detected: {:.2} MB/sec",
159                        avg_growth
160                    ),
161                });
162            }
163        }
164
165        // Check for allocation patterns
166        let total_allocations: u64 = recent_metrics.iter().map(|m| m.allocated_objects).sum();
167        let total_deallocations: u64 = recent_metrics.iter().map(|m| m.deallocated_objects).sum();
168
169        if total_allocations > total_deallocations + self.allocation_pattern_threshold as u64 {
170            let imbalance_ratio = total_allocations as f64 / (total_deallocations + 1) as f64;
171            if imbalance_ratio > 1.2 {
172                alerts.push(LeakAlert {
173                    alert_type: LeakAlertType::AllocationImbalance,
174                    confidence: (imbalance_ratio - 1.0).min(1.0),
175                    description: format!(
176                        "Allocation/deallocation imbalance: ratio {:.2}",
177                        imbalance_ratio
178                    ),
179                });
180            }
181        }
182
183        alerts
184    }
185
186    fn calculate_leak_confidence(&self, growth_rates: &[f64]) -> f64 {
187        if growth_rates.is_empty() {
188            return 0.0;
189        }
190
191        // Calculate consistency of growth (low variance = higher confidence)
192        let mean = growth_rates.iter().sum::<f64>() / growth_rates.len() as f64;
193        let variance = growth_rates.iter().map(|x| (x - mean).powi(2)).sum::<f64>()
194            / growth_rates.len() as f64;
195
196        let consistency = 1.0 / (1.0 + variance);
197        let magnitude = (mean / self.sustained_growth_threshold).min(1.0);
198
199        (consistency * magnitude).min(1.0)
200    }
201}
202
203#[derive(Debug, Clone)]
204pub struct LeakAlert {
205    pub alert_type: LeakAlertType,
206    pub confidence: f64,
207    pub description: String,
208}
209
210#[derive(Debug, Clone)]
211pub enum LeakAlertType {
212    SustainedGrowth,
213    AllocationImbalance,
214    PatternAnomaly,
215}
216
217/// Memory usage predictor using trend analysis
218#[derive(Debug, Clone)]
219pub struct MemoryPredictor {
220    trend_window: usize,
221    prediction_horizon_secs: u64,
222    confidence_threshold: f64,
223    linear_regression_cache: Option<LinearRegression>,
224}
225
226impl Default for MemoryPredictor {
227    fn default() -> Self {
228        Self {
229            trend_window: 60,             // Last 60 data points
230            prediction_horizon_secs: 300, // 5 minutes ahead
231            confidence_threshold: 0.7,
232            linear_regression_cache: None,
233        }
234    }
235}
236
237impl MemoryPredictor {
238    /// Predict future memory usage based on current trends
239    pub fn predict_memory_usage(
240        &mut self,
241        metrics: &[MemoryMetrics],
242        horizon_secs: Option<u64>,
243    ) -> Option<MemoryPrediction> {
244        if metrics.len() < self.trend_window {
245            return None;
246        }
247
248        let recent_metrics = &metrics[metrics.len() - self.trend_window..];
249        let horizon = horizon_secs.unwrap_or(self.prediction_horizon_secs);
250
251        // Convert to time series data
252        let data_points: Vec<(f64, f64)> = recent_metrics
253            .iter()
254            .enumerate()
255            .map(|(i, m)| (i as f64, m.total_memory_mb))
256            .collect();
257
258        let regression = self.calculate_linear_regression(&data_points);
259        let confidence = regression.correlation.abs();
260
261        if confidence < self.confidence_threshold {
262            return None; // Low confidence prediction
263        }
264
265        let future_time = data_points.len() as f64 + (horizon as f64 / 60.0); // Assuming 1-minute intervals
266        let predicted_memory = regression.slope * future_time + regression.intercept;
267
268        self.linear_regression_cache = Some(regression.clone());
269
270        Some(MemoryPrediction {
271            predicted_memory_mb: predicted_memory,
272            confidence,
273            horizon_secs: horizon,
274            trend_slope: regression.slope,
275        })
276    }
277
278    fn calculate_linear_regression(&self, data: &[(f64, f64)]) -> LinearRegression {
279        let n = data.len() as f64;
280        let sum_x: f64 = data.iter().map(|(x, _)| x).sum();
281        let sum_y: f64 = data.iter().map(|(_, y)| y).sum();
282        let sum_xy: f64 = data.iter().map(|(x, y)| x * y).sum();
283        let sum_x2: f64 = data.iter().map(|(x, _)| x * x).sum();
284        let sum_y2: f64 = data.iter().map(|(_, y)| y * y).sum();
285
286        let slope = (n * sum_xy - sum_x * sum_y) / (n * sum_x2 - sum_x * sum_x);
287        let intercept = (sum_y - slope * sum_x) / n;
288
289        // Calculate correlation coefficient
290        let numerator = n * sum_xy - sum_x * sum_y;
291        let denominator = ((n * sum_x2 - sum_x * sum_x) * (n * sum_y2 - sum_y * sum_y)).sqrt();
292        let correlation = if denominator != 0.0 { numerator / denominator } else { 0.0 };
293
294        LinearRegression {
295            slope,
296            intercept,
297            correlation,
298            last_computed: SystemTime::now(),
299        }
300    }
301}
302
303/// Simple linear regression for memory prediction
304#[derive(Debug, Clone)]
305pub struct LinearRegression {
306    pub slope: f64,
307    pub intercept: f64,
308    pub correlation: f64,
309    pub last_computed: SystemTime,
310}
311
312#[derive(Debug, Clone)]
313pub struct MemoryPrediction {
314    pub predicted_memory_mb: f64,
315    pub confidence: f64,
316    pub horizon_secs: u64,
317    pub trend_slope: f64,
318}
319
320/// Advanced statistical analyzer for memory patterns
321#[derive(Debug, Clone)]
322pub struct StatisticalAnalyzer {
323    #[allow(dead_code)]
324    confidence_interval: f64,
325    outlier_threshold: f64,
326}
327
328impl StatisticalAnalyzer {
329    pub fn new(confidence_interval: f64) -> Self {
330        Self {
331            confidence_interval,
332            outlier_threshold: 2.0, // 2 standard deviations
333        }
334    }
335
336    /// Calculate memory usage statistics with confidence intervals
337    pub fn calculate_usage_statistics(&self, metrics: &[MemoryMetrics]) -> MemoryStatistics {
338        if metrics.is_empty() {
339            return MemoryStatistics::default();
340        }
341
342        let memory_values: Vec<f64> = metrics.iter().map(|m| m.total_memory_mb).collect();
343
344        let mean = self.calculate_mean(&memory_values);
345        let std_dev = self.calculate_std_dev(&memory_values, mean);
346        let median = self.calculate_median(&memory_values);
347        let (p25, p75) = self.calculate_quartiles(&memory_values);
348
349        let confidence_lower = mean - (1.96 * std_dev / (memory_values.len() as f64).sqrt());
350        let confidence_upper = mean + (1.96 * std_dev / (memory_values.len() as f64).sqrt());
351
352        let outliers = self.detect_outliers(&memory_values, mean, std_dev);
353        let trend = self.calculate_trend(&memory_values);
354
355        MemoryStatistics {
356            mean,
357            median,
358            std_dev,
359            min: memory_values.iter().fold(f64::INFINITY, |a, &b| a.min(b)),
360            max: memory_values.iter().fold(f64::NEG_INFINITY, |a, &b| a.max(b)),
361            percentile_25: p25,
362            percentile_75: p75,
363            confidence_interval_lower: confidence_lower,
364            confidence_interval_upper: confidence_upper,
365            outlier_count: outliers.len(),
366            trend_slope: trend,
367            coefficient_of_variation: if mean != 0.0 { std_dev / mean } else { 0.0 },
368        }
369    }
370
371    /// Detect memory usage anomalies using statistical methods
372    pub fn detect_anomalies(&self, metrics: &[MemoryMetrics]) -> Vec<MemoryAnomaly> {
373        let mut anomalies = Vec::new();
374
375        if metrics.len() < 10 {
376            return anomalies; // Need sufficient data for anomaly detection
377        }
378
379        let memory_values: Vec<f64> = metrics.iter().map(|m| m.total_memory_mb).collect();
380        let mean = self.calculate_mean(&memory_values);
381        let std_dev = self.calculate_std_dev(&memory_values, mean);
382
383        // Detect sudden spikes
384        for (i, &value) in memory_values.iter().enumerate() {
385            let z_score = (value - mean).abs() / std_dev;
386
387            if z_score > self.outlier_threshold {
388                anomalies.push(MemoryAnomaly {
389                    timestamp: metrics[i].timestamp,
390                    anomaly_type: if value > mean {
391                        AnomalyType::SuddenSpike
392                    } else {
393                        AnomalyType::SuddenDrop
394                    },
395                    severity: if z_score > 3.0 {
396                        AnomalySeverity::High
397                    } else {
398                        AnomalySeverity::Medium
399                    },
400                    value,
401                    expected_value: mean,
402                    confidence_score: ((z_score - self.outlier_threshold) / self.outlier_threshold).min(1.0),
403                    description: format!("Memory usage {} ({}MB) deviates significantly from expected value ({}MB). Z-score: {:.2}",
404                                       if value > mean { "spike" } else { "drop" },
405                                       value, mean, z_score),
406                });
407            }
408        }
409
410        // Detect sustained growth patterns
411        if let Some(growth_anomaly) = self.detect_sustained_growth(&memory_values, metrics) {
412            anomalies.push(growth_anomaly);
413        }
414
415        anomalies
416    }
417
418    fn calculate_mean(&self, values: &[f64]) -> f64 {
419        values.iter().sum::<f64>() / values.len() as f64
420    }
421
422    fn calculate_std_dev(&self, values: &[f64], mean: f64) -> f64 {
423        let variance =
424            values.iter().map(|&x| (x - mean).powi(2)).sum::<f64>() / values.len() as f64;
425        variance.sqrt()
426    }
427
428    fn calculate_median(&self, values: &[f64]) -> f64 {
429        let mut sorted = values.to_vec();
430        sorted.sort_by(|a, b| a.partial_cmp(b).unwrap_or(std::cmp::Ordering::Equal));
431        let mid = sorted.len() / 2;
432        if sorted.len().is_multiple_of(2) {
433            (sorted[mid - 1] + sorted[mid]) / 2.0
434        } else {
435            sorted[mid]
436        }
437    }
438
439    fn calculate_quartiles(&self, values: &[f64]) -> (f64, f64) {
440        let mut sorted = values.to_vec();
441        sorted.sort_by(|a, b| a.partial_cmp(b).unwrap_or(std::cmp::Ordering::Equal));
442        let n = sorted.len();
443        let q1_idx = n / 4;
444        let q3_idx = 3 * n / 4;
445        (sorted[q1_idx], sorted[q3_idx])
446    }
447
448    fn detect_outliers(&self, values: &[f64], mean: f64, std_dev: f64) -> Vec<f64> {
449        values
450            .iter()
451            .filter(|&&x| (x - mean).abs() / std_dev > self.outlier_threshold)
452            .copied()
453            .collect()
454    }
455
456    fn calculate_trend(&self, values: &[f64]) -> f64 {
457        if values.len() < 2 {
458            return 0.0;
459        }
460
461        let n = values.len() as f64;
462        let x_mean = (n - 1.0) / 2.0;
463        let y_mean = self.calculate_mean(values);
464
465        let mut numerator = 0.0;
466        let mut denominator = 0.0;
467
468        for (i, &y) in values.iter().enumerate() {
469            let x = i as f64;
470            numerator += (x - x_mean) * (y - y_mean);
471            denominator += (x - x_mean).powi(2);
472        }
473
474        if denominator != 0.0 {
475            numerator / denominator
476        } else {
477            0.0
478        }
479    }
480
481    fn detect_sustained_growth(
482        &self,
483        values: &[f64],
484        metrics: &[MemoryMetrics],
485    ) -> Option<MemoryAnomaly> {
486        if values.len() < 20 {
487            return None;
488        }
489
490        let recent_window = &values[values.len() - 10..];
491        let earlier_window = &values[values.len() - 20..values.len() - 10];
492
493        let recent_mean = self.calculate_mean(recent_window);
494        let earlier_mean = self.calculate_mean(earlier_window);
495
496        let growth_rate = (recent_mean - earlier_mean) / earlier_mean;
497
498        if growth_rate > 0.2 {
499            // 20% growth is considered anomalous
500            Some(MemoryAnomaly {
501                timestamp: metrics[metrics.len() - 1].timestamp,
502                anomaly_type: AnomalyType::SustainedGrowth,
503                severity: if growth_rate > 0.5 {
504                    AnomalySeverity::High
505                } else {
506                    AnomalySeverity::Medium
507                },
508                value: recent_mean,
509                expected_value: earlier_mean,
510                confidence_score: (growth_rate / 0.5).min(1.0),
511                description: format!(
512                    "Sustained memory growth detected: {:.1}% increase over recent observations",
513                    growth_rate * 100.0
514                ),
515            })
516        } else {
517            None
518        }
519    }
520}
521
522/// Memory usage statistics
523#[derive(Debug, Clone, Serialize, Deserialize)]
524pub struct MemoryStatistics {
525    pub mean: f64,
526    pub median: f64,
527    pub std_dev: f64,
528    pub min: f64,
529    pub max: f64,
530    pub percentile_25: f64,
531    pub percentile_75: f64,
532    pub confidence_interval_lower: f64,
533    pub confidence_interval_upper: f64,
534    pub outlier_count: usize,
535    pub trend_slope: f64,
536    pub coefficient_of_variation: f64,
537}
538
539impl Default for MemoryStatistics {
540    fn default() -> Self {
541        Self {
542            mean: 0.0,
543            median: 0.0,
544            std_dev: 0.0,
545            min: 0.0,
546            max: 0.0,
547            percentile_25: 0.0,
548            percentile_75: 0.0,
549            confidence_interval_lower: 0.0,
550            confidence_interval_upper: 0.0,
551            outlier_count: 0,
552            trend_slope: 0.0,
553            coefficient_of_variation: 0.0,
554        }
555    }
556}
557
558/// Memory anomaly detection
559#[derive(Debug, Clone, Serialize, Deserialize)]
560pub struct MemoryAnomaly {
561    pub timestamp: SystemTime,
562    pub anomaly_type: AnomalyType,
563    pub severity: AnomalySeverity,
564    pub value: f64,
565    pub expected_value: f64,
566    pub confidence_score: f64,
567    pub description: String,
568}
569
570#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
571pub enum AnomalyType {
572    SuddenSpike,
573    SuddenDrop,
574    SustainedGrowth,
575    UnusualPattern,
576}
577
578#[derive(Debug, Clone, Serialize, Deserialize)]
579pub enum AnomalySeverity {
580    Low,
581    Medium,
582    High,
583    Critical,
584}