1use super::types::MemoryMetrics;
7use serde::{Deserialize, Serialize};
8use std::time::{Duration, SystemTime};
9
10#[derive(Debug, Clone)]
12pub struct AlertRecommendations {
13 pub high_memory: Vec<String>,
14 pub rapid_growth: Vec<String>,
15 pub fragmentation: Vec<String>,
16 pub memory_leak: Vec<String>,
17 pub gc_pressure: Vec<String>,
18}
19
20impl Default for AlertRecommendations {
21 fn default() -> Self {
22 Self::new()
23 }
24}
25
26impl AlertRecommendations {
27 pub fn new() -> Self {
28 Self {
29 high_memory: vec![
30 "Consider reducing batch size".to_string(),
31 "Enable gradient checkpointing".to_string(),
32 "Use mixed precision training".to_string(),
33 "Implement tensor sharding for large models".to_string(),
34 ],
35 rapid_growth: vec![
36 "Check for memory leaks".to_string(),
37 "Review tensor lifetime management".to_string(),
38 "Consider explicit garbage collection".to_string(),
39 "Monitor unclosed file handles and resources".to_string(),
40 ],
41 fragmentation: vec![
42 "Restart the process to defragment memory".to_string(),
43 "Use memory pools for frequent allocations".to_string(),
44 "Implement custom allocators for large tensors".to_string(),
45 ],
46 memory_leak: vec![
47 "Use weak references for circular dependencies".to_string(),
48 "Implement explicit cleanup for model components".to_string(),
49 "Review closure captures for tensor references".to_string(),
50 "Consider using RAII patterns for resource management".to_string(),
51 ],
52 gc_pressure: vec![
53 "Increase heap size if possible".to_string(),
54 "Reduce allocation frequency in hot paths".to_string(),
55 "Implement object pooling for frequent allocations".to_string(),
56 "Use pre-allocated buffers for intermediate results".to_string(),
57 ],
58 }
59 }
60}
61
62#[derive(Debug, Clone, Serialize, Deserialize)]
64pub struct AdaptiveThresholds {
65 pub base_memory_threshold: f64,
66 pub growth_rate_threshold: f64,
67 pub fragmentation_threshold: f64,
68 pub adaptation_factor: f64,
69 pub last_updated: SystemTime,
70}
71
72impl Default for AdaptiveThresholds {
73 fn default() -> Self {
74 Self {
75 base_memory_threshold: 1024.0,
76 growth_rate_threshold: 50.0,
77 fragmentation_threshold: 0.3,
78 adaptation_factor: 0.1,
79 last_updated: SystemTime::now(),
80 }
81 }
82}
83
84impl AdaptiveThresholds {
85 pub fn update_thresholds(&mut self, recent_metrics: &[MemoryMetrics]) {
87 if recent_metrics.is_empty() {
88 return;
89 }
90
91 let avg_memory = recent_metrics.iter().map(|m| m.total_memory_mb).sum::<f64>()
92 / recent_metrics.len() as f64;
93
94 let avg_growth =
95 recent_metrics.iter().map(|m| m.memory_growth_rate_mb_per_sec).sum::<f64>()
96 / recent_metrics.len() as f64;
97
98 let avg_fragmentation =
99 recent_metrics.iter().map(|m| m.memory_fragmentation_ratio).sum::<f64>()
100 / recent_metrics.len() as f64;
101
102 self.base_memory_threshold = self.base_memory_threshold * (1.0 - self.adaptation_factor)
104 + avg_memory * 1.2 * self.adaptation_factor;
105 self.growth_rate_threshold = self.growth_rate_threshold * (1.0 - self.adaptation_factor)
106 + avg_growth * 2.0 * self.adaptation_factor;
107 self.fragmentation_threshold = self.fragmentation_threshold
108 * (1.0 - self.adaptation_factor)
109 + avg_fragmentation * 1.5 * self.adaptation_factor;
110
111 self.last_updated = SystemTime::now();
112 }
113}
114
115#[derive(Debug, Clone)]
117pub struct LeakDetectionHeuristics {
118 pub sustained_growth_threshold: f64,
119 pub growth_duration_threshold: Duration,
120 pub allocation_pattern_threshold: usize,
121 pub false_positive_filter: f64,
122}
123
124impl Default for LeakDetectionHeuristics {
125 fn default() -> Self {
126 Self {
127 sustained_growth_threshold: 10.0, growth_duration_threshold: Duration::from_secs(300), allocation_pattern_threshold: 1000,
130 false_positive_filter: 0.8, }
132 }
133}
134
135impl LeakDetectionHeuristics {
136 pub fn detect_potential_leaks(&self, metrics: &[MemoryMetrics]) -> Vec<LeakAlert> {
138 let mut alerts = Vec::new();
139
140 if metrics.len() < 10 {
141 return alerts; }
143
144 let recent_metrics = &metrics[metrics.len().saturating_sub(60)..];
146 let growth_rates: Vec<f64> =
147 recent_metrics.iter().map(|m| m.memory_growth_rate_mb_per_sec).collect();
148
149 let avg_growth = growth_rates.iter().sum::<f64>() / growth_rates.len() as f64;
150
151 if avg_growth > self.sustained_growth_threshold {
152 let confidence = self.calculate_leak_confidence(&growth_rates);
153 if confidence > self.false_positive_filter {
154 alerts.push(LeakAlert {
155 alert_type: LeakAlertType::SustainedGrowth,
156 confidence,
157 description: format!(
158 "Sustained memory growth detected: {:.2} MB/sec",
159 avg_growth
160 ),
161 });
162 }
163 }
164
165 let total_allocations: u64 = recent_metrics.iter().map(|m| m.allocated_objects).sum();
167 let total_deallocations: u64 = recent_metrics.iter().map(|m| m.deallocated_objects).sum();
168
169 if total_allocations > total_deallocations + self.allocation_pattern_threshold as u64 {
170 let imbalance_ratio = total_allocations as f64 / (total_deallocations + 1) as f64;
171 if imbalance_ratio > 1.2 {
172 alerts.push(LeakAlert {
173 alert_type: LeakAlertType::AllocationImbalance,
174 confidence: (imbalance_ratio - 1.0).min(1.0),
175 description: format!(
176 "Allocation/deallocation imbalance: ratio {:.2}",
177 imbalance_ratio
178 ),
179 });
180 }
181 }
182
183 alerts
184 }
185
186 fn calculate_leak_confidence(&self, growth_rates: &[f64]) -> f64 {
187 if growth_rates.is_empty() {
188 return 0.0;
189 }
190
191 let mean = growth_rates.iter().sum::<f64>() / growth_rates.len() as f64;
193 let variance = growth_rates.iter().map(|x| (x - mean).powi(2)).sum::<f64>()
194 / growth_rates.len() as f64;
195
196 let consistency = 1.0 / (1.0 + variance);
197 let magnitude = (mean / self.sustained_growth_threshold).min(1.0);
198
199 (consistency * magnitude).min(1.0)
200 }
201}
202
203#[derive(Debug, Clone)]
204pub struct LeakAlert {
205 pub alert_type: LeakAlertType,
206 pub confidence: f64,
207 pub description: String,
208}
209
210#[derive(Debug, Clone)]
211pub enum LeakAlertType {
212 SustainedGrowth,
213 AllocationImbalance,
214 PatternAnomaly,
215}
216
217#[derive(Debug, Clone)]
219pub struct MemoryPredictor {
220 trend_window: usize,
221 prediction_horizon_secs: u64,
222 confidence_threshold: f64,
223 linear_regression_cache: Option<LinearRegression>,
224}
225
226impl Default for MemoryPredictor {
227 fn default() -> Self {
228 Self {
229 trend_window: 60, prediction_horizon_secs: 300, confidence_threshold: 0.7,
232 linear_regression_cache: None,
233 }
234 }
235}
236
237impl MemoryPredictor {
238 pub fn predict_memory_usage(
240 &mut self,
241 metrics: &[MemoryMetrics],
242 horizon_secs: Option<u64>,
243 ) -> Option<MemoryPrediction> {
244 if metrics.len() < self.trend_window {
245 return None;
246 }
247
248 let recent_metrics = &metrics[metrics.len() - self.trend_window..];
249 let horizon = horizon_secs.unwrap_or(self.prediction_horizon_secs);
250
251 let data_points: Vec<(f64, f64)> = recent_metrics
253 .iter()
254 .enumerate()
255 .map(|(i, m)| (i as f64, m.total_memory_mb))
256 .collect();
257
258 let regression = self.calculate_linear_regression(&data_points);
259 let confidence = regression.correlation.abs();
260
261 if confidence < self.confidence_threshold {
262 return None; }
264
265 let future_time = data_points.len() as f64 + (horizon as f64 / 60.0); let predicted_memory = regression.slope * future_time + regression.intercept;
267
268 self.linear_regression_cache = Some(regression.clone());
269
270 Some(MemoryPrediction {
271 predicted_memory_mb: predicted_memory,
272 confidence,
273 horizon_secs: horizon,
274 trend_slope: regression.slope,
275 })
276 }
277
278 fn calculate_linear_regression(&self, data: &[(f64, f64)]) -> LinearRegression {
279 let n = data.len() as f64;
280 let sum_x: f64 = data.iter().map(|(x, _)| x).sum();
281 let sum_y: f64 = data.iter().map(|(_, y)| y).sum();
282 let sum_xy: f64 = data.iter().map(|(x, y)| x * y).sum();
283 let sum_x2: f64 = data.iter().map(|(x, _)| x * x).sum();
284 let sum_y2: f64 = data.iter().map(|(_, y)| y * y).sum();
285
286 let slope = (n * sum_xy - sum_x * sum_y) / (n * sum_x2 - sum_x * sum_x);
287 let intercept = (sum_y - slope * sum_x) / n;
288
289 let numerator = n * sum_xy - sum_x * sum_y;
291 let denominator = ((n * sum_x2 - sum_x * sum_x) * (n * sum_y2 - sum_y * sum_y)).sqrt();
292 let correlation = if denominator != 0.0 { numerator / denominator } else { 0.0 };
293
294 LinearRegression {
295 slope,
296 intercept,
297 correlation,
298 last_computed: SystemTime::now(),
299 }
300 }
301}
302
303#[derive(Debug, Clone)]
305pub struct LinearRegression {
306 pub slope: f64,
307 pub intercept: f64,
308 pub correlation: f64,
309 pub last_computed: SystemTime,
310}
311
312#[derive(Debug, Clone)]
313pub struct MemoryPrediction {
314 pub predicted_memory_mb: f64,
315 pub confidence: f64,
316 pub horizon_secs: u64,
317 pub trend_slope: f64,
318}
319
320#[derive(Debug, Clone)]
322pub struct StatisticalAnalyzer {
323 #[allow(dead_code)]
324 confidence_interval: f64,
325 outlier_threshold: f64,
326}
327
328impl StatisticalAnalyzer {
329 pub fn new(confidence_interval: f64) -> Self {
330 Self {
331 confidence_interval,
332 outlier_threshold: 2.0, }
334 }
335
336 pub fn calculate_usage_statistics(&self, metrics: &[MemoryMetrics]) -> MemoryStatistics {
338 if metrics.is_empty() {
339 return MemoryStatistics::default();
340 }
341
342 let memory_values: Vec<f64> = metrics.iter().map(|m| m.total_memory_mb).collect();
343
344 let mean = self.calculate_mean(&memory_values);
345 let std_dev = self.calculate_std_dev(&memory_values, mean);
346 let median = self.calculate_median(&memory_values);
347 let (p25, p75) = self.calculate_quartiles(&memory_values);
348
349 let confidence_lower = mean - (1.96 * std_dev / (memory_values.len() as f64).sqrt());
350 let confidence_upper = mean + (1.96 * std_dev / (memory_values.len() as f64).sqrt());
351
352 let outliers = self.detect_outliers(&memory_values, mean, std_dev);
353 let trend = self.calculate_trend(&memory_values);
354
355 MemoryStatistics {
356 mean,
357 median,
358 std_dev,
359 min: memory_values.iter().fold(f64::INFINITY, |a, &b| a.min(b)),
360 max: memory_values.iter().fold(f64::NEG_INFINITY, |a, &b| a.max(b)),
361 percentile_25: p25,
362 percentile_75: p75,
363 confidence_interval_lower: confidence_lower,
364 confidence_interval_upper: confidence_upper,
365 outlier_count: outliers.len(),
366 trend_slope: trend,
367 coefficient_of_variation: if mean != 0.0 { std_dev / mean } else { 0.0 },
368 }
369 }
370
371 pub fn detect_anomalies(&self, metrics: &[MemoryMetrics]) -> Vec<MemoryAnomaly> {
373 let mut anomalies = Vec::new();
374
375 if metrics.len() < 10 {
376 return anomalies; }
378
379 let memory_values: Vec<f64> = metrics.iter().map(|m| m.total_memory_mb).collect();
380 let mean = self.calculate_mean(&memory_values);
381 let std_dev = self.calculate_std_dev(&memory_values, mean);
382
383 for (i, &value) in memory_values.iter().enumerate() {
385 let z_score = (value - mean).abs() / std_dev;
386
387 if z_score > self.outlier_threshold {
388 anomalies.push(MemoryAnomaly {
389 timestamp: metrics[i].timestamp,
390 anomaly_type: if value > mean {
391 AnomalyType::SuddenSpike
392 } else {
393 AnomalyType::SuddenDrop
394 },
395 severity: if z_score > 3.0 {
396 AnomalySeverity::High
397 } else {
398 AnomalySeverity::Medium
399 },
400 value,
401 expected_value: mean,
402 confidence_score: ((z_score - self.outlier_threshold) / self.outlier_threshold).min(1.0),
403 description: format!("Memory usage {} ({}MB) deviates significantly from expected value ({}MB). Z-score: {:.2}",
404 if value > mean { "spike" } else { "drop" },
405 value, mean, z_score),
406 });
407 }
408 }
409
410 if let Some(growth_anomaly) = self.detect_sustained_growth(&memory_values, metrics) {
412 anomalies.push(growth_anomaly);
413 }
414
415 anomalies
416 }
417
418 fn calculate_mean(&self, values: &[f64]) -> f64 {
419 values.iter().sum::<f64>() / values.len() as f64
420 }
421
422 fn calculate_std_dev(&self, values: &[f64], mean: f64) -> f64 {
423 let variance =
424 values.iter().map(|&x| (x - mean).powi(2)).sum::<f64>() / values.len() as f64;
425 variance.sqrt()
426 }
427
428 fn calculate_median(&self, values: &[f64]) -> f64 {
429 let mut sorted = values.to_vec();
430 sorted.sort_by(|a, b| a.partial_cmp(b).unwrap_or(std::cmp::Ordering::Equal));
431 let mid = sorted.len() / 2;
432 if sorted.len().is_multiple_of(2) {
433 (sorted[mid - 1] + sorted[mid]) / 2.0
434 } else {
435 sorted[mid]
436 }
437 }
438
439 fn calculate_quartiles(&self, values: &[f64]) -> (f64, f64) {
440 let mut sorted = values.to_vec();
441 sorted.sort_by(|a, b| a.partial_cmp(b).unwrap_or(std::cmp::Ordering::Equal));
442 let n = sorted.len();
443 let q1_idx = n / 4;
444 let q3_idx = 3 * n / 4;
445 (sorted[q1_idx], sorted[q3_idx])
446 }
447
448 fn detect_outliers(&self, values: &[f64], mean: f64, std_dev: f64) -> Vec<f64> {
449 values
450 .iter()
451 .filter(|&&x| (x - mean).abs() / std_dev > self.outlier_threshold)
452 .copied()
453 .collect()
454 }
455
456 fn calculate_trend(&self, values: &[f64]) -> f64 {
457 if values.len() < 2 {
458 return 0.0;
459 }
460
461 let n = values.len() as f64;
462 let x_mean = (n - 1.0) / 2.0;
463 let y_mean = self.calculate_mean(values);
464
465 let mut numerator = 0.0;
466 let mut denominator = 0.0;
467
468 for (i, &y) in values.iter().enumerate() {
469 let x = i as f64;
470 numerator += (x - x_mean) * (y - y_mean);
471 denominator += (x - x_mean).powi(2);
472 }
473
474 if denominator != 0.0 {
475 numerator / denominator
476 } else {
477 0.0
478 }
479 }
480
481 fn detect_sustained_growth(
482 &self,
483 values: &[f64],
484 metrics: &[MemoryMetrics],
485 ) -> Option<MemoryAnomaly> {
486 if values.len() < 20 {
487 return None;
488 }
489
490 let recent_window = &values[values.len() - 10..];
491 let earlier_window = &values[values.len() - 20..values.len() - 10];
492
493 let recent_mean = self.calculate_mean(recent_window);
494 let earlier_mean = self.calculate_mean(earlier_window);
495
496 let growth_rate = (recent_mean - earlier_mean) / earlier_mean;
497
498 if growth_rate > 0.2 {
499 Some(MemoryAnomaly {
501 timestamp: metrics[metrics.len() - 1].timestamp,
502 anomaly_type: AnomalyType::SustainedGrowth,
503 severity: if growth_rate > 0.5 {
504 AnomalySeverity::High
505 } else {
506 AnomalySeverity::Medium
507 },
508 value: recent_mean,
509 expected_value: earlier_mean,
510 confidence_score: (growth_rate / 0.5).min(1.0),
511 description: format!(
512 "Sustained memory growth detected: {:.1}% increase over recent observations",
513 growth_rate * 100.0
514 ),
515 })
516 } else {
517 None
518 }
519 }
520}
521
522#[derive(Debug, Clone, Serialize, Deserialize)]
524pub struct MemoryStatistics {
525 pub mean: f64,
526 pub median: f64,
527 pub std_dev: f64,
528 pub min: f64,
529 pub max: f64,
530 pub percentile_25: f64,
531 pub percentile_75: f64,
532 pub confidence_interval_lower: f64,
533 pub confidence_interval_upper: f64,
534 pub outlier_count: usize,
535 pub trend_slope: f64,
536 pub coefficient_of_variation: f64,
537}
538
539impl Default for MemoryStatistics {
540 fn default() -> Self {
541 Self {
542 mean: 0.0,
543 median: 0.0,
544 std_dev: 0.0,
545 min: 0.0,
546 max: 0.0,
547 percentile_25: 0.0,
548 percentile_75: 0.0,
549 confidence_interval_lower: 0.0,
550 confidence_interval_upper: 0.0,
551 outlier_count: 0,
552 trend_slope: 0.0,
553 coefficient_of_variation: 0.0,
554 }
555 }
556}
557
558#[derive(Debug, Clone, Serialize, Deserialize)]
560pub struct MemoryAnomaly {
561 pub timestamp: SystemTime,
562 pub anomaly_type: AnomalyType,
563 pub severity: AnomalySeverity,
564 pub value: f64,
565 pub expected_value: f64,
566 pub confidence_score: f64,
567 pub description: String,
568}
569
570#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
571pub enum AnomalyType {
572 SuddenSpike,
573 SuddenDrop,
574 SustainedGrowth,
575 UnusualPattern,
576}
577
578#[derive(Debug, Clone, Serialize, Deserialize)]
579pub enum AnomalySeverity {
580 Low,
581 Medium,
582 High,
583 Critical,
584}