trustformers_models/memory_profiling/
system.rs1use anyhow::{anyhow, Result};
9use std::sync::atomic::{AtomicU64, Ordering};
10use std::sync::Mutex;
11use sysinfo::{MemoryRefreshKind, ProcessRefreshKind, ProcessesToUpdate, RefreshKind, System};
12
13use super::types::*;
14
15const BYTES_PER_MB: f64 = 1024.0 * 1024.0;
16
17#[derive(Debug, Clone)]
22pub struct GpuMemoryInfo {
23 pub total_mb: f64,
24 pub used_mb: f64,
25 pub free_mb: f64,
26}
27
28#[derive(Debug, Clone)]
30pub struct ProcessMemoryInfo {
31 pub resident_mb: f64,
33 pub virtual_mb: f64,
35 pub peak_resident_mb: f64,
40}
41
42#[derive(Debug, Clone, Copy, Default)]
47pub struct TrackedAllocationStats {
48 pub allocated_objects: u64,
50 pub deallocated_objects: u64,
52 pub active_allocations: u64,
54 pub active_bytes: u64,
56}
57
58static PEAK_RESIDENT_BYTES: AtomicU64 = AtomicU64::new(0);
60
61fn system_handle() -> &'static Mutex<System> {
67 static SYSTEM: std::sync::OnceLock<Mutex<System>> = std::sync::OnceLock::new();
68 SYSTEM.get_or_init(|| {
69 Mutex::new(System::new_with_specifics(
70 RefreshKind::nothing().with_memory(MemoryRefreshKind::everything()),
71 ))
72 })
73}
74
75impl super::profiler::MemoryProfiler {
76 pub async fn collect_memory_metrics(tracked: TrackedAllocationStats) -> Result<MemoryMetrics> {
81 let process_info = Self::get_process_memory_info()?;
82 let gpu_info = Self::get_gpu_memory_info().await?;
83
84 Ok(MemoryMetrics {
85 timestamp: std::time::SystemTime::now(),
86 total_memory_mb: process_info.resident_mb,
87 virtual_memory_mb: process_info.virtual_mb,
88 heap_memory_mb: None,
89 stack_memory_mb: None,
90 gpu_memory_mb: gpu_info.map(|info| info.used_mb),
91 peak_memory_mb: process_info.peak_resident_mb,
92 allocated_objects: tracked.allocated_objects,
93 deallocated_objects: tracked.deallocated_objects,
94 active_allocations: tracked.active_allocations,
95 memory_fragmentation_ratio: Self::calculate_fragmentation_ratio(&process_info, tracked),
96 memory_growth_rate_mb_per_sec: 0.0, })
98 }
99
100 pub fn get_system_memory_info() -> Result<SystemMemoryInfo> {
105 let mut system = system_handle()
106 .lock()
107 .map_err(|_| anyhow!("system information handle is poisoned"))?;
108 system.refresh_memory();
109
110 let total_memory = system.total_memory();
111 if total_memory == 0 {
112 return Err(anyhow!(
113 "the operating system reported a total memory of 0 bytes; \
114 memory statistics are unavailable on this platform"
115 ));
116 }
117
118 Ok(SystemMemoryInfo {
119 total_memory,
120 available_memory: system.available_memory(),
121 used_memory: system.used_memory(),
122 free_memory: system.free_memory(),
123 cached_memory: None,
124 buffer_memory: None,
125 })
126 }
127
128 pub fn get_process_memory_info() -> Result<ProcessMemoryInfo> {
130 let pid = sysinfo::get_current_pid()
131 .map_err(|e| anyhow!("failed to determine the current process id: {e}"))?;
132
133 let mut system = system_handle()
134 .lock()
135 .map_err(|_| anyhow!("system information handle is poisoned"))?;
136 system.refresh_processes_specifics(
137 ProcessesToUpdate::Some(&[pid]),
138 true,
139 ProcessRefreshKind::nothing().with_memory(),
140 );
141
142 let process = system
143 .process(pid)
144 .ok_or_else(|| anyhow!("the operating system reported no process with id {pid}"))?;
145
146 let resident_bytes = process.memory();
147 let virtual_bytes = process.virtual_memory();
148 drop(system);
149
150 if resident_bytes == 0 {
151 return Err(anyhow!(
152 "the operating system reported a resident set size of 0 bytes for this process; \
153 process memory statistics are unavailable on this platform"
154 ));
155 }
156
157 let peak_bytes = PEAK_RESIDENT_BYTES.fetch_max(resident_bytes, Ordering::Relaxed);
158
159 Ok(ProcessMemoryInfo {
160 resident_mb: resident_bytes as f64 / BYTES_PER_MB,
161 virtual_mb: virtual_bytes as f64 / BYTES_PER_MB,
162 peak_resident_mb: peak_bytes.max(resident_bytes) as f64 / BYTES_PER_MB,
163 })
164 }
165
166 pub async fn get_gpu_memory_info() -> Result<Option<GpuMemoryInfo>> {
172 Ok(None)
173 }
174
175 pub fn calculate_fragmentation_ratio(
180 info: &ProcessMemoryInfo,
181 tracked: TrackedAllocationStats,
182 ) -> f64 {
183 if tracked.active_bytes == 0 || info.resident_mb <= 0.0 {
184 return 0.0;
185 }
186 let tracked_mb = tracked.active_bytes as f64 / BYTES_PER_MB;
187 ((info.resident_mb - tracked_mb) / info.resident_mb).clamp(0.0, 1.0)
188 }
189
190 pub async fn update_adaptive_thresholds(
192 current_metrics: &MemoryMetrics,
193 adaptive_thresholds: &std::sync::Arc<
194 std::sync::Mutex<super::analytics::AdaptiveThresholds>,
195 >,
196 ) {
197 let should_update = {
199 let thresholds = match adaptive_thresholds.lock() {
200 Ok(guard) => guard,
201 Err(_) => return, };
203 current_metrics
204 .timestamp
205 .duration_since(thresholds.last_updated)
206 .unwrap_or_default()
207 .as_secs()
208 > 300
209 };
210
211 if should_update {
212 let mut thresholds = match adaptive_thresholds.lock() {
213 Ok(guard) => guard,
214 Err(_) => return, };
216 let adaptation_factor = 0.1;
218 thresholds.base_memory_threshold = thresholds.base_memory_threshold
219 * (1.0 - adaptation_factor)
220 + current_metrics.total_memory_mb * 1.2 * adaptation_factor;
221 thresholds.growth_rate_threshold = thresholds.growth_rate_threshold
222 * (1.0 - adaptation_factor)
223 + current_metrics.memory_growth_rate_mb_per_sec.abs() * 2.0 * adaptation_factor;
224 thresholds.fragmentation_threshold = thresholds.fragmentation_threshold
225 * (1.0 - adaptation_factor)
226 + current_metrics.memory_fragmentation_ratio * 1.5 * adaptation_factor;
227 thresholds.last_updated = current_metrics.timestamp;
228 }
229 }
230
231 pub async fn detect_memory_leaks(
233 current_metrics: &MemoryMetrics,
234 previous_metrics: &Option<MemoryMetrics>,
235 leak_detection: &std::sync::Arc<
236 std::sync::Mutex<super::analytics::LeakDetectionHeuristics>,
237 >,
238 alerts: &std::sync::Arc<std::sync::Mutex<Vec<MemoryAlert>>>,
239 cached_recommendations: &super::analytics::AlertRecommendations,
240 ) {
241 if let Some(_prev) = previous_metrics {
242 let growth_rate = current_metrics.memory_growth_rate_mb_per_sec;
243 let detection = match leak_detection.lock() {
244 Ok(guard) => guard,
245 Err(_) => return, };
247
248 if growth_rate > detection.sustained_growth_threshold {
250 let mut alerts_vec = match alerts.lock() {
251 Ok(guard) => guard,
252 Err(_) => return, };
254 alerts_vec.push(MemoryAlert {
255 id: uuid::Uuid::new_v4(),
256 timestamp: current_metrics.timestamp,
257 alert_type: MemoryAlertType::MemoryLeak,
258 severity: if growth_rate > detection.sustained_growth_threshold * 2.0 {
259 AlertSeverity::Critical
260 } else {
261 AlertSeverity::Warning
262 },
263 message: format!(
264 "Potential memory leak detected: {:.2} MB/sec growth",
265 growth_rate
266 ),
267 details: std::collections::HashMap::new(),
268 recommendations: cached_recommendations.memory_leak.clone(),
269 });
270 }
271 }
272 }
273
274 pub async fn update_memory_prediction(
276 _current_metrics: &MemoryMetrics,
277 memory_predictor: &std::sync::Arc<std::sync::Mutex<super::analytics::MemoryPredictor>>,
278 metrics_history: &std::sync::Arc<
279 std::sync::Mutex<std::collections::VecDeque<MemoryMetrics>>,
280 >,
281 ) {
282 let history = match metrics_history.lock() {
283 Ok(guard) => guard,
284 Err(_) => return, };
286 if history.len() >= 10 {
287 let recent_metrics: Vec<MemoryMetrics> = history.iter().cloned().collect();
288 drop(history);
289
290 let mut predictor = match memory_predictor.lock() {
291 Ok(guard) => guard,
292 Err(_) => return, };
294 let _prediction = predictor.predict_memory_usage(&recent_metrics, Some(300));
295 }
298 }
299
300 pub async fn analyze_for_alerts_adaptive(
302 current: &MemoryMetrics,
303 _previous: &Option<MemoryMetrics>,
304 alerts: &std::sync::Arc<std::sync::Mutex<Vec<MemoryAlert>>>,
305 _config: &super::types::ProfilerConfig,
306 cached_recommendations: &super::analytics::AlertRecommendations,
307 adaptive_thresholds: &std::sync::Arc<
308 std::sync::Mutex<super::analytics::AdaptiveThresholds>,
309 >,
310 ) {
311 let thresholds = match adaptive_thresholds.lock() {
312 Ok(guard) => guard,
313 Err(_) => return, };
315 let mut new_alerts = Vec::new();
316
317 if current.total_memory_mb > thresholds.base_memory_threshold {
319 new_alerts.push(MemoryAlert {
320 id: uuid::Uuid::new_v4(),
321 timestamp: current.timestamp,
322 alert_type: MemoryAlertType::HighMemoryUsage,
323 severity: if current.total_memory_mb > thresholds.base_memory_threshold * 1.5 {
324 AlertSeverity::Critical
325 } else {
326 AlertSeverity::Warning
327 },
328 message: format!(
329 "Memory usage is {:.2}MB, exceeding adaptive threshold of {:.2}MB",
330 current.total_memory_mb, thresholds.base_memory_threshold
331 ),
332 details: std::collections::HashMap::new(),
333 recommendations: cached_recommendations.high_memory.clone(),
334 });
335 }
336
337 if current.memory_growth_rate_mb_per_sec > thresholds.growth_rate_threshold {
339 new_alerts.push(MemoryAlert {
340 id: uuid::Uuid::new_v4(),
341 timestamp: current.timestamp,
342 alert_type: MemoryAlertType::RapidGrowth,
343 severity: AlertSeverity::Warning,
344 message: format!(
345 "Rapid memory growth detected: {:.2}MB/sec",
346 current.memory_growth_rate_mb_per_sec
347 ),
348 details: std::collections::HashMap::new(),
349 recommendations: cached_recommendations.rapid_growth.clone(),
350 });
351 }
352
353 if current.memory_fragmentation_ratio > thresholds.fragmentation_threshold {
355 new_alerts.push(MemoryAlert {
356 id: uuid::Uuid::new_v4(),
357 timestamp: current.timestamp,
358 alert_type: MemoryAlertType::FragmentationHigh,
359 severity: AlertSeverity::Info,
360 message: format!(
361 "High memory fragmentation: {:.2}%",
362 current.memory_fragmentation_ratio * 100.0
363 ),
364 details: std::collections::HashMap::new(),
365 recommendations: cached_recommendations.fragmentation.clone(),
366 });
367 }
368
369 if !new_alerts.is_empty() {
371 let mut alerts_vec = match alerts.lock() {
372 Ok(guard) => guard,
373 Err(_) => {
374 drop(thresholds);
375 return; },
377 };
378 alerts_vec.extend(new_alerts);
379
380 if alerts_vec.len() > 1000 {
382 let excess = alerts_vec.len() - 1000;
383 alerts_vec.drain(0..excess);
384 }
385 }
386 drop(thresholds);
387 }
388
389 pub async fn update_patterns(
391 current_metrics: &MemoryMetrics,
392 patterns: &std::sync::Arc<std::sync::Mutex<Vec<MemoryPattern>>>,
393 ) {
394 let mut patterns_vec = match patterns.lock() {
395 Ok(guard) => guard,
396 Err(_) => return, };
398
399 if current_metrics.total_memory_mb > 1000.0 {
401 let has_large_pattern = patterns_vec
403 .iter()
404 .any(|p| matches!(p.pattern_type, PatternType::LargeAllocations));
405
406 if !has_large_pattern {
407 patterns_vec.push(MemoryPattern {
408 pattern_type: PatternType::LargeAllocations,
409 frequency: 1,
410 average_size: current_metrics.total_memory_mb,
411 total_size: current_metrics.total_memory_mb,
412 locations: vec!["system".to_string()],
413 trend: PatternTrend::Increasing,
414 });
415 } else {
416 if let Some(pattern) = patterns_vec
418 .iter_mut()
419 .find(|p| matches!(p.pattern_type, PatternType::LargeAllocations))
420 {
421 pattern.frequency += 1;
422 pattern.average_size =
423 (pattern.average_size + current_metrics.total_memory_mb) / 2.0;
424 pattern.total_size += current_metrics.total_memory_mb;
425 }
426 }
427 }
428
429 if patterns_vec.len() > 100 {
431 let excess = patterns_vec.len() - 100;
432 patterns_vec.drain(0..excess);
433 }
434 }
435}