Skip to main content

trustformers_models/memory_profiling/
system.rs

1//! System memory collection utilities
2//!
3//! Every figure produced here is a **real measurement** taken from the operating
4//! system through the pure-Rust [`sysinfo`] crate. Nothing is simulated: when a
5//! value cannot be obtained on the current platform the corresponding field is
6//! `None` (or the call returns an error) rather than a plausible-looking constant.
7
8use anyhow::{anyhow, Result};
9use std::sync::atomic::{AtomicU64, Ordering};
10use std::sync::Mutex;
11use sysinfo::{MemoryRefreshKind, ProcessRefreshKind, ProcessesToUpdate, RefreshKind, System};
12
13use super::types::*;
14
15const BYTES_PER_MB: f64 = 1024.0 * 1024.0;
16
17/// GPU memory information.
18///
19/// Only produced when a GPU backend is compiled in; there is deliberately no
20/// fallback that invents a device.
21#[derive(Debug, Clone)]
22pub struct GpuMemoryInfo {
23    pub total_mb: f64,
24    pub used_mb: f64,
25    pub free_mb: f64,
26}
27
28/// Measured memory footprint of the current process.
29#[derive(Debug, Clone)]
30pub struct ProcessMemoryInfo {
31    /// Resident set size (physical memory currently held), in MB.
32    pub resident_mb: f64,
33    /// Virtual memory size, in MB.
34    pub virtual_mb: f64,
35    /// Largest resident set size observed by this process since start-up, in MB.
36    ///
37    /// This is a running maximum over the samples this profiler has taken, not a
38    /// kernel-reported high-water mark.
39    pub peak_resident_mb: f64,
40}
41
42/// Counts of allocations the caller has explicitly registered with the profiler.
43///
44/// The profiler cannot see allocations it was not told about, so these counters
45/// describe *tracked* allocations only.
46#[derive(Debug, Clone, Copy, Default)]
47pub struct TrackedAllocationStats {
48    /// Cumulative number of allocations registered.
49    pub allocated_objects: u64,
50    /// Cumulative number of deallocations registered.
51    pub deallocated_objects: u64,
52    /// Allocations currently registered as live.
53    pub active_allocations: u64,
54    /// Total size of the live allocations, in bytes.
55    pub active_bytes: u64,
56}
57
58/// Running maximum of the resident set size, in bytes.
59static PEAK_RESIDENT_BYTES: AtomicU64 = AtomicU64::new(0);
60
61/// Shared `sysinfo` handle.
62///
63/// `System` caches per-process bookkeeping between refreshes, so a single
64/// long-lived instance is both cheaper and more accurate than creating one per
65/// sample.
66fn system_handle() -> &'static Mutex<System> {
67    static SYSTEM: std::sync::OnceLock<Mutex<System>> = std::sync::OnceLock::new();
68    SYSTEM.get_or_init(|| {
69        Mutex::new(System::new_with_specifics(
70            RefreshKind::nothing().with_memory(MemoryRefreshKind::everything()),
71        ))
72    })
73}
74
75impl super::profiler::MemoryProfiler {
76    /// Collect current memory metrics from the operating system.
77    ///
78    /// `tracked` carries the allocation counters the profiler maintains for
79    /// allocations that were explicitly registered by the caller.
80    pub async fn collect_memory_metrics(tracked: TrackedAllocationStats) -> Result<MemoryMetrics> {
81        let process_info = Self::get_process_memory_info()?;
82        let gpu_info = Self::get_gpu_memory_info().await?;
83
84        Ok(MemoryMetrics {
85            timestamp: std::time::SystemTime::now(),
86            total_memory_mb: process_info.resident_mb,
87            virtual_memory_mb: process_info.virtual_mb,
88            heap_memory_mb: None,
89            stack_memory_mb: None,
90            gpu_memory_mb: gpu_info.map(|info| info.used_mb),
91            peak_memory_mb: process_info.peak_resident_mb,
92            allocated_objects: tracked.allocated_objects,
93            deallocated_objects: tracked.deallocated_objects,
94            active_allocations: tracked.active_allocations,
95            memory_fragmentation_ratio: Self::calculate_fragmentation_ratio(&process_info, tracked),
96            memory_growth_rate_mb_per_sec: 0.0, // Filled in by the sampling loop
97        })
98    }
99
100    /// Read the machine's memory statistics.
101    ///
102    /// `cached_memory` and `buffer_memory` are `None`: they are not exposed
103    /// portably, and reporting a guess would be worse than reporting nothing.
104    pub fn get_system_memory_info() -> Result<SystemMemoryInfo> {
105        let mut system = system_handle()
106            .lock()
107            .map_err(|_| anyhow!("system information handle is poisoned"))?;
108        system.refresh_memory();
109
110        let total_memory = system.total_memory();
111        if total_memory == 0 {
112            return Err(anyhow!(
113                "the operating system reported a total memory of 0 bytes; \
114                 memory statistics are unavailable on this platform"
115            ));
116        }
117
118        Ok(SystemMemoryInfo {
119            total_memory,
120            available_memory: system.available_memory(),
121            used_memory: system.used_memory(),
122            free_memory: system.free_memory(),
123            cached_memory: None,
124            buffer_memory: None,
125        })
126    }
127
128    /// Read this process's resident and virtual memory footprint.
129    pub fn get_process_memory_info() -> Result<ProcessMemoryInfo> {
130        let pid = sysinfo::get_current_pid()
131            .map_err(|e| anyhow!("failed to determine the current process id: {e}"))?;
132
133        let mut system = system_handle()
134            .lock()
135            .map_err(|_| anyhow!("system information handle is poisoned"))?;
136        system.refresh_processes_specifics(
137            ProcessesToUpdate::Some(&[pid]),
138            true,
139            ProcessRefreshKind::nothing().with_memory(),
140        );
141
142        let process = system
143            .process(pid)
144            .ok_or_else(|| anyhow!("the operating system reported no process with id {pid}"))?;
145
146        let resident_bytes = process.memory();
147        let virtual_bytes = process.virtual_memory();
148        drop(system);
149
150        if resident_bytes == 0 {
151            return Err(anyhow!(
152                "the operating system reported a resident set size of 0 bytes for this process; \
153                 process memory statistics are unavailable on this platform"
154            ));
155        }
156
157        let peak_bytes = PEAK_RESIDENT_BYTES.fetch_max(resident_bytes, Ordering::Relaxed);
158
159        Ok(ProcessMemoryInfo {
160            resident_mb: resident_bytes as f64 / BYTES_PER_MB,
161            virtual_mb: virtual_bytes as f64 / BYTES_PER_MB,
162            peak_resident_mb: peak_bytes.max(resident_bytes) as f64 / BYTES_PER_MB,
163        })
164    }
165
166    /// Read GPU memory usage.
167    ///
168    /// Returns `Ok(None)` when no GPU backend is compiled in — this crate has no
169    /// way to query a device that is not there, and a fabricated card would make
170    /// every downstream report fiction.
171    pub async fn get_gpu_memory_info() -> Result<Option<GpuMemoryInfo>> {
172        Ok(None)
173    }
174
175    /// Share of the resident set that is *not* covered by tracked allocations.
176    ///
177    /// With no registered allocations there is nothing to compare the resident
178    /// set against, so the ratio is `0.0` rather than an invented percentage.
179    pub fn calculate_fragmentation_ratio(
180        info: &ProcessMemoryInfo,
181        tracked: TrackedAllocationStats,
182    ) -> f64 {
183        if tracked.active_bytes == 0 || info.resident_mb <= 0.0 {
184            return 0.0;
185        }
186        let tracked_mb = tracked.active_bytes as f64 / BYTES_PER_MB;
187        ((info.resident_mb - tracked_mb) / info.resident_mb).clamp(0.0, 1.0)
188    }
189
190    /// Update adaptive thresholds based on current metrics
191    pub async fn update_adaptive_thresholds(
192        current_metrics: &MemoryMetrics,
193        adaptive_thresholds: &std::sync::Arc<
194            std::sync::Mutex<super::analytics::AdaptiveThresholds>,
195        >,
196    ) {
197        // Update thresholds every 5 minutes
198        let should_update = {
199            let thresholds = match adaptive_thresholds.lock() {
200                Ok(guard) => guard,
201                Err(_) => return, // Skip update on lock failure
202            };
203            current_metrics
204                .timestamp
205                .duration_since(thresholds.last_updated)
206                .unwrap_or_default()
207                .as_secs()
208                > 300
209        };
210
211        if should_update {
212            let mut thresholds = match adaptive_thresholds.lock() {
213                Ok(guard) => guard,
214                Err(_) => return, // Skip update on lock failure
215            };
216            // Simple adaptive update based on current usage
217            let adaptation_factor = 0.1;
218            thresholds.base_memory_threshold = thresholds.base_memory_threshold
219                * (1.0 - adaptation_factor)
220                + current_metrics.total_memory_mb * 1.2 * adaptation_factor;
221            thresholds.growth_rate_threshold = thresholds.growth_rate_threshold
222                * (1.0 - adaptation_factor)
223                + current_metrics.memory_growth_rate_mb_per_sec.abs() * 2.0 * adaptation_factor;
224            thresholds.fragmentation_threshold = thresholds.fragmentation_threshold
225                * (1.0 - adaptation_factor)
226                + current_metrics.memory_fragmentation_ratio * 1.5 * adaptation_factor;
227            thresholds.last_updated = current_metrics.timestamp;
228        }
229    }
230
231    /// Detect memory leaks using heuristics
232    pub async fn detect_memory_leaks(
233        current_metrics: &MemoryMetrics,
234        previous_metrics: &Option<MemoryMetrics>,
235        leak_detection: &std::sync::Arc<
236            std::sync::Mutex<super::analytics::LeakDetectionHeuristics>,
237        >,
238        alerts: &std::sync::Arc<std::sync::Mutex<Vec<MemoryAlert>>>,
239        cached_recommendations: &super::analytics::AlertRecommendations,
240    ) {
241        if let Some(_prev) = previous_metrics {
242            let growth_rate = current_metrics.memory_growth_rate_mb_per_sec;
243            let detection = match leak_detection.lock() {
244                Ok(guard) => guard,
245                Err(_) => return, // Skip detection on lock failure
246            };
247
248            // Check for sustained growth
249            if growth_rate > detection.sustained_growth_threshold {
250                let mut alerts_vec = match alerts.lock() {
251                    Ok(guard) => guard,
252                    Err(_) => return, // Skip alert on lock failure
253                };
254                alerts_vec.push(MemoryAlert {
255                    id: uuid::Uuid::new_v4(),
256                    timestamp: current_metrics.timestamp,
257                    alert_type: MemoryAlertType::MemoryLeak,
258                    severity: if growth_rate > detection.sustained_growth_threshold * 2.0 {
259                        AlertSeverity::Critical
260                    } else {
261                        AlertSeverity::Warning
262                    },
263                    message: format!(
264                        "Potential memory leak detected: {:.2} MB/sec growth",
265                        growth_rate
266                    ),
267                    details: std::collections::HashMap::new(),
268                    recommendations: cached_recommendations.memory_leak.clone(),
269                });
270            }
271        }
272    }
273
274    /// Update memory prediction
275    pub async fn update_memory_prediction(
276        _current_metrics: &MemoryMetrics,
277        memory_predictor: &std::sync::Arc<std::sync::Mutex<super::analytics::MemoryPredictor>>,
278        metrics_history: &std::sync::Arc<
279            std::sync::Mutex<std::collections::VecDeque<MemoryMetrics>>,
280        >,
281    ) {
282        let history = match metrics_history.lock() {
283            Ok(guard) => guard,
284            Err(_) => return, // Skip prediction on lock failure
285        };
286        if history.len() >= 10 {
287            let recent_metrics: Vec<MemoryMetrics> = history.iter().cloned().collect();
288            drop(history);
289
290            let mut predictor = match memory_predictor.lock() {
291                Ok(guard) => guard,
292                Err(_) => return, // Skip prediction on lock failure
293            };
294            let _prediction = predictor.predict_memory_usage(&recent_metrics, Some(300));
295            // 5 minutes
296            // Prediction result would be stored or used for alerts in a real implementation
297        }
298    }
299
300    /// Analyze for alerts with adaptive thresholds
301    pub async fn analyze_for_alerts_adaptive(
302        current: &MemoryMetrics,
303        _previous: &Option<MemoryMetrics>,
304        alerts: &std::sync::Arc<std::sync::Mutex<Vec<MemoryAlert>>>,
305        _config: &super::types::ProfilerConfig,
306        cached_recommendations: &super::analytics::AlertRecommendations,
307        adaptive_thresholds: &std::sync::Arc<
308            std::sync::Mutex<super::analytics::AdaptiveThresholds>,
309        >,
310    ) {
311        let thresholds = match adaptive_thresholds.lock() {
312            Ok(guard) => guard,
313            Err(_) => return, // Skip analysis on lock failure
314        };
315        let mut new_alerts = Vec::new();
316
317        // High memory usage alert using adaptive threshold
318        if current.total_memory_mb > thresholds.base_memory_threshold {
319            new_alerts.push(MemoryAlert {
320                id: uuid::Uuid::new_v4(),
321                timestamp: current.timestamp,
322                alert_type: MemoryAlertType::HighMemoryUsage,
323                severity: if current.total_memory_mb > thresholds.base_memory_threshold * 1.5 {
324                    AlertSeverity::Critical
325                } else {
326                    AlertSeverity::Warning
327                },
328                message: format!(
329                    "Memory usage is {:.2}MB, exceeding adaptive threshold of {:.2}MB",
330                    current.total_memory_mb, thresholds.base_memory_threshold
331                ),
332                details: std::collections::HashMap::new(),
333                recommendations: cached_recommendations.high_memory.clone(),
334            });
335        }
336
337        // Rapid growth alert
338        if current.memory_growth_rate_mb_per_sec > thresholds.growth_rate_threshold {
339            new_alerts.push(MemoryAlert {
340                id: uuid::Uuid::new_v4(),
341                timestamp: current.timestamp,
342                alert_type: MemoryAlertType::RapidGrowth,
343                severity: AlertSeverity::Warning,
344                message: format!(
345                    "Rapid memory growth detected: {:.2}MB/sec",
346                    current.memory_growth_rate_mb_per_sec
347                ),
348                details: std::collections::HashMap::new(),
349                recommendations: cached_recommendations.rapid_growth.clone(),
350            });
351        }
352
353        // Fragmentation alert
354        if current.memory_fragmentation_ratio > thresholds.fragmentation_threshold {
355            new_alerts.push(MemoryAlert {
356                id: uuid::Uuid::new_v4(),
357                timestamp: current.timestamp,
358                alert_type: MemoryAlertType::FragmentationHigh,
359                severity: AlertSeverity::Info,
360                message: format!(
361                    "High memory fragmentation: {:.2}%",
362                    current.memory_fragmentation_ratio * 100.0
363                ),
364                details: std::collections::HashMap::new(),
365                recommendations: cached_recommendations.fragmentation.clone(),
366            });
367        }
368
369        // Store alerts if any were generated
370        if !new_alerts.is_empty() {
371            let mut alerts_vec = match alerts.lock() {
372                Ok(guard) => guard,
373                Err(_) => {
374                    drop(thresholds);
375                    return; // Skip storing alerts on lock failure
376                },
377            };
378            alerts_vec.extend(new_alerts);
379
380            // Keep only recent alerts (last 1000)
381            if alerts_vec.len() > 1000 {
382                let excess = alerts_vec.len() - 1000;
383                alerts_vec.drain(0..excess);
384            }
385        }
386        drop(thresholds);
387    }
388
389    /// Update memory patterns
390    pub async fn update_patterns(
391        current_metrics: &MemoryMetrics,
392        patterns: &std::sync::Arc<std::sync::Mutex<Vec<MemoryPattern>>>,
393    ) {
394        let mut patterns_vec = match patterns.lock() {
395            Ok(guard) => guard,
396            Err(_) => return, // Skip pattern update on lock failure
397        };
398
399        // Simple pattern detection - in practice this would be much more sophisticated
400        if current_metrics.total_memory_mb > 1000.0 {
401            // Check if we already have a large allocation pattern
402            let has_large_pattern = patterns_vec
403                .iter()
404                .any(|p| matches!(p.pattern_type, PatternType::LargeAllocations));
405
406            if !has_large_pattern {
407                patterns_vec.push(MemoryPattern {
408                    pattern_type: PatternType::LargeAllocations,
409                    frequency: 1,
410                    average_size: current_metrics.total_memory_mb,
411                    total_size: current_metrics.total_memory_mb,
412                    locations: vec!["system".to_string()],
413                    trend: PatternTrend::Increasing,
414                });
415            } else {
416                // Update existing pattern
417                if let Some(pattern) = patterns_vec
418                    .iter_mut()
419                    .find(|p| matches!(p.pattern_type, PatternType::LargeAllocations))
420                {
421                    pattern.frequency += 1;
422                    pattern.average_size =
423                        (pattern.average_size + current_metrics.total_memory_mb) / 2.0;
424                    pattern.total_size += current_metrics.total_memory_mb;
425                }
426            }
427        }
428
429        // Keep patterns list bounded
430        if patterns_vec.len() > 100 {
431            let excess = patterns_vec.len() - 100;
432            patterns_vec.drain(0..excess);
433        }
434    }
435}