Skip to main content

dataprof_core/
memory_sampler.rs

1use std::time::{Duration, Instant};
2
3use sysinfo::{Pid, ProcessRefreshKind, ProcessesToUpdate, System};
4
5/// Minimum spacing between two throttled samples.
6///
7/// One refresh costs about 14 ms on Windows, where sysinfo walks the process
8/// table even when asked for a single process, and the streaming engines
9/// sample once per chunk. Small chunks paid that cost thousands of times: a
10/// 300,000-row CSV took 345 s with 1 KB chunks against 1.6 s with 1 MB ones.
11/// Resident memory moves slowly next to that, so sampling at most four times
12/// a second loses nothing a peak reading needs, and callers take one
13/// unthrottled sample at the end.
14const MIN_SAMPLE_INTERVAL: Duration = Duration::from_millis(250);
15
16/// Tracks the peak resident set size (RSS) of the current process across a
17/// profiling run.
18///
19/// Engines call [`sample`](Self::sample) at chunk/batch boundaries and
20/// [`sample_now`](Self::sample_now) once at the end; the peak observed feeds
21/// `ExecutionMetadata::memory_peak_mb`. The reading is the
22/// whole-process RSS, so on embedded surfaces (e.g. the Python extension) it
23/// includes host-process memory — it is an upper bound on what profiling used,
24/// suitable for verifying the bounded-memory claim, not an exact attribution.
25pub struct PeakMemorySampler {
26    /// `None` on platforms sysinfo does not support.
27    system: Option<System>,
28    pid: Option<Pid>,
29    peak_bytes: u64,
30    last_sample: Option<Instant>,
31    refreshes: u64,
32}
33
34impl PeakMemorySampler {
35    /// Create a sampler and take an initial reading.
36    pub fn new() -> Self {
37        let mut sampler = Self {
38            system: System::new().ok(),
39            pid: sysinfo::get_current_pid().ok(),
40            peak_bytes: 0,
41            last_sample: None,
42            refreshes: 0,
43        };
44        sampler.sample_now();
45        sampler
46    }
47
48    /// Sample unless the previous sample is less than [`MIN_SAMPLE_INTERVAL`]
49    /// old. For per-chunk calls inside a loop.
50    pub fn sample(&mut self) {
51        if self
52            .last_sample
53            .is_some_and(|last| last.elapsed() < MIN_SAMPLE_INTERVAL)
54        {
55            return;
56        }
57        self.sample_now();
58    }
59
60    /// Refresh the process RSS and fold it into the running peak, whatever
61    /// the interval. For the final reading of a run, so the reported peak
62    /// covers the work done after the last throttled sample.
63    pub fn sample_now(&mut self) {
64        self.last_sample = Some(Instant::now());
65        let (Some(system), Some(pid)) = (self.system.as_mut(), self.pid) else {
66            return;
67        };
68        self.refreshes += 1;
69        system.refresh_processes_specifics(
70            ProcessesToUpdate::Some(&[pid]),
71            false,
72            ProcessRefreshKind::nothing().with_memory(),
73        );
74        if let Some(process) = system.process(pid) {
75            self.peak_bytes = self.peak_bytes.max(process.memory());
76        }
77    }
78
79    /// Peak RSS observed so far, in megabytes.
80    ///
81    /// `None` when the platform provided no reading — callers should leave
82    /// `memory_peak_mb` absent (not analyzed) rather than report zero.
83    pub fn peak_mb(&self) -> Option<f64> {
84        (self.peak_bytes > 0).then(|| self.peak_bytes as f64 / (1024.0 * 1024.0))
85    }
86}
87
88impl Default for PeakMemorySampler {
89    fn default() -> Self {
90        Self::new()
91    }
92}
93
94#[cfg(test)]
95mod tests {
96    use super::*;
97
98    #[test]
99    fn repeated_samples_within_the_interval_do_not_refresh() {
100        let mut sampler = PeakMemorySampler::new();
101        let before = sampler.refreshes;
102        for _ in 0..1_000 {
103            sampler.sample();
104        }
105        assert_eq!(sampler.refreshes, before, "a tight loop must not refresh");
106        sampler.sample_now();
107        assert_eq!(sampler.refreshes, before + 1, "sample_now always refreshes");
108    }
109
110    #[test]
111    fn test_sampler_reports_positive_peak() {
112        let sampler = PeakMemorySampler::new();
113        let peak = sampler
114            .peak_mb()
115            .expect("current process RSS should be readable on supported platforms");
116        assert!(peak > 0.0, "peak RSS must be positive, got {peak}");
117    }
118
119    #[test]
120    fn test_peak_is_monotonic_across_samples() {
121        let mut sampler = PeakMemorySampler::new();
122        let first = sampler.peak_mb().expect("initial reading");
123        // Allocate enough that RSS cannot legitimately shrink below the first peak.
124        let ballast = vec![1u8; 4 * 1024 * 1024];
125        sampler.sample_now();
126        let second = sampler.peak_mb().expect("second reading");
127        assert!(
128            second >= first,
129            "peak must never decrease: {first} -> {second}"
130        );
131        drop(ballast);
132    }
133}