Skip to main content

mtop/
sampler.rs

1use std::collections::HashMap;
2use std::sync::mpsc::{Receiver, RecvTimeoutError, Sender};
3use std::thread;
4use std::time::{Duration, Instant};
5
6use crate::model::{
7    Detail, Focus, PSS_DENIED_TTL, PSS_MAX_AGE, PSS_REFRESH_AFTER, ProcRow, PssInfo, Snapshot,
8};
9use crate::procfs;
10
11pub const TICK: Duration = Duration::from_millis(1000);
12/// Per-tick ceiling for page-table walks. Measured: one walk is ~1.5 ms average
13/// but 16-19 ms for a browser process, so a fixed row count is the wrong knob.
14pub const PSS_BUDGET: Duration = Duration::from_millis(150);
15/// Detail view re-reads full smaps (~30-40 ms), so it is refreshed lazily.
16pub const DETAIL_MAX_AGE: Duration = Duration::from_millis(2000);
17const MEMINFO_EVERY: u32 = 5;
18
19struct State {
20    focus: Option<u32>,
21    want_detail: bool,
22    pss: HashMap<u32, PssInfo>,
23    prev_rss: HashMap<u32, u64>,
24    detail: Option<Detail>,
25    detail_error: Option<String>,
26    denied: HashMap<u32, Instant>,
27    no_mm: HashMap<u32, Instant>,
28    ticks: u32,
29    mem: procfs::MemInfo,
30    cgroups: Vec<crate::cgroup::CgroupMem>,
31    pid_cgroup: std::sync::Arc<HashMap<u32, String>>,
32    pid_exe: std::sync::Arc<HashMap<u32, String>>,
33    oom_prev: HashMap<String, u64>,
34    scan_seq: u64,
35}
36
37impl State {
38    fn new() -> Self {
39        State {
40            focus: None,
41            want_detail: false,
42            pss: HashMap::new(),
43            prev_rss: HashMap::new(),
44            detail: None,
45            detail_error: None,
46            denied: HashMap::new(),
47            no_mm: HashMap::new(),
48            ticks: 0,
49            mem: procfs::read_meminfo(),
50            cgroups: crate::cgroup::scan(std::path::Path::new(crate::cgroup::ROOT)),
51            pid_cgroup: std::sync::Arc::new(crate::cgroup::pid_map()),
52            pid_exe: std::sync::Arc::new(procfs::pid_exe()),
53            oom_prev: HashMap::new(),
54            scan_seq: 1,
55        }
56    }
57}
58
59/// One synchronous snapshot with a permissive budget, for `--json`.
60///
61/// The TUI spreads page-table walks over ticks; a one-shot caller cannot wait
62/// for that, so the whole readable set is walked in a single pass (~700 ms).
63pub fn snapshot_once(focus: Option<u32>, want_detail: bool) -> Snapshot {
64    let mut st = State::new();
65    st.focus = focus;
66    st.want_detail = want_detail;
67    tick(&mut st, Duration::from_secs(2))
68}
69
70pub fn spawn(rx: Receiver<Focus>, tx: Sender<Snapshot>) {
71    thread::Builder::new()
72        .name("sampler".into())
73        .spawn(move || run(rx, tx))
74        .expect("spawn sampler thread");
75}
76
77fn run(rx: Receiver<Focus>, tx: Sender<Snapshot>) {
78    let mut st = State::new();
79    let mut next = Instant::now();
80    let mut urgent = false;
81
82    loop {
83        let wait = if urgent {
84            Duration::ZERO
85        } else {
86            next.saturating_duration_since(Instant::now())
87        };
88        match rx.recv_timeout(wait) {
89            Ok(Focus::Set { pid, want_detail }) => {
90                st.focus = pid;
91                st.want_detail = want_detail;
92                while let Ok(Focus::Set { pid, want_detail }) = rx.try_recv() {
93                    st.focus = pid;
94                    st.want_detail = want_detail;
95                }
96                urgent = true;
97            }
98            Err(RecvTimeoutError::Timeout) => {
99                let snap = tick(&mut st, PSS_BUDGET);
100                if tx.send(snap).is_err() {
101                    return;
102                }
103                next = Instant::now() + TICK;
104                urgent = false;
105            }
106            Err(RecvTimeoutError::Disconnected) => return,
107        }
108    }
109}
110
111/// Order in which to spend the page-table budget this tick.
112///
113/// Focused pid first (always), then pids with no cached rollup in RSS order —
114/// the biggest unknowns are exactly the rows whose rank a PSS sort would get
115/// wrong — then cached pids oldest-first. Fresh entries are skipped so the
116/// budget is never burned re-reading the same process.
117pub fn pss_targets(
118    focus: Option<u32>,
119    rows: &[ProcRow],
120    pss: &HashMap<u32, PssInfo>,
121    denied: &HashMap<u32, Instant>,
122    no_mm: &HashMap<u32, Instant>,
123    refresh_after: Duration,
124    denied_ttl: Duration,
125) -> Vec<u32> {
126    // Unreadable pids keep their place in the row list but must not consume the
127    // budget again until the denial expires.
128    let allowed = |pid: u32| {
129        let fresh =
130            |m: &HashMap<u32, Instant>| m.get(&pid).is_some_and(|at| at.elapsed() < denied_ttl);
131        !fresh(denied) && !fresh(no_mm)
132    };
133    let mut order: Vec<u32> = Vec::with_capacity(rows.len() + 1);
134    if let Some(pid) = focus
135        && allowed(pid)
136    {
137        order.push(pid);
138    }
139    order.extend(
140        rows.iter()
141            .filter(|r| !pss.contains_key(&r.pid) && allowed(r.pid))
142            .map(|r| r.pid),
143    );
144    let mut stale: Vec<(Instant, u32)> = rows
145        .iter()
146        .filter_map(|r| {
147            let info = pss.get(&r.pid)?;
148            (info.at.elapsed() > refresh_after && allowed(r.pid)).then_some((info.at, r.pid))
149        })
150        .collect();
151    stale.sort_by_key(|(at, _)| *at);
152    order.extend(stale.into_iter().map(|(_, pid)| pid));
153    order
154}
155
156fn tick(st: &mut State, pss_budget: Duration) -> Snapshot {
157    let page_kb = procfs::page_size() / 1024;
158    let t_sweep = Instant::now();
159
160    let pids = procfs::list_pids();
161    let mut rows: Vec<ProcRow> = Vec::with_capacity(pids.len());
162    for pid in pids {
163        let (Some(rss_kb), Some(comm)) = (procfs::read_rss(pid, page_kb), procfs::read_comm(pid))
164        else {
165            continue;
166        };
167        let delta_kb = match st.prev_rss.get(&pid) {
168            Some(prev) => rss_kb as i64 - *prev as i64,
169            None => 0,
170        };
171        st.prev_rss.insert(pid, rss_kb);
172        rows.push(ProcRow {
173            pid,
174            comm,
175            rss_kb,
176            delta_kb,
177        });
178    }
179    rows.sort_by_key(|r| std::cmp::Reverse(r.rss_kb));
180    let sweep_ms = t_sweep.elapsed().as_secs_f32() * 1000.0;
181
182    // Keep the previous map from growing forever.
183    st.prev_rss
184        .retain(|pid, _| rows.iter().any(|r| r.pid == *pid));
185
186    // --- Tier 2: exact PSS/USS, budgeted -------------------------------------
187    let t_pss = Instant::now();
188    for pid in pss_targets(
189        st.focus,
190        &rows,
191        &st.pss,
192        &st.denied,
193        &st.no_mm,
194        PSS_REFRESH_AFTER,
195        PSS_DENIED_TTL,
196    ) {
197        if t_pss.elapsed() > pss_budget && Some(pid) != st.focus {
198            break;
199        }
200        match procfs::read_rollup(pid) {
201            Ok(rollup) => {
202                st.pss.insert(
203                    pid,
204                    PssInfo {
205                        rollup,
206                        at: Instant::now(),
207                    },
208                );
209                st.denied.remove(&pid);
210                st.no_mm.remove(&pid);
211            }
212            Err(procfs::RollupError::NoMemoryMap) => {
213                st.pss.remove(&pid);
214                st.no_mm.insert(pid, Instant::now());
215            }
216            Err(_) => {
217                st.pss.remove(&pid);
218                st.denied.insert(pid, Instant::now());
219            }
220        }
221    }
222    // Forget dead pids so coverage numbers stay honest.
223    let alive: std::collections::HashSet<u32> = rows.iter().map(|r| r.pid).collect();
224    st.pss.retain(|pid, _| alive.contains(pid));
225    st.denied
226        .retain(|pid, at| alive.contains(pid) && at.elapsed() < PSS_DENIED_TTL);
227    st.no_mm
228        .retain(|pid, at| alive.contains(pid) && at.elapsed() < PSS_DENIED_TTL);
229    let pss_ms = t_pss.elapsed().as_secs_f32() * 1000.0;
230
231    // --- Detail: full smaps for the focused pid, only when asked -------------
232    let mut detail = None;
233    if st.want_detail {
234        if let Some(pid) = st.focus {
235            let stale = st
236                .detail
237                .as_ref()
238                .is_none_or(|d| d.pid != pid || d.at.elapsed() > DETAIL_MAX_AGE);
239            if stale {
240                let t = Instant::now();
241                match procfs::read_smaps(pid) {
242                    Ok(report) => {
243                        let rollup = st
244                            .pss
245                            .get(&pid)
246                            .map(|p| p.rollup)
247                            .or_else(|| procfs::read_rollup(pid).ok())
248                            .unwrap_or_default();
249                        st.detail = Some(Detail {
250                            pid,
251                            comm: rows
252                                .iter()
253                                .find(|r| r.pid == pid)
254                                .map(|r| r.comm.clone())
255                                .unwrap_or_default(),
256                            cmdline: procfs::read_cmdline(pid),
257                            report,
258                            rollup,
259                            parse_ms: t.elapsed().as_secs_f32() * 1000.0,
260                            at: Instant::now(),
261                        });
262                        st.detail_error = None;
263                    }
264                    Err(e) => {
265                        st.detail = None;
266                        st.detail_error = Some(format!("/proc/{pid}/smaps: {e}"));
267                    }
268                }
269            }
270        }
271    } else {
272        st.detail = None;
273        st.detail_error = None;
274    }
275    if let Some(d) = &st.detail
276        && st.focus == Some(d.pid)
277    {
278        detail = Some(d.clone());
279    }
280
281    st.ticks = st.ticks.wrapping_add(1);
282    if st.ticks.is_multiple_of(MEMINFO_EVERY) {
283        st.mem = procfs::read_meminfo();
284        st.cgroups = crate::cgroup::scan(std::path::Path::new(crate::cgroup::ROOT));
285        // OOM events are cumulative; report the delta since the last scan and
286        // treat the first sighting as "no events" rather than a spike.
287        for cg in &mut st.cgroups {
288            let prev = st.oom_prev.get(&cg.path).copied().unwrap_or(cg.oom_kills);
289            cg.oom_recent = cg.oom_kills.saturating_sub(prev);
290        }
291        st.oom_prev = st
292            .cgroups
293            .iter()
294            .map(|c| (c.path.clone(), c.oom_kills))
295            .collect();
296        st.scan_seq = st.scan_seq.wrapping_add(1);
297        st.pid_cgroup = std::sync::Arc::new(crate::cgroup::pid_map());
298        st.pid_exe = std::sync::Arc::new(procfs::pid_exe());
299    }
300
301    let fresh: Vec<&PssInfo> = st
302        .pss
303        .values()
304        .filter(|p| p.at.elapsed() < PSS_MAX_AGE)
305        .collect();
306    let covered_pss_kb = fresh.iter().map(|p| p.rollup.pss_kb).sum();
307    let covered = fresh.len();
308
309    Snapshot {
310        rows,
311        pss: st.pss.clone(),
312        mem: st.mem.clone(),
313        pressure: procfs::read_pressure(),
314        scan_seq: st.scan_seq,
315        cgroups: st.cgroups.clone(),
316        pid_cgroup: st.pid_cgroup.clone(),
317        pid_exe: st.pid_exe.clone(),
318        detail,
319        detail_error: if st.want_detail {
320            st.detail_error.clone()
321        } else {
322            None
323        },
324        sweep_ms,
325        pss_ms,
326        covered_pss_kb,
327        covered,
328        denied: st.denied.keys().copied().collect(),
329        no_mm: st.no_mm.keys().copied().collect(),
330        at: Instant::now(),
331    }
332}
333
334#[cfg(test)]
335mod tests {
336    use super::*;
337    use crate::procfs::Rollup;
338    use std::time::Duration;
339
340    fn row(pid: u32, rss_kb: u64) -> ProcRow {
341        ProcRow {
342            pid,
343            comm: format!("p{pid}"),
344            rss_kb,
345            delta_kb: 0,
346        }
347    }
348
349    fn cached(age: Duration) -> PssInfo {
350        PssInfo {
351            rollup: Rollup::default(),
352            at: Instant::now() - age,
353        }
354    }
355
356    #[test]
357    fn focus_is_always_first_even_when_fresh() {
358        let rows = [row(2, 100), row(1, 300)];
359        let pss = HashMap::from([(2, cached(Duration::ZERO))]);
360        let got = pss_targets(
361            Some(2),
362            &rows,
363            &pss,
364            &HashMap::new(),
365            &HashMap::new(),
366            Duration::from_secs(2),
367            Duration::from_secs(30),
368        );
369        assert_eq!(got[0], 2, "focus must be sampled first: {got:?}");
370        assert!(got.contains(&1), "unsampled pid must be queued: {got:?}");
371    }
372
373    #[test]
374    fn unsampled_rows_are_queued_in_rss_order() {
375        // rows arrive sorted by rss desc; the biggest unknown is the row a PSS
376        // sort would misrank, so it must be resolved first.
377        let rows = [row(10, 500), row(11, 400), row(12, 300)];
378        let got = pss_targets(
379            None,
380            &rows,
381            &HashMap::new(),
382            &HashMap::new(),
383            &HashMap::new(),
384            Duration::from_secs(2),
385            Duration::from_secs(30),
386        );
387        assert_eq!(got, vec![10, 11, 12]);
388    }
389
390    #[test]
391    fn stale_entries_go_oldest_first_and_fresh_ones_are_skipped() {
392        let rows = [row(1, 500), row(2, 400), row(3, 300)];
393        let pss = HashMap::from([
394            (1, cached(Duration::from_secs(10))),
395            (2, cached(Duration::from_secs(3))),
396            (3, cached(Duration::from_secs(1))),
397        ]);
398        let got = pss_targets(
399            None,
400            &rows,
401            &pss,
402            &HashMap::new(),
403            &HashMap::new(),
404            Duration::from_secs(2),
405            Duration::from_secs(30),
406        );
407        assert_eq!(
408            got,
409            vec![1, 2],
410            "oldest first, fresh pid 3 untouched: {got:?}"
411        );
412    }
413
414    #[test]
415    fn empty_system_yields_no_targets() {
416        assert!(
417            pss_targets(
418                None,
419                &[],
420                &HashMap::new(),
421                &HashMap::new(),
422                &HashMap::new(),
423                Duration::from_secs(2),
424                Duration::from_secs(30)
425            )
426            .is_empty()
427        );
428    }
429
430    #[test]
431    fn denied_pids_do_not_consume_the_budget_until_the_denial_expires() {
432        // 234 of ~450 pids are EACCES as a normal user; retrying them every
433        // tick is what starved the readable ones.
434        let rows = [row(1, 500), row(2, 400), row(3, 300)];
435        let denied = HashMap::from([
436            (1, Instant::now()),
437            (3, Instant::now() - Duration::from_secs(31)),
438        ]);
439        let got = pss_targets(
440            Some(1),
441            &rows,
442            &HashMap::new(),
443            &denied,
444            &HashMap::new(),
445            Duration::from_secs(2),
446            Duration::from_secs(30),
447        );
448        assert_eq!(
449            got,
450            vec![2, 3],
451            "fresh denials skipped, expired retried: {got:?}"
452        );
453    }
454}