rigtop_core/linux/
oom.rs

1//! Processes stopped by the kernel's OOM killer, from the `oom_kill`
2//! counter of `/proc/vmstat` (no root needed; it counts system-wide and
3//! cgroup OOM kills, not those of userspace killers such as systemd-oomd).
4//!
5//! The kernel log that names the process needs root, so the likely one is
6//! the largest process that went away in the same interval, if it was
7//! large enough to be one ([`LIKELY_MIN_BYTES`]): one that started and was
8//! stopped between two samples is not known, and a small process that
9//! merely exited would be a wrong guess.
10
11use std::collections::{HashMap, HashSet};
12use std::fs;
13
14use super::SysRoots;
15use crate::snapshot::{OomKill, ProcessInfo};
16
17/// How long a kill stays in the snapshot (and so in the alerts).
18pub const OOM_KILL_WINDOW_MS: u64 = 60 * 60 * 1000;
19
20/// Smallest process taken for the one the OOM killer stopped.
21const LIKELY_MIN_BYTES: u64 = 64 << 20;
22
23/// Counts OOM kills between samples.
24#[derive(Debug, Default)]
25pub struct OomTracker {
26    /// The counter at the previous sample.
27    last_count: Option<u64>,
28    /// Memory of each process at the previous sample: pid -> (name, bytes).
29    previous: HashMap<u32, (String, u64)>,
30    /// Kills seen within [`OOM_KILL_WINDOW_MS`], oldest first.
31    kills: Vec<OomKill>,
32}
33
34impl OomTracker {
35    /// Creates a tracker; the first sample only sets the starting count.
36    pub fn new() -> Self {
37        Self::default()
38    }
39
40    /// Kills seen in the last [`OOM_KILL_WINDOW_MS`] up to `now_ms`, given
41    /// the processes of this sample.
42    pub fn sample(
43        &mut self,
44        roots: &SysRoots,
45        processes: &[ProcessInfo],
46        now_ms: u64,
47    ) -> Vec<OomKill> {
48        let count = fs::read_to_string(roots.proc.join("vmstat"))
49            .ok()
50            .and_then(|text| parse_oom_kill(&text));
51        if let (Some(count), Some(last)) = (count, self.last_count) {
52            let new = count.saturating_sub(last);
53            if new > 0 {
54                let likely = likely_victims(&self.previous, processes, new as usize);
55                for i in 0..new as usize {
56                    self.kills.push(OomKill {
57                        at_ms: now_ms,
58                        likely_process: likely.get(i).cloned(),
59                    });
60                }
61            }
62        }
63        if count.is_some() {
64            self.last_count = count;
65        }
66        self.previous = processes
67            .iter()
68            .map(|p| (p.pid, (p.name.clone(), p.memory_bytes)))
69            .collect();
70        self.kills
71            .retain(|k| now_ms.saturating_sub(k.at_ms) < OOM_KILL_WINDOW_MS);
72        self.kills.clone()
73    }
74}
75
76/// The value of the `oom_kill` line of `/proc/vmstat` text.
77pub(crate) fn parse_oom_kill(text: &str) -> Option<u64> {
78    text.lines()
79        .find_map(|line| line.strip_prefix("oom_kill "))
80        .and_then(|value| value.trim().parse().ok())
81}
82
83/// Names of the `count` largest processes of `previous` missing from
84/// `current`, largest first.
85fn likely_victims(
86    previous: &HashMap<u32, (String, u64)>,
87    current: &[ProcessInfo],
88    count: usize,
89) -> Vec<String> {
90    let alive: HashSet<u32> = current.iter().map(|p| p.pid).collect();
91    let mut gone: Vec<&(String, u64)> = previous
92        .iter()
93        .filter(|(pid, (_, bytes))| !alive.contains(pid) && *bytes >= LIKELY_MIN_BYTES)
94        .map(|(_, process)| process)
95        .collect();
96    gone.sort_by_key(|(_, bytes)| std::cmp::Reverse(*bytes));
97    gone.into_iter()
98        .take(count)
99        .map(|(name, _)| name.clone())
100        .collect()
101}
102
103#[cfg(test)]
104mod tests {
105    use super::*;
106    use crate::linux::testutil::write;
107
108    fn process(pid: u32, name: &str, gib: u64) -> ProcessInfo {
109        ProcessInfo {
110            pid,
111            name: name.to_owned(),
112            memory_bytes: gib << 30,
113            ..Default::default()
114        }
115    }
116
117    #[test]
118    fn parses_the_counter() {
119        let text = "pgfault 123\noom_kill 4\nnr_free_pages 5\n";
120        assert_eq!(parse_oom_kill(text), Some(4));
121        // Kernels before 4.13 have no such line.
122        assert_eq!(parse_oom_kill("pgfault 123\n"), None);
123    }
124
125    #[test]
126    fn records_kills_with_the_likely_process_and_forgets_them() {
127        let tmp = tempfile::tempdir().unwrap();
128        let roots = SysRoots {
129            sys: tmp.path().join("sys"),
130            proc: tmp.path().join("proc"),
131        };
132        let mut tracker = OomTracker::new();
133        let before = vec![
134            process(1, "systemd", 0),
135            process(10, "python3", 60),
136            process(11, "bash", 0),
137        ];
138
139        // Kills before rigtop started are not reported.
140        write(tmp.path(), "proc/vmstat", "oom_kill 3\n");
141        assert!(tracker.sample(&roots, &before, 1_000).is_empty());
142
143        // The large python3 and the small bash went away; two kills, but
144        // bash is too small to be the other one.
145        write(tmp.path(), "proc/vmstat", "oom_kill 5\n");
146        let after = vec![process(1, "systemd", 0)];
147        let kills = tracker.sample(&roots, &after, 2_000);
148        let kill = |name: Option<&str>| OomKill {
149            at_ms: 2_000,
150            likely_process: name.map(str::to_owned),
151        };
152        assert_eq!(kills, vec![kill(Some("python3")), kill(None)]);
153
154        // Still reported within the window, gone after it.
155        assert_eq!(tracker.sample(&roots, &after, 3_000).len(), 2);
156        assert!(
157            tracker
158                .sample(&roots, &after, 2_000 + OOM_KILL_WINDOW_MS)
159                .is_empty()
160        );
161    }
162}
rigtop_core/linux/oom.rs

rigtop_core/linux/
oom.rs
