runner_manager_platform/
resource_usage.rs

1//! What a runner's process tree uses right now: resident memory and CPU time
2//! summed over a root process and every descendant (coordinator stage 4, job
3//! weight statistics).
4//!
5//! One call is one cheap look, taken every few seconds by the daemon off its
6//! reconcile loop:
7//!
8//! * **Linux** reads `/proc/<pid>/stat` for every process (parent, CPU ticks,
9//!   resident pages). A native runner does not get a cgroup of its own, so
10//!   the tree is the unit.
11//! * **macOS** lists every PID (`proc_listallpids`), reads each parent
12//!   (`PROC_PIDT_SHORTBSDINFO`, which any account may read, so a `sudo` step
13//!   keeps its place in the tree) and, for the tree, `PROC_PIDTASKINFO`
14//!   (resident size, user and system time in Mach ticks).
15//! * **Windows** takes one Toolhelp snapshot for the parent links and, for
16//!   the tree, `GetProcessMemoryInfo` (working set) and `GetProcessTimes`. A
17//!   native runner is not started inside a Job Object, so there is no job
18//!   accounting to read. Toolhelp keeps a dead parent's PID, which Windows
19//!   soon hands to another process, so a child created before its supposed
20//!   parent is not counted as its child.
21//!
22//! The CPU total of a tree drops when a descendant exits (except on Linux,
23//! which folds a reaped child into its parent's `cutime`/`cstime`); the
24//! caller adds up the increases between looks, so at most a look's worth of
25//! an exiting process's time is lost. A process this account may not
26//! inspect counts as using nothing. Work that leaves the tree is not seen:
27//! containers run by `dockerd`, and daemons that detach (a Gradle daemon, a
28//! `nohup` server) and are re-parented to init.
29
30use std::collections::{HashMap, HashSet, VecDeque};
31use std::time::Duration;
32
33/// One look at a process tree.
34#[derive(Debug, Clone, Copy, Default, PartialEq, Eq)]
35pub struct TreeUsage {
36    /// Resident memory of every process in the tree, summed.
37    pub rss_bytes: u64,
38    /// CPU time the tree has used, summed.
39    pub cpu_time: Duration,
40    /// How many processes the tree held.
41    pub processes: u32,
42}
43
44/// What one process uses: resident memory and CPU time.
45type Facts = (u64, Duration);
46
47/// The tree under `root`, `root` included, or `None` when `root` is gone.
48#[must_use]
49pub fn sample_tree(root: u32) -> Option<TreeUsage> {
50    sample_trees(&[root]).remove(&root)
51}
52
53/// The trees under each of `roots`, from one look at the process table: a
54/// root that is gone has no entry.
55#[must_use]
56pub fn sample_trees(roots: &[u32]) -> HashMap<u32, TreeUsage> {
57    if roots.is_empty() {
58        return HashMap::new();
59    }
60    sys::sample_trees(roots)
61}
62
63/// Sums `facts` over the tree under each of `roots`, given every process's
64/// `(pid, parent)`. A process `facts` cannot read counts as using nothing; a
65/// cycle (a recycled PID naming a descendant as its parent) is walked once.
66fn trees(
67    roots: &[u32],
68    parents: &[(u32, u32)],
69    mut facts: impl FnMut(u32) -> Option<Facts>,
70    mut is_child: impl FnMut(u32, u32) -> bool,
71) -> HashMap<u32, TreeUsage> {
72    let mut children: HashMap<u32, Vec<u32>> = HashMap::new();
73    for (pid, parent) in parents {
74        if pid != parent {
75            children.entry(*parent).or_default().push(*pid);
76        }
77    }
78    let alive: HashSet<u32> = parents.iter().map(|(pid, _)| *pid).collect();
79    let mut found = HashMap::new();
80    for root in roots.iter().copied().filter(|root| alive.contains(root)) {
81        let mut usage = TreeUsage::default();
82        let mut seen = HashSet::from([root]);
83        let mut queue = VecDeque::from([root]);
84        while let Some(pid) = queue.pop_front() {
85            let (rss, cpu) = facts(pid).unwrap_or_default();
86            usage.rss_bytes = usage.rss_bytes.saturating_add(rss);
87            usage.cpu_time = usage.cpu_time.saturating_add(cpu);
88            usage.processes = usage.processes.saturating_add(1);
89            for child in children.get(&pid).into_iter().flatten() {
90                if !seen.contains(child) && is_child(*child, pid) {
91                    seen.insert(*child);
92                    queue.push_back(*child);
93                }
94            }
95        }
96        found.insert(root, usage);
97    }
98    found
99}
100
101/// Fields of one `/proc/<pid>/stat` line: the parent, the CPU ticks the
102/// process and its waited-for children used, and its resident pages. The
103/// command name may hold spaces and parentheses, so the fields are counted
104/// from the last `)` ([`crate::process::proc_stat_after_comm`]).
105#[cfg_attr(not(target_os = "linux"), allow(dead_code))]
106fn parse_proc_stat(line: &str) -> Option<(u32, u64, u64)> {
107    let fields: Vec<&str> = crate::process::proc_stat_after_comm(line)?
108        .split_whitespace()
109        .collect();
110    // After the name: state(3) ppid(4) ... utime(14) stime(15) cutime(16)
111    // cstime(17) ... rss(24); index 0 here is field 3.
112    let field = |number: usize| fields.get(number - 3).copied();
113    let parent = field(4)?.parse().ok()?;
114    let ticks = [14, 15, 16, 17]
115        .into_iter()
116        .map(|number| field(number)?.parse::<i64>().ok())
117        .try_fold(0u64, |sum, value| {
118            Some(sum.saturating_add(u64::try_from(value?).unwrap_or(0)))
119        })?;
120    let pages = u64::try_from(field(24)?.parse::<i64>().ok()?).unwrap_or(0);
121    Some((parent, ticks, pages))
122}
123
124/// Whether one `/proc/<pid>/stat` line is of a process that has exited. A zombie
125/// stays in `/proc` until its parent reaps it, and a runner's parent is the daemon,
126/// which reaps it in the pass this look exists to bring forward.
127#[cfg_attr(not(target_os = "linux"), allow(dead_code))]
128fn proc_stat_exited(line: &str) -> bool {
129    crate::process::proc_stat_after_comm(line)
130        .and_then(|fields| fields.split_whitespace().next())
131        .is_some_and(|state| matches!(state, "Z" | "X" | "x"))
132}
133
134#[cfg(target_os = "linux")]
135mod sys {
136    use std::collections::HashMap;
137    use std::time::Duration;
138
139    use super::{Facts, TreeUsage, parse_proc_stat, proc_stat_exited, trees};
140
141    pub(super) fn sample_trees(roots: &[u32]) -> HashMap<u32, TreeUsage> {
142        // SAFETY: `sysconf` reads a constant and cannot fail in a way that
143        // matters here; a non-positive answer is replaced below.
144        let ticks_per_second = u64::try_from(unsafe { libc::sysconf(libc::_SC_CLK_TCK) })
145            .ok()
146            .filter(|ticks| *ticks > 0)
147            .unwrap_or(100);
148        // SAFETY: as above.
149        let page = u64::try_from(unsafe { libc::sysconf(libc::_SC_PAGESIZE) })
150            .ok()
151            .filter(|page| *page > 0)
152            .unwrap_or(4096);
153        let mut parents = Vec::new();
154        let mut facts: HashMap<u32, Facts> = HashMap::new();
155        let Ok(entries) = std::fs::read_dir("/proc") else {
156            return HashMap::new();
157        };
158        for entry in entries.flatten() {
159            let Some(pid) = entry
160                .file_name()
161                .to_str()
162                .and_then(|n| n.parse::<u32>().ok())
163            else {
164                continue;
165            };
166            // Gone meanwhile, or not readable: skipped.
167            let Ok(line) = std::fs::read_to_string(entry.path().join("stat")) else {
168                continue;
169            };
170            // A root that exited is gone, or its slot waits for the next scheduled
171            // poll. Not a descendant: it would drop out of the tree's CPU total, and
172            // return to it through its parent's once reaped, counted twice.
173            if roots.contains(&pid) && proc_stat_exited(&line) {
174                continue;
175            }
176            if let Some((parent, ticks, pages)) = parse_proc_stat(&line) {
177                parents.push((pid, parent));
178                facts.insert(
179                    pid,
180                    (
181                        pages.saturating_mul(page),
182                        Duration::from_nanos(
183                            ticks.saturating_mul(1_000_000_000) / ticks_per_second,
184                        ),
185                    ),
186                );
187            }
188        }
189        trees(roots, &parents, |pid| facts.get(&pid).copied(), |_, _| true)
190    }
191}
192
193#[cfg(target_os = "macos")]
194mod sys {
195    use std::collections::HashMap;
196    use std::time::Duration;
197
198    use super::{TreeUsage, trees};
199
200    fn info<T>(pid: i32, flavor: i32) -> Option<T> {
201        let size = i32::try_from(std::mem::size_of::<T>()).ok()?;
202        let mut value = std::mem::MaybeUninit::<T>::zeroed();
203        // SAFETY: `value` is a zeroed buffer of exactly `size` bytes for the
204        // plain-data structure `flavor` fills; the call writes at most that.
205        let written =
206            unsafe { libc::proc_pidinfo(pid, flavor, 0, value.as_mut_ptr().cast(), size) };
207        // SAFETY: the kernel filled the whole structure (`written == size`),
208        // and zeroed is a valid value of these plain-data types anyway.
209        (written == size).then(|| unsafe { value.assume_init() })
210    }
211
212    /// Mach ticks to nanoseconds: `PROC_PIDTASKINFO` times are in ticks,
213    /// which are nanoseconds on Intel and not on Apple silicon.
214    #[allow(
215        deprecated,
216        reason = "libc marks the mach bindings deprecated in favour of a crate this workspace does not take; the call is stable"
217    )]
218    fn ticks_to_nanos(ticks: u64) -> u64 {
219        static BASE: std::sync::OnceLock<(u32, u32)> = std::sync::OnceLock::new();
220        let (numer, denom) = *BASE.get_or_init(|| {
221            let mut base = libc::mach_timebase_info { numer: 0, denom: 0 };
222            // SAFETY: `base` is a live structure owned by this frame.
223            let ok = unsafe { libc::mach_timebase_info(&raw mut base) } == 0;
224            if ok && base.denom != 0 {
225                (base.numer, base.denom)
226            } else {
227                (1, 1)
228            }
229        });
230        u64::try_from(u128::from(ticks) * u128::from(numer) / u128::from(denom)).unwrap_or(u64::MAX)
231    }
232
233    fn all_pids() -> Vec<i32> {
234        // SAFETY: a null buffer asks for the count only.
235        let count = unsafe { libc::proc_listallpids(std::ptr::null_mut(), 0) };
236        let Ok(count) = usize::try_from(count) else {
237            return Vec::new();
238        };
239        // Room for processes started since the count.
240        let mut pids = vec![0i32; count + 64];
241        let bytes = i32::try_from(pids.len() * std::mem::size_of::<i32>()).unwrap_or(i32::MAX);
242        // SAFETY: `pids` holds `bytes` bytes and outlives the call.
243        let filled = unsafe { libc::proc_listallpids(pids.as_mut_ptr().cast(), bytes) };
244        pids.truncate(usize::try_from(filled).unwrap_or(0).min(pids.len()));
245        pids
246    }
247
248    pub(super) fn sample_trees(roots: &[u32]) -> HashMap<u32, TreeUsage> {
249        let parents: Vec<(u32, u32)> = all_pids()
250            .into_iter()
251            .filter(|pid| *pid > 0)
252            .filter_map(|pid| {
253                let bsd: libc::proc_bsdshortinfo = info(pid, libc::PROC_PIDT_SHORTBSDINFO)?;
254                Some((u32::try_from(pid).ok()?, bsd.pbsi_ppid))
255            })
256            .collect();
257        trees(
258            roots,
259            &parents,
260            |pid| {
261                let task: libc::proc_taskinfo =
262                    info(i32::try_from(pid).ok()?, libc::PROC_PIDTASKINFO)?;
263                Some((
264                    task.pti_resident_size,
265                    Duration::from_nanos(ticks_to_nanos(
266                        task.pti_total_user.saturating_add(task.pti_total_system),
267                    )),
268                ))
269            },
270            |_, _| true,
271        )
272    }
273}
274
275#[cfg(windows)]
276mod sys {
277    use std::collections::HashMap;
278    use std::time::Duration;
279
280    use windows::Win32::Foundation::{CloseHandle, FILETIME};
281    use windows::Win32::System::Diagnostics::ToolHelp::{
282        CreateToolhelp32Snapshot, PROCESSENTRY32W, Process32FirstW, Process32NextW,
283        TH32CS_SNAPPROCESS,
284    };
285    use windows::Win32::System::ProcessStatus::{GetProcessMemoryInfo, PROCESS_MEMORY_COUNTERS};
286    use windows::Win32::System::Threading::{
287        GetProcessTimes, OpenProcess, PROCESS_QUERY_LIMITED_INFORMATION,
288    };
289
290    use super::{Facts, TreeUsage, trees};
291
292    fn parents() -> Vec<(u32, u32)> {
293        // SAFETY: a process snapshot takes no pointer; the handle is closed
294        // below exactly once.
295        let Ok(snapshot) = (unsafe { CreateToolhelp32Snapshot(TH32CS_SNAPPROCESS, 0) }) else {
296            return Vec::new();
297        };
298        let mut entry = PROCESSENTRY32W {
299            dwSize: u32::try_from(std::mem::size_of::<PROCESSENTRY32W>()).unwrap_or(0),
300            ..PROCESSENTRY32W::default()
301        };
302        let mut pairs = Vec::new();
303        // SAFETY: `entry` is a live, size-stamped structure owned by this
304        // frame, and `snapshot` is open.
305        let mut more = unsafe { Process32FirstW(snapshot, &raw mut entry) }.is_ok();
306        while more {
307            pairs.push((entry.th32ProcessID, entry.th32ParentProcessID));
308            // SAFETY: as above.
309            more = unsafe { Process32NextW(snapshot, &raw mut entry) }.is_ok();
310        }
311        // SAFETY: opened above, closed once.
312        let _ = unsafe { CloseHandle(snapshot) };
313        pairs
314    }
315
316    /// A `FILETIME` in 100 ns units.
317    fn hundreds(time: FILETIME) -> u64 {
318        (u64::from(time.dwHighDateTime) << 32) | u64::from(time.dwLowDateTime)
319    }
320
321    fn filetime(time: FILETIME) -> Duration {
322        Duration::from_nanos(hundreds(time).saturating_mul(100))
323    }
324
325    /// What one process uses, and when it was created; `None` when this
326    /// account cannot open it.
327    fn facts(pid: u32) -> Option<(Facts, Option<u64>)> {
328        // SAFETY: a plain call; the handle is closed below exactly once.
329        // Querying memory and times needs no more than limited information.
330        let handle = unsafe { OpenProcess(PROCESS_QUERY_LIMITED_INFORMATION, false, pid) }.ok()?;
331        let mut memory = PROCESS_MEMORY_COUNTERS {
332            cb: u32::try_from(std::mem::size_of::<PROCESS_MEMORY_COUNTERS>()).unwrap_or(0),
333            ..PROCESS_MEMORY_COUNTERS::default()
334        };
335        // SAFETY: `memory` is a live, size-stamped structure owned by this
336        // frame; `handle` is open.
337        let rss = if unsafe { GetProcessMemoryInfo(handle, &raw mut memory, memory.cb) }.is_ok() {
338            u64::try_from(memory.WorkingSetSize).unwrap_or(u64::MAX)
339        } else {
340            0
341        };
342        let (mut created, mut exited, mut kernel, mut user) = (
343            FILETIME::default(),
344            FILETIME::default(),
345            FILETIME::default(),
346            FILETIME::default(),
347        );
348        // SAFETY: four live out-parameters owned by this frame.
349        let timed = unsafe {
350            GetProcessTimes(
351                handle,
352                &raw mut created,
353                &raw mut exited,
354                &raw mut kernel,
355                &raw mut user,
356            )
357        };
358        let (cpu, born) = if timed.is_ok() {
359            (
360                filetime(kernel).saturating_add(filetime(user)),
361                Some(hundreds(created)),
362            )
363        } else {
364            (Duration::ZERO, None)
365        };
366        // SAFETY: opened above, closed once.
367        let _ = unsafe { CloseHandle(handle) };
368        Some(((rss, cpu), born))
369    }
370
371    pub(super) fn sample_trees(roots: &[u32]) -> HashMap<u32, TreeUsage> {
372        let known = std::cell::RefCell::new(HashMap::<u32, Option<(Facts, Option<u64>)>>::new());
373        let look = |pid: u32| *known.borrow_mut().entry(pid).or_insert_with(|| facts(pid));
374        trees(
375            roots,
376            &parents(),
377            |pid| look(pid).map(|(facts, _)| facts),
378            // A child created before its parent names a dead parent whose
379            // PID was reused: not this tree's.
380            |child, parent| match (look(child), look(parent)) {
381                (Some((_, Some(child))), Some((_, Some(parent)))) => child >= parent,
382                _ => true,
383            },
384        )
385    }
386}
387
388#[cfg(not(any(target_os = "linux", target_os = "macos", windows)))]
389mod sys {
390    use std::collections::HashMap;
391
392    use super::TreeUsage;
393
394    pub(super) fn sample_trees(_: &[u32]) -> HashMap<u32, TreeUsage> {
395        HashMap::new()
396    }
397}
398
399#[cfg(test)]
400mod tests {
401    use super::*;
402
403    #[test]
404    fn a_stat_line_parses_whatever_the_command_name_holds() {
405        let line = "4242 (Runner.Worker (x) 1) S 4200 4242 4200 0 -1 4194560 100 0 0 0 \
406                    250 50 7 3 20 0 12 0 123456 1048576000 25600 18446744073709551615";
407        assert_eq!(parse_proc_stat(line), Some((4200, 310, 25600)));
408        assert_eq!(parse_proc_stat("garbage"), None);
409        assert_eq!(parse_proc_stat("1 (init) S"), None);
410    }
411
412    #[test]
413    fn a_tree_sums_its_descendants_only_and_survives_a_cycle() {
414        let parents = [
415            (1, 0),
416            (10, 1),
417            (11, 10),
418            (12, 11),
419            (20, 1),
420            (30, 12),
421            (12, 30),
422        ];
423        let facts: HashMap<u32, Facts> = [
424            (1, (1, 1)),
425            (10, (100, 10)),
426            (11, (20, 2)),
427            (12, (30, 3)),
428            (20, (999, 99)),
429            (30, (5, 1)),
430        ]
431        .into_iter()
432        .map(|(pid, (rss, secs))| (pid, (rss, Duration::from_secs(secs))))
433        .collect();
434        let found = trees(
435            &[10, 99],
436            &parents,
437            |pid| facts.get(&pid).copied(),
438            |_, _| true,
439        );
440        assert_eq!(
441            found,
442            HashMap::from([(
443                10,
444                TreeUsage {
445                    rss_bytes: 155,
446                    cpu_time: Duration::from_secs(16),
447                    processes: 4,
448                }
449            )]),
450            "the gone root 99 has no entry"
451        );
452        // An unreadable process still counts as one of the tree.
453        let unreadable = trees(&[10], &parents, |_| None, |_, _| true);
454        assert_eq!(unreadable[&10].processes, 4);
455        // A link the platform disowns (a stale parent PID) cuts that subtree.
456        let cut = trees(
457            &[10],
458            &parents,
459            |pid| facts.get(&pid).copied(),
460            |child, _| child != 11,
461        );
462        assert_eq!(cut[&10].processes, 1);
463    }
464
465    #[test]
466    fn a_stat_line_says_whether_the_process_exited() {
467        let line = |state: &str| {
468            format!("7 (run.sh) {state} 1 7 7 0 -1 0 0 0 0 0 5 6 7 8 20 0 1 0 100 4096 42 0")
469        };
470        assert!(!proc_stat_exited(&line("S")));
471        assert!(!proc_stat_exited(&line("R")));
472        assert!(proc_stat_exited(&line("Z")));
473        assert!(proc_stat_exited(&line("X")));
474        assert!(!proc_stat_exited("garbage"));
475    }
476
477    /// A runner that exited stays in the process table until its parent reaps
478    /// it, and nothing reaps it but the pass this look exists to bring forward. So
479    /// it has to read as gone, or a slot waits for the next scheduled poll.
480    #[test]
481    fn a_process_that_exited_and_was_not_reaped_is_gone() {
482        #[cfg(unix)]
483        let mut child = std::process::Command::new("sh")
484            .args(["-c", "exit 0"])
485            .spawn()
486            .expect("sh starts");
487        #[cfg(windows)]
488        let mut child = std::process::Command::new("cmd")
489            .args(["/C", "exit 0"])
490            .spawn()
491            .expect("cmd starts");
492        let root = child.id();
493        let mut gone = false;
494        for _ in 0..50 {
495            std::thread::sleep(Duration::from_millis(100));
496            if sample_tree(root).is_none() {
497                gone = true;
498                break;
499            }
500        }
501        let reaped = child.wait();
502        assert!(reaped.is_ok(), "it was still ours to reap: {reaped:?}");
503        assert!(gone, "an exited process still reads as running");
504    }
505
506    /// A real child tree of this process: the measurement on the current OS.
507    #[cfg(unix)]
508    #[test]
509    fn a_spawned_process_tree_is_measured_on_this_os() {
510        // A shell that starts a second process and burns some CPU, so both
511        // the tree and its CPU time are non-trivial.
512        let mut child = std::process::Command::new("sh")
513            .args([
514                "-c",
515                "sleep 30 & i=0; while [ $i -lt 200000 ]; do i=$((i+1)); done; wait",
516            ])
517            .spawn()
518            .expect("sh starts");
519        let root = child.id();
520        let mut best = None;
521        for _ in 0..50 {
522            std::thread::sleep(Duration::from_millis(100));
523            if let Some(usage) = sample_tree(root)
524                && usage.processes >= 2
525                && usage.cpu_time > Duration::ZERO
526            {
527                best = Some(usage);
528                break;
529            }
530        }
531        let _ = child.kill();
532        let _ = child.wait();
533        let usage = best.expect("the tree was seen with its child and some CPU");
534        assert!(usage.rss_bytes > 0, "{usage:?}");
535        assert_eq!(sample_tree(root), None, "a reaped root is gone");
536    }
537
538    #[cfg(windows)]
539    #[test]
540    fn a_spawned_process_tree_is_measured_on_this_os() {
541        let mut child = std::process::Command::new("cmd")
542            .args(["/C", "ping -n 6 127.0.0.1 > NUL"])
543            .spawn()
544            .expect("cmd starts");
545        let root = child.id();
546        let mut best = None;
547        for _ in 0..50 {
548            std::thread::sleep(Duration::from_millis(100));
549            if let Some(usage) = sample_tree(root)
550                && usage.processes >= 2
551            {
552                best = Some(usage);
553                break;
554            }
555        }
556        let _ = child.kill();
557        let _ = child.wait();
558        let usage = best.expect("the tree was seen with its child");
559        assert!(usage.rss_bytes > 0, "{usage:?}");
560    }
561}
runner_manager_platform/resource_usage.rs

runner_manager_platform/
resource_usage.rs
