runner_manager_platform/
resource_usage.rs

1//! What a runner's process tree uses right now: resident memory and CPU time
2//! summed over a root process and every descendant (coordinator stage 4, job
3//! weight statistics).
4//!
5//! One call is one cheap look, taken every few seconds by the daemon off its
6//! reconcile loop:
7//!
8//! * **Linux** reads `/proc/<pid>/stat` for every process (parent, CPU ticks,
9//!   resident pages). A native runner does not get a cgroup of its own, so
10//!   the tree is the unit.
11//! * **macOS** lists every PID (`proc_listallpids`), reads each parent
12//!   (`PROC_PIDT_SHORTBSDINFO`, which any account may read, so a `sudo` step
13//!   keeps its place in the tree) and, for the tree, `PROC_PIDTASKINFO`
14//!   (resident size, user and system time in Mach ticks).
15//! * **Windows** takes one Toolhelp snapshot for the parent links and, for
16//!   the tree, `GetProcessMemoryInfo` (working set) and `GetProcessTimes`. A
17//!   native runner is not started inside a Job Object, so there is no job
18//!   accounting to read. Toolhelp keeps a dead parent's PID, which Windows
19//!   soon hands to another process, so a child created before its supposed
20//!   parent is not counted as its child.
21//!
22//! The CPU total of a tree drops when a descendant exits (except on Linux,
23//! which folds a reaped child into its parent's `cutime`/`cstime`); the
24//! caller adds up the increases between looks, so at most a look's worth of
25//! an exiting process's time is lost. A process this account may not
26//! inspect counts as using nothing. Work that leaves the tree is not seen:
27//! containers run by `dockerd`, and daemons that detach (a Gradle daemon, a
28//! `nohup` server) and are re-parented to init.
29
30use std::collections::{HashMap, HashSet, VecDeque};
31use std::time::Duration;
32
33/// One look at a process tree.
34#[derive(Debug, Clone, Copy, Default, PartialEq, Eq)]
35pub struct TreeUsage {
36    /// Resident memory of every process in the tree, summed.
37    pub rss_bytes: u64,
38    /// CPU time the tree has used, summed.
39    pub cpu_time: Duration,
40    /// How many processes the tree held.
41    pub processes: u32,
42}
43
44/// What one process uses: resident memory and CPU time.
45type Facts = (u64, Duration);
46
47/// The tree under `root`, `root` included, or `None` when `root` is gone.
48#[must_use]
49pub fn sample_tree(root: u32) -> Option<TreeUsage> {
50    sample_trees(&[root]).remove(&root)
51}
52
53/// The trees under each of `roots`, from one look at the process table: a
54/// root that is gone has no entry.
55#[must_use]
56pub fn sample_trees(roots: &[u32]) -> HashMap<u32, TreeUsage> {
57    if roots.is_empty() {
58        return HashMap::new();
59    }
60    sys::sample_trees(roots)
61}
62
63/// Sums `facts` over the tree under each of `roots`, given every process's
64/// `(pid, parent)`. A process `facts` cannot read counts as using nothing; a
65/// cycle (a recycled PID naming a descendant as its parent) is walked once.
66fn trees(
67    roots: &[u32],
68    parents: &[(u32, u32)],
69    mut facts: impl FnMut(u32) -> Option<Facts>,
70    mut is_child: impl FnMut(u32, u32) -> bool,
71) -> HashMap<u32, TreeUsage> {
72    let mut children: HashMap<u32, Vec<u32>> = HashMap::new();
73    for (pid, parent) in parents {
74        if pid != parent {
75            children.entry(*parent).or_default().push(*pid);
76        }
77    }
78    let alive: HashSet<u32> = parents.iter().map(|(pid, _)| *pid).collect();
79    let mut found = HashMap::new();
80    for root in roots.iter().copied().filter(|root| alive.contains(root)) {
81        let mut usage = TreeUsage::default();
82        let mut seen = HashSet::from([root]);
83        let mut queue = VecDeque::from([root]);
84        while let Some(pid) = queue.pop_front() {
85            let (rss, cpu) = facts(pid).unwrap_or_default();
86            usage.rss_bytes = usage.rss_bytes.saturating_add(rss);
87            usage.cpu_time = usage.cpu_time.saturating_add(cpu);
88            usage.processes = usage.processes.saturating_add(1);
89            for child in children.get(&pid).into_iter().flatten() {
90                if !seen.contains(child) && is_child(*child, pid) {
91                    seen.insert(*child);
92                    queue.push_back(*child);
93                }
94            }
95        }
96        found.insert(root, usage);
97    }
98    found
99}
100
101/// Fields of one `/proc/<pid>/stat` line: the parent, the CPU ticks the
102/// process and its waited-for children used, and its resident pages. The
103/// command name may hold spaces and parentheses, so the fields are counted
104/// from the last `)` ([`crate::process::proc_stat_after_comm`]).
105#[cfg_attr(not(target_os = "linux"), allow(dead_code))]
106fn parse_proc_stat(line: &str) -> Option<(u32, u64, u64)> {
107    let fields: Vec<&str> = crate::process::proc_stat_after_comm(line)?
108        .split_whitespace()
109        .collect();
110    // After the name: state(3) ppid(4) ... utime(14) stime(15) cutime(16)
111    // cstime(17) ... rss(24); index 0 here is field 3.
112    let field = |number: usize| fields.get(number - 3).copied();
113    let parent = field(4)?.parse().ok()?;
114    let ticks = [14, 15, 16, 17]
115        .into_iter()
116        .map(|number| field(number)?.parse::<i64>().ok())
117        .try_fold(0u64, |sum, value| {
118            Some(sum.saturating_add(u64::try_from(value?).unwrap_or(0)))
119        })?;
120    let pages = u64::try_from(field(24)?.parse::<i64>().ok()?).unwrap_or(0);
121    Some((parent, ticks, pages))
122}
123
124#[cfg(target_os = "linux")]
125mod sys {
126    use std::collections::HashMap;
127    use std::time::Duration;
128
129    use super::{Facts, TreeUsage, parse_proc_stat, trees};
130
131    pub(super) fn sample_trees(roots: &[u32]) -> HashMap<u32, TreeUsage> {
132        // SAFETY: `sysconf` reads a constant and cannot fail in a way that
133        // matters here; a non-positive answer is replaced below.
134        let ticks_per_second = u64::try_from(unsafe { libc::sysconf(libc::_SC_CLK_TCK) })
135            .ok()
136            .filter(|ticks| *ticks > 0)
137            .unwrap_or(100);
138        // SAFETY: as above.
139        let page = u64::try_from(unsafe { libc::sysconf(libc::_SC_PAGESIZE) })
140            .ok()
141            .filter(|page| *page > 0)
142            .unwrap_or(4096);
143        let mut parents = Vec::new();
144        let mut facts: HashMap<u32, Facts> = HashMap::new();
145        let Ok(entries) = std::fs::read_dir("/proc") else {
146            return HashMap::new();
147        };
148        for entry in entries.flatten() {
149            let Some(pid) = entry
150                .file_name()
151                .to_str()
152                .and_then(|n| n.parse::<u32>().ok())
153            else {
154                continue;
155            };
156            // Gone meanwhile, or not readable: skipped.
157            let Ok(line) = std::fs::read_to_string(entry.path().join("stat")) else {
158                continue;
159            };
160            if let Some((parent, ticks, pages)) = parse_proc_stat(&line) {
161                parents.push((pid, parent));
162                facts.insert(
163                    pid,
164                    (
165                        pages.saturating_mul(page),
166                        Duration::from_nanos(
167                            ticks.saturating_mul(1_000_000_000) / ticks_per_second,
168                        ),
169                    ),
170                );
171            }
172        }
173        trees(roots, &parents, |pid| facts.get(&pid).copied(), |_, _| true)
174    }
175}
176
177#[cfg(target_os = "macos")]
178mod sys {
179    use std::collections::HashMap;
180    use std::time::Duration;
181
182    use super::{TreeUsage, trees};
183
184    fn info<T>(pid: i32, flavor: i32) -> Option<T> {
185        let size = i32::try_from(std::mem::size_of::<T>()).ok()?;
186        let mut value = std::mem::MaybeUninit::<T>::zeroed();
187        // SAFETY: `value` is a zeroed buffer of exactly `size` bytes for the
188        // plain-data structure `flavor` fills; the call writes at most that.
189        let written =
190            unsafe { libc::proc_pidinfo(pid, flavor, 0, value.as_mut_ptr().cast(), size) };
191        // SAFETY: the kernel filled the whole structure (`written == size`),
192        // and zeroed is a valid value of these plain-data types anyway.
193        (written == size).then(|| unsafe { value.assume_init() })
194    }
195
196    /// Mach ticks to nanoseconds: `PROC_PIDTASKINFO` times are in ticks,
197    /// which are nanoseconds on Intel and not on Apple silicon.
198    #[allow(
199        deprecated,
200        reason = "libc marks the mach bindings deprecated in favour of a crate this workspace does not take; the call is stable"
201    )]
202    fn ticks_to_nanos(ticks: u64) -> u64 {
203        static BASE: std::sync::OnceLock<(u32, u32)> = std::sync::OnceLock::new();
204        let (numer, denom) = *BASE.get_or_init(|| {
205            let mut base = libc::mach_timebase_info { numer: 0, denom: 0 };
206            // SAFETY: `base` is a live structure owned by this frame.
207            let ok = unsafe { libc::mach_timebase_info(&raw mut base) } == 0;
208            if ok && base.denom != 0 {
209                (base.numer, base.denom)
210            } else {
211                (1, 1)
212            }
213        });
214        u64::try_from(u128::from(ticks) * u128::from(numer) / u128::from(denom)).unwrap_or(u64::MAX)
215    }
216
217    fn all_pids() -> Vec<i32> {
218        // SAFETY: a null buffer asks for the count only.
219        let count = unsafe { libc::proc_listallpids(std::ptr::null_mut(), 0) };
220        let Ok(count) = usize::try_from(count) else {
221            return Vec::new();
222        };
223        // Room for processes started since the count.
224        let mut pids = vec![0i32; count + 64];
225        let bytes = i32::try_from(pids.len() * std::mem::size_of::<i32>()).unwrap_or(i32::MAX);
226        // SAFETY: `pids` holds `bytes` bytes and outlives the call.
227        let filled = unsafe { libc::proc_listallpids(pids.as_mut_ptr().cast(), bytes) };
228        pids.truncate(usize::try_from(filled).unwrap_or(0).min(pids.len()));
229        pids
230    }
231
232    pub(super) fn sample_trees(roots: &[u32]) -> HashMap<u32, TreeUsage> {
233        let parents: Vec<(u32, u32)> = all_pids()
234            .into_iter()
235            .filter(|pid| *pid > 0)
236            .filter_map(|pid| {
237                let bsd: libc::proc_bsdshortinfo = info(pid, libc::PROC_PIDT_SHORTBSDINFO)?;
238                Some((u32::try_from(pid).ok()?, bsd.pbsi_ppid))
239            })
240            .collect();
241        trees(
242            roots,
243            &parents,
244            |pid| {
245                let task: libc::proc_taskinfo =
246                    info(i32::try_from(pid).ok()?, libc::PROC_PIDTASKINFO)?;
247                Some((
248                    task.pti_resident_size,
249                    Duration::from_nanos(ticks_to_nanos(
250                        task.pti_total_user.saturating_add(task.pti_total_system),
251                    )),
252                ))
253            },
254            |_, _| true,
255        )
256    }
257}
258
259#[cfg(windows)]
260mod sys {
261    use std::collections::HashMap;
262    use std::time::Duration;
263
264    use windows::Win32::Foundation::{CloseHandle, FILETIME};
265    use windows::Win32::System::Diagnostics::ToolHelp::{
266        CreateToolhelp32Snapshot, PROCESSENTRY32W, Process32FirstW, Process32NextW,
267        TH32CS_SNAPPROCESS,
268    };
269    use windows::Win32::System::ProcessStatus::{GetProcessMemoryInfo, PROCESS_MEMORY_COUNTERS};
270    use windows::Win32::System::Threading::{
271        GetProcessTimes, OpenProcess, PROCESS_QUERY_LIMITED_INFORMATION,
272    };
273
274    use super::{Facts, TreeUsage, trees};
275
276    fn parents() -> Vec<(u32, u32)> {
277        // SAFETY: a process snapshot takes no pointer; the handle is closed
278        // below exactly once.
279        let Ok(snapshot) = (unsafe { CreateToolhelp32Snapshot(TH32CS_SNAPPROCESS, 0) }) else {
280            return Vec::new();
281        };
282        let mut entry = PROCESSENTRY32W {
283            dwSize: u32::try_from(std::mem::size_of::<PROCESSENTRY32W>()).unwrap_or(0),
284            ..PROCESSENTRY32W::default()
285        };
286        let mut pairs = Vec::new();
287        // SAFETY: `entry` is a live, size-stamped structure owned by this
288        // frame, and `snapshot` is open.
289        let mut more = unsafe { Process32FirstW(snapshot, &raw mut entry) }.is_ok();
290        while more {
291            pairs.push((entry.th32ProcessID, entry.th32ParentProcessID));
292            // SAFETY: as above.
293            more = unsafe { Process32NextW(snapshot, &raw mut entry) }.is_ok();
294        }
295        // SAFETY: opened above, closed once.
296        let _ = unsafe { CloseHandle(snapshot) };
297        pairs
298    }
299
300    /// A `FILETIME` in 100 ns units.
301    fn hundreds(time: FILETIME) -> u64 {
302        (u64::from(time.dwHighDateTime) << 32) | u64::from(time.dwLowDateTime)
303    }
304
305    fn filetime(time: FILETIME) -> Duration {
306        Duration::from_nanos(hundreds(time).saturating_mul(100))
307    }
308
309    /// What one process uses, and when it was created; `None` when this
310    /// account cannot open it.
311    fn facts(pid: u32) -> Option<(Facts, Option<u64>)> {
312        // SAFETY: a plain call; the handle is closed below exactly once.
313        // Querying memory and times needs no more than limited information.
314        let handle = unsafe { OpenProcess(PROCESS_QUERY_LIMITED_INFORMATION, false, pid) }.ok()?;
315        let mut memory = PROCESS_MEMORY_COUNTERS {
316            cb: u32::try_from(std::mem::size_of::<PROCESS_MEMORY_COUNTERS>()).unwrap_or(0),
317            ..PROCESS_MEMORY_COUNTERS::default()
318        };
319        // SAFETY: `memory` is a live, size-stamped structure owned by this
320        // frame; `handle` is open.
321        let rss = if unsafe { GetProcessMemoryInfo(handle, &raw mut memory, memory.cb) }.is_ok() {
322            u64::try_from(memory.WorkingSetSize).unwrap_or(u64::MAX)
323        } else {
324            0
325        };
326        let (mut created, mut exited, mut kernel, mut user) = (
327            FILETIME::default(),
328            FILETIME::default(),
329            FILETIME::default(),
330            FILETIME::default(),
331        );
332        // SAFETY: four live out-parameters owned by this frame.
333        let timed = unsafe {
334            GetProcessTimes(
335                handle,
336                &raw mut created,
337                &raw mut exited,
338                &raw mut kernel,
339                &raw mut user,
340            )
341        };
342        let (cpu, born) = if timed.is_ok() {
343            (
344                filetime(kernel).saturating_add(filetime(user)),
345                Some(hundreds(created)),
346            )
347        } else {
348            (Duration::ZERO, None)
349        };
350        // SAFETY: opened above, closed once.
351        let _ = unsafe { CloseHandle(handle) };
352        Some(((rss, cpu), born))
353    }
354
355    pub(super) fn sample_trees(roots: &[u32]) -> HashMap<u32, TreeUsage> {
356        let known = std::cell::RefCell::new(HashMap::<u32, Option<(Facts, Option<u64>)>>::new());
357        let look = |pid: u32| *known.borrow_mut().entry(pid).or_insert_with(|| facts(pid));
358        trees(
359            roots,
360            &parents(),
361            |pid| look(pid).map(|(facts, _)| facts),
362            // A child created before its parent names a dead parent whose
363            // PID was reused: not this tree's.
364            |child, parent| match (look(child), look(parent)) {
365                (Some((_, Some(child))), Some((_, Some(parent)))) => child >= parent,
366                _ => true,
367            },
368        )
369    }
370}
371
372#[cfg(not(any(target_os = "linux", target_os = "macos", windows)))]
373mod sys {
374    use std::collections::HashMap;
375
376    use super::TreeUsage;
377
378    pub(super) fn sample_trees(_: &[u32]) -> HashMap<u32, TreeUsage> {
379        HashMap::new()
380    }
381}
382
383#[cfg(test)]
384mod tests {
385    use super::*;
386
387    #[test]
388    fn a_stat_line_parses_whatever_the_command_name_holds() {
389        let line = "4242 (Runner.Worker (x) 1) S 4200 4242 4200 0 -1 4194560 100 0 0 0 \
390                    250 50 7 3 20 0 12 0 123456 1048576000 25600 18446744073709551615";
391        assert_eq!(parse_proc_stat(line), Some((4200, 310, 25600)));
392        assert_eq!(parse_proc_stat("garbage"), None);
393        assert_eq!(parse_proc_stat("1 (init) S"), None);
394    }
395
396    #[test]
397    fn a_tree_sums_its_descendants_only_and_survives_a_cycle() {
398        let parents = [
399            (1, 0),
400            (10, 1),
401            (11, 10),
402            (12, 11),
403            (20, 1),
404            (30, 12),
405            (12, 30),
406        ];
407        let facts: HashMap<u32, Facts> = [
408            (1, (1, 1)),
409            (10, (100, 10)),
410            (11, (20, 2)),
411            (12, (30, 3)),
412            (20, (999, 99)),
413            (30, (5, 1)),
414        ]
415        .into_iter()
416        .map(|(pid, (rss, secs))| (pid, (rss, Duration::from_secs(secs))))
417        .collect();
418        let found = trees(
419            &[10, 99],
420            &parents,
421            |pid| facts.get(&pid).copied(),
422            |_, _| true,
423        );
424        assert_eq!(
425            found,
426            HashMap::from([(
427                10,
428                TreeUsage {
429                    rss_bytes: 155,
430                    cpu_time: Duration::from_secs(16),
431                    processes: 4,
432                }
433            )]),
434            "the gone root 99 has no entry"
435        );
436        // An unreadable process still counts as one of the tree.
437        let unreadable = trees(&[10], &parents, |_| None, |_, _| true);
438        assert_eq!(unreadable[&10].processes, 4);
439        // A link the platform disowns (a stale parent PID) cuts that subtree.
440        let cut = trees(
441            &[10],
442            &parents,
443            |pid| facts.get(&pid).copied(),
444            |child, _| child != 11,
445        );
446        assert_eq!(cut[&10].processes, 1);
447    }
448
449    /// A real child tree of this process: the measurement on the current OS.
450    #[cfg(unix)]
451    #[test]
452    fn a_spawned_process_tree_is_measured_on_this_os() {
453        // A shell that starts a second process and burns some CPU, so both
454        // the tree and its CPU time are non-trivial.
455        let mut child = std::process::Command::new("sh")
456            .args([
457                "-c",
458                "sleep 30 & i=0; while [ $i -lt 200000 ]; do i=$((i+1)); done; wait",
459            ])
460            .spawn()
461            .expect("sh starts");
462        let root = child.id();
463        let mut best = None;
464        for _ in 0..50 {
465            std::thread::sleep(Duration::from_millis(100));
466            if let Some(usage) = sample_tree(root)
467                && usage.processes >= 2
468                && usage.cpu_time > Duration::ZERO
469            {
470                best = Some(usage);
471                break;
472            }
473        }
474        let _ = child.kill();
475        let _ = child.wait();
476        let usage = best.expect("the tree was seen with its child and some CPU");
477        assert!(usage.rss_bytes > 0, "{usage:?}");
478        assert_eq!(sample_tree(root), None, "a reaped root is gone");
479    }
480
481    #[cfg(windows)]
482    #[test]
483    fn a_spawned_process_tree_is_measured_on_this_os() {
484        let mut child = std::process::Command::new("cmd")
485            .args(["/C", "ping -n 6 127.0.0.1 > NUL"])
486            .spawn()
487            .expect("cmd starts");
488        let root = child.id();
489        let mut best = None;
490        for _ in 0..50 {
491            std::thread::sleep(Duration::from_millis(100));
492            if let Some(usage) = sample_tree(root)
493                && usage.processes >= 2
494            {
495                best = Some(usage);
496                break;
497            }
498        }
499        let _ = child.kill();
500        let _ = child.wait();
501        let usage = best.expect("the tree was seen with its child");
502        assert!(usage.rss_bytes > 0, "{usage:?}");
503    }
504}
runner_manager_platform/resource_usage.rs

runner_manager_platform/
resource_usage.rs
