Skip to main content

runnerd_core/observe/
disk.rs

1//! What the disk under Docker's root writes: which block devices hold that root (`Disks::resolve`, from
2//! `/proc/self/mountinfo` and sysfs), the bytes a cgroup wrote to them (`io.stat`), and over windows of
3//! `WINDOW` the bytes the drive wrote (`/sys/class/block/<disk>/stat`), the bytes the runners' slice wrote and the
4//! btrfs commits of the file system (`/sys/fs/btrfs/<uuid>/commit_stats`); and the free disk Docker has
5//! (`free_space`), down through a loop device's backing file. A figure the host does not give, as on
6//! cgroup v1, a kernel without the io controller, a file system other than btrfs or an unreadable sysfs, is absent,
7//! never 0. Reads files only; it writes nothing to the host and decides nothing.
8//!
9//! Only the disks under Docker's root count: runners write their layers, slot volumes and shared caches there, and
10//! a cgroup's `io.stat` also lists other devices, such as zram's swap, whose bytes never reach the drive. A
11//! partition counts as its whole disk, which is how `io.stat` keys it; a device-mapper volume counts as itself, not
12//! the disks under it, whose `io.stat` lines would count the same bytes again.
13//!
14//! When Docker's root is a file system in a file, on a loop device, a cgroup's figure is the loop device's line:
15//! what it wrote to that file system. The loop's own writes to its backing file reach the drive charged to the
16//! cgroup only some of the time, sometimes to none, so that line is never counted. The drive's bytes and commits
17//! follow the backing file (`/sys/block/loopN/loop/backing_file`) to the disk and file system under it.
18
19use std::collections::BTreeSet;
20use std::path::{Path, PathBuf};
21use std::time::Duration;
22
23use super::metrics::Paths;
24
25/// How long a stored window of the drive's figures spans at most. A daemon's stop stores the window it is in.
26pub const WINDOW: Duration = Duration::from_mins(10);
27
28/// A block device, as a cgroup's `io.stat` keys it.
29#[derive(Debug, Clone, PartialEq, Eq, PartialOrd, Ord)]
30pub struct Device {
31    major: u32,
32    minor: u32,
33    name: String,
34}
35
36/// How deep a loop device's backing file is followed: a loop on a loop on a disk.
37const LOOP_DEPTH: u8 = 4;
38
39/// The devices Docker's root sits on: those a cgroup's writes to it are counted on, the physical disks under them,
40/// and the btrfs file system's UUID when the disks hold one.
41#[derive(Debug, Clone, PartialEq, Eq)]
42pub struct Disks {
43    /// What `io.stat` counts: the disks, or the loop devices when the root is in a file.
44    charged: Vec<Device>,
45    /// The physical disks: the drive.
46    drive: Vec<Device>,
47    btrfs: Option<String>,
48}
49
50impl Disks {
51    /// What `root`, Docker's root directory, is on, from the mount that holds it: its file system's block devices,
52    /// each a partition's disk, and its btrfs UUID; through a loop device, the disks and file system its backing
53    /// file is on. `None` when the mount or its devices can't be read, as without `/proc/self/mountinfo` or sysfs,
54    /// or on a file system without a block device.
55    pub fn resolve(paths: &Paths, root: &Path) -> Option<Disks> {
56        let mountinfo = std::fs::read_to_string(paths.proc.join("self/mountinfo")).ok()?;
57        Disks::under(paths, &mountinfo, root, LOOP_DEPTH)
58    }
59
60    fn under(paths: &Paths, mountinfo: &str, path: &Path, depth: u8) -> Option<Disks> {
61        let held = Held::find(paths, mountinfo, path)?;
62        let mut btrfs = held.btrfs;
63        let mut charged = BTreeSet::new();
64        let mut drive = BTreeSet::new();
65        for disk in held.disks {
66            let device = Device::read(&disk)?;
67            if let Some(backing) = backing_file(&disk) {
68                let under = Disks::under(paths, mountinfo, &backing, depth.checked_sub(1)?)?;
69                drive.extend(under.drive);
70                btrfs = under.btrfs;
71            } else {
72                drive.insert(device.clone());
73            }
74            charged.insert(device);
75        }
76        (!charged.is_empty()).then(|| Disks {
77            charged: charged.into_iter().collect(),
78            drive: drive.into_iter().collect(),
79            btrfs,
80        })
81    }
82
83    /// The loop devices Docker's root is on, joined by `+`, when it is a file system in a file: `loop0`.
84    pub fn store(&self) -> Option<String> {
85        (self.charged != self.drive).then(|| join(&self.charged))
86    }
87
88    /// The physical disks' names, joined by `+`: `nvme2n1`.
89    pub fn name(&self) -> String {
90        join(&self.drive)
91    }
92
93    /// The bytes and write requests a cgroup and its descendants have sent the disks so far (`io.stat` `wbytes`,
94    /// `wios`), or the loop devices when the root is in a file: what they wrote to its file system. `None` when the
95    /// cgroup has no `io.stat`, as without the io controller or once it is gone; a disk without a line in it has had
96    /// none of the cgroup's I/O yet, which is 0.
97    pub fn written(&self, cgroup: &Path) -> Option<Written> {
98        let stat = std::fs::read_to_string(cgroup.join("io.stat")).ok()?;
99        let mut written = Written::default();
100        for line in stat.lines() {
101            let mut fields = line.split_whitespace();
102            let Some((major, minor)) = fields.next().and_then(|device| device.split_once(':'))
103            else {
104                continue;
105            };
106            if !self.charged.iter().any(|device| {
107                major.parse() == Ok(device.major) && minor.parse() == Ok(device.minor)
108            }) {
109                continue;
110            }
111            for field in fields {
112                let Some((key, value)) = field.split_once('=') else {
113                    continue;
114                };
115                let Ok(value) = value.parse::<u64>() else {
116                    return None;
117                };
118                match key {
119                    "wbytes" => written.bytes = written.bytes.saturating_add(value),
120                    "wios" => written.ios = written.ios.saturating_add(value),
121                    _ => {}
122                }
123            }
124        }
125        Some(written)
126    }
127
128    /// The disks', the slice's and the file system's counters now, at `at` (ms).
129    pub fn read(&self, paths: &Paths, slice: &Path, at: u64) -> Reading {
130        let mut drive = Some(0u64);
131        for device in &self.drive {
132            // Sectors written, the seventh field, in 512-byte units whatever the disk's own sector size.
133            let sectors = std::fs::read_to_string(
134                paths
135                    .sys
136                    .join("class/block")
137                    .join(&device.name)
138                    .join("stat"),
139            )
140            .ok()
141            .and_then(|stat| stat.split_whitespace().nth(6)?.parse::<u64>().ok());
142            drive = drive
143                .zip(sectors)
144                .map(|(sum, sectors)| sum.saturating_add(sectors.saturating_mul(512)));
145        }
146        let commits = self.btrfs.as_ref().and_then(|uuid| {
147            let stats =
148                std::fs::read_to_string(paths.sys.join("fs/btrfs").join(uuid).join("commit_stats"))
149                    .ok()?;
150            stats
151                .lines()
152                .find_map(|line| line.strip_prefix("commits ")?.trim().parse().ok())
153        });
154        // The daemon's own: what it sent to any block device, its history among it.
155        let daemon = std::fs::read_to_string(paths.proc.join("self/io"))
156            .ok()
157            .and_then(|io| {
158                io.lines()
159                    .find_map(|line| line.strip_prefix("write_bytes:")?.trim().parse().ok())
160            });
161        Reading {
162            at,
163            drive,
164            slice: self.written(slice).map(|written| written.bytes),
165            daemon,
166            commits,
167            device: self.name(),
168            store: self.store(),
169        }
170    }
171}
172
173/// What a cgroup has written to the disks so far.
174#[derive(Debug, Clone, Copy, Default, PartialEq, Eq)]
175pub struct Written {
176    /// Bytes (`wbytes`).
177    pub bytes: u64,
178    /// Write requests (`wios`).
179    pub ios: u64,
180}
181
182/// One line of `/proc/self/mountinfo`, as far as `Disks::resolve` reads it.
183struct Mount {
184    /// `major:minor` of the mounted file system; an anonymous one on btrfs.
185    device: String,
186    point: PathBuf,
187    kind: String,
188    source: String,
189}
190
191impl Mount {
192    /// The mount in `mountinfo` that holds `path`: the deepest whose point it is under.
193    fn holding(mountinfo: &str, path: &Path) -> Option<Mount> {
194        mountinfo
195            .lines()
196            .filter_map(Mount::parse)
197            .filter(|mount| path.starts_with(&mount.point))
198            .max_by_key(|mount| mount.point.components().count())
199    }
200
201    fn parse(line: &str) -> Option<Mount> {
202        let (front, back) = line.split_once(" - ")?;
203        let mut front = front.split(' ');
204        let device = front.nth(2)?.to_owned();
205        let point = PathBuf::from(unescape(front.nth(1)?));
206        let mut back = back.split(' ');
207        Some(Mount {
208            device,
209            point,
210            kind: back.next()?.to_owned(),
211            source: unescape(back.next()?),
212        })
213    }
214}
215
216/// A mountinfo field with its octal escapes (`\040` for a space) undone.
217fn unescape(field: &str) -> String {
218    let bytes = field.as_bytes();
219    let mut out = Vec::with_capacity(bytes.len());
220    let mut at = 0;
221    while at < bytes.len() {
222        let code = bytes
223            .get(at + 1..at + 4)
224            .filter(|_| bytes[at] == b'\\')
225            .and_then(|digits| u8::from_str_radix(std::str::from_utf8(digits).ok()?, 8).ok());
226        if let Some(code) = code {
227            out.push(code);
228            at += 4;
229        } else {
230            out.push(bytes[at]);
231            at += 1;
232        }
233    }
234    String::from_utf8_lossy(&out).into_owned()
235}
236
237/// The file system that holds a path: its btrfs UUID when it is btrfs, and the sysfs directories of the whole disks
238/// under it.
239struct Held {
240    btrfs: Option<String>,
241    disks: Vec<PathBuf>,
242}
243
244impl Held {
245    /// What holds `path`, from `mountinfo` and sysfs; `None` when the mount or a device can't be read.
246    fn find(paths: &Paths, mountinfo: &str, path: &Path) -> Option<Held> {
247        let mount = Mount::holding(mountinfo, path)?;
248        let class = paths.sys.join("class/block");
249        let btrfs = (mount.kind == "btrfs")
250            .then(|| btrfs_uuid(paths, &mount.source))
251            .flatten();
252        let names: Vec<String> = if let Some(uuid) = &btrfs {
253            std::fs::read_dir(paths.sys.join("fs/btrfs").join(uuid).join("devices"))
254                .ok()?
255                .flatten()
256                .filter_map(|entry| entry.file_name().into_string().ok())
257                .collect()
258        } else {
259            let link = paths.sys.join("dev/block").join(&mount.device);
260            vec![file_name(&std::fs::canonicalize(link).ok()?)?]
261        };
262        let mut disks = Vec::new();
263        for name in names {
264            let dir = class.join(&name);
265            // A partition's directory sits in its disk's: the disk is what `io.stat` and the drive's `stat` count.
266            disks.push(if dir.join("partition").is_file() {
267                std::fs::canonicalize(&dir).ok()?.parent()?.to_path_buf()
268            } else {
269                std::fs::canonicalize(&dir).ok()?
270            });
271        }
272        Some(Held { btrfs, disks })
273    }
274}
275
276/// The file a loop device whose sysfs directory is `disk` reads and writes; `None` for another device. A deleted
277/// backing file keeps its path, marked; the mount that held it still holds its blocks.
278fn backing_file(disk: &Path) -> Option<PathBuf> {
279    let backing = std::fs::read_to_string(disk.join("loop/backing_file")).ok()?;
280    let backing = backing.trim_end();
281    Some(PathBuf::from(
282        backing.strip_suffix(" (deleted)").unwrap_or(backing),
283    ))
284}
285
286/// The bytes Docker can still write under `root`, its root directory (ADR 0054): `statfs`'s free space for an
287/// unprivileged user, `free`, on the file system that holds it. When that file system is in a file, on a loop device,
288/// the file can grow only while the file system under it has room too, so it is the least of the free space of each
289/// file system down to the disk, nested loops included. A file system under it that can't be found or measured
290/// leaves the figure of those above it, never 0. `free` is the one boundary to the host, so tests give their own.
291///
292/// # Errors
293///
294/// `free`'s error for `root` itself.
295pub fn free_space(
296    paths: &Paths,
297    root: &Path,
298    free: impl Fn(&Path) -> std::io::Result<u64>,
299) -> std::io::Result<u64> {
300    let own = free(root)?;
301    let under = std::fs::read_to_string(paths.proc.join("self/mountinfo"))
302        .ok()
303        .and_then(|mountinfo| free_under(paths, &mountinfo, root, &free, LOOP_DEPTH));
304    Ok(under.map_or(own, |under| own.min(under)))
305}
306
307/// The least free space of the file systems that hold the backing files of the loop devices `path`'s file system is
308/// on, and of those under them; `None` when it is on no loop device or none of them could be measured.
309fn free_under(
310    paths: &Paths,
311    mountinfo: &str,
312    path: &Path,
313    free: &impl Fn(&Path) -> std::io::Result<u64>,
314    depth: u8,
315) -> Option<u64> {
316    let depth = depth.checked_sub(1)?;
317    let held = Held::find(paths, mountinfo, path)?;
318    held.disks
319        .iter()
320        .filter_map(|disk| backing_file(disk))
321        .filter_map(|backing| {
322            // The mount point, not the file: runnerd may not be allowed to search the directories above the file.
323            let point = Mount::holding(mountinfo, &backing)?.point;
324            let own = free(&point).ok();
325            let under = free_under(paths, mountinfo, &backing, free, depth);
326            own.into_iter().chain(under).min()
327        })
328        .min()
329}
330
331impl Device {
332    /// The block device whose sysfs directory is `dir`.
333    fn read(dir: &Path) -> Option<Device> {
334        let number = std::fs::read_to_string(dir.join("dev")).ok()?;
335        let (major, minor) = number.trim().split_once(':')?;
336        Some(Device {
337            major: major.parse().ok()?,
338            minor: minor.parse().ok()?,
339            name: file_name(dir)?,
340        })
341    }
342}
343
344/// Devices' names joined by `+`.
345fn join(devices: &[Device]) -> String {
346    devices
347        .iter()
348        .map(|device| device.name.as_str())
349        .collect::<Vec<_>>()
350        .join("+")
351}
352
353fn file_name(path: &Path) -> Option<String> {
354    path.file_name()?.to_str().map(str::to_owned)
355}
356
357/// The UUID of the btrfs file system `source` (`/dev/nvme0n1p2`, `/dev/mapper/root`) belongs to: the one whose
358/// `devices` lists the device it names.
359fn btrfs_uuid(paths: &Paths, source: &str) -> Option<String> {
360    let source = Path::new(source);
361    // `/dev/mapper/root` is a link to `/dev/dm-0`, the name sysfs lists.
362    let name = std::fs::canonicalize(source)
363        .ok()
364        .and_then(|path| file_name(&path))
365        .or_else(|| file_name(source))?;
366    std::fs::read_dir(paths.sys.join("fs/btrfs"))
367        .ok()?
368        .flatten()
369        .find(|entry| entry.path().join("devices").join(&name).exists())
370        .and_then(|entry| entry.file_name().into_string().ok())
371}
372
373/// The drive's, the slice's and the file system's counters at one moment.
374#[derive(Debug, Clone, PartialEq, Eq)]
375pub struct Reading {
376    at: u64,
377    drive: Option<u64>,
378    slice: Option<u64>,
379    daemon: Option<u64>,
380    commits: Option<u64>,
381    device: String,
382    store: Option<String>,
383}
384
385/// What a window's readings add up to, as the history stores it.
386#[derive(Debug, Clone, PartialEq, Eq)]
387pub struct Window {
388    /// Its first reading, in ms.
389    pub start: u64,
390    /// Its last reading, in ms.
391    pub end: u64,
392    /// The disks it read: `nvme2n1`.
393    pub device: String,
394    /// Bytes the disks wrote; `None` when a reading lacked them.
395    pub drive: Option<u64>,
396    /// Bytes the runners' slice wrote to them, its containers' and those it had that are gone; `None` when a
397    /// reading lacked them.
398    pub slice: Option<u64>,
399    /// Bytes the daemon itself wrote (`/proc/self/io`'s `write_bytes`), to any disk; `None` when a reading lacked
400    /// them.
401    pub daemon: Option<u64>,
402    /// btrfs transaction commits; `None` on another file system or when a reading lacked them.
403    pub commits: Option<u64>,
404    /// The loop devices Docker's root is on when it is a file system in a file on `device`: `loop0`. The runner
405    /// slice's figure is then what it wrote to that file system.
406    pub store: Option<String>,
407}
408
409/// The window being added up: from its first reading to its last, and what the counters rose by between them.
410#[derive(Debug)]
411struct Open {
412    first: u64,
413    last: Reading,
414    drive: Option<u64>,
415    slice: Option<u64>,
416    daemon: Option<u64>,
417    commits: Option<u64>,
418}
419
420impl Open {
421    fn new(reading: Reading) -> Open {
422        Open {
423            first: reading.at,
424            drive: Some(0),
425            slice: Some(0),
426            daemon: Some(0),
427            commits: reading.commits.map(|_| 0),
428            last: reading,
429        }
430    }
431
432    /// Adds what each counter rose by since the last reading. A counter either reading lacks makes the window's
433    /// absent; one that fell, as a disk's after it was taken out and put back, adds nothing.
434    fn add(&mut self, reading: Reading) {
435        let rose = |sum: Option<u64>, before: Option<u64>, now: Option<u64>| {
436            let (sum, before, now) = (sum?, before?, now?);
437            Some(sum.saturating_add(now.saturating_sub(before)))
438        };
439        self.drive = rose(self.drive, self.last.drive, reading.drive);
440        self.slice = rose(self.slice, self.last.slice, reading.slice);
441        self.daemon = rose(self.daemon, self.last.daemon, reading.daemon);
442        self.commits = rose(self.commits, self.last.commits, reading.commits);
443        self.last = reading;
444    }
445
446    fn close(&self) -> Window {
447        Window {
448            start: self.first,
449            end: self.last.at,
450            device: self.last.device.clone(),
451            store: self.last.store.clone(),
452            drive: self.drive,
453            slice: self.slice,
454            daemon: self.daemon,
455            commits: self.commits,
456        }
457    }
458}
459
460/// Adds up the readings of the drive into windows of `WINDOW`, each starting where the last ended. A reload leaves
461/// it be; a restart starts a new window at its first reading, so the time no daemon ran is in none.
462#[derive(Debug, Default)]
463pub struct Windows {
464    open: Option<Open>,
465}
466
467impl Windows {
468    /// Adds a reading; returns the window it closes, once the window spans `WINDOW`.
469    pub fn add(&mut self, reading: Reading) -> Option<Window> {
470        let Some(open) = self.open.as_mut() else {
471            self.open = Some(Open::new(reading));
472            return None;
473        };
474        if reading.device != open.last.device || reading.store != open.last.store {
475            let closed = open.close();
476            self.open = Some(Open::new(reading));
477            return Some(closed).filter(|window| window.end > window.start);
478        }
479        open.add(reading);
480        let span = u64::try_from(WINDOW.as_millis()).unwrap_or(u64::MAX);
481        if open.last.at.saturating_sub(open.first) < span {
482            return None;
483        }
484        let closed = open.close();
485        let last = open.last.clone();
486        self.open = Some(Open::new(last));
487        Some(closed)
488    }
489
490    /// The window being added up, if it spans any time: what a stop stores.
491    pub fn close(&mut self) -> Option<Window> {
492        self.open
493            .take()
494            .map(|open| open.close())
495            .filter(|window| window.end > window.start)
496    }
497}