Skip to main content

runnerd_core/api/
disk.rs

1//! What runner jobs and the drive under Docker's root wrote (ADR 0143), as clients read it, and every word they show
2//! of it: a job's figure in a table and on its page, and the disk-writes section of `GET /v1/insights`. Sizes are
3//! bytes, times seconds since the Unix epoch.
4
5use std::fmt::Write as _;
6
7use serde::{Deserialize, Serialize};
8
9use super::{Tone, duration};
10use crate::config::ByteSize;
11
12/// What a job's runner container wrote to the disks under Docker's root, from its cgroup's `io.stat`: what its
13/// processes, the Docker Engine inside it and that engine's containers sent the disks from the job's start to the
14/// container's last read. Bytes still in the page cache then go to the disk after, on the runner slice's account.
15#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
16pub struct DiskWritten {
17    /// Bytes written.
18    pub bytes: u64,
19    /// Write requests.
20    pub ios: u64,
21    /// What its container had written to the page cache and the kernel not yet to the disk at its last read: written
22    /// after, unless deleted first. `None` where the kernel does not say.
23    #[serde(default, skip_serializing_if = "Option::is_none")]
24    pub dirty: Option<u64>,
25    /// Whether that is all it wrote.
26    pub measure: Measure,
27}
28
29/// Whether a job's [`DiskWritten`] is all it wrote. A job no reading measured has no figure at all.
30#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
31#[serde(rename_all = "kebab-case")]
32pub enum Measure {
33    /// Read from before the job's start to within a sample of its container's end.
34    Complete,
35    /// Part of it was not read: its container's end, as when it ended while no daemon ran, or the counter's start,
36    /// as when the container was started again, or its start, as when the counter was not read before the job.
37    /// What was read is a floor.
38    Partial,
39    /// A measure a later release names; read as partial.
40    #[serde(other)]
41    Unknown,
42}
43
44impl DiskWritten {
45    /// It in a table cell: `1.2G`, or `≥1.2G` for a partial figure.
46    pub fn short(&self) -> String {
47        match self.measure {
48            Measure::Complete => ByteSize(self.bytes).to_string(),
49            Measure::Partial | Measure::Unknown => format!("≥{}", ByteSize(self.bytes)),
50        }
51    }
52
53    /// It on a job's page: `1.2G in 3402 writes`, then `; 80M more waited to be written back as its container
54    /// ended` when some did, or `at least 1.2G in 3402 writes: …` with why a partial figure is one.
55    pub fn line(&self) -> String {
56        let writes = if self.ios == 1 { "write" } else { "writes" };
57        let mut line = match self.measure {
58            Measure::Complete => format!("{} in {} {writes}", ByteSize(self.bytes), self.ios),
59            Measure::Partial | Measure::Unknown => format!(
60                "at least {} in {} {writes}: part of its run was not read",
61                ByteSize(self.bytes),
62                self.ios
63            ),
64        };
65        if let Some(dirty) = self.dirty.filter(|dirty| *dirty > 0) {
66            let _ = write!(
67                line,
68                "; {} more waited to be written back as its container ended",
69                ByteSize(dirty)
70            );
71        }
72        line
73    }
74
75    /// What it means: a partial figure is a floor, so it reads in the background.
76    pub fn tone(&self) -> Tone {
77        match self.measure {
78            Measure::Complete => Tone::Neutral,
79            Measure::Partial | Measure::Unknown => Tone::Quiet,
80        }
81    }
82}
83
84/// The disk-writes section of [`Insights`](super::Insights): what the jobs that ended in the window wrote, the
85/// workflows and jobs that wrote most, and what the drive wrote while the daemon read it.
86#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
87pub struct DiskWrites {
88    /// The jobs' figures together.
89    pub jobs: JobWrites,
90    /// The workflows whose jobs wrote most, most first.
91    pub workflows: Vec<WriteTop>,
92    /// The jobs, by repository, workflow and name, that wrote most, most first.
93    pub top_jobs: Vec<WriteTop>,
94    /// The drive's figures; `None` when the daemon read none in the window.
95    #[serde(default, skip_serializing_if = "Option::is_none")]
96    pub drive: Option<DriveWrites>,
97}
98
99/// What the jobs that ended in a window wrote, each its runner container's [`DiskWritten`].
100#[derive(Debug, Clone, Copy, Default, PartialEq, Eq, Serialize, Deserialize)]
101pub struct JobWrites {
102    /// Jobs with a figure.
103    pub measured: u32,
104    /// Of those, the partial ones.
105    pub partial: u32,
106    /// Bytes they wrote together.
107    pub total: u64,
108    /// The median of one job's bytes.
109    pub median: u64,
110    /// The 90th percentile, by nearest rank.
111    pub p90: u64,
112    /// What waited to be written back as their containers ended, together.
113    pub dirty: u64,
114}
115
116/// A workflow, or a job, and what its jobs in the window wrote.
117#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
118pub struct WriteTop {
119    /// `owner/name`.
120    pub repository: String,
121    /// The workflow's name, when known.
122    #[serde(default, skip_serializing_if = "Option::is_none")]
123    pub workflow: Option<String>,
124    /// The job's name, for a job.
125    #[serde(default, skip_serializing_if = "Option::is_none")]
126    pub job: Option<String>,
127    /// Its jobs with a figure.
128    pub jobs: u32,
129    /// Bytes they wrote together.
130    pub total: u64,
131}
132
133/// What the disks under Docker's root wrote while the daemon read them in a window: from `/sys/class/block`, the
134/// runner slice's `io.stat`, which keeps what its containers wrote after they are gone, the daemon's own
135/// `/proc/self/io`, and btrfs's commit count.
136#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
137pub struct DriveWrites {
138    /// The disks: `nvme2n1`.
139    pub device: String,
140    /// Seconds the daemon read them.
141    pub watched: u64,
142    /// Bytes they wrote then.
143    pub written: u64,
144    /// Whose those bytes were.
145    pub shares: DriveShares,
146    /// btrfs transaction commits; `None` on another file system.
147    #[serde(default, skip_serializing_if = "Option::is_none")]
148    pub commits: Option<Commits>,
149    /// The loop devices Docker's root is on when it is a file system in a file on the disks: `loop0`. Jobs' and the
150    /// runner slice's figures are then bytes written to that file system, which the disks write with its journal
151    /// and the overhead of the file system under it.
152    #[serde(default, skip_serializing_if = "Option::is_none")]
153    pub store: Option<String>,
154}
155
156/// Whose the drive's bytes were, over the stretches the daemon read the runner slice, or over all it read the drive
157/// when it never read the slice: a stretch that missed one figure leaves the others' shares be.
158#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
159pub struct DriveShares {
160    /// Seconds these were read over.
161    pub watched: u64,
162    /// Bytes the drive wrote then.
163    pub written: u64,
164    /// Of those, the figures of the jobs that ended then.
165    pub jobs: u64,
166    /// Of those, the runner slice's; `None` without its `io.stat`.
167    #[serde(default, skip_serializing_if = "Option::is_none")]
168    pub slice: Option<u64>,
169    /// What the daemon itself wrote then, to any disk; `None` where it could not read its own counters.
170    #[serde(default, skip_serializing_if = "Option::is_none")]
171    pub daemon: Option<u64>,
172}
173
174/// btrfs's transaction commits over the stretches the daemon read them.
175#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
176pub struct Commits {
177    /// Commits.
178    pub count: u64,
179    /// Seconds they were read over.
180    pub watched: u64,
181}
182
183/// `part` of `whole` in whole percent, rounded, at most 100.
184fn percent(part: u64, whole: u64) -> u64 {
185    if whole == 0 {
186        return 0;
187    }
188    let share = (u128::from(part) * 100 + u128::from(whole) / 2) / u128::from(whole);
189    u64::try_from(share.min(100)).unwrap_or(100)
190}
191
192impl JobWrites {
193    /// Them in a line: `529 jobs wrote 144.7G: median 49M, p90 612M a job`, then `; 12 partial, counted as far as
194    /// read` and `; 3.1G more waited to be written back as their containers ended` when so.
195    pub fn line(&self) -> String {
196        let jobs = if self.measured == 1 { "job" } else { "jobs" };
197        let mut line = format!(
198            "{} {jobs} wrote {}: median {}, p90 {} a job",
199            self.measured,
200            ByteSize(self.total),
201            ByteSize(self.median),
202            ByteSize(self.p90)
203        );
204        if self.partial > 0 {
205            let _ = write!(line, "; {} partial, counted as far as read", self.partial);
206        }
207        if self.dirty > 0 {
208            let _ = write!(
209                line,
210                "; {} more waited to be written back as their containers ended",
211                ByteSize(self.dirty)
212            );
213        }
214        line
215    }
216}
217
218impl WriteTop {
219    /// It in a line: `halyard/api CI / build: 12.4G over 31 jobs, 410M a job`; a workflow's without the job.
220    pub fn line(&self) -> String {
221        let mut name = self.repository.clone();
222        if let Some(workflow) = &self.workflow {
223            name.push(' ');
224            name.push_str(workflow);
225        }
226        if let Some(job) = &self.job {
227            name.push_str(if self.workflow.is_some() { " / " } else { " " });
228            name.push_str(job);
229        }
230        let jobs = if self.jobs == 1 { "job" } else { "jobs" };
231        format!(
232            "{name}: {} over {} {jobs}, {} a job",
233            ByteSize(self.total),
234            self.jobs,
235            ByteSize(self.total / u64::from(self.jobs.max(1)))
236        )
237    }
238}
239
240impl DriveWrites {
241    /// What a day at the window's rate writes, in bytes.
242    pub fn per_day(&self) -> u64 {
243        let day = u128::from(self.written) * 86_400 / u128::from(self.watched.max(1));
244        u64::try_from(day).unwrap_or(u64::MAX)
245    }
246
247    /// Them in lines: what the drive wrote and its daily rate; what the figures count when Docker's root is in a
248    /// file on it; whose it was, runner jobs', the rest of the runner
249    /// slice's, runnerd's own and the share no one in runnerd owns; and btrfs's commits a second.
250    pub fn lines(&self) -> Vec<String> {
251        let mut lines = vec![format!(
252            "{} wrote {} over the {} runnerd read it: {} a day at this rate",
253            self.device,
254            ByteSize(self.written),
255            duration(self.watched),
256            ByteSize(self.per_day())
257        )];
258        if let Some(store) = &self.store {
259            lines.push(format!(
260                "Docker's root is a file system in a file on it ({store}): the jobs' and the runner slice's figures \
261                 are bytes written to that file system; {} also wrote its journal and the overhead of the file \
262                 system under it",
263                self.device
264            ));
265        }
266        lines.push(self.shares.line(self.watched));
267        if let Some(commits) = self.commits {
268            let tenths = u128::from(commits.count) * 10 / u128::from(commits.watched.max(1));
269            lines.push(format!(
270                "btrfs committed {}.{} times a second",
271                tenths / 10,
272                tenths % 10
273            ));
274        }
275        lines
276    }
277}
278
279impl DriveShares {
280    /// Them in a line, saying the stretch they cover when it is shorter than the `watched` seconds the drive was
281    /// read: `Runner jobs 25%; the rest of the runner slice 8% (...); runnerd itself 1%; no owner runnerd knows 66%`.
282    /// No share passes the whole.
283    pub fn line(&self, watched: u64) -> String {
284        let whole = self.written;
285        // Jobs that ended in the stretch may count what they wrote before it: never past the whole.
286        let counted = self.jobs.min(whole);
287        let mut line = if self.watched < watched {
288            format!(
289                "Over the {} runnerd read the runner slice: runner jobs",
290                duration(self.watched)
291            )
292        } else {
293            "Runner jobs".to_owned()
294        };
295        let _ = write!(line, " {}%", percent(counted, whole));
296        let owned = match self.slice {
297            Some(slice) => {
298                let slice = slice.clamp(counted, whole);
299                let _ = write!(
300                    line,
301                    "; the rest of the runner slice {}% (seeds, the mirror, runners before and after their jobs, \
302                     write-back after a container ended)",
303                    percent(slice - counted, whole)
304                );
305                slice
306            }
307            None => counted,
308        };
309        let daemon = self.daemon.map(|daemon| daemon.min(whole - owned));
310        if let Some(daemon) = daemon {
311            let _ = write!(line, "; runnerd itself {}%", percent(daemon, whole));
312        }
313        let _ = write!(
314            line,
315            "; no owner runnerd knows {}%",
316            percent(whole - owned - daemon.unwrap_or(0), whole)
317        );
318        if self.slice.is_none() {
319            line.push_str(", the runner slice's share unread");
320        }
321        line
322    }
323}
324
325/// A part of the disk-writes section, under its title.
326#[derive(Debug, Clone, PartialEq, Eq)]
327pub struct WriteSection {
328    /// `Jobs`, `Drive`, `Workflows that wrote most`, `Jobs that wrote most`.
329    pub title: &'static str,
330    /// Its lines, in order.
331    pub lines: Vec<String>,
332}
333
334impl DiskWrites {
335    /// The section's title: `Disk writes`.
336    pub const TITLE: &'static str = "Disk writes";
337
338    /// Its parts in order, each with what it says, none empty: the jobs' figures, the drive's, and the workflows and
339    /// the jobs that wrote most.
340    pub fn sections(&self) -> Vec<WriteSection> {
341        let mut sections = Vec::new();
342        if self.jobs.measured > 0 {
343            sections.push(WriteSection {
344                title: "Jobs",
345                lines: vec![self.jobs.line()],
346            });
347        }
348        if let Some(drive) = &self.drive {
349            sections.push(WriteSection {
350                title: "Drive",
351                lines: drive.lines(),
352            });
353        }
354        for (title, top) in [
355            ("Workflows that wrote most", &self.workflows),
356            ("Jobs that wrote most", &self.top_jobs),
357        ] {
358            if !top.is_empty() {
359                sections.push(WriteSection {
360                    title,
361                    lines: top.iter().map(WriteTop::line).collect(),
362                });
363            }
364        }
365        sections
366    }
367}
368
369#[cfg(test)]
370mod tests {
371    use super::*;
372
373    const M: u64 = 1 << 20;
374    const G: u64 = 1 << 30;
375
376    #[test]
377    fn a_partial_figure_reads_as_a_floor_and_says_why() {
378        let mut written = DiskWritten {
379            bytes: 640 * M,
380            ios: 1,
381            dirty: Some(0),
382            measure: Measure::Complete,
383        };
384        assert_eq!(written.short(), "640M");
385        assert_eq!(written.line(), "640M in 1 write");
386        written.measure = Measure::Partial;
387        written.dirty = Some(96 * M);
388        assert_eq!(written.short(), "≥640M");
389        assert_eq!(written.tone(), Tone::Quiet, "a floor is in the background");
390        assert_eq!(
391            written.line(),
392            "at least 640M in 1 write: part of its run was not read; 96M more waited to be written back as its \
393             container ended"
394        );
395    }
396
397    #[test]
398    fn the_drive_s_lines_share_its_bytes_out_without_passing_the_whole() {
399        let shares = DriveShares {
400            watched: 6 * 3600,
401            written: 600 * G,
402            jobs: 150 * G,
403            slice: Some(200 * G),
404            daemon: Some(6 * G),
405        };
406        let drive = DriveWrites {
407            device: "nvme0n1".to_owned(),
408            watched: 6 * 3600,
409            written: 600 * G,
410            shares,
411            commits: Some(Commits {
412                count: 43_200,
413                watched: 6 * 3600,
414            }),
415            store: None,
416        };
417        assert_eq!(
418            drive.lines(),
419            [
420                "nvme0n1 wrote 600G over the 6h 00m runnerd read it: 2.34T a day at this rate",
421                "Runner jobs 25%; the rest of the runner slice 8% (seeds, the mirror, runners before and after \
422                 their jobs, write-back after a container ended); runnerd itself 1%; no owner runnerd knows 66%",
423                "btrfs committed 2.0 times a second",
424            ]
425        );
426        // Jobs that ended in the window may count bytes written before it: never past the whole.
427        let over = DriveWrites {
428            shares: DriveShares {
429                slice: Some(100 * G),
430                jobs: 700 * G,
431                ..shares
432            },
433            commits: None,
434            ..drive.clone()
435        };
436        assert_eq!(
437            over.lines()[1],
438            "Runner jobs 100%; the rest of the runner slice 0% (seeds, the mirror, runners before and after their \
439             jobs, write-back after a container ended); runnerd itself 0%; no owner runnerd knows 0%"
440        );
441        assert_eq!(over.lines().len(), 2, "no commits line off btrfs");
442        let in_a_file = DriveWrites {
443            store: Some("loop0".to_owned()),
444            ..over.clone()
445        };
446        assert_eq!(
447            in_a_file.lines()[1],
448            "Docker's root is a file system in a file on it (loop0): the jobs' and the runner slice's figures are \
449             bytes written to that file system; nvme0n1 also wrote its journal and the overhead of the file system \
450             under it"
451        );
452        // The slice read over only part of the time: its shares say over which.
453        let part = DriveWrites {
454            shares: DriveShares {
455                watched: 3600,
456                written: 100 * G,
457                jobs: 50 * G,
458                slice: Some(60 * G),
459                daemon: None,
460            },
461            ..drive
462        };
463        assert_eq!(
464            part.lines()[1],
465            "Over the 1h 00m runnerd read the runner slice: runner jobs 50%; the rest of the runner slice 10% (seeds, \
466             the mirror, runners before and after their jobs, write-back after a container ended); no owner runnerd \
467             knows 40%"
468        );
469    }
470}