Skip to main content

runnerd_core/api/
disk.rs

1//! What runner jobs and the drive under Docker's root wrote (ADR 0143), as clients read it, and every word they show
2//! of it: a job's figure in a table and on its page, and the disk-writes section of `GET /v1/insights`. Sizes are
3//! bytes, times seconds since the Unix epoch.
4
5use std::fmt::Write as _;
6
7use serde::{Deserialize, Serialize};
8
9use super::{JOBS_HEADING, LabelledFigure, Tone, duration};
10use crate::config::ByteSize;
11
12/// What a job's runner container wrote to the disks under Docker's root, from its cgroup's `io.stat`: what its
13/// processes, the Docker Engine inside it and that engine's containers sent the disks from the job's start to the
14/// container's last read. Bytes still in the page cache then go to the disk after, on the runner slice's account.
15#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
16pub struct DiskWritten {
17    /// Bytes written.
18    pub bytes: u64,
19    /// Write requests.
20    pub ios: u64,
21    /// What its container had written to the page cache and the kernel not yet to the disk at its last read: written
22    /// after, unless deleted first. `None` where the kernel does not say.
23    #[serde(default, skip_serializing_if = "Option::is_none")]
24    pub dirty: Option<u64>,
25    /// Whether that is all it wrote.
26    pub measure: Measure,
27}
28
29/// Whether a job's [`DiskWritten`] is all it wrote. A job no reading measured has no figure at all.
30#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
31#[serde(rename_all = "kebab-case")]
32pub enum Measure {
33    /// Read from before the job's start to within a sample of its container's end.
34    Complete,
35    /// Part of it was not read: its container's end, as when it ended while no daemon ran, or the counter's start,
36    /// as when the container was started again, or its start, as when the counter was not read before the job.
37    /// What was read is a floor.
38    Partial,
39    /// A measure a later release names; read as partial.
40    #[serde(other)]
41    Unknown,
42}
43
44impl DiskWritten {
45    /// It in a table cell: `1.2G`, or `≥1.2G` for a partial figure.
46    pub fn short(&self) -> String {
47        match self.measure {
48            Measure::Complete => ByteSize(self.bytes).to_string(),
49            Measure::Partial | Measure::Unknown => format!("≥{}", ByteSize(self.bytes)),
50        }
51    }
52
53    /// It on a job's page: `1.2G in 3402 writes`, then `; 80M more waited to be written back as its container
54    /// ended` when some did, or `at least 1.2G in 3402 writes: …` with why a partial figure is one.
55    pub fn line(&self) -> String {
56        let writes = if self.ios == 1 { "write" } else { "writes" };
57        let mut line = match self.measure {
58            Measure::Complete => format!("{} in {} {writes}", ByteSize(self.bytes), self.ios),
59            Measure::Partial | Measure::Unknown => format!(
60                "at least {} in {} {writes}: part of its run was not read",
61                ByteSize(self.bytes),
62                self.ios
63            ),
64        };
65        if let Some(dirty) = self.dirty.filter(|dirty| *dirty > 0) {
66            let _ = write!(
67                line,
68                "; {} more waited to be written back as its container ended",
69                ByteSize(dirty)
70            );
71        }
72        line
73    }
74
75    /// What it means: a partial figure is a floor, so it reads in the background.
76    pub fn tone(&self) -> Tone {
77        match self.measure {
78            Measure::Complete => Tone::Neutral,
79            Measure::Partial | Measure::Unknown => Tone::Quiet,
80        }
81    }
82}
83
84/// The disk-writes section of [`Insights`](super::Insights): what the jobs that ended in the window wrote, the
85/// workflows and jobs that wrote most, and what the drive wrote while the daemon read it.
86#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
87pub struct DiskWrites {
88    /// The jobs' figures together.
89    pub jobs: JobWrites,
90    /// The workflows whose jobs wrote most, most first.
91    pub workflows: Vec<WriteTop>,
92    /// The jobs, by repository, workflow and name, that wrote most, most first.
93    pub top_jobs: Vec<WriteTop>,
94    /// The drive's figures; `None` when the daemon read none in the window.
95    #[serde(default, skip_serializing_if = "Option::is_none")]
96    pub drive: Option<DriveWrites>,
97}
98
99/// What the jobs that ended in a window wrote, each its runner container's [`DiskWritten`].
100#[derive(Debug, Clone, Copy, Default, PartialEq, Eq, Serialize, Deserialize)]
101pub struct JobWrites {
102    /// Jobs with a figure.
103    pub measured: u32,
104    /// Of those, the partial ones.
105    pub partial: u32,
106    /// Bytes they wrote together.
107    pub total: u64,
108    /// The median of one job's bytes.
109    pub median: u64,
110    /// The 90th percentile, by nearest rank.
111    pub p90: u64,
112    /// What waited to be written back as their containers ended, together.
113    pub dirty: u64,
114}
115
116/// A workflow, or a job, and what its jobs in the window wrote.
117#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
118pub struct WriteTop {
119    /// `owner/name`.
120    pub repository: String,
121    /// The workflow's name, when known.
122    #[serde(default, skip_serializing_if = "Option::is_none")]
123    pub workflow: Option<String>,
124    /// The job's name, for a job.
125    #[serde(default, skip_serializing_if = "Option::is_none")]
126    pub job: Option<String>,
127    /// Its jobs with a figure.
128    pub jobs: u32,
129    /// Bytes they wrote together.
130    pub total: u64,
131}
132
133/// What the disks under Docker's root wrote while the daemon read them in a window: from `/sys/class/block`, the
134/// runner slice's `io.stat`, which keeps what its containers wrote after they are gone, the daemon's own
135/// `/proc/self/io`, and btrfs's commit count.
136#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
137pub struct DriveWrites {
138    /// The disks: `nvme2n1`.
139    pub device: String,
140    /// Seconds the daemon read them.
141    pub watched: u64,
142    /// Bytes they wrote then.
143    pub written: u64,
144    /// Whose those bytes were.
145    pub shares: DriveShares,
146    /// btrfs transaction commits; `None` on another file system.
147    #[serde(default, skip_serializing_if = "Option::is_none")]
148    pub commits: Option<Commits>,
149    /// The loop devices Docker's root is on when it is a file system in a file on the disks: `loop0`. Jobs' and the
150    /// runner slice's figures are then bytes written to that file system, which the disks write with its journal
151    /// and the overhead of the file system under it.
152    #[serde(default, skip_serializing_if = "Option::is_none")]
153    pub store: Option<String>,
154}
155
156/// Whose the drive's bytes were, over the stretches the daemon read the runner slice, or over all it read the drive
157/// when it never read the slice: a stretch that missed one figure leaves the others' shares be.
158#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
159pub struct DriveShares {
160    /// Seconds these were read over.
161    pub watched: u64,
162    /// Bytes the drive wrote then.
163    pub written: u64,
164    /// Of those, the figures of the jobs that ended then.
165    pub jobs: u64,
166    /// Of those, the runner slice's; `None` without its `io.stat`.
167    #[serde(default, skip_serializing_if = "Option::is_none")]
168    pub slice: Option<u64>,
169    /// What the daemon itself wrote then, to any disk; `None` where it could not read its own counters.
170    #[serde(default, skip_serializing_if = "Option::is_none")]
171    pub daemon: Option<u64>,
172}
173
174/// btrfs's transaction commits over the stretches the daemon read them.
175#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
176pub struct Commits {
177    /// Commits.
178    pub count: u64,
179    /// Seconds they were read over.
180    pub watched: u64,
181}
182
183/// Sets each part's `percent` of `whole`, whose bytes they add up to, so that the percents add up to 100: each its
184/// share rounded down, and the points left over to the parts with the largest remainders, the first of equal ones.
185/// Rounding each alone could show 1%, 1%, 1% and 99%.
186fn largest_remainder(parts: &mut [DriveShare], whole: u64) {
187    if whole == 0 {
188        return;
189    }
190    let whole = u128::from(whole);
191    let mut left = 100u128;
192    let mut remainders = Vec::with_capacity(parts.len());
193    for (index, part) in parts.iter_mut().enumerate() {
194        let scaled = u128::from(part.bytes) * 100;
195        let floor = scaled / whole;
196        part.percent = u64::try_from(floor).unwrap_or(100);
197        left = left.saturating_sub(floor);
198        remainders.push((scaled % whole, index));
199    }
200    remainders.sort_by(|a, b| b.0.cmp(&a.0).then(a.1.cmp(&b.1)));
201    for (_, index) in remainders
202        .into_iter()
203        .take(usize::try_from(left).unwrap_or(0))
204    {
205        if let Some(part) = parts.get_mut(index) {
206            part.percent += 1;
207        }
208    }
209}
210
211impl JobWrites {
212    /// Them in a line: `529 jobs wrote 144.7G: median 49M, p90 612M a job`, then `; 12 partial, counted as far as
213    /// read` and `; 3.1G more waited to be written back as their containers ended` when so.
214    pub fn line(&self) -> String {
215        let jobs = if self.measured == 1 { "job" } else { "jobs" };
216        let mut line = format!(
217            "{} {jobs} wrote {}: median {}, p90 {} a job",
218            self.measured,
219            ByteSize(self.total),
220            ByteSize(self.median),
221            ByteSize(self.p90)
222        );
223        for note in [self.partial_note(), self.dirty_note()]
224            .into_iter()
225            .flatten()
226        {
227            let _ = write!(line, "; {note}");
228        }
229        line
230    }
231
232    /// `12 partial, counted as far as read`, when a job was read only in part.
233    fn partial_note(&self) -> Option<String> {
234        (self.partial > 0).then(|| format!("{} partial, counted as far as read", self.partial))
235    }
236
237    /// `3.1G more waited to be written back as their containers ended`, when any did.
238    fn dirty_note(&self) -> Option<String> {
239        (self.dirty > 0).then(|| {
240            format!(
241                "{} more waited to be written back as their containers ended",
242                ByteSize(self.dirty)
243            )
244        })
245    }
246}
247
248impl WriteTop {
249    /// Its name: `halyard/api CI / build`; a workflow's without the job.
250    pub fn name(&self) -> String {
251        let mut name = self.repository.clone();
252        if let Some(workflow) = &self.workflow {
253            name.push(' ');
254            name.push_str(workflow);
255        }
256        if let Some(job) = &self.job {
257            name.push_str(if self.workflow.is_some() { " / " } else { " " });
258            name.push_str(job);
259        }
260        name
261    }
262
263    /// What one of its jobs wrote on average, in bytes.
264    pub fn per_job(&self) -> u64 {
265        self.total / u64::from(self.jobs.max(1))
266    }
267
268    /// What its total is over, beside it: `31 jobs, 410M a job`.
269    pub fn note(&self) -> String {
270        let jobs = if self.jobs == 1 { "job" } else { "jobs" };
271        format!("{} {jobs}, {} a job", self.jobs, ByteSize(self.per_job()))
272    }
273
274    /// It in a line: `halyard/api CI / build: 12.4G over 31 jobs, 410M a job`; a workflow's without the job.
275    pub fn line(&self) -> String {
276        format!(
277            "{}: {} over {}",
278            self.name(),
279            ByteSize(self.total),
280            self.note()
281        )
282    }
283}
284
285impl DriveWrites {
286    /// What a day at the window's rate writes, in bytes.
287    pub fn per_day(&self) -> u64 {
288        let day = u128::from(self.written) * 86_400 / u128::from(self.watched.max(1));
289        u64::try_from(day).unwrap_or(u64::MAX)
290    }
291
292    /// Them in lines: what the drive wrote and its daily rate; what the figures count when Docker's root is in a
293    /// file on it; whose it was, runner jobs', the rest of the runner slice's, runnerd's own and the share no one in
294    /// runnerd owns, unless it wrote nothing over the shares; and btrfs's commits a second.
295    pub fn lines(&self) -> Vec<String> {
296        let mut lines = vec![format!(
297            "{} wrote {} over the {} runnerd read it: {} a day at this rate",
298            self.device,
299            ByteSize(self.written),
300            duration(self.watched),
301            ByteSize(self.per_day())
302        )];
303        lines.extend(self.store_note());
304        if !self.shares.is_empty() {
305            lines.push(self.shares.line(self.watched));
306        }
307        if let Some(commits) = self.commits {
308            lines.push(format!(
309                "btrfs committed {} times a second",
310                commits.per_second()
311            ));
312        }
313        lines
314    }
315}
316
317impl DriveWrites {
318    /// What its bytes are called among the figures: `nvme0n1 wrote`.
319    fn label(&self) -> String {
320        format!("{} wrote", self.device)
321    }
322
323    /// What the figures count when Docker's root is a file system in a file on the drive.
324    fn store_note(&self) -> Option<String> {
325        self.store.as_ref().map(|store| {
326            format!(
327                "Docker's root is a file system in a file on it ({store}): the jobs' and the runner slice's figures \
328                 are bytes written to that file system; {} also wrote its journal and the overhead of the file \
329                 system under it",
330                self.device
331            )
332        })
333    }
334}
335
336impl Commits {
337    /// Commits a second, to a tenth: `2.6`.
338    pub fn per_second(&self) -> String {
339        let tenths = u128::from(self.count) * 10 / u128::from(self.watched.max(1));
340        format!("{}.{}", tenths / 10, tenths % 10)
341    }
342}
343
344/// Whose a share of what the drive wrote was.
345#[derive(Debug, Clone, Copy, PartialEq, Eq)]
346pub enum ShareOwner {
347    /// The runner jobs that ended then: what their containers wrote.
348    Jobs,
349    /// The rest of the runner slice: seeds, the mirror, runners before and after their jobs, write-back after a
350    /// container ended.
351    Slice,
352    /// runnerd itself.
353    Daemon,
354    /// No owner runnerd knows.
355    Unknown,
356}
357
358impl ShareOwner {
359    /// Its name, lower case: `runner jobs`, `the rest of the runner slice`, `runnerd itself`, `no owner runnerd
360    /// knows`.
361    pub fn label(self) -> &'static str {
362        match self {
363            ShareOwner::Jobs => "runner jobs",
364            ShareOwner::Slice => "the rest of the runner slice",
365            ShareOwner::Daemon => "runnerd itself",
366            ShareOwner::Unknown => "no owner runnerd knows",
367        }
368    }
369}
370
371/// One owner's share of what the drive wrote.
372#[derive(Debug, Clone, Copy, PartialEq, Eq)]
373pub struct DriveShare {
374    /// Whose.
375    pub owner: ShareOwner,
376    /// Its bytes.
377    pub bytes: u64,
378    /// Its part of the whole, in whole percent; the parts' add up to 100.
379    pub percent: u64,
380}
381
382impl DriveShares {
383    /// Whether they have nothing to show: the drive wrote nothing over them, as on the first read after a start, so
384    /// every part is 0 bytes and its percents would not add up to 100. A client draws no shares then.
385    pub fn is_empty(&self) -> bool {
386        self.written == 0
387    }
388
389    /// Whose the bytes were, in the order the line says them: runner jobs', the rest of the runner slice's when its
390    /// `io.stat` was read, runnerd's own when read, and the rest, no owner runnerd knows. Their bytes add up to the
391    /// whole; jobs that ended in the stretch may count what they wrote before it, so no share passes the whole.
392    pub fn parts(&self) -> Vec<DriveShare> {
393        let whole = self.written;
394        let share = |owner, bytes| DriveShare {
395            owner,
396            bytes,
397            percent: 0,
398        };
399        let counted = self.jobs.min(whole);
400        let mut parts = vec![share(ShareOwner::Jobs, counted)];
401        let owned = match self.slice {
402            Some(slice) => {
403                let slice = slice.clamp(counted, whole);
404                parts.push(share(ShareOwner::Slice, slice - counted));
405                slice
406            }
407            None => counted,
408        };
409        let daemon = self.daemon.map(|daemon| daemon.min(whole - owned));
410        if let Some(daemon) = daemon {
411            parts.push(share(ShareOwner::Daemon, daemon));
412        }
413        parts.push(share(
414            ShareOwner::Unknown,
415            whole - owned - daemon.unwrap_or(0),
416        ));
417        largest_remainder(&mut parts, whole);
418        parts
419    }
420
421    /// Them in a line, saying the stretch they cover when it is shorter than the `watched` seconds the drive was
422    /// read: `Runner jobs 25%; the rest of the runner slice 8% (...); runnerd itself 1%; no owner runnerd knows 66%`.
423    /// No share passes the whole.
424    pub fn line(&self, watched: u64) -> String {
425        let mut line = self
426            .stretch_note(watched)
427            .map(|stretch| format!("{stretch}: "))
428            .unwrap_or_default();
429        let said: Vec<String> = self
430            .parts()
431            .iter()
432            .map(|part| {
433                let note = if part.owner == ShareOwner::Slice {
434                    " (seeds, the mirror, runners before and after their jobs, write-back after a container ended)"
435                } else {
436                    ""
437                };
438                format!("{} {}%{note}", part.owner.label(), part.percent)
439            })
440            .collect();
441        line.push_str(&said.join("; "));
442        if self.slice.is_none() {
443            line.push_str(", ");
444            line.push_str(Self::SLICE_UNREAD);
445        }
446        sentence_case(&line)
447    }
448
449    /// What the shares say when the runner slice was not read.
450    const SLICE_UNREAD: &'static str = "the runner slice's share unread";
451
452    /// `over the 1h 00m runnerd read the runner slice`, when that is shorter than the `watched` seconds the drive was
453    /// read.
454    fn stretch_note(&self, watched: u64) -> Option<String> {
455        (self.watched < watched).then(|| {
456            format!(
457                "over the {} runnerd read the runner slice",
458                duration(self.watched)
459            )
460        })
461    }
462}
463
464/// `line` starting as a sentence does, its first letter a capital.
465fn sentence_case(line: &str) -> String {
466    let mut chars = line.chars();
467    chars.next().map_or_else(String::new, |first| {
468        first.to_uppercase().chain(chars).collect()
469    })
470}
471
472/// A part of the disk-writes section, under its title.
473#[derive(Debug, Clone, PartialEq, Eq)]
474pub struct WriteSection {
475    /// `Jobs`, `Drive`, [`DiskWrites::WORKFLOWS_TITLE`], [`DiskWrites::TOP_JOBS_TITLE`].
476    pub title: &'static str,
477    /// Its lines, in order.
478    pub lines: Vec<String>,
479}
480
481impl DiskWrites {
482    /// The section's title: `Disk writes`.
483    pub const TITLE: &'static str = "Disk writes";
484    /// What the drive's shares are called over them.
485    pub const SHARES_TITLE: &'static str = "Who wrote to the drive";
486    /// What the workflows that wrote most are called.
487    pub const WORKFLOWS_TITLE: &'static str = "Workflows that wrote most";
488    /// What the jobs that wrote most are called.
489    pub const TOP_JOBS_TITLE: &'static str = "Jobs that wrote most";
490    /// What the jobs' bytes are called among the figures.
491    const JOBS_WROTE: &'static str = "Jobs wrote";
492
493    /// What the figures leave out that changes how they read, a line each, in the order the figures show, each
494    /// starting with the label of the figure or the title it is about: jobs read only in part, bytes that waited to be
495    /// written back, Docker's root as a file system in a file on the drive, and the drive's shares read over a
496    /// shorter stretch than the drive or without the runner slice's, when there are shares to show. The same words as
497    /// [`DiskWrites::sections`].
498    pub fn caveats(&self) -> Vec<String> {
499        let mut caveats = Vec::new();
500        if self.jobs.measured > 0 {
501            caveats.extend(
502                [self.jobs.partial_note(), self.jobs.dirty_note()]
503                    .into_iter()
504                    .flatten()
505                    .map(|note| format!("{}: {note}", Self::JOBS_WROTE)),
506            );
507        }
508        if let Some(drive) = &self.drive {
509            caveats.extend(
510                drive
511                    .store_note()
512                    .map(|note| format!("{}: {note}", drive.label())),
513            );
514            caveats.extend(
515                (!drive.shares.is_empty())
516                    .then(|| drive.shares.stretch_note(drive.watched))
517                    .flatten()
518                    .into_iter()
519                    .chain(
520                        (!drive.shares.is_empty() && drive.shares.slice.is_none())
521                            .then(|| DriveShares::SLICE_UNREAD.to_owned()),
522                    )
523                    .map(|note| format!("{}: {note}", Self::SHARES_TITLE)),
524            );
525        }
526        caveats
527    }
528
529    /// Its figures, in the order they show: the jobs' bytes, count, median and p90 when a job was measured; the
530    /// drive's bytes, daily rate and btrfs commits when it was read.
531    pub fn figures(&self) -> Vec<LabelledFigure> {
532        let mut figures = Vec::new();
533        let jobs = &self.jobs;
534        if jobs.measured > 0 {
535            figures.extend([
536                LabelledFigure::new(Self::JOBS_WROTE, ByteSize(jobs.total).to_string()),
537                LabelledFigure::new(JOBS_HEADING, jobs.measured.to_string()),
538                LabelledFigure::new("Median a job", ByteSize(jobs.median).to_string()),
539                LabelledFigure::new("p90 a job", ByteSize(jobs.p90).to_string()),
540            ]);
541        }
542        if let Some(drive) = &self.drive {
543            figures.push(LabelledFigure::new(
544                drive.label(),
545                ByteSize(drive.written).to_string(),
546            ));
547            figures.push(LabelledFigure::new(
548                "A day at this rate",
549                ByteSize(drive.per_day()).to_string(),
550            ));
551            if let Some(commits) = drive.commits {
552                figures.push(LabelledFigure::new(
553                    "btrfs commits a second",
554                    commits.per_second(),
555                ));
556            }
557        }
558        figures
559    }
560
561    /// Its parts in order, each with what it says, none empty: the jobs' figures, the drive's, and the workflows and
562    /// the jobs that wrote most.
563    pub fn sections(&self) -> Vec<WriteSection> {
564        let mut sections = Vec::new();
565        if self.jobs.measured > 0 {
566            sections.push(WriteSection {
567                title: "Jobs",
568                lines: vec![self.jobs.line()],
569            });
570        }
571        if let Some(drive) = &self.drive {
572            sections.push(WriteSection {
573                title: "Drive",
574                lines: drive.lines(),
575            });
576        }
577        for (title, top) in [
578            (Self::WORKFLOWS_TITLE, &self.workflows),
579            (Self::TOP_JOBS_TITLE, &self.top_jobs),
580        ] {
581            if !top.is_empty() {
582                sections.push(WriteSection {
583                    title,
584                    lines: top.iter().map(WriteTop::line).collect(),
585                });
586            }
587        }
588        sections
589    }
590}
591
592#[cfg(test)]
593mod tests {
594    use super::*;
595
596    const M: u64 = 1 << 20;
597    const G: u64 = 1 << 30;
598
599    #[test]
600    fn a_partial_figure_reads_as_a_floor_and_says_why() {
601        let mut written = DiskWritten {
602            bytes: 640 * M,
603            ios: 1,
604            dirty: Some(0),
605            measure: Measure::Complete,
606        };
607        assert_eq!(written.short(), "640M");
608        assert_eq!(written.line(), "640M in 1 write");
609        written.measure = Measure::Partial;
610        written.dirty = Some(96 * M);
611        assert_eq!(written.short(), "≥640M");
612        assert_eq!(written.tone(), Tone::Quiet, "a floor is in the background");
613        assert_eq!(
614            written.line(),
615            "at least 640M in 1 write: part of its run was not read; 96M more waited to be written back as its \
616             container ended"
617        );
618    }
619
620    #[test]
621    fn the_drive_s_lines_share_its_bytes_out_without_passing_the_whole() {
622        let shares = DriveShares {
623            watched: 6 * 3600,
624            written: 600 * G,
625            jobs: 150 * G,
626            slice: Some(200 * G),
627            daemon: Some(6 * G),
628        };
629        let drive = DriveWrites {
630            device: "nvme0n1".to_owned(),
631            watched: 6 * 3600,
632            written: 600 * G,
633            shares,
634            commits: Some(Commits {
635                count: 43_200,
636                watched: 6 * 3600,
637            }),
638            store: None,
639        };
640        assert_eq!(
641            drive.lines(),
642            [
643                "nvme0n1 wrote 600G over the 6h 00m runnerd read it: 2.34T a day at this rate",
644                "Runner jobs 25%; the rest of the runner slice 8% (seeds, the mirror, runners before and after \
645                 their jobs, write-back after a container ended); runnerd itself 1%; no owner runnerd knows 66%",
646                "btrfs committed 2.0 times a second",
647            ]
648        );
649        // Jobs that ended in the window may count bytes written before it: never past the whole.
650        let over = DriveWrites {
651            shares: DriveShares {
652                slice: Some(100 * G),
653                jobs: 700 * G,
654                ..shares
655            },
656            commits: None,
657            ..drive.clone()
658        };
659        assert_eq!(
660            over.lines()[1],
661            "Runner jobs 100%; the rest of the runner slice 0% (seeds, the mirror, runners before and after their \
662             jobs, write-back after a container ended); runnerd itself 0%; no owner runnerd knows 0%"
663        );
664        assert_eq!(over.lines().len(), 2, "no commits line off btrfs");
665        let in_a_file = DriveWrites {
666            store: Some("loop0".to_owned()),
667            ..over.clone()
668        };
669        assert_eq!(
670            in_a_file.lines()[1],
671            "Docker's root is a file system in a file on it (loop0): the jobs' and the runner slice's figures are \
672             bytes written to that file system; nvme0n1 also wrote its journal and the overhead of the file system \
673             under it"
674        );
675        // The slice read over only part of the time: its shares say over which.
676        let part = DriveWrites {
677            shares: DriveShares {
678                watched: 3600,
679                written: 100 * G,
680                jobs: 50 * G,
681                slice: Some(60 * G),
682                daemon: None,
683            },
684            ..drive
685        };
686        assert_eq!(
687            part.lines()[1],
688            "Over the 1h 00m runnerd read the runner slice: runner jobs 50%; the rest of the runner slice 10% (seeds, \
689             the mirror, runners before and after their jobs, write-back after a container ended); no owner runnerd \
690             knows 40%"
691        );
692    }
693
694    /// A client draws the shares as one bar: a share whose bytes ran past the whole, or that left a gap, would draw
695    /// a bar longer or shorter than the drive's.
696    #[test]
697    fn the_drive_s_shares_as_parts_fill_the_whole_and_match_its_line() {
698        let shares = DriveShares {
699            watched: 3600,
700            written: 100 * G,
701            // Jobs that ended in the stretch count what they wrote before it.
702            jobs: 150 * G,
703            slice: Some(90 * G),
704            daemon: Some(G),
705        };
706        let parts = shares.parts();
707        assert_eq!(
708            parts.iter().map(|part| part.bytes).sum::<u64>(),
709            100 * G,
710            "{parts:?}"
711        );
712        assert_eq!(parts[0].owner, ShareOwner::Jobs);
713        assert_eq!(parts[0].percent, 100);
714        assert_eq!(
715            shares.line(3600),
716            "Runner jobs 100%; the rest of the runner slice 0% (seeds, the mirror, runners before and after their \
717             jobs, write-back after a container ended); runnerd itself 0%; no owner runnerd knows 0%"
718        );
719    }
720
721    /// A key under the share bar reads as a whole: rounding each share alone showed 1%, 1%, 1% and 99%.
722    #[test]
723    fn the_drive_s_shares_add_up_to_100_percent_by_their_largest_remainders() {
724        let percents = |shares: DriveShares| -> Vec<u64> {
725            shares.parts().iter().map(|part| part.percent).collect()
726        };
727        let halves = DriveShares {
728            watched: 3600,
729            written: 1000,
730            jobs: 5,
731            slice: Some(10),
732            daemon: Some(5),
733        };
734        assert_eq!(
735            percents(halves),
736            [1, 1, 0, 98],
737            "the first of equal remainders take the points left"
738        );
739        let thirds = DriveShares {
740            watched: 3600,
741            written: 3,
742            jobs: 1,
743            slice: Some(2),
744            daemon: Some(1),
745        };
746        assert_eq!(percents(thirds), [34, 33, 33, 0]);
747        let none = DriveShares {
748            watched: 3600,
749            written: 0,
750            jobs: 0,
751            slice: None,
752            daemon: None,
753        };
754        assert_eq!(percents(none), [0, 0]);
755    }
756
757    /// The figures every client shows of the section, worded once: a client that worded its own would drift.
758    #[test]
759    fn disk_writes_read_as_labelled_figures_and_each_top_as_its_note() {
760        let top = WriteTop {
761            repository: "halyard/ledger".to_owned(),
762            workflow: Some("CI".to_owned()),
763            job: None,
764            jobs: 1,
765            total: 410 * M,
766        };
767        assert_eq!(top.note(), "1 job, 410M a job");
768        assert_eq!(top.line(), "halyard/ledger CI: 410M over 1 job, 410M a job");
769        let disk = DiskWrites {
770            jobs: JobWrites {
771                measured: 31,
772                partial: 0,
773                total: 12 * G,
774                median: 120 * M,
775                p90: 780 * M,
776                dirty: 0,
777            },
778            workflows: vec![top],
779            top_jobs: Vec::new(),
780            drive: Some(DriveWrites {
781                device: "nvme0n1".to_owned(),
782                watched: 43_200,
783                written: 40 * G,
784                shares: DriveShares {
785                    watched: 43_200,
786                    written: 40 * G,
787                    jobs: 12 * G,
788                    slice: None,
789                    daemon: None,
790                },
791                commits: Some(Commits {
792                    count: 112_320,
793                    watched: 43_200,
794                }),
795                store: None,
796            }),
797        };
798        let shown: Vec<(String, String)> = disk
799            .figures()
800            .into_iter()
801            .map(|figure| (figure.label, figure.value))
802            .collect();
803        let expected = [
804            ("Jobs wrote", "12G"),
805            ("Jobs", "31"),
806            ("Median a job", "120M"),
807            ("p90 a job", "780M"),
808            ("nvme0n1 wrote", "40G"),
809            ("A day at this rate", "80G"),
810            ("btrfs commits a second", "2.6"),
811        ]
812        .map(|(label, value)| (label.to_owned(), value.to_owned()));
813        assert_eq!(shown, expected);
814        let unmeasured = DiskWrites {
815            jobs: JobWrites::default(),
816            drive: None,
817            ..disk
818        };
819        assert!(
820            unmeasured.figures().is_empty(),
821            "no figures without a measured job or a read drive"
822        );
823    }
824
825    /// The figures leave out what the sentences say about how to read them: a client that shows only the figures
826    /// would show bytes from a partly read job, or shares of a drive behind a file system in a file, as whole.
827    #[test]
828    fn the_caveats_say_what_the_figures_leave_out_in_the_sentences_words() {
829        let shares = DriveShares {
830            watched: 3600,
831            written: 100 * G,
832            jobs: 50 * G,
833            slice: None,
834            daemon: None,
835        };
836        let disk = DiskWrites {
837            jobs: JobWrites {
838                measured: 529,
839                partial: 12,
840                total: 145 * G,
841                median: 49 * M,
842                p90: 612 * M,
843                dirty: 3 * G,
844            },
845            workflows: Vec::new(),
846            top_jobs: Vec::new(),
847            drive: Some(DriveWrites {
848                device: "nvme0n1".to_owned(),
849                watched: 2 * 3600,
850                written: 200 * G,
851                shares,
852                commits: None,
853                store: Some("loop0".to_owned()),
854            }),
855        };
856        assert_eq!(
857            disk.caveats(),
858            [
859                "Jobs wrote: 12 partial, counted as far as read",
860                "Jobs wrote: 3G more waited to be written back as their containers ended",
861                "nvme0n1 wrote: Docker's root is a file system in a file on it (loop0): the jobs' and the runner slice's \
862                 figures are bytes written to that file system; nvme0n1 also wrote its journal and the overhead of the \
863                 file system under it",
864                "Who wrote to the drive: over the 1h 00m runnerd read the runner slice",
865                "Who wrote to the drive: the runner slice's share unread",
866            ]
867        );
868        // The sentences say the same, in the same words.
869        let sentences: Vec<String> = disk
870            .sections()
871            .into_iter()
872            .flat_map(|section| section.lines)
873            .collect();
874        let said = sentences.join(" ").to_lowercase();
875        for caveat in disk.caveats() {
876            let note = [
877                DiskWrites::JOBS_WROTE,
878                DiskWrites::SHARES_TITLE,
879                "nvme0n1 wrote",
880            ]
881            .iter()
882            .find_map(|title| caveat.strip_prefix(&format!("{title}: ")))
883            .unwrap_or_else(|| panic!("{caveat} names its figure"));
884            assert!(said.contains(&note.to_lowercase()), "{note}: {sentences:?}");
885        }
886        // Read whole, the figures need no caveat.
887        let whole = DiskWrites {
888            jobs: JobWrites {
889                partial: 0,
890                dirty: 0,
891                ..disk.jobs
892            },
893            drive: disk.drive.map(|drive| DriveWrites {
894                watched: 3600,
895                store: None,
896                shares: DriveShares {
897                    slice: Some(60 * G),
898                    ..shares
899                },
900                ..drive
901            }),
902            ..disk
903        };
904        assert_eq!(whole.caveats(), Vec::<String>::new(), "nothing to say");
905    }
906
907    /// A drive that wrote nothing, as on the first read after a start, has shares of 0 bytes whose percents do not
908    /// add up: no client draws them, and core says nothing about them.
909    #[test]
910    fn a_drive_that_wrote_nothing_has_no_shares_to_show_or_speak_of() {
911        let quiet = DriveShares {
912            watched: 3600,
913            written: 0,
914            jobs: 0,
915            slice: None,
916            daemon: None,
917        };
918        assert!(quiet.is_empty(), "nothing to draw");
919        assert!(
920            !DriveShares {
921                written: G,
922                ..quiet
923            }
924            .is_empty(),
925            "the drive wrote"
926        );
927        let drive = DriveWrites {
928            device: "nvme0n1".to_owned(),
929            watched: 2 * 3600,
930            written: 0,
931            shares: quiet,
932            commits: None,
933            store: None,
934        };
935        assert_eq!(
936            drive.lines(),
937            ["nvme0n1 wrote 0B over the 2h 00m runnerd read it: 0B a day at this rate"],
938            "no shares line of zeros"
939        );
940        let disk = DiskWrites {
941            jobs: JobWrites::default(),
942            workflows: Vec::new(),
943            top_jobs: Vec::new(),
944            drive: Some(drive),
945        };
946        assert_eq!(
947            disk.caveats(),
948            Vec::<String>::new(),
949            "neither a shorter stretch nor an unread slice is said of shares not shown"
950        );
951    }
952}