Skip to main content

runnerd_core/api/
storage.rs

1//! What the disk runnerd keeps means as every client shows it: what each cache volume holds and what its limit does,
2//! what the build cache did for a job and over a week, where GitHub Actions' cache is served, how full the records are (the doctor's `records` check too) and what became of a job's record,
3//! and how much of a job's log is stored. Plain derivations of the API's types; clients only lay them out.
4
5use super::{
6    BuildCacheCounts, BuildCacheStats, CacheKind, CacheVolume, Check, CheckStatus, LabelledFigure,
7    LogSummary, RecordLimits, RecordNote, RecordRemoval, Records, Tone, period,
8};
9use crate::config::ByteSize;
10
11/// The share of `[records] max_total` past which the records are near it, in percent: new ones then remove the
12/// oldest to fit.
13pub const RECORDS_NEAR_FULL: u64 = 80;
14
15/// Whether records taking `bytes` are near `max_total` (`RECORDS_NEAR_FULL`).
16pub fn records_near_full(bytes: u64, max_total: u64) -> bool {
17    u128::from(bytes) * 100 >= u128::from(max_total) * u128::from(RECORDS_NEAR_FULL)
18}
19
20/// The doctor's `records` check for records taking `bytes` of `[records] max_total`: a warning with what to run when
21/// they are near it (`records_near_full`), else how much they take.
22pub fn records_check(bytes: u64, max_total: u64) -> Check {
23    let taken = format!(
24        "the records of failed jobs take {} of [records] max_total = {}",
25        ByteSize(bytes),
26        ByteSize(max_total)
27    );
28    if records_near_full(bytes, max_total) {
29        Check {
30            name: "records".to_owned(),
31            status: CheckStatus::Warn,
32            detail: format!("{taken}; new ones remove the oldest to fit"),
33            fix: Some(
34                "runnerctl records rm --older-than 2d, or raise max_total under [records]"
35                    .to_owned(),
36            ),
37        }
38    } else {
39        Check {
40            name: "records".to_owned(),
41            status: CheckStatus::Info,
42            detail: taken,
43            fix: None,
44        }
45    }
46}
47
48impl CacheKind {
49    /// What it holds, as every client names it: `jvm snapshot`, `jvm slot 2 layer`, `pnpm store, shared`, `Maven
50    /// repository, shared`, `Docker Hub mirror` or `build cache, shared`.
51    pub fn description(&self) -> String {
52        match self {
53            CacheKind::Snapshot { profile } => format!("{profile} snapshot"),
54            CacheKind::Slot { profile, slot } => format!("{profile} slot {slot} layer"),
55            CacheKind::Pnpm => "pnpm store, shared".to_owned(),
56            CacheKind::Maven => "Maven repository, shared".to_owned(),
57            CacheKind::Mirror => "Docker Hub mirror".to_owned(),
58            CacheKind::Build => "build cache, shared".to_owned(),
59        }
60    }
61
62    /// The profile it belongs to; `None` for a shared one.
63    pub fn profile(&self) -> Option<&str> {
64        match self {
65            CacheKind::Snapshot { profile } | CacheKind::Slot { profile, .. } => Some(profile),
66            CacheKind::Pnpm | CacheKind::Maven | CacheKind::Mirror | CacheKind::Build => None,
67        }
68    }
69}
70
71/// A cache volume against its limit, from `CacheVolume::limit_status`.
72#[derive(Debug, Clone, PartialEq, Eq)]
73pub struct CacheLimit {
74    /// In bytes.
75    pub limit: u64,
76    /// Whether the volume is past it, so runnerd acts on it.
77    pub over: bool,
78    /// What runnerd does at it: `emptied above 5G`, `trimmed to 20G`, or for a snapshot `started over at its next
79    /// reseed above 30G`.
80    pub label: String,
81}
82
83impl CacheLimit {
84    /// What its size means: past the limit is worth a look.
85    pub fn tone(&self) -> Tone {
86        if self.over { Tone::Warn } else { Tone::Neutral }
87    }
88}
89
90impl CacheVolume {
91    /// Its size against its limit, when it has one.
92    pub fn limit_status(&self) -> Option<CacheLimit> {
93        let limit = self.limit?;
94        let size = ByteSize(limit);
95        let label = match self.kind {
96            CacheKind::Build => format!("trimmed to {size}"),
97            CacheKind::Snapshot { .. } => format!("started over at its next reseed above {size}"),
98            CacheKind::Slot { .. } | CacheKind::Pnpm | CacheKind::Maven | CacheKind::Mirror => {
99                format!("emptied above {size}")
100            }
101        };
102        Some(CacheLimit {
103            limit,
104            over: self.bytes > limit,
105            label,
106        })
107    }
108}
109
110/// What the build cache did for one job, from `BuildCacheCounts::activity`.
111#[derive(Debug, Clone, Copy, PartialEq, Eq)]
112pub struct BuildCacheActivity {
113    /// The counts.
114    pub counts: BuildCacheCounts,
115}
116
117impl BuildCacheCounts {
118    /// What it did, for a job that used it at all: looked an entry up, stored one, or was refused one. `None` for
119    /// a job that never used it.
120    pub fn activity(&self) -> Option<BuildCacheActivity> {
121        (self.lookups() > 0 || self.written_bytes > 0 || self.refused > 0)
122            .then_some(BuildCacheActivity { counts: *self })
123    }
124}
125
126impl BuildCacheActivity {
127    /// Its lookups: `3 of 4 lookups hit (75%)`, or `no lookups`.
128    pub fn lookups_line(&self) -> String {
129        let counts = &self.counts;
130        match counts.hit_percent() {
131            Some(percent) => format!(
132                "{} of {} lookups hit ({percent}%)",
133                counts.hits,
134                counts.lookups()
135            ),
136            None => "no lookups".to_owned(),
137        }
138    }
139
140    /// What it served: `1.5G served`.
141    pub fn served_line(&self) -> String {
142        format!("{} served", ByteSize(self.counts.read_bytes))
143    }
144
145    /// What it stored: `2G stored`, or `read only` for a job that may not write.
146    pub fn stored_line(&self) -> String {
147        if self.counts.writer {
148            format!("{} stored", ByteSize(self.counts.written_bytes))
149        } else {
150            "read only".to_owned()
151        }
152    }
153
154    /// The uploads it refused, when it refused any: `3 uploads refused`.
155    pub fn refused_line(&self) -> Option<String> {
156        (self.counts.refused > 0).then(|| format!("{} uploads refused", self.counts.refused))
157    }
158
159    /// A line for a client with little room: the hit rate and what it served, or what a job that only wrote stored,
160    /// and always the uploads it refused: `75% hit, 1.5G served, 3 uploads refused`, `read only`.
161    pub fn short_line(&self) -> String {
162        let mut parts = match self.counts.hit_percent() {
163            Some(percent) => vec![format!("{percent}% hit"), self.served_line()],
164            None => vec![self.stored_line()],
165        };
166        parts.extend(self.refused_line());
167        parts.join(", ")
168    }
169
170    /// All of it in a line: `3 of 4 lookups hit (75%), 1.5G served, 2G stored, 3 uploads refused`.
171    pub fn line(&self) -> String {
172        let mut parts = vec![self.lookups_line(), self.served_line(), self.stored_line()];
173        parts.extend(self.refused_line());
174        parts.join(", ")
175    }
176}
177
178/// The tools whose caches the build cache serves, as clients name them over its counts (ADR 0052, 0083). The image
179/// points each at it but Nx, which fails its run when the cache does not answer.
180pub const BUILD_CACHE_TOOLS: &str =
181    "Gradle, sccache, Bazel and Turborepo, and Nx where a workflow opts in";
182
183/// What doctor and the Caches page name GitHub Actions' cache.
184pub const ACTIONS_CACHE: &str = "GitHub Actions' cache";
185
186/// Where jobs' GitHub Actions cache is (ADR 0082), with `[caches.build] actions_cache` at `served`.
187pub fn actions_cache_place(served: bool) -> &'static str {
188    if served {
189        "served from the build cache on this host, in place of GitHub's"
190    } else {
191        "GitHub's, as [caches.build] actions_cache = false sets"
192    }
193}
194
195impl BuildCacheStats {
196    /// Where jobs' GitHub Actions cache is, and while it is here, where its hits are counted and how to turn it off:
197    /// `GitHub Actions' cache: served from the build cache on this host, in place of GitHub's, and counted with it;
198    /// [caches.build] actions_cache = false turns it off`.
199    pub fn actions_cache_line(&self) -> String {
200        let place = actions_cache_place(self.actions_cache);
201        if self.actions_cache {
202            format!(
203                "{ACTIONS_CACHE}: {place}, and counted with it; [caches.build] actions_cache = false turns it \
204                 off"
205            )
206        } else {
207            format!("{ACTIONS_CACHE}: {place}")
208        }
209    }
210
211    /// Its entries and the last 7 days' jobs as figures, in the order they show: its entries; the share of lookups
212    /// that hit, out of how many, and what it served, or that nothing looked an entry up; what it stored, when a job
213    /// used it; the uploads it refused, when any. The same facts as [`BuildCacheStats::week_line`].
214    pub fn figures(&self) -> Vec<LabelledFigure> {
215        let week = &self.week;
216        let mut figures = vec![LabelledFigure::new("Entries", self.entries.to_string())];
217        match week.hit_percent() {
218            Some(percent) => {
219                figures.push(
220                    LabelledFigure::new("Lookups hit, 7 days", format!("{percent}%"))
221                        .of(format!("of {}", week.lookups())),
222                );
223                figures.push(LabelledFigure::new(
224                    "Served",
225                    ByteSize(week.read_bytes).to_string(),
226                ));
227            }
228            None => figures.push(LabelledFigure::new("Lookups, 7 days", "0")),
229        }
230        if week.activity().is_some() {
231            figures.push(LabelledFigure::new(
232                "Stored",
233                ByteSize(week.written_bytes).to_string(),
234            ));
235        }
236        if week.refused > 0 {
237            figures.push(LabelledFigure::new(
238                "Uploads refused",
239                week.refused.to_string(),
240            ));
241        }
242        figures
243    }
244
245    /// Its entries and the last 7 days' jobs: `120 entries; 75% of 400 lookups hit in 7 days, 3G served, 1G
246    /// stored, 2 uploads refused`, `120 entries; no lookups in 7 days, 1G stored`, or `120 entries; no lookups in 7
247    /// days`.
248    pub fn week_line(&self) -> String {
249        let week = &self.week;
250        let mut parts = match week.hit_percent() {
251            Some(percent) => vec![
252                format!("{percent}% of {} lookups hit in 7 days", week.lookups()),
253                format!("{} served", ByteSize(week.read_bytes)),
254            ],
255            None => vec!["no lookups in 7 days".to_owned()],
256        };
257        if week.activity().is_some() {
258            parts.push(format!("{} stored", ByteSize(week.written_bytes)));
259        }
260        if week.refused > 0 {
261            parts.push(format!("{} uploads refused", week.refused));
262        }
263        format!("{} entries; {}", self.entries, parts.join(", "))
264    }
265}
266
267impl RecordLimits {
268    /// Whether records taking `bytes` are near `max_total` (`records_near_full`).
269    pub fn near_full(&self, bytes: u64) -> bool {
270        records_near_full(bytes, self.max_total)
271    }
272}
273
274impl Records {
275    /// How many there are and how much they take of their limits, and when near `max_total`, `Records::pressure`:
276    /// `3 of 20 records, 1.2G of 10G, each kept 7 days; near max_total, so new ones remove the oldest`.
277    pub fn summary(&self) -> String {
278        match self.pressure() {
279            Some(pressure) => format!("{}; {pressure}", self.usage()),
280            None => self.usage(),
281        }
282    }
283
284    /// How many there are and how much they take of their limits: `3 of 20 records, 1.2G of 10G, each kept 7 days`.
285    pub fn usage(&self) -> String {
286        let limits = &self.limits;
287        // `[records] max_age` is in days more often than not.
288        let age = match limits.max_age / 86_400 {
289            1 if limits.max_age == 86_400 => "1 day".to_owned(),
290            days if days > 1 && limits.max_age.is_multiple_of(86_400) => format!("{days} days"),
291            _ => period(limits.max_age),
292        };
293        format!(
294            "{} of {} records, {} of {}, each kept {age}",
295            self.records.len(),
296            limits.max_count,
297            ByteSize(self.bytes()),
298            ByteSize(limits.max_total),
299        )
300    }
301
302    /// What being near `max_total` means, when they are: `near max_total, so new ones remove the oldest`.
303    pub fn pressure(&self) -> Option<&'static str> {
304        self.limits
305            .near_full(self.bytes())
306            .then_some("near max_total, so new ones remove the oldest")
307    }
308
309    /// What the summary means: near `max_total` is worth a look.
310    pub fn summary_tone(&self) -> Tone {
311        if self.limits.near_full(self.bytes()) {
312            Tone::Warn
313        } else {
314            Tone::Neutral
315        }
316    }
317
318    /// Why there are none: `none kept: a profile with record_failures = true keeps each failed job's runner as a
319    /// record, to open a shell where it failed`.
320    pub fn empty_reason() -> &'static str {
321        "none kept: a profile with record_failures = true keeps each failed job's runner as a record, to open a \
322         shell where it failed"
323    }
324
325    /// What a kept record says when the job history no longer has its job, as `GET /v1/jobs/{id}` answers 404 once
326    /// `history.jobs` pruned it: its image is kept, so its shell still opens, but it has no job page.
327    pub fn history_gone() -> &'static str {
328        "its job is past the history; its shell still opens"
329    }
330
331    /// Whether a shell is open in `job`'s record.
332    pub fn shell_open(&self, job: u64) -> bool {
333        self.shells.contains(&job)
334    }
335}
336
337/// What became of a failed job's record, from `RecordNote::state`.
338#[derive(Debug, Clone, Copy, PartialEq, Eq)]
339pub enum RecordState<'note> {
340    /// None was made.
341    Skipped {
342        /// Why, such as too little free disk.
343        why: &'note str,
344    },
345    /// It was made, then removed.
346    Removed {
347        /// When.
348        at: u64,
349        /// Why, when known.
350        by: Option<RecordRemoval>,
351    },
352    /// Its image went from Docker without runnerd removing it: when and why are not known.
353    Gone {
354        /// When runnerd found it gone.
355        noticed_at: u64,
356    },
357    /// It is kept: a shell can open in it.
358    Kept {
359        /// Its size when made, when known.
360        bytes: Option<u64>,
361    },
362}
363
364impl RecordNote {
365    /// What became of the record: not made, made and removed, gone outside runnerd, or kept, in that order of
366    /// precedence.
367    pub fn state(&self) -> RecordState<'_> {
368        match (&self.skipped, self.removed_at, self.gone_at) {
369            (Some(why), _, _) => RecordState::Skipped { why },
370            (None, Some(at), _) => RecordState::Removed {
371                at,
372                by: self.removed_by,
373            },
374            (None, None, Some(noticed_at)) => RecordState::Gone { noticed_at },
375            (None, None, None) => RecordState::Kept { bytes: self.bytes },
376        }
377    }
378
379    /// The command that opens a shell in `job`'s record: `runnerctl shell 42`.
380    pub fn shell_command(job: u64) -> String {
381        format!("runnerctl shell {job}")
382    }
383}
384
385impl RecordState<'_> {
386    /// What a record found gone from Docker says, on its job's page and for `record.gone`.
387    pub fn gone_words() -> &'static str {
388        "removed outside runnerd, when and why unknown"
389    }
390
391    /// Its words, before when it was removed for a removed one: `not kept: too little free disk`, `removed past
392    /// max_age`, or `record kept, 2G`.
393    pub fn line(&self) -> String {
394        match self {
395            RecordState::Skipped { why } => format!("not kept: {why}"),
396            RecordState::Removed { by, .. } => {
397                by.map_or_else(|| "removed".to_owned(), |by| by.to_string())
398            }
399            RecordState::Gone { .. } => RecordState::gone_words().to_owned(),
400            RecordState::Kept { bytes: Some(bytes) } => {
401                format!("record kept, {}", ByteSize(*bytes))
402            }
403            RecordState::Kept { bytes: None } => "record kept".to_owned(),
404        }
405    }
406}
407
408impl LogSummary {
409    /// How much of the log is stored: `120 lines, 4M`, and `, cut at history.log_max` once later lines were
410    /// dropped.
411    pub fn description(&self) -> String {
412        let line = format!("{} lines, {}", self.lines, ByteSize(self.bytes));
413        if self.truncated {
414            format!("{line}, cut at history.log_max")
415        } else {
416            line
417        }
418    }
419}
420
421#[cfg(test)]
422mod tests;