Skip to main content

runnerd_core/api/
insights.rs

1//! Insights as clients read them: what got slower, what is flaky, what nears its memory cap, what memory pressure
2//! throttled, what its CPU cap held back, jobs held for the host's memory and how long jobs wait, computed from the job history (ADR 0051); the daily digest; and what the
3//! daemon's notifications carry. Times are seconds since the Unix epoch, durations seconds, sizes bytes.
4
5use serde::{Deserialize, Serialize};
6
7use super::RunnerFailure;
8use super::jobs::JobSummary;
9
10mod shown;
11
12pub use shown::InsightFigure;
13pub use shown::InsightsHeading;
14pub use shown::WaitFigure;
15
16/// The window `GET /v1/insights` reads by default, in days.
17pub const INSIGHT_DAYS: u32 = 14;
18
19/// The longest window `GET /v1/insights` reads, in days.
20pub const INSIGHT_DAYS_MAX: u32 = 90;
21
22/// `GET /v1/insights` query parameters.
23#[derive(Debug, Clone, Default, PartialEq, Eq, Serialize, Deserialize)]
24pub struct InsightsQuery {
25    /// Days of history to read; default [`INSIGHT_DAYS`], at most [`INSIGHT_DAYS_MAX`].
26    pub days: Option<u32>,
27}
28
29/// `GET /v1/insights`: what the history says about the window.
30#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
31pub struct Insights {
32    /// The start of the window.
33    pub since: u64,
34    /// When it was read: the end of the window.
35    pub until: u64,
36    /// Warnings first, then by repository, workflow and job.
37    pub insights: Vec<Insight>,
38    /// Each repository's waits this week and the week before, by repository. Always the last 14 days, whatever the
39    /// window.
40    pub waits: Vec<RepositoryWaits>,
41    /// What the host's memory came to on each UTC day of the window that CI ran, newest first (ADR 0108); absent from
42    /// a daemon before 0.0.30.
43    #[serde(default, skip_serializing_if = "Vec::is_empty")]
44    pub host: Vec<HostHazard>,
45    /// What the jobs that ended in the window and the drive under Docker's root wrote (ADR 0143); `None` when no job
46    /// was measured and the daemon read no drive, and from a daemon before 0.1.3.
47    #[serde(default, skip_serializing_if = "Option::is_none")]
48    pub disk: Option<super::DiskWrites>,
49    /// Why each profile's jobs that ended in the window queued, split by what held them (ADR 0154), the profile whose
50    /// jobs waited longest first; empty from a daemon before 0.1.5.
51    #[serde(default, skip_serializing_if = "Vec::is_empty")]
52    pub queue: Vec<ProfileQueue>,
53}
54
55/// Why one profile's jobs queued over a window, from what each plan that held them said (ADR 0154). Unlike the rest
56/// of insights, its times are milliseconds, so shares of short waits come out right. A wait runs from GitHub queueing
57/// a job to its start, less what passed while no runnerd ran, summed over the jobs counted; the causes and the two
58/// parts no cause covers add up to `waited_ms`.
59#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
60pub struct ProfileQueue {
61    /// The profile.
62    pub profile: String,
63    /// The jobs whose waits it splits.
64    pub jobs: u32,
65    /// Their waits together.
66    pub waited_ms: u64,
67    /// What each cause held them for, longest first; none of 0.
68    pub causes: Vec<CauseWait>,
69    /// Of `waited_ms`, what passed before runnerd first listed the jobs queued: GitHub's listing lag and the time to
70    /// runnerd's next listing, less any downtime in it.
71    pub before_listed_ms: u64,
72    /// Of `waited_ms`, what no plan held them for once listed: after a restart until the first plan, and between a
73    /// planned start and its runner.
74    pub uncovered_ms: u64,
75    /// The jobs left out as their history rows are from before runnerd counted why jobs wait.
76    pub unrecorded: u32,
77    /// The jobs whose causes were counted past their start, as when a runner took one before runnerd read it, and
78    /// were scaled down into the wait left after the time before the listing: their causes are estimates.
79    #[serde(default, skip_serializing_if = "is_zero")]
80    pub estimated: u32,
81}
82
83/// Whether a count is 0, for serde to skip it.
84#[expect(
85    clippy::trivially_copy_pass_by_ref,
86    reason = "serde's `skip_serializing_if` passes a reference"
87)]
88fn is_zero(count: &u32) -> bool {
89    *count == 0
90}
91
92/// What one cause held a profile's jobs for.
93#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
94pub struct CauseWait {
95    /// The cause.
96    pub cause: super::WaitCause,
97    /// How long, in ms, summed over the jobs.
98    pub waited_ms: u64,
99}
100
101/// What the host's memory came to while CI ran over a window, from the readings the budget's events stored (ADR 0108):
102/// how long `MemAvailable` stood under each line, the least it and `SwapFree` fell to, and how much of that came while
103/// the host's limit took a credit. Times are seconds, sizes bytes.
104#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
105pub struct HostHazard {
106    /// The start of the window: a UTC day's, or the digest's 24 hours'.
107    pub since: u64,
108    /// Its end.
109    pub until: u64,
110    /// How long a runner ran with a reading of the host.
111    pub running: u64,
112    /// Of that, how long a host credit stood.
113    pub credited: u64,
114    /// The line under which sockets start to throttle: the host's margin and a quarter of its memory (ADR 0099).
115    pub low_line: u64,
116    /// The line under which systemd-oomd may act once swap is full too: a tenth of the host's memory.
117    pub critical_line: u64,
118    /// How long `MemAvailable` stood under `low_line`.
119    pub low: HostSpan,
120    /// How long it stood under `critical_line`.
121    pub critical: HostSpan,
122    /// The least `MemAvailable` read.
123    pub lowest: u64,
124    /// The least read while a host credit stood; `None` when none did.
125    #[serde(default, skip_serializing_if = "Option::is_none")]
126    pub lowest_credited: Option<u64>,
127    /// The least `SwapFree` read; `None` on a host without swap.
128    #[serde(default, skip_serializing_if = "Option::is_none")]
129    pub lowest_swap_free: Option<u64>,
130}
131
132/// How long the host stood under a line while CI ran, in seconds.
133#[derive(Debug, Clone, Copy, Default, PartialEq, Eq, Serialize, Deserialize)]
134pub struct HostSpan {
135    /// In all.
136    pub seconds: u64,
137    /// Of that, while a host credit stood.
138    pub credited: u64,
139}
140
141/// How much an insight matters.
142#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Serialize, Deserialize)]
143#[serde(rename_all = "kebab-case")]
144pub enum Severity {
145    /// Worth acting on: notifications send these.
146    Warning,
147    /// Worth knowing.
148    Notice,
149}
150
151/// One thing the history says, about one job or one repository.
152#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
153pub struct Insight {
154    /// The same while it holds, such as `slower:halyard/api/CI/test`.
155    pub key: String,
156    /// How much it matters.
157    pub severity: Severity,
158    /// `owner/name`.
159    pub repository: String,
160    /// The workflow's name; `None` for a repository's waits, or when unknown.
161    pub workflow: Option<String>,
162    /// The job's name; `None` for a repository's waits.
163    pub job: Option<String>,
164    /// What it is, in one line.
165    pub sentence: String,
166    /// What it is, with its figures.
167    #[serde(flatten)]
168    pub kind: InsightKind,
169    /// The runs that show it, newest first, at most 5.
170    pub runs: Vec<RunLink>,
171}
172
173/// What an insight is, tagged by `kind`.
174#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
175#[serde(rename_all = "kebab-case", tag = "kind")]
176pub enum InsightKind {
177    /// A job's successful runs on its busiest branch take longer than they did, and stayed so.
178    Slower {
179        /// The branch.
180        branch: Option<String>,
181        /// The median run time before, in seconds.
182        before: u64,
183        /// The median run time of the last runs, in seconds.
184        now: u64,
185        /// The first slow run after which no two runs in a row were fast.
186        since: Option<RunLink>,
187    },
188    /// A job failed and then passed on a re-run of the same run.
189    Flaky {
190        /// Runs in which it did.
191        flaky: u32,
192        /// The job's runs in the window.
193        total: u32,
194    },
195    /// A job's memory peaks near its profile's cap.
196    NearCap {
197        /// The profile.
198        profile: String,
199        /// The 95th percentile of its last runs' memory peaks.
200        peak: u64,
201        /// The profile's `memory`.
202        cap: u64,
203        /// What auto sizing does about it.
204        auto: AutoNote,
205    },
206    /// A job's runs that write to the build cache find almost nothing in it: its cache keys differ from run to run.
207    ColdCache {
208        /// The branch those runs are on.
209        branch: Option<String>,
210        /// Lookups that found an entry, over those runs.
211        hits: u64,
212        /// Lookups over those runs.
213        lookups: u64,
214    },
215    /// The kernel throttled a repository's jobs' downloads for memory pressure (`JobMetrics::sock_throttled`):
216    /// the host, or a job at its cap, was short of memory.
217    Throttled {
218        /// Runs it throttled.
219        throttled: u32,
220        /// The repository's runs in the window.
221        total: u32,
222    },
223    /// A repository's jobs wait longer this week than the week before.
224    Waits {
225        /// The 95th percentile of waits the week before, in seconds.
226        before: u64,
227        /// The 95th percentile of waits this week, in seconds.
228        now: u64,
229    },
230    /// A job's CPU cap held back its runs: their quota throttled them in 15% of their run or more (ADR 0079).
231    CpuHeld {
232        /// The profile its runs ran in.
233        profile: String,
234        /// The profile's `cpus`, the cap those runs had.
235        cpus: u32,
236        /// Runs the cap held back.
237        held: u32,
238        /// The job's last runs at that cap that measured their throttling.
239        total: u32,
240        /// Under `measured` admission with auto sizing, the `cpus` the profile was raised from when that raise did
241        /// not make this job faster or less throttled: its runs ask for no further raise. What auto sizing does
242        /// next for the profile, on the other jobs' runs or on the averages, is `runnerctl sizing`'s to say.
243        raised_from: Option<u32>,
244        /// The `cpus` to set by hand; `None` while auto sizing raises them.
245        set: Option<u32>,
246    },
247    /// A queued job has waited 30 minutes or more for the host's memory, and a job waits for it now: programs
248    /// outside CI hold it, which closing is the owner's to do (ADR 0100).
249    HostHeld {
250        /// Jobs waiting for the host's memory now.
251        jobs: u32,
252        /// The longest of their waits for it, in seconds.
253        waited: u64,
254    },
255    /// On a UTC day, the host's `MemAvailable` fell under a tenth of its memory while a host credit let runners start
256    /// past their reservations (ADR 0108): the buffer the credit spends ran out. Told once a day at most.
257    HostLow {
258        /// The start of the day.
259        day: u64,
260        /// How long it stood there while a host credit stood, in seconds.
261        seconds: u64,
262        /// The least `MemAvailable` read while one stood, in bytes.
263        lowest: u64,
264    },
265    /// A kind from a later release, which this one reads only by its sentence. The daemon never sends it.
266    #[serde(other)]
267    Unknown,
268}
269
270/// What auto sizing does about a job near its cap.
271#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
272#[serde(rename_all = "kebab-case")]
273pub enum AutoNote {
274    /// It raises the profile's `memory` when a run meets the cap.
275    Raises,
276    /// The profile is at its `max_memory`: it cannot.
277    AtMax,
278    /// It is off.
279    Off,
280}
281
282/// One run of a job, to open in the history or on GitHub.
283#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
284pub struct RunLink {
285    /// GitHub's job ID, the history's too.
286    pub job: u64,
287    /// The workflow run.
288    pub run_id: u64,
289    /// The run's attempt.
290    pub run_attempt: u32,
291    /// The branch.
292    pub branch: Option<String>,
293    /// The job's page on GitHub.
294    pub url: Option<String>,
295    /// When it finished.
296    pub at: u64,
297}
298
299/// A repository's waits this week against the week before.
300#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
301pub struct RepositoryWaits {
302    /// `owner/name`.
303    pub repository: String,
304    /// The jobs that started in the last 7 days.
305    pub now: WeekWaits,
306    /// The jobs that started in the 7 days before.
307    pub before: WeekWaits,
308}
309
310/// The waits of the jobs that started in a week, from queued to started, leaving out the time before the history's
311/// start and the time no runnerd ran (ADR 0081).
312#[derive(Debug, Clone, Default, PartialEq, Eq, Serialize, Deserialize)]
313pub struct WeekWaits {
314    /// Every job.
315    pub all: WaitSpread,
316    /// The jobs that found a runner already started: a warm runner (ADR 0050).
317    pub warm: WaitSpread,
318    /// The jobs whose runner started after they were queued.
319    pub cold: WaitSpread,
320}
321
322/// How long some jobs waited.
323#[derive(Debug, Clone, Default, PartialEq, Eq, Serialize, Deserialize)]
324pub struct WaitSpread {
325    /// How many.
326    pub jobs: u32,
327    /// The median, in seconds; `None` without jobs.
328    pub p50: Option<u64>,
329    /// The 95th percentile by nearest rank, in seconds; `None` without jobs.
330    pub p95: Option<u64>,
331}
332
333/// `GET /v1/digest`: the last 24 hours.
334#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
335pub struct Digest {
336    /// 24 hours before `until`.
337    pub since: u64,
338    /// When it was read.
339    pub until: u64,
340    /// Jobs that finished: completed, or lost with their runner.
341    pub finished: u64,
342    /// Of those, the ones that failed, timed out or were lost.
343    pub failed: u64,
344    /// The insights that hold now and did not 24 hours ago.
345    pub new: Vec<Insight>,
346    /// The three longest waits of the jobs that started, longest first.
347    pub waits: Vec<LongWait>,
348    /// What the month's job minutes would have cost on GitHub's runners (ADR 0072); `None` when the history could
349    /// not say.
350    #[serde(default, skip_serializing_if = "Option::is_none")]
351    pub savings: Option<super::SavedWindow>,
352    /// What the host's memory came to while CI ran (ADR 0108); `None` when no runner ran with a reading of it.
353    #[serde(default, skip_serializing_if = "Option::is_none")]
354    pub host: Option<HostHazard>,
355}
356
357/// A job that waited.
358#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
359pub struct LongWait {
360    /// GitHub's job ID.
361    pub job: u64,
362    /// `owner/name`.
363    pub repository: String,
364    /// The workflow's name.
365    pub workflow: Option<String>,
366    /// The job's name.
367    pub name: String,
368    /// Seconds from queued to started, leaving out the time before the history's start and the time no runnerd
369    /// ran (ADR 0081).
370    pub waited: u64,
371    /// The job's page on GitHub.
372    pub url: Option<String>,
373}
374
375/// What a notification target is sent, and the events `[notify]` targets choose from.
376#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Serialize, Deserialize)]
377#[serde(rename_all = "kebab-case")]
378pub enum NotifyEvent {
379    /// A job failed or timed out.
380    Failed,
381    /// A job's runner vanished before GitHub said the job completed.
382    Lost,
383    /// A runner could not be registered or started, and its repository is held back.
384    Runner,
385    /// A new warning insight.
386    Insights,
387    /// The daily digest.
388    Digest,
389    /// `runnerctl notify test`; not one a target chooses.
390    Test,
391}
392
393/// How urgent a notification is, as ntfy ranks it.
394#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
395#[serde(rename_all = "kebab-case")]
396pub enum Priority {
397    /// ntfy's 2: no sound.
398    Low,
399    /// ntfy's 3.
400    Default,
401    /// ntfy's 4: a longer vibration.
402    High,
403}
404
405/// What a webhook target is posted, as JSON; an ntfy target gets its title, message, priority and link.
406#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
407pub struct Notification {
408    /// What it is about.
409    pub event: NotifyEvent,
410    /// One short line.
411    pub title: String,
412    /// A few lines.
413    pub message: String,
414    /// What to open: a job's page on GitHub, or nothing.
415    pub url: Option<String>,
416    /// How urgent.
417    pub priority: Priority,
418    /// The jobs it is about: failed or lost.
419    #[serde(default, skip_serializing_if = "Vec::is_empty")]
420    pub jobs: Vec<JobSummary>,
421    /// The runners that could not start.
422    #[serde(default, skip_serializing_if = "Vec::is_empty")]
423    pub runners: Vec<RunnerFailure>,
424    /// The insights it is about.
425    #[serde(default, skip_serializing_if = "Vec::is_empty")]
426    pub insights: Vec<Insight>,
427    /// The digest.
428    #[serde(default, skip_serializing_if = "Option::is_none")]
429    pub digest: Option<Digest>,
430}
431
432/// `POST /v1/notify/test` query parameters.
433#[derive(Debug, Clone, Default, PartialEq, Eq, Serialize, Deserialize)]
434pub struct NotifyTestQuery {
435    /// Only the target with this name; default every one.
436    pub target: Option<String>,
437}
438
439/// `POST /v1/notify/test`: what each target did with a test message.
440#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
441pub struct NotifyTest {
442    /// In the config's order.
443    pub targets: Vec<Delivery>,
444}
445
446/// What one target did with a message.
447#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
448pub struct Delivery {
449    /// The target's name.
450    pub target: String,
451    /// Why it was not delivered: an HTTP status or a connection error, never the URL. `None` when it was.
452    pub error: Option<String>,
453}