runnerd_core/api/insights.rs
1//! Insights as clients read them: what got slower, what is flaky, what nears its memory cap, what memory pressure
2//! throttled, what its CPU cap held back, jobs held for the host's memory and how long jobs wait, computed from the job history (ADR 0051); the daily digest; and what the
3//! daemon's notifications carry. Times are seconds since the Unix epoch, durations seconds, sizes bytes.
4
5use serde::{Deserialize, Serialize};
6
7use super::RunnerFailure;
8use super::jobs::JobSummary;
9
10mod shown;
11
12pub use shown::InsightFigure;
13pub use shown::InsightsHeading;
14pub use shown::WaitFigure;
15
16/// The window `GET /v1/insights` reads by default, in days.
17pub const INSIGHT_DAYS: u32 = 14;
18
19/// The longest window `GET /v1/insights` reads, in days.
20pub const INSIGHT_DAYS_MAX: u32 = 90;
21
22/// `GET /v1/insights` query parameters.
23#[derive(Debug, Clone, Default, PartialEq, Eq, Serialize, Deserialize)]
24pub struct InsightsQuery {
25 /// Days of history to read; default [`INSIGHT_DAYS`], at most [`INSIGHT_DAYS_MAX`].
26 pub days: Option<u32>,
27}
28
29/// `GET /v1/insights`: what the history says about the window.
30#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
31pub struct Insights {
32 /// The start of the window.
33 pub since: u64,
34 /// When it was read: the end of the window.
35 pub until: u64,
36 /// Warnings first, then by repository, workflow and job.
37 pub insights: Vec<Insight>,
38 /// Each repository's waits this week and the week before, by repository. Always the last 14 days, whatever the
39 /// window.
40 pub waits: Vec<RepositoryWaits>,
41 /// What the host's memory came to on each UTC day of the window that CI ran, newest first (ADR 0108); absent from
42 /// a daemon before 0.0.30.
43 #[serde(default, skip_serializing_if = "Vec::is_empty")]
44 pub host: Vec<HostHazard>,
45 /// What the jobs that ended in the window and the drive under Docker's root wrote (ADR 0143); `None` when no job
46 /// was measured and the daemon read no drive, and from a daemon before 0.1.3.
47 #[serde(default, skip_serializing_if = "Option::is_none")]
48 pub disk: Option<super::DiskWrites>,
49 /// Why each profile's jobs that ended in the window queued, split by what held them (ADR 0154), the profile whose
50 /// jobs waited longest first; empty from a daemon before 0.1.5.
51 #[serde(default, skip_serializing_if = "Vec::is_empty")]
52 pub queue: Vec<ProfileQueue>,
53}
54
55/// Why one profile's jobs queued over a window, from what each plan that held them said (ADR 0154). Unlike the rest
56/// of insights, its times are milliseconds, so shares of short waits come out right. A wait runs from GitHub queueing
57/// a job to its start, less what passed while no runnerd ran, summed over the jobs counted; the causes and the two
58/// parts no cause covers add up to `waited_ms`.
59#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
60pub struct ProfileQueue {
61 /// The profile.
62 pub profile: String,
63 /// The jobs whose waits it splits.
64 pub jobs: u32,
65 /// Their waits together.
66 pub waited_ms: u64,
67 /// What each cause held them for, longest first; none of 0.
68 pub causes: Vec<CauseWait>,
69 /// Of `waited_ms`, what passed before runnerd first listed the jobs queued: GitHub's listing lag and the time to
70 /// runnerd's next listing, less any downtime in it.
71 pub before_listed_ms: u64,
72 /// Of `waited_ms`, what no plan held them for once listed: after a restart until the first plan, and between a
73 /// planned start and its runner.
74 pub uncovered_ms: u64,
75 /// The jobs left out as their history rows are from before runnerd counted why jobs wait.
76 pub unrecorded: u32,
77 /// The jobs whose causes were counted past their start, as when a runner took one before runnerd read it, and
78 /// were scaled down into the wait left after the time before the listing: their causes are estimates.
79 #[serde(default, skip_serializing_if = "is_zero")]
80 pub estimated: u32,
81}
82
83/// Whether a count is 0, for serde to skip it.
84#[expect(
85 clippy::trivially_copy_pass_by_ref,
86 reason = "serde's `skip_serializing_if` passes a reference"
87)]
88fn is_zero(count: &u32) -> bool {
89 *count == 0
90}
91
92/// What one cause held a profile's jobs for.
93#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
94pub struct CauseWait {
95 /// The cause.
96 pub cause: super::WaitCause,
97 /// How long, in ms, summed over the jobs.
98 pub waited_ms: u64,
99}
100
101/// What the host's memory came to while CI ran over a window, from the readings the budget's events stored (ADR 0108):
102/// how long `MemAvailable` stood under each line, the least it and `SwapFree` fell to, and how much of that came while
103/// the host's limit took a credit. Times are seconds, sizes bytes.
104#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
105pub struct HostHazard {
106 /// The start of the window: a UTC day's, or the digest's 24 hours'.
107 pub since: u64,
108 /// Its end.
109 pub until: u64,
110 /// How long a runner ran with a reading of the host.
111 pub running: u64,
112 /// Of that, how long a host credit stood.
113 pub credited: u64,
114 /// The line under which sockets start to throttle: the host's margin and a quarter of its memory (ADR 0099).
115 pub low_line: u64,
116 /// The line under which systemd-oomd may act once swap is full too: a tenth of the host's memory.
117 pub critical_line: u64,
118 /// How long `MemAvailable` stood under `low_line`.
119 pub low: HostSpan,
120 /// How long it stood under `critical_line`.
121 pub critical: HostSpan,
122 /// The least `MemAvailable` read.
123 pub lowest: u64,
124 /// The least read while a host credit stood; `None` when none did.
125 #[serde(default, skip_serializing_if = "Option::is_none")]
126 pub lowest_credited: Option<u64>,
127 /// The least `SwapFree` read; `None` on a host without swap.
128 #[serde(default, skip_serializing_if = "Option::is_none")]
129 pub lowest_swap_free: Option<u64>,
130}
131
132/// How long the host stood under a line while CI ran, in seconds.
133#[derive(Debug, Clone, Copy, Default, PartialEq, Eq, Serialize, Deserialize)]
134pub struct HostSpan {
135 /// In all.
136 pub seconds: u64,
137 /// Of that, while a host credit stood.
138 pub credited: u64,
139}
140
141/// How much an insight matters.
142#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Serialize, Deserialize)]
143#[serde(rename_all = "kebab-case")]
144pub enum Severity {
145 /// Worth acting on: notifications send these.
146 Warning,
147 /// Worth knowing.
148 Notice,
149}
150
151/// One thing the history says, about one job or one repository.
152#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
153pub struct Insight {
154 /// The same while it holds, such as `slower:halyard/api/CI/test`.
155 pub key: String,
156 /// How much it matters.
157 pub severity: Severity,
158 /// `owner/name`.
159 pub repository: String,
160 /// The workflow's name; `None` for a repository's waits, or when unknown.
161 pub workflow: Option<String>,
162 /// The job's name; `None` for a repository's waits.
163 pub job: Option<String>,
164 /// What it is, in one line.
165 pub sentence: String,
166 /// What it is, with its figures.
167 #[serde(flatten)]
168 pub kind: InsightKind,
169 /// The runs that show it, newest first, at most 5.
170 pub runs: Vec<RunLink>,
171}
172
173/// What an insight is, tagged by `kind`.
174#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
175#[serde(rename_all = "kebab-case", tag = "kind")]
176pub enum InsightKind {
177 /// A job's successful runs on its busiest branch take longer than they did, and stayed so.
178 Slower {
179 /// The branch.
180 branch: Option<String>,
181 /// The median run time before, in seconds.
182 before: u64,
183 /// The median run time of the last runs, in seconds.
184 now: u64,
185 /// The first slow run after which no two runs in a row were fast.
186 since: Option<RunLink>,
187 },
188 /// A job failed and then passed on a re-run of the same run.
189 Flaky {
190 /// Runs in which it did.
191 flaky: u32,
192 /// The job's runs in the window.
193 total: u32,
194 },
195 /// A job's memory peaks near its profile's cap.
196 NearCap {
197 /// The profile.
198 profile: String,
199 /// The 95th percentile of its last runs' memory peaks.
200 peak: u64,
201 /// The profile's `memory`.
202 cap: u64,
203 /// What auto sizing does about it.
204 auto: AutoNote,
205 },
206 /// A job's runs that write to the build cache find almost nothing in it: its cache keys differ from run to run.
207 ColdCache {
208 /// The branch those runs are on.
209 branch: Option<String>,
210 /// Lookups that found an entry, over those runs.
211 hits: u64,
212 /// Lookups over those runs.
213 lookups: u64,
214 },
215 /// The kernel throttled a repository's jobs' downloads for memory pressure (`JobMetrics::sock_throttled`):
216 /// the host, or a job at its cap, was short of memory.
217 Throttled {
218 /// Runs it throttled.
219 throttled: u32,
220 /// The repository's runs in the window.
221 total: u32,
222 },
223 /// A repository's jobs wait longer this week than the week before.
224 Waits {
225 /// The 95th percentile of waits the week before, in seconds.
226 before: u64,
227 /// The 95th percentile of waits this week, in seconds.
228 now: u64,
229 },
230 /// A job's CPU cap held back its runs: their quota throttled them in 15% of their run or more (ADR 0079).
231 CpuHeld {
232 /// The profile its runs ran in.
233 profile: String,
234 /// The profile's `cpus`, the cap those runs had.
235 cpus: u32,
236 /// Runs the cap held back.
237 held: u32,
238 /// The job's last runs at that cap that measured their throttling.
239 total: u32,
240 /// Under `measured` admission with auto sizing, the `cpus` the profile was raised from when that raise did
241 /// not make this job faster or less throttled: its runs ask for no further raise. What auto sizing does
242 /// next for the profile, on the other jobs' runs or on the averages, is `runnerctl sizing`'s to say.
243 raised_from: Option<u32>,
244 /// The `cpus` to set by hand; `None` while auto sizing raises them.
245 set: Option<u32>,
246 },
247 /// A queued job has waited 30 minutes or more for the host's memory, and a job waits for it now: programs
248 /// outside CI hold it, which closing is the owner's to do (ADR 0100).
249 HostHeld {
250 /// Jobs waiting for the host's memory now.
251 jobs: u32,
252 /// The longest of their waits for it, in seconds.
253 waited: u64,
254 },
255 /// On a UTC day, the host's `MemAvailable` fell under a tenth of its memory while a host credit let runners start
256 /// past their reservations (ADR 0108): the buffer the credit spends ran out. Told once a day at most.
257 HostLow {
258 /// The start of the day.
259 day: u64,
260 /// How long it stood there while a host credit stood, in seconds.
261 seconds: u64,
262 /// The least `MemAvailable` read while one stood, in bytes.
263 lowest: u64,
264 },
265 /// A kind from a later release, which this one reads only by its sentence. The daemon never sends it.
266 #[serde(other)]
267 Unknown,
268}
269
270/// What auto sizing does about a job near its cap.
271#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
272#[serde(rename_all = "kebab-case")]
273pub enum AutoNote {
274 /// It raises the profile's `memory` when a run meets the cap.
275 Raises,
276 /// The profile is at its `max_memory`: it cannot.
277 AtMax,
278 /// It is off.
279 Off,
280}
281
282/// One run of a job, to open in the history or on GitHub.
283#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
284pub struct RunLink {
285 /// GitHub's job ID, the history's too.
286 pub job: u64,
287 /// The workflow run.
288 pub run_id: u64,
289 /// The run's attempt.
290 pub run_attempt: u32,
291 /// The branch.
292 pub branch: Option<String>,
293 /// The job's page on GitHub.
294 pub url: Option<String>,
295 /// When it finished.
296 pub at: u64,
297}
298
299/// A repository's waits this week against the week before.
300#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
301pub struct RepositoryWaits {
302 /// `owner/name`.
303 pub repository: String,
304 /// The jobs that started in the last 7 days.
305 pub now: WeekWaits,
306 /// The jobs that started in the 7 days before.
307 pub before: WeekWaits,
308}
309
310/// The waits of the jobs that started in a week, from queued to started, leaving out the time before the history's
311/// start and the time no runnerd ran (ADR 0081).
312#[derive(Debug, Clone, Default, PartialEq, Eq, Serialize, Deserialize)]
313pub struct WeekWaits {
314 /// Every job.
315 pub all: WaitSpread,
316 /// The jobs that found a runner already started: a warm runner (ADR 0050).
317 pub warm: WaitSpread,
318 /// The jobs whose runner started after they were queued.
319 pub cold: WaitSpread,
320}
321
322/// How long some jobs waited.
323#[derive(Debug, Clone, Default, PartialEq, Eq, Serialize, Deserialize)]
324pub struct WaitSpread {
325 /// How many.
326 pub jobs: u32,
327 /// The median, in seconds; `None` without jobs.
328 pub p50: Option<u64>,
329 /// The 95th percentile by nearest rank, in seconds; `None` without jobs.
330 pub p95: Option<u64>,
331}
332
333/// `GET /v1/digest`: the last 24 hours.
334#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
335pub struct Digest {
336 /// 24 hours before `until`.
337 pub since: u64,
338 /// When it was read.
339 pub until: u64,
340 /// Jobs that finished: completed, or lost with their runner.
341 pub finished: u64,
342 /// Of those, the ones that failed, timed out or were lost.
343 pub failed: u64,
344 /// The insights that hold now and did not 24 hours ago.
345 pub new: Vec<Insight>,
346 /// The three longest waits of the jobs that started, longest first.
347 pub waits: Vec<LongWait>,
348 /// What the month's job minutes would have cost on GitHub's runners (ADR 0072); `None` when the history could
349 /// not say.
350 #[serde(default, skip_serializing_if = "Option::is_none")]
351 pub savings: Option<super::SavedWindow>,
352 /// What the host's memory came to while CI ran (ADR 0108); `None` when no runner ran with a reading of it.
353 #[serde(default, skip_serializing_if = "Option::is_none")]
354 pub host: Option<HostHazard>,
355}
356
357/// A job that waited.
358#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
359pub struct LongWait {
360 /// GitHub's job ID.
361 pub job: u64,
362 /// `owner/name`.
363 pub repository: String,
364 /// The workflow's name.
365 pub workflow: Option<String>,
366 /// The job's name.
367 pub name: String,
368 /// Seconds from queued to started, leaving out the time before the history's start and the time no runnerd
369 /// ran (ADR 0081).
370 pub waited: u64,
371 /// The job's page on GitHub.
372 pub url: Option<String>,
373}
374
375/// What a notification target is sent, and the events `[notify]` targets choose from.
376#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Serialize, Deserialize)]
377#[serde(rename_all = "kebab-case")]
378pub enum NotifyEvent {
379 /// A job failed or timed out.
380 Failed,
381 /// A job's runner vanished before GitHub said the job completed.
382 Lost,
383 /// A runner could not be registered or started, and its repository is held back.
384 Runner,
385 /// A new warning insight.
386 Insights,
387 /// The daily digest.
388 Digest,
389 /// `runnerctl notify test`; not one a target chooses.
390 Test,
391}
392
393/// How urgent a notification is, as ntfy ranks it.
394#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
395#[serde(rename_all = "kebab-case")]
396pub enum Priority {
397 /// ntfy's 2: no sound.
398 Low,
399 /// ntfy's 3.
400 Default,
401 /// ntfy's 4: a longer vibration.
402 High,
403}
404
405/// What a webhook target is posted, as JSON; an ntfy target gets its title, message, priority and link.
406#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
407pub struct Notification {
408 /// What it is about.
409 pub event: NotifyEvent,
410 /// One short line.
411 pub title: String,
412 /// A few lines.
413 pub message: String,
414 /// What to open: a job's page on GitHub, or nothing.
415 pub url: Option<String>,
416 /// How urgent.
417 pub priority: Priority,
418 /// The jobs it is about: failed or lost.
419 #[serde(default, skip_serializing_if = "Vec::is_empty")]
420 pub jobs: Vec<JobSummary>,
421 /// The runners that could not start.
422 #[serde(default, skip_serializing_if = "Vec::is_empty")]
423 pub runners: Vec<RunnerFailure>,
424 /// The insights it is about.
425 #[serde(default, skip_serializing_if = "Vec::is_empty")]
426 pub insights: Vec<Insight>,
427 /// The digest.
428 #[serde(default, skip_serializing_if = "Option::is_none")]
429 pub digest: Option<Digest>,
430}
431
432/// `POST /v1/notify/test` query parameters.
433#[derive(Debug, Clone, Default, PartialEq, Eq, Serialize, Deserialize)]
434pub struct NotifyTestQuery {
435 /// Only the target with this name; default every one.
436 pub target: Option<String>,
437}
438
439/// `POST /v1/notify/test`: what each target did with a test message.
440#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
441pub struct NotifyTest {
442 /// In the config's order.
443 pub targets: Vec<Delivery>,
444}
445
446/// What one target did with a message.
447#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
448pub struct Delivery {
449 /// The target's name.
450 pub target: String,
451 /// Why it was not delivered: an HTTP status or a connection error, never the URL. `None` when it was.
452 pub error: Option<String>,
453}