runnerd_core/api/insights.rs
1//! Insights as clients read them: what got slower, what is flaky, what nears its memory cap, what memory pressure
2//! throttled, what its CPU cap held back, jobs held for the host's memory and how long jobs wait, computed from the job history (ADR 0051); the daily digest; and what the
3//! daemon's notifications carry. Times are seconds since the Unix epoch, durations seconds, sizes bytes.
4
5use serde::{Deserialize, Serialize};
6
7use super::RunnerFailure;
8use super::jobs::JobSummary;
9
10mod shown;
11
12/// The window `GET /v1/insights` reads by default, in days.
13pub const INSIGHT_DAYS: u32 = 14;
14
15/// The longest window `GET /v1/insights` reads, in days.
16pub const INSIGHT_DAYS_MAX: u32 = 90;
17
18/// `GET /v1/insights` query parameters.
19#[derive(Debug, Clone, Default, PartialEq, Eq, Serialize, Deserialize)]
20pub struct InsightsQuery {
21 /// Days of history to read; default [`INSIGHT_DAYS`], at most [`INSIGHT_DAYS_MAX`].
22 pub days: Option<u32>,
23}
24
25/// `GET /v1/insights`: what the history says about the window.
26#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
27pub struct Insights {
28 /// The start of the window.
29 pub since: u64,
30 /// When it was read: the end of the window.
31 pub until: u64,
32 /// Warnings first, then by repository, workflow and job.
33 pub insights: Vec<Insight>,
34 /// Each repository's waits this week and the week before, by repository. Always the last 14 days, whatever the
35 /// window.
36 pub waits: Vec<RepositoryWaits>,
37 /// What the host's memory came to on each UTC day of the window that CI ran, newest first (ADR 0108); absent from
38 /// a daemon before 0.0.30.
39 #[serde(default, skip_serializing_if = "Vec::is_empty")]
40 pub host: Vec<HostHazard>,
41}
42
43/// What the host's memory came to while CI ran over a window, from the readings the budget's events stored (ADR 0108):
44/// how long `MemAvailable` stood under each line, the least it and `SwapFree` fell to, and how much of that came while
45/// the host's limit took a credit. Times are seconds, sizes bytes.
46#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
47pub struct HostHazard {
48 /// The start of the window: a UTC day's, or the digest's 24 hours'.
49 pub since: u64,
50 /// Its end.
51 pub until: u64,
52 /// How long a runner ran with a reading of the host.
53 pub running: u64,
54 /// Of that, how long a host credit stood.
55 pub credited: u64,
56 /// The line under which sockets start to throttle: the host's margin and a quarter of its memory (ADR 0099).
57 pub low_line: u64,
58 /// The line under which systemd-oomd may act once swap is full too: a tenth of the host's memory.
59 pub critical_line: u64,
60 /// How long `MemAvailable` stood under `low_line`.
61 pub low: HostSpan,
62 /// How long it stood under `critical_line`.
63 pub critical: HostSpan,
64 /// The least `MemAvailable` read.
65 pub lowest: u64,
66 /// The least read while a host credit stood; `None` when none did.
67 #[serde(default, skip_serializing_if = "Option::is_none")]
68 pub lowest_credited: Option<u64>,
69 /// The least `SwapFree` read; `None` on a host without swap.
70 #[serde(default, skip_serializing_if = "Option::is_none")]
71 pub lowest_swap_free: Option<u64>,
72}
73
74/// How long the host stood under a line while CI ran, in seconds.
75#[derive(Debug, Clone, Copy, Default, PartialEq, Eq, Serialize, Deserialize)]
76pub struct HostSpan {
77 /// In all.
78 pub seconds: u64,
79 /// Of that, while a host credit stood.
80 pub credited: u64,
81}
82
83/// How much an insight matters.
84#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Serialize, Deserialize)]
85#[serde(rename_all = "kebab-case")]
86pub enum Severity {
87 /// Worth acting on: notifications send these.
88 Warning,
89 /// Worth knowing.
90 Notice,
91}
92
93/// One thing the history says, about one job or one repository.
94#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
95pub struct Insight {
96 /// The same while it holds, such as `slower:halyard/api/CI/test`.
97 pub key: String,
98 /// How much it matters.
99 pub severity: Severity,
100 /// `owner/name`.
101 pub repository: String,
102 /// The workflow's name; `None` for a repository's waits, or when unknown.
103 pub workflow: Option<String>,
104 /// The job's name; `None` for a repository's waits.
105 pub job: Option<String>,
106 /// What it is, in one line.
107 pub sentence: String,
108 /// What it is, with its figures.
109 #[serde(flatten)]
110 pub kind: InsightKind,
111 /// The runs that show it, newest first, at most 5.
112 pub runs: Vec<RunLink>,
113}
114
115/// What an insight is, tagged by `kind`.
116#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
117#[serde(rename_all = "kebab-case", tag = "kind")]
118pub enum InsightKind {
119 /// A job's successful runs on its busiest branch take longer than they did, and stayed so.
120 Slower {
121 /// The branch.
122 branch: Option<String>,
123 /// The median run time before, in seconds.
124 before: u64,
125 /// The median run time of the last runs, in seconds.
126 now: u64,
127 /// The first slow run after which no two runs in a row were fast.
128 since: Option<RunLink>,
129 },
130 /// A job failed and then passed on a re-run of the same run.
131 Flaky {
132 /// Runs in which it did.
133 flaky: u32,
134 /// The job's runs in the window.
135 total: u32,
136 },
137 /// A job's memory peaks near its profile's cap.
138 NearCap {
139 /// The profile.
140 profile: String,
141 /// The 95th percentile of its last runs' memory peaks.
142 peak: u64,
143 /// The profile's `memory`.
144 cap: u64,
145 /// What auto sizing does about it.
146 auto: AutoNote,
147 },
148 /// A job's runs that write to the build cache find almost nothing in it: its cache keys differ from run to run.
149 ColdCache {
150 /// The branch those runs are on.
151 branch: Option<String>,
152 /// Lookups that found an entry, over those runs.
153 hits: u64,
154 /// Lookups over those runs.
155 lookups: u64,
156 },
157 /// The kernel throttled a repository's jobs' downloads for memory pressure (`JobMetrics::sock_throttled`):
158 /// the host, or a job at its cap, was short of memory.
159 Throttled {
160 /// Runs it throttled.
161 throttled: u32,
162 /// The repository's runs in the window.
163 total: u32,
164 },
165 /// A repository's jobs wait longer this week than the week before.
166 Waits {
167 /// The 95th percentile of waits the week before, in seconds.
168 before: u64,
169 /// The 95th percentile of waits this week, in seconds.
170 now: u64,
171 },
172 /// A job's CPU cap held back its runs: their quota throttled them in 15% of their run or more (ADR 0079).
173 CpuHeld {
174 /// The profile its runs ran in.
175 profile: String,
176 /// The profile's `cpus`, the cap those runs had.
177 cpus: u32,
178 /// Runs the cap held back.
179 held: u32,
180 /// The job's last runs at that cap that measured their throttling.
181 total: u32,
182 /// Under `measured` admission with auto sizing, the `cpus` the profile was raised from when that raise did
183 /// not make this job faster or less throttled: its runs ask for no further raise. What auto sizing does
184 /// next for the profile, on the other jobs' runs or on the averages, is `runnerctl sizing`'s to say.
185 raised_from: Option<u32>,
186 /// The `cpus` to set by hand; `None` while auto sizing raises them.
187 set: Option<u32>,
188 },
189 /// A queued job has waited 30 minutes or more for the host's memory, and a job waits for it now: programs
190 /// outside CI hold it, which closing is the owner's to do (ADR 0100).
191 HostHeld {
192 /// Jobs waiting for the host's memory now.
193 jobs: u32,
194 /// The longest of their waits for it, in seconds.
195 waited: u64,
196 },
197 /// On a UTC day, the host's `MemAvailable` fell under a tenth of its memory while a host credit let runners start
198 /// past their reservations (ADR 0108): the buffer the credit spends ran out. Told once a day at most.
199 HostLow {
200 /// The start of the day.
201 day: u64,
202 /// How long it stood there while a host credit stood, in seconds.
203 seconds: u64,
204 /// The least `MemAvailable` read while one stood, in bytes.
205 lowest: u64,
206 },
207 /// A kind from a later release, which this one reads only by its sentence. The daemon never sends it.
208 #[serde(other)]
209 Unknown,
210}
211
212/// What auto sizing does about a job near its cap.
213#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
214#[serde(rename_all = "kebab-case")]
215pub enum AutoNote {
216 /// It raises the profile's `memory` when a run meets the cap.
217 Raises,
218 /// The profile is at its `max_memory`: it cannot.
219 AtMax,
220 /// It is off.
221 Off,
222}
223
224/// One run of a job, to open in the history or on GitHub.
225#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
226pub struct RunLink {
227 /// GitHub's job ID, the history's too.
228 pub job: u64,
229 /// The workflow run.
230 pub run_id: u64,
231 /// The run's attempt.
232 pub run_attempt: u32,
233 /// The branch.
234 pub branch: Option<String>,
235 /// The job's page on GitHub.
236 pub url: Option<String>,
237 /// When it finished.
238 pub at: u64,
239}
240
241/// A repository's waits this week against the week before.
242#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
243pub struct RepositoryWaits {
244 /// `owner/name`.
245 pub repository: String,
246 /// The jobs that started in the last 7 days.
247 pub now: WeekWaits,
248 /// The jobs that started in the 7 days before.
249 pub before: WeekWaits,
250}
251
252/// The waits of the jobs that started in a week, from queued to started, leaving out the time before the history's
253/// start and the time no runnerd ran (ADR 0081).
254#[derive(Debug, Clone, Default, PartialEq, Eq, Serialize, Deserialize)]
255pub struct WeekWaits {
256 /// Every job.
257 pub all: WaitSpread,
258 /// The jobs that found a runner already started: a warm runner (ADR 0050).
259 pub warm: WaitSpread,
260 /// The jobs whose runner started after they were queued.
261 pub cold: WaitSpread,
262}
263
264/// How long some jobs waited.
265#[derive(Debug, Clone, Default, PartialEq, Eq, Serialize, Deserialize)]
266pub struct WaitSpread {
267 /// How many.
268 pub jobs: u32,
269 /// The median, in seconds; `None` without jobs.
270 pub p50: Option<u64>,
271 /// The 95th percentile by nearest rank, in seconds; `None` without jobs.
272 pub p95: Option<u64>,
273}
274
275/// `GET /v1/digest`: the last 24 hours.
276#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
277pub struct Digest {
278 /// 24 hours before `until`.
279 pub since: u64,
280 /// When it was read.
281 pub until: u64,
282 /// Jobs that finished: completed, or lost with their runner.
283 pub finished: u64,
284 /// Of those, the ones that failed, timed out or were lost.
285 pub failed: u64,
286 /// The insights that hold now and did not 24 hours ago.
287 pub new: Vec<Insight>,
288 /// The three longest waits of the jobs that started, longest first.
289 pub waits: Vec<LongWait>,
290 /// What the month's job minutes would have cost on GitHub's runners (ADR 0072); `None` when the history could
291 /// not say.
292 #[serde(default, skip_serializing_if = "Option::is_none")]
293 pub savings: Option<super::SavedWindow>,
294 /// What the host's memory came to while CI ran (ADR 0108); `None` when no runner ran with a reading of it.
295 #[serde(default, skip_serializing_if = "Option::is_none")]
296 pub host: Option<HostHazard>,
297}
298
299/// A job that waited.
300#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
301pub struct LongWait {
302 /// GitHub's job ID.
303 pub job: u64,
304 /// `owner/name`.
305 pub repository: String,
306 /// The workflow's name.
307 pub workflow: Option<String>,
308 /// The job's name.
309 pub name: String,
310 /// Seconds from queued to started, leaving out the time before the history's start and the time no runnerd
311 /// ran (ADR 0081).
312 pub waited: u64,
313 /// The job's page on GitHub.
314 pub url: Option<String>,
315}
316
317/// What a notification target is sent, and the events `[notify]` targets choose from.
318#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Serialize, Deserialize)]
319#[serde(rename_all = "kebab-case")]
320pub enum NotifyEvent {
321 /// A job failed or timed out.
322 Failed,
323 /// A job's runner vanished before GitHub said the job completed.
324 Lost,
325 /// A runner could not be registered or started, and its repository is held back.
326 Runner,
327 /// A new warning insight.
328 Insights,
329 /// The daily digest.
330 Digest,
331 /// `runnerctl notify test`; not one a target chooses.
332 Test,
333}
334
335/// How urgent a notification is, as ntfy ranks it.
336#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
337#[serde(rename_all = "kebab-case")]
338pub enum Priority {
339 /// ntfy's 2: no sound.
340 Low,
341 /// ntfy's 3.
342 Default,
343 /// ntfy's 4: a longer vibration.
344 High,
345}
346
347/// What a webhook target is posted, as JSON; an ntfy target gets its title, message, priority and link.
348#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
349pub struct Notification {
350 /// What it is about.
351 pub event: NotifyEvent,
352 /// One short line.
353 pub title: String,
354 /// A few lines.
355 pub message: String,
356 /// What to open: a job's page on GitHub, or nothing.
357 pub url: Option<String>,
358 /// How urgent.
359 pub priority: Priority,
360 /// The jobs it is about: failed or lost.
361 #[serde(default, skip_serializing_if = "Vec::is_empty")]
362 pub jobs: Vec<JobSummary>,
363 /// The runners that could not start.
364 #[serde(default, skip_serializing_if = "Vec::is_empty")]
365 pub runners: Vec<RunnerFailure>,
366 /// The insights it is about.
367 #[serde(default, skip_serializing_if = "Vec::is_empty")]
368 pub insights: Vec<Insight>,
369 /// The digest.
370 #[serde(default, skip_serializing_if = "Option::is_none")]
371 pub digest: Option<Digest>,
372}
373
374/// `POST /v1/notify/test` query parameters.
375#[derive(Debug, Clone, Default, PartialEq, Eq, Serialize, Deserialize)]
376pub struct NotifyTestQuery {
377 /// Only the target with this name; default every one.
378 pub target: Option<String>,
379}
380
381/// `POST /v1/notify/test`: what each target did with a test message.
382#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
383pub struct NotifyTest {
384 /// In the config's order.
385 pub targets: Vec<Delivery>,
386}
387
388/// What one target did with a message.
389#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
390pub struct Delivery {
391 /// The target's name.
392 pub target: String,
393 /// Why it was not delivered: an HTTP status or a connection error, never the URL. `None` when it was.
394 pub error: Option<String>,
395}