runnerd_core/api/insights.rs
1//! Insights as clients read them: what got slower, what is flaky, what nears its memory cap, what memory pressure
2//! throttled, what its CPU cap held back, jobs held for the host's memory and how long jobs wait, computed from the job history (ADR 0051); the daily digest; and what the
3//! daemon's notifications carry. Times are seconds since the Unix epoch, durations seconds, sizes bytes.
4
5use serde::{Deserialize, Serialize};
6
7use super::RunnerFailure;
8use super::jobs::JobSummary;
9
10mod shown;
11
12/// The window `GET /v1/insights` reads by default, in days.
13pub const INSIGHT_DAYS: u32 = 14;
14
15/// The longest window `GET /v1/insights` reads, in days.
16pub const INSIGHT_DAYS_MAX: u32 = 90;
17
18/// `GET /v1/insights` query parameters.
19#[derive(Debug, Clone, Default, PartialEq, Eq, Serialize, Deserialize)]
20pub struct InsightsQuery {
21 /// Days of history to read; default [`INSIGHT_DAYS`], at most [`INSIGHT_DAYS_MAX`].
22 pub days: Option<u32>,
23}
24
25/// `GET /v1/insights`: what the history says about the window.
26#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
27pub struct Insights {
28 /// The start of the window.
29 pub since: u64,
30 /// When it was read: the end of the window.
31 pub until: u64,
32 /// Warnings first, then by repository, workflow and job.
33 pub insights: Vec<Insight>,
34 /// Each repository's waits this week and the week before, by repository. Always the last 14 days, whatever the
35 /// window.
36 pub waits: Vec<RepositoryWaits>,
37}
38
39/// How much an insight matters.
40#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Serialize, Deserialize)]
41#[serde(rename_all = "kebab-case")]
42pub enum Severity {
43 /// Worth acting on: notifications send these.
44 Warning,
45 /// Worth knowing.
46 Notice,
47}
48
49/// One thing the history says, about one job or one repository.
50#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
51pub struct Insight {
52 /// The same while it holds, such as `slower:halyard/api/CI/test`.
53 pub key: String,
54 /// How much it matters.
55 pub severity: Severity,
56 /// `owner/name`.
57 pub repository: String,
58 /// The workflow's name; `None` for a repository's waits, or when unknown.
59 pub workflow: Option<String>,
60 /// The job's name; `None` for a repository's waits.
61 pub job: Option<String>,
62 /// What it is, in one line.
63 pub sentence: String,
64 /// What it is, with its figures.
65 #[serde(flatten)]
66 pub kind: InsightKind,
67 /// The runs that show it, newest first, at most 5.
68 pub runs: Vec<RunLink>,
69}
70
71/// What an insight is, tagged by `kind`.
72#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
73#[serde(rename_all = "kebab-case", tag = "kind")]
74pub enum InsightKind {
75 /// A job's successful runs on its busiest branch take longer than they did, and stayed so.
76 Slower {
77 /// The branch.
78 branch: Option<String>,
79 /// The median run time before, in seconds.
80 before: u64,
81 /// The median run time of the last runs, in seconds.
82 now: u64,
83 /// The first slow run after which no two runs in a row were fast.
84 since: Option<RunLink>,
85 },
86 /// A job failed and then passed on a re-run of the same run.
87 Flaky {
88 /// Runs in which it did.
89 flaky: u32,
90 /// The job's runs in the window.
91 total: u32,
92 },
93 /// A job's memory peaks near its profile's cap.
94 NearCap {
95 /// The profile.
96 profile: String,
97 /// The 95th percentile of its last runs' memory peaks.
98 peak: u64,
99 /// The profile's `memory`.
100 cap: u64,
101 /// What auto sizing does about it.
102 auto: AutoNote,
103 },
104 /// A job's runs that write to the build cache find almost nothing in it: its cache keys differ from run to run.
105 ColdCache {
106 /// The branch those runs are on.
107 branch: Option<String>,
108 /// Lookups that found an entry, over those runs.
109 hits: u64,
110 /// Lookups over those runs.
111 lookups: u64,
112 },
113 /// The kernel throttled a repository's jobs' downloads for memory pressure (`JobMetrics::sock_throttled`):
114 /// the host, or a job at its cap, was short of memory.
115 Throttled {
116 /// Runs it throttled.
117 throttled: u32,
118 /// The repository's runs in the window.
119 total: u32,
120 },
121 /// A repository's jobs wait longer this week than the week before.
122 Waits {
123 /// The 95th percentile of waits the week before, in seconds.
124 before: u64,
125 /// The 95th percentile of waits this week, in seconds.
126 now: u64,
127 },
128 /// A job's CPU cap held back its runs: their quota throttled them in 15% of their run or more (ADR 0079).
129 CpuHeld {
130 /// The profile its runs ran in.
131 profile: String,
132 /// The profile's `cpus`, the cap those runs had.
133 cpus: u32,
134 /// Runs the cap held back.
135 held: u32,
136 /// The job's last runs at that cap that measured their throttling.
137 total: u32,
138 /// Under `measured` admission with auto sizing, the `cpus` the profile was raised from when that raise did
139 /// not make this job faster or less throttled: its runs ask for no further raise. What auto sizing does
140 /// next for the profile, on the other jobs' runs or on the averages, is `runnerctl sizing`'s to say.
141 raised_from: Option<u32>,
142 /// The `cpus` to set by hand; `None` while auto sizing raises them.
143 set: Option<u32>,
144 },
145 /// A queued job has waited 30 minutes or more for the host's memory, and a job waits for it now: programs
146 /// outside CI hold it, which closing is the owner's to do (ADR 0100).
147 HostHeld {
148 /// Jobs waiting for the host's memory now.
149 jobs: u32,
150 /// The longest of their waits for it, in seconds.
151 waited: u64,
152 },
153 /// A kind from a later release, which this one reads only by its sentence. The daemon never sends it.
154 #[serde(other)]
155 Unknown,
156}
157
158/// What auto sizing does about a job near its cap.
159#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
160#[serde(rename_all = "kebab-case")]
161pub enum AutoNote {
162 /// It raises the profile's `memory` when a run meets the cap.
163 Raises,
164 /// The profile is at its `max_memory`: it cannot.
165 AtMax,
166 /// It is off.
167 Off,
168}
169
170/// One run of a job, to open in the history or on GitHub.
171#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
172pub struct RunLink {
173 /// GitHub's job ID, the history's too.
174 pub job: u64,
175 /// The workflow run.
176 pub run_id: u64,
177 /// The run's attempt.
178 pub run_attempt: u32,
179 /// The branch.
180 pub branch: Option<String>,
181 /// The job's page on GitHub.
182 pub url: Option<String>,
183 /// When it finished.
184 pub at: u64,
185}
186
187/// A repository's waits this week against the week before.
188#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
189pub struct RepositoryWaits {
190 /// `owner/name`.
191 pub repository: String,
192 /// The jobs that started in the last 7 days.
193 pub now: WeekWaits,
194 /// The jobs that started in the 7 days before.
195 pub before: WeekWaits,
196}
197
198/// The waits of the jobs that started in a week, from queued to started, leaving out the time before the history's
199/// start and the time no runnerd ran (ADR 0081).
200#[derive(Debug, Clone, Default, PartialEq, Eq, Serialize, Deserialize)]
201pub struct WeekWaits {
202 /// Every job.
203 pub all: WaitSpread,
204 /// The jobs that found a runner already started: a warm runner (ADR 0050).
205 pub warm: WaitSpread,
206 /// The jobs whose runner started after they were queued.
207 pub cold: WaitSpread,
208}
209
210/// How long some jobs waited.
211#[derive(Debug, Clone, Default, PartialEq, Eq, Serialize, Deserialize)]
212pub struct WaitSpread {
213 /// How many.
214 pub jobs: u32,
215 /// The median, in seconds; `None` without jobs.
216 pub p50: Option<u64>,
217 /// The 95th percentile by nearest rank, in seconds; `None` without jobs.
218 pub p95: Option<u64>,
219}
220
221/// `GET /v1/digest`: the last 24 hours.
222#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
223pub struct Digest {
224 /// 24 hours before `until`.
225 pub since: u64,
226 /// When it was read.
227 pub until: u64,
228 /// Jobs that finished: completed, or lost with their runner.
229 pub finished: u64,
230 /// Of those, the ones that failed, timed out or were lost.
231 pub failed: u64,
232 /// The insights that hold now and did not 24 hours ago.
233 pub new: Vec<Insight>,
234 /// The three longest waits of the jobs that started, longest first.
235 pub waits: Vec<LongWait>,
236 /// What the month's job minutes would have cost on GitHub's runners (ADR 0072); `None` when the history could
237 /// not say.
238 #[serde(default, skip_serializing_if = "Option::is_none")]
239 pub savings: Option<super::SavedWindow>,
240}
241
242/// A job that waited.
243#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
244pub struct LongWait {
245 /// GitHub's job ID.
246 pub job: u64,
247 /// `owner/name`.
248 pub repository: String,
249 /// The workflow's name.
250 pub workflow: Option<String>,
251 /// The job's name.
252 pub name: String,
253 /// Seconds from queued to started, leaving out the time before the history's start and the time no runnerd
254 /// ran (ADR 0081).
255 pub waited: u64,
256 /// The job's page on GitHub.
257 pub url: Option<String>,
258}
259
260/// What a notification target is sent, and the events `[notify]` targets choose from.
261#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Serialize, Deserialize)]
262#[serde(rename_all = "kebab-case")]
263pub enum NotifyEvent {
264 /// A job failed or timed out.
265 Failed,
266 /// A job's runner vanished before GitHub said the job completed.
267 Lost,
268 /// A runner could not be registered or started, and its repository is held back.
269 Runner,
270 /// A new warning insight.
271 Insights,
272 /// The daily digest.
273 Digest,
274 /// `runnerctl notify test`; not one a target chooses.
275 Test,
276}
277
278/// How urgent a notification is, as ntfy ranks it.
279#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
280#[serde(rename_all = "kebab-case")]
281pub enum Priority {
282 /// ntfy's 2: no sound.
283 Low,
284 /// ntfy's 3.
285 Default,
286 /// ntfy's 4: a longer vibration.
287 High,
288}
289
290/// What a webhook target is posted, as JSON; an ntfy target gets its title, message, priority and link.
291#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
292pub struct Notification {
293 /// What it is about.
294 pub event: NotifyEvent,
295 /// One short line.
296 pub title: String,
297 /// A few lines.
298 pub message: String,
299 /// What to open: a job's page on GitHub, or nothing.
300 pub url: Option<String>,
301 /// How urgent.
302 pub priority: Priority,
303 /// The jobs it is about: failed or lost.
304 #[serde(default, skip_serializing_if = "Vec::is_empty")]
305 pub jobs: Vec<JobSummary>,
306 /// The runners that could not start.
307 #[serde(default, skip_serializing_if = "Vec::is_empty")]
308 pub runners: Vec<RunnerFailure>,
309 /// The insights it is about.
310 #[serde(default, skip_serializing_if = "Vec::is_empty")]
311 pub insights: Vec<Insight>,
312 /// The digest.
313 #[serde(default, skip_serializing_if = "Option::is_none")]
314 pub digest: Option<Digest>,
315}
316
317/// `POST /v1/notify/test` query parameters.
318#[derive(Debug, Clone, Default, PartialEq, Eq, Serialize, Deserialize)]
319pub struct NotifyTestQuery {
320 /// Only the target with this name; default every one.
321 pub target: Option<String>,
322}
323
324/// `POST /v1/notify/test`: what each target did with a test message.
325#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
326pub struct NotifyTest {
327 /// In the config's order.
328 pub targets: Vec<Delivery>,
329}
330
331/// What one target did with a message.
332#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
333pub struct Delivery {
334 /// The target's name.
335 pub target: String,
336 /// Why it was not delivered: an HTTP status or a connection error, never the URL. `None` when it was.
337 pub error: Option<String>,
338}