Skip to main content

runnerd_core/api/
insights.rs

1//! Insights as clients read them: what got slower, what is flaky, what nears its memory cap, what memory pressure
2//! throttled, what its CPU cap held back, jobs held for the host's memory and how long jobs wait, computed from the job history (ADR 0051); the daily digest; and what the
3//! daemon's notifications carry. Times are seconds since the Unix epoch, durations seconds, sizes bytes.
4
5use serde::{Deserialize, Serialize};
6
7use super::RunnerFailure;
8use super::jobs::JobSummary;
9
10mod shown;
11
12/// The window `GET /v1/insights` reads by default, in days.
13pub const INSIGHT_DAYS: u32 = 14;
14
15/// The longest window `GET /v1/insights` reads, in days.
16pub const INSIGHT_DAYS_MAX: u32 = 90;
17
18/// `GET /v1/insights` query parameters.
19#[derive(Debug, Clone, Default, PartialEq, Eq, Serialize, Deserialize)]
20pub struct InsightsQuery {
21    /// Days of history to read; default [`INSIGHT_DAYS`], at most [`INSIGHT_DAYS_MAX`].
22    pub days: Option<u32>,
23}
24
25/// `GET /v1/insights`: what the history says about the window.
26#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
27pub struct Insights {
28    /// The start of the window.
29    pub since: u64,
30    /// When it was read: the end of the window.
31    pub until: u64,
32    /// Warnings first, then by repository, workflow and job.
33    pub insights: Vec<Insight>,
34    /// Each repository's waits this week and the week before, by repository. Always the last 14 days, whatever the
35    /// window.
36    pub waits: Vec<RepositoryWaits>,
37    /// What the host's memory came to on each UTC day of the window that CI ran, newest first (ADR 0108); absent from
38    /// a daemon before 0.0.30.
39    #[serde(default, skip_serializing_if = "Vec::is_empty")]
40    pub host: Vec<HostHazard>,
41    /// What the jobs that ended in the window and the drive under Docker's root wrote (ADR 0143); `None` when no job
42    /// was measured and the daemon read no drive, and from a daemon before 0.1.3.
43    #[serde(default, skip_serializing_if = "Option::is_none")]
44    pub disk: Option<super::DiskWrites>,
45}
46
47/// What the host's memory came to while CI ran over a window, from the readings the budget's events stored (ADR 0108):
48/// how long `MemAvailable` stood under each line, the least it and `SwapFree` fell to, and how much of that came while
49/// the host's limit took a credit. Times are seconds, sizes bytes.
50#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
51pub struct HostHazard {
52    /// The start of the window: a UTC day's, or the digest's 24 hours'.
53    pub since: u64,
54    /// Its end.
55    pub until: u64,
56    /// How long a runner ran with a reading of the host.
57    pub running: u64,
58    /// Of that, how long a host credit stood.
59    pub credited: u64,
60    /// The line under which sockets start to throttle: the host's margin and a quarter of its memory (ADR 0099).
61    pub low_line: u64,
62    /// The line under which systemd-oomd may act once swap is full too: a tenth of the host's memory.
63    pub critical_line: u64,
64    /// How long `MemAvailable` stood under `low_line`.
65    pub low: HostSpan,
66    /// How long it stood under `critical_line`.
67    pub critical: HostSpan,
68    /// The least `MemAvailable` read.
69    pub lowest: u64,
70    /// The least read while a host credit stood; `None` when none did.
71    #[serde(default, skip_serializing_if = "Option::is_none")]
72    pub lowest_credited: Option<u64>,
73    /// The least `SwapFree` read; `None` on a host without swap.
74    #[serde(default, skip_serializing_if = "Option::is_none")]
75    pub lowest_swap_free: Option<u64>,
76}
77
78/// How long the host stood under a line while CI ran, in seconds.
79#[derive(Debug, Clone, Copy, Default, PartialEq, Eq, Serialize, Deserialize)]
80pub struct HostSpan {
81    /// In all.
82    pub seconds: u64,
83    /// Of that, while a host credit stood.
84    pub credited: u64,
85}
86
87/// How much an insight matters.
88#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Serialize, Deserialize)]
89#[serde(rename_all = "kebab-case")]
90pub enum Severity {
91    /// Worth acting on: notifications send these.
92    Warning,
93    /// Worth knowing.
94    Notice,
95}
96
97/// One thing the history says, about one job or one repository.
98#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
99pub struct Insight {
100    /// The same while it holds, such as `slower:halyard/api/CI/test`.
101    pub key: String,
102    /// How much it matters.
103    pub severity: Severity,
104    /// `owner/name`.
105    pub repository: String,
106    /// The workflow's name; `None` for a repository's waits, or when unknown.
107    pub workflow: Option<String>,
108    /// The job's name; `None` for a repository's waits.
109    pub job: Option<String>,
110    /// What it is, in one line.
111    pub sentence: String,
112    /// What it is, with its figures.
113    #[serde(flatten)]
114    pub kind: InsightKind,
115    /// The runs that show it, newest first, at most 5.
116    pub runs: Vec<RunLink>,
117}
118
119/// What an insight is, tagged by `kind`.
120#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
121#[serde(rename_all = "kebab-case", tag = "kind")]
122pub enum InsightKind {
123    /// A job's successful runs on its busiest branch take longer than they did, and stayed so.
124    Slower {
125        /// The branch.
126        branch: Option<String>,
127        /// The median run time before, in seconds.
128        before: u64,
129        /// The median run time of the last runs, in seconds.
130        now: u64,
131        /// The first slow run after which no two runs in a row were fast.
132        since: Option<RunLink>,
133    },
134    /// A job failed and then passed on a re-run of the same run.
135    Flaky {
136        /// Runs in which it did.
137        flaky: u32,
138        /// The job's runs in the window.
139        total: u32,
140    },
141    /// A job's memory peaks near its profile's cap.
142    NearCap {
143        /// The profile.
144        profile: String,
145        /// The 95th percentile of its last runs' memory peaks.
146        peak: u64,
147        /// The profile's `memory`.
148        cap: u64,
149        /// What auto sizing does about it.
150        auto: AutoNote,
151    },
152    /// A job's runs that write to the build cache find almost nothing in it: its cache keys differ from run to run.
153    ColdCache {
154        /// The branch those runs are on.
155        branch: Option<String>,
156        /// Lookups that found an entry, over those runs.
157        hits: u64,
158        /// Lookups over those runs.
159        lookups: u64,
160    },
161    /// The kernel throttled a repository's jobs' downloads for memory pressure (`JobMetrics::sock_throttled`):
162    /// the host, or a job at its cap, was short of memory.
163    Throttled {
164        /// Runs it throttled.
165        throttled: u32,
166        /// The repository's runs in the window.
167        total: u32,
168    },
169    /// A repository's jobs wait longer this week than the week before.
170    Waits {
171        /// The 95th percentile of waits the week before, in seconds.
172        before: u64,
173        /// The 95th percentile of waits this week, in seconds.
174        now: u64,
175    },
176    /// A job's CPU cap held back its runs: their quota throttled them in 15% of their run or more (ADR 0079).
177    CpuHeld {
178        /// The profile its runs ran in.
179        profile: String,
180        /// The profile's `cpus`, the cap those runs had.
181        cpus: u32,
182        /// Runs the cap held back.
183        held: u32,
184        /// The job's last runs at that cap that measured their throttling.
185        total: u32,
186        /// Under `measured` admission with auto sizing, the `cpus` the profile was raised from when that raise did
187        /// not make this job faster or less throttled: its runs ask for no further raise. What auto sizing does
188        /// next for the profile, on the other jobs' runs or on the averages, is `runnerctl sizing`'s to say.
189        raised_from: Option<u32>,
190        /// The `cpus` to set by hand; `None` while auto sizing raises them.
191        set: Option<u32>,
192    },
193    /// A queued job has waited 30 minutes or more for the host's memory, and a job waits for it now: programs
194    /// outside CI hold it, which closing is the owner's to do (ADR 0100).
195    HostHeld {
196        /// Jobs waiting for the host's memory now.
197        jobs: u32,
198        /// The longest of their waits for it, in seconds.
199        waited: u64,
200    },
201    /// On a UTC day, the host's `MemAvailable` fell under a tenth of its memory while a host credit let runners start
202    /// past their reservations (ADR 0108): the buffer the credit spends ran out. Told once a day at most.
203    HostLow {
204        /// The start of the day.
205        day: u64,
206        /// How long it stood there while a host credit stood, in seconds.
207        seconds: u64,
208        /// The least `MemAvailable` read while one stood, in bytes.
209        lowest: u64,
210    },
211    /// A kind from a later release, which this one reads only by its sentence. The daemon never sends it.
212    #[serde(other)]
213    Unknown,
214}
215
216/// What auto sizing does about a job near its cap.
217#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
218#[serde(rename_all = "kebab-case")]
219pub enum AutoNote {
220    /// It raises the profile's `memory` when a run meets the cap.
221    Raises,
222    /// The profile is at its `max_memory`: it cannot.
223    AtMax,
224    /// It is off.
225    Off,
226}
227
228/// One run of a job, to open in the history or on GitHub.
229#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
230pub struct RunLink {
231    /// GitHub's job ID, the history's too.
232    pub job: u64,
233    /// The workflow run.
234    pub run_id: u64,
235    /// The run's attempt.
236    pub run_attempt: u32,
237    /// The branch.
238    pub branch: Option<String>,
239    /// The job's page on GitHub.
240    pub url: Option<String>,
241    /// When it finished.
242    pub at: u64,
243}
244
245/// A repository's waits this week against the week before.
246#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
247pub struct RepositoryWaits {
248    /// `owner/name`.
249    pub repository: String,
250    /// The jobs that started in the last 7 days.
251    pub now: WeekWaits,
252    /// The jobs that started in the 7 days before.
253    pub before: WeekWaits,
254}
255
256/// The waits of the jobs that started in a week, from queued to started, leaving out the time before the history's
257/// start and the time no runnerd ran (ADR 0081).
258#[derive(Debug, Clone, Default, PartialEq, Eq, Serialize, Deserialize)]
259pub struct WeekWaits {
260    /// Every job.
261    pub all: WaitSpread,
262    /// The jobs that found a runner already started: a warm runner (ADR 0050).
263    pub warm: WaitSpread,
264    /// The jobs whose runner started after they were queued.
265    pub cold: WaitSpread,
266}
267
268/// How long some jobs waited.
269#[derive(Debug, Clone, Default, PartialEq, Eq, Serialize, Deserialize)]
270pub struct WaitSpread {
271    /// How many.
272    pub jobs: u32,
273    /// The median, in seconds; `None` without jobs.
274    pub p50: Option<u64>,
275    /// The 95th percentile by nearest rank, in seconds; `None` without jobs.
276    pub p95: Option<u64>,
277}
278
279/// `GET /v1/digest`: the last 24 hours.
280#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
281pub struct Digest {
282    /// 24 hours before `until`.
283    pub since: u64,
284    /// When it was read.
285    pub until: u64,
286    /// Jobs that finished: completed, or lost with their runner.
287    pub finished: u64,
288    /// Of those, the ones that failed, timed out or were lost.
289    pub failed: u64,
290    /// The insights that hold now and did not 24 hours ago.
291    pub new: Vec<Insight>,
292    /// The three longest waits of the jobs that started, longest first.
293    pub waits: Vec<LongWait>,
294    /// What the month's job minutes would have cost on GitHub's runners (ADR 0072); `None` when the history could
295    /// not say.
296    #[serde(default, skip_serializing_if = "Option::is_none")]
297    pub savings: Option<super::SavedWindow>,
298    /// What the host's memory came to while CI ran (ADR 0108); `None` when no runner ran with a reading of it.
299    #[serde(default, skip_serializing_if = "Option::is_none")]
300    pub host: Option<HostHazard>,
301}
302
303/// A job that waited.
304#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
305pub struct LongWait {
306    /// GitHub's job ID.
307    pub job: u64,
308    /// `owner/name`.
309    pub repository: String,
310    /// The workflow's name.
311    pub workflow: Option<String>,
312    /// The job's name.
313    pub name: String,
314    /// Seconds from queued to started, leaving out the time before the history's start and the time no runnerd
315    /// ran (ADR 0081).
316    pub waited: u64,
317    /// The job's page on GitHub.
318    pub url: Option<String>,
319}
320
321/// What a notification target is sent, and the events `[notify]` targets choose from.
322#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Serialize, Deserialize)]
323#[serde(rename_all = "kebab-case")]
324pub enum NotifyEvent {
325    /// A job failed or timed out.
326    Failed,
327    /// A job's runner vanished before GitHub said the job completed.
328    Lost,
329    /// A runner could not be registered or started, and its repository is held back.
330    Runner,
331    /// A new warning insight.
332    Insights,
333    /// The daily digest.
334    Digest,
335    /// `runnerctl notify test`; not one a target chooses.
336    Test,
337}
338
339/// How urgent a notification is, as ntfy ranks it.
340#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
341#[serde(rename_all = "kebab-case")]
342pub enum Priority {
343    /// ntfy's 2: no sound.
344    Low,
345    /// ntfy's 3.
346    Default,
347    /// ntfy's 4: a longer vibration.
348    High,
349}
350
351/// What a webhook target is posted, as JSON; an ntfy target gets its title, message, priority and link.
352#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
353pub struct Notification {
354    /// What it is about.
355    pub event: NotifyEvent,
356    /// One short line.
357    pub title: String,
358    /// A few lines.
359    pub message: String,
360    /// What to open: a job's page on GitHub, or nothing.
361    pub url: Option<String>,
362    /// How urgent.
363    pub priority: Priority,
364    /// The jobs it is about: failed or lost.
365    #[serde(default, skip_serializing_if = "Vec::is_empty")]
366    pub jobs: Vec<JobSummary>,
367    /// The runners that could not start.
368    #[serde(default, skip_serializing_if = "Vec::is_empty")]
369    pub runners: Vec<RunnerFailure>,
370    /// The insights it is about.
371    #[serde(default, skip_serializing_if = "Vec::is_empty")]
372    pub insights: Vec<Insight>,
373    /// The digest.
374    #[serde(default, skip_serializing_if = "Option::is_none")]
375    pub digest: Option<Digest>,
376}
377
378/// `POST /v1/notify/test` query parameters.
379#[derive(Debug, Clone, Default, PartialEq, Eq, Serialize, Deserialize)]
380pub struct NotifyTestQuery {
381    /// Only the target with this name; default every one.
382    pub target: Option<String>,
383}
384
385/// `POST /v1/notify/test`: what each target did with a test message.
386#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
387pub struct NotifyTest {
388    /// In the config's order.
389    pub targets: Vec<Delivery>,
390}
391
392/// What one target did with a message.
393#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
394pub struct Delivery {
395    /// The target's name.
396    pub target: String,
397    /// Why it was not delivered: an HTTP status or a connection error, never the URL. `None` when it was.
398    pub error: Option<String>,
399}