Skip to main content

runnerd_core/api/
insights.rs

1//! Insights as clients read them: what got slower, what is flaky, what nears its memory cap, what memory pressure
2//! throttled, what its CPU cap held back, jobs held for the host's memory and how long jobs wait, computed from the job history (ADR 0051); the daily digest; and what the
3//! daemon's notifications carry. Times are seconds since the Unix epoch, durations seconds, sizes bytes.
4
5use serde::{Deserialize, Serialize};
6
7use super::RunnerFailure;
8use super::jobs::JobSummary;
9
10mod shown;
11
12pub use shown::InsightFigure;
13pub use shown::InsightsHeading;
14pub use shown::WaitFigure;
15
16/// The window `GET /v1/insights` reads by default, in days.
17pub const INSIGHT_DAYS: u32 = 14;
18
19/// The longest window `GET /v1/insights` reads, in days.
20pub const INSIGHT_DAYS_MAX: u32 = 90;
21
22/// `GET /v1/insights` query parameters.
23#[derive(Debug, Clone, Default, PartialEq, Eq, Serialize, Deserialize)]
24pub struct InsightsQuery {
25    /// Days of history to read; default [`INSIGHT_DAYS`], at most [`INSIGHT_DAYS_MAX`].
26    pub days: Option<u32>,
27}
28
29/// `GET /v1/insights`: what the history says about the window.
30#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
31pub struct Insights {
32    /// The start of the window.
33    pub since: u64,
34    /// When it was read: the end of the window.
35    pub until: u64,
36    /// Warnings first, then by repository, workflow and job.
37    pub insights: Vec<Insight>,
38    /// Each repository's waits this week and the week before, by repository. Always the last 14 days, whatever the
39    /// window.
40    pub waits: Vec<RepositoryWaits>,
41    /// What the host's memory came to on each UTC day of the window that CI ran, newest first (ADR 0108); absent from
42    /// a daemon before 0.0.30.
43    #[serde(default, skip_serializing_if = "Vec::is_empty")]
44    pub host: Vec<HostHazard>,
45    /// What the jobs that ended in the window and the drive under Docker's root wrote (ADR 0143); `None` when no job
46    /// was measured and the daemon read no drive, and from a daemon before 0.1.3.
47    #[serde(default, skip_serializing_if = "Option::is_none")]
48    pub disk: Option<super::DiskWrites>,
49}
50
51/// What the host's memory came to while CI ran over a window, from the readings the budget's events stored (ADR 0108):
52/// how long `MemAvailable` stood under each line, the least it and `SwapFree` fell to, and how much of that came while
53/// the host's limit took a credit. Times are seconds, sizes bytes.
54#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
55pub struct HostHazard {
56    /// The start of the window: a UTC day's, or the digest's 24 hours'.
57    pub since: u64,
58    /// Its end.
59    pub until: u64,
60    /// How long a runner ran with a reading of the host.
61    pub running: u64,
62    /// Of that, how long a host credit stood.
63    pub credited: u64,
64    /// The line under which sockets start to throttle: the host's margin and a quarter of its memory (ADR 0099).
65    pub low_line: u64,
66    /// The line under which systemd-oomd may act once swap is full too: a tenth of the host's memory.
67    pub critical_line: u64,
68    /// How long `MemAvailable` stood under `low_line`.
69    pub low: HostSpan,
70    /// How long it stood under `critical_line`.
71    pub critical: HostSpan,
72    /// The least `MemAvailable` read.
73    pub lowest: u64,
74    /// The least read while a host credit stood; `None` when none did.
75    #[serde(default, skip_serializing_if = "Option::is_none")]
76    pub lowest_credited: Option<u64>,
77    /// The least `SwapFree` read; `None` on a host without swap.
78    #[serde(default, skip_serializing_if = "Option::is_none")]
79    pub lowest_swap_free: Option<u64>,
80}
81
82/// How long the host stood under a line while CI ran, in seconds.
83#[derive(Debug, Clone, Copy, Default, PartialEq, Eq, Serialize, Deserialize)]
84pub struct HostSpan {
85    /// In all.
86    pub seconds: u64,
87    /// Of that, while a host credit stood.
88    pub credited: u64,
89}
90
91/// How much an insight matters.
92#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Serialize, Deserialize)]
93#[serde(rename_all = "kebab-case")]
94pub enum Severity {
95    /// Worth acting on: notifications send these.
96    Warning,
97    /// Worth knowing.
98    Notice,
99}
100
101/// One thing the history says, about one job or one repository.
102#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
103pub struct Insight {
104    /// The same while it holds, such as `slower:halyard/api/CI/test`.
105    pub key: String,
106    /// How much it matters.
107    pub severity: Severity,
108    /// `owner/name`.
109    pub repository: String,
110    /// The workflow's name; `None` for a repository's waits, or when unknown.
111    pub workflow: Option<String>,
112    /// The job's name; `None` for a repository's waits.
113    pub job: Option<String>,
114    /// What it is, in one line.
115    pub sentence: String,
116    /// What it is, with its figures.
117    #[serde(flatten)]
118    pub kind: InsightKind,
119    /// The runs that show it, newest first, at most 5.
120    pub runs: Vec<RunLink>,
121}
122
123/// What an insight is, tagged by `kind`.
124#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
125#[serde(rename_all = "kebab-case", tag = "kind")]
126pub enum InsightKind {
127    /// A job's successful runs on its busiest branch take longer than they did, and stayed so.
128    Slower {
129        /// The branch.
130        branch: Option<String>,
131        /// The median run time before, in seconds.
132        before: u64,
133        /// The median run time of the last runs, in seconds.
134        now: u64,
135        /// The first slow run after which no two runs in a row were fast.
136        since: Option<RunLink>,
137    },
138    /// A job failed and then passed on a re-run of the same run.
139    Flaky {
140        /// Runs in which it did.
141        flaky: u32,
142        /// The job's runs in the window.
143        total: u32,
144    },
145    /// A job's memory peaks near its profile's cap.
146    NearCap {
147        /// The profile.
148        profile: String,
149        /// The 95th percentile of its last runs' memory peaks.
150        peak: u64,
151        /// The profile's `memory`.
152        cap: u64,
153        /// What auto sizing does about it.
154        auto: AutoNote,
155    },
156    /// A job's runs that write to the build cache find almost nothing in it: its cache keys differ from run to run.
157    ColdCache {
158        /// The branch those runs are on.
159        branch: Option<String>,
160        /// Lookups that found an entry, over those runs.
161        hits: u64,
162        /// Lookups over those runs.
163        lookups: u64,
164    },
165    /// The kernel throttled a repository's jobs' downloads for memory pressure (`JobMetrics::sock_throttled`):
166    /// the host, or a job at its cap, was short of memory.
167    Throttled {
168        /// Runs it throttled.
169        throttled: u32,
170        /// The repository's runs in the window.
171        total: u32,
172    },
173    /// A repository's jobs wait longer this week than the week before.
174    Waits {
175        /// The 95th percentile of waits the week before, in seconds.
176        before: u64,
177        /// The 95th percentile of waits this week, in seconds.
178        now: u64,
179    },
180    /// A job's CPU cap held back its runs: their quota throttled them in 15% of their run or more (ADR 0079).
181    CpuHeld {
182        /// The profile its runs ran in.
183        profile: String,
184        /// The profile's `cpus`, the cap those runs had.
185        cpus: u32,
186        /// Runs the cap held back.
187        held: u32,
188        /// The job's last runs at that cap that measured their throttling.
189        total: u32,
190        /// Under `measured` admission with auto sizing, the `cpus` the profile was raised from when that raise did
191        /// not make this job faster or less throttled: its runs ask for no further raise. What auto sizing does
192        /// next for the profile, on the other jobs' runs or on the averages, is `runnerctl sizing`'s to say.
193        raised_from: Option<u32>,
194        /// The `cpus` to set by hand; `None` while auto sizing raises them.
195        set: Option<u32>,
196    },
197    /// A queued job has waited 30 minutes or more for the host's memory, and a job waits for it now: programs
198    /// outside CI hold it, which closing is the owner's to do (ADR 0100).
199    HostHeld {
200        /// Jobs waiting for the host's memory now.
201        jobs: u32,
202        /// The longest of their waits for it, in seconds.
203        waited: u64,
204    },
205    /// On a UTC day, the host's `MemAvailable` fell under a tenth of its memory while a host credit let runners start
206    /// past their reservations (ADR 0108): the buffer the credit spends ran out. Told once a day at most.
207    HostLow {
208        /// The start of the day.
209        day: u64,
210        /// How long it stood there while a host credit stood, in seconds.
211        seconds: u64,
212        /// The least `MemAvailable` read while one stood, in bytes.
213        lowest: u64,
214    },
215    /// A kind from a later release, which this one reads only by its sentence. The daemon never sends it.
216    #[serde(other)]
217    Unknown,
218}
219
220/// What auto sizing does about a job near its cap.
221#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
222#[serde(rename_all = "kebab-case")]
223pub enum AutoNote {
224    /// It raises the profile's `memory` when a run meets the cap.
225    Raises,
226    /// The profile is at its `max_memory`: it cannot.
227    AtMax,
228    /// It is off.
229    Off,
230}
231
232/// One run of a job, to open in the history or on GitHub.
233#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
234pub struct RunLink {
235    /// GitHub's job ID, the history's too.
236    pub job: u64,
237    /// The workflow run.
238    pub run_id: u64,
239    /// The run's attempt.
240    pub run_attempt: u32,
241    /// The branch.
242    pub branch: Option<String>,
243    /// The job's page on GitHub.
244    pub url: Option<String>,
245    /// When it finished.
246    pub at: u64,
247}
248
249/// A repository's waits this week against the week before.
250#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
251pub struct RepositoryWaits {
252    /// `owner/name`.
253    pub repository: String,
254    /// The jobs that started in the last 7 days.
255    pub now: WeekWaits,
256    /// The jobs that started in the 7 days before.
257    pub before: WeekWaits,
258}
259
260/// The waits of the jobs that started in a week, from queued to started, leaving out the time before the history's
261/// start and the time no runnerd ran (ADR 0081).
262#[derive(Debug, Clone, Default, PartialEq, Eq, Serialize, Deserialize)]
263pub struct WeekWaits {
264    /// Every job.
265    pub all: WaitSpread,
266    /// The jobs that found a runner already started: a warm runner (ADR 0050).
267    pub warm: WaitSpread,
268    /// The jobs whose runner started after they were queued.
269    pub cold: WaitSpread,
270}
271
272/// How long some jobs waited.
273#[derive(Debug, Clone, Default, PartialEq, Eq, Serialize, Deserialize)]
274pub struct WaitSpread {
275    /// How many.
276    pub jobs: u32,
277    /// The median, in seconds; `None` without jobs.
278    pub p50: Option<u64>,
279    /// The 95th percentile by nearest rank, in seconds; `None` without jobs.
280    pub p95: Option<u64>,
281}
282
283/// `GET /v1/digest`: the last 24 hours.
284#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
285pub struct Digest {
286    /// 24 hours before `until`.
287    pub since: u64,
288    /// When it was read.
289    pub until: u64,
290    /// Jobs that finished: completed, or lost with their runner.
291    pub finished: u64,
292    /// Of those, the ones that failed, timed out or were lost.
293    pub failed: u64,
294    /// The insights that hold now and did not 24 hours ago.
295    pub new: Vec<Insight>,
296    /// The three longest waits of the jobs that started, longest first.
297    pub waits: Vec<LongWait>,
298    /// What the month's job minutes would have cost on GitHub's runners (ADR 0072); `None` when the history could
299    /// not say.
300    #[serde(default, skip_serializing_if = "Option::is_none")]
301    pub savings: Option<super::SavedWindow>,
302    /// What the host's memory came to while CI ran (ADR 0108); `None` when no runner ran with a reading of it.
303    #[serde(default, skip_serializing_if = "Option::is_none")]
304    pub host: Option<HostHazard>,
305}
306
307/// A job that waited.
308#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
309pub struct LongWait {
310    /// GitHub's job ID.
311    pub job: u64,
312    /// `owner/name`.
313    pub repository: String,
314    /// The workflow's name.
315    pub workflow: Option<String>,
316    /// The job's name.
317    pub name: String,
318    /// Seconds from queued to started, leaving out the time before the history's start and the time no runnerd
319    /// ran (ADR 0081).
320    pub waited: u64,
321    /// The job's page on GitHub.
322    pub url: Option<String>,
323}
324
325/// What a notification target is sent, and the events `[notify]` targets choose from.
326#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Serialize, Deserialize)]
327#[serde(rename_all = "kebab-case")]
328pub enum NotifyEvent {
329    /// A job failed or timed out.
330    Failed,
331    /// A job's runner vanished before GitHub said the job completed.
332    Lost,
333    /// A runner could not be registered or started, and its repository is held back.
334    Runner,
335    /// A new warning insight.
336    Insights,
337    /// The daily digest.
338    Digest,
339    /// `runnerctl notify test`; not one a target chooses.
340    Test,
341}
342
343/// How urgent a notification is, as ntfy ranks it.
344#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
345#[serde(rename_all = "kebab-case")]
346pub enum Priority {
347    /// ntfy's 2: no sound.
348    Low,
349    /// ntfy's 3.
350    Default,
351    /// ntfy's 4: a longer vibration.
352    High,
353}
354
355/// What a webhook target is posted, as JSON; an ntfy target gets its title, message, priority and link.
356#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
357pub struct Notification {
358    /// What it is about.
359    pub event: NotifyEvent,
360    /// One short line.
361    pub title: String,
362    /// A few lines.
363    pub message: String,
364    /// What to open: a job's page on GitHub, or nothing.
365    pub url: Option<String>,
366    /// How urgent.
367    pub priority: Priority,
368    /// The jobs it is about: failed or lost.
369    #[serde(default, skip_serializing_if = "Vec::is_empty")]
370    pub jobs: Vec<JobSummary>,
371    /// The runners that could not start.
372    #[serde(default, skip_serializing_if = "Vec::is_empty")]
373    pub runners: Vec<RunnerFailure>,
374    /// The insights it is about.
375    #[serde(default, skip_serializing_if = "Vec::is_empty")]
376    pub insights: Vec<Insight>,
377    /// The digest.
378    #[serde(default, skip_serializing_if = "Option::is_none")]
379    pub digest: Option<Digest>,
380}
381
382/// `POST /v1/notify/test` query parameters.
383#[derive(Debug, Clone, Default, PartialEq, Eq, Serialize, Deserialize)]
384pub struct NotifyTestQuery {
385    /// Only the target with this name; default every one.
386    pub target: Option<String>,
387}
388
389/// `POST /v1/notify/test`: what each target did with a test message.
390#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
391pub struct NotifyTest {
392    /// In the config's order.
393    pub targets: Vec<Delivery>,
394}
395
396/// What one target did with a message.
397#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
398pub struct Delivery {
399    /// The target's name.
400    pub target: String,
401    /// Why it was not delivered: an HTTP status or a connection error, never the URL. `None` when it was.
402    pub error: Option<String>,
403}