runnerd-core 0.1.4

Config, domain logic and API types shared by runnerd and its clients
Documentation
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
//! Insights as clients read them: what got slower, what is flaky, what nears its memory cap, what memory pressure
//! throttled, what its CPU cap held back, jobs held for the host's memory and how long jobs wait, computed from the job history (ADR 0051); the daily digest; and what the
//! daemon's notifications carry. Times are seconds since the Unix epoch, durations seconds, sizes bytes.

use serde::{Deserialize, Serialize};

use super::RunnerFailure;
use super::jobs::JobSummary;

mod shown;

pub use shown::InsightFigure;
pub use shown::InsightsHeading;
pub use shown::WaitFigure;

/// The window `GET /v1/insights` reads by default, in days.
pub const INSIGHT_DAYS: u32 = 14;

/// The longest window `GET /v1/insights` reads, in days.
pub const INSIGHT_DAYS_MAX: u32 = 90;

/// `GET /v1/insights` query parameters.
#[derive(Debug, Clone, Default, PartialEq, Eq, Serialize, Deserialize)]
pub struct InsightsQuery {
    /// Days of history to read; default [`INSIGHT_DAYS`], at most [`INSIGHT_DAYS_MAX`].
    pub days: Option<u32>,
}

/// `GET /v1/insights`: what the history says about the window.
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
pub struct Insights {
    /// The start of the window.
    pub since: u64,
    /// When it was read: the end of the window.
    pub until: u64,
    /// Warnings first, then by repository, workflow and job.
    pub insights: Vec<Insight>,
    /// Each repository's waits this week and the week before, by repository. Always the last 14 days, whatever the
    /// window.
    pub waits: Vec<RepositoryWaits>,
    /// What the host's memory came to on each UTC day of the window that CI ran, newest first (ADR 0108); absent from
    /// a daemon before 0.0.30.
    #[serde(default, skip_serializing_if = "Vec::is_empty")]
    pub host: Vec<HostHazard>,
    /// What the jobs that ended in the window and the drive under Docker's root wrote (ADR 0143); `None` when no job
    /// was measured and the daemon read no drive, and from a daemon before 0.1.3.
    #[serde(default, skip_serializing_if = "Option::is_none")]
    pub disk: Option<super::DiskWrites>,
}

/// What the host's memory came to while CI ran over a window, from the readings the budget's events stored (ADR 0108):
/// how long `MemAvailable` stood under each line, the least it and `SwapFree` fell to, and how much of that came while
/// the host's limit took a credit. Times are seconds, sizes bytes.
#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
pub struct HostHazard {
    /// The start of the window: a UTC day's, or the digest's 24 hours'.
    pub since: u64,
    /// Its end.
    pub until: u64,
    /// How long a runner ran with a reading of the host.
    pub running: u64,
    /// Of that, how long a host credit stood.
    pub credited: u64,
    /// The line under which sockets start to throttle: the host's margin and a quarter of its memory (ADR 0099).
    pub low_line: u64,
    /// The line under which systemd-oomd may act once swap is full too: a tenth of the host's memory.
    pub critical_line: u64,
    /// How long `MemAvailable` stood under `low_line`.
    pub low: HostSpan,
    /// How long it stood under `critical_line`.
    pub critical: HostSpan,
    /// The least `MemAvailable` read.
    pub lowest: u64,
    /// The least read while a host credit stood; `None` when none did.
    #[serde(default, skip_serializing_if = "Option::is_none")]
    pub lowest_credited: Option<u64>,
    /// The least `SwapFree` read; `None` on a host without swap.
    #[serde(default, skip_serializing_if = "Option::is_none")]
    pub lowest_swap_free: Option<u64>,
}

/// How long the host stood under a line while CI ran, in seconds.
#[derive(Debug, Clone, Copy, Default, PartialEq, Eq, Serialize, Deserialize)]
pub struct HostSpan {
    /// In all.
    pub seconds: u64,
    /// Of that, while a host credit stood.
    pub credited: u64,
}

/// How much an insight matters.
#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Serialize, Deserialize)]
#[serde(rename_all = "kebab-case")]
pub enum Severity {
    /// Worth acting on: notifications send these.
    Warning,
    /// Worth knowing.
    Notice,
}

/// One thing the history says, about one job or one repository.
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
pub struct Insight {
    /// The same while it holds, such as `slower:halyard/api/CI/test`.
    pub key: String,
    /// How much it matters.
    pub severity: Severity,
    /// `owner/name`.
    pub repository: String,
    /// The workflow's name; `None` for a repository's waits, or when unknown.
    pub workflow: Option<String>,
    /// The job's name; `None` for a repository's waits.
    pub job: Option<String>,
    /// What it is, in one line.
    pub sentence: String,
    /// What it is, with its figures.
    #[serde(flatten)]
    pub kind: InsightKind,
    /// The runs that show it, newest first, at most 5.
    pub runs: Vec<RunLink>,
}

/// What an insight is, tagged by `kind`.
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
#[serde(rename_all = "kebab-case", tag = "kind")]
pub enum InsightKind {
    /// A job's successful runs on its busiest branch take longer than they did, and stayed so.
    Slower {
        /// The branch.
        branch: Option<String>,
        /// The median run time before, in seconds.
        before: u64,
        /// The median run time of the last runs, in seconds.
        now: u64,
        /// The first slow run after which no two runs in a row were fast.
        since: Option<RunLink>,
    },
    /// A job failed and then passed on a re-run of the same run.
    Flaky {
        /// Runs in which it did.
        flaky: u32,
        /// The job's runs in the window.
        total: u32,
    },
    /// A job's memory peaks near its profile's cap.
    NearCap {
        /// The profile.
        profile: String,
        /// The 95th percentile of its last runs' memory peaks.
        peak: u64,
        /// The profile's `memory`.
        cap: u64,
        /// What auto sizing does about it.
        auto: AutoNote,
    },
    /// A job's runs that write to the build cache find almost nothing in it: its cache keys differ from run to run.
    ColdCache {
        /// The branch those runs are on.
        branch: Option<String>,
        /// Lookups that found an entry, over those runs.
        hits: u64,
        /// Lookups over those runs.
        lookups: u64,
    },
    /// The kernel throttled a repository's jobs' downloads for memory pressure (`JobMetrics::sock_throttled`):
    /// the host, or a job at its cap, was short of memory.
    Throttled {
        /// Runs it throttled.
        throttled: u32,
        /// The repository's runs in the window.
        total: u32,
    },
    /// A repository's jobs wait longer this week than the week before.
    Waits {
        /// The 95th percentile of waits the week before, in seconds.
        before: u64,
        /// The 95th percentile of waits this week, in seconds.
        now: u64,
    },
    /// A job's CPU cap held back its runs: their quota throttled them in 15% of their run or more (ADR 0079).
    CpuHeld {
        /// The profile its runs ran in.
        profile: String,
        /// The profile's `cpus`, the cap those runs had.
        cpus: u32,
        /// Runs the cap held back.
        held: u32,
        /// The job's last runs at that cap that measured their throttling.
        total: u32,
        /// Under `measured` admission with auto sizing, the `cpus` the profile was raised from when that raise did
        /// not make this job faster or less throttled: its runs ask for no further raise. What auto sizing does
        /// next for the profile, on the other jobs' runs or on the averages, is `runnerctl sizing`'s to say.
        raised_from: Option<u32>,
        /// The `cpus` to set by hand; `None` while auto sizing raises them.
        set: Option<u32>,
    },
    /// A queued job has waited 30 minutes or more for the host's memory, and a job waits for it now: programs
    /// outside CI hold it, which closing is the owner's to do (ADR 0100).
    HostHeld {
        /// Jobs waiting for the host's memory now.
        jobs: u32,
        /// The longest of their waits for it, in seconds.
        waited: u64,
    },
    /// On a UTC day, the host's `MemAvailable` fell under a tenth of its memory while a host credit let runners start
    /// past their reservations (ADR 0108): the buffer the credit spends ran out. Told once a day at most.
    HostLow {
        /// The start of the day.
        day: u64,
        /// How long it stood there while a host credit stood, in seconds.
        seconds: u64,
        /// The least `MemAvailable` read while one stood, in bytes.
        lowest: u64,
    },
    /// A kind from a later release, which this one reads only by its sentence. The daemon never sends it.
    #[serde(other)]
    Unknown,
}

/// What auto sizing does about a job near its cap.
#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
#[serde(rename_all = "kebab-case")]
pub enum AutoNote {
    /// It raises the profile's `memory` when a run meets the cap.
    Raises,
    /// The profile is at its `max_memory`: it cannot.
    AtMax,
    /// It is off.
    Off,
}

/// One run of a job, to open in the history or on GitHub.
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
pub struct RunLink {
    /// GitHub's job ID, the history's too.
    pub job: u64,
    /// The workflow run.
    pub run_id: u64,
    /// The run's attempt.
    pub run_attempt: u32,
    /// The branch.
    pub branch: Option<String>,
    /// The job's page on GitHub.
    pub url: Option<String>,
    /// When it finished.
    pub at: u64,
}

/// A repository's waits this week against the week before.
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
pub struct RepositoryWaits {
    /// `owner/name`.
    pub repository: String,
    /// The jobs that started in the last 7 days.
    pub now: WeekWaits,
    /// The jobs that started in the 7 days before.
    pub before: WeekWaits,
}

/// The waits of the jobs that started in a week, from queued to started, leaving out the time before the history's
/// start and the time no runnerd ran (ADR 0081).
#[derive(Debug, Clone, Default, PartialEq, Eq, Serialize, Deserialize)]
pub struct WeekWaits {
    /// Every job.
    pub all: WaitSpread,
    /// The jobs that found a runner already started: a warm runner (ADR 0050).
    pub warm: WaitSpread,
    /// The jobs whose runner started after they were queued.
    pub cold: WaitSpread,
}

/// How long some jobs waited.
#[derive(Debug, Clone, Default, PartialEq, Eq, Serialize, Deserialize)]
pub struct WaitSpread {
    /// How many.
    pub jobs: u32,
    /// The median, in seconds; `None` without jobs.
    pub p50: Option<u64>,
    /// The 95th percentile by nearest rank, in seconds; `None` without jobs.
    pub p95: Option<u64>,
}

/// `GET /v1/digest`: the last 24 hours.
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
pub struct Digest {
    /// 24 hours before `until`.
    pub since: u64,
    /// When it was read.
    pub until: u64,
    /// Jobs that finished: completed, or lost with their runner.
    pub finished: u64,
    /// Of those, the ones that failed, timed out or were lost.
    pub failed: u64,
    /// The insights that hold now and did not 24 hours ago.
    pub new: Vec<Insight>,
    /// The three longest waits of the jobs that started, longest first.
    pub waits: Vec<LongWait>,
    /// What the month's job minutes would have cost on GitHub's runners (ADR 0072); `None` when the history could
    /// not say.
    #[serde(default, skip_serializing_if = "Option::is_none")]
    pub savings: Option<super::SavedWindow>,
    /// What the host's memory came to while CI ran (ADR 0108); `None` when no runner ran with a reading of it.
    #[serde(default, skip_serializing_if = "Option::is_none")]
    pub host: Option<HostHazard>,
}

/// A job that waited.
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
pub struct LongWait {
    /// GitHub's job ID.
    pub job: u64,
    /// `owner/name`.
    pub repository: String,
    /// The workflow's name.
    pub workflow: Option<String>,
    /// The job's name.
    pub name: String,
    /// Seconds from queued to started, leaving out the time before the history's start and the time no runnerd
    /// ran (ADR 0081).
    pub waited: u64,
    /// The job's page on GitHub.
    pub url: Option<String>,
}

/// What a notification target is sent, and the events `[notify]` targets choose from.
#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Serialize, Deserialize)]
#[serde(rename_all = "kebab-case")]
pub enum NotifyEvent {
    /// A job failed or timed out.
    Failed,
    /// A job's runner vanished before GitHub said the job completed.
    Lost,
    /// A runner could not be registered or started, and its repository is held back.
    Runner,
    /// A new warning insight.
    Insights,
    /// The daily digest.
    Digest,
    /// `runnerctl notify test`; not one a target chooses.
    Test,
}

/// How urgent a notification is, as ntfy ranks it.
#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
#[serde(rename_all = "kebab-case")]
pub enum Priority {
    /// ntfy's 2: no sound.
    Low,
    /// ntfy's 3.
    Default,
    /// ntfy's 4: a longer vibration.
    High,
}

/// What a webhook target is posted, as JSON; an ntfy target gets its title, message, priority and link.
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
pub struct Notification {
    /// What it is about.
    pub event: NotifyEvent,
    /// One short line.
    pub title: String,
    /// A few lines.
    pub message: String,
    /// What to open: a job's page on GitHub, or nothing.
    pub url: Option<String>,
    /// How urgent.
    pub priority: Priority,
    /// The jobs it is about: failed or lost.
    #[serde(default, skip_serializing_if = "Vec::is_empty")]
    pub jobs: Vec<JobSummary>,
    /// The runners that could not start.
    #[serde(default, skip_serializing_if = "Vec::is_empty")]
    pub runners: Vec<RunnerFailure>,
    /// The insights it is about.
    #[serde(default, skip_serializing_if = "Vec::is_empty")]
    pub insights: Vec<Insight>,
    /// The digest.
    #[serde(default, skip_serializing_if = "Option::is_none")]
    pub digest: Option<Digest>,
}

/// `POST /v1/notify/test` query parameters.
#[derive(Debug, Clone, Default, PartialEq, Eq, Serialize, Deserialize)]
pub struct NotifyTestQuery {
    /// Only the target with this name; default every one.
    pub target: Option<String>,
}

/// `POST /v1/notify/test`: what each target did with a test message.
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
pub struct NotifyTest {
    /// In the config's order.
    pub targets: Vec<Delivery>,
}

/// What one target did with a message.
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
pub struct Delivery {
    /// The target's name.
    pub target: String,
    /// Why it was not delivered: an HTTP status or a connection error, never the URL. `None` when it was.
    pub error: Option<String>,
}