runnerd_core/api.rs
1//! The daemon's JSON API, version 1: the request and response types every client deserializes, and the
2//! principal a transport authenticates. Plain data only: no I/O, no Docker or GitHub types, no secrets but a
3//! [`Pairing`]'s one-time link, which its `Debug` leaves out.
4//!
5//! Routes, all under the authenticated principal:
6//!
7//! | Route | Body |
8//! |---|---|
9//! | `GET /v1/status` | [`Status`] |
10//! | `POST /v1/repositories/{owner}/{name}/pause?`[`PauseQuery`] | [`Paused`]; 409 [`Busy`] while it runs a job |
11//! | `POST /v1/repositories/{owner}/{name}/resume` | none, 204 |
12//! | `POST /v1/repositories/pause?`[`PauseQuery`] | [`Paused`]; 409 [`Busy`] while any runs a job, pausing none |
13//! | `POST /v1/repositories/resume` | none, 204: every repository resumed |
14//! | `POST /v1/profiles/{name}/reseed` | none, 202; 409 while it reseeds; `cache.reseeded` when done |
15//! | `POST /v1/caches/{name}/clean` | none, 204; 409 [`Busy`] while a runner uses it |
16//! | `GET /v1/config` | [`ConfigFile`] |
17//! | `POST /v1/config/check` | TOML in, [`ConfigCheck`] |
18//! | `PUT /v1/config` | TOML in, [`ConfigCheck`]; 422 [`ConfigCheck`] when invalid |
19//! | `GET /v1/doctor` | [`Doctor`]; 503 [`ErrorResponse`] while no run has finished |
20//! | `GET /v1/attention` | [`Attention`]; 404 from a daemon before 0.2 |
21//! | `POST /v1/shutdown` | [`ShutdownRequest`] in, 202 [`ShutdownAnswer`] |
22//! | `GET /v1/dashboard` | [`Dashboard`] |
23//! | `POST /v1/dashboard/sign-out` | none, 204 |
24//! | `POST /v1/dashboard/pairings?`[`PairingQuery`] | [`Pairings`]; 409 when the dashboard is off or has no `[web] origins` |
25//! | `GET /v1/jobs?`[`JobQuery`] | [`JobPage`] |
26//! | `GET /v1/jobs/{id}` | [`Job`] |
27//! | `GET /v1/jobs/{id}/samples` | [`SampleSeries`] |
28//! | `GET /v1/jobs/{id}/logs?`[`LogQuery`] | `text/plain`, one line per log line |
29//! | `GET /v1/host/samples?`[`HostSampleQuery`] | [`HostSamples`] |
30//! | `GET /v1/stats?`[`StatsQuery`] | [`Stats`] |
31//! | `GET /v1/activity?`[`ActivityQuery`] | [`Activity`] |
32//! | `GET /v1/sizing?`[`SizingQuery`] | [`Sizing`] |
33//! | `GET /v1/insights?`[`InsightsQuery`] | [`Insights`] |
34//! | `GET /v1/digest` | [`Digest`] |
35//! | `GET /v1/savings` | [`Savings`] |
36//! | `POST /v1/notify/test?`[`NotifyTestQuery`] | [`NotifyTest`]; 404 without `[notify]` targets or with no such one |
37//! | `GET /v1/caches` | [`Caches`] |
38//! | `GET /v1/caches/history?`[`CacheHistoryQuery`] | [`CacheHistory`] |
39//! | `GET /v1/events?`[`EventQuery`] | `text/event-stream` of [`EventEnvelope`] |
40//! | `GET /v1/records` | [`Records`] |
41//! | `DELETE /v1/records/{job}` | [`RemovedRecords`]; 409 while a shell is open in it |
42//! | `DELETE /v1/records?`[`RemoveRecordsQuery`] | [`RemovedRecords`]; records with an open shell stay |
43//! | `POST /v1/records/{job}/shell?`[`ShellQuery`] | `Upgrade:` [`SHELL_PROTOCOL`], 101, then the shell's terminal; 409 when the budget is full |
44//!
45//! Errors carry an [`ErrorResponse`].
46
47use serde::{Deserialize, Serialize};
48
49mod actions;
50mod admission;
51mod attention;
52mod budget;
53mod control;
54mod disk;
55mod events;
56mod figure;
57mod insights;
58mod jobs;
59mod labels;
60mod records;
61mod results;
62mod savings;
63mod sizing;
64mod slots;
65mod stats;
66mod storage;
67mod tone;
68mod units;
69
70pub use actions::{
71 ActionAnswer, ActionOutcome, ActionPreview, ActionQuery, ActionRefusal, ActionResult,
72 ActionTarget, CONFIRM_SECS, JOB_ACTIONS_CHECK, JobAction, JobActions, PreviewFacts, RunSeen,
73 SIBLINGS_MAX, Superseded, UnknownCause, actions_in_flight, actions_no_run,
74 actions_not_configured, actions_off,
75};
76pub use admission::{Admission, AdmissionPolicy, Basis, CpuAdmission, JobPeak, cores, cpus};
77pub use attention::{
78 ATTENTION_MAX, Attention, AttentionItem, AttentionKind, Fix, Subject, attention_items,
79};
80pub use budget::{
81 BudgetParts, ClosedKind, ClosedPart, FreePart, HeldKind, HeldPart, HostFigure, Reading,
82 SlotForm, SlotPart, SlotReading,
83};
84pub use control::{
85 Busy, Check, CheckCount, CheckStatus, ConfigCheck, ConfigFile, Doctor, DoctorQuery,
86 DoctorState, DoctorSummary, PauseQuery, Paused, ReplaceQuery, RunningJob,
87};
88pub use disk::{
89 Commits, DiskWrites, DiskWritten, DriveShare, DriveShares, DriveWrites, JobWrites, Measure,
90 ShareOwner, WriteSection, WriteTop,
91};
92pub use events::{Event, EventEnvelope, EventQuery, RunnerFailure};
93pub use figure::LabelledFigure;
94pub use insights::{
95 AutoNote, Delivery, Digest, HostHazard, HostSpan, INSIGHT_DAYS, INSIGHT_DAYS_MAX, Insight,
96 InsightFigure, InsightKind, Insights, InsightsHeading, InsightsQuery, LongWait, Notification,
97 NotifyEvent, NotifyTest, NotifyTestQuery, Priority, RepositoryWaits, RunLink, Severity,
98 WaitFigure, WaitSpread, WeekWaits,
99};
100#[cfg(feature = "host")]
101pub(crate) use jobs::SAMPLE_GAP_INTERVALS;
102#[cfg(feature = "host")]
103pub(crate) use jobs::TITLE_SOURCE_MAX;
104pub use jobs::{
105 Activity, ActivityQuery, Bucket, BuildCacheCounts, BuildCacheStats, CacheHistory,
106 CacheHistoryQuery, CacheKind, CacheSize, CacheVolume, Caches, ChartLine, ChartLineKind,
107 ChartLines, Holder, HostMemory, HostSample, HostSampleQuery, HostSamples, Job, JobMetrics,
108 JobPage, JobQuery, JobRunner, JobState, JobSummary, JobTitle, Limit, LimitSource, Limits,
109 LogQuery, LogSummary, Milestone, Outcome, OutcomeKind, Phase, PhaseKind, Pressure,
110 RunComparison, Sample, SampleSeries, SampleUsage, Started, Stats, StatsQuery, StatsRow, Step,
111 Swap, Timeline, Total, host_margin, job_title,
112};
113pub use jobs::{
114 BarEnd, Baseline, BaselineRead, COMPARE_STEP_BYTES, COMPARE_STEPS_MAX, ChangeKind,
115 CompareQuery, ComparedSteps, FigureDelta, FigureKind, Gap, GapKind, JOB_LABELS_MAX,
116 JOB_SEARCH_MAX, JOB_TEXT_MAX, JOBS_PAGE_BYTES, JOBS_PAGE_DEFAULT, JOBS_PAGE_MAX, JobComparison,
117 JobList, JobOrder, JobRange, JobSort, Lane, RecordFilter, RecordMark, RunnerSpan, SlotKey,
118 SlotTimeline, SlottedBar, SlottedRunner, SortOrder, Span, StepDelta, StepMatch, StepUsage,
119 TIMELINE_BARS_MAX, TIMELINE_DEFAULT, TIMELINE_WINDOW_MAX, TimelineBar, TimelineQuery,
120 TimelineRows, Unavailable, row_bytes,
121};
122pub use jobs::{
123 Color, FailureSummary, LINE_BYTES_MAX, LINES_DEFAULT, LINES_MAX, LineKind, LineRange, LineRead,
124 LogEvidence, LogGroup, LogLinesQuery, LogMark, LogNeedle, LogOutline, LogSearch,
125 LogSearchQuery, MarkKind, OutlineChange, OutlineParts, Outliner, RUNS_MAX, Run,
126 SEARCH_QUERY_MAX, SavedOutline, Search, StepLines, StepLog, StoredLine, StoredLog, StyledLine,
127};
128pub use labels::{Column, Feature, JOBS_HEADING, NOT_SUPPORTED, Section, View};
129pub use records::{
130 Record, RecordLimits, RecordNote, RecordRemoval, Records, RemoveRecordsQuery, RemovedRecords,
131 SHELL_PROTOCOL, ShellQuery,
132};
133pub use results::{force_pause_consequence, reseed_requested, reseeded};
134pub use savings::{
135 Period, PricedRun, PricedRunner, Pricing, PricingHistory, SELF_HOSTED_RATE, SavedWindow,
136 Savings, Sku, Usd, Visibility, billed_minutes, reads_since,
137};
138pub use sizing::{
139 Advice, AutoPlan, GRANULE, JobSizing, MemoryBound, MemoryHold, Need, PAGE_CACHE_ROOM, Peaks,
140 ProfilePlan, ProfileSizing, Resize, RevertCause, SAMPLE_INTERVAL, SLOT_DISK, Sizing,
141 SizingChange, SizingHold, SizingQuery, SizingRefused, SizingRevert, Spread, Starts, need,
142 period, sampled_note,
143};
144pub use slots::{Figure, Health, HealthKind, SlotState};
145pub use stats::StatsSummary;
146pub use storage::{
147 ACTIONS_CACHE, BUILD_CACHE_TOOLS, BuildCacheActivity, CacheLimit, RECORDS_NEAR_FULL,
148 RecordState, actions_cache_place, records_check, records_near_full,
149};
150pub use tone::{Tone, Toned};
151pub(crate) use units::days_from_civil;
152pub use units::{ago, cores_used, cpu_time, duration, in_cores};
153
154/// Who sent a request, as the transport authenticated it. Handlers get it from one place, so a new transport only
155/// has to produce one.
156#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
157#[serde(rename_all = "kebab-case", tag = "kind")]
158pub enum Principal {
159 /// The user the daemon runs as, on the local Unix socket.
160 LocalOwner {
161 /// The Unix user ID the socket's peer runs as.
162 uid: u32,
163 },
164}
165
166/// `GET /v1/status`: what the daemon serves and runs right now.
167#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
168pub struct Status {
169 /// The daemon's version, from its `Cargo.toml`.
170 pub version: String,
171 /// Set once the daemon stops polling to shut down. [`Status::stopping_line`] says how.
172 pub shutting_down: bool,
173 /// How it stops, once it does; absent from a daemon before it said.
174 #[serde(default, skip_serializing_if = "Option::is_none")]
175 pub stopping: Option<Stopping>,
176 /// Until when GitHub's rate limit is used up, in seconds since the Unix epoch: no runner starts before.
177 /// [`Status::github_line`] says it. Absent while it is not.
178 #[serde(default, skip_serializing_if = "Option::is_none")]
179 pub github_limited_until: Option<u64>,
180 /// The memory budget and what runners use of it.
181 pub budget: Budget,
182 /// Every configured profile with its slots.
183 pub profiles: Vec<ProfileStatus>,
184 /// Every configured repository.
185 pub repositories: Vec<RepositoryStatus>,
186 /// Queued jobs the repositories' runners can take and none holds yet, at the last poll, oldest first. No event
187 /// carries the list: `job.queued` announces a job, `job.started` and `job.finished` its leaving the queue.
188 /// A repository whose last listing failed, as while GitHub rate-limits runnerd, or that is no longer polled
189 /// because runnerd is shutting down, shows no jobs here, while the history keeps its jobs queued.
190 pub queued: Vec<QueuedJob>,
191 /// Whether the daemon cancels and re-runs jobs (ADR 0134); a daemon before 0.2 sends none, read as
192 /// [`JobActions::Unsupported`].
193 #[serde(default)]
194 pub job_actions: JobActions,
195}
196
197/// The memory budget runners and the mirror share, in bytes.
198#[derive(Debug, Clone, Default, PartialEq, Eq, Serialize, Deserialize)]
199pub struct Budget {
200 /// The systemd slice runner containers run in.
201 pub slice: String,
202 /// The slice's `MemoryMax`; `None` when it is unlimited.
203 pub memory_max: Option<u64>,
204 /// Memory the running containers reserve: the runners' caps, or under measured admission the larger of each
205 /// runner's predicted peak and its last sampled memory, nothing for one whose container is gone, and the caps of
206 /// the seed containers, shells and the mirror.
207 pub memory_in_use: u64,
208 /// What pooling takes off `memory_in_use` (ADR 0099, 0130); absent while it takes nothing.
209 #[serde(default, skip_serializing_if = "Option::is_none")]
210 pub pooled: Option<Pooled>,
211 /// The growth count both budgets go by, and its brake (ADR 0130); `pooled` and `host.credit` say what it takes off
212 /// `memory_in_use`. Absent under caps admission, with `[host] pooled_count` off, and from an older daemon.
213 #[serde(default, skip_serializing_if = "Option::is_none")]
214 pub counted: Option<Counted>,
215 /// What a new runner must fit in beside them.
216 pub admission: AdmissionPolicy,
217 /// Under measured admission, whether the slice is at the high-water mark, so no runner starts.
218 pub held: bool,
219 /// The mirror's part of `memory_in_use`: its cap while it is enabled or still running, else 0.
220 pub mirror_memory: u64,
221 /// The seed containers' part of `memory_in_use`, one per profile that reseeds, by profile name.
222 pub seed_memory: Vec<SeedMemory>,
223 /// The shells' part of `memory_in_use`, one per shell open in a record, by job.
224 pub shell_memory: Vec<ShellMemory>,
225 /// What the host's memory leaves runners beside its other programs, from a recent reading of it; absent
226 /// without one.
227 #[serde(default, skip_serializing_if = "Option::is_none")]
228 pub host: Option<HostBudget>,
229 /// The CPU budget, under `[host] cpu_overcommit`. Absent without one.
230 #[serde(default, skip_serializing_if = "Option::is_none")]
231 pub cpu: Option<CpuBudget>,
232 /// The CPUs the slice may use, in millicores: its `AllowedCPUs` or the host's, less under a lower `CPUQuota`,
233 /// whether or not a CPU budget admits runners. The runners' CPU limits are quotas that may add up past it.
234 /// Absent when it could not be read.
235 #[serde(default, skip_serializing_if = "Option::is_none")]
236 pub slice_cpus: Option<u64>,
237}
238
239/// The count the budgets go by while pooling takes something off `Budget::memory_in_use` (ADR 0099).
240#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
241pub struct Pooled {
242 /// What it takes off, in bytes.
243 pub credit: u64,
244 /// `Budget::memory_in_use` less `credit`: what new runners fit beside under `MemoryMax`. The host's limit reads
245 /// `memory_in_use`.
246 pub counted: u64,
247 /// How many runners the credit counts on.
248 pub runners: u64,
249}
250
251/// How far back the growth count's brake compares what runs uses: 10 minutes (ADR 0130,
252/// `admission::growth::GROWTH_WINDOW`).
253pub const GROWTH_WINDOW: std::time::Duration = std::time::Duration::from_mins(10);
254
255/// The growth count (ADR 0130): what the runners use, and the room kept for what they may still grow by.
256#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
257pub struct Counted {
258 /// What the runners use, by their recent samples, in bytes.
259 pub used: u64,
260 /// The room it keeps for their growth, in bytes; absent while it rests (`brake`).
261 #[serde(default, skip_serializing_if = "Option::is_none")]
262 pub kept: Option<u64>,
263 /// How many of the largest growths the room covers at least.
264 pub jobs: u64,
265 /// Why it rests, while it does: the budgets then go by 0099's and 0108's credits.
266 #[serde(default, skip_serializing_if = "Option::is_none")]
267 pub brake: Option<CountBrake>,
268}
269
270/// Why the growth count rests (ADR 0130).
271#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
272#[serde(rename_all = "kebab-case", tag = "kind")]
273pub enum CountBrake {
274 /// What ran grew past the room kept within `GROWTH_WINDOW`.
275 Grew {
276 /// How much the runners present at both looks grew, together, in bytes.
277 grew: u64,
278 /// The room the earlier look kept, in bytes.
279 kept: u64,
280 /// Until when, in seconds since the Unix epoch.
281 until: u64,
282 },
283 /// The daemon started, or a reload changed what the count reads, too recently to judge growth.
284 Restarted {
285 /// Until when, in seconds since the Unix epoch.
286 until: u64,
287 },
288 /// The host is under the guard's line, or a reading is missing, now or at a look within 5 minutes.
289 Guard {
290 /// Until when, in seconds since the Unix epoch; absent for a moment the brake has not looked at yet.
291 #[serde(default, skip_serializing_if = "Option::is_none")]
292 until: Option<u64>,
293 },
294 /// A brake a newer daemon names; this client shows only that the count rests.
295 #[serde(other)]
296 Unknown,
297}
298
299/// What the host's memory leaves the runners (ADR 0055). New runners fit in the lower of `MemoryMax` and
300/// `limit`; running ones are never stopped for it.
301#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
302pub struct HostBudget {
303 /// The reading it comes from.
304 pub memory: HostMemory,
305 /// What the slice may hold now: what its runners hold without page cache plus the host's `MemAvailable`, less
306 /// a margin kept free for the host's page cache; with a ceiling, plus what the host can still grow by
307 /// (`HostMemory::growable`, ADR 0084).
308 pub limit: u64,
309 /// What that leaves new runners beside what the running ones reserve, in bytes.
310 pub spare: u64,
311 /// Whether the host's tasks stall for memory (`full` pressure), so no runner starts until they stop.
312 pub stalled: bool,
313 /// What the host's limit takes off `Budget::memory_in_use` for runners that may not all grow at once (ADR 0108);
314 /// absent while it takes nothing, and from a daemon before 0.0.30.
315 #[serde(default, skip_serializing_if = "Option::is_none")]
316 pub credit: Option<HostCredit>,
317}
318
319/// What the host's limit takes off the plain count (ADR 0108): `Budget::memory_in_use` less `bytes` is what it counts,
320/// `used` of that is what the runners use, and the rest is room kept for what runs to grow into.
321#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
322pub struct HostCredit {
323 /// What it takes off, in bytes.
324 pub bytes: u64,
325 /// What the runners use, by their recent samples, in bytes.
326 pub used: u64,
327}
328
329impl HostBudget {
330 /// Whether the host itself lacks `needs` bytes for a new runner: its tasks stall, `HostMemory::room` is less, or
331 /// that room is under one more margin, so `MemAvailable` is under twice the plain margin, the line `host::stalls`
332 /// reads. There a small job fits only by what the host is about to run out of, and what closing programs frees is
333 /// what starts it. Otherwise a job the host limit holds waits for what runs, whose reservations reach the limit
334 /// (ADR 0103).
335 pub fn short_of(&self, needs: u64) -> bool {
336 let room = self.memory.room();
337 self.stalled || needs > room || room < host_margin(self.memory.grown())
338 }
339
340 /// What the host's limit counts of what runs, `reserved` being `Budget::memory_in_use`: that less the credit
341 /// (ADR 0108).
342 pub fn counts(&self, reserved: u64) -> u64 {
343 reserved.saturating_sub(self.credit.map_or(0, |credit| credit.bytes))
344 }
345
346 /// What that count is made of while a credit stands, as the host's line, a job's wait and the log say it: `it uses
347 /// 14G and keeps 9G for growth`. `None` without a credit, when it counts every reservation.
348 pub fn growth_clause(&self, reserved: u64) -> Option<String> {
349 let credit = self.credit?;
350 let counts = self.counts(reserved);
351 Some(format!(
352 "it uses {} and keeps {} for growth",
353 crate::config::ByteSize(credit.used.min(counts)),
354 crate::config::ByteSize(counts.saturating_sub(credit.used))
355 ))
356 }
357}
358
359/// Why a job waits for the host's limit, needing `needs` bytes with `free` left of it, by the latest reading: for what
360/// runs, or for a host that lacks the room, whose programs it names. While the host's limit takes a credit, what runs
361/// is what it uses and the room it keeps for growth, not its reservations (ADR 0108); `reserved` is
362/// `Budget::memory_in_use`.
363fn host_reason(needs: u64, free: u64, host: Option<&HostBudget>, reserved: u64) -> String {
364 let size = |bytes: u64| crate::config::ByteSize(bytes).to_string();
365 if let Some(host) = host.filter(|host| !host.short_of(needs)) {
366 if let Some(growth) = host.growth_clause(reserved) {
367 return format!(
368 "The host has {}, but what runs may still grow into it: {growth}; needs {}, {} is free",
369 host.memory.available_line(),
370 size(needs),
371 size(free)
372 );
373 }
374 return format!(
375 "The host has {}, but what runs may still grow into it: needs {}; {} of the {} it leaves the slice is free",
376 host.memory.available_line(),
377 size(needs),
378 size(free),
379 size(host.limit)
380 );
381 }
382 let reason = if host.is_some_and(|host| host.stalled) {
383 "The host is short of memory: its programs stall waiting for it, so nothing starts"
384 .to_owned()
385 } else {
386 format!(
387 "The host is short of memory: needs {}; the host has {} to spare",
388 size(needs),
389 size(free)
390 )
391 };
392 match host {
393 Some(host) => format!("{reason}; {}", host.memory.outside_line()),
394 None => reason,
395 }
396}
397
398/// Hundredths of a percent as a percentage: `1.5%`, `12%`.
399pub fn percent(hundredths: u32) -> String {
400 let whole = hundredths / 100;
401 match hundredths % 100 / 10 {
402 0 => format!("{whole}%"),
403 tenth => format!("{whole}.{tenth}%"),
404 }
405}
406
407/// The CPU budget runners share, in millicores.
408#[derive(Debug, Clone, Copy, Default, PartialEq, Eq, Serialize, Deserialize)]
409pub struct CpuBudget {
410 /// The CPUs the slice may use: its `AllowedCPUs` or the host's, less under a lower `CPUQuota`.
411 pub cpus: u64,
412 /// Those times `cpu_overcommit`: what the runners' reservations must fit.
413 pub budget: u64,
414 /// What the runners reserve: each its predicted average CPU under measured admission, else its `cpus`.
415 pub in_use: u64,
416 /// Whether a queued job waits for CPU that memory and a slot have room for.
417 pub held: bool,
418}
419
420impl CpuBudget {
421 /// The budget's line as clients show it beside memory, such as `CPU 4.5 of 32 CPUs reserved` or, while a job
422 /// waits for it, `CPU 30 of 32 CPUs reserved; a job waits for CPU`.
423 pub fn line(&self) -> String {
424 let mut line = format!(
425 "CPU {} of {} CPUs reserved",
426 cores(self.in_use),
427 cores(self.budget)
428 );
429 if self.held {
430 line.push_str("; a job waits for CPU");
431 }
432 line
433 }
434}
435
436impl Budget {
437 /// How runners are admitted, as clients add it to the budget's line: nothing under caps, else `by predicted
438 /// peaks`, and why nothing starts while the slice is at the high-water mark.
439 pub fn admission_note(&self) -> Option<&'static str> {
440 match (self.admission, self.held) {
441 (AdmissionPolicy::Caps, _) => None,
442 (AdmissionPolicy::Measured, false) => Some("by predicted peaks"),
443 (AdmissionPolicy::Measured, true) => Some(
444 "by predicted peaks; the slice is past 90% of its MemoryMax, so no runner starts",
445 ),
446 }
447 }
448}
449
450/// A shell container's memory cap: its record's profile's limit when the shell opened.
451#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
452pub struct ShellMemory {
453 /// The job whose record the shell runs.
454 pub job: u64,
455 /// Its cap in bytes.
456 pub memory: u64,
457}
458
459/// A seed container's memory cap: its profile's limit when the reseed started, or, for a requested reseed whose
460/// profile drained, the room it holds until it starts.
461#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
462pub struct SeedMemory {
463 /// The profile it reseeds.
464 pub profile: String,
465 /// Its cap in bytes.
466 pub memory: u64,
467}
468
469/// A runner profile and what runs in its slots.
470#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
471pub struct ProfileStatus {
472 /// The name under `[profiles]` in `runnerd.toml`.
473 pub name: String,
474 /// The image its runners run, such as `runnerd-runner:base`.
475 pub image: String,
476 /// CPUs per slot.
477 pub cpus: u32,
478 /// Bytes per slot.
479 pub memory: u64,
480 /// Set while slot 1's layers become the profile's snapshot.
481 pub reseeding: bool,
482 /// Index 0 is slot 1.
483 pub slots: Vec<SlotStatus>,
484}
485
486/// One slot of a profile: room for one job.
487#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
488pub struct SlotStatus {
489 /// 1-based.
490 pub slot: usize,
491 /// `None` when the slot is free.
492 pub runner: Option<RunnerStatus>,
493}
494
495/// A runner registered for one job, in its container.
496#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
497pub struct RunnerStatus {
498 /// `owner/name` the runner is registered on.
499 pub repository: String,
500 /// The runner's name, also its container's.
501 pub name: String,
502 /// The runner's ID on GitHub.
503 pub runner_id: u64,
504 /// The Docker container ID.
505 pub container: String,
506 /// Whether it runs a job, as GitHub or its log reports it: its log names the job before GitHub lists it busy.
507 pub busy: bool,
508 /// Seconds since the Unix epoch.
509 pub started_at: u64,
510 /// What it takes of the budget: its cap under caps; under measured admission the larger of its predicted peak
511 /// and its last sampled memory.
512 pub reserved: u64,
513 /// A warm runner: idle, and no queued job of its repository waits for it; it takes the next one GitHub hands
514 /// out.
515 pub warm: bool,
516}
517
518/// A configured repository and its queue.
519#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
520pub struct RepositoryStatus {
521 /// `owner/name` on GitHub.
522 pub name: String,
523 /// Its pools in config order, one per `[[repositories]]` entry: a job runs in the one whose labels match it
524 /// (ADR 0058).
525 pub pools: Vec<PoolStatus>,
526 /// Paused repositories get no new runners; their running jobs finish.
527 pub paused: bool,
528 /// Its jobs in `Status::queued`: queued ones none of its runners took yet, those no pool serves too.
529 pub queued_jobs: usize,
530 /// Why its queued jobs cannot be listed, while the listings fail (ADR 0132); absent while they answer, before the
531 /// first, while GitHub's rate limit stops them, and from a daemon before 0.2.
532 #[serde(default, skip_serializing_if = "Option::is_none")]
533 pub listing: Option<ListingFailure>,
534}
535
536/// Why runnerd cannot list a repository's queued jobs (ADR 0132): what class of failure GitHub's answer was, never its
537/// text. The engine keeps it until a listing answers or a reload removes the repository; a restart forgets it.
538#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
539pub struct ListingFailure {
540 /// When the listings began to fail, in seconds since the Unix epoch.
541 pub since: u64,
542 /// Why the last one failed.
543 pub problem: ListingProblem,
544}
545
546/// What a failed listing ran into.
547#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
548#[serde(rename_all = "kebab-case", tag = "kind")]
549pub enum ListingProblem {
550 /// runnerd could read no GitHub token.
551 NoToken,
552 /// GitHub refused the token: 401.
553 Unauthorized,
554 /// GitHub answered 403, 404 or 410: the token may not read the repository's Actions, cannot see it, or it is gone.
555 Refused {
556 /// The HTTP status.
557 status: u16,
558 },
559 /// GitHub answered another error status, such as a 5xx.
560 Failed {
561 /// The HTTP status.
562 status: u16,
563 },
564 /// No answer came: the network, DNS or TLS failed.
565 Unreachable,
566 /// The answer was not what GitHub sends.
567 Unexpected,
568 /// A problem from a newer daemon than this client knows.
569 #[serde(other)]
570 Unknown,
571}
572
573impl ListingProblem {
574 /// Whether it is the token's, and so every repository's: none read, or GitHub refused it.
575 pub fn is_token(&self) -> bool {
576 matches!(self, ListingProblem::NoToken | ListingProblem::Unauthorized)
577 }
578}
579
580impl ListingFailure {
581 /// What fails, as every client says it: `Cannot list halyard/ledger's queued jobs: GitHub answers 404; the token
582 /// cannot see the repository, or it is gone`.
583 pub fn line(&self, repository: &str) -> String {
584 format!(
585 "Cannot list {repository}'s queued jobs: {}",
586 self.problem.why()
587 )
588 }
589}
590
591impl ListingProblem {
592 /// Why the listing fails, as the daemon's log and every client say it: `GitHub answers 502`. Never GitHub's text.
593 pub fn why(&self) -> String {
594 match self {
595 ListingProblem::NoToken => "runnerd has no GitHub token".to_owned(),
596 ListingProblem::Unauthorized => "GitHub refuses runnerd's token (401)".to_owned(),
597 ListingProblem::Refused { status: 403 } => {
598 "GitHub answers 403; the token may not read the repository's Actions".to_owned()
599 }
600 ListingProblem::Refused { status } => format!(
601 "GitHub answers {status}; the token cannot see the repository, or it is gone"
602 ),
603 ListingProblem::Failed { status } => format!("GitHub answers {status}"),
604 ListingProblem::Unreachable => "GitHub does not answer".to_owned(),
605 ListingProblem::Unexpected => "GitHub's answer is not what runnerd reads".to_owned(),
606 ListingProblem::Unknown => "for a reason this client does not know".to_owned(),
607 }
608 }
609}
610
611/// One of a repository's pools: its runners in one profile.
612#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
613pub struct PoolStatus {
614 /// The profile its jobs run in.
615 pub profile: String,
616 /// The labels its runners register with.
617 pub labels: Vec<String>,
618 /// Its jobs in `Status::queued`: those its labels match.
619 pub queued_jobs: usize,
620}
621
622/// A queued job runners of this daemon can take, as GitHub listed it at the last poll.
623#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
624pub struct QueuedJob {
625 /// GitHub's job ID.
626 pub id: u64,
627 /// The repository whose runners can take it, `owner/name`.
628 pub repository: String,
629 /// The profile of its repository's pool whose labels match it; `None` when no pool's do.
630 #[serde(default, skip_serializing_if = "Option::is_none")]
631 pub profile: Option<String>,
632 /// The workflow run it belongs to.
633 pub run_id: u64,
634 /// 1 for the first attempt of the run, higher for re-runs.
635 pub run_attempt: u32,
636 /// The workflow's name.
637 pub workflow: Option<String>,
638 /// The job's name in the workflow.
639 pub name: String,
640 /// The branch the run is for.
641 pub branch: Option<String>,
642 /// The job's page on GitHub.
643 pub url: Option<String>,
644 /// When GitHub queued it, in seconds since the Unix epoch; 0 if GitHub did not say.
645 pub queued_at: u64,
646 /// Why no runner has it yet, as the last plan saw it. `None` before the daemon has planned with it, and while
647 /// it shuts down.
648 #[serde(default, skip_serializing_if = "Option::is_none")]
649 pub waiting: Option<Waiting>,
650}
651
652/// The slot of an idle runner of `job`'s repository, in its pool's profile when known, such as `jvm 2`.
653fn starting_for(job: &QueuedJob, status: &Status) -> Option<String> {
654 status
655 .profiles
656 .iter()
657 .filter(|profile| job.profile.as_ref().is_none_or(|own| *own == profile.name))
658 .find_map(|profile| {
659 profile
660 .slots
661 .iter()
662 .find(|slot| {
663 slot.runner
664 .as_ref()
665 .is_some_and(|runner| runner.repository == job.repository && !runner.busy)
666 })
667 .map(|slot| format!("{}\u{a0}{}", profile.name, slot.slot))
668 })
669}
670
671impl QueuedJob {
672 /// Where it runs, as every client names it: its repository, and its pool's profile when the repository has
673 /// several pools in `status`, such as `halyard/ledger · light`.
674 pub fn place(&self, status: &Status) -> String {
675 let pools = status
676 .repositories
677 .iter()
678 .find(|repository| repository.name == self.repository)
679 .map_or(0, |repository| repository.pools.len());
680 match self.profile.as_deref().filter(|_| pools > 1) {
681 Some(profile) => format!("{}\u{a0}· {profile}", self.repository),
682 None => self.repository.clone(),
683 }
684 }
685}
686
687impl RepositoryStatus {
688 /// The profile of its first pool, which every client colours it by.
689 pub fn first_profile(&self) -> &str {
690 self.pools.first().map_or("", |pool| pool.profile.as_str())
691 }
692
693 /// Its pools' profiles, as every client lists them on one line: `jvm` or `jvm + light`.
694 pub fn profiles(&self) -> String {
695 self.pools
696 .iter()
697 .map(|pool| pool.profile.as_str())
698 .collect::<Vec<_>>()
699 .join(" + ")
700 }
701
702 /// The profile of its pool whose labels match a job asking for `labels`, as the daemon picks it.
703 pub fn pool_for(&self, labels: &[String]) -> Option<String> {
704 self.pools
705 .iter()
706 .find(|pool| crate::config::labels_match(labels, &pool.labels))
707 .map(|pool| pool.profile.clone())
708 }
709}
710
711/// What failed, that holds a repository's runners back for a while.
712#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
713#[serde(rename_all = "kebab-case")]
714pub enum HoldCause {
715 /// Its runners ended without connecting, so without taking a job (ADR 0104).
716 NoJob,
717 /// Its runners connected and ended without a job, one after the other.
718 IdleExits,
719 /// GitHub did not register its runner.
720 Register,
721 /// Docker did not start its runner's container.
722 Start,
723}
724
725/// Why a queued job has no runner of its own yet. Jobs are served oldest first; each takes the first of these that
726/// holds for it.
727#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
728#[serde(rename_all = "kebab-case", tag = "kind")]
729pub enum Waiting {
730 /// Its repository is paused.
731 Paused,
732 /// Runners of its repository failed, so none starts before `until` (seconds since the Unix epoch).
733 Backoff {
734 /// When runners may start again.
735 until: u64,
736 /// What failed.
737 cause: HoldCause,
738 },
739 /// A runner started for it and waits for GitHub to hand it the job: a runner takes any job its labels match,
740 /// so not necessarily this one.
741 RunnerStarting,
742 /// Slot 1 of its profile is becoming the profile's snapshot, so none of the profile's slots starts.
743 Reseeding {
744 /// The profile.
745 profile: String,
746 },
747 /// Its profile's image is being pulled, so none of the profile's slots starts.
748 PullingImage {
749 /// The profile.
750 profile: String,
751 },
752 /// Every slot of its profile holds a runner.
753 NoSlot {
754 /// The profile.
755 profile: String,
756 },
757 /// What its runner would reserve does not fit the memory budget left. Under measured admission, `free` is 0
758 /// while the slice is at the high-water mark (`Budget::held`).
759 Memory {
760 /// What its runner would reserve, in bytes.
761 needs: u64,
762 /// What the budget has left, in bytes.
763 free: u64,
764 },
765 /// What its runner would reserve does not fit what the host's memory has to spare beside its other programs
766 /// (`HostBudget::spare`), though it fits the slice's budget. `free` is 0 while the host's tasks stall for memory
767 /// (`HostBudget::stalled`).
768 HostMemory {
769 /// What its runner would reserve, in bytes.
770 needs: u64,
771 /// What the host has to spare, in bytes.
772 free: u64,
773 },
774 /// What its runner would reserve does not fit the CPU budget left, in millicores.
775 Cpu {
776 /// What its runner would reserve.
777 needs: u64,
778 /// What the CPU budget has left.
779 free: u64,
780 },
781 /// An older job has waited long enough for CPU that no younger one takes CPU before it.
782 BehindCpu {
783 /// The older job's ID.
784 job: u64,
785 },
786 /// An older job has waited long enough for memory that no younger one starts before it (ADR 0096).
787 BehindMemory {
788 /// The older job's ID.
789 job: u64,
790 },
791 /// An older job is about to have a runner start for it, which takes the slot, memory or CPU this one would
792 /// otherwise start with: what holds it is that start, not what runs now.
793 BehindStart {
794 /// The older job's ID.
795 job: u64,
796 },
797 /// No pool of its repository has every label it asks for, so no runner of this daemon can take it (ADR 0058).
798 NoPool {
799 /// The labels it asks for.
800 labels: Vec<String>,
801 },
802 /// A host crash may have torn what runners wrote without syncing, and the daemon has not emptied it yet, so no
803 /// runner starts (ADR 0141, 0142).
804 Torn {
805 /// The volumes a container keeps it from removing, as of its last try: the owner's to remove once nothing
806 /// mounts them. Empty while it has not tried, or Docker failed.
807 volumes: Vec<String>,
808 },
809 /// A reason from a newer daemon than this client knows, so a reason added later does not keep the client from
810 /// reading the status.
811 #[serde(other)]
812 Unknown,
813}
814
815/// What a [`Waiting`] reason waits on, which clients draw it by.
816#[derive(Debug, Clone, Copy, PartialEq, Eq)]
817pub enum WaitKind {
818 /// Something the owner can act on: a pause, runners that failed to take a job, or labels no pool serves.
819 Act,
820 /// Something that clears by itself shortly: a runner starting, a reseed, an image pull, an older job's start.
821 Soon,
822 /// Room: a slot, memory or CPU that a running job holds.
823 Full,
824 /// A reason this client does not know (`Waiting::Unknown`), drawn neutral.
825 Other,
826}
827
828impl Waiting {
829 /// Whether runnerd cannot start the job with the config it runs with: its repository is paused, or no pool serves
830 /// its labels. Not that nothing ever will: a resume or a config change can, and another machine's runner may take
831 /// it. A backoff ends by itself, and a reason this client does not know says nothing (ADR 0132).
832 pub fn cannot_start(&self) -> bool {
833 matches!(self, Waiting::Paused | Waiting::NoPool { .. })
834 }
835
836 /// What it waits on.
837 pub fn kind(&self) -> WaitKind {
838 match self {
839 Waiting::Paused | Waiting::Backoff { .. } | Waiting::NoPool { .. } => WaitKind::Act,
840 Waiting::Torn { volumes } if !volumes.is_empty() => WaitKind::Act,
841 Waiting::Unknown => WaitKind::Other,
842 Waiting::Torn { .. }
843 | Waiting::RunnerStarting
844 | Waiting::Reseeding { .. }
845 | Waiting::PullingImage { .. }
846 | Waiting::BehindStart { .. } => WaitKind::Soon,
847 Waiting::NoSlot { .. }
848 | Waiting::Memory { .. }
849 | Waiting::HostMemory { .. }
850 | Waiting::Cpu { .. }
851 | Waiting::BehindCpu { .. }
852 | Waiting::BehindMemory { .. } => WaitKind::Full,
853 }
854 }
855
856 /// Why `job` waits, in the owner's words with the numbers that hold it, as every client says it: such as
857 /// `All 2 node slots are busy` or `Needs 9G of memory; 2.5G of the budget is free`. `status` is the one `job`
858 /// is queued in, which names the slot of a starting runner and the older job one waits behind.
859 pub fn reason(&self, job: &QueuedJob, status: &Status, now: u64) -> String {
860 let size = |bytes: u64| crate::config::ByteSize(bytes).to_string();
861 let older_name = |id: u64| {
862 status
863 .queued
864 .iter()
865 .find(|candidate| candidate.id == id)
866 .map_or_else(|| "an older job".to_owned(), |older| older.name.clone())
867 };
868 match self {
869 Waiting::Paused => format!("{} is paused", job.repository),
870 Waiting::Unknown => "Waits for a reason this client does not know yet".to_owned(),
871 Waiting::NoPool { labels } if labels.is_empty() => {
872 "It asks for no labels, so no profile serves it".to_owned()
873 }
874 Waiting::NoPool { labels } => format!(
875 "No profile serves the {} {}",
876 if labels.len() == 1 { "label" } else { "labels" },
877 labels.join(", ")
878 ),
879 Waiting::Backoff { until, cause } => format!(
880 "{}; retrying in {}",
881 match cause {
882 HoldCause::NoJob => "Runners failed to take a job",
883 HoldCause::IdleExits => "Runners ended without a job after waiting for one",
884 HoldCause::Register => "Runners cannot register with GitHub",
885 HoldCause::Start => "Runners cannot start",
886 },
887 duration(until.saturating_sub(now))
888 ),
889 Waiting::RunnerStarting => match starting_for(job, status) {
890 Some(slot) => format!("A runner is starting for it in {slot}"),
891 None => "A runner is starting for it".to_owned(),
892 },
893 Waiting::Reseeding { profile } => format!("{profile} is rebuilding its snapshot"),
894 Waiting::Torn { volumes } if volumes.is_empty() => {
895 "Emptying the caches a host crash may have torn".to_owned()
896 }
897 Waiting::Torn { volumes } => format!(
898 "A host crash may have torn caches a container holds; once nothing mounts them: docker volume rm {}",
899 volumes.join(" ")
900 ),
901 Waiting::PullingImage { profile } => format!("Pulling {profile}'s image"),
902 Waiting::NoSlot { profile } => {
903 let slots = status
904 .profiles
905 .iter()
906 .find(|candidate| &candidate.name == profile)
907 .map_or(0, |profile| profile.slots.len());
908 if slots == 1 {
909 format!("The {profile} slot is busy")
910 } else {
911 format!("All {slots} {profile} slots are busy")
912 }
913 }
914 Waiting::Memory { .. } if status.budget.held => {
915 "The slice is past 90% of its MemoryMax, so nothing starts".to_owned()
916 }
917 Waiting::Memory { needs, free } => format!(
918 "Needs {} of memory; {} of the budget is free",
919 size(*needs),
920 size(*free)
921 ),
922 Waiting::HostMemory { needs, free } => host_reason(
923 *needs,
924 *free,
925 status.budget.host.as_ref(),
926 status.budget.memory_in_use,
927 ),
928 Waiting::Cpu { needs, free } => {
929 format!(
930 "Needs {}; {} are free",
931 cpus(*needs),
932 admission::free_cores(*free)
933 )
934 }
935 Waiting::BehindCpu { job: older } => {
936 format!("Behind {}, which waits for CPU", older_name(*older))
937 }
938 Waiting::BehindMemory { job: older } => {
939 format!("Behind {}, which waits for memory", older_name(*older))
940 }
941 Waiting::BehindStart { job: older } => {
942 format!(
943 "Next after {}, whose runner starts first",
944 older_name(*older)
945 )
946 }
947 }
948 }
949}
950
951/// `GET /v1/dashboard`: the dashboard's `[web]` settings, as the daemon read them when it started.
952#[derive(Debug, Clone, Default, PartialEq, Eq, Serialize, Deserialize)]
953pub struct Dashboard {
954 /// `[web] open`: the one of `[web] origins` that `runnerctl web` opens the dashboard on; the loopback address
955 /// when none.
956 #[serde(default, skip_serializing_if = "Option::is_none")]
957 pub open: Option<String>,
958}
959
960/// `POST /v1/dashboard/pairings`: which of `[web] origins` to make a link for.
961#[derive(Debug, Clone, Default, PartialEq, Eq, Serialize, Deserialize)]
962pub struct PairingQuery {
963 /// One of `[web] origins`; each of them when none.
964 #[serde(default, skip_serializing_if = "Option::is_none")]
965 pub origin: Option<String>,
966}
967
968/// `POST /v1/dashboard/pairings`: fresh one-time links that sign a device in to the dashboard.
969#[derive(Debug, Clone, Default, PartialEq, Eq, Serialize, Deserialize)]
970pub struct Pairings {
971 /// One per origin asked for.
972 pub links: Vec<Pairing>,
973}
974
975/// A link that signs one device in to the dashboard on `origin`, and nowhere else, within `lifetime`.
976#[derive(Clone, PartialEq, Eq, Serialize, Deserialize)]
977pub struct Pairing {
978 /// One of `[web] origins`.
979 pub origin: String,
980 /// The link with its one-time code: a credential, which `Debug` leaves out.
981 pub url: String,
982 /// Seconds it works for from when it was made.
983 pub lifetime: u64,
984}
985
986impl std::fmt::Debug for Pairing {
987 fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
988 formatter
989 .debug_struct("Pairing")
990 .field("origin", &self.origin)
991 .field("lifetime", &self.lifetime)
992 .finish_non_exhaustive()
993 }
994}
995
996/// `POST /v1/shutdown`.
997#[derive(Debug, Clone, Default, PartialEq, Eq, Serialize, Deserialize)]
998pub struct ShutdownRequest {
999 /// Kill running jobs and delete their runners instead of waiting for them.
1000 #[serde(default)]
1001 pub force: bool,
1002}
1003
1004/// What `POST /v1/shutdown` answers: how the daemon stops now, which a stop asked for earlier may have made more
1005/// urgent than the request.
1006#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
1007pub struct ShutdownAnswer {
1008 /// How it stops.
1009 pub stopping: Stopping,
1010}
1011
1012/// How a stopping daemon treats running jobs (ADR 0091).
1013#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
1014#[serde(rename_all = "kebab-case")]
1015pub enum Stopping {
1016 /// `runnerd stop`: it waits for running jobs, then removes every runner.
1017 WaitsForJobs,
1018 /// A signal: it removes idle runners and leaves running jobs to the next start.
1019 LeavesJobs,
1020 /// `runnerd stop --force` or a second signal: it kills running jobs.
1021 KillsJobs,
1022 /// A kind of stop from a newer daemon than this client knows, so one added later does not keep the client from
1023 /// reading the status.
1024 #[serde(other)]
1025 Unknown,
1026}
1027
1028impl Stopping {
1029 /// What it does, as every client says it: `shutting down: waits for running jobs`.
1030 pub fn line(self) -> &'static str {
1031 match self {
1032 Stopping::WaitsForJobs => "shutting down: waits for running jobs",
1033 Stopping::LeavesJobs => "shutting down: leaves running jobs to the next start",
1034 Stopping::KillsJobs => "shutting down: kills running jobs",
1035 Stopping::Unknown => "shutting down",
1036 }
1037 }
1038}
1039
1040impl Status {
1041 /// What a stopping daemon does, as every client says it ([`Stopping::line`]); `shutting down` from a daemon
1042 /// that does not say how; `None` while it runs.
1043 pub fn stopping_line(&self) -> Option<&'static str> {
1044 match (self.stopping, self.shutting_down) {
1045 (Some(stopping), _) => Some(stopping.line()),
1046 (None, true) => Some("shutting down"),
1047 (None, false) => None,
1048 }
1049 }
1050
1051 /// Why no runner starts while GitHub's rate limit is used up, as every client says it; `None` while it is not.
1052 pub fn github_line(&self) -> Option<&'static str> {
1053 self.github_limited_until
1054 .map(|_| "GitHub's rate limit is used up: no runner starts until it resets")
1055 }
1056}
1057
1058/// The body of every error response.
1059#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
1060pub struct ErrorResponse {
1061 /// What went wrong, for people to read.
1062 pub message: String,
1063}