runnerd_core/api.rs
1//! The daemon's JSON API, version 1: the request and response types every client deserializes, and the
2//! principal a transport authenticates. Plain data only: no I/O, no Docker or GitHub types, no secrets but a
3//! [`Pairing`]'s one-time link, which its `Debug` leaves out.
4//!
5//! Routes, all under the authenticated principal:
6//!
7//! | Route | Body |
8//! |---|---|
9//! | `GET /v1/status` | [`Status`] |
10//! | `POST /v1/repositories/{owner}/{name}/pause?`[`PauseQuery`] | [`Paused`]; 409 [`Busy`] while it runs a job |
11//! | `POST /v1/repositories/{owner}/{name}/resume` | none, 204 |
12//! | `POST /v1/repositories/pause?`[`PauseQuery`] | [`Paused`]; 409 [`Busy`] while any runs a job, pausing none |
13//! | `POST /v1/repositories/resume` | none, 204: every repository resumed |
14//! | `POST /v1/profiles/{name}/reseed` | none, 202; 409 while it reseeds; `cache.reseeded` when done |
15//! | `POST /v1/caches/{name}/clean` | none, 204; 409 [`Busy`] while a runner uses it |
16//! | `GET /v1/config` | [`ConfigFile`] |
17//! | `POST /v1/config/check` | TOML in, [`ConfigCheck`] |
18//! | `PUT /v1/config` | TOML in, [`ConfigCheck`]; 422 [`ConfigCheck`] when invalid |
19//! | `GET /v1/doctor` | [`Doctor`]; 503 [`ErrorResponse`] while no run has finished |
20//! | `GET /v1/attention` | [`Attention`]; 404 from a daemon before 0.2 |
21//! | `POST /v1/shutdown` | [`ShutdownRequest`] in, 202 [`ShutdownAnswer`] |
22//! | `GET /v1/dashboard` | [`Dashboard`] |
23//! | `POST /v1/dashboard/sign-out` | none, 204 |
24//! | `POST /v1/dashboard/pairings?`[`PairingQuery`] | [`Pairings`]; 409 when the dashboard is off or has no `[web] origins` |
25//! | `GET /v1/jobs?`[`JobQuery`] | [`JobPage`] |
26//! | `GET /v1/jobs/{id}` | [`Job`] |
27//! | `GET /v1/jobs/{id}/samples` | [`SampleSeries`] |
28//! | `GET /v1/jobs/{id}/logs?`[`LogQuery`] | `text/plain`, one line per log line |
29//! | `GET /v1/host/samples?`[`HostSampleQuery`] | [`HostSamples`] |
30//! | `GET /v1/stats?`[`StatsQuery`] | [`Stats`] |
31//! | `GET /v1/activity?`[`ActivityQuery`] | [`Activity`] |
32//! | `GET /v1/sizing?`[`SizingQuery`] | [`Sizing`] |
33//! | `GET /v1/insights?`[`InsightsQuery`] | [`Insights`] |
34//! | `GET /v1/digest` | [`Digest`] |
35//! | `GET /v1/savings` | [`Savings`] |
36//! | `POST /v1/notify/test?`[`NotifyTestQuery`] | [`NotifyTest`]; 404 without `[notify]` targets or with no such one |
37//! | `GET /v1/caches` | [`Caches`] |
38//! | `GET /v1/caches/history?`[`CacheHistoryQuery`] | [`CacheHistory`] |
39//! | `GET /v1/events?`[`EventQuery`] | `text/event-stream` of [`EventEnvelope`] |
40//! | `GET /v1/records` | [`Records`] |
41//! | `DELETE /v1/records/{job}` | [`RemovedRecords`]; 409 while a shell is open in it |
42//! | `DELETE /v1/records?`[`RemoveRecordsQuery`] | [`RemovedRecords`]; records with an open shell stay |
43//! | `POST /v1/records/{job}/shell?`[`ShellQuery`] | `Upgrade:` [`SHELL_PROTOCOL`], 101, then the shell's terminal; 409 when the budget is full |
44//!
45//! Errors carry an [`ErrorResponse`].
46
47use serde::{Deserialize, Serialize};
48
49mod actions;
50mod admission;
51mod attention;
52mod budget;
53mod control;
54mod events;
55mod insights;
56mod jobs;
57mod labels;
58mod records;
59mod results;
60mod savings;
61mod sizing;
62mod slots;
63mod stats;
64mod storage;
65mod tone;
66mod units;
67
68pub use actions::{
69 ActionAnswer, ActionOutcome, ActionPreview, ActionQuery, ActionRefusal, ActionResult,
70 ActionTarget, CONFIRM_SECS, JOB_ACTIONS_CHECK, JobAction, JobActions, PreviewFacts, RunSeen,
71 SIBLINGS_MAX, Superseded, UnknownCause, actions_in_flight, actions_no_run,
72 actions_not_configured, actions_off,
73};
74pub use admission::{Admission, AdmissionPolicy, Basis, CpuAdmission, JobPeak, cores, cpus};
75pub use attention::{
76 ATTENTION_MAX, Attention, AttentionItem, AttentionKind, Fix, Subject, attention_items,
77};
78pub use budget::{
79 BudgetParts, ClosedKind, ClosedPart, FreePart, HeldKind, HeldPart, HostFigure, Reading,
80 SlotPart,
81};
82pub use control::{
83 Busy, Check, CheckStatus, ConfigCheck, ConfigFile, Doctor, DoctorState, PauseQuery, Paused,
84 ReplaceQuery, RunningJob,
85};
86pub use events::{Event, EventEnvelope, EventQuery, RunnerFailure};
87pub use insights::{
88 AutoNote, Delivery, Digest, HostHazard, HostSpan, INSIGHT_DAYS, INSIGHT_DAYS_MAX, Insight,
89 InsightKind, Insights, InsightsQuery, LongWait, Notification, NotifyEvent, NotifyTest,
90 NotifyTestQuery, Priority, RepositoryWaits, RunLink, Severity, WaitSpread, WeekWaits,
91};
92#[cfg(feature = "host")]
93pub(crate) use jobs::SAMPLE_GAP_INTERVALS;
94#[cfg(feature = "host")]
95pub(crate) use jobs::TITLE_SOURCE_MAX;
96pub use jobs::{
97 Activity, ActivityQuery, Bucket, BuildCacheCounts, BuildCacheStats, CacheHistory,
98 CacheHistoryQuery, CacheKind, CacheSize, CacheVolume, Caches, ChartLine, ChartLineKind,
99 ChartLines, Holder, HostMemory, HostSample, HostSampleQuery, HostSamples, Job, JobMetrics,
100 JobPage, JobQuery, JobRunner, JobState, JobSummary, JobTitle, Limit, LimitSource, Limits,
101 LogQuery, LogSummary, Milestone, Outcome, OutcomeKind, Phase, PhaseKind, Pressure,
102 RunComparison, Sample, SampleSeries, SampleUsage, Started, Stats, StatsQuery, StatsRow, Step,
103 Swap, Timeline, Total, host_margin, job_title,
104};
105pub use jobs::{
106 BarEnd, Baseline, BaselineRead, COMPARE_STEP_BYTES, COMPARE_STEPS_MAX, ChangeKind,
107 CompareQuery, ComparedSteps, FigureDelta, FigureKind, Gap, GapKind, JOB_LABELS_MAX,
108 JOB_SEARCH_MAX, JOB_TEXT_MAX, JOBS_PAGE_BYTES, JOBS_PAGE_DEFAULT, JOBS_PAGE_MAX, JobComparison,
109 JobList, JobOrder, JobRange, JobSort, Lane, RecordFilter, RecordMark, RunnerSpan, SlotKey,
110 SlotTimeline, SlottedBar, SlottedRunner, SortOrder, Span, StepDelta, StepMatch, StepUsage,
111 TIMELINE_BARS_MAX, TIMELINE_DEFAULT, TIMELINE_WINDOW_MAX, TimelineBar, TimelineQuery,
112 TimelineRows, Unavailable, row_bytes,
113};
114pub use jobs::{
115 Color, FailureSummary, LINE_BYTES_MAX, LINES_DEFAULT, LINES_MAX, LineKind, LineRange, LineRead,
116 LogEvidence, LogGroup, LogLinesQuery, LogMark, LogNeedle, LogOutline, LogSearch,
117 LogSearchQuery, MarkKind, OutlineChange, OutlineParts, Outliner, RUNS_MAX, Run,
118 SEARCH_QUERY_MAX, SavedOutline, Search, StepLines, StepLog, StoredLine, StoredLog, StyledLine,
119};
120pub use labels::{Column, Feature, JOBS_HEADING, NOT_SUPPORTED, Section, View};
121pub use records::{
122 Record, RecordLimits, RecordNote, RecordRemoval, Records, RemoveRecordsQuery, RemovedRecords,
123 SHELL_PROTOCOL, ShellQuery,
124};
125pub use results::{force_pause_consequence, reseed_requested, reseeded};
126pub use savings::{
127 Period, PricedRun, PricedRunner, Pricing, PricingHistory, SELF_HOSTED_RATE, SavedWindow,
128 Savings, Sku, Usd, Visibility, billed_minutes, reads_since,
129};
130pub use sizing::{
131 Advice, AutoPlan, GRANULE, JobSizing, MemoryBound, MemoryHold, Need, PAGE_CACHE_ROOM, Peaks,
132 ProfilePlan, ProfileSizing, Resize, RevertCause, SAMPLE_INTERVAL, SLOT_DISK, Sizing,
133 SizingChange, SizingHold, SizingQuery, SizingRefused, SizingRevert, Spread, Starts, need,
134 period, sampled_note,
135};
136pub use slots::{Figure, Health, HealthKind, SlotState};
137pub use stats::StatsSummary;
138pub use storage::{
139 ACTIONS_CACHE, BUILD_CACHE_TOOLS, BuildCacheActivity, CacheLimit, RECORDS_NEAR_FULL,
140 RecordState, actions_cache_place, records_check, records_near_full,
141};
142pub use tone::{Tone, Toned};
143pub(crate) use units::days_from_civil;
144pub use units::{ago, cores_used, cpu_time, duration, in_cores};
145
146/// Who sent a request, as the transport authenticated it. Handlers get it from one place, so a new transport only
147/// has to produce one.
148#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
149#[serde(rename_all = "kebab-case", tag = "kind")]
150pub enum Principal {
151 /// The user the daemon runs as, on the local Unix socket.
152 LocalOwner {
153 /// The Unix user ID the socket's peer runs as.
154 uid: u32,
155 },
156}
157
158/// `GET /v1/status`: what the daemon serves and runs right now.
159#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
160pub struct Status {
161 /// The daemon's version, from its `Cargo.toml`.
162 pub version: String,
163 /// Set once the daemon stops polling to shut down. [`Status::stopping_line`] says how.
164 pub shutting_down: bool,
165 /// How it stops, once it does; absent from a daemon before it said.
166 #[serde(default, skip_serializing_if = "Option::is_none")]
167 pub stopping: Option<Stopping>,
168 /// Until when GitHub's rate limit is used up, in seconds since the Unix epoch: no runner starts before.
169 /// [`Status::github_line`] says it. Absent while it is not.
170 #[serde(default, skip_serializing_if = "Option::is_none")]
171 pub github_limited_until: Option<u64>,
172 /// The memory budget and what runners use of it.
173 pub budget: Budget,
174 /// Every configured profile with its slots.
175 pub profiles: Vec<ProfileStatus>,
176 /// Every configured repository.
177 pub repositories: Vec<RepositoryStatus>,
178 /// Queued jobs the repositories' runners can take and none holds yet, at the last poll, oldest first. No event
179 /// carries the list: `job.queued` announces a job, `job.started` and `job.finished` its leaving the queue.
180 /// A repository whose last listing failed, as while GitHub rate-limits runnerd, or that is no longer polled
181 /// because runnerd is shutting down, shows no jobs here, while the history keeps its jobs queued.
182 pub queued: Vec<QueuedJob>,
183 /// Whether the daemon cancels and re-runs jobs (ADR 0134); a daemon before 0.2 sends none, read as
184 /// [`JobActions::Unsupported`].
185 #[serde(default)]
186 pub job_actions: JobActions,
187}
188
189/// The memory budget runners and the mirror share, in bytes.
190#[derive(Debug, Clone, Default, PartialEq, Eq, Serialize, Deserialize)]
191pub struct Budget {
192 /// The systemd slice runner containers run in.
193 pub slice: String,
194 /// The slice's `MemoryMax`; `None` when it is unlimited.
195 pub memory_max: Option<u64>,
196 /// Memory the running containers reserve: the runners' caps, or under measured admission the larger of each
197 /// runner's predicted peak and its last sampled memory, nothing for one whose container is gone, and the caps of
198 /// the seed containers, shells and the mirror.
199 pub memory_in_use: u64,
200 /// What pooling takes off `memory_in_use` (ADR 0099, 0130); absent while it takes nothing.
201 #[serde(default, skip_serializing_if = "Option::is_none")]
202 pub pooled: Option<Pooled>,
203 /// The growth count both budgets go by, and its brake (ADR 0130); `pooled` and `host.credit` say what it takes off
204 /// `memory_in_use`. Absent under caps admission, with `[host] pooled_count` off, and from an older daemon.
205 #[serde(default, skip_serializing_if = "Option::is_none")]
206 pub counted: Option<Counted>,
207 /// What a new runner must fit in beside them.
208 pub admission: AdmissionPolicy,
209 /// Under measured admission, whether the slice is at the high-water mark, so no runner starts.
210 pub held: bool,
211 /// The mirror's part of `memory_in_use`: its cap while it is enabled or still running, else 0.
212 pub mirror_memory: u64,
213 /// The seed containers' part of `memory_in_use`, one per profile that reseeds, by profile name.
214 pub seed_memory: Vec<SeedMemory>,
215 /// The shells' part of `memory_in_use`, one per shell open in a record, by job.
216 pub shell_memory: Vec<ShellMemory>,
217 /// What the host's memory leaves runners beside its other programs, from a recent reading of it; absent
218 /// without one.
219 #[serde(default, skip_serializing_if = "Option::is_none")]
220 pub host: Option<HostBudget>,
221 /// The CPU budget, under `[host] cpu_overcommit`. Absent without one.
222 #[serde(default, skip_serializing_if = "Option::is_none")]
223 pub cpu: Option<CpuBudget>,
224 /// The CPUs the slice may use, in millicores: its `AllowedCPUs` or the host's, less under a lower `CPUQuota`,
225 /// whether or not a CPU budget admits runners. The runners' CPU limits are quotas that may add up past it.
226 /// Absent when it could not be read.
227 #[serde(default, skip_serializing_if = "Option::is_none")]
228 pub slice_cpus: Option<u64>,
229}
230
231/// The count the budgets go by while pooling takes something off `Budget::memory_in_use` (ADR 0099).
232#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
233pub struct Pooled {
234 /// What it takes off, in bytes.
235 pub credit: u64,
236 /// `Budget::memory_in_use` less `credit`: what new runners fit beside under `MemoryMax`. The host's limit reads
237 /// `memory_in_use`.
238 pub counted: u64,
239 /// How many runners the credit counts on.
240 pub runners: u64,
241}
242
243/// How far back the growth count's brake compares what runs uses: 10 minutes (ADR 0130,
244/// `admission::growth::GROWTH_WINDOW`).
245pub const GROWTH_WINDOW: std::time::Duration = std::time::Duration::from_mins(10);
246
247/// The growth count (ADR 0130): what the runners use, and the room kept for what they may still grow by.
248#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
249pub struct Counted {
250 /// What the runners use, by their recent samples, in bytes.
251 pub used: u64,
252 /// The room it keeps for their growth, in bytes; absent while it rests (`brake`).
253 #[serde(default, skip_serializing_if = "Option::is_none")]
254 pub kept: Option<u64>,
255 /// How many of the largest growths the room covers at least.
256 pub jobs: u64,
257 /// Why it rests, while it does: the budgets then go by 0099's and 0108's credits.
258 #[serde(default, skip_serializing_if = "Option::is_none")]
259 pub brake: Option<CountBrake>,
260}
261
262/// Why the growth count rests (ADR 0130).
263#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
264#[serde(rename_all = "kebab-case", tag = "kind")]
265pub enum CountBrake {
266 /// What ran grew past the room kept within `GROWTH_WINDOW`.
267 Grew {
268 /// How much the runners present at both looks grew, together, in bytes.
269 grew: u64,
270 /// The room the earlier look kept, in bytes.
271 kept: u64,
272 /// Until when, in seconds since the Unix epoch.
273 until: u64,
274 },
275 /// The daemon started, or a reload changed what the count reads, too recently to judge growth.
276 Restarted {
277 /// Until when, in seconds since the Unix epoch.
278 until: u64,
279 },
280 /// The host is under the guard's line, or a reading is missing, now or at a look within 5 minutes.
281 Guard {
282 /// Until when, in seconds since the Unix epoch; absent for a moment the brake has not looked at yet.
283 #[serde(default, skip_serializing_if = "Option::is_none")]
284 until: Option<u64>,
285 },
286 /// A brake a newer daemon names; this client shows only that the count rests.
287 #[serde(other)]
288 Unknown,
289}
290
291/// What the host's memory leaves the runners (ADR 0055). New runners fit in the lower of `MemoryMax` and
292/// `limit`; running ones are never stopped for it.
293#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
294pub struct HostBudget {
295 /// The reading it comes from.
296 pub memory: HostMemory,
297 /// What the slice may hold now: what its runners hold without page cache plus the host's `MemAvailable`, less
298 /// a margin kept free for the host's page cache; with a ceiling, plus what the host can still grow by
299 /// (`HostMemory::growable`, ADR 0084).
300 pub limit: u64,
301 /// What that leaves new runners beside what the running ones reserve, in bytes.
302 pub spare: u64,
303 /// Whether the host's tasks stall for memory (`full` pressure), so no runner starts until they stop.
304 pub stalled: bool,
305 /// What the host's limit takes off `Budget::memory_in_use` for runners that may not all grow at once (ADR 0108);
306 /// absent while it takes nothing, and from a daemon before 0.0.30.
307 #[serde(default, skip_serializing_if = "Option::is_none")]
308 pub credit: Option<HostCredit>,
309}
310
311/// What the host's limit takes off the plain count (ADR 0108): `Budget::memory_in_use` less `bytes` is what it counts,
312/// `used` of that is what the runners use, and the rest is room kept for what runs to grow into.
313#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
314pub struct HostCredit {
315 /// What it takes off, in bytes.
316 pub bytes: u64,
317 /// What the runners use, by their recent samples, in bytes.
318 pub used: u64,
319}
320
321impl HostBudget {
322 /// Whether the host itself lacks `needs` bytes for a new runner: its tasks stall, `HostMemory::room` is less, or
323 /// that room is under one more margin, so `MemAvailable` is under twice the plain margin, the line `host::stalls`
324 /// reads. There a small job fits only by what the host is about to run out of, and what closing programs frees is
325 /// what starts it. Otherwise a job the host limit holds waits for what runs, whose reservations reach the limit
326 /// (ADR 0103).
327 pub fn short_of(&self, needs: u64) -> bool {
328 let room = self.memory.room();
329 self.stalled || needs > room || room < host_margin(self.memory.grown())
330 }
331
332 /// What the host's limit counts of what runs, `reserved` being `Budget::memory_in_use`: that less the credit
333 /// (ADR 0108).
334 pub fn counts(&self, reserved: u64) -> u64 {
335 reserved.saturating_sub(self.credit.map_or(0, |credit| credit.bytes))
336 }
337
338 /// What that count is made of while a credit stands, as the host's line, a job's wait and the log say it: `it uses
339 /// 14G and keeps 9G for growth`. `None` without a credit, when it counts every reservation.
340 pub fn growth_clause(&self, reserved: u64) -> Option<String> {
341 let credit = self.credit?;
342 let counts = self.counts(reserved);
343 Some(format!(
344 "it uses {} and keeps {} for growth",
345 crate::config::ByteSize(credit.used.min(counts)),
346 crate::config::ByteSize(counts.saturating_sub(credit.used))
347 ))
348 }
349}
350
351/// Why a job waits for the host's limit, needing `needs` bytes with `free` left of it, by the latest reading: for what
352/// runs, or for a host that lacks the room, whose programs it names. While the host's limit takes a credit, what runs
353/// is what it uses and the room it keeps for growth, not its reservations (ADR 0108); `reserved` is
354/// `Budget::memory_in_use`.
355fn host_reason(needs: u64, free: u64, host: Option<&HostBudget>, reserved: u64) -> String {
356 let size = |bytes: u64| crate::config::ByteSize(bytes).to_string();
357 if let Some(host) = host.filter(|host| !host.short_of(needs)) {
358 if let Some(growth) = host.growth_clause(reserved) {
359 return format!(
360 "The host has {}, but what runs may still grow into it: {growth}; needs {}, {} is free",
361 host.memory.available_line(),
362 size(needs),
363 size(free)
364 );
365 }
366 return format!(
367 "The host has {}, but what runs may still grow into it: needs {}; {} of the {} it leaves the slice is free",
368 host.memory.available_line(),
369 size(needs),
370 size(free),
371 size(host.limit)
372 );
373 }
374 let reason = if host.is_some_and(|host| host.stalled) {
375 "The host is short of memory: its programs stall waiting for it, so nothing starts"
376 .to_owned()
377 } else {
378 format!(
379 "The host is short of memory: needs {}; the host has {} to spare",
380 size(needs),
381 size(free)
382 )
383 };
384 match host {
385 Some(host) => format!("{reason}; {}", host.memory.outside_line()),
386 None => reason,
387 }
388}
389
390/// Hundredths of a percent as a percentage: `1.5%`, `12%`.
391pub fn percent(hundredths: u32) -> String {
392 let whole = hundredths / 100;
393 match hundredths % 100 / 10 {
394 0 => format!("{whole}%"),
395 tenth => format!("{whole}.{tenth}%"),
396 }
397}
398
399/// The CPU budget runners share, in millicores.
400#[derive(Debug, Clone, Copy, Default, PartialEq, Eq, Serialize, Deserialize)]
401pub struct CpuBudget {
402 /// The CPUs the slice may use: its `AllowedCPUs` or the host's, less under a lower `CPUQuota`.
403 pub cpus: u64,
404 /// Those times `cpu_overcommit`: what the runners' reservations must fit.
405 pub budget: u64,
406 /// What the runners reserve: each its predicted average CPU under measured admission, else its `cpus`.
407 pub in_use: u64,
408 /// Whether a queued job waits for CPU that memory and a slot have room for.
409 pub held: bool,
410}
411
412impl CpuBudget {
413 /// The budget's line as clients show it beside memory, such as `CPU 4.5 of 32 CPUs reserved` or, while a job
414 /// waits for it, `CPU 30 of 32 CPUs reserved; a job waits for CPU`.
415 pub fn line(&self) -> String {
416 let mut line = format!(
417 "CPU {} of {} CPUs reserved",
418 cores(self.in_use),
419 cores(self.budget)
420 );
421 if self.held {
422 line.push_str("; a job waits for CPU");
423 }
424 line
425 }
426}
427
428impl Budget {
429 /// How runners are admitted, as clients add it to the budget's line: nothing under caps, else `by predicted
430 /// peaks`, and why nothing starts while the slice is at the high-water mark.
431 pub fn admission_note(&self) -> Option<&'static str> {
432 match (self.admission, self.held) {
433 (AdmissionPolicy::Caps, _) => None,
434 (AdmissionPolicy::Measured, false) => Some("by predicted peaks"),
435 (AdmissionPolicy::Measured, true) => Some(
436 "by predicted peaks; the slice is past 90% of its MemoryMax, so no runner starts",
437 ),
438 }
439 }
440}
441
442/// A shell container's memory cap: its record's profile's limit when the shell opened.
443#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
444pub struct ShellMemory {
445 /// The job whose record the shell runs.
446 pub job: u64,
447 /// Its cap in bytes.
448 pub memory: u64,
449}
450
451/// A seed container's memory cap: its profile's limit when the reseed started, or, for a requested reseed whose
452/// profile drained, the room it holds until it starts.
453#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
454pub struct SeedMemory {
455 /// The profile it reseeds.
456 pub profile: String,
457 /// Its cap in bytes.
458 pub memory: u64,
459}
460
461/// A runner profile and what runs in its slots.
462#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
463pub struct ProfileStatus {
464 /// The name under `[profiles]` in `runnerd.toml`.
465 pub name: String,
466 /// The image its runners run, such as `runnerd-runner:base`.
467 pub image: String,
468 /// CPUs per slot.
469 pub cpus: u32,
470 /// Bytes per slot.
471 pub memory: u64,
472 /// Set while slot 1's layers become the profile's snapshot.
473 pub reseeding: bool,
474 /// Index 0 is slot 1.
475 pub slots: Vec<SlotStatus>,
476}
477
478/// One slot of a profile: room for one job.
479#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
480pub struct SlotStatus {
481 /// 1-based.
482 pub slot: usize,
483 /// `None` when the slot is free.
484 pub runner: Option<RunnerStatus>,
485}
486
487/// A runner registered for one job, in its container.
488#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
489pub struct RunnerStatus {
490 /// `owner/name` the runner is registered on.
491 pub repository: String,
492 /// The runner's name, also its container's.
493 pub name: String,
494 /// The runner's ID on GitHub.
495 pub runner_id: u64,
496 /// The Docker container ID.
497 pub container: String,
498 /// Whether it runs a job, as GitHub or its log reports it: its log names the job before GitHub lists it busy.
499 pub busy: bool,
500 /// Seconds since the Unix epoch.
501 pub started_at: u64,
502 /// What it takes of the budget: its cap under caps; under measured admission the larger of its predicted peak
503 /// and its last sampled memory.
504 pub reserved: u64,
505 /// A warm runner: idle, and no queued job of its repository waits for it; it takes the next one GitHub hands
506 /// out.
507 pub warm: bool,
508}
509
510/// A configured repository and its queue.
511#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
512pub struct RepositoryStatus {
513 /// `owner/name` on GitHub.
514 pub name: String,
515 /// Its pools in config order, one per `[[repositories]]` entry: a job runs in the one whose labels match it
516 /// (ADR 0058).
517 pub pools: Vec<PoolStatus>,
518 /// Paused repositories get no new runners; their running jobs finish.
519 pub paused: bool,
520 /// Its jobs in `Status::queued`: queued ones none of its runners took yet, those no pool serves too.
521 pub queued_jobs: usize,
522 /// Why its queued jobs cannot be listed, while the listings fail (ADR 0132); absent while they answer, before the
523 /// first, while GitHub's rate limit stops them, and from a daemon before 0.2.
524 #[serde(default, skip_serializing_if = "Option::is_none")]
525 pub listing: Option<ListingFailure>,
526}
527
528/// Why runnerd cannot list a repository's queued jobs (ADR 0132): what class of failure GitHub's answer was, never its
529/// text. The engine keeps it until a listing answers or a reload removes the repository; a restart forgets it.
530#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
531pub struct ListingFailure {
532 /// When the listings began to fail, in seconds since the Unix epoch.
533 pub since: u64,
534 /// Why the last one failed.
535 pub problem: ListingProblem,
536}
537
538/// What a failed listing ran into.
539#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
540#[serde(rename_all = "kebab-case", tag = "kind")]
541pub enum ListingProblem {
542 /// runnerd could read no GitHub token.
543 NoToken,
544 /// GitHub refused the token: 401.
545 Unauthorized,
546 /// GitHub answered 403, 404 or 410: the token may not read the repository's Actions, cannot see it, or it is gone.
547 Refused {
548 /// The HTTP status.
549 status: u16,
550 },
551 /// GitHub answered another error status, such as a 5xx.
552 Failed {
553 /// The HTTP status.
554 status: u16,
555 },
556 /// No answer came: the network, DNS or TLS failed.
557 Unreachable,
558 /// The answer was not what GitHub sends.
559 Unexpected,
560 /// A problem from a newer daemon than this client knows.
561 #[serde(other)]
562 Unknown,
563}
564
565impl ListingProblem {
566 /// Whether it is the token's, and so every repository's: none read, or GitHub refused it.
567 pub fn is_token(&self) -> bool {
568 matches!(self, ListingProblem::NoToken | ListingProblem::Unauthorized)
569 }
570}
571
572impl ListingFailure {
573 /// What fails, as every client says it: `Cannot list halyard/ledger's queued jobs: GitHub answers 404; the token
574 /// cannot see the repository, or it is gone`.
575 pub fn line(&self, repository: &str) -> String {
576 format!(
577 "Cannot list {repository}'s queued jobs: {}",
578 self.problem.why()
579 )
580 }
581}
582
583impl ListingProblem {
584 /// Why the listing fails, as the daemon's log and every client say it: `GitHub answers 502`. Never GitHub's text.
585 pub fn why(&self) -> String {
586 match self {
587 ListingProblem::NoToken => "runnerd has no GitHub token".to_owned(),
588 ListingProblem::Unauthorized => "GitHub refuses runnerd's token (401)".to_owned(),
589 ListingProblem::Refused { status: 403 } => {
590 "GitHub answers 403; the token may not read the repository's Actions".to_owned()
591 }
592 ListingProblem::Refused { status } => format!(
593 "GitHub answers {status}; the token cannot see the repository, or it is gone"
594 ),
595 ListingProblem::Failed { status } => format!("GitHub answers {status}"),
596 ListingProblem::Unreachable => "GitHub does not answer".to_owned(),
597 ListingProblem::Unexpected => "GitHub's answer is not what runnerd reads".to_owned(),
598 ListingProblem::Unknown => "for a reason this client does not know".to_owned(),
599 }
600 }
601}
602
603/// One of a repository's pools: its runners in one profile.
604#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
605pub struct PoolStatus {
606 /// The profile its jobs run in.
607 pub profile: String,
608 /// The labels its runners register with.
609 pub labels: Vec<String>,
610 /// Its jobs in `Status::queued`: those its labels match.
611 pub queued_jobs: usize,
612}
613
614/// A queued job runners of this daemon can take, as GitHub listed it at the last poll.
615#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
616pub struct QueuedJob {
617 /// GitHub's job ID.
618 pub id: u64,
619 /// The repository whose runners can take it, `owner/name`.
620 pub repository: String,
621 /// The profile of its repository's pool whose labels match it; `None` when no pool's do.
622 #[serde(default, skip_serializing_if = "Option::is_none")]
623 pub profile: Option<String>,
624 /// The workflow run it belongs to.
625 pub run_id: u64,
626 /// 1 for the first attempt of the run, higher for re-runs.
627 pub run_attempt: u32,
628 /// The workflow's name.
629 pub workflow: Option<String>,
630 /// The job's name in the workflow.
631 pub name: String,
632 /// The branch the run is for.
633 pub branch: Option<String>,
634 /// The job's page on GitHub.
635 pub url: Option<String>,
636 /// When GitHub queued it, in seconds since the Unix epoch; 0 if GitHub did not say.
637 pub queued_at: u64,
638 /// Why no runner has it yet, as the last plan saw it. `None` before the daemon has planned with it, and while
639 /// it shuts down.
640 #[serde(default, skip_serializing_if = "Option::is_none")]
641 pub waiting: Option<Waiting>,
642}
643
644/// The slot of an idle runner of `job`'s repository, in its pool's profile when known, such as `jvm 2`.
645fn starting_for(job: &QueuedJob, status: &Status) -> Option<String> {
646 status
647 .profiles
648 .iter()
649 .filter(|profile| job.profile.as_ref().is_none_or(|own| *own == profile.name))
650 .find_map(|profile| {
651 profile
652 .slots
653 .iter()
654 .find(|slot| {
655 slot.runner
656 .as_ref()
657 .is_some_and(|runner| runner.repository == job.repository && !runner.busy)
658 })
659 .map(|slot| format!("{}\u{a0}{}", profile.name, slot.slot))
660 })
661}
662
663impl QueuedJob {
664 /// Where it runs, as every client names it: its repository, and its pool's profile when the repository has
665 /// several pools in `status`, such as `halyard/ledger · light`.
666 pub fn place(&self, status: &Status) -> String {
667 let pools = status
668 .repositories
669 .iter()
670 .find(|repository| repository.name == self.repository)
671 .map_or(0, |repository| repository.pools.len());
672 match self.profile.as_deref().filter(|_| pools > 1) {
673 Some(profile) => format!("{}\u{a0}· {profile}", self.repository),
674 None => self.repository.clone(),
675 }
676 }
677}
678
679impl RepositoryStatus {
680 /// The profile of its first pool, which every client colours it by.
681 pub fn first_profile(&self) -> &str {
682 self.pools.first().map_or("", |pool| pool.profile.as_str())
683 }
684
685 /// Its pools' profiles, as every client lists them on one line: `jvm` or `jvm + light`.
686 pub fn profiles(&self) -> String {
687 self.pools
688 .iter()
689 .map(|pool| pool.profile.as_str())
690 .collect::<Vec<_>>()
691 .join(" + ")
692 }
693
694 /// The profile of its pool whose labels match a job asking for `labels`, as the daemon picks it.
695 pub fn pool_for(&self, labels: &[String]) -> Option<String> {
696 self.pools
697 .iter()
698 .find(|pool| crate::config::labels_match(labels, &pool.labels))
699 .map(|pool| pool.profile.clone())
700 }
701}
702
703/// What failed, that holds a repository's runners back for a while.
704#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
705#[serde(rename_all = "kebab-case")]
706pub enum HoldCause {
707 /// Its runners ended without connecting, so without taking a job (ADR 0104).
708 NoJob,
709 /// Its runners connected and ended without a job, one after the other.
710 IdleExits,
711 /// GitHub did not register its runner.
712 Register,
713 /// Docker did not start its runner's container.
714 Start,
715}
716
717/// Why a queued job has no runner of its own yet. Jobs are served oldest first; each takes the first of these that
718/// holds for it.
719#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
720#[serde(rename_all = "kebab-case", tag = "kind")]
721pub enum Waiting {
722 /// Its repository is paused.
723 Paused,
724 /// Runners of its repository failed, so none starts before `until` (seconds since the Unix epoch).
725 Backoff {
726 /// When runners may start again.
727 until: u64,
728 /// What failed.
729 cause: HoldCause,
730 },
731 /// A runner started for it and waits for GitHub to hand it the job: a runner takes any job its labels match,
732 /// so not necessarily this one.
733 RunnerStarting,
734 /// Slot 1 of its profile is becoming the profile's snapshot, so none of the profile's slots starts.
735 Reseeding {
736 /// The profile.
737 profile: String,
738 },
739 /// Its profile's image is being pulled, so none of the profile's slots starts.
740 PullingImage {
741 /// The profile.
742 profile: String,
743 },
744 /// Every slot of its profile holds a runner.
745 NoSlot {
746 /// The profile.
747 profile: String,
748 },
749 /// What its runner would reserve does not fit the memory budget left. Under measured admission, `free` is 0
750 /// while the slice is at the high-water mark (`Budget::held`).
751 Memory {
752 /// What its runner would reserve, in bytes.
753 needs: u64,
754 /// What the budget has left, in bytes.
755 free: u64,
756 },
757 /// What its runner would reserve does not fit what the host's memory has to spare beside its other programs
758 /// (`HostBudget::spare`), though it fits the slice's budget. `free` is 0 while the host's tasks stall for memory
759 /// (`HostBudget::stalled`).
760 HostMemory {
761 /// What its runner would reserve, in bytes.
762 needs: u64,
763 /// What the host has to spare, in bytes.
764 free: u64,
765 },
766 /// What its runner would reserve does not fit the CPU budget left, in millicores.
767 Cpu {
768 /// What its runner would reserve.
769 needs: u64,
770 /// What the CPU budget has left.
771 free: u64,
772 },
773 /// An older job has waited long enough for CPU that no younger one takes CPU before it.
774 BehindCpu {
775 /// The older job's ID.
776 job: u64,
777 },
778 /// An older job has waited long enough for memory that no younger one starts before it (ADR 0096).
779 BehindMemory {
780 /// The older job's ID.
781 job: u64,
782 },
783 /// An older job is about to have a runner start for it, which takes the slot, memory or CPU this one would
784 /// otherwise start with: what holds it is that start, not what runs now.
785 BehindStart {
786 /// The older job's ID.
787 job: u64,
788 },
789 /// No pool of its repository has every label it asks for, so no runner of this daemon can take it (ADR 0058).
790 NoPool {
791 /// The labels it asks for.
792 labels: Vec<String>,
793 },
794 /// A reason from a newer daemon than this client knows, so a reason added later does not keep the client from
795 /// reading the status.
796 #[serde(other)]
797 Unknown,
798}
799
800/// What a [`Waiting`] reason waits on, which clients draw it by.
801#[derive(Debug, Clone, Copy, PartialEq, Eq)]
802pub enum WaitKind {
803 /// Something the owner can act on: a pause, runners that failed to take a job, or labels no pool serves.
804 Act,
805 /// Something that clears by itself shortly: a runner starting, a reseed, an image pull, an older job's start.
806 Soon,
807 /// Room: a slot, memory or CPU that a running job holds.
808 Full,
809 /// A reason this client does not know (`Waiting::Unknown`), drawn neutral.
810 Other,
811}
812
813impl Waiting {
814 /// Whether runnerd cannot start the job with the config it runs with: its repository is paused, or no pool serves
815 /// its labels. Not that nothing ever will: a resume or a config change can, and another machine's runner may take
816 /// it. A backoff ends by itself, and a reason this client does not know says nothing (ADR 0132).
817 pub fn cannot_start(&self) -> bool {
818 matches!(self, Waiting::Paused | Waiting::NoPool { .. })
819 }
820
821 /// What it waits on.
822 pub fn kind(&self) -> WaitKind {
823 match self {
824 Waiting::Paused | Waiting::Backoff { .. } | Waiting::NoPool { .. } => WaitKind::Act,
825 Waiting::Unknown => WaitKind::Other,
826 Waiting::RunnerStarting
827 | Waiting::Reseeding { .. }
828 | Waiting::PullingImage { .. }
829 | Waiting::BehindStart { .. } => WaitKind::Soon,
830 Waiting::NoSlot { .. }
831 | Waiting::Memory { .. }
832 | Waiting::HostMemory { .. }
833 | Waiting::Cpu { .. }
834 | Waiting::BehindCpu { .. }
835 | Waiting::BehindMemory { .. } => WaitKind::Full,
836 }
837 }
838
839 /// Why `job` waits, in the owner's words with the numbers that hold it, as every client says it: such as
840 /// `All 2 node slots are busy` or `Needs 9G of memory; 2.5G of the budget is free`. `status` is the one `job`
841 /// is queued in, which names the slot of a starting runner and the older job one waits behind.
842 pub fn reason(&self, job: &QueuedJob, status: &Status, now: u64) -> String {
843 let size = |bytes: u64| crate::config::ByteSize(bytes).to_string();
844 let older_name = |id: u64| {
845 status
846 .queued
847 .iter()
848 .find(|candidate| candidate.id == id)
849 .map_or_else(|| "an older job".to_owned(), |older| older.name.clone())
850 };
851 match self {
852 Waiting::Paused => format!("{} is paused", job.repository),
853 Waiting::Unknown => "Waits for a reason this client does not know yet".to_owned(),
854 Waiting::NoPool { labels } if labels.is_empty() => {
855 "It asks for no labels, so no profile serves it".to_owned()
856 }
857 Waiting::NoPool { labels } => format!(
858 "No profile serves the {} {}",
859 if labels.len() == 1 { "label" } else { "labels" },
860 labels.join(", ")
861 ),
862 Waiting::Backoff { until, cause } => format!(
863 "{}; retrying in {}",
864 match cause {
865 HoldCause::NoJob => "Runners failed to take a job",
866 HoldCause::IdleExits => "Runners ended without a job after waiting for one",
867 HoldCause::Register => "Runners cannot register with GitHub",
868 HoldCause::Start => "Runners cannot start",
869 },
870 duration(until.saturating_sub(now))
871 ),
872 Waiting::RunnerStarting => match starting_for(job, status) {
873 Some(slot) => format!("A runner is starting for it in {slot}"),
874 None => "A runner is starting for it".to_owned(),
875 },
876 Waiting::Reseeding { profile } => format!("{profile} is rebuilding its snapshot"),
877 Waiting::PullingImage { profile } => format!("Pulling {profile}'s image"),
878 Waiting::NoSlot { profile } => {
879 let slots = status
880 .profiles
881 .iter()
882 .find(|candidate| &candidate.name == profile)
883 .map_or(0, |profile| profile.slots.len());
884 if slots == 1 {
885 format!("The {profile} slot is busy")
886 } else {
887 format!("All {slots} {profile} slots are busy")
888 }
889 }
890 Waiting::Memory { .. } if status.budget.held => {
891 "The slice is past 90% of its MemoryMax, so nothing starts".to_owned()
892 }
893 Waiting::Memory { needs, free } => format!(
894 "Needs {} of memory; {} of the budget is free",
895 size(*needs),
896 size(*free)
897 ),
898 Waiting::HostMemory { needs, free } => host_reason(
899 *needs,
900 *free,
901 status.budget.host.as_ref(),
902 status.budget.memory_in_use,
903 ),
904 Waiting::Cpu { needs, free } => {
905 format!(
906 "Needs {}; {} are free",
907 cpus(*needs),
908 admission::free_cores(*free)
909 )
910 }
911 Waiting::BehindCpu { job: older } => {
912 format!("Behind {}, which waits for CPU", older_name(*older))
913 }
914 Waiting::BehindMemory { job: older } => {
915 format!("Behind {}, which waits for memory", older_name(*older))
916 }
917 Waiting::BehindStart { job: older } => {
918 format!(
919 "Next after {}, whose runner starts first",
920 older_name(*older)
921 )
922 }
923 }
924 }
925}
926
927/// `GET /v1/dashboard`: the dashboard's `[web]` settings, as the daemon read them when it started.
928#[derive(Debug, Clone, Default, PartialEq, Eq, Serialize, Deserialize)]
929pub struct Dashboard {
930 /// `[web] open`: the one of `[web] origins` that `runnerctl web` opens the dashboard on; the loopback address
931 /// when none.
932 #[serde(default, skip_serializing_if = "Option::is_none")]
933 pub open: Option<String>,
934}
935
936/// `POST /v1/dashboard/pairings`: which of `[web] origins` to make a link for.
937#[derive(Debug, Clone, Default, PartialEq, Eq, Serialize, Deserialize)]
938pub struct PairingQuery {
939 /// One of `[web] origins`; each of them when none.
940 #[serde(default, skip_serializing_if = "Option::is_none")]
941 pub origin: Option<String>,
942}
943
944/// `POST /v1/dashboard/pairings`: fresh one-time links that sign a device in to the dashboard.
945#[derive(Debug, Clone, Default, PartialEq, Eq, Serialize, Deserialize)]
946pub struct Pairings {
947 /// One per origin asked for.
948 pub links: Vec<Pairing>,
949}
950
951/// A link that signs one device in to the dashboard on `origin`, and nowhere else, within `lifetime`.
952#[derive(Clone, PartialEq, Eq, Serialize, Deserialize)]
953pub struct Pairing {
954 /// One of `[web] origins`.
955 pub origin: String,
956 /// The link with its one-time code: a credential, which `Debug` leaves out.
957 pub url: String,
958 /// Seconds it works for from when it was made.
959 pub lifetime: u64,
960}
961
962impl std::fmt::Debug for Pairing {
963 fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
964 formatter
965 .debug_struct("Pairing")
966 .field("origin", &self.origin)
967 .field("lifetime", &self.lifetime)
968 .finish_non_exhaustive()
969 }
970}
971
972/// `POST /v1/shutdown`.
973#[derive(Debug, Clone, Default, PartialEq, Eq, Serialize, Deserialize)]
974pub struct ShutdownRequest {
975 /// Kill running jobs and delete their runners instead of waiting for them.
976 #[serde(default)]
977 pub force: bool,
978}
979
980/// What `POST /v1/shutdown` answers: how the daemon stops now, which a stop asked for earlier may have made more
981/// urgent than the request.
982#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
983pub struct ShutdownAnswer {
984 /// How it stops.
985 pub stopping: Stopping,
986}
987
988/// How a stopping daemon treats running jobs (ADR 0091).
989#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
990#[serde(rename_all = "kebab-case")]
991pub enum Stopping {
992 /// `runnerd stop`: it waits for running jobs, then removes every runner.
993 WaitsForJobs,
994 /// A signal: it removes idle runners and leaves running jobs to the next start.
995 LeavesJobs,
996 /// `runnerd stop --force` or a second signal: it kills running jobs.
997 KillsJobs,
998 /// A kind of stop from a newer daemon than this client knows, so one added later does not keep the client from
999 /// reading the status.
1000 #[serde(other)]
1001 Unknown,
1002}
1003
1004impl Stopping {
1005 /// What it does, as every client says it: `shutting down: waits for running jobs`.
1006 pub fn line(self) -> &'static str {
1007 match self {
1008 Stopping::WaitsForJobs => "shutting down: waits for running jobs",
1009 Stopping::LeavesJobs => "shutting down: leaves running jobs to the next start",
1010 Stopping::KillsJobs => "shutting down: kills running jobs",
1011 Stopping::Unknown => "shutting down",
1012 }
1013 }
1014}
1015
1016impl Status {
1017 /// What a stopping daemon does, as every client says it ([`Stopping::line`]); `shutting down` from a daemon
1018 /// that does not say how; `None` while it runs.
1019 pub fn stopping_line(&self) -> Option<&'static str> {
1020 match (self.stopping, self.shutting_down) {
1021 (Some(stopping), _) => Some(stopping.line()),
1022 (None, true) => Some("shutting down"),
1023 (None, false) => None,
1024 }
1025 }
1026
1027 /// Why no runner starts while GitHub's rate limit is used up, as every client says it; `None` while it is not.
1028 pub fn github_line(&self) -> Option<&'static str> {
1029 self.github_limited_until
1030 .map(|_| "GitHub's rate limit is used up: no runner starts until it resets")
1031 }
1032}
1033
1034/// The body of every error response.
1035#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
1036pub struct ErrorResponse {
1037 /// What went wrong, for people to read.
1038 pub message: String,
1039}