runnerd_core/api.rs
1//! The daemon's JSON API, version 1: the request and response types every client deserializes, and the
2//! principal a transport authenticates. Plain data only: no I/O, no Docker or GitHub types, no secrets but a
3//! [`Pairing`]'s one-time link, which its `Debug` leaves out.
4//!
5//! Routes, all under the authenticated principal:
6//!
7//! | Route | Body |
8//! |---|---|
9//! | `GET /v1/status` | [`Status`] |
10//! | `POST /v1/repositories/{owner}/{name}/pause?`[`PauseQuery`] | [`Paused`]; 409 [`Busy`] while it runs a job |
11//! | `POST /v1/repositories/{owner}/{name}/resume` | none, 204 |
12//! | `POST /v1/repositories/pause?`[`PauseQuery`] | [`Paused`]; 409 [`Busy`] while any runs a job, pausing none |
13//! | `POST /v1/repositories/resume` | none, 204: every repository resumed |
14//! | `POST /v1/profiles/{name}/reseed` | none, 202; 409 while it reseeds; `cache.reseeded` when done |
15//! | `POST /v1/caches/{name}/clean` | none, 204; 409 [`Busy`] while a runner uses it |
16//! | `GET /v1/config` | [`ConfigFile`] |
17//! | `POST /v1/config/check` | TOML in, [`ConfigCheck`] |
18//! | `PUT /v1/config` | TOML in, [`ConfigCheck`]; 422 [`ConfigCheck`] when invalid |
19//! | `GET /v1/doctor` | [`Doctor`]; 503 [`ErrorResponse`] while no run has finished |
20//! | `GET /v1/attention` | [`Attention`]; 404 from a daemon before 0.2 |
21//! | `POST /v1/shutdown` | [`ShutdownRequest`] in, 202 [`ShutdownAnswer`] |
22//! | `GET /v1/dashboard` | [`Dashboard`] |
23//! | `POST /v1/dashboard/sign-out` | none, 204 |
24//! | `POST /v1/dashboard/pairings?`[`PairingQuery`] | [`Pairings`]; 409 when the dashboard is off or has no `[web] origins` |
25//! | `GET /v1/jobs?`[`JobQuery`] | [`JobPage`] |
26//! | `GET /v1/jobs/{id}` | [`Job`] |
27//! | `GET /v1/jobs/{id}/samples` | [`SampleSeries`] |
28//! | `GET /v1/jobs/{id}/logs?`[`LogQuery`] | `text/plain`, one line per log line |
29//! | `GET /v1/host/samples?`[`HostSampleQuery`] | [`HostSamples`] |
30//! | `GET /v1/stats?`[`StatsQuery`] | [`Stats`] |
31//! | `GET /v1/activity?`[`ActivityQuery`] | [`Activity`] |
32//! | `GET /v1/sizing?`[`SizingQuery`] | [`Sizing`] |
33//! | `GET /v1/insights?`[`InsightsQuery`] | [`Insights`] |
34//! | `GET /v1/digest` | [`Digest`] |
35//! | `GET /v1/savings` | [`Savings`] |
36//! | `POST /v1/notify/test?`[`NotifyTestQuery`] | [`NotifyTest`]; 404 without `[notify]` targets or with no such one |
37//! | `GET /v1/caches` | [`Caches`] |
38//! | `GET /v1/caches/history?`[`CacheHistoryQuery`] | [`CacheHistory`] |
39//! | `GET /v1/events?`[`EventQuery`] | `text/event-stream` of [`EventEnvelope`] |
40//! | `GET /v1/records` | [`Records`] |
41//! | `DELETE /v1/records/{job}` | [`RemovedRecords`]; 409 while a shell is open in it |
42//! | `DELETE /v1/records?`[`RemoveRecordsQuery`] | [`RemovedRecords`]; records with an open shell stay |
43//! | `POST /v1/records/{job}/shell?`[`ShellQuery`] | `Upgrade:` [`SHELL_PROTOCOL`], 101, then the shell's terminal; 409 when the budget is full |
44//!
45//! Errors carry an [`ErrorResponse`].
46
47use serde::{Deserialize, Serialize};
48
49mod actions;
50mod admission;
51mod attention;
52mod budget;
53mod control;
54mod disk;
55mod events;
56mod insights;
57mod jobs;
58mod labels;
59mod records;
60mod results;
61mod savings;
62mod sizing;
63mod slots;
64mod stats;
65mod storage;
66mod tone;
67mod units;
68
69pub use actions::{
70 ActionAnswer, ActionOutcome, ActionPreview, ActionQuery, ActionRefusal, ActionResult,
71 ActionTarget, CONFIRM_SECS, JOB_ACTIONS_CHECK, JobAction, JobActions, PreviewFacts, RunSeen,
72 SIBLINGS_MAX, Superseded, UnknownCause, actions_in_flight, actions_no_run,
73 actions_not_configured, actions_off,
74};
75pub use admission::{Admission, AdmissionPolicy, Basis, CpuAdmission, JobPeak, cores, cpus};
76pub use attention::{
77 ATTENTION_MAX, Attention, AttentionItem, AttentionKind, Fix, Subject, attention_items,
78};
79pub use budget::{
80 BudgetParts, ClosedKind, ClosedPart, FreePart, HeldKind, HeldPart, HostFigure, Reading,
81 SlotPart,
82};
83pub use control::{
84 Busy, Check, CheckStatus, ConfigCheck, ConfigFile, Doctor, DoctorState, PauseQuery, Paused,
85 ReplaceQuery, RunningJob,
86};
87pub use disk::{
88 Commits, DiskWrites, DiskWritten, DriveShares, DriveWrites, JobWrites, Measure, WriteSection,
89 WriteTop,
90};
91pub use events::{Event, EventEnvelope, EventQuery, RunnerFailure};
92pub use insights::{
93 AutoNote, Delivery, Digest, HostHazard, HostSpan, INSIGHT_DAYS, INSIGHT_DAYS_MAX, Insight,
94 InsightKind, Insights, InsightsQuery, LongWait, Notification, NotifyEvent, NotifyTest,
95 NotifyTestQuery, Priority, RepositoryWaits, RunLink, Severity, WaitSpread, WeekWaits,
96};
97#[cfg(feature = "host")]
98pub(crate) use jobs::SAMPLE_GAP_INTERVALS;
99#[cfg(feature = "host")]
100pub(crate) use jobs::TITLE_SOURCE_MAX;
101pub use jobs::{
102 Activity, ActivityQuery, Bucket, BuildCacheCounts, BuildCacheStats, CacheHistory,
103 CacheHistoryQuery, CacheKind, CacheSize, CacheVolume, Caches, ChartLine, ChartLineKind,
104 ChartLines, Holder, HostMemory, HostSample, HostSampleQuery, HostSamples, Job, JobMetrics,
105 JobPage, JobQuery, JobRunner, JobState, JobSummary, JobTitle, Limit, LimitSource, Limits,
106 LogQuery, LogSummary, Milestone, Outcome, OutcomeKind, Phase, PhaseKind, Pressure,
107 RunComparison, Sample, SampleSeries, SampleUsage, Started, Stats, StatsQuery, StatsRow, Step,
108 Swap, Timeline, Total, host_margin, job_title,
109};
110pub use jobs::{
111 BarEnd, Baseline, BaselineRead, COMPARE_STEP_BYTES, COMPARE_STEPS_MAX, ChangeKind,
112 CompareQuery, ComparedSteps, FigureDelta, FigureKind, Gap, GapKind, JOB_LABELS_MAX,
113 JOB_SEARCH_MAX, JOB_TEXT_MAX, JOBS_PAGE_BYTES, JOBS_PAGE_DEFAULT, JOBS_PAGE_MAX, JobComparison,
114 JobList, JobOrder, JobRange, JobSort, Lane, RecordFilter, RecordMark, RunnerSpan, SlotKey,
115 SlotTimeline, SlottedBar, SlottedRunner, SortOrder, Span, StepDelta, StepMatch, StepUsage,
116 TIMELINE_BARS_MAX, TIMELINE_DEFAULT, TIMELINE_WINDOW_MAX, TimelineBar, TimelineQuery,
117 TimelineRows, Unavailable, row_bytes,
118};
119pub use jobs::{
120 Color, FailureSummary, LINE_BYTES_MAX, LINES_DEFAULT, LINES_MAX, LineKind, LineRange, LineRead,
121 LogEvidence, LogGroup, LogLinesQuery, LogMark, LogNeedle, LogOutline, LogSearch,
122 LogSearchQuery, MarkKind, OutlineChange, OutlineParts, Outliner, RUNS_MAX, Run,
123 SEARCH_QUERY_MAX, SavedOutline, Search, StepLines, StepLog, StoredLine, StoredLog, StyledLine,
124};
125pub use labels::{Column, Feature, JOBS_HEADING, NOT_SUPPORTED, Section, View};
126pub use records::{
127 Record, RecordLimits, RecordNote, RecordRemoval, Records, RemoveRecordsQuery, RemovedRecords,
128 SHELL_PROTOCOL, ShellQuery,
129};
130pub use results::{force_pause_consequence, reseed_requested, reseeded};
131pub use savings::{
132 Period, PricedRun, PricedRunner, Pricing, PricingHistory, SELF_HOSTED_RATE, SavedWindow,
133 Savings, Sku, Usd, Visibility, billed_minutes, reads_since,
134};
135pub use sizing::{
136 Advice, AutoPlan, GRANULE, JobSizing, MemoryBound, MemoryHold, Need, PAGE_CACHE_ROOM, Peaks,
137 ProfilePlan, ProfileSizing, Resize, RevertCause, SAMPLE_INTERVAL, SLOT_DISK, Sizing,
138 SizingChange, SizingHold, SizingQuery, SizingRefused, SizingRevert, Spread, Starts, need,
139 period, sampled_note,
140};
141pub use slots::{Figure, Health, HealthKind, SlotState};
142pub use stats::StatsSummary;
143pub use storage::{
144 ACTIONS_CACHE, BUILD_CACHE_TOOLS, BuildCacheActivity, CacheLimit, RECORDS_NEAR_FULL,
145 RecordState, actions_cache_place, records_check, records_near_full,
146};
147pub use tone::{Tone, Toned};
148pub(crate) use units::days_from_civil;
149pub use units::{ago, cores_used, cpu_time, duration, in_cores};
150
151/// Who sent a request, as the transport authenticated it. Handlers get it from one place, so a new transport only
152/// has to produce one.
153#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
154#[serde(rename_all = "kebab-case", tag = "kind")]
155pub enum Principal {
156 /// The user the daemon runs as, on the local Unix socket.
157 LocalOwner {
158 /// The Unix user ID the socket's peer runs as.
159 uid: u32,
160 },
161}
162
163/// `GET /v1/status`: what the daemon serves and runs right now.
164#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
165pub struct Status {
166 /// The daemon's version, from its `Cargo.toml`.
167 pub version: String,
168 /// Set once the daemon stops polling to shut down. [`Status::stopping_line`] says how.
169 pub shutting_down: bool,
170 /// How it stops, once it does; absent from a daemon before it said.
171 #[serde(default, skip_serializing_if = "Option::is_none")]
172 pub stopping: Option<Stopping>,
173 /// Until when GitHub's rate limit is used up, in seconds since the Unix epoch: no runner starts before.
174 /// [`Status::github_line`] says it. Absent while it is not.
175 #[serde(default, skip_serializing_if = "Option::is_none")]
176 pub github_limited_until: Option<u64>,
177 /// The memory budget and what runners use of it.
178 pub budget: Budget,
179 /// Every configured profile with its slots.
180 pub profiles: Vec<ProfileStatus>,
181 /// Every configured repository.
182 pub repositories: Vec<RepositoryStatus>,
183 /// Queued jobs the repositories' runners can take and none holds yet, at the last poll, oldest first. No event
184 /// carries the list: `job.queued` announces a job, `job.started` and `job.finished` its leaving the queue.
185 /// A repository whose last listing failed, as while GitHub rate-limits runnerd, or that is no longer polled
186 /// because runnerd is shutting down, shows no jobs here, while the history keeps its jobs queued.
187 pub queued: Vec<QueuedJob>,
188 /// Whether the daemon cancels and re-runs jobs (ADR 0134); a daemon before 0.2 sends none, read as
189 /// [`JobActions::Unsupported`].
190 #[serde(default)]
191 pub job_actions: JobActions,
192}
193
194/// The memory budget runners and the mirror share, in bytes.
195#[derive(Debug, Clone, Default, PartialEq, Eq, Serialize, Deserialize)]
196pub struct Budget {
197 /// The systemd slice runner containers run in.
198 pub slice: String,
199 /// The slice's `MemoryMax`; `None` when it is unlimited.
200 pub memory_max: Option<u64>,
201 /// Memory the running containers reserve: the runners' caps, or under measured admission the larger of each
202 /// runner's predicted peak and its last sampled memory, nothing for one whose container is gone, and the caps of
203 /// the seed containers, shells and the mirror.
204 pub memory_in_use: u64,
205 /// What pooling takes off `memory_in_use` (ADR 0099, 0130); absent while it takes nothing.
206 #[serde(default, skip_serializing_if = "Option::is_none")]
207 pub pooled: Option<Pooled>,
208 /// The growth count both budgets go by, and its brake (ADR 0130); `pooled` and `host.credit` say what it takes off
209 /// `memory_in_use`. Absent under caps admission, with `[host] pooled_count` off, and from an older daemon.
210 #[serde(default, skip_serializing_if = "Option::is_none")]
211 pub counted: Option<Counted>,
212 /// What a new runner must fit in beside them.
213 pub admission: AdmissionPolicy,
214 /// Under measured admission, whether the slice is at the high-water mark, so no runner starts.
215 pub held: bool,
216 /// The mirror's part of `memory_in_use`: its cap while it is enabled or still running, else 0.
217 pub mirror_memory: u64,
218 /// The seed containers' part of `memory_in_use`, one per profile that reseeds, by profile name.
219 pub seed_memory: Vec<SeedMemory>,
220 /// The shells' part of `memory_in_use`, one per shell open in a record, by job.
221 pub shell_memory: Vec<ShellMemory>,
222 /// What the host's memory leaves runners beside its other programs, from a recent reading of it; absent
223 /// without one.
224 #[serde(default, skip_serializing_if = "Option::is_none")]
225 pub host: Option<HostBudget>,
226 /// The CPU budget, under `[host] cpu_overcommit`. Absent without one.
227 #[serde(default, skip_serializing_if = "Option::is_none")]
228 pub cpu: Option<CpuBudget>,
229 /// The CPUs the slice may use, in millicores: its `AllowedCPUs` or the host's, less under a lower `CPUQuota`,
230 /// whether or not a CPU budget admits runners. The runners' CPU limits are quotas that may add up past it.
231 /// Absent when it could not be read.
232 #[serde(default, skip_serializing_if = "Option::is_none")]
233 pub slice_cpus: Option<u64>,
234}
235
236/// The count the budgets go by while pooling takes something off `Budget::memory_in_use` (ADR 0099).
237#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
238pub struct Pooled {
239 /// What it takes off, in bytes.
240 pub credit: u64,
241 /// `Budget::memory_in_use` less `credit`: what new runners fit beside under `MemoryMax`. The host's limit reads
242 /// `memory_in_use`.
243 pub counted: u64,
244 /// How many runners the credit counts on.
245 pub runners: u64,
246}
247
248/// How far back the growth count's brake compares what runs uses: 10 minutes (ADR 0130,
249/// `admission::growth::GROWTH_WINDOW`).
250pub const GROWTH_WINDOW: std::time::Duration = std::time::Duration::from_mins(10);
251
252/// The growth count (ADR 0130): what the runners use, and the room kept for what they may still grow by.
253#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
254pub struct Counted {
255 /// What the runners use, by their recent samples, in bytes.
256 pub used: u64,
257 /// The room it keeps for their growth, in bytes; absent while it rests (`brake`).
258 #[serde(default, skip_serializing_if = "Option::is_none")]
259 pub kept: Option<u64>,
260 /// How many of the largest growths the room covers at least.
261 pub jobs: u64,
262 /// Why it rests, while it does: the budgets then go by 0099's and 0108's credits.
263 #[serde(default, skip_serializing_if = "Option::is_none")]
264 pub brake: Option<CountBrake>,
265}
266
267/// Why the growth count rests (ADR 0130).
268#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
269#[serde(rename_all = "kebab-case", tag = "kind")]
270pub enum CountBrake {
271 /// What ran grew past the room kept within `GROWTH_WINDOW`.
272 Grew {
273 /// How much the runners present at both looks grew, together, in bytes.
274 grew: u64,
275 /// The room the earlier look kept, in bytes.
276 kept: u64,
277 /// Until when, in seconds since the Unix epoch.
278 until: u64,
279 },
280 /// The daemon started, or a reload changed what the count reads, too recently to judge growth.
281 Restarted {
282 /// Until when, in seconds since the Unix epoch.
283 until: u64,
284 },
285 /// The host is under the guard's line, or a reading is missing, now or at a look within 5 minutes.
286 Guard {
287 /// Until when, in seconds since the Unix epoch; absent for a moment the brake has not looked at yet.
288 #[serde(default, skip_serializing_if = "Option::is_none")]
289 until: Option<u64>,
290 },
291 /// A brake a newer daemon names; this client shows only that the count rests.
292 #[serde(other)]
293 Unknown,
294}
295
296/// What the host's memory leaves the runners (ADR 0055). New runners fit in the lower of `MemoryMax` and
297/// `limit`; running ones are never stopped for it.
298#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
299pub struct HostBudget {
300 /// The reading it comes from.
301 pub memory: HostMemory,
302 /// What the slice may hold now: what its runners hold without page cache plus the host's `MemAvailable`, less
303 /// a margin kept free for the host's page cache; with a ceiling, plus what the host can still grow by
304 /// (`HostMemory::growable`, ADR 0084).
305 pub limit: u64,
306 /// What that leaves new runners beside what the running ones reserve, in bytes.
307 pub spare: u64,
308 /// Whether the host's tasks stall for memory (`full` pressure), so no runner starts until they stop.
309 pub stalled: bool,
310 /// What the host's limit takes off `Budget::memory_in_use` for runners that may not all grow at once (ADR 0108);
311 /// absent while it takes nothing, and from a daemon before 0.0.30.
312 #[serde(default, skip_serializing_if = "Option::is_none")]
313 pub credit: Option<HostCredit>,
314}
315
316/// What the host's limit takes off the plain count (ADR 0108): `Budget::memory_in_use` less `bytes` is what it counts,
317/// `used` of that is what the runners use, and the rest is room kept for what runs to grow into.
318#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
319pub struct HostCredit {
320 /// What it takes off, in bytes.
321 pub bytes: u64,
322 /// What the runners use, by their recent samples, in bytes.
323 pub used: u64,
324}
325
326impl HostBudget {
327 /// Whether the host itself lacks `needs` bytes for a new runner: its tasks stall, `HostMemory::room` is less, or
328 /// that room is under one more margin, so `MemAvailable` is under twice the plain margin, the line `host::stalls`
329 /// reads. There a small job fits only by what the host is about to run out of, and what closing programs frees is
330 /// what starts it. Otherwise a job the host limit holds waits for what runs, whose reservations reach the limit
331 /// (ADR 0103).
332 pub fn short_of(&self, needs: u64) -> bool {
333 let room = self.memory.room();
334 self.stalled || needs > room || room < host_margin(self.memory.grown())
335 }
336
337 /// What the host's limit counts of what runs, `reserved` being `Budget::memory_in_use`: that less the credit
338 /// (ADR 0108).
339 pub fn counts(&self, reserved: u64) -> u64 {
340 reserved.saturating_sub(self.credit.map_or(0, |credit| credit.bytes))
341 }
342
343 /// What that count is made of while a credit stands, as the host's line, a job's wait and the log say it: `it uses
344 /// 14G and keeps 9G for growth`. `None` without a credit, when it counts every reservation.
345 pub fn growth_clause(&self, reserved: u64) -> Option<String> {
346 let credit = self.credit?;
347 let counts = self.counts(reserved);
348 Some(format!(
349 "it uses {} and keeps {} for growth",
350 crate::config::ByteSize(credit.used.min(counts)),
351 crate::config::ByteSize(counts.saturating_sub(credit.used))
352 ))
353 }
354}
355
356/// Why a job waits for the host's limit, needing `needs` bytes with `free` left of it, by the latest reading: for what
357/// runs, or for a host that lacks the room, whose programs it names. While the host's limit takes a credit, what runs
358/// is what it uses and the room it keeps for growth, not its reservations (ADR 0108); `reserved` is
359/// `Budget::memory_in_use`.
360fn host_reason(needs: u64, free: u64, host: Option<&HostBudget>, reserved: u64) -> String {
361 let size = |bytes: u64| crate::config::ByteSize(bytes).to_string();
362 if let Some(host) = host.filter(|host| !host.short_of(needs)) {
363 if let Some(growth) = host.growth_clause(reserved) {
364 return format!(
365 "The host has {}, but what runs may still grow into it: {growth}; needs {}, {} is free",
366 host.memory.available_line(),
367 size(needs),
368 size(free)
369 );
370 }
371 return format!(
372 "The host has {}, but what runs may still grow into it: needs {}; {} of the {} it leaves the slice is free",
373 host.memory.available_line(),
374 size(needs),
375 size(free),
376 size(host.limit)
377 );
378 }
379 let reason = if host.is_some_and(|host| host.stalled) {
380 "The host is short of memory: its programs stall waiting for it, so nothing starts"
381 .to_owned()
382 } else {
383 format!(
384 "The host is short of memory: needs {}; the host has {} to spare",
385 size(needs),
386 size(free)
387 )
388 };
389 match host {
390 Some(host) => format!("{reason}; {}", host.memory.outside_line()),
391 None => reason,
392 }
393}
394
395/// Hundredths of a percent as a percentage: `1.5%`, `12%`.
396pub fn percent(hundredths: u32) -> String {
397 let whole = hundredths / 100;
398 match hundredths % 100 / 10 {
399 0 => format!("{whole}%"),
400 tenth => format!("{whole}.{tenth}%"),
401 }
402}
403
404/// The CPU budget runners share, in millicores.
405#[derive(Debug, Clone, Copy, Default, PartialEq, Eq, Serialize, Deserialize)]
406pub struct CpuBudget {
407 /// The CPUs the slice may use: its `AllowedCPUs` or the host's, less under a lower `CPUQuota`.
408 pub cpus: u64,
409 /// Those times `cpu_overcommit`: what the runners' reservations must fit.
410 pub budget: u64,
411 /// What the runners reserve: each its predicted average CPU under measured admission, else its `cpus`.
412 pub in_use: u64,
413 /// Whether a queued job waits for CPU that memory and a slot have room for.
414 pub held: bool,
415}
416
417impl CpuBudget {
418 /// The budget's line as clients show it beside memory, such as `CPU 4.5 of 32 CPUs reserved` or, while a job
419 /// waits for it, `CPU 30 of 32 CPUs reserved; a job waits for CPU`.
420 pub fn line(&self) -> String {
421 let mut line = format!(
422 "CPU {} of {} CPUs reserved",
423 cores(self.in_use),
424 cores(self.budget)
425 );
426 if self.held {
427 line.push_str("; a job waits for CPU");
428 }
429 line
430 }
431}
432
433impl Budget {
434 /// How runners are admitted, as clients add it to the budget's line: nothing under caps, else `by predicted
435 /// peaks`, and why nothing starts while the slice is at the high-water mark.
436 pub fn admission_note(&self) -> Option<&'static str> {
437 match (self.admission, self.held) {
438 (AdmissionPolicy::Caps, _) => None,
439 (AdmissionPolicy::Measured, false) => Some("by predicted peaks"),
440 (AdmissionPolicy::Measured, true) => Some(
441 "by predicted peaks; the slice is past 90% of its MemoryMax, so no runner starts",
442 ),
443 }
444 }
445}
446
447/// A shell container's memory cap: its record's profile's limit when the shell opened.
448#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
449pub struct ShellMemory {
450 /// The job whose record the shell runs.
451 pub job: u64,
452 /// Its cap in bytes.
453 pub memory: u64,
454}
455
456/// A seed container's memory cap: its profile's limit when the reseed started, or, for a requested reseed whose
457/// profile drained, the room it holds until it starts.
458#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
459pub struct SeedMemory {
460 /// The profile it reseeds.
461 pub profile: String,
462 /// Its cap in bytes.
463 pub memory: u64,
464}
465
466/// A runner profile and what runs in its slots.
467#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
468pub struct ProfileStatus {
469 /// The name under `[profiles]` in `runnerd.toml`.
470 pub name: String,
471 /// The image its runners run, such as `runnerd-runner:base`.
472 pub image: String,
473 /// CPUs per slot.
474 pub cpus: u32,
475 /// Bytes per slot.
476 pub memory: u64,
477 /// Set while slot 1's layers become the profile's snapshot.
478 pub reseeding: bool,
479 /// Index 0 is slot 1.
480 pub slots: Vec<SlotStatus>,
481}
482
483/// One slot of a profile: room for one job.
484#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
485pub struct SlotStatus {
486 /// 1-based.
487 pub slot: usize,
488 /// `None` when the slot is free.
489 pub runner: Option<RunnerStatus>,
490}
491
492/// A runner registered for one job, in its container.
493#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
494pub struct RunnerStatus {
495 /// `owner/name` the runner is registered on.
496 pub repository: String,
497 /// The runner's name, also its container's.
498 pub name: String,
499 /// The runner's ID on GitHub.
500 pub runner_id: u64,
501 /// The Docker container ID.
502 pub container: String,
503 /// Whether it runs a job, as GitHub or its log reports it: its log names the job before GitHub lists it busy.
504 pub busy: bool,
505 /// Seconds since the Unix epoch.
506 pub started_at: u64,
507 /// What it takes of the budget: its cap under caps; under measured admission the larger of its predicted peak
508 /// and its last sampled memory.
509 pub reserved: u64,
510 /// A warm runner: idle, and no queued job of its repository waits for it; it takes the next one GitHub hands
511 /// out.
512 pub warm: bool,
513}
514
515/// A configured repository and its queue.
516#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
517pub struct RepositoryStatus {
518 /// `owner/name` on GitHub.
519 pub name: String,
520 /// Its pools in config order, one per `[[repositories]]` entry: a job runs in the one whose labels match it
521 /// (ADR 0058).
522 pub pools: Vec<PoolStatus>,
523 /// Paused repositories get no new runners; their running jobs finish.
524 pub paused: bool,
525 /// Its jobs in `Status::queued`: queued ones none of its runners took yet, those no pool serves too.
526 pub queued_jobs: usize,
527 /// Why its queued jobs cannot be listed, while the listings fail (ADR 0132); absent while they answer, before the
528 /// first, while GitHub's rate limit stops them, and from a daemon before 0.2.
529 #[serde(default, skip_serializing_if = "Option::is_none")]
530 pub listing: Option<ListingFailure>,
531}
532
533/// Why runnerd cannot list a repository's queued jobs (ADR 0132): what class of failure GitHub's answer was, never its
534/// text. The engine keeps it until a listing answers or a reload removes the repository; a restart forgets it.
535#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
536pub struct ListingFailure {
537 /// When the listings began to fail, in seconds since the Unix epoch.
538 pub since: u64,
539 /// Why the last one failed.
540 pub problem: ListingProblem,
541}
542
543/// What a failed listing ran into.
544#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
545#[serde(rename_all = "kebab-case", tag = "kind")]
546pub enum ListingProblem {
547 /// runnerd could read no GitHub token.
548 NoToken,
549 /// GitHub refused the token: 401.
550 Unauthorized,
551 /// GitHub answered 403, 404 or 410: the token may not read the repository's Actions, cannot see it, or it is gone.
552 Refused {
553 /// The HTTP status.
554 status: u16,
555 },
556 /// GitHub answered another error status, such as a 5xx.
557 Failed {
558 /// The HTTP status.
559 status: u16,
560 },
561 /// No answer came: the network, DNS or TLS failed.
562 Unreachable,
563 /// The answer was not what GitHub sends.
564 Unexpected,
565 /// A problem from a newer daemon than this client knows.
566 #[serde(other)]
567 Unknown,
568}
569
570impl ListingProblem {
571 /// Whether it is the token's, and so every repository's: none read, or GitHub refused it.
572 pub fn is_token(&self) -> bool {
573 matches!(self, ListingProblem::NoToken | ListingProblem::Unauthorized)
574 }
575}
576
577impl ListingFailure {
578 /// What fails, as every client says it: `Cannot list halyard/ledger's queued jobs: GitHub answers 404; the token
579 /// cannot see the repository, or it is gone`.
580 pub fn line(&self, repository: &str) -> String {
581 format!(
582 "Cannot list {repository}'s queued jobs: {}",
583 self.problem.why()
584 )
585 }
586}
587
588impl ListingProblem {
589 /// Why the listing fails, as the daemon's log and every client say it: `GitHub answers 502`. Never GitHub's text.
590 pub fn why(&self) -> String {
591 match self {
592 ListingProblem::NoToken => "runnerd has no GitHub token".to_owned(),
593 ListingProblem::Unauthorized => "GitHub refuses runnerd's token (401)".to_owned(),
594 ListingProblem::Refused { status: 403 } => {
595 "GitHub answers 403; the token may not read the repository's Actions".to_owned()
596 }
597 ListingProblem::Refused { status } => format!(
598 "GitHub answers {status}; the token cannot see the repository, or it is gone"
599 ),
600 ListingProblem::Failed { status } => format!("GitHub answers {status}"),
601 ListingProblem::Unreachable => "GitHub does not answer".to_owned(),
602 ListingProblem::Unexpected => "GitHub's answer is not what runnerd reads".to_owned(),
603 ListingProblem::Unknown => "for a reason this client does not know".to_owned(),
604 }
605 }
606}
607
608/// One of a repository's pools: its runners in one profile.
609#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
610pub struct PoolStatus {
611 /// The profile its jobs run in.
612 pub profile: String,
613 /// The labels its runners register with.
614 pub labels: Vec<String>,
615 /// Its jobs in `Status::queued`: those its labels match.
616 pub queued_jobs: usize,
617}
618
619/// A queued job runners of this daemon can take, as GitHub listed it at the last poll.
620#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
621pub struct QueuedJob {
622 /// GitHub's job ID.
623 pub id: u64,
624 /// The repository whose runners can take it, `owner/name`.
625 pub repository: String,
626 /// The profile of its repository's pool whose labels match it; `None` when no pool's do.
627 #[serde(default, skip_serializing_if = "Option::is_none")]
628 pub profile: Option<String>,
629 /// The workflow run it belongs to.
630 pub run_id: u64,
631 /// 1 for the first attempt of the run, higher for re-runs.
632 pub run_attempt: u32,
633 /// The workflow's name.
634 pub workflow: Option<String>,
635 /// The job's name in the workflow.
636 pub name: String,
637 /// The branch the run is for.
638 pub branch: Option<String>,
639 /// The job's page on GitHub.
640 pub url: Option<String>,
641 /// When GitHub queued it, in seconds since the Unix epoch; 0 if GitHub did not say.
642 pub queued_at: u64,
643 /// Why no runner has it yet, as the last plan saw it. `None` before the daemon has planned with it, and while
644 /// it shuts down.
645 #[serde(default, skip_serializing_if = "Option::is_none")]
646 pub waiting: Option<Waiting>,
647}
648
649/// The slot of an idle runner of `job`'s repository, in its pool's profile when known, such as `jvm 2`.
650fn starting_for(job: &QueuedJob, status: &Status) -> Option<String> {
651 status
652 .profiles
653 .iter()
654 .filter(|profile| job.profile.as_ref().is_none_or(|own| *own == profile.name))
655 .find_map(|profile| {
656 profile
657 .slots
658 .iter()
659 .find(|slot| {
660 slot.runner
661 .as_ref()
662 .is_some_and(|runner| runner.repository == job.repository && !runner.busy)
663 })
664 .map(|slot| format!("{}\u{a0}{}", profile.name, slot.slot))
665 })
666}
667
668impl QueuedJob {
669 /// Where it runs, as every client names it: its repository, and its pool's profile when the repository has
670 /// several pools in `status`, such as `halyard/ledger · light`.
671 pub fn place(&self, status: &Status) -> String {
672 let pools = status
673 .repositories
674 .iter()
675 .find(|repository| repository.name == self.repository)
676 .map_or(0, |repository| repository.pools.len());
677 match self.profile.as_deref().filter(|_| pools > 1) {
678 Some(profile) => format!("{}\u{a0}· {profile}", self.repository),
679 None => self.repository.clone(),
680 }
681 }
682}
683
684impl RepositoryStatus {
685 /// The profile of its first pool, which every client colours it by.
686 pub fn first_profile(&self) -> &str {
687 self.pools.first().map_or("", |pool| pool.profile.as_str())
688 }
689
690 /// Its pools' profiles, as every client lists them on one line: `jvm` or `jvm + light`.
691 pub fn profiles(&self) -> String {
692 self.pools
693 .iter()
694 .map(|pool| pool.profile.as_str())
695 .collect::<Vec<_>>()
696 .join(" + ")
697 }
698
699 /// The profile of its pool whose labels match a job asking for `labels`, as the daemon picks it.
700 pub fn pool_for(&self, labels: &[String]) -> Option<String> {
701 self.pools
702 .iter()
703 .find(|pool| crate::config::labels_match(labels, &pool.labels))
704 .map(|pool| pool.profile.clone())
705 }
706}
707
708/// What failed, that holds a repository's runners back for a while.
709#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
710#[serde(rename_all = "kebab-case")]
711pub enum HoldCause {
712 /// Its runners ended without connecting, so without taking a job (ADR 0104).
713 NoJob,
714 /// Its runners connected and ended without a job, one after the other.
715 IdleExits,
716 /// GitHub did not register its runner.
717 Register,
718 /// Docker did not start its runner's container.
719 Start,
720}
721
722/// Why a queued job has no runner of its own yet. Jobs are served oldest first; each takes the first of these that
723/// holds for it.
724#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
725#[serde(rename_all = "kebab-case", tag = "kind")]
726pub enum Waiting {
727 /// Its repository is paused.
728 Paused,
729 /// Runners of its repository failed, so none starts before `until` (seconds since the Unix epoch).
730 Backoff {
731 /// When runners may start again.
732 until: u64,
733 /// What failed.
734 cause: HoldCause,
735 },
736 /// A runner started for it and waits for GitHub to hand it the job: a runner takes any job its labels match,
737 /// so not necessarily this one.
738 RunnerStarting,
739 /// Slot 1 of its profile is becoming the profile's snapshot, so none of the profile's slots starts.
740 Reseeding {
741 /// The profile.
742 profile: String,
743 },
744 /// Its profile's image is being pulled, so none of the profile's slots starts.
745 PullingImage {
746 /// The profile.
747 profile: String,
748 },
749 /// Every slot of its profile holds a runner.
750 NoSlot {
751 /// The profile.
752 profile: String,
753 },
754 /// What its runner would reserve does not fit the memory budget left. Under measured admission, `free` is 0
755 /// while the slice is at the high-water mark (`Budget::held`).
756 Memory {
757 /// What its runner would reserve, in bytes.
758 needs: u64,
759 /// What the budget has left, in bytes.
760 free: u64,
761 },
762 /// What its runner would reserve does not fit what the host's memory has to spare beside its other programs
763 /// (`HostBudget::spare`), though it fits the slice's budget. `free` is 0 while the host's tasks stall for memory
764 /// (`HostBudget::stalled`).
765 HostMemory {
766 /// What its runner would reserve, in bytes.
767 needs: u64,
768 /// What the host has to spare, in bytes.
769 free: u64,
770 },
771 /// What its runner would reserve does not fit the CPU budget left, in millicores.
772 Cpu {
773 /// What its runner would reserve.
774 needs: u64,
775 /// What the CPU budget has left.
776 free: u64,
777 },
778 /// An older job has waited long enough for CPU that no younger one takes CPU before it.
779 BehindCpu {
780 /// The older job's ID.
781 job: u64,
782 },
783 /// An older job has waited long enough for memory that no younger one starts before it (ADR 0096).
784 BehindMemory {
785 /// The older job's ID.
786 job: u64,
787 },
788 /// An older job is about to have a runner start for it, which takes the slot, memory or CPU this one would
789 /// otherwise start with: what holds it is that start, not what runs now.
790 BehindStart {
791 /// The older job's ID.
792 job: u64,
793 },
794 /// No pool of its repository has every label it asks for, so no runner of this daemon can take it (ADR 0058).
795 NoPool {
796 /// The labels it asks for.
797 labels: Vec<String>,
798 },
799 /// A host crash may have torn what runners wrote without syncing, and the daemon has not emptied it yet, so no
800 /// runner starts (ADR 0141, 0142).
801 Torn {
802 /// The volumes a container keeps it from removing, as of its last try: the owner's to remove once nothing
803 /// mounts them. Empty while it has not tried, or Docker failed.
804 volumes: Vec<String>,
805 },
806 /// A reason from a newer daemon than this client knows, so a reason added later does not keep the client from
807 /// reading the status.
808 #[serde(other)]
809 Unknown,
810}
811
812/// What a [`Waiting`] reason waits on, which clients draw it by.
813#[derive(Debug, Clone, Copy, PartialEq, Eq)]
814pub enum WaitKind {
815 /// Something the owner can act on: a pause, runners that failed to take a job, or labels no pool serves.
816 Act,
817 /// Something that clears by itself shortly: a runner starting, a reseed, an image pull, an older job's start.
818 Soon,
819 /// Room: a slot, memory or CPU that a running job holds.
820 Full,
821 /// A reason this client does not know (`Waiting::Unknown`), drawn neutral.
822 Other,
823}
824
825impl Waiting {
826 /// Whether runnerd cannot start the job with the config it runs with: its repository is paused, or no pool serves
827 /// its labels. Not that nothing ever will: a resume or a config change can, and another machine's runner may take
828 /// it. A backoff ends by itself, and a reason this client does not know says nothing (ADR 0132).
829 pub fn cannot_start(&self) -> bool {
830 matches!(self, Waiting::Paused | Waiting::NoPool { .. })
831 }
832
833 /// What it waits on.
834 pub fn kind(&self) -> WaitKind {
835 match self {
836 Waiting::Paused | Waiting::Backoff { .. } | Waiting::NoPool { .. } => WaitKind::Act,
837 Waiting::Torn { volumes } if !volumes.is_empty() => WaitKind::Act,
838 Waiting::Unknown => WaitKind::Other,
839 Waiting::Torn { .. }
840 | Waiting::RunnerStarting
841 | Waiting::Reseeding { .. }
842 | Waiting::PullingImage { .. }
843 | Waiting::BehindStart { .. } => WaitKind::Soon,
844 Waiting::NoSlot { .. }
845 | Waiting::Memory { .. }
846 | Waiting::HostMemory { .. }
847 | Waiting::Cpu { .. }
848 | Waiting::BehindCpu { .. }
849 | Waiting::BehindMemory { .. } => WaitKind::Full,
850 }
851 }
852
853 /// Why `job` waits, in the owner's words with the numbers that hold it, as every client says it: such as
854 /// `All 2 node slots are busy` or `Needs 9G of memory; 2.5G of the budget is free`. `status` is the one `job`
855 /// is queued in, which names the slot of a starting runner and the older job one waits behind.
856 pub fn reason(&self, job: &QueuedJob, status: &Status, now: u64) -> String {
857 let size = |bytes: u64| crate::config::ByteSize(bytes).to_string();
858 let older_name = |id: u64| {
859 status
860 .queued
861 .iter()
862 .find(|candidate| candidate.id == id)
863 .map_or_else(|| "an older job".to_owned(), |older| older.name.clone())
864 };
865 match self {
866 Waiting::Paused => format!("{} is paused", job.repository),
867 Waiting::Unknown => "Waits for a reason this client does not know yet".to_owned(),
868 Waiting::NoPool { labels } if labels.is_empty() => {
869 "It asks for no labels, so no profile serves it".to_owned()
870 }
871 Waiting::NoPool { labels } => format!(
872 "No profile serves the {} {}",
873 if labels.len() == 1 { "label" } else { "labels" },
874 labels.join(", ")
875 ),
876 Waiting::Backoff { until, cause } => format!(
877 "{}; retrying in {}",
878 match cause {
879 HoldCause::NoJob => "Runners failed to take a job",
880 HoldCause::IdleExits => "Runners ended without a job after waiting for one",
881 HoldCause::Register => "Runners cannot register with GitHub",
882 HoldCause::Start => "Runners cannot start",
883 },
884 duration(until.saturating_sub(now))
885 ),
886 Waiting::RunnerStarting => match starting_for(job, status) {
887 Some(slot) => format!("A runner is starting for it in {slot}"),
888 None => "A runner is starting for it".to_owned(),
889 },
890 Waiting::Reseeding { profile } => format!("{profile} is rebuilding its snapshot"),
891 Waiting::Torn { volumes } if volumes.is_empty() => {
892 "Emptying the caches a host crash may have torn".to_owned()
893 }
894 Waiting::Torn { volumes } => format!(
895 "A host crash may have torn caches a container holds; once nothing mounts them: docker volume rm {}",
896 volumes.join(" ")
897 ),
898 Waiting::PullingImage { profile } => format!("Pulling {profile}'s image"),
899 Waiting::NoSlot { profile } => {
900 let slots = status
901 .profiles
902 .iter()
903 .find(|candidate| &candidate.name == profile)
904 .map_or(0, |profile| profile.slots.len());
905 if slots == 1 {
906 format!("The {profile} slot is busy")
907 } else {
908 format!("All {slots} {profile} slots are busy")
909 }
910 }
911 Waiting::Memory { .. } if status.budget.held => {
912 "The slice is past 90% of its MemoryMax, so nothing starts".to_owned()
913 }
914 Waiting::Memory { needs, free } => format!(
915 "Needs {} of memory; {} of the budget is free",
916 size(*needs),
917 size(*free)
918 ),
919 Waiting::HostMemory { needs, free } => host_reason(
920 *needs,
921 *free,
922 status.budget.host.as_ref(),
923 status.budget.memory_in_use,
924 ),
925 Waiting::Cpu { needs, free } => {
926 format!(
927 "Needs {}; {} are free",
928 cpus(*needs),
929 admission::free_cores(*free)
930 )
931 }
932 Waiting::BehindCpu { job: older } => {
933 format!("Behind {}, which waits for CPU", older_name(*older))
934 }
935 Waiting::BehindMemory { job: older } => {
936 format!("Behind {}, which waits for memory", older_name(*older))
937 }
938 Waiting::BehindStart { job: older } => {
939 format!(
940 "Next after {}, whose runner starts first",
941 older_name(*older)
942 )
943 }
944 }
945 }
946}
947
948/// `GET /v1/dashboard`: the dashboard's `[web]` settings, as the daemon read them when it started.
949#[derive(Debug, Clone, Default, PartialEq, Eq, Serialize, Deserialize)]
950pub struct Dashboard {
951 /// `[web] open`: the one of `[web] origins` that `runnerctl web` opens the dashboard on; the loopback address
952 /// when none.
953 #[serde(default, skip_serializing_if = "Option::is_none")]
954 pub open: Option<String>,
955}
956
957/// `POST /v1/dashboard/pairings`: which of `[web] origins` to make a link for.
958#[derive(Debug, Clone, Default, PartialEq, Eq, Serialize, Deserialize)]
959pub struct PairingQuery {
960 /// One of `[web] origins`; each of them when none.
961 #[serde(default, skip_serializing_if = "Option::is_none")]
962 pub origin: Option<String>,
963}
964
965/// `POST /v1/dashboard/pairings`: fresh one-time links that sign a device in to the dashboard.
966#[derive(Debug, Clone, Default, PartialEq, Eq, Serialize, Deserialize)]
967pub struct Pairings {
968 /// One per origin asked for.
969 pub links: Vec<Pairing>,
970}
971
972/// A link that signs one device in to the dashboard on `origin`, and nowhere else, within `lifetime`.
973#[derive(Clone, PartialEq, Eq, Serialize, Deserialize)]
974pub struct Pairing {
975 /// One of `[web] origins`.
976 pub origin: String,
977 /// The link with its one-time code: a credential, which `Debug` leaves out.
978 pub url: String,
979 /// Seconds it works for from when it was made.
980 pub lifetime: u64,
981}
982
983impl std::fmt::Debug for Pairing {
984 fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
985 formatter
986 .debug_struct("Pairing")
987 .field("origin", &self.origin)
988 .field("lifetime", &self.lifetime)
989 .finish_non_exhaustive()
990 }
991}
992
993/// `POST /v1/shutdown`.
994#[derive(Debug, Clone, Default, PartialEq, Eq, Serialize, Deserialize)]
995pub struct ShutdownRequest {
996 /// Kill running jobs and delete their runners instead of waiting for them.
997 #[serde(default)]
998 pub force: bool,
999}
1000
1001/// What `POST /v1/shutdown` answers: how the daemon stops now, which a stop asked for earlier may have made more
1002/// urgent than the request.
1003#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
1004pub struct ShutdownAnswer {
1005 /// How it stops.
1006 pub stopping: Stopping,
1007}
1008
1009/// How a stopping daemon treats running jobs (ADR 0091).
1010#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
1011#[serde(rename_all = "kebab-case")]
1012pub enum Stopping {
1013 /// `runnerd stop`: it waits for running jobs, then removes every runner.
1014 WaitsForJobs,
1015 /// A signal: it removes idle runners and leaves running jobs to the next start.
1016 LeavesJobs,
1017 /// `runnerd stop --force` or a second signal: it kills running jobs.
1018 KillsJobs,
1019 /// A kind of stop from a newer daemon than this client knows, so one added later does not keep the client from
1020 /// reading the status.
1021 #[serde(other)]
1022 Unknown,
1023}
1024
1025impl Stopping {
1026 /// What it does, as every client says it: `shutting down: waits for running jobs`.
1027 pub fn line(self) -> &'static str {
1028 match self {
1029 Stopping::WaitsForJobs => "shutting down: waits for running jobs",
1030 Stopping::LeavesJobs => "shutting down: leaves running jobs to the next start",
1031 Stopping::KillsJobs => "shutting down: kills running jobs",
1032 Stopping::Unknown => "shutting down",
1033 }
1034 }
1035}
1036
1037impl Status {
1038 /// What a stopping daemon does, as every client says it ([`Stopping::line`]); `shutting down` from a daemon
1039 /// that does not say how; `None` while it runs.
1040 pub fn stopping_line(&self) -> Option<&'static str> {
1041 match (self.stopping, self.shutting_down) {
1042 (Some(stopping), _) => Some(stopping.line()),
1043 (None, true) => Some("shutting down"),
1044 (None, false) => None,
1045 }
1046 }
1047
1048 /// Why no runner starts while GitHub's rate limit is used up, as every client says it; `None` while it is not.
1049 pub fn github_line(&self) -> Option<&'static str> {
1050 self.github_limited_until
1051 .map(|_| "GitHub's rate limit is used up: no runner starts until it resets")
1052 }
1053}
1054
1055/// The body of every error response.
1056#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
1057pub struct ErrorResponse {
1058 /// What went wrong, for people to read.
1059 pub message: String,
1060}