Skip to main content

runnerd_core/
api.rs

1//! The daemon's JSON API, version 1: the request and response types every client deserializes, and the
2//! principal a transport authenticates. Plain data only: no I/O, no Docker or GitHub types, no secrets but a
3//! [`Pairing`]'s one-time link, which its `Debug` leaves out.
4//!
5//! Routes, all under the authenticated principal:
6//!
7//! | Route | Body |
8//! |---|---|
9//! | `GET /v1/status` | [`Status`] |
10//! | `POST /v1/repositories/{owner}/{name}/pause?`[`PauseQuery`] | [`Paused`]; 409 [`Busy`] while it runs a job |
11//! | `POST /v1/repositories/{owner}/{name}/resume` | none, 204 |
12//! | `POST /v1/repositories/pause?`[`PauseQuery`] | [`Paused`]; 409 [`Busy`] while any runs a job, pausing none |
13//! | `POST /v1/repositories/resume` | none, 204: every repository resumed |
14//! | `POST /v1/profiles/{name}/reseed` | none, 202; 409 while it reseeds; `cache.reseeded` when done |
15//! | `POST /v1/caches/{name}/clean` | none, 204; 409 [`Busy`] while a runner uses it |
16//! | `GET /v1/config` | [`ConfigFile`] |
17//! | `POST /v1/config/check` | TOML in, [`ConfigCheck`] |
18//! | `PUT /v1/config` | TOML in, [`ConfigCheck`]; 422 [`ConfigCheck`] when invalid |
19//! | `GET /v1/doctor` | [`Doctor`] |
20//! | `POST /v1/shutdown` | [`ShutdownRequest`] in, 202 [`ShutdownAnswer`] |
21//! | `GET /v1/dashboard` | [`Dashboard`] |
22//! | `POST /v1/dashboard/sign-out` | none, 204 |
23//! | `POST /v1/dashboard/pairings?`[`PairingQuery`] | [`Pairings`]; 409 when the dashboard is off or has no `[web] origins` |
24//! | `GET /v1/jobs?`[`JobQuery`] | [`JobPage`] |
25//! | `GET /v1/jobs/{id}` | [`Job`] |
26//! | `GET /v1/jobs/{id}/samples` | [`SampleSeries`] |
27//! | `GET /v1/jobs/{id}/logs?`[`LogQuery`] | `text/plain`, one line per log line |
28//! | `GET /v1/host/samples?`[`HostSampleQuery`] | [`HostSamples`] |
29//! | `GET /v1/stats?`[`StatsQuery`] | [`Stats`] |
30//! | `GET /v1/activity?`[`ActivityQuery`] | [`Activity`] |
31//! | `GET /v1/sizing?`[`SizingQuery`] | [`Sizing`] |
32//! | `GET /v1/insights?`[`InsightsQuery`] | [`Insights`] |
33//! | `GET /v1/digest` | [`Digest`] |
34//! | `GET /v1/savings` | [`Savings`] |
35//! | `POST /v1/notify/test?`[`NotifyTestQuery`] | [`NotifyTest`]; 404 without `[notify]` targets or with no such one |
36//! | `GET /v1/caches` | [`Caches`] |
37//! | `GET /v1/caches/history?`[`CacheHistoryQuery`] | [`CacheHistory`] |
38//! | `GET /v1/events?`[`EventQuery`] | `text/event-stream` of [`EventEnvelope`] |
39//! | `GET /v1/records` | [`Records`] |
40//! | `DELETE /v1/records/{job}` | [`RemovedRecords`]; 409 while a shell is open in it |
41//! | `DELETE /v1/records?`[`RemoveRecordsQuery`] | [`RemovedRecords`]; records with an open shell stay |
42//! | `POST /v1/records/{job}/shell?`[`ShellQuery`] | `Upgrade:` [`SHELL_PROTOCOL`], 101, then the shell's terminal; 409 when the budget is full |
43//!
44//! Errors carry an [`ErrorResponse`].
45
46use serde::{Deserialize, Serialize};
47
48mod admission;
49mod budget;
50mod control;
51mod events;
52mod insights;
53mod jobs;
54mod labels;
55mod records;
56mod results;
57mod savings;
58mod sizing;
59mod slots;
60mod stats;
61mod storage;
62mod tone;
63mod units;
64
65pub use admission::{Admission, AdmissionPolicy, Basis, CpuAdmission, JobPeak, cores, cpus};
66pub use budget::{
67    BudgetParts, ClosedKind, ClosedPart, FreePart, HeldKind, HeldPart, Reading, SlotPart,
68};
69pub use control::{
70    Busy, Check, CheckStatus, ConfigCheck, ConfigFile, Doctor, PauseQuery, Paused, ReplaceQuery,
71    RunningJob,
72};
73pub use events::{Event, EventEnvelope, EventQuery, RunnerFailure};
74pub use insights::{
75    AutoNote, Delivery, Digest, HostHazard, HostSpan, INSIGHT_DAYS, INSIGHT_DAYS_MAX, Insight,
76    InsightKind, Insights, InsightsQuery, LongWait, Notification, NotifyEvent, NotifyTest,
77    NotifyTestQuery, Priority, RepositoryWaits, RunLink, Severity, WaitSpread, WeekWaits,
78};
79pub use jobs::{
80    Activity, ActivityQuery, Bucket, BuildCacheCounts, BuildCacheStats, CacheHistory,
81    CacheHistoryQuery, CacheKind, CacheSize, CacheVolume, Caches, ChartLine, ChartLineKind,
82    ChartLines, Holder, HostMemory, HostSample, HostSampleQuery, HostSamples, Job, JobMetrics,
83    JobPage, JobQuery, JobRunner, JobState, JobSummary, JobTitle, Limit, LimitSource, Limits,
84    LogQuery, LogSummary, Milestone, Outcome, OutcomeKind, Phase, PhaseKind, Pressure,
85    RunComparison, Sample, SampleSeries, SampleUsage, Started, Stats, StatsQuery, StatsRow, Step,
86    Swap, Timeline, Total, host_margin, job_title,
87};
88pub use labels::{Column, View};
89pub use records::{
90    Record, RecordLimits, RecordNote, RecordRemoval, Records, RemoveRecordsQuery, RemovedRecords,
91    SHELL_PROTOCOL, ShellQuery,
92};
93pub use results::{force_pause_consequence, reseed_requested, reseeded};
94pub use savings::{
95    Period, PricedRun, PricedRunner, Pricing, PricingHistory, SELF_HOSTED_RATE, SavedWindow,
96    Savings, Sku, Usd, Visibility, billed_minutes, reads_since,
97};
98pub use sizing::{
99    Advice, AutoPlan, GRANULE, JobSizing, MemoryBound, MemoryHold, Need, PAGE_CACHE_ROOM, Peaks,
100    ProfilePlan, ProfileSizing, Resize, RevertCause, SAMPLE_INTERVAL, SLOT_DISK, Sizing,
101    SizingChange, SizingHold, SizingQuery, SizingRefused, SizingRevert, Spread, Starts, need,
102    period, sampled_note,
103};
104pub use slots::{Figure, Health, SlotState};
105pub use stats::StatsSummary;
106pub use storage::{
107    ACTIONS_CACHE, BUILD_CACHE_TOOLS, BuildCacheActivity, CacheLimit, RECORDS_NEAR_FULL,
108    RecordState, actions_cache_place, records_near_full,
109};
110pub use tone::{Tone, Toned};
111#[cfg(feature = "host")]
112pub(crate) use units::days_from_civil;
113pub use units::{ago, cores_used, cpu_time, duration, in_cores};
114
115/// Who sent a request, as the transport authenticated it. Handlers get it from one place, so a new transport only
116/// has to produce one.
117#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
118#[serde(rename_all = "kebab-case", tag = "kind")]
119pub enum Principal {
120    /// The user the daemon runs as, on the local Unix socket.
121    LocalOwner {
122        /// The Unix user ID the socket's peer runs as.
123        uid: u32,
124    },
125}
126
127/// `GET /v1/status`: what the daemon serves and runs right now.
128#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
129pub struct Status {
130    /// The daemon's version, from its `Cargo.toml`.
131    pub version: String,
132    /// Set once the daemon stops polling to shut down. [`Status::stopping_line`] says how.
133    pub shutting_down: bool,
134    /// How it stops, once it does; absent from a daemon before it said.
135    #[serde(default, skip_serializing_if = "Option::is_none")]
136    pub stopping: Option<Stopping>,
137    /// Until when GitHub's rate limit is used up, in seconds since the Unix epoch: no runner starts before.
138    /// [`Status::github_line`] says it. Absent while it is not.
139    #[serde(default, skip_serializing_if = "Option::is_none")]
140    pub github_limited_until: Option<u64>,
141    /// The memory budget and what runners use of it.
142    pub budget: Budget,
143    /// Every configured profile with its slots.
144    pub profiles: Vec<ProfileStatus>,
145    /// Every configured repository.
146    pub repositories: Vec<RepositoryStatus>,
147    /// Queued jobs the repositories' runners can take and none holds yet, at the last poll, oldest first. No event
148    /// carries the list: `job.queued` announces a job, `job.started` and `job.finished` its leaving the queue.
149    /// A repository whose last listing failed, as while GitHub rate-limits runnerd, or that is no longer polled
150    /// because runnerd is shutting down, shows no jobs here, while the history keeps its jobs queued.
151    pub queued: Vec<QueuedJob>,
152}
153
154/// The memory budget runners and the mirror share, in bytes.
155#[derive(Debug, Clone, Default, PartialEq, Eq, Serialize, Deserialize)]
156pub struct Budget {
157    /// The systemd slice runner containers run in.
158    pub slice: String,
159    /// The slice's `MemoryMax`; `None` when it is unlimited.
160    pub memory_max: Option<u64>,
161    /// Memory the running containers reserve: the runners' caps, or under measured admission the larger of each
162    /// runner's predicted peak and its last sampled memory, nothing for one whose container is gone, and the caps of
163    /// the seed containers, shells and the mirror.
164    pub memory_in_use: u64,
165    /// What pooling takes off `memory_in_use` (ADR 0099, 0130); absent while it takes nothing.
166    #[serde(default, skip_serializing_if = "Option::is_none")]
167    pub pooled: Option<Pooled>,
168    /// The growth count both budgets go by, and its brake (ADR 0130); `pooled` and `host.credit` say what it takes off
169    /// `memory_in_use`. Absent under caps admission, with `[host] pooled_count` off, and from an older daemon.
170    #[serde(default, skip_serializing_if = "Option::is_none")]
171    pub counted: Option<Counted>,
172    /// What a new runner must fit in beside them.
173    pub admission: AdmissionPolicy,
174    /// Under measured admission, whether the slice is at the high-water mark, so no runner starts.
175    pub held: bool,
176    /// The mirror's part of `memory_in_use`: its cap while it is enabled or still running, else 0.
177    pub mirror_memory: u64,
178    /// The seed containers' part of `memory_in_use`, one per profile that reseeds, by profile name.
179    pub seed_memory: Vec<SeedMemory>,
180    /// The shells' part of `memory_in_use`, one per shell open in a record, by job.
181    pub shell_memory: Vec<ShellMemory>,
182    /// What the host's memory leaves runners beside its other programs, from a recent reading of it; absent
183    /// without one.
184    #[serde(default, skip_serializing_if = "Option::is_none")]
185    pub host: Option<HostBudget>,
186    /// The CPU budget, under `[host] cpu_overcommit`. Absent without one.
187    #[serde(default, skip_serializing_if = "Option::is_none")]
188    pub cpu: Option<CpuBudget>,
189    /// The CPUs the slice may use, in millicores: its `AllowedCPUs` or the host's, less under a lower `CPUQuota`,
190    /// whether or not a CPU budget admits runners. The runners' CPU limits are quotas that may add up past it.
191    /// Absent when it could not be read.
192    #[serde(default, skip_serializing_if = "Option::is_none")]
193    pub slice_cpus: Option<u64>,
194}
195
196/// The count the budgets go by while pooling takes something off `Budget::memory_in_use` (ADR 0099).
197#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
198pub struct Pooled {
199    /// What it takes off, in bytes.
200    pub credit: u64,
201    /// `Budget::memory_in_use` less `credit`: what new runners fit beside under `MemoryMax`. The host's limit reads
202    /// `memory_in_use`.
203    pub counted: u64,
204    /// How many runners the credit counts on.
205    pub runners: u64,
206}
207
208/// How far back the growth count's brake compares what runs uses: 10 minutes (ADR 0130,
209/// `admission::growth::GROWTH_WINDOW`).
210pub const GROWTH_WINDOW: std::time::Duration = std::time::Duration::from_mins(10);
211
212/// The growth count (ADR 0130): what the runners use, and the room kept for what they may still grow by.
213#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
214pub struct Counted {
215    /// What the runners use, by their recent samples, in bytes.
216    pub used: u64,
217    /// The room it keeps for their growth, in bytes; absent while it rests (`brake`).
218    #[serde(default, skip_serializing_if = "Option::is_none")]
219    pub kept: Option<u64>,
220    /// How many of the largest growths the room covers at least.
221    pub jobs: u64,
222    /// Why it rests, while it does: the budgets then go by 0099's and 0108's credits.
223    #[serde(default, skip_serializing_if = "Option::is_none")]
224    pub brake: Option<CountBrake>,
225}
226
227/// Why the growth count rests (ADR 0130).
228#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
229#[serde(rename_all = "kebab-case", tag = "kind")]
230pub enum CountBrake {
231    /// What ran grew past the room kept within `GROWTH_WINDOW`.
232    Grew {
233        /// How much the runners present at both looks grew, together, in bytes.
234        grew: u64,
235        /// The room the earlier look kept, in bytes.
236        kept: u64,
237        /// Until when, in seconds since the Unix epoch.
238        until: u64,
239    },
240    /// The daemon started, or a reload changed what the count reads, too recently to judge growth.
241    Restarted {
242        /// Until when, in seconds since the Unix epoch.
243        until: u64,
244    },
245    /// The host is under the guard's line, or a reading is missing, now or at a look within 5 minutes.
246    Guard {
247        /// Until when, in seconds since the Unix epoch; absent for a moment the brake has not looked at yet.
248        #[serde(default, skip_serializing_if = "Option::is_none")]
249        until: Option<u64>,
250    },
251    /// A brake a newer daemon names; this client shows only that the count rests.
252    #[serde(other)]
253    Unknown,
254}
255
256/// What the host's memory leaves the runners (ADR 0055). New runners fit in the lower of `MemoryMax` and
257/// `limit`; running ones are never stopped for it.
258#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
259pub struct HostBudget {
260    /// The reading it comes from.
261    pub memory: HostMemory,
262    /// What the slice may hold now: what its runners hold without page cache plus the host's `MemAvailable`, less
263    /// a margin kept free for the host's page cache; with a ceiling, plus what the host can still grow by
264    /// (`HostMemory::growable`, ADR 0084).
265    pub limit: u64,
266    /// What that leaves new runners beside what the running ones reserve, in bytes.
267    pub spare: u64,
268    /// Whether the host's tasks stall for memory (`full` pressure), so no runner starts until they stop.
269    pub stalled: bool,
270    /// What the host's limit takes off `Budget::memory_in_use` for runners that may not all grow at once (ADR 0108);
271    /// absent while it takes nothing, and from a daemon before 0.0.30.
272    #[serde(default, skip_serializing_if = "Option::is_none")]
273    pub credit: Option<HostCredit>,
274}
275
276/// What the host's limit takes off the plain count (ADR 0108): `Budget::memory_in_use` less `bytes` is what it counts,
277/// `used` of that is what the runners use, and the rest is room kept for what runs to grow into.
278#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
279pub struct HostCredit {
280    /// What it takes off, in bytes.
281    pub bytes: u64,
282    /// What the runners use, by their recent samples, in bytes.
283    pub used: u64,
284}
285
286impl HostBudget {
287    /// Whether the host itself lacks `needs` bytes for a new runner: its tasks stall, `HostMemory::room` is less, or
288    /// that room is under one more margin, so `MemAvailable` is under twice the plain margin, the line `host::stalls`
289    /// reads. There a small job fits only by what the host is about to run out of, and what closing programs frees is
290    /// what starts it. Otherwise a job the host limit holds waits for what runs, whose reservations reach the limit
291    /// (ADR 0103).
292    pub fn short_of(&self, needs: u64) -> bool {
293        let room = self.memory.room();
294        self.stalled || needs > room || room < host_margin(self.memory.grown())
295    }
296
297    /// What the host's limit counts of what runs, `reserved` being `Budget::memory_in_use`: that less the credit
298    /// (ADR 0108).
299    pub fn counts(&self, reserved: u64) -> u64 {
300        reserved.saturating_sub(self.credit.map_or(0, |credit| credit.bytes))
301    }
302
303    /// What that count is made of while a credit stands, as the host's line, a job's wait and the log say it: `it uses
304    /// 14G and keeps 9G for growth`. `None` without a credit, when it counts every reservation.
305    pub fn growth_clause(&self, reserved: u64) -> Option<String> {
306        let credit = self.credit?;
307        let counts = self.counts(reserved);
308        Some(format!(
309            "it uses {} and keeps {} for growth",
310            crate::config::ByteSize(credit.used.min(counts)),
311            crate::config::ByteSize(counts.saturating_sub(credit.used))
312        ))
313    }
314}
315
316/// Why a job waits for the host's limit, needing `needs` bytes with `free` left of it, by the latest reading: for what
317/// runs, or for a host that lacks the room, whose programs it names. While the host's limit takes a credit, what runs
318/// is what it uses and the room it keeps for growth, not its reservations (ADR 0108); `reserved` is
319/// `Budget::memory_in_use`.
320fn host_reason(needs: u64, free: u64, host: Option<&HostBudget>, reserved: u64) -> String {
321    let size = |bytes: u64| crate::config::ByteSize(bytes).to_string();
322    if let Some(host) = host.filter(|host| !host.short_of(needs)) {
323        if let Some(growth) = host.growth_clause(reserved) {
324            return format!(
325                "The host has {}, but what runs may still grow into it: {growth}; needs {}, {} is free",
326                host.memory.available_line(),
327                size(needs),
328                size(free)
329            );
330        }
331        return format!(
332            "The host has {}, but what runs may still grow into it: needs {}; {} of the {} it leaves the slice is free",
333            host.memory.available_line(),
334            size(needs),
335            size(free),
336            size(host.limit)
337        );
338    }
339    let reason = if host.is_some_and(|host| host.stalled) {
340        "The host is short of memory: its programs stall waiting for it, so nothing starts"
341            .to_owned()
342    } else {
343        format!(
344            "The host is short of memory: needs {}; the host has {} to spare",
345            size(needs),
346            size(free)
347        )
348    };
349    match host {
350        Some(host) => format!("{reason}; {}", host.memory.outside_line()),
351        None => reason,
352    }
353}
354
355/// Hundredths of a percent as a percentage: `1.5%`, `12%`.
356pub fn percent(hundredths: u32) -> String {
357    let whole = hundredths / 100;
358    match hundredths % 100 / 10 {
359        0 => format!("{whole}%"),
360        tenth => format!("{whole}.{tenth}%"),
361    }
362}
363
364/// The CPU budget runners share, in millicores.
365#[derive(Debug, Clone, Copy, Default, PartialEq, Eq, Serialize, Deserialize)]
366pub struct CpuBudget {
367    /// The CPUs the slice may use: its `AllowedCPUs` or the host's, less under a lower `CPUQuota`.
368    pub cpus: u64,
369    /// Those times `cpu_overcommit`: what the runners' reservations must fit.
370    pub budget: u64,
371    /// What the runners reserve: each its predicted average CPU under measured admission, else its `cpus`.
372    pub in_use: u64,
373    /// Whether a queued job waits for CPU that memory and a slot have room for.
374    pub held: bool,
375}
376
377impl CpuBudget {
378    /// The budget's line as clients show it beside memory, such as `CPU 4.5 of 32 CPUs reserved` or, while a job
379    /// waits for it, `CPU 30 of 32 CPUs reserved; a job waits for CPU`.
380    pub fn line(&self) -> String {
381        let mut line = format!(
382            "CPU {} of {} CPUs reserved",
383            cores(self.in_use),
384            cores(self.budget)
385        );
386        if self.held {
387            line.push_str("; a job waits for CPU");
388        }
389        line
390    }
391}
392
393impl Budget {
394    /// How runners are admitted, as clients add it to the budget's line: nothing under caps, else `by predicted
395    /// peaks`, and why nothing starts while the slice is at the high-water mark.
396    pub fn admission_note(&self) -> Option<&'static str> {
397        match (self.admission, self.held) {
398            (AdmissionPolicy::Caps, _) => None,
399            (AdmissionPolicy::Measured, false) => Some("by predicted peaks"),
400            (AdmissionPolicy::Measured, true) => Some(
401                "by predicted peaks; the slice is past 90% of its MemoryMax, so no runner starts",
402            ),
403        }
404    }
405}
406
407/// A shell container's memory cap: its record's profile's limit when the shell opened.
408#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
409pub struct ShellMemory {
410    /// The job whose record the shell runs.
411    pub job: u64,
412    /// Its cap in bytes.
413    pub memory: u64,
414}
415
416/// A seed container's memory cap: its profile's limit when the reseed started, or, for a requested reseed whose
417/// profile drained, the room it holds until it starts.
418#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
419pub struct SeedMemory {
420    /// The profile it reseeds.
421    pub profile: String,
422    /// Its cap in bytes.
423    pub memory: u64,
424}
425
426/// A runner profile and what runs in its slots.
427#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
428pub struct ProfileStatus {
429    /// The name under `[profiles]` in `runnerd.toml`.
430    pub name: String,
431    /// The image its runners run, such as `runnerd-runner:base`.
432    pub image: String,
433    /// CPUs per slot.
434    pub cpus: u32,
435    /// Bytes per slot.
436    pub memory: u64,
437    /// Set while slot 1's layers become the profile's snapshot.
438    pub reseeding: bool,
439    /// Index 0 is slot 1.
440    pub slots: Vec<SlotStatus>,
441}
442
443/// One slot of a profile: room for one job.
444#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
445pub struct SlotStatus {
446    /// 1-based.
447    pub slot: usize,
448    /// `None` when the slot is free.
449    pub runner: Option<RunnerStatus>,
450}
451
452/// A runner registered for one job, in its container.
453#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
454pub struct RunnerStatus {
455    /// `owner/name` the runner is registered on.
456    pub repository: String,
457    /// The runner's name, also its container's.
458    pub name: String,
459    /// The runner's ID on GitHub.
460    pub runner_id: u64,
461    /// The Docker container ID.
462    pub container: String,
463    /// Whether it runs a job, as GitHub or its log reports it: its log names the job before GitHub lists it busy.
464    pub busy: bool,
465    /// Seconds since the Unix epoch.
466    pub started_at: u64,
467    /// What it takes of the budget: its cap under caps; under measured admission the larger of its predicted peak
468    /// and its last sampled memory.
469    pub reserved: u64,
470    /// A warm runner: idle, and no queued job of its repository waits for it; it takes the next one GitHub hands
471    /// out.
472    pub warm: bool,
473}
474
475/// A configured repository and its queue.
476#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
477pub struct RepositoryStatus {
478    /// `owner/name` on GitHub.
479    pub name: String,
480    /// Its pools in config order, one per `[[repositories]]` entry: a job runs in the one whose labels match it
481    /// (ADR 0058).
482    pub pools: Vec<PoolStatus>,
483    /// Paused repositories get no new runners; their running jobs finish.
484    pub paused: bool,
485    /// Its jobs in `Status::queued`: queued ones none of its runners took yet, those no pool serves too.
486    pub queued_jobs: usize,
487}
488
489/// One of a repository's pools: its runners in one profile.
490#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
491pub struct PoolStatus {
492    /// The profile its jobs run in.
493    pub profile: String,
494    /// The labels its runners register with.
495    pub labels: Vec<String>,
496    /// Its jobs in `Status::queued`: those its labels match.
497    pub queued_jobs: usize,
498}
499
500/// A queued job runners of this daemon can take, as GitHub listed it at the last poll.
501#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
502pub struct QueuedJob {
503    /// GitHub's job ID.
504    pub id: u64,
505    /// The repository whose runners can take it, `owner/name`.
506    pub repository: String,
507    /// The profile of its repository's pool whose labels match it; `None` when no pool's do.
508    #[serde(default, skip_serializing_if = "Option::is_none")]
509    pub profile: Option<String>,
510    /// The workflow run it belongs to.
511    pub run_id: u64,
512    /// 1 for the first attempt of the run, higher for re-runs.
513    pub run_attempt: u32,
514    /// The workflow's name.
515    pub workflow: Option<String>,
516    /// The job's name in the workflow.
517    pub name: String,
518    /// The branch the run is for.
519    pub branch: Option<String>,
520    /// The job's page on GitHub.
521    pub url: Option<String>,
522    /// When GitHub queued it, in seconds since the Unix epoch; 0 if GitHub did not say.
523    pub queued_at: u64,
524    /// Why no runner has it yet, as the last plan saw it. `None` before the daemon has planned with it, and while
525    /// it shuts down.
526    #[serde(default, skip_serializing_if = "Option::is_none")]
527    pub waiting: Option<Waiting>,
528}
529
530/// The slot of an idle runner of `job`'s repository, in its pool's profile when known, such as `jvm 2`.
531fn starting_for(job: &QueuedJob, status: &Status) -> Option<String> {
532    status
533        .profiles
534        .iter()
535        .filter(|profile| job.profile.as_ref().is_none_or(|own| *own == profile.name))
536        .find_map(|profile| {
537            profile
538                .slots
539                .iter()
540                .find(|slot| {
541                    slot.runner
542                        .as_ref()
543                        .is_some_and(|runner| runner.repository == job.repository && !runner.busy)
544                })
545                .map(|slot| format!("{}\u{a0}{}", profile.name, slot.slot))
546        })
547}
548
549impl QueuedJob {
550    /// Where it runs, as every client names it: its repository, and its pool's profile when the repository has
551    /// several pools in `status`, such as `halyard/ledger · light`.
552    pub fn place(&self, status: &Status) -> String {
553        let pools = status
554            .repositories
555            .iter()
556            .find(|repository| repository.name == self.repository)
557            .map_or(0, |repository| repository.pools.len());
558        match self.profile.as_deref().filter(|_| pools > 1) {
559            Some(profile) => format!("{}\u{a0}· {profile}", self.repository),
560            None => self.repository.clone(),
561        }
562    }
563}
564
565impl RepositoryStatus {
566    /// The profile of its first pool, which every client colours it by.
567    pub fn first_profile(&self) -> &str {
568        self.pools.first().map_or("", |pool| pool.profile.as_str())
569    }
570
571    /// Its pools' profiles, as every client lists them on one line: `jvm` or `jvm + light`.
572    pub fn profiles(&self) -> String {
573        self.pools
574            .iter()
575            .map(|pool| pool.profile.as_str())
576            .collect::<Vec<_>>()
577            .join(" + ")
578    }
579
580    /// The profile of its pool whose labels match a job asking for `labels`, as the daemon picks it.
581    pub fn pool_for(&self, labels: &[String]) -> Option<String> {
582        self.pools
583            .iter()
584            .find(|pool| crate::config::labels_match(labels, &pool.labels))
585            .map(|pool| pool.profile.clone())
586    }
587}
588
589/// What failed, that holds a repository's runners back for a while.
590#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
591#[serde(rename_all = "kebab-case")]
592pub enum HoldCause {
593    /// Its runners ended without connecting, so without taking a job (ADR 0104).
594    NoJob,
595    /// Its runners connected and ended without a job, one after the other.
596    IdleExits,
597    /// GitHub did not register its runner.
598    Register,
599    /// Docker did not start its runner's container.
600    Start,
601}
602
603/// Why a queued job has no runner of its own yet. Jobs are served oldest first; each takes the first of these that
604/// holds for it.
605#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
606#[serde(rename_all = "kebab-case", tag = "kind")]
607pub enum Waiting {
608    /// Its repository is paused.
609    Paused,
610    /// Runners of its repository failed, so none starts before `until` (seconds since the Unix epoch).
611    Backoff {
612        /// When runners may start again.
613        until: u64,
614        /// What failed.
615        cause: HoldCause,
616    },
617    /// A runner started for it and waits for GitHub to hand it the job: a runner takes any job its labels match,
618    /// so not necessarily this one.
619    RunnerStarting,
620    /// Slot 1 of its profile is becoming the profile's snapshot, so none of the profile's slots starts.
621    Reseeding {
622        /// The profile.
623        profile: String,
624    },
625    /// Its profile's image is being pulled, so none of the profile's slots starts.
626    PullingImage {
627        /// The profile.
628        profile: String,
629    },
630    /// Every slot of its profile holds a runner.
631    NoSlot {
632        /// The profile.
633        profile: String,
634    },
635    /// What its runner would reserve does not fit the memory budget left. Under measured admission, `free` is 0
636    /// while the slice is at the high-water mark (`Budget::held`).
637    Memory {
638        /// What its runner would reserve, in bytes.
639        needs: u64,
640        /// What the budget has left, in bytes.
641        free: u64,
642    },
643    /// What its runner would reserve does not fit what the host's memory has to spare beside its other programs
644    /// (`HostBudget::spare`), though it fits the slice's budget. `free` is 0 while the host's tasks stall for memory
645    /// (`HostBudget::stalled`).
646    HostMemory {
647        /// What its runner would reserve, in bytes.
648        needs: u64,
649        /// What the host has to spare, in bytes.
650        free: u64,
651    },
652    /// What its runner would reserve does not fit the CPU budget left, in millicores.
653    Cpu {
654        /// What its runner would reserve.
655        needs: u64,
656        /// What the CPU budget has left.
657        free: u64,
658    },
659    /// An older job has waited long enough for CPU that no younger one takes CPU before it.
660    BehindCpu {
661        /// The older job's ID.
662        job: u64,
663    },
664    /// An older job has waited long enough for memory that no younger one starts before it (ADR 0096).
665    BehindMemory {
666        /// The older job's ID.
667        job: u64,
668    },
669    /// An older job is about to have a runner start for it, which takes the slot, memory or CPU this one would
670    /// otherwise start with: what holds it is that start, not what runs now.
671    BehindStart {
672        /// The older job's ID.
673        job: u64,
674    },
675    /// No pool of its repository has every label it asks for, so no runner of this daemon can take it (ADR 0058).
676    NoPool {
677        /// The labels it asks for.
678        labels: Vec<String>,
679    },
680    /// A reason from a newer daemon than this client knows, so a reason added later does not keep the client from
681    /// reading the status.
682    #[serde(other)]
683    Unknown,
684}
685
686/// What a [`Waiting`] reason waits on, which clients draw it by.
687#[derive(Debug, Clone, Copy, PartialEq, Eq)]
688pub enum WaitKind {
689    /// Something the owner can act on: a pause, runners that failed to take a job, or labels no pool serves.
690    Act,
691    /// Something that clears by itself shortly: a runner starting, a reseed, an image pull, an older job's start.
692    Soon,
693    /// Room: a slot, memory or CPU that a running job holds.
694    Full,
695    /// A reason this client does not know (`Waiting::Unknown`), drawn neutral.
696    Other,
697}
698
699impl Waiting {
700    /// What it waits on.
701    pub fn kind(&self) -> WaitKind {
702        match self {
703            Waiting::Paused | Waiting::Backoff { .. } | Waiting::NoPool { .. } => WaitKind::Act,
704            Waiting::Unknown => WaitKind::Other,
705            Waiting::RunnerStarting
706            | Waiting::Reseeding { .. }
707            | Waiting::PullingImage { .. }
708            | Waiting::BehindStart { .. } => WaitKind::Soon,
709            Waiting::NoSlot { .. }
710            | Waiting::Memory { .. }
711            | Waiting::HostMemory { .. }
712            | Waiting::Cpu { .. }
713            | Waiting::BehindCpu { .. }
714            | Waiting::BehindMemory { .. } => WaitKind::Full,
715        }
716    }
717
718    /// Why `job` waits, in the owner's words with the numbers that hold it, as every client says it: such as
719    /// `All 2 node slots are busy` or `Needs 9G of memory; 2.5G of the budget is free`. `status` is the one `job`
720    /// is queued in, which names the slot of a starting runner and the older job one waits behind.
721    pub fn reason(&self, job: &QueuedJob, status: &Status, now: u64) -> String {
722        let size = |bytes: u64| crate::config::ByteSize(bytes).to_string();
723        let older_name = |id: u64| {
724            status
725                .queued
726                .iter()
727                .find(|candidate| candidate.id == id)
728                .map_or_else(|| "an older job".to_owned(), |older| older.name.clone())
729        };
730        match self {
731            Waiting::Paused => format!("{} is paused", job.repository),
732            Waiting::Unknown => "Waits for a reason this client does not know yet".to_owned(),
733            Waiting::NoPool { labels } if labels.is_empty() => {
734                "It asks for no labels, so no profile serves it".to_owned()
735            }
736            Waiting::NoPool { labels } => format!(
737                "No profile serves the {} {}",
738                if labels.len() == 1 { "label" } else { "labels" },
739                labels.join(", ")
740            ),
741            Waiting::Backoff { until, cause } => format!(
742                "{}; retrying in {}",
743                match cause {
744                    HoldCause::NoJob => "Runners failed to take a job",
745                    HoldCause::IdleExits => "Runners ended without a job after waiting for one",
746                    HoldCause::Register => "Runners cannot register with GitHub",
747                    HoldCause::Start => "Runners cannot start",
748                },
749                duration(until.saturating_sub(now))
750            ),
751            Waiting::RunnerStarting => match starting_for(job, status) {
752                Some(slot) => format!("A runner is starting for it in {slot}"),
753                None => "A runner is starting for it".to_owned(),
754            },
755            Waiting::Reseeding { profile } => format!("{profile} is rebuilding its snapshot"),
756            Waiting::PullingImage { profile } => format!("Pulling {profile}'s image"),
757            Waiting::NoSlot { profile } => {
758                let slots = status
759                    .profiles
760                    .iter()
761                    .find(|candidate| &candidate.name == profile)
762                    .map_or(0, |profile| profile.slots.len());
763                if slots == 1 {
764                    format!("The {profile} slot is busy")
765                } else {
766                    format!("All {slots} {profile} slots are busy")
767                }
768            }
769            Waiting::Memory { .. } if status.budget.held => {
770                "The slice is past 90% of its MemoryMax, so nothing starts".to_owned()
771            }
772            Waiting::Memory { needs, free } => format!(
773                "Needs {} of memory; {} of the budget is free",
774                size(*needs),
775                size(*free)
776            ),
777            Waiting::HostMemory { needs, free } => host_reason(
778                *needs,
779                *free,
780                status.budget.host.as_ref(),
781                status.budget.memory_in_use,
782            ),
783            Waiting::Cpu { needs, free } => {
784                format!(
785                    "Needs {}; {} are free",
786                    cpus(*needs),
787                    admission::free_cores(*free)
788                )
789            }
790            Waiting::BehindCpu { job: older } => {
791                format!("Behind {}, which waits for CPU", older_name(*older))
792            }
793            Waiting::BehindMemory { job: older } => {
794                format!("Behind {}, which waits for memory", older_name(*older))
795            }
796            Waiting::BehindStart { job: older } => {
797                format!(
798                    "Next after {}, whose runner starts first",
799                    older_name(*older)
800                )
801            }
802        }
803    }
804}
805
806/// `GET /v1/dashboard`: the dashboard's `[web]` settings, as the daemon read them when it started.
807#[derive(Debug, Clone, Default, PartialEq, Eq, Serialize, Deserialize)]
808pub struct Dashboard {
809    /// `[web] open`: the one of `[web] origins` that `runnerctl web` opens the dashboard on; the loopback address
810    /// when none.
811    #[serde(default, skip_serializing_if = "Option::is_none")]
812    pub open: Option<String>,
813}
814
815/// `POST /v1/dashboard/pairings`: which of `[web] origins` to make a link for.
816#[derive(Debug, Clone, Default, PartialEq, Eq, Serialize, Deserialize)]
817pub struct PairingQuery {
818    /// One of `[web] origins`; each of them when none.
819    #[serde(default, skip_serializing_if = "Option::is_none")]
820    pub origin: Option<String>,
821}
822
823/// `POST /v1/dashboard/pairings`: fresh one-time links that sign a device in to the dashboard.
824#[derive(Debug, Clone, Default, PartialEq, Eq, Serialize, Deserialize)]
825pub struct Pairings {
826    /// One per origin asked for.
827    pub links: Vec<Pairing>,
828}
829
830/// A link that signs one device in to the dashboard on `origin`, and nowhere else, within `lifetime`.
831#[derive(Clone, PartialEq, Eq, Serialize, Deserialize)]
832pub struct Pairing {
833    /// One of `[web] origins`.
834    pub origin: String,
835    /// The link with its one-time code: a credential, which `Debug` leaves out.
836    pub url: String,
837    /// Seconds it works for from when it was made.
838    pub lifetime: u64,
839}
840
841impl std::fmt::Debug for Pairing {
842    fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
843        formatter
844            .debug_struct("Pairing")
845            .field("origin", &self.origin)
846            .field("lifetime", &self.lifetime)
847            .finish_non_exhaustive()
848    }
849}
850
851/// `POST /v1/shutdown`.
852#[derive(Debug, Clone, Default, PartialEq, Eq, Serialize, Deserialize)]
853pub struct ShutdownRequest {
854    /// Kill running jobs and delete their runners instead of waiting for them.
855    #[serde(default)]
856    pub force: bool,
857}
858
859/// What `POST /v1/shutdown` answers: how the daemon stops now, which a stop asked for earlier may have made more
860/// urgent than the request.
861#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
862pub struct ShutdownAnswer {
863    /// How it stops.
864    pub stopping: Stopping,
865}
866
867/// How a stopping daemon treats running jobs (ADR 0091).
868#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
869#[serde(rename_all = "kebab-case")]
870pub enum Stopping {
871    /// `runnerd stop`: it waits for running jobs, then removes every runner.
872    WaitsForJobs,
873    /// A signal: it removes idle runners and leaves running jobs to the next start.
874    LeavesJobs,
875    /// `runnerd stop --force` or a second signal: it kills running jobs.
876    KillsJobs,
877    /// A kind of stop from a newer daemon than this client knows, so one added later does not keep the client from
878    /// reading the status.
879    #[serde(other)]
880    Unknown,
881}
882
883impl Stopping {
884    /// What it does, as every client says it: `shutting down: waits for running jobs`.
885    pub fn line(self) -> &'static str {
886        match self {
887            Stopping::WaitsForJobs => "shutting down: waits for running jobs",
888            Stopping::LeavesJobs => "shutting down: leaves running jobs to the next start",
889            Stopping::KillsJobs => "shutting down: kills running jobs",
890            Stopping::Unknown => "shutting down",
891        }
892    }
893}
894
895impl Status {
896    /// What a stopping daemon does, as every client says it ([`Stopping::line`]); `shutting down` from a daemon
897    /// that does not say how; `None` while it runs.
898    pub fn stopping_line(&self) -> Option<&'static str> {
899        match (self.stopping, self.shutting_down) {
900            (Some(stopping), _) => Some(stopping.line()),
901            (None, true) => Some("shutting down"),
902            (None, false) => None,
903        }
904    }
905
906    /// Why no runner starts while GitHub's rate limit is used up, as every client says it; `None` while it is not.
907    pub fn github_line(&self) -> Option<&'static str> {
908        self.github_limited_until
909            .map(|_| "GitHub's rate limit is used up: no runner starts until it resets")
910    }
911}
912
913/// The body of every error response.
914#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
915pub struct ErrorResponse {
916    /// What went wrong, for people to read.
917    pub message: String,
918}