Skip to main content

runnerd_core/
api.rs

1//! The daemon's JSON API, version 1: the request and response types every client deserializes, and the
2//! principal a transport authenticates. Plain data only: no I/O, no Docker or GitHub types, no secrets but a
3//! [`Pairing`]'s one-time link, which its `Debug` leaves out.
4//!
5//! Routes, all under the authenticated principal:
6//!
7//! | Route | Body |
8//! |---|---|
9//! | `GET /v1/status` | [`Status`] |
10//! | `POST /v1/repositories/{owner}/{name}/pause?`[`PauseQuery`] | [`Paused`]; 409 [`Busy`] while it runs a job |
11//! | `POST /v1/repositories/{owner}/{name}/resume` | none, 204 |
12//! | `POST /v1/repositories/pause?`[`PauseQuery`] | [`Paused`]; 409 [`Busy`] while any runs a job, pausing none |
13//! | `POST /v1/repositories/resume` | none, 204: every repository resumed |
14//! | `POST /v1/profiles/{name}/reseed` | none, 202; 409 while it reseeds; `cache.reseeded` when done |
15//! | `POST /v1/caches/{name}/clean` | none, 204; 409 [`Busy`] while a runner uses it |
16//! | `GET /v1/config` | [`ConfigFile`] |
17//! | `POST /v1/config/check` | TOML in, [`ConfigCheck`] |
18//! | `PUT /v1/config` | TOML in, [`ConfigCheck`]; 422 [`ConfigCheck`] when invalid |
19//! | `GET /v1/doctor` | [`Doctor`]; 503 [`ErrorResponse`] while no run has finished |
20//! | `GET /v1/attention` | [`Attention`]; 404 from a daemon before 0.2 |
21//! | `POST /v1/shutdown` | [`ShutdownRequest`] in, 202 [`ShutdownAnswer`] |
22//! | `GET /v1/dashboard` | [`Dashboard`] |
23//! | `POST /v1/dashboard/sign-out` | none, 204 |
24//! | `POST /v1/dashboard/pairings?`[`PairingQuery`] | [`Pairings`]; 409 when the dashboard is off or has no `[web] origins` |
25//! | `GET /v1/jobs?`[`JobQuery`] | [`JobPage`] |
26//! | `GET /v1/jobs/{id}` | [`Job`] |
27//! | `GET /v1/jobs/{id}/samples` | [`SampleSeries`] |
28//! | `GET /v1/jobs/{id}/logs?`[`LogQuery`] | `text/plain`, one line per log line |
29//! | `GET /v1/host/samples?`[`HostSampleQuery`] | [`HostSamples`] |
30//! | `GET /v1/stats?`[`StatsQuery`] | [`Stats`] |
31//! | `GET /v1/activity?`[`ActivityQuery`] | [`Activity`] |
32//! | `GET /v1/sizing?`[`SizingQuery`] | [`Sizing`] |
33//! | `GET /v1/insights?`[`InsightsQuery`] | [`Insights`] |
34//! | `GET /v1/digest` | [`Digest`] |
35//! | `GET /v1/savings` | [`Savings`] |
36//! | `POST /v1/notify/test?`[`NotifyTestQuery`] | [`NotifyTest`]; 404 without `[notify]` targets or with no such one |
37//! | `GET /v1/caches` | [`Caches`] |
38//! | `GET /v1/caches/history?`[`CacheHistoryQuery`] | [`CacheHistory`] |
39//! | `GET /v1/events?`[`EventQuery`] | `text/event-stream` of [`EventEnvelope`] |
40//! | `GET /v1/records` | [`Records`] |
41//! | `DELETE /v1/records/{job}` | [`RemovedRecords`]; 409 while a shell is open in it |
42//! | `DELETE /v1/records?`[`RemoveRecordsQuery`] | [`RemovedRecords`]; records with an open shell stay |
43//! | `POST /v1/records/{job}/shell?`[`ShellQuery`] | `Upgrade:` [`SHELL_PROTOCOL`], 101, then the shell's terminal; 409 when the budget is full |
44//!
45//! Errors carry an [`ErrorResponse`].
46
47use serde::{Deserialize, Serialize};
48
49mod actions;
50mod admission;
51mod attention;
52mod budget;
53mod control;
54mod events;
55mod insights;
56mod jobs;
57mod labels;
58mod records;
59mod results;
60mod savings;
61mod sizing;
62mod slots;
63mod stats;
64mod storage;
65mod tone;
66mod units;
67
68pub use actions::{
69    ActionAnswer, ActionOutcome, ActionPreview, ActionQuery, ActionRefusal, ActionResult,
70    ActionTarget, CONFIRM_SECS, JOB_ACTIONS_CHECK, JobAction, JobActions, PreviewFacts, RunSeen,
71    SIBLINGS_MAX, Superseded, UnknownCause, actions_in_flight, actions_no_run,
72    actions_not_configured, actions_off,
73};
74pub use admission::{Admission, AdmissionPolicy, Basis, CpuAdmission, JobPeak, cores, cpus};
75pub use attention::{
76    ATTENTION_MAX, Attention, AttentionItem, AttentionKind, Fix, Subject, attention_items,
77};
78pub use budget::{
79    BudgetParts, ClosedKind, ClosedPart, FreePart, HeldKind, HeldPart, HostFigure, Reading,
80    SlotPart,
81};
82pub use control::{
83    Busy, Check, CheckStatus, ConfigCheck, ConfigFile, Doctor, DoctorState, PauseQuery, Paused,
84    ReplaceQuery, RunningJob,
85};
86pub use events::{Event, EventEnvelope, EventQuery, RunnerFailure};
87pub use insights::{
88    AutoNote, Delivery, Digest, HostHazard, HostSpan, INSIGHT_DAYS, INSIGHT_DAYS_MAX, Insight,
89    InsightKind, Insights, InsightsQuery, LongWait, Notification, NotifyEvent, NotifyTest,
90    NotifyTestQuery, Priority, RepositoryWaits, RunLink, Severity, WaitSpread, WeekWaits,
91};
92#[cfg(feature = "host")]
93pub(crate) use jobs::SAMPLE_GAP_INTERVALS;
94#[cfg(feature = "host")]
95pub(crate) use jobs::TITLE_SOURCE_MAX;
96pub use jobs::{
97    Activity, ActivityQuery, Bucket, BuildCacheCounts, BuildCacheStats, CacheHistory,
98    CacheHistoryQuery, CacheKind, CacheSize, CacheVolume, Caches, ChartLine, ChartLineKind,
99    ChartLines, Holder, HostMemory, HostSample, HostSampleQuery, HostSamples, Job, JobMetrics,
100    JobPage, JobQuery, JobRunner, JobState, JobSummary, JobTitle, Limit, LimitSource, Limits,
101    LogQuery, LogSummary, Milestone, Outcome, OutcomeKind, Phase, PhaseKind, Pressure,
102    RunComparison, Sample, SampleSeries, SampleUsage, Started, Stats, StatsQuery, StatsRow, Step,
103    Swap, Timeline, Total, host_margin, job_title,
104};
105pub use jobs::{
106    BarEnd, Baseline, BaselineRead, COMPARE_STEP_BYTES, COMPARE_STEPS_MAX, ChangeKind,
107    CompareQuery, ComparedSteps, FigureDelta, FigureKind, Gap, GapKind, JOB_LABELS_MAX,
108    JOB_SEARCH_MAX, JOB_TEXT_MAX, JOBS_PAGE_BYTES, JOBS_PAGE_DEFAULT, JOBS_PAGE_MAX, JobComparison,
109    JobList, JobOrder, JobRange, JobSort, Lane, RecordFilter, RecordMark, RunnerSpan, SlotKey,
110    SlotTimeline, SlottedBar, SlottedRunner, SortOrder, Span, StepDelta, StepMatch, StepUsage,
111    TIMELINE_BARS_MAX, TIMELINE_DEFAULT, TIMELINE_WINDOW_MAX, TimelineBar, TimelineQuery,
112    TimelineRows, Unavailable, row_bytes,
113};
114pub use jobs::{
115    Color, FailureSummary, LINE_BYTES_MAX, LINES_DEFAULT, LINES_MAX, LineKind, LineRange, LineRead,
116    LogEvidence, LogGroup, LogLinesQuery, LogMark, LogNeedle, LogOutline, LogSearch,
117    LogSearchQuery, MarkKind, OutlineChange, OutlineParts, Outliner, RUNS_MAX, Run,
118    SEARCH_QUERY_MAX, SavedOutline, Search, StepLines, StepLog, StoredLine, StoredLog, StyledLine,
119};
120pub use labels::{Column, Feature, JOBS_HEADING, NOT_SUPPORTED, Section, View};
121pub use records::{
122    Record, RecordLimits, RecordNote, RecordRemoval, Records, RemoveRecordsQuery, RemovedRecords,
123    SHELL_PROTOCOL, ShellQuery,
124};
125pub use results::{force_pause_consequence, reseed_requested, reseeded};
126pub use savings::{
127    Period, PricedRun, PricedRunner, Pricing, PricingHistory, SELF_HOSTED_RATE, SavedWindow,
128    Savings, Sku, Usd, Visibility, billed_minutes, reads_since,
129};
130pub use sizing::{
131    Advice, AutoPlan, GRANULE, JobSizing, MemoryBound, MemoryHold, Need, PAGE_CACHE_ROOM, Peaks,
132    ProfilePlan, ProfileSizing, Resize, RevertCause, SAMPLE_INTERVAL, SLOT_DISK, Sizing,
133    SizingChange, SizingHold, SizingQuery, SizingRefused, SizingRevert, Spread, Starts, need,
134    period, sampled_note,
135};
136pub use slots::{Figure, Health, HealthKind, SlotState};
137pub use stats::StatsSummary;
138pub use storage::{
139    ACTIONS_CACHE, BUILD_CACHE_TOOLS, BuildCacheActivity, CacheLimit, RECORDS_NEAR_FULL,
140    RecordState, actions_cache_place, records_check, records_near_full,
141};
142pub use tone::{Tone, Toned};
143pub(crate) use units::days_from_civil;
144pub use units::{ago, cores_used, cpu_time, duration, in_cores};
145
146/// Who sent a request, as the transport authenticated it. Handlers get it from one place, so a new transport only
147/// has to produce one.
148#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
149#[serde(rename_all = "kebab-case", tag = "kind")]
150pub enum Principal {
151    /// The user the daemon runs as, on the local Unix socket.
152    LocalOwner {
153        /// The Unix user ID the socket's peer runs as.
154        uid: u32,
155    },
156}
157
158/// `GET /v1/status`: what the daemon serves and runs right now.
159#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
160pub struct Status {
161    /// The daemon's version, from its `Cargo.toml`.
162    pub version: String,
163    /// Set once the daemon stops polling to shut down. [`Status::stopping_line`] says how.
164    pub shutting_down: bool,
165    /// How it stops, once it does; absent from a daemon before it said.
166    #[serde(default, skip_serializing_if = "Option::is_none")]
167    pub stopping: Option<Stopping>,
168    /// Until when GitHub's rate limit is used up, in seconds since the Unix epoch: no runner starts before.
169    /// [`Status::github_line`] says it. Absent while it is not.
170    #[serde(default, skip_serializing_if = "Option::is_none")]
171    pub github_limited_until: Option<u64>,
172    /// The memory budget and what runners use of it.
173    pub budget: Budget,
174    /// Every configured profile with its slots.
175    pub profiles: Vec<ProfileStatus>,
176    /// Every configured repository.
177    pub repositories: Vec<RepositoryStatus>,
178    /// Queued jobs the repositories' runners can take and none holds yet, at the last poll, oldest first. No event
179    /// carries the list: `job.queued` announces a job, `job.started` and `job.finished` its leaving the queue.
180    /// A repository whose last listing failed, as while GitHub rate-limits runnerd, or that is no longer polled
181    /// because runnerd is shutting down, shows no jobs here, while the history keeps its jobs queued.
182    pub queued: Vec<QueuedJob>,
183    /// Whether the daemon cancels and re-runs jobs (ADR 0134); a daemon before 0.2 sends none, read as
184    /// [`JobActions::Unsupported`].
185    #[serde(default)]
186    pub job_actions: JobActions,
187}
188
189/// The memory budget runners and the mirror share, in bytes.
190#[derive(Debug, Clone, Default, PartialEq, Eq, Serialize, Deserialize)]
191pub struct Budget {
192    /// The systemd slice runner containers run in.
193    pub slice: String,
194    /// The slice's `MemoryMax`; `None` when it is unlimited.
195    pub memory_max: Option<u64>,
196    /// Memory the running containers reserve: the runners' caps, or under measured admission the larger of each
197    /// runner's predicted peak and its last sampled memory, nothing for one whose container is gone, and the caps of
198    /// the seed containers, shells and the mirror.
199    pub memory_in_use: u64,
200    /// What pooling takes off `memory_in_use` (ADR 0099, 0130); absent while it takes nothing.
201    #[serde(default, skip_serializing_if = "Option::is_none")]
202    pub pooled: Option<Pooled>,
203    /// The growth count both budgets go by, and its brake (ADR 0130); `pooled` and `host.credit` say what it takes off
204    /// `memory_in_use`. Absent under caps admission, with `[host] pooled_count` off, and from an older daemon.
205    #[serde(default, skip_serializing_if = "Option::is_none")]
206    pub counted: Option<Counted>,
207    /// What a new runner must fit in beside them.
208    pub admission: AdmissionPolicy,
209    /// Under measured admission, whether the slice is at the high-water mark, so no runner starts.
210    pub held: bool,
211    /// The mirror's part of `memory_in_use`: its cap while it is enabled or still running, else 0.
212    pub mirror_memory: u64,
213    /// The seed containers' part of `memory_in_use`, one per profile that reseeds, by profile name.
214    pub seed_memory: Vec<SeedMemory>,
215    /// The shells' part of `memory_in_use`, one per shell open in a record, by job.
216    pub shell_memory: Vec<ShellMemory>,
217    /// What the host's memory leaves runners beside its other programs, from a recent reading of it; absent
218    /// without one.
219    #[serde(default, skip_serializing_if = "Option::is_none")]
220    pub host: Option<HostBudget>,
221    /// The CPU budget, under `[host] cpu_overcommit`. Absent without one.
222    #[serde(default, skip_serializing_if = "Option::is_none")]
223    pub cpu: Option<CpuBudget>,
224    /// The CPUs the slice may use, in millicores: its `AllowedCPUs` or the host's, less under a lower `CPUQuota`,
225    /// whether or not a CPU budget admits runners. The runners' CPU limits are quotas that may add up past it.
226    /// Absent when it could not be read.
227    #[serde(default, skip_serializing_if = "Option::is_none")]
228    pub slice_cpus: Option<u64>,
229}
230
231/// The count the budgets go by while pooling takes something off `Budget::memory_in_use` (ADR 0099).
232#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
233pub struct Pooled {
234    /// What it takes off, in bytes.
235    pub credit: u64,
236    /// `Budget::memory_in_use` less `credit`: what new runners fit beside under `MemoryMax`. The host's limit reads
237    /// `memory_in_use`.
238    pub counted: u64,
239    /// How many runners the credit counts on.
240    pub runners: u64,
241}
242
243/// How far back the growth count's brake compares what runs uses: 10 minutes (ADR 0130,
244/// `admission::growth::GROWTH_WINDOW`).
245pub const GROWTH_WINDOW: std::time::Duration = std::time::Duration::from_mins(10);
246
247/// The growth count (ADR 0130): what the runners use, and the room kept for what they may still grow by.
248#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
249pub struct Counted {
250    /// What the runners use, by their recent samples, in bytes.
251    pub used: u64,
252    /// The room it keeps for their growth, in bytes; absent while it rests (`brake`).
253    #[serde(default, skip_serializing_if = "Option::is_none")]
254    pub kept: Option<u64>,
255    /// How many of the largest growths the room covers at least.
256    pub jobs: u64,
257    /// Why it rests, while it does: the budgets then go by 0099's and 0108's credits.
258    #[serde(default, skip_serializing_if = "Option::is_none")]
259    pub brake: Option<CountBrake>,
260}
261
262/// Why the growth count rests (ADR 0130).
263#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
264#[serde(rename_all = "kebab-case", tag = "kind")]
265pub enum CountBrake {
266    /// What ran grew past the room kept within `GROWTH_WINDOW`.
267    Grew {
268        /// How much the runners present at both looks grew, together, in bytes.
269        grew: u64,
270        /// The room the earlier look kept, in bytes.
271        kept: u64,
272        /// Until when, in seconds since the Unix epoch.
273        until: u64,
274    },
275    /// The daemon started, or a reload changed what the count reads, too recently to judge growth.
276    Restarted {
277        /// Until when, in seconds since the Unix epoch.
278        until: u64,
279    },
280    /// The host is under the guard's line, or a reading is missing, now or at a look within 5 minutes.
281    Guard {
282        /// Until when, in seconds since the Unix epoch; absent for a moment the brake has not looked at yet.
283        #[serde(default, skip_serializing_if = "Option::is_none")]
284        until: Option<u64>,
285    },
286    /// A brake a newer daemon names; this client shows only that the count rests.
287    #[serde(other)]
288    Unknown,
289}
290
291/// What the host's memory leaves the runners (ADR 0055). New runners fit in the lower of `MemoryMax` and
292/// `limit`; running ones are never stopped for it.
293#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
294pub struct HostBudget {
295    /// The reading it comes from.
296    pub memory: HostMemory,
297    /// What the slice may hold now: what its runners hold without page cache plus the host's `MemAvailable`, less
298    /// a margin kept free for the host's page cache; with a ceiling, plus what the host can still grow by
299    /// (`HostMemory::growable`, ADR 0084).
300    pub limit: u64,
301    /// What that leaves new runners beside what the running ones reserve, in bytes.
302    pub spare: u64,
303    /// Whether the host's tasks stall for memory (`full` pressure), so no runner starts until they stop.
304    pub stalled: bool,
305    /// What the host's limit takes off `Budget::memory_in_use` for runners that may not all grow at once (ADR 0108);
306    /// absent while it takes nothing, and from a daemon before 0.0.30.
307    #[serde(default, skip_serializing_if = "Option::is_none")]
308    pub credit: Option<HostCredit>,
309}
310
311/// What the host's limit takes off the plain count (ADR 0108): `Budget::memory_in_use` less `bytes` is what it counts,
312/// `used` of that is what the runners use, and the rest is room kept for what runs to grow into.
313#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
314pub struct HostCredit {
315    /// What it takes off, in bytes.
316    pub bytes: u64,
317    /// What the runners use, by their recent samples, in bytes.
318    pub used: u64,
319}
320
321impl HostBudget {
322    /// Whether the host itself lacks `needs` bytes for a new runner: its tasks stall, `HostMemory::room` is less, or
323    /// that room is under one more margin, so `MemAvailable` is under twice the plain margin, the line `host::stalls`
324    /// reads. There a small job fits only by what the host is about to run out of, and what closing programs frees is
325    /// what starts it. Otherwise a job the host limit holds waits for what runs, whose reservations reach the limit
326    /// (ADR 0103).
327    pub fn short_of(&self, needs: u64) -> bool {
328        let room = self.memory.room();
329        self.stalled || needs > room || room < host_margin(self.memory.grown())
330    }
331
332    /// What the host's limit counts of what runs, `reserved` being `Budget::memory_in_use`: that less the credit
333    /// (ADR 0108).
334    pub fn counts(&self, reserved: u64) -> u64 {
335        reserved.saturating_sub(self.credit.map_or(0, |credit| credit.bytes))
336    }
337
338    /// What that count is made of while a credit stands, as the host's line, a job's wait and the log say it: `it uses
339    /// 14G and keeps 9G for growth`. `None` without a credit, when it counts every reservation.
340    pub fn growth_clause(&self, reserved: u64) -> Option<String> {
341        let credit = self.credit?;
342        let counts = self.counts(reserved);
343        Some(format!(
344            "it uses {} and keeps {} for growth",
345            crate::config::ByteSize(credit.used.min(counts)),
346            crate::config::ByteSize(counts.saturating_sub(credit.used))
347        ))
348    }
349}
350
351/// Why a job waits for the host's limit, needing `needs` bytes with `free` left of it, by the latest reading: for what
352/// runs, or for a host that lacks the room, whose programs it names. While the host's limit takes a credit, what runs
353/// is what it uses and the room it keeps for growth, not its reservations (ADR 0108); `reserved` is
354/// `Budget::memory_in_use`.
355fn host_reason(needs: u64, free: u64, host: Option<&HostBudget>, reserved: u64) -> String {
356    let size = |bytes: u64| crate::config::ByteSize(bytes).to_string();
357    if let Some(host) = host.filter(|host| !host.short_of(needs)) {
358        if let Some(growth) = host.growth_clause(reserved) {
359            return format!(
360                "The host has {}, but what runs may still grow into it: {growth}; needs {}, {} is free",
361                host.memory.available_line(),
362                size(needs),
363                size(free)
364            );
365        }
366        return format!(
367            "The host has {}, but what runs may still grow into it: needs {}; {} of the {} it leaves the slice is free",
368            host.memory.available_line(),
369            size(needs),
370            size(free),
371            size(host.limit)
372        );
373    }
374    let reason = if host.is_some_and(|host| host.stalled) {
375        "The host is short of memory: its programs stall waiting for it, so nothing starts"
376            .to_owned()
377    } else {
378        format!(
379            "The host is short of memory: needs {}; the host has {} to spare",
380            size(needs),
381            size(free)
382        )
383    };
384    match host {
385        Some(host) => format!("{reason}; {}", host.memory.outside_line()),
386        None => reason,
387    }
388}
389
390/// Hundredths of a percent as a percentage: `1.5%`, `12%`.
391pub fn percent(hundredths: u32) -> String {
392    let whole = hundredths / 100;
393    match hundredths % 100 / 10 {
394        0 => format!("{whole}%"),
395        tenth => format!("{whole}.{tenth}%"),
396    }
397}
398
399/// The CPU budget runners share, in millicores.
400#[derive(Debug, Clone, Copy, Default, PartialEq, Eq, Serialize, Deserialize)]
401pub struct CpuBudget {
402    /// The CPUs the slice may use: its `AllowedCPUs` or the host's, less under a lower `CPUQuota`.
403    pub cpus: u64,
404    /// Those times `cpu_overcommit`: what the runners' reservations must fit.
405    pub budget: u64,
406    /// What the runners reserve: each its predicted average CPU under measured admission, else its `cpus`.
407    pub in_use: u64,
408    /// Whether a queued job waits for CPU that memory and a slot have room for.
409    pub held: bool,
410}
411
412impl CpuBudget {
413    /// The budget's line as clients show it beside memory, such as `CPU 4.5 of 32 CPUs reserved` or, while a job
414    /// waits for it, `CPU 30 of 32 CPUs reserved; a job waits for CPU`.
415    pub fn line(&self) -> String {
416        let mut line = format!(
417            "CPU {} of {} CPUs reserved",
418            cores(self.in_use),
419            cores(self.budget)
420        );
421        if self.held {
422            line.push_str("; a job waits for CPU");
423        }
424        line
425    }
426}
427
428impl Budget {
429    /// How runners are admitted, as clients add it to the budget's line: nothing under caps, else `by predicted
430    /// peaks`, and why nothing starts while the slice is at the high-water mark.
431    pub fn admission_note(&self) -> Option<&'static str> {
432        match (self.admission, self.held) {
433            (AdmissionPolicy::Caps, _) => None,
434            (AdmissionPolicy::Measured, false) => Some("by predicted peaks"),
435            (AdmissionPolicy::Measured, true) => Some(
436                "by predicted peaks; the slice is past 90% of its MemoryMax, so no runner starts",
437            ),
438        }
439    }
440}
441
442/// A shell container's memory cap: its record's profile's limit when the shell opened.
443#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
444pub struct ShellMemory {
445    /// The job whose record the shell runs.
446    pub job: u64,
447    /// Its cap in bytes.
448    pub memory: u64,
449}
450
451/// A seed container's memory cap: its profile's limit when the reseed started, or, for a requested reseed whose
452/// profile drained, the room it holds until it starts.
453#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
454pub struct SeedMemory {
455    /// The profile it reseeds.
456    pub profile: String,
457    /// Its cap in bytes.
458    pub memory: u64,
459}
460
461/// A runner profile and what runs in its slots.
462#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
463pub struct ProfileStatus {
464    /// The name under `[profiles]` in `runnerd.toml`.
465    pub name: String,
466    /// The image its runners run, such as `runnerd-runner:base`.
467    pub image: String,
468    /// CPUs per slot.
469    pub cpus: u32,
470    /// Bytes per slot.
471    pub memory: u64,
472    /// Set while slot 1's layers become the profile's snapshot.
473    pub reseeding: bool,
474    /// Index 0 is slot 1.
475    pub slots: Vec<SlotStatus>,
476}
477
478/// One slot of a profile: room for one job.
479#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
480pub struct SlotStatus {
481    /// 1-based.
482    pub slot: usize,
483    /// `None` when the slot is free.
484    pub runner: Option<RunnerStatus>,
485}
486
487/// A runner registered for one job, in its container.
488#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
489pub struct RunnerStatus {
490    /// `owner/name` the runner is registered on.
491    pub repository: String,
492    /// The runner's name, also its container's.
493    pub name: String,
494    /// The runner's ID on GitHub.
495    pub runner_id: u64,
496    /// The Docker container ID.
497    pub container: String,
498    /// Whether it runs a job, as GitHub or its log reports it: its log names the job before GitHub lists it busy.
499    pub busy: bool,
500    /// Seconds since the Unix epoch.
501    pub started_at: u64,
502    /// What it takes of the budget: its cap under caps; under measured admission the larger of its predicted peak
503    /// and its last sampled memory.
504    pub reserved: u64,
505    /// A warm runner: idle, and no queued job of its repository waits for it; it takes the next one GitHub hands
506    /// out.
507    pub warm: bool,
508}
509
510/// A configured repository and its queue.
511#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
512pub struct RepositoryStatus {
513    /// `owner/name` on GitHub.
514    pub name: String,
515    /// Its pools in config order, one per `[[repositories]]` entry: a job runs in the one whose labels match it
516    /// (ADR 0058).
517    pub pools: Vec<PoolStatus>,
518    /// Paused repositories get no new runners; their running jobs finish.
519    pub paused: bool,
520    /// Its jobs in `Status::queued`: queued ones none of its runners took yet, those no pool serves too.
521    pub queued_jobs: usize,
522    /// Why its queued jobs cannot be listed, while the listings fail (ADR 0132); absent while they answer, before the
523    /// first, while GitHub's rate limit stops them, and from a daemon before 0.2.
524    #[serde(default, skip_serializing_if = "Option::is_none")]
525    pub listing: Option<ListingFailure>,
526}
527
528/// Why runnerd cannot list a repository's queued jobs (ADR 0132): what class of failure GitHub's answer was, never its
529/// text. The engine keeps it until a listing answers or a reload removes the repository; a restart forgets it.
530#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
531pub struct ListingFailure {
532    /// When the listings began to fail, in seconds since the Unix epoch.
533    pub since: u64,
534    /// Why the last one failed.
535    pub problem: ListingProblem,
536}
537
538/// What a failed listing ran into.
539#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
540#[serde(rename_all = "kebab-case", tag = "kind")]
541pub enum ListingProblem {
542    /// runnerd could read no GitHub token.
543    NoToken,
544    /// GitHub refused the token: 401.
545    Unauthorized,
546    /// GitHub answered 403, 404 or 410: the token may not read the repository's Actions, cannot see it, or it is gone.
547    Refused {
548        /// The HTTP status.
549        status: u16,
550    },
551    /// GitHub answered another error status, such as a 5xx.
552    Failed {
553        /// The HTTP status.
554        status: u16,
555    },
556    /// No answer came: the network, DNS or TLS failed.
557    Unreachable,
558    /// The answer was not what GitHub sends.
559    Unexpected,
560    /// A problem from a newer daemon than this client knows.
561    #[serde(other)]
562    Unknown,
563}
564
565impl ListingProblem {
566    /// Whether it is the token's, and so every repository's: none read, or GitHub refused it.
567    pub fn is_token(&self) -> bool {
568        matches!(self, ListingProblem::NoToken | ListingProblem::Unauthorized)
569    }
570}
571
572impl ListingFailure {
573    /// What fails, as every client says it: `Cannot list halyard/ledger's queued jobs: GitHub answers 404; the token
574    /// cannot see the repository, or it is gone`.
575    pub fn line(&self, repository: &str) -> String {
576        format!(
577            "Cannot list {repository}'s queued jobs: {}",
578            self.problem.why()
579        )
580    }
581}
582
583impl ListingProblem {
584    /// Why the listing fails, as the daemon's log and every client say it: `GitHub answers 502`. Never GitHub's text.
585    pub fn why(&self) -> String {
586        match self {
587            ListingProblem::NoToken => "runnerd has no GitHub token".to_owned(),
588            ListingProblem::Unauthorized => "GitHub refuses runnerd's token (401)".to_owned(),
589            ListingProblem::Refused { status: 403 } => {
590                "GitHub answers 403; the token may not read the repository's Actions".to_owned()
591            }
592            ListingProblem::Refused { status } => format!(
593                "GitHub answers {status}; the token cannot see the repository, or it is gone"
594            ),
595            ListingProblem::Failed { status } => format!("GitHub answers {status}"),
596            ListingProblem::Unreachable => "GitHub does not answer".to_owned(),
597            ListingProblem::Unexpected => "GitHub's answer is not what runnerd reads".to_owned(),
598            ListingProblem::Unknown => "for a reason this client does not know".to_owned(),
599        }
600    }
601}
602
603/// One of a repository's pools: its runners in one profile.
604#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
605pub struct PoolStatus {
606    /// The profile its jobs run in.
607    pub profile: String,
608    /// The labels its runners register with.
609    pub labels: Vec<String>,
610    /// Its jobs in `Status::queued`: those its labels match.
611    pub queued_jobs: usize,
612}
613
614/// A queued job runners of this daemon can take, as GitHub listed it at the last poll.
615#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
616pub struct QueuedJob {
617    /// GitHub's job ID.
618    pub id: u64,
619    /// The repository whose runners can take it, `owner/name`.
620    pub repository: String,
621    /// The profile of its repository's pool whose labels match it; `None` when no pool's do.
622    #[serde(default, skip_serializing_if = "Option::is_none")]
623    pub profile: Option<String>,
624    /// The workflow run it belongs to.
625    pub run_id: u64,
626    /// 1 for the first attempt of the run, higher for re-runs.
627    pub run_attempt: u32,
628    /// The workflow's name.
629    pub workflow: Option<String>,
630    /// The job's name in the workflow.
631    pub name: String,
632    /// The branch the run is for.
633    pub branch: Option<String>,
634    /// The job's page on GitHub.
635    pub url: Option<String>,
636    /// When GitHub queued it, in seconds since the Unix epoch; 0 if GitHub did not say.
637    pub queued_at: u64,
638    /// Why no runner has it yet, as the last plan saw it. `None` before the daemon has planned with it, and while
639    /// it shuts down.
640    #[serde(default, skip_serializing_if = "Option::is_none")]
641    pub waiting: Option<Waiting>,
642}
643
644/// The slot of an idle runner of `job`'s repository, in its pool's profile when known, such as `jvm 2`.
645fn starting_for(job: &QueuedJob, status: &Status) -> Option<String> {
646    status
647        .profiles
648        .iter()
649        .filter(|profile| job.profile.as_ref().is_none_or(|own| *own == profile.name))
650        .find_map(|profile| {
651            profile
652                .slots
653                .iter()
654                .find(|slot| {
655                    slot.runner
656                        .as_ref()
657                        .is_some_and(|runner| runner.repository == job.repository && !runner.busy)
658                })
659                .map(|slot| format!("{}\u{a0}{}", profile.name, slot.slot))
660        })
661}
662
663impl QueuedJob {
664    /// Where it runs, as every client names it: its repository, and its pool's profile when the repository has
665    /// several pools in `status`, such as `halyard/ledger · light`.
666    pub fn place(&self, status: &Status) -> String {
667        let pools = status
668            .repositories
669            .iter()
670            .find(|repository| repository.name == self.repository)
671            .map_or(0, |repository| repository.pools.len());
672        match self.profile.as_deref().filter(|_| pools > 1) {
673            Some(profile) => format!("{}\u{a0}· {profile}", self.repository),
674            None => self.repository.clone(),
675        }
676    }
677}
678
679impl RepositoryStatus {
680    /// The profile of its first pool, which every client colours it by.
681    pub fn first_profile(&self) -> &str {
682        self.pools.first().map_or("", |pool| pool.profile.as_str())
683    }
684
685    /// Its pools' profiles, as every client lists them on one line: `jvm` or `jvm + light`.
686    pub fn profiles(&self) -> String {
687        self.pools
688            .iter()
689            .map(|pool| pool.profile.as_str())
690            .collect::<Vec<_>>()
691            .join(" + ")
692    }
693
694    /// The profile of its pool whose labels match a job asking for `labels`, as the daemon picks it.
695    pub fn pool_for(&self, labels: &[String]) -> Option<String> {
696        self.pools
697            .iter()
698            .find(|pool| crate::config::labels_match(labels, &pool.labels))
699            .map(|pool| pool.profile.clone())
700    }
701}
702
703/// What failed, that holds a repository's runners back for a while.
704#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
705#[serde(rename_all = "kebab-case")]
706pub enum HoldCause {
707    /// Its runners ended without connecting, so without taking a job (ADR 0104).
708    NoJob,
709    /// Its runners connected and ended without a job, one after the other.
710    IdleExits,
711    /// GitHub did not register its runner.
712    Register,
713    /// Docker did not start its runner's container.
714    Start,
715}
716
717/// Why a queued job has no runner of its own yet. Jobs are served oldest first; each takes the first of these that
718/// holds for it.
719#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
720#[serde(rename_all = "kebab-case", tag = "kind")]
721pub enum Waiting {
722    /// Its repository is paused.
723    Paused,
724    /// Runners of its repository failed, so none starts before `until` (seconds since the Unix epoch).
725    Backoff {
726        /// When runners may start again.
727        until: u64,
728        /// What failed.
729        cause: HoldCause,
730    },
731    /// A runner started for it and waits for GitHub to hand it the job: a runner takes any job its labels match,
732    /// so not necessarily this one.
733    RunnerStarting,
734    /// Slot 1 of its profile is becoming the profile's snapshot, so none of the profile's slots starts.
735    Reseeding {
736        /// The profile.
737        profile: String,
738    },
739    /// Its profile's image is being pulled, so none of the profile's slots starts.
740    PullingImage {
741        /// The profile.
742        profile: String,
743    },
744    /// Every slot of its profile holds a runner.
745    NoSlot {
746        /// The profile.
747        profile: String,
748    },
749    /// What its runner would reserve does not fit the memory budget left. Under measured admission, `free` is 0
750    /// while the slice is at the high-water mark (`Budget::held`).
751    Memory {
752        /// What its runner would reserve, in bytes.
753        needs: u64,
754        /// What the budget has left, in bytes.
755        free: u64,
756    },
757    /// What its runner would reserve does not fit what the host's memory has to spare beside its other programs
758    /// (`HostBudget::spare`), though it fits the slice's budget. `free` is 0 while the host's tasks stall for memory
759    /// (`HostBudget::stalled`).
760    HostMemory {
761        /// What its runner would reserve, in bytes.
762        needs: u64,
763        /// What the host has to spare, in bytes.
764        free: u64,
765    },
766    /// What its runner would reserve does not fit the CPU budget left, in millicores.
767    Cpu {
768        /// What its runner would reserve.
769        needs: u64,
770        /// What the CPU budget has left.
771        free: u64,
772    },
773    /// An older job has waited long enough for CPU that no younger one takes CPU before it.
774    BehindCpu {
775        /// The older job's ID.
776        job: u64,
777    },
778    /// An older job has waited long enough for memory that no younger one starts before it (ADR 0096).
779    BehindMemory {
780        /// The older job's ID.
781        job: u64,
782    },
783    /// An older job is about to have a runner start for it, which takes the slot, memory or CPU this one would
784    /// otherwise start with: what holds it is that start, not what runs now.
785    BehindStart {
786        /// The older job's ID.
787        job: u64,
788    },
789    /// No pool of its repository has every label it asks for, so no runner of this daemon can take it (ADR 0058).
790    NoPool {
791        /// The labels it asks for.
792        labels: Vec<String>,
793    },
794    /// A reason from a newer daemon than this client knows, so a reason added later does not keep the client from
795    /// reading the status.
796    #[serde(other)]
797    Unknown,
798}
799
800/// What a [`Waiting`] reason waits on, which clients draw it by.
801#[derive(Debug, Clone, Copy, PartialEq, Eq)]
802pub enum WaitKind {
803    /// Something the owner can act on: a pause, runners that failed to take a job, or labels no pool serves.
804    Act,
805    /// Something that clears by itself shortly: a runner starting, a reseed, an image pull, an older job's start.
806    Soon,
807    /// Room: a slot, memory or CPU that a running job holds.
808    Full,
809    /// A reason this client does not know (`Waiting::Unknown`), drawn neutral.
810    Other,
811}
812
813impl Waiting {
814    /// Whether runnerd cannot start the job with the config it runs with: its repository is paused, or no pool serves
815    /// its labels. Not that nothing ever will: a resume or a config change can, and another machine's runner may take
816    /// it. A backoff ends by itself, and a reason this client does not know says nothing (ADR 0132).
817    pub fn cannot_start(&self) -> bool {
818        matches!(self, Waiting::Paused | Waiting::NoPool { .. })
819    }
820
821    /// What it waits on.
822    pub fn kind(&self) -> WaitKind {
823        match self {
824            Waiting::Paused | Waiting::Backoff { .. } | Waiting::NoPool { .. } => WaitKind::Act,
825            Waiting::Unknown => WaitKind::Other,
826            Waiting::RunnerStarting
827            | Waiting::Reseeding { .. }
828            | Waiting::PullingImage { .. }
829            | Waiting::BehindStart { .. } => WaitKind::Soon,
830            Waiting::NoSlot { .. }
831            | Waiting::Memory { .. }
832            | Waiting::HostMemory { .. }
833            | Waiting::Cpu { .. }
834            | Waiting::BehindCpu { .. }
835            | Waiting::BehindMemory { .. } => WaitKind::Full,
836        }
837    }
838
839    /// Why `job` waits, in the owner's words with the numbers that hold it, as every client says it: such as
840    /// `All 2 node slots are busy` or `Needs 9G of memory; 2.5G of the budget is free`. `status` is the one `job`
841    /// is queued in, which names the slot of a starting runner and the older job one waits behind.
842    pub fn reason(&self, job: &QueuedJob, status: &Status, now: u64) -> String {
843        let size = |bytes: u64| crate::config::ByteSize(bytes).to_string();
844        let older_name = |id: u64| {
845            status
846                .queued
847                .iter()
848                .find(|candidate| candidate.id == id)
849                .map_or_else(|| "an older job".to_owned(), |older| older.name.clone())
850        };
851        match self {
852            Waiting::Paused => format!("{} is paused", job.repository),
853            Waiting::Unknown => "Waits for a reason this client does not know yet".to_owned(),
854            Waiting::NoPool { labels } if labels.is_empty() => {
855                "It asks for no labels, so no profile serves it".to_owned()
856            }
857            Waiting::NoPool { labels } => format!(
858                "No profile serves the {} {}",
859                if labels.len() == 1 { "label" } else { "labels" },
860                labels.join(", ")
861            ),
862            Waiting::Backoff { until, cause } => format!(
863                "{}; retrying in {}",
864                match cause {
865                    HoldCause::NoJob => "Runners failed to take a job",
866                    HoldCause::IdleExits => "Runners ended without a job after waiting for one",
867                    HoldCause::Register => "Runners cannot register with GitHub",
868                    HoldCause::Start => "Runners cannot start",
869                },
870                duration(until.saturating_sub(now))
871            ),
872            Waiting::RunnerStarting => match starting_for(job, status) {
873                Some(slot) => format!("A runner is starting for it in {slot}"),
874                None => "A runner is starting for it".to_owned(),
875            },
876            Waiting::Reseeding { profile } => format!("{profile} is rebuilding its snapshot"),
877            Waiting::PullingImage { profile } => format!("Pulling {profile}'s image"),
878            Waiting::NoSlot { profile } => {
879                let slots = status
880                    .profiles
881                    .iter()
882                    .find(|candidate| &candidate.name == profile)
883                    .map_or(0, |profile| profile.slots.len());
884                if slots == 1 {
885                    format!("The {profile} slot is busy")
886                } else {
887                    format!("All {slots} {profile} slots are busy")
888                }
889            }
890            Waiting::Memory { .. } if status.budget.held => {
891                "The slice is past 90% of its MemoryMax, so nothing starts".to_owned()
892            }
893            Waiting::Memory { needs, free } => format!(
894                "Needs {} of memory; {} of the budget is free",
895                size(*needs),
896                size(*free)
897            ),
898            Waiting::HostMemory { needs, free } => host_reason(
899                *needs,
900                *free,
901                status.budget.host.as_ref(),
902                status.budget.memory_in_use,
903            ),
904            Waiting::Cpu { needs, free } => {
905                format!(
906                    "Needs {}; {} are free",
907                    cpus(*needs),
908                    admission::free_cores(*free)
909                )
910            }
911            Waiting::BehindCpu { job: older } => {
912                format!("Behind {}, which waits for CPU", older_name(*older))
913            }
914            Waiting::BehindMemory { job: older } => {
915                format!("Behind {}, which waits for memory", older_name(*older))
916            }
917            Waiting::BehindStart { job: older } => {
918                format!(
919                    "Next after {}, whose runner starts first",
920                    older_name(*older)
921                )
922            }
923        }
924    }
925}
926
927/// `GET /v1/dashboard`: the dashboard's `[web]` settings, as the daemon read them when it started.
928#[derive(Debug, Clone, Default, PartialEq, Eq, Serialize, Deserialize)]
929pub struct Dashboard {
930    /// `[web] open`: the one of `[web] origins` that `runnerctl web` opens the dashboard on; the loopback address
931    /// when none.
932    #[serde(default, skip_serializing_if = "Option::is_none")]
933    pub open: Option<String>,
934}
935
936/// `POST /v1/dashboard/pairings`: which of `[web] origins` to make a link for.
937#[derive(Debug, Clone, Default, PartialEq, Eq, Serialize, Deserialize)]
938pub struct PairingQuery {
939    /// One of `[web] origins`; each of them when none.
940    #[serde(default, skip_serializing_if = "Option::is_none")]
941    pub origin: Option<String>,
942}
943
944/// `POST /v1/dashboard/pairings`: fresh one-time links that sign a device in to the dashboard.
945#[derive(Debug, Clone, Default, PartialEq, Eq, Serialize, Deserialize)]
946pub struct Pairings {
947    /// One per origin asked for.
948    pub links: Vec<Pairing>,
949}
950
951/// A link that signs one device in to the dashboard on `origin`, and nowhere else, within `lifetime`.
952#[derive(Clone, PartialEq, Eq, Serialize, Deserialize)]
953pub struct Pairing {
954    /// One of `[web] origins`.
955    pub origin: String,
956    /// The link with its one-time code: a credential, which `Debug` leaves out.
957    pub url: String,
958    /// Seconds it works for from when it was made.
959    pub lifetime: u64,
960}
961
962impl std::fmt::Debug for Pairing {
963    fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
964        formatter
965            .debug_struct("Pairing")
966            .field("origin", &self.origin)
967            .field("lifetime", &self.lifetime)
968            .finish_non_exhaustive()
969    }
970}
971
972/// `POST /v1/shutdown`.
973#[derive(Debug, Clone, Default, PartialEq, Eq, Serialize, Deserialize)]
974pub struct ShutdownRequest {
975    /// Kill running jobs and delete their runners instead of waiting for them.
976    #[serde(default)]
977    pub force: bool,
978}
979
980/// What `POST /v1/shutdown` answers: how the daemon stops now, which a stop asked for earlier may have made more
981/// urgent than the request.
982#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
983pub struct ShutdownAnswer {
984    /// How it stops.
985    pub stopping: Stopping,
986}
987
988/// How a stopping daemon treats running jobs (ADR 0091).
989#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
990#[serde(rename_all = "kebab-case")]
991pub enum Stopping {
992    /// `runnerd stop`: it waits for running jobs, then removes every runner.
993    WaitsForJobs,
994    /// A signal: it removes idle runners and leaves running jobs to the next start.
995    LeavesJobs,
996    /// `runnerd stop --force` or a second signal: it kills running jobs.
997    KillsJobs,
998    /// A kind of stop from a newer daemon than this client knows, so one added later does not keep the client from
999    /// reading the status.
1000    #[serde(other)]
1001    Unknown,
1002}
1003
1004impl Stopping {
1005    /// What it does, as every client says it: `shutting down: waits for running jobs`.
1006    pub fn line(self) -> &'static str {
1007        match self {
1008            Stopping::WaitsForJobs => "shutting down: waits for running jobs",
1009            Stopping::LeavesJobs => "shutting down: leaves running jobs to the next start",
1010            Stopping::KillsJobs => "shutting down: kills running jobs",
1011            Stopping::Unknown => "shutting down",
1012        }
1013    }
1014}
1015
1016impl Status {
1017    /// What a stopping daemon does, as every client says it ([`Stopping::line`]); `shutting down` from a daemon
1018    /// that does not say how; `None` while it runs.
1019    pub fn stopping_line(&self) -> Option<&'static str> {
1020        match (self.stopping, self.shutting_down) {
1021            (Some(stopping), _) => Some(stopping.line()),
1022            (None, true) => Some("shutting down"),
1023            (None, false) => None,
1024        }
1025    }
1026
1027    /// Why no runner starts while GitHub's rate limit is used up, as every client says it; `None` while it is not.
1028    pub fn github_line(&self) -> Option<&'static str> {
1029        self.github_limited_until
1030            .map(|_| "GitHub's rate limit is used up: no runner starts until it resets")
1031    }
1032}
1033
1034/// The body of every error response.
1035#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
1036pub struct ErrorResponse {
1037    /// What went wrong, for people to read.
1038    pub message: String,
1039}