Skip to main content

runnerd_core/
api.rs

1//! The daemon's JSON API, version 1: the request and response types every client deserializes, and the
2//! principal a transport authenticates. Plain data only: no I/O, no Docker or GitHub types, no secrets but a
3//! [`Pairing`]'s one-time link, which its `Debug` leaves out.
4//!
5//! Routes, all under the authenticated principal:
6//!
7//! | Route | Body |
8//! |---|---|
9//! | `GET /v1/status` | [`Status`] |
10//! | `POST /v1/repositories/{owner}/{name}/pause?`[`PauseQuery`] | [`Paused`]; 409 [`Busy`] while it runs a job |
11//! | `POST /v1/repositories/{owner}/{name}/resume` | none, 204 |
12//! | `POST /v1/repositories/pause?`[`PauseQuery`] | [`Paused`]; 409 [`Busy`] while any runs a job, pausing none |
13//! | `POST /v1/repositories/resume` | none, 204: every repository resumed |
14//! | `POST /v1/profiles/{name}/reseed` | none, 202; 409 while it reseeds; `cache.reseeded` when done |
15//! | `POST /v1/caches/{name}/clean` | none, 204; 409 [`Busy`] while a runner uses it |
16//! | `GET /v1/config` | [`ConfigFile`] |
17//! | `POST /v1/config/check` | TOML in, [`ConfigCheck`] |
18//! | `PUT /v1/config` | TOML in, [`ConfigCheck`]; 422 [`ConfigCheck`] when invalid |
19//! | `GET /v1/doctor` | [`Doctor`]; 503 [`ErrorResponse`] while no run has finished |
20//! | `GET /v1/attention` | [`Attention`]; 404 from a daemon before 0.2 |
21//! | `POST /v1/shutdown` | [`ShutdownRequest`] in, 202 [`ShutdownAnswer`] |
22//! | `GET /v1/dashboard` | [`Dashboard`] |
23//! | `POST /v1/dashboard/sign-out` | none, 204 |
24//! | `POST /v1/dashboard/pairings?`[`PairingQuery`] | [`Pairings`]; 409 when the dashboard is off or has no `[web] origins` |
25//! | `GET /v1/jobs?`[`JobQuery`] | [`JobPage`] |
26//! | `GET /v1/jobs/{id}` | [`Job`] |
27//! | `GET /v1/jobs/{id}/samples` | [`SampleSeries`] |
28//! | `GET /v1/jobs/{id}/logs?`[`LogQuery`] | `text/plain`, one line per log line |
29//! | `GET /v1/host/samples?`[`HostSampleQuery`] | [`HostSamples`] |
30//! | `GET /v1/stats?`[`StatsQuery`] | [`Stats`] |
31//! | `GET /v1/activity?`[`ActivityQuery`] | [`Activity`] |
32//! | `GET /v1/sizing?`[`SizingQuery`] | [`Sizing`] |
33//! | `GET /v1/insights?`[`InsightsQuery`] | [`Insights`] |
34//! | `GET /v1/digest` | [`Digest`] |
35//! | `GET /v1/savings` | [`Savings`] |
36//! | `POST /v1/notify/test?`[`NotifyTestQuery`] | [`NotifyTest`]; 404 without `[notify]` targets or with no such one |
37//! | `GET /v1/caches` | [`Caches`] |
38//! | `GET /v1/caches/history?`[`CacheHistoryQuery`] | [`CacheHistory`] |
39//! | `GET /v1/events?`[`EventQuery`] | `text/event-stream` of [`EventEnvelope`] |
40//! | `GET /v1/records` | [`Records`] |
41//! | `DELETE /v1/records/{job}` | [`RemovedRecords`]; 409 while a shell is open in it |
42//! | `DELETE /v1/records?`[`RemoveRecordsQuery`] | [`RemovedRecords`]; records with an open shell stay |
43//! | `POST /v1/records/{job}/shell?`[`ShellQuery`] | `Upgrade:` [`SHELL_PROTOCOL`], 101, then the shell's terminal; 409 when the budget is full |
44//!
45//! Errors carry an [`ErrorResponse`].
46
47use serde::{Deserialize, Serialize};
48
49mod actions;
50mod admission;
51mod attention;
52mod budget;
53mod control;
54mod disk;
55mod events;
56mod figure;
57mod insights;
58mod jobs;
59mod labels;
60mod records;
61mod results;
62mod savings;
63mod sizing;
64mod slots;
65mod stats;
66mod storage;
67mod tone;
68mod units;
69
70pub use actions::{
71    ActionAnswer, ActionOutcome, ActionPreview, ActionQuery, ActionRefusal, ActionResult,
72    ActionTarget, CONFIRM_SECS, JOB_ACTIONS_CHECK, JobAction, JobActions, PreviewFacts, RunSeen,
73    SIBLINGS_MAX, Superseded, UnknownCause, actions_in_flight, actions_no_run,
74    actions_not_configured, actions_off,
75};
76pub use admission::{Admission, AdmissionPolicy, Basis, CpuAdmission, JobPeak, cores, cpus};
77pub use attention::{
78    ATTENTION_MAX, Attention, AttentionItem, AttentionKind, Fix, Subject, attention_items,
79};
80pub use budget::{
81    BudgetParts, ClosedKind, ClosedPart, FreePart, HeldKind, HeldPart, HostFigure, Reading,
82    SlotForm, SlotPart, SlotReading,
83};
84pub use control::{
85    Busy, Check, CheckCount, CheckStatus, ConfigCheck, ConfigFile, Doctor, DoctorQuery,
86    DoctorState, DoctorSummary, PauseQuery, Paused, ReplaceQuery, RunningJob,
87};
88pub use disk::{
89    Commits, DiskWrites, DiskWritten, DriveShare, DriveShares, DriveWrites, JobWrites, Measure,
90    ShareOwner, WriteSection, WriteTop,
91};
92pub use events::{Event, EventEnvelope, EventQuery, RunnerFailure};
93pub use figure::LabelledFigure;
94pub use insights::{
95    AutoNote, Delivery, Digest, HostHazard, HostSpan, INSIGHT_DAYS, INSIGHT_DAYS_MAX, Insight,
96    InsightFigure, InsightKind, Insights, InsightsHeading, InsightsQuery, LongWait, Notification,
97    NotifyEvent, NotifyTest, NotifyTestQuery, Priority, RepositoryWaits, RunLink, Severity,
98    WaitFigure, WaitSpread, WeekWaits,
99};
100#[cfg(feature = "host")]
101pub(crate) use jobs::SAMPLE_GAP_INTERVALS;
102#[cfg(feature = "host")]
103pub(crate) use jobs::TITLE_SOURCE_MAX;
104pub use jobs::{
105    Activity, ActivityQuery, Bucket, BuildCacheCounts, BuildCacheStats, CacheHistory,
106    CacheHistoryQuery, CacheKind, CacheSize, CacheVolume, Caches, ChartLine, ChartLineKind,
107    ChartLines, Holder, HostMemory, HostSample, HostSampleQuery, HostSamples, Job, JobMetrics,
108    JobPage, JobQuery, JobRunner, JobState, JobSummary, JobTitle, Limit, LimitSource, Limits,
109    LogQuery, LogSummary, Milestone, Outcome, OutcomeKind, Phase, PhaseKind, Pressure,
110    RunComparison, Sample, SampleSeries, SampleUsage, Started, Stats, StatsQuery, StatsRow, Step,
111    Swap, Timeline, Total, host_margin, job_title,
112};
113pub use jobs::{
114    BarEnd, Baseline, BaselineRead, COMPARE_STEP_BYTES, COMPARE_STEPS_MAX, ChangeKind,
115    CompareQuery, ComparedSteps, FigureDelta, FigureKind, Gap, GapKind, JOB_LABELS_MAX,
116    JOB_SEARCH_MAX, JOB_TEXT_MAX, JOBS_PAGE_BYTES, JOBS_PAGE_DEFAULT, JOBS_PAGE_MAX, JobComparison,
117    JobList, JobOrder, JobRange, JobSort, Lane, RecordFilter, RecordMark, RunnerSpan, SlotKey,
118    SlotTimeline, SlottedBar, SlottedRunner, SortOrder, Span, StepDelta, StepMatch, StepUsage,
119    TIMELINE_BARS_MAX, TIMELINE_DEFAULT, TIMELINE_WINDOW_MAX, TimelineBar, TimelineQuery,
120    TimelineRows, Unavailable, row_bytes,
121};
122pub use jobs::{
123    Color, FailureSummary, LINE_BYTES_MAX, LINES_DEFAULT, LINES_MAX, LineKind, LineRange, LineRead,
124    LogEvidence, LogGroup, LogLinesQuery, LogMark, LogNeedle, LogOutline, LogSearch,
125    LogSearchQuery, MarkKind, OutlineChange, OutlineParts, Outliner, RUNS_MAX, Run,
126    SEARCH_QUERY_MAX, SavedOutline, Search, StepLines, StepLog, StoredLine, StoredLog, StyledLine,
127};
128pub use labels::{Column, Feature, JOBS_HEADING, NOT_SUPPORTED, Section, View};
129pub use records::{
130    Record, RecordLimits, RecordNote, RecordRemoval, Records, RemoveRecordsQuery, RemovedRecords,
131    SHELL_PROTOCOL, ShellQuery,
132};
133pub use results::{force_pause_consequence, reseed_requested, reseeded};
134pub use savings::{
135    Period, PricedRun, PricedRunner, Pricing, PricingHistory, SELF_HOSTED_RATE, SavedWindow,
136    Savings, Sku, Usd, Visibility, billed_minutes, reads_since,
137};
138pub use sizing::{
139    Advice, AutoPlan, GRANULE, JobSizing, MemoryBound, MemoryHold, Need, PAGE_CACHE_ROOM, Peaks,
140    ProfilePlan, ProfileSizing, Resize, RevertCause, SAMPLE_INTERVAL, SLOT_DISK, Sizing,
141    SizingChange, SizingHold, SizingQuery, SizingRefused, SizingRevert, Spread, Starts, need,
142    period, sampled_note,
143};
144pub use slots::{Figure, Health, HealthKind, SlotState};
145pub use stats::StatsSummary;
146pub use storage::{
147    ACTIONS_CACHE, BUILD_CACHE_TOOLS, BuildCacheActivity, CacheLimit, RECORDS_NEAR_FULL,
148    RecordState, actions_cache_place, records_check, records_near_full,
149};
150pub use tone::{Tone, Toned};
151pub(crate) use units::days_from_civil;
152pub use units::{ago, cores_used, cpu_time, duration, in_cores};
153
154/// Who sent a request, as the transport authenticated it. Handlers get it from one place, so a new transport only
155/// has to produce one.
156#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
157#[serde(rename_all = "kebab-case", tag = "kind")]
158pub enum Principal {
159    /// The user the daemon runs as, on the local Unix socket.
160    LocalOwner {
161        /// The Unix user ID the socket's peer runs as.
162        uid: u32,
163    },
164}
165
166/// `GET /v1/status`: what the daemon serves and runs right now.
167#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
168pub struct Status {
169    /// The daemon's version, from its `Cargo.toml`.
170    pub version: String,
171    /// Set once the daemon stops polling to shut down. [`Status::stopping_line`] says how.
172    pub shutting_down: bool,
173    /// How it stops, once it does; absent from a daemon before it said.
174    #[serde(default, skip_serializing_if = "Option::is_none")]
175    pub stopping: Option<Stopping>,
176    /// Until when GitHub's rate limit is used up, in seconds since the Unix epoch: no runner starts before.
177    /// [`Status::github_line`] says it. Absent while it is not.
178    #[serde(default, skip_serializing_if = "Option::is_none")]
179    pub github_limited_until: Option<u64>,
180    /// The memory budget and what runners use of it.
181    pub budget: Budget,
182    /// Every configured profile with its slots.
183    pub profiles: Vec<ProfileStatus>,
184    /// Every configured repository.
185    pub repositories: Vec<RepositoryStatus>,
186    /// Queued jobs the repositories' runners can take and none holds yet, at the last poll, oldest first. No event
187    /// carries the list: `job.queued` announces a job, `job.started` and `job.finished` its leaving the queue.
188    /// A repository whose last listing failed, as while GitHub rate-limits runnerd, or that is no longer polled
189    /// because runnerd is shutting down, shows no jobs here, while the history keeps its jobs queued.
190    pub queued: Vec<QueuedJob>,
191    /// Whether the daemon cancels and re-runs jobs (ADR 0134); a daemon before 0.2 sends none, read as
192    /// [`JobActions::Unsupported`].
193    #[serde(default)]
194    pub job_actions: JobActions,
195}
196
197/// The memory budget runners and the mirror share, in bytes.
198#[derive(Debug, Clone, Default, PartialEq, Eq, Serialize, Deserialize)]
199pub struct Budget {
200    /// The systemd slice runner containers run in.
201    pub slice: String,
202    /// The slice's `MemoryMax`; `None` when it is unlimited.
203    pub memory_max: Option<u64>,
204    /// Memory the running containers reserve: the runners' caps, or under measured admission the larger of each
205    /// runner's predicted peak and its last sampled memory, nothing for one whose container is gone, and the caps of
206    /// the seed containers, shells and the mirror.
207    pub memory_in_use: u64,
208    /// What pooling takes off `memory_in_use` (ADR 0099, 0130); absent while it takes nothing.
209    #[serde(default, skip_serializing_if = "Option::is_none")]
210    pub pooled: Option<Pooled>,
211    /// The growth count both budgets go by, and its brake (ADR 0130); `pooled` and `host.credit` say what it takes off
212    /// `memory_in_use`. Absent under caps admission, with `[host] pooled_count` off, and from an older daemon.
213    #[serde(default, skip_serializing_if = "Option::is_none")]
214    pub counted: Option<Counted>,
215    /// What a new runner must fit in beside them.
216    pub admission: AdmissionPolicy,
217    /// Under measured admission, whether the slice is at the high-water mark, so no runner starts.
218    pub held: bool,
219    /// The mirror's part of `memory_in_use`: its cap while it is enabled or still running, else 0.
220    pub mirror_memory: u64,
221    /// The seed containers' part of `memory_in_use`, one per profile that reseeds, by profile name.
222    pub seed_memory: Vec<SeedMemory>,
223    /// The shells' part of `memory_in_use`, one per shell open in a record, by job.
224    pub shell_memory: Vec<ShellMemory>,
225    /// What the host's memory leaves runners beside its other programs, from a recent reading of it; absent
226    /// without one.
227    #[serde(default, skip_serializing_if = "Option::is_none")]
228    pub host: Option<HostBudget>,
229    /// The CPU budget, under `[host] cpu_overcommit`. Absent without one.
230    #[serde(default, skip_serializing_if = "Option::is_none")]
231    pub cpu: Option<CpuBudget>,
232    /// The CPUs the slice may use, in millicores: its `AllowedCPUs` or the host's, less under a lower `CPUQuota`,
233    /// whether or not a CPU budget admits runners. The runners' CPU limits are quotas that may add up past it.
234    /// Absent when it could not be read.
235    #[serde(default, skip_serializing_if = "Option::is_none")]
236    pub slice_cpus: Option<u64>,
237}
238
239/// The count the budgets go by while pooling takes something off `Budget::memory_in_use` (ADR 0099).
240#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
241pub struct Pooled {
242    /// What it takes off, in bytes.
243    pub credit: u64,
244    /// `Budget::memory_in_use` less `credit`: what new runners fit beside under `MemoryMax`. The host's limit reads
245    /// `memory_in_use`.
246    pub counted: u64,
247    /// How many runners the credit counts on.
248    pub runners: u64,
249}
250
251/// How far back the growth count's brake compares what runs uses: 10 minutes (ADR 0130,
252/// `admission::growth::GROWTH_WINDOW`).
253pub const GROWTH_WINDOW: std::time::Duration = std::time::Duration::from_mins(10);
254
255/// The growth count (ADR 0130): what the runners use, and the room kept for what they may still grow by.
256#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
257pub struct Counted {
258    /// What the runners use, by their recent samples, in bytes.
259    pub used: u64,
260    /// The room it keeps for their growth, in bytes; absent while it rests (`brake`).
261    #[serde(default, skip_serializing_if = "Option::is_none")]
262    pub kept: Option<u64>,
263    /// How many of the largest growths the room covers at least.
264    pub jobs: u64,
265    /// Why it rests, while it does: the budgets then go by 0099's and 0108's credits.
266    #[serde(default, skip_serializing_if = "Option::is_none")]
267    pub brake: Option<CountBrake>,
268}
269
270/// Why the growth count rests (ADR 0130).
271#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
272#[serde(rename_all = "kebab-case", tag = "kind")]
273pub enum CountBrake {
274    /// What ran grew past the room kept within `GROWTH_WINDOW`.
275    Grew {
276        /// How much the runners present at both looks grew, together, in bytes.
277        grew: u64,
278        /// The room the earlier look kept, in bytes.
279        kept: u64,
280        /// Until when, in seconds since the Unix epoch.
281        until: u64,
282    },
283    /// The daemon started, or a reload changed what the count reads, too recently to judge growth.
284    Restarted {
285        /// Until when, in seconds since the Unix epoch.
286        until: u64,
287    },
288    /// The host is under the guard's line, or a reading is missing, now or at a look within 5 minutes.
289    Guard {
290        /// Until when, in seconds since the Unix epoch; absent for a moment the brake has not looked at yet.
291        #[serde(default, skip_serializing_if = "Option::is_none")]
292        until: Option<u64>,
293    },
294    /// A brake a newer daemon names; this client shows only that the count rests.
295    #[serde(other)]
296    Unknown,
297}
298
299/// What the host's memory leaves the runners (ADR 0055). New runners fit in the lower of `MemoryMax` and
300/// `limit`; running ones are never stopped for it.
301#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
302pub struct HostBudget {
303    /// The reading it comes from.
304    pub memory: HostMemory,
305    /// What the slice may hold now: what its runners hold without page cache plus the host's `MemAvailable`, less
306    /// a margin kept free for the host's page cache; with a ceiling, plus what the host can still grow by
307    /// (`HostMemory::growable`, ADR 0084).
308    pub limit: u64,
309    /// What that leaves new runners beside what the running ones reserve, in bytes.
310    pub spare: u64,
311    /// Whether the host's tasks stall for memory (`full` pressure), so no runner starts until they stop.
312    pub stalled: bool,
313    /// What the host's limit takes off `Budget::memory_in_use` for runners that may not all grow at once (ADR 0108);
314    /// absent while it takes nothing, and from a daemon before 0.0.30.
315    #[serde(default, skip_serializing_if = "Option::is_none")]
316    pub credit: Option<HostCredit>,
317}
318
319/// What the host's limit takes off the plain count (ADR 0108): `Budget::memory_in_use` less `bytes` is what it counts,
320/// `used` of that is what the runners use, and the rest is room kept for what runs to grow into.
321#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
322pub struct HostCredit {
323    /// What it takes off, in bytes.
324    pub bytes: u64,
325    /// What the runners use, by their recent samples, in bytes.
326    pub used: u64,
327}
328
329impl HostBudget {
330    /// Whether the host itself lacks `needs` bytes for a new runner: its tasks stall, `HostMemory::room` is less, or
331    /// that room is under one more margin, so `MemAvailable` is under twice the plain margin, the line `host::stalls`
332    /// reads. There a small job fits only by what the host is about to run out of, and what closing programs frees is
333    /// what starts it. Otherwise a job the host limit holds waits for what runs, whose reservations reach the limit
334    /// (ADR 0103).
335    pub fn short_of(&self, needs: u64) -> bool {
336        let room = self.memory.room();
337        self.stalled || needs > room || room < host_margin(self.memory.grown())
338    }
339
340    /// What the host's limit counts of what runs, `reserved` being `Budget::memory_in_use`: that less the credit
341    /// (ADR 0108).
342    pub fn counts(&self, reserved: u64) -> u64 {
343        reserved.saturating_sub(self.credit.map_or(0, |credit| credit.bytes))
344    }
345
346    /// What that count is made of while a credit stands, as the host's line, a job's wait and the log say it: `it uses
347    /// 14G and keeps 9G for growth`. `None` without a credit, when it counts every reservation.
348    pub fn growth_clause(&self, reserved: u64) -> Option<String> {
349        let credit = self.credit?;
350        let counts = self.counts(reserved);
351        Some(format!(
352            "it uses {} and keeps {} for growth",
353            crate::config::ByteSize(credit.used.min(counts)),
354            crate::config::ByteSize(counts.saturating_sub(credit.used))
355        ))
356    }
357}
358
359/// Why a job waits for the host's limit, needing `needs` bytes with `free` left of it, by the latest reading: for what
360/// runs, or for a host that lacks the room, whose programs it names. While the host's limit takes a credit, what runs
361/// is what it uses and the room it keeps for growth, not its reservations (ADR 0108); `reserved` is
362/// `Budget::memory_in_use`.
363fn host_reason(needs: u64, free: u64, host: Option<&HostBudget>, reserved: u64) -> String {
364    let size = |bytes: u64| crate::config::ByteSize(bytes).to_string();
365    if let Some(host) = host.filter(|host| !host.short_of(needs)) {
366        if let Some(growth) = host.growth_clause(reserved) {
367            return format!(
368                "The host has {}, but what runs may still grow into it: {growth}; needs {}, {} is free",
369                host.memory.available_line(),
370                size(needs),
371                size(free)
372            );
373        }
374        return format!(
375            "The host has {}, but what runs may still grow into it: needs {}; {} of the {} it leaves the slice is free",
376            host.memory.available_line(),
377            size(needs),
378            size(free),
379            size(host.limit)
380        );
381    }
382    let reason = if host.is_some_and(|host| host.stalled) {
383        "The host is short of memory: its programs stall waiting for it, so nothing starts"
384            .to_owned()
385    } else {
386        format!(
387            "The host is short of memory: needs {}; the host has {} to spare",
388            size(needs),
389            size(free)
390        )
391    };
392    match host {
393        Some(host) => format!("{reason}; {}", host.memory.outside_line()),
394        None => reason,
395    }
396}
397
398/// Hundredths of a percent as a percentage: `1.5%`, `12%`.
399pub fn percent(hundredths: u32) -> String {
400    let whole = hundredths / 100;
401    match hundredths % 100 / 10 {
402        0 => format!("{whole}%"),
403        tenth => format!("{whole}.{tenth}%"),
404    }
405}
406
407/// The CPU budget runners share, in millicores.
408#[derive(Debug, Clone, Copy, Default, PartialEq, Eq, Serialize, Deserialize)]
409pub struct CpuBudget {
410    /// The CPUs the slice may use: its `AllowedCPUs` or the host's, less under a lower `CPUQuota`.
411    pub cpus: u64,
412    /// Those times `cpu_overcommit`: what the runners' reservations must fit.
413    pub budget: u64,
414    /// What the runners reserve: each its predicted average CPU under measured admission, else its `cpus`.
415    pub in_use: u64,
416    /// Whether a queued job waits for CPU that memory and a slot have room for.
417    pub held: bool,
418}
419
420impl CpuBudget {
421    /// The budget's line as clients show it beside memory, such as `CPU 4.5 of 32 CPUs reserved` or, while a job
422    /// waits for it, `CPU 30 of 32 CPUs reserved; a job waits for CPU`.
423    pub fn line(&self) -> String {
424        let mut line = format!(
425            "CPU {} of {} CPUs reserved",
426            cores(self.in_use),
427            cores(self.budget)
428        );
429        if self.held {
430            line.push_str("; a job waits for CPU");
431        }
432        line
433    }
434}
435
436impl Budget {
437    /// How runners are admitted, as clients add it to the budget's line: nothing under caps, else `by predicted
438    /// peaks`, and why nothing starts while the slice is at the high-water mark.
439    pub fn admission_note(&self) -> Option<&'static str> {
440        match (self.admission, self.held) {
441            (AdmissionPolicy::Caps, _) => None,
442            (AdmissionPolicy::Measured, false) => Some("by predicted peaks"),
443            (AdmissionPolicy::Measured, true) => Some(
444                "by predicted peaks; the slice is past 90% of its MemoryMax, so no runner starts",
445            ),
446        }
447    }
448}
449
450/// A shell container's memory cap: its record's profile's limit when the shell opened.
451#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
452pub struct ShellMemory {
453    /// The job whose record the shell runs.
454    pub job: u64,
455    /// Its cap in bytes.
456    pub memory: u64,
457}
458
459/// A seed container's memory cap: its profile's limit when the reseed started, or, for a requested reseed whose
460/// profile drained, the room it holds until it starts.
461#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
462pub struct SeedMemory {
463    /// The profile it reseeds.
464    pub profile: String,
465    /// Its cap in bytes.
466    pub memory: u64,
467}
468
469/// A runner profile and what runs in its slots.
470#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
471pub struct ProfileStatus {
472    /// The name under `[profiles]` in `runnerd.toml`.
473    pub name: String,
474    /// The image its runners run, such as `runnerd-runner:base`.
475    pub image: String,
476    /// CPUs per slot.
477    pub cpus: u32,
478    /// Bytes per slot.
479    pub memory: u64,
480    /// Set while slot 1's layers become the profile's snapshot.
481    pub reseeding: bool,
482    /// Index 0 is slot 1.
483    pub slots: Vec<SlotStatus>,
484}
485
486/// One slot of a profile: room for one job.
487#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
488pub struct SlotStatus {
489    /// 1-based.
490    pub slot: usize,
491    /// `None` when the slot is free.
492    pub runner: Option<RunnerStatus>,
493}
494
495/// A runner registered for one job, in its container.
496#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
497pub struct RunnerStatus {
498    /// `owner/name` the runner is registered on.
499    pub repository: String,
500    /// The runner's name, also its container's.
501    pub name: String,
502    /// The runner's ID on GitHub.
503    pub runner_id: u64,
504    /// The Docker container ID.
505    pub container: String,
506    /// Whether it runs a job, as GitHub or its log reports it: its log names the job before GitHub lists it busy.
507    pub busy: bool,
508    /// Seconds since the Unix epoch.
509    pub started_at: u64,
510    /// What it takes of the budget: its cap under caps; under measured admission the larger of its predicted peak
511    /// and its last sampled memory.
512    pub reserved: u64,
513    /// A warm runner: idle, and no queued job of its repository waits for it; it takes the next one GitHub hands
514    /// out.
515    pub warm: bool,
516}
517
518/// A configured repository and its queue.
519#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
520pub struct RepositoryStatus {
521    /// `owner/name` on GitHub.
522    pub name: String,
523    /// Its pools in config order, one per `[[repositories]]` entry: a job runs in the one whose labels match it
524    /// (ADR 0058).
525    pub pools: Vec<PoolStatus>,
526    /// Paused repositories get no new runners; their running jobs finish.
527    pub paused: bool,
528    /// Its jobs in `Status::queued`: queued ones none of its runners took yet, those no pool serves too.
529    pub queued_jobs: usize,
530    /// Why its queued jobs cannot be listed, while the listings fail (ADR 0132); absent while they answer, before the
531    /// first, while GitHub's rate limit stops them, and from a daemon before 0.2.
532    #[serde(default, skip_serializing_if = "Option::is_none")]
533    pub listing: Option<ListingFailure>,
534}
535
536/// Why runnerd cannot list a repository's queued jobs (ADR 0132): what class of failure GitHub's answer was, never its
537/// text. The engine keeps it until a listing answers or a reload removes the repository; a restart forgets it.
538#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
539pub struct ListingFailure {
540    /// When the listings began to fail, in seconds since the Unix epoch.
541    pub since: u64,
542    /// Why the last one failed.
543    pub problem: ListingProblem,
544}
545
546/// What a failed listing ran into.
547#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
548#[serde(rename_all = "kebab-case", tag = "kind")]
549pub enum ListingProblem {
550    /// runnerd could read no GitHub token.
551    NoToken,
552    /// GitHub refused the token: 401.
553    Unauthorized,
554    /// GitHub answered 403, 404 or 410: the token may not read the repository's Actions, cannot see it, or it is gone.
555    Refused {
556        /// The HTTP status.
557        status: u16,
558    },
559    /// GitHub answered another error status, such as a 5xx.
560    Failed {
561        /// The HTTP status.
562        status: u16,
563    },
564    /// No answer came: the network, DNS or TLS failed.
565    Unreachable,
566    /// The answer was not what GitHub sends.
567    Unexpected,
568    /// A problem from a newer daemon than this client knows.
569    #[serde(other)]
570    Unknown,
571}
572
573impl ListingProblem {
574    /// Whether it is the token's, and so every repository's: none read, or GitHub refused it.
575    pub fn is_token(&self) -> bool {
576        matches!(self, ListingProblem::NoToken | ListingProblem::Unauthorized)
577    }
578}
579
580impl ListingFailure {
581    /// What fails, as every client says it: `Cannot list halyard/ledger's queued jobs: GitHub answers 404; the token
582    /// cannot see the repository, or it is gone`.
583    pub fn line(&self, repository: &str) -> String {
584        format!(
585            "Cannot list {repository}'s queued jobs: {}",
586            self.problem.why()
587        )
588    }
589}
590
591impl ListingProblem {
592    /// Why the listing fails, as the daemon's log and every client say it: `GitHub answers 502`. Never GitHub's text.
593    pub fn why(&self) -> String {
594        match self {
595            ListingProblem::NoToken => "runnerd has no GitHub token".to_owned(),
596            ListingProblem::Unauthorized => "GitHub refuses runnerd's token (401)".to_owned(),
597            ListingProblem::Refused { status: 403 } => {
598                "GitHub answers 403; the token may not read the repository's Actions".to_owned()
599            }
600            ListingProblem::Refused { status } => format!(
601                "GitHub answers {status}; the token cannot see the repository, or it is gone"
602            ),
603            ListingProblem::Failed { status } => format!("GitHub answers {status}"),
604            ListingProblem::Unreachable => "GitHub does not answer".to_owned(),
605            ListingProblem::Unexpected => "GitHub's answer is not what runnerd reads".to_owned(),
606            ListingProblem::Unknown => "for a reason this client does not know".to_owned(),
607        }
608    }
609}
610
611/// One of a repository's pools: its runners in one profile.
612#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
613pub struct PoolStatus {
614    /// The profile its jobs run in.
615    pub profile: String,
616    /// The labels its runners register with.
617    pub labels: Vec<String>,
618    /// Its jobs in `Status::queued`: those its labels match.
619    pub queued_jobs: usize,
620}
621
622/// A queued job runners of this daemon can take, as GitHub listed it at the last poll.
623#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
624pub struct QueuedJob {
625    /// GitHub's job ID.
626    pub id: u64,
627    /// The repository whose runners can take it, `owner/name`.
628    pub repository: String,
629    /// The profile of its repository's pool whose labels match it; `None` when no pool's do.
630    #[serde(default, skip_serializing_if = "Option::is_none")]
631    pub profile: Option<String>,
632    /// The workflow run it belongs to.
633    pub run_id: u64,
634    /// 1 for the first attempt of the run, higher for re-runs.
635    pub run_attempt: u32,
636    /// The workflow's name.
637    pub workflow: Option<String>,
638    /// The job's name in the workflow.
639    pub name: String,
640    /// The branch the run is for.
641    pub branch: Option<String>,
642    /// The job's page on GitHub.
643    pub url: Option<String>,
644    /// When GitHub queued it, in seconds since the Unix epoch; 0 if GitHub did not say.
645    pub queued_at: u64,
646    /// Why no runner has it yet, as the last plan saw it. `None` before the daemon has planned with it, and while
647    /// it shuts down.
648    #[serde(default, skip_serializing_if = "Option::is_none")]
649    pub waiting: Option<Waiting>,
650}
651
652/// The slot of an idle runner of `job`'s repository, in its pool's profile when known, such as `jvm 2`.
653fn starting_for(job: &QueuedJob, status: &Status) -> Option<String> {
654    status
655        .profiles
656        .iter()
657        .filter(|profile| job.profile.as_ref().is_none_or(|own| *own == profile.name))
658        .find_map(|profile| {
659            profile
660                .slots
661                .iter()
662                .find(|slot| {
663                    slot.runner
664                        .as_ref()
665                        .is_some_and(|runner| runner.repository == job.repository && !runner.busy)
666                })
667                .map(|slot| format!("{}\u{a0}{}", profile.name, slot.slot))
668        })
669}
670
671impl QueuedJob {
672    /// Where it runs, as every client names it: its repository, and its pool's profile when the repository has
673    /// several pools in `status`, such as `halyard/ledger · light`.
674    pub fn place(&self, status: &Status) -> String {
675        let pools = status
676            .repositories
677            .iter()
678            .find(|repository| repository.name == self.repository)
679            .map_or(0, |repository| repository.pools.len());
680        match self.profile.as_deref().filter(|_| pools > 1) {
681            Some(profile) => format!("{}\u{a0}· {profile}", self.repository),
682            None => self.repository.clone(),
683        }
684    }
685}
686
687impl RepositoryStatus {
688    /// The profile of its first pool, which every client colours it by.
689    pub fn first_profile(&self) -> &str {
690        self.pools.first().map_or("", |pool| pool.profile.as_str())
691    }
692
693    /// Its pools' profiles, as every client lists them on one line: `jvm` or `jvm + light`.
694    pub fn profiles(&self) -> String {
695        self.pools
696            .iter()
697            .map(|pool| pool.profile.as_str())
698            .collect::<Vec<_>>()
699            .join(" + ")
700    }
701
702    /// The profile of its pool whose labels match a job asking for `labels`, as the daemon picks it.
703    pub fn pool_for(&self, labels: &[String]) -> Option<String> {
704        self.pools
705            .iter()
706            .find(|pool| crate::config::labels_match(labels, &pool.labels))
707            .map(|pool| pool.profile.clone())
708    }
709}
710
711/// What failed, that holds a repository's runners back for a while.
712#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
713#[serde(rename_all = "kebab-case")]
714pub enum HoldCause {
715    /// Its runners ended without connecting, so without taking a job (ADR 0104).
716    NoJob,
717    /// Its runners connected and ended without a job, one after the other.
718    IdleExits,
719    /// GitHub did not register its runner.
720    Register,
721    /// Docker did not start its runner's container.
722    Start,
723}
724
725/// Why a queued job has no runner of its own yet. Jobs are served oldest first; each takes the first of these that
726/// holds for it.
727#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
728#[serde(rename_all = "kebab-case", tag = "kind")]
729pub enum Waiting {
730    /// Its repository is paused.
731    Paused,
732    /// Runners of its repository failed, so none starts before `until` (seconds since the Unix epoch).
733    Backoff {
734        /// When runners may start again.
735        until: u64,
736        /// What failed.
737        cause: HoldCause,
738    },
739    /// A runner started for it and waits for GitHub to hand it the job: a runner takes any job its labels match,
740    /// so not necessarily this one.
741    RunnerStarting,
742    /// Slot 1 of its profile is becoming the profile's snapshot, so none of the profile's slots starts.
743    Reseeding {
744        /// The profile.
745        profile: String,
746    },
747    /// Its profile's image is being pulled, so none of the profile's slots starts.
748    PullingImage {
749        /// The profile.
750        profile: String,
751    },
752    /// Every slot of its profile holds a runner.
753    NoSlot {
754        /// The profile.
755        profile: String,
756    },
757    /// What its runner would reserve does not fit the memory budget left. Under measured admission, `free` is 0
758    /// while the slice is at the high-water mark (`Budget::held`).
759    Memory {
760        /// What its runner would reserve, in bytes.
761        needs: u64,
762        /// What the budget has left, in bytes.
763        free: u64,
764    },
765    /// What its runner would reserve does not fit what the host's memory has to spare beside its other programs
766    /// (`HostBudget::spare`), though it fits the slice's budget. `free` is 0 while the host's tasks stall for memory
767    /// (`HostBudget::stalled`).
768    HostMemory {
769        /// What its runner would reserve, in bytes.
770        needs: u64,
771        /// What the host has to spare, in bytes.
772        free: u64,
773    },
774    /// What its runner would reserve does not fit the CPU budget left, in millicores.
775    Cpu {
776        /// What its runner would reserve.
777        needs: u64,
778        /// What the CPU budget has left.
779        free: u64,
780    },
781    /// An older job has waited long enough for CPU that no younger one takes CPU before it.
782    BehindCpu {
783        /// The older job's ID.
784        job: u64,
785    },
786    /// An older job has waited long enough for memory that no younger one starts before it (ADR 0096).
787    BehindMemory {
788        /// The older job's ID.
789        job: u64,
790    },
791    /// An older job is about to have a runner start for it, which takes the slot, memory or CPU this one would
792    /// otherwise start with: what holds it is that start, not what runs now.
793    BehindStart {
794        /// The older job's ID.
795        job: u64,
796    },
797    /// No pool of its repository has every label it asks for, so no runner of this daemon can take it (ADR 0058).
798    NoPool {
799        /// The labels it asks for.
800        labels: Vec<String>,
801    },
802    /// A host crash may have torn what runners wrote without syncing, and the daemon has not emptied it yet, so no
803    /// runner starts (ADR 0141, 0142).
804    Torn {
805        /// The volumes a container keeps it from removing, as of its last try: the owner's to remove once nothing
806        /// mounts them. Empty while it has not tried, or Docker failed.
807        volumes: Vec<String>,
808    },
809    /// A reason from a newer daemon than this client knows, so a reason added later does not keep the client from
810    /// reading the status.
811    #[serde(other)]
812    Unknown,
813}
814
815/// What a [`Waiting`] reason waits on, which clients draw it by.
816#[derive(Debug, Clone, Copy, PartialEq, Eq)]
817pub enum WaitKind {
818    /// Something the owner can act on: a pause, runners that failed to take a job, or labels no pool serves.
819    Act,
820    /// Something that clears by itself shortly: a runner starting, a reseed, an image pull, an older job's start.
821    Soon,
822    /// Room: a slot, memory or CPU that a running job holds.
823    Full,
824    /// A reason this client does not know (`Waiting::Unknown`), drawn neutral.
825    Other,
826}
827
828impl Waiting {
829    /// Whether runnerd cannot start the job with the config it runs with: its repository is paused, or no pool serves
830    /// its labels. Not that nothing ever will: a resume or a config change can, and another machine's runner may take
831    /// it. A backoff ends by itself, and a reason this client does not know says nothing (ADR 0132).
832    pub fn cannot_start(&self) -> bool {
833        matches!(self, Waiting::Paused | Waiting::NoPool { .. })
834    }
835
836    /// What it waits on.
837    pub fn kind(&self) -> WaitKind {
838        match self {
839            Waiting::Paused | Waiting::Backoff { .. } | Waiting::NoPool { .. } => WaitKind::Act,
840            Waiting::Torn { volumes } if !volumes.is_empty() => WaitKind::Act,
841            Waiting::Unknown => WaitKind::Other,
842            Waiting::Torn { .. }
843            | Waiting::RunnerStarting
844            | Waiting::Reseeding { .. }
845            | Waiting::PullingImage { .. }
846            | Waiting::BehindStart { .. } => WaitKind::Soon,
847            Waiting::NoSlot { .. }
848            | Waiting::Memory { .. }
849            | Waiting::HostMemory { .. }
850            | Waiting::Cpu { .. }
851            | Waiting::BehindCpu { .. }
852            | Waiting::BehindMemory { .. } => WaitKind::Full,
853        }
854    }
855
856    /// Why `job` waits, in the owner's words with the numbers that hold it, as every client says it: such as
857    /// `All 2 node slots are busy` or `Needs 9G of memory; 2.5G of the budget is free`. `status` is the one `job`
858    /// is queued in, which names the slot of a starting runner and the older job one waits behind.
859    pub fn reason(&self, job: &QueuedJob, status: &Status, now: u64) -> String {
860        let size = |bytes: u64| crate::config::ByteSize(bytes).to_string();
861        let older_name = |id: u64| {
862            status
863                .queued
864                .iter()
865                .find(|candidate| candidate.id == id)
866                .map_or_else(|| "an older job".to_owned(), |older| older.name.clone())
867        };
868        match self {
869            Waiting::Paused => format!("{} is paused", job.repository),
870            Waiting::Unknown => "Waits for a reason this client does not know yet".to_owned(),
871            Waiting::NoPool { labels } if labels.is_empty() => {
872                "It asks for no labels, so no profile serves it".to_owned()
873            }
874            Waiting::NoPool { labels } => format!(
875                "No profile serves the {} {}",
876                if labels.len() == 1 { "label" } else { "labels" },
877                labels.join(", ")
878            ),
879            Waiting::Backoff { until, cause } => format!(
880                "{}; retrying in {}",
881                match cause {
882                    HoldCause::NoJob => "Runners failed to take a job",
883                    HoldCause::IdleExits => "Runners ended without a job after waiting for one",
884                    HoldCause::Register => "Runners cannot register with GitHub",
885                    HoldCause::Start => "Runners cannot start",
886                },
887                duration(until.saturating_sub(now))
888            ),
889            Waiting::RunnerStarting => match starting_for(job, status) {
890                Some(slot) => format!("A runner is starting for it in {slot}"),
891                None => "A runner is starting for it".to_owned(),
892            },
893            Waiting::Reseeding { profile } => format!("{profile} is rebuilding its snapshot"),
894            Waiting::Torn { volumes } if volumes.is_empty() => {
895                "Emptying the caches a host crash may have torn".to_owned()
896            }
897            Waiting::Torn { volumes } => format!(
898                "A host crash may have torn caches a container holds; once nothing mounts them: docker volume rm {}",
899                volumes.join(" ")
900            ),
901            Waiting::PullingImage { profile } => format!("Pulling {profile}'s image"),
902            Waiting::NoSlot { profile } => {
903                let slots = status
904                    .profiles
905                    .iter()
906                    .find(|candidate| &candidate.name == profile)
907                    .map_or(0, |profile| profile.slots.len());
908                if slots == 1 {
909                    format!("The {profile} slot is busy")
910                } else {
911                    format!("All {slots} {profile} slots are busy")
912                }
913            }
914            Waiting::Memory { .. } if status.budget.held => {
915                "The slice is past 90% of its MemoryMax, so nothing starts".to_owned()
916            }
917            Waiting::Memory { needs, free } => format!(
918                "Needs {} of memory; {} of the budget is free",
919                size(*needs),
920                size(*free)
921            ),
922            Waiting::HostMemory { needs, free } => host_reason(
923                *needs,
924                *free,
925                status.budget.host.as_ref(),
926                status.budget.memory_in_use,
927            ),
928            Waiting::Cpu { needs, free } => {
929                format!(
930                    "Needs {}; {} are free",
931                    cpus(*needs),
932                    admission::free_cores(*free)
933                )
934            }
935            Waiting::BehindCpu { job: older } => {
936                format!("Behind {}, which waits for CPU", older_name(*older))
937            }
938            Waiting::BehindMemory { job: older } => {
939                format!("Behind {}, which waits for memory", older_name(*older))
940            }
941            Waiting::BehindStart { job: older } => {
942                format!(
943                    "Next after {}, whose runner starts first",
944                    older_name(*older)
945                )
946            }
947        }
948    }
949}
950
951/// `GET /v1/dashboard`: the dashboard's `[web]` settings, as the daemon read them when it started.
952#[derive(Debug, Clone, Default, PartialEq, Eq, Serialize, Deserialize)]
953pub struct Dashboard {
954    /// `[web] open`: the one of `[web] origins` that `runnerctl web` opens the dashboard on; the loopback address
955    /// when none.
956    #[serde(default, skip_serializing_if = "Option::is_none")]
957    pub open: Option<String>,
958}
959
960/// `POST /v1/dashboard/pairings`: which of `[web] origins` to make a link for.
961#[derive(Debug, Clone, Default, PartialEq, Eq, Serialize, Deserialize)]
962pub struct PairingQuery {
963    /// One of `[web] origins`; each of them when none.
964    #[serde(default, skip_serializing_if = "Option::is_none")]
965    pub origin: Option<String>,
966}
967
968/// `POST /v1/dashboard/pairings`: fresh one-time links that sign a device in to the dashboard.
969#[derive(Debug, Clone, Default, PartialEq, Eq, Serialize, Deserialize)]
970pub struct Pairings {
971    /// One per origin asked for.
972    pub links: Vec<Pairing>,
973}
974
975/// A link that signs one device in to the dashboard on `origin`, and nowhere else, within `lifetime`.
976#[derive(Clone, PartialEq, Eq, Serialize, Deserialize)]
977pub struct Pairing {
978    /// One of `[web] origins`.
979    pub origin: String,
980    /// The link with its one-time code: a credential, which `Debug` leaves out.
981    pub url: String,
982    /// Seconds it works for from when it was made.
983    pub lifetime: u64,
984}
985
986impl std::fmt::Debug for Pairing {
987    fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
988        formatter
989            .debug_struct("Pairing")
990            .field("origin", &self.origin)
991            .field("lifetime", &self.lifetime)
992            .finish_non_exhaustive()
993    }
994}
995
996/// `POST /v1/shutdown`.
997#[derive(Debug, Clone, Default, PartialEq, Eq, Serialize, Deserialize)]
998pub struct ShutdownRequest {
999    /// Kill running jobs and delete their runners instead of waiting for them.
1000    #[serde(default)]
1001    pub force: bool,
1002}
1003
1004/// What `POST /v1/shutdown` answers: how the daemon stops now, which a stop asked for earlier may have made more
1005/// urgent than the request.
1006#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
1007pub struct ShutdownAnswer {
1008    /// How it stops.
1009    pub stopping: Stopping,
1010}
1011
1012/// How a stopping daemon treats running jobs (ADR 0091).
1013#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
1014#[serde(rename_all = "kebab-case")]
1015pub enum Stopping {
1016    /// `runnerd stop`: it waits for running jobs, then removes every runner.
1017    WaitsForJobs,
1018    /// A signal: it removes idle runners and leaves running jobs to the next start.
1019    LeavesJobs,
1020    /// `runnerd stop --force` or a second signal: it kills running jobs.
1021    KillsJobs,
1022    /// A kind of stop from a newer daemon than this client knows, so one added later does not keep the client from
1023    /// reading the status.
1024    #[serde(other)]
1025    Unknown,
1026}
1027
1028impl Stopping {
1029    /// What it does, as every client says it: `shutting down: waits for running jobs`.
1030    pub fn line(self) -> &'static str {
1031        match self {
1032            Stopping::WaitsForJobs => "shutting down: waits for running jobs",
1033            Stopping::LeavesJobs => "shutting down: leaves running jobs to the next start",
1034            Stopping::KillsJobs => "shutting down: kills running jobs",
1035            Stopping::Unknown => "shutting down",
1036        }
1037    }
1038}
1039
1040impl Status {
1041    /// What a stopping daemon does, as every client says it ([`Stopping::line`]); `shutting down` from a daemon
1042    /// that does not say how; `None` while it runs.
1043    pub fn stopping_line(&self) -> Option<&'static str> {
1044        match (self.stopping, self.shutting_down) {
1045            (Some(stopping), _) => Some(stopping.line()),
1046            (None, true) => Some("shutting down"),
1047            (None, false) => None,
1048        }
1049    }
1050
1051    /// Why no runner starts while GitHub's rate limit is used up, as every client says it; `None` while it is not.
1052    pub fn github_line(&self) -> Option<&'static str> {
1053        self.github_limited_until
1054            .map(|_| "GitHub's rate limit is used up: no runner starts until it resets")
1055    }
1056}
1057
1058/// The body of every error response.
1059#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
1060pub struct ErrorResponse {
1061    /// What went wrong, for people to read.
1062    pub message: String,
1063}