Skip to main content

runnerd_core/
api.rs

1//! The daemon's JSON API, version 1: the request and response types every client deserializes, and the
2//! principal a transport authenticates. Plain data only: no I/O, no Docker or GitHub types, no secrets but a
3//! [`Pairing`]'s one-time link, which its `Debug` leaves out.
4//!
5//! Routes, all under the authenticated principal:
6//!
7//! | Route | Body |
8//! |---|---|
9//! | `GET /v1/status` | [`Status`] |
10//! | `POST /v1/repositories/{owner}/{name}/pause?`[`PauseQuery`] | [`Paused`]; 409 [`Busy`] while it runs a job |
11//! | `POST /v1/repositories/{owner}/{name}/resume` | none, 204 |
12//! | `POST /v1/repositories/pause?`[`PauseQuery`] | [`Paused`]; 409 [`Busy`] while any runs a job, pausing none |
13//! | `POST /v1/repositories/resume` | none, 204: every repository resumed |
14//! | `POST /v1/profiles/{name}/reseed` | none, 202; 409 while it reseeds; `cache.reseeded` when done |
15//! | `POST /v1/caches/{name}/clean` | none, 204; 409 [`Busy`] while a runner uses it |
16//! | `GET /v1/config` | [`ConfigFile`] |
17//! | `POST /v1/config/check` | TOML in, [`ConfigCheck`] |
18//! | `PUT /v1/config` | TOML in, [`ConfigCheck`]; 422 [`ConfigCheck`] when invalid |
19//! | `GET /v1/doctor` | [`Doctor`]; 503 [`ErrorResponse`] while no run has finished |
20//! | `GET /v1/attention` | [`Attention`]; 404 from a daemon before 0.2 |
21//! | `POST /v1/shutdown` | [`ShutdownRequest`] in, 202 [`ShutdownAnswer`] |
22//! | `GET /v1/dashboard` | [`Dashboard`] |
23//! | `POST /v1/dashboard/sign-out` | none, 204 |
24//! | `POST /v1/dashboard/pairings?`[`PairingQuery`] | [`Pairings`]; 409 when the dashboard is off or has no `[web] origins` |
25//! | `GET /v1/jobs?`[`JobQuery`] | [`JobPage`] |
26//! | `GET /v1/jobs/{id}` | [`Job`] |
27//! | `GET /v1/jobs/{id}/samples` | [`SampleSeries`] |
28//! | `GET /v1/jobs/{id}/logs?`[`LogQuery`] | `text/plain`, one line per log line |
29//! | `GET /v1/host/samples?`[`HostSampleQuery`] | [`HostSamples`] |
30//! | `GET /v1/stats?`[`StatsQuery`] | [`Stats`] |
31//! | `GET /v1/activity?`[`ActivityQuery`] | [`Activity`] |
32//! | `GET /v1/sizing?`[`SizingQuery`] | [`Sizing`] |
33//! | `GET /v1/insights?`[`InsightsQuery`] | [`Insights`] |
34//! | `GET /v1/digest` | [`Digest`] |
35//! | `GET /v1/savings` | [`Savings`] |
36//! | `POST /v1/notify/test?`[`NotifyTestQuery`] | [`NotifyTest`]; 404 without `[notify]` targets or with no such one |
37//! | `GET /v1/caches` | [`Caches`] |
38//! | `GET /v1/caches/history?`[`CacheHistoryQuery`] | [`CacheHistory`] |
39//! | `GET /v1/events?`[`EventQuery`] | `text/event-stream` of [`EventEnvelope`] |
40//! | `GET /v1/records` | [`Records`] |
41//! | `DELETE /v1/records/{job}` | [`RemovedRecords`]; 409 while a shell is open in it |
42//! | `DELETE /v1/records?`[`RemoveRecordsQuery`] | [`RemovedRecords`]; records with an open shell stay |
43//! | `POST /v1/records/{job}/shell?`[`ShellQuery`] | `Upgrade:` [`SHELL_PROTOCOL`], 101, then the shell's terminal; 409 when the budget is full |
44//!
45//! Errors carry an [`ErrorResponse`].
46
47use serde::{Deserialize, Serialize};
48
49mod actions;
50mod admission;
51mod attention;
52mod budget;
53mod control;
54mod disk;
55mod events;
56mod insights;
57mod jobs;
58mod labels;
59mod records;
60mod results;
61mod savings;
62mod sizing;
63mod slots;
64mod stats;
65mod storage;
66mod tone;
67mod units;
68
69pub use actions::{
70    ActionAnswer, ActionOutcome, ActionPreview, ActionQuery, ActionRefusal, ActionResult,
71    ActionTarget, CONFIRM_SECS, JOB_ACTIONS_CHECK, JobAction, JobActions, PreviewFacts, RunSeen,
72    SIBLINGS_MAX, Superseded, UnknownCause, actions_in_flight, actions_no_run,
73    actions_not_configured, actions_off,
74};
75pub use admission::{Admission, AdmissionPolicy, Basis, CpuAdmission, JobPeak, cores, cpus};
76pub use attention::{
77    ATTENTION_MAX, Attention, AttentionItem, AttentionKind, Fix, Subject, attention_items,
78};
79pub use budget::{
80    BudgetParts, ClosedKind, ClosedPart, FreePart, HeldKind, HeldPart, HostFigure, Reading,
81    SlotPart,
82};
83pub use control::{
84    Busy, Check, CheckStatus, ConfigCheck, ConfigFile, Doctor, DoctorState, PauseQuery, Paused,
85    ReplaceQuery, RunningJob,
86};
87pub use disk::{
88    Commits, DiskWrites, DiskWritten, DriveShares, DriveWrites, JobWrites, Measure, WriteSection,
89    WriteTop,
90};
91pub use events::{Event, EventEnvelope, EventQuery, RunnerFailure};
92pub use insights::{
93    AutoNote, Delivery, Digest, HostHazard, HostSpan, INSIGHT_DAYS, INSIGHT_DAYS_MAX, Insight,
94    InsightKind, Insights, InsightsQuery, LongWait, Notification, NotifyEvent, NotifyTest,
95    NotifyTestQuery, Priority, RepositoryWaits, RunLink, Severity, WaitSpread, WeekWaits,
96};
97#[cfg(feature = "host")]
98pub(crate) use jobs::SAMPLE_GAP_INTERVALS;
99#[cfg(feature = "host")]
100pub(crate) use jobs::TITLE_SOURCE_MAX;
101pub use jobs::{
102    Activity, ActivityQuery, Bucket, BuildCacheCounts, BuildCacheStats, CacheHistory,
103    CacheHistoryQuery, CacheKind, CacheSize, CacheVolume, Caches, ChartLine, ChartLineKind,
104    ChartLines, Holder, HostMemory, HostSample, HostSampleQuery, HostSamples, Job, JobMetrics,
105    JobPage, JobQuery, JobRunner, JobState, JobSummary, JobTitle, Limit, LimitSource, Limits,
106    LogQuery, LogSummary, Milestone, Outcome, OutcomeKind, Phase, PhaseKind, Pressure,
107    RunComparison, Sample, SampleSeries, SampleUsage, Started, Stats, StatsQuery, StatsRow, Step,
108    Swap, Timeline, Total, host_margin, job_title,
109};
110pub use jobs::{
111    BarEnd, Baseline, BaselineRead, COMPARE_STEP_BYTES, COMPARE_STEPS_MAX, ChangeKind,
112    CompareQuery, ComparedSteps, FigureDelta, FigureKind, Gap, GapKind, JOB_LABELS_MAX,
113    JOB_SEARCH_MAX, JOB_TEXT_MAX, JOBS_PAGE_BYTES, JOBS_PAGE_DEFAULT, JOBS_PAGE_MAX, JobComparison,
114    JobList, JobOrder, JobRange, JobSort, Lane, RecordFilter, RecordMark, RunnerSpan, SlotKey,
115    SlotTimeline, SlottedBar, SlottedRunner, SortOrder, Span, StepDelta, StepMatch, StepUsage,
116    TIMELINE_BARS_MAX, TIMELINE_DEFAULT, TIMELINE_WINDOW_MAX, TimelineBar, TimelineQuery,
117    TimelineRows, Unavailable, row_bytes,
118};
119pub use jobs::{
120    Color, FailureSummary, LINE_BYTES_MAX, LINES_DEFAULT, LINES_MAX, LineKind, LineRange, LineRead,
121    LogEvidence, LogGroup, LogLinesQuery, LogMark, LogNeedle, LogOutline, LogSearch,
122    LogSearchQuery, MarkKind, OutlineChange, OutlineParts, Outliner, RUNS_MAX, Run,
123    SEARCH_QUERY_MAX, SavedOutline, Search, StepLines, StepLog, StoredLine, StoredLog, StyledLine,
124};
125pub use labels::{Column, Feature, JOBS_HEADING, NOT_SUPPORTED, Section, View};
126pub use records::{
127    Record, RecordLimits, RecordNote, RecordRemoval, Records, RemoveRecordsQuery, RemovedRecords,
128    SHELL_PROTOCOL, ShellQuery,
129};
130pub use results::{force_pause_consequence, reseed_requested, reseeded};
131pub use savings::{
132    Period, PricedRun, PricedRunner, Pricing, PricingHistory, SELF_HOSTED_RATE, SavedWindow,
133    Savings, Sku, Usd, Visibility, billed_minutes, reads_since,
134};
135pub use sizing::{
136    Advice, AutoPlan, GRANULE, JobSizing, MemoryBound, MemoryHold, Need, PAGE_CACHE_ROOM, Peaks,
137    ProfilePlan, ProfileSizing, Resize, RevertCause, SAMPLE_INTERVAL, SLOT_DISK, Sizing,
138    SizingChange, SizingHold, SizingQuery, SizingRefused, SizingRevert, Spread, Starts, need,
139    period, sampled_note,
140};
141pub use slots::{Figure, Health, HealthKind, SlotState};
142pub use stats::StatsSummary;
143pub use storage::{
144    ACTIONS_CACHE, BUILD_CACHE_TOOLS, BuildCacheActivity, CacheLimit, RECORDS_NEAR_FULL,
145    RecordState, actions_cache_place, records_check, records_near_full,
146};
147pub use tone::{Tone, Toned};
148pub(crate) use units::days_from_civil;
149pub use units::{ago, cores_used, cpu_time, duration, in_cores};
150
151/// Who sent a request, as the transport authenticated it. Handlers get it from one place, so a new transport only
152/// has to produce one.
153#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
154#[serde(rename_all = "kebab-case", tag = "kind")]
155pub enum Principal {
156    /// The user the daemon runs as, on the local Unix socket.
157    LocalOwner {
158        /// The Unix user ID the socket's peer runs as.
159        uid: u32,
160    },
161}
162
163/// `GET /v1/status`: what the daemon serves and runs right now.
164#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
165pub struct Status {
166    /// The daemon's version, from its `Cargo.toml`.
167    pub version: String,
168    /// Set once the daemon stops polling to shut down. [`Status::stopping_line`] says how.
169    pub shutting_down: bool,
170    /// How it stops, once it does; absent from a daemon before it said.
171    #[serde(default, skip_serializing_if = "Option::is_none")]
172    pub stopping: Option<Stopping>,
173    /// Until when GitHub's rate limit is used up, in seconds since the Unix epoch: no runner starts before.
174    /// [`Status::github_line`] says it. Absent while it is not.
175    #[serde(default, skip_serializing_if = "Option::is_none")]
176    pub github_limited_until: Option<u64>,
177    /// The memory budget and what runners use of it.
178    pub budget: Budget,
179    /// Every configured profile with its slots.
180    pub profiles: Vec<ProfileStatus>,
181    /// Every configured repository.
182    pub repositories: Vec<RepositoryStatus>,
183    /// Queued jobs the repositories' runners can take and none holds yet, at the last poll, oldest first. No event
184    /// carries the list: `job.queued` announces a job, `job.started` and `job.finished` its leaving the queue.
185    /// A repository whose last listing failed, as while GitHub rate-limits runnerd, or that is no longer polled
186    /// because runnerd is shutting down, shows no jobs here, while the history keeps its jobs queued.
187    pub queued: Vec<QueuedJob>,
188    /// Whether the daemon cancels and re-runs jobs (ADR 0134); a daemon before 0.2 sends none, read as
189    /// [`JobActions::Unsupported`].
190    #[serde(default)]
191    pub job_actions: JobActions,
192}
193
194/// The memory budget runners and the mirror share, in bytes.
195#[derive(Debug, Clone, Default, PartialEq, Eq, Serialize, Deserialize)]
196pub struct Budget {
197    /// The systemd slice runner containers run in.
198    pub slice: String,
199    /// The slice's `MemoryMax`; `None` when it is unlimited.
200    pub memory_max: Option<u64>,
201    /// Memory the running containers reserve: the runners' caps, or under measured admission the larger of each
202    /// runner's predicted peak and its last sampled memory, nothing for one whose container is gone, and the caps of
203    /// the seed containers, shells and the mirror.
204    pub memory_in_use: u64,
205    /// What pooling takes off `memory_in_use` (ADR 0099, 0130); absent while it takes nothing.
206    #[serde(default, skip_serializing_if = "Option::is_none")]
207    pub pooled: Option<Pooled>,
208    /// The growth count both budgets go by, and its brake (ADR 0130); `pooled` and `host.credit` say what it takes off
209    /// `memory_in_use`. Absent under caps admission, with `[host] pooled_count` off, and from an older daemon.
210    #[serde(default, skip_serializing_if = "Option::is_none")]
211    pub counted: Option<Counted>,
212    /// What a new runner must fit in beside them.
213    pub admission: AdmissionPolicy,
214    /// Under measured admission, whether the slice is at the high-water mark, so no runner starts.
215    pub held: bool,
216    /// The mirror's part of `memory_in_use`: its cap while it is enabled or still running, else 0.
217    pub mirror_memory: u64,
218    /// The seed containers' part of `memory_in_use`, one per profile that reseeds, by profile name.
219    pub seed_memory: Vec<SeedMemory>,
220    /// The shells' part of `memory_in_use`, one per shell open in a record, by job.
221    pub shell_memory: Vec<ShellMemory>,
222    /// What the host's memory leaves runners beside its other programs, from a recent reading of it; absent
223    /// without one.
224    #[serde(default, skip_serializing_if = "Option::is_none")]
225    pub host: Option<HostBudget>,
226    /// The CPU budget, under `[host] cpu_overcommit`. Absent without one.
227    #[serde(default, skip_serializing_if = "Option::is_none")]
228    pub cpu: Option<CpuBudget>,
229    /// The CPUs the slice may use, in millicores: its `AllowedCPUs` or the host's, less under a lower `CPUQuota`,
230    /// whether or not a CPU budget admits runners. The runners' CPU limits are quotas that may add up past it.
231    /// Absent when it could not be read.
232    #[serde(default, skip_serializing_if = "Option::is_none")]
233    pub slice_cpus: Option<u64>,
234}
235
236/// The count the budgets go by while pooling takes something off `Budget::memory_in_use` (ADR 0099).
237#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
238pub struct Pooled {
239    /// What it takes off, in bytes.
240    pub credit: u64,
241    /// `Budget::memory_in_use` less `credit`: what new runners fit beside under `MemoryMax`. The host's limit reads
242    /// `memory_in_use`.
243    pub counted: u64,
244    /// How many runners the credit counts on.
245    pub runners: u64,
246}
247
248/// How far back the growth count's brake compares what runs uses: 10 minutes (ADR 0130,
249/// `admission::growth::GROWTH_WINDOW`).
250pub const GROWTH_WINDOW: std::time::Duration = std::time::Duration::from_mins(10);
251
252/// The growth count (ADR 0130): what the runners use, and the room kept for what they may still grow by.
253#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
254pub struct Counted {
255    /// What the runners use, by their recent samples, in bytes.
256    pub used: u64,
257    /// The room it keeps for their growth, in bytes; absent while it rests (`brake`).
258    #[serde(default, skip_serializing_if = "Option::is_none")]
259    pub kept: Option<u64>,
260    /// How many of the largest growths the room covers at least.
261    pub jobs: u64,
262    /// Why it rests, while it does: the budgets then go by 0099's and 0108's credits.
263    #[serde(default, skip_serializing_if = "Option::is_none")]
264    pub brake: Option<CountBrake>,
265}
266
267/// Why the growth count rests (ADR 0130).
268#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
269#[serde(rename_all = "kebab-case", tag = "kind")]
270pub enum CountBrake {
271    /// What ran grew past the room kept within `GROWTH_WINDOW`.
272    Grew {
273        /// How much the runners present at both looks grew, together, in bytes.
274        grew: u64,
275        /// The room the earlier look kept, in bytes.
276        kept: u64,
277        /// Until when, in seconds since the Unix epoch.
278        until: u64,
279    },
280    /// The daemon started, or a reload changed what the count reads, too recently to judge growth.
281    Restarted {
282        /// Until when, in seconds since the Unix epoch.
283        until: u64,
284    },
285    /// The host is under the guard's line, or a reading is missing, now or at a look within 5 minutes.
286    Guard {
287        /// Until when, in seconds since the Unix epoch; absent for a moment the brake has not looked at yet.
288        #[serde(default, skip_serializing_if = "Option::is_none")]
289        until: Option<u64>,
290    },
291    /// A brake a newer daemon names; this client shows only that the count rests.
292    #[serde(other)]
293    Unknown,
294}
295
296/// What the host's memory leaves the runners (ADR 0055). New runners fit in the lower of `MemoryMax` and
297/// `limit`; running ones are never stopped for it.
298#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
299pub struct HostBudget {
300    /// The reading it comes from.
301    pub memory: HostMemory,
302    /// What the slice may hold now: what its runners hold without page cache plus the host's `MemAvailable`, less
303    /// a margin kept free for the host's page cache; with a ceiling, plus what the host can still grow by
304    /// (`HostMemory::growable`, ADR 0084).
305    pub limit: u64,
306    /// What that leaves new runners beside what the running ones reserve, in bytes.
307    pub spare: u64,
308    /// Whether the host's tasks stall for memory (`full` pressure), so no runner starts until they stop.
309    pub stalled: bool,
310    /// What the host's limit takes off `Budget::memory_in_use` for runners that may not all grow at once (ADR 0108);
311    /// absent while it takes nothing, and from a daemon before 0.0.30.
312    #[serde(default, skip_serializing_if = "Option::is_none")]
313    pub credit: Option<HostCredit>,
314}
315
316/// What the host's limit takes off the plain count (ADR 0108): `Budget::memory_in_use` less `bytes` is what it counts,
317/// `used` of that is what the runners use, and the rest is room kept for what runs to grow into.
318#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
319pub struct HostCredit {
320    /// What it takes off, in bytes.
321    pub bytes: u64,
322    /// What the runners use, by their recent samples, in bytes.
323    pub used: u64,
324}
325
326impl HostBudget {
327    /// Whether the host itself lacks `needs` bytes for a new runner: its tasks stall, `HostMemory::room` is less, or
328    /// that room is under one more margin, so `MemAvailable` is under twice the plain margin, the line `host::stalls`
329    /// reads. There a small job fits only by what the host is about to run out of, and what closing programs frees is
330    /// what starts it. Otherwise a job the host limit holds waits for what runs, whose reservations reach the limit
331    /// (ADR 0103).
332    pub fn short_of(&self, needs: u64) -> bool {
333        let room = self.memory.room();
334        self.stalled || needs > room || room < host_margin(self.memory.grown())
335    }
336
337    /// What the host's limit counts of what runs, `reserved` being `Budget::memory_in_use`: that less the credit
338    /// (ADR 0108).
339    pub fn counts(&self, reserved: u64) -> u64 {
340        reserved.saturating_sub(self.credit.map_or(0, |credit| credit.bytes))
341    }
342
343    /// What that count is made of while a credit stands, as the host's line, a job's wait and the log say it: `it uses
344    /// 14G and keeps 9G for growth`. `None` without a credit, when it counts every reservation.
345    pub fn growth_clause(&self, reserved: u64) -> Option<String> {
346        let credit = self.credit?;
347        let counts = self.counts(reserved);
348        Some(format!(
349            "it uses {} and keeps {} for growth",
350            crate::config::ByteSize(credit.used.min(counts)),
351            crate::config::ByteSize(counts.saturating_sub(credit.used))
352        ))
353    }
354}
355
356/// Why a job waits for the host's limit, needing `needs` bytes with `free` left of it, by the latest reading: for what
357/// runs, or for a host that lacks the room, whose programs it names. While the host's limit takes a credit, what runs
358/// is what it uses and the room it keeps for growth, not its reservations (ADR 0108); `reserved` is
359/// `Budget::memory_in_use`.
360fn host_reason(needs: u64, free: u64, host: Option<&HostBudget>, reserved: u64) -> String {
361    let size = |bytes: u64| crate::config::ByteSize(bytes).to_string();
362    if let Some(host) = host.filter(|host| !host.short_of(needs)) {
363        if let Some(growth) = host.growth_clause(reserved) {
364            return format!(
365                "The host has {}, but what runs may still grow into it: {growth}; needs {}, {} is free",
366                host.memory.available_line(),
367                size(needs),
368                size(free)
369            );
370        }
371        return format!(
372            "The host has {}, but what runs may still grow into it: needs {}; {} of the {} it leaves the slice is free",
373            host.memory.available_line(),
374            size(needs),
375            size(free),
376            size(host.limit)
377        );
378    }
379    let reason = if host.is_some_and(|host| host.stalled) {
380        "The host is short of memory: its programs stall waiting for it, so nothing starts"
381            .to_owned()
382    } else {
383        format!(
384            "The host is short of memory: needs {}; the host has {} to spare",
385            size(needs),
386            size(free)
387        )
388    };
389    match host {
390        Some(host) => format!("{reason}; {}", host.memory.outside_line()),
391        None => reason,
392    }
393}
394
395/// Hundredths of a percent as a percentage: `1.5%`, `12%`.
396pub fn percent(hundredths: u32) -> String {
397    let whole = hundredths / 100;
398    match hundredths % 100 / 10 {
399        0 => format!("{whole}%"),
400        tenth => format!("{whole}.{tenth}%"),
401    }
402}
403
404/// The CPU budget runners share, in millicores.
405#[derive(Debug, Clone, Copy, Default, PartialEq, Eq, Serialize, Deserialize)]
406pub struct CpuBudget {
407    /// The CPUs the slice may use: its `AllowedCPUs` or the host's, less under a lower `CPUQuota`.
408    pub cpus: u64,
409    /// Those times `cpu_overcommit`: what the runners' reservations must fit.
410    pub budget: u64,
411    /// What the runners reserve: each its predicted average CPU under measured admission, else its `cpus`.
412    pub in_use: u64,
413    /// Whether a queued job waits for CPU that memory and a slot have room for.
414    pub held: bool,
415}
416
417impl CpuBudget {
418    /// The budget's line as clients show it beside memory, such as `CPU 4.5 of 32 CPUs reserved` or, while a job
419    /// waits for it, `CPU 30 of 32 CPUs reserved; a job waits for CPU`.
420    pub fn line(&self) -> String {
421        let mut line = format!(
422            "CPU {} of {} CPUs reserved",
423            cores(self.in_use),
424            cores(self.budget)
425        );
426        if self.held {
427            line.push_str("; a job waits for CPU");
428        }
429        line
430    }
431}
432
433impl Budget {
434    /// How runners are admitted, as clients add it to the budget's line: nothing under caps, else `by predicted
435    /// peaks`, and why nothing starts while the slice is at the high-water mark.
436    pub fn admission_note(&self) -> Option<&'static str> {
437        match (self.admission, self.held) {
438            (AdmissionPolicy::Caps, _) => None,
439            (AdmissionPolicy::Measured, false) => Some("by predicted peaks"),
440            (AdmissionPolicy::Measured, true) => Some(
441                "by predicted peaks; the slice is past 90% of its MemoryMax, so no runner starts",
442            ),
443        }
444    }
445}
446
447/// A shell container's memory cap: its record's profile's limit when the shell opened.
448#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
449pub struct ShellMemory {
450    /// The job whose record the shell runs.
451    pub job: u64,
452    /// Its cap in bytes.
453    pub memory: u64,
454}
455
456/// A seed container's memory cap: its profile's limit when the reseed started, or, for a requested reseed whose
457/// profile drained, the room it holds until it starts.
458#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
459pub struct SeedMemory {
460    /// The profile it reseeds.
461    pub profile: String,
462    /// Its cap in bytes.
463    pub memory: u64,
464}
465
466/// A runner profile and what runs in its slots.
467#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
468pub struct ProfileStatus {
469    /// The name under `[profiles]` in `runnerd.toml`.
470    pub name: String,
471    /// The image its runners run, such as `runnerd-runner:base`.
472    pub image: String,
473    /// CPUs per slot.
474    pub cpus: u32,
475    /// Bytes per slot.
476    pub memory: u64,
477    /// Set while slot 1's layers become the profile's snapshot.
478    pub reseeding: bool,
479    /// Index 0 is slot 1.
480    pub slots: Vec<SlotStatus>,
481}
482
483/// One slot of a profile: room for one job.
484#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
485pub struct SlotStatus {
486    /// 1-based.
487    pub slot: usize,
488    /// `None` when the slot is free.
489    pub runner: Option<RunnerStatus>,
490}
491
492/// A runner registered for one job, in its container.
493#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
494pub struct RunnerStatus {
495    /// `owner/name` the runner is registered on.
496    pub repository: String,
497    /// The runner's name, also its container's.
498    pub name: String,
499    /// The runner's ID on GitHub.
500    pub runner_id: u64,
501    /// The Docker container ID.
502    pub container: String,
503    /// Whether it runs a job, as GitHub or its log reports it: its log names the job before GitHub lists it busy.
504    pub busy: bool,
505    /// Seconds since the Unix epoch.
506    pub started_at: u64,
507    /// What it takes of the budget: its cap under caps; under measured admission the larger of its predicted peak
508    /// and its last sampled memory.
509    pub reserved: u64,
510    /// A warm runner: idle, and no queued job of its repository waits for it; it takes the next one GitHub hands
511    /// out.
512    pub warm: bool,
513}
514
515/// A configured repository and its queue.
516#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
517pub struct RepositoryStatus {
518    /// `owner/name` on GitHub.
519    pub name: String,
520    /// Its pools in config order, one per `[[repositories]]` entry: a job runs in the one whose labels match it
521    /// (ADR 0058).
522    pub pools: Vec<PoolStatus>,
523    /// Paused repositories get no new runners; their running jobs finish.
524    pub paused: bool,
525    /// Its jobs in `Status::queued`: queued ones none of its runners took yet, those no pool serves too.
526    pub queued_jobs: usize,
527    /// Why its queued jobs cannot be listed, while the listings fail (ADR 0132); absent while they answer, before the
528    /// first, while GitHub's rate limit stops them, and from a daemon before 0.2.
529    #[serde(default, skip_serializing_if = "Option::is_none")]
530    pub listing: Option<ListingFailure>,
531}
532
533/// Why runnerd cannot list a repository's queued jobs (ADR 0132): what class of failure GitHub's answer was, never its
534/// text. The engine keeps it until a listing answers or a reload removes the repository; a restart forgets it.
535#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
536pub struct ListingFailure {
537    /// When the listings began to fail, in seconds since the Unix epoch.
538    pub since: u64,
539    /// Why the last one failed.
540    pub problem: ListingProblem,
541}
542
543/// What a failed listing ran into.
544#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
545#[serde(rename_all = "kebab-case", tag = "kind")]
546pub enum ListingProblem {
547    /// runnerd could read no GitHub token.
548    NoToken,
549    /// GitHub refused the token: 401.
550    Unauthorized,
551    /// GitHub answered 403, 404 or 410: the token may not read the repository's Actions, cannot see it, or it is gone.
552    Refused {
553        /// The HTTP status.
554        status: u16,
555    },
556    /// GitHub answered another error status, such as a 5xx.
557    Failed {
558        /// The HTTP status.
559        status: u16,
560    },
561    /// No answer came: the network, DNS or TLS failed.
562    Unreachable,
563    /// The answer was not what GitHub sends.
564    Unexpected,
565    /// A problem from a newer daemon than this client knows.
566    #[serde(other)]
567    Unknown,
568}
569
570impl ListingProblem {
571    /// Whether it is the token's, and so every repository's: none read, or GitHub refused it.
572    pub fn is_token(&self) -> bool {
573        matches!(self, ListingProblem::NoToken | ListingProblem::Unauthorized)
574    }
575}
576
577impl ListingFailure {
578    /// What fails, as every client says it: `Cannot list halyard/ledger's queued jobs: GitHub answers 404; the token
579    /// cannot see the repository, or it is gone`.
580    pub fn line(&self, repository: &str) -> String {
581        format!(
582            "Cannot list {repository}'s queued jobs: {}",
583            self.problem.why()
584        )
585    }
586}
587
588impl ListingProblem {
589    /// Why the listing fails, as the daemon's log and every client say it: `GitHub answers 502`. Never GitHub's text.
590    pub fn why(&self) -> String {
591        match self {
592            ListingProblem::NoToken => "runnerd has no GitHub token".to_owned(),
593            ListingProblem::Unauthorized => "GitHub refuses runnerd's token (401)".to_owned(),
594            ListingProblem::Refused { status: 403 } => {
595                "GitHub answers 403; the token may not read the repository's Actions".to_owned()
596            }
597            ListingProblem::Refused { status } => format!(
598                "GitHub answers {status}; the token cannot see the repository, or it is gone"
599            ),
600            ListingProblem::Failed { status } => format!("GitHub answers {status}"),
601            ListingProblem::Unreachable => "GitHub does not answer".to_owned(),
602            ListingProblem::Unexpected => "GitHub's answer is not what runnerd reads".to_owned(),
603            ListingProblem::Unknown => "for a reason this client does not know".to_owned(),
604        }
605    }
606}
607
608/// One of a repository's pools: its runners in one profile.
609#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
610pub struct PoolStatus {
611    /// The profile its jobs run in.
612    pub profile: String,
613    /// The labels its runners register with.
614    pub labels: Vec<String>,
615    /// Its jobs in `Status::queued`: those its labels match.
616    pub queued_jobs: usize,
617}
618
619/// A queued job runners of this daemon can take, as GitHub listed it at the last poll.
620#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
621pub struct QueuedJob {
622    /// GitHub's job ID.
623    pub id: u64,
624    /// The repository whose runners can take it, `owner/name`.
625    pub repository: String,
626    /// The profile of its repository's pool whose labels match it; `None` when no pool's do.
627    #[serde(default, skip_serializing_if = "Option::is_none")]
628    pub profile: Option<String>,
629    /// The workflow run it belongs to.
630    pub run_id: u64,
631    /// 1 for the first attempt of the run, higher for re-runs.
632    pub run_attempt: u32,
633    /// The workflow's name.
634    pub workflow: Option<String>,
635    /// The job's name in the workflow.
636    pub name: String,
637    /// The branch the run is for.
638    pub branch: Option<String>,
639    /// The job's page on GitHub.
640    pub url: Option<String>,
641    /// When GitHub queued it, in seconds since the Unix epoch; 0 if GitHub did not say.
642    pub queued_at: u64,
643    /// Why no runner has it yet, as the last plan saw it. `None` before the daemon has planned with it, and while
644    /// it shuts down.
645    #[serde(default, skip_serializing_if = "Option::is_none")]
646    pub waiting: Option<Waiting>,
647}
648
649/// The slot of an idle runner of `job`'s repository, in its pool's profile when known, such as `jvm 2`.
650fn starting_for(job: &QueuedJob, status: &Status) -> Option<String> {
651    status
652        .profiles
653        .iter()
654        .filter(|profile| job.profile.as_ref().is_none_or(|own| *own == profile.name))
655        .find_map(|profile| {
656            profile
657                .slots
658                .iter()
659                .find(|slot| {
660                    slot.runner
661                        .as_ref()
662                        .is_some_and(|runner| runner.repository == job.repository && !runner.busy)
663                })
664                .map(|slot| format!("{}\u{a0}{}", profile.name, slot.slot))
665        })
666}
667
668impl QueuedJob {
669    /// Where it runs, as every client names it: its repository, and its pool's profile when the repository has
670    /// several pools in `status`, such as `halyard/ledger · light`.
671    pub fn place(&self, status: &Status) -> String {
672        let pools = status
673            .repositories
674            .iter()
675            .find(|repository| repository.name == self.repository)
676            .map_or(0, |repository| repository.pools.len());
677        match self.profile.as_deref().filter(|_| pools > 1) {
678            Some(profile) => format!("{}\u{a0}· {profile}", self.repository),
679            None => self.repository.clone(),
680        }
681    }
682}
683
684impl RepositoryStatus {
685    /// The profile of its first pool, which every client colours it by.
686    pub fn first_profile(&self) -> &str {
687        self.pools.first().map_or("", |pool| pool.profile.as_str())
688    }
689
690    /// Its pools' profiles, as every client lists them on one line: `jvm` or `jvm + light`.
691    pub fn profiles(&self) -> String {
692        self.pools
693            .iter()
694            .map(|pool| pool.profile.as_str())
695            .collect::<Vec<_>>()
696            .join(" + ")
697    }
698
699    /// The profile of its pool whose labels match a job asking for `labels`, as the daemon picks it.
700    pub fn pool_for(&self, labels: &[String]) -> Option<String> {
701        self.pools
702            .iter()
703            .find(|pool| crate::config::labels_match(labels, &pool.labels))
704            .map(|pool| pool.profile.clone())
705    }
706}
707
708/// What failed, that holds a repository's runners back for a while.
709#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
710#[serde(rename_all = "kebab-case")]
711pub enum HoldCause {
712    /// Its runners ended without connecting, so without taking a job (ADR 0104).
713    NoJob,
714    /// Its runners connected and ended without a job, one after the other.
715    IdleExits,
716    /// GitHub did not register its runner.
717    Register,
718    /// Docker did not start its runner's container.
719    Start,
720}
721
722/// Why a queued job has no runner of its own yet. Jobs are served oldest first; each takes the first of these that
723/// holds for it.
724#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
725#[serde(rename_all = "kebab-case", tag = "kind")]
726pub enum Waiting {
727    /// Its repository is paused.
728    Paused,
729    /// Runners of its repository failed, so none starts before `until` (seconds since the Unix epoch).
730    Backoff {
731        /// When runners may start again.
732        until: u64,
733        /// What failed.
734        cause: HoldCause,
735    },
736    /// A runner started for it and waits for GitHub to hand it the job: a runner takes any job its labels match,
737    /// so not necessarily this one.
738    RunnerStarting,
739    /// Slot 1 of its profile is becoming the profile's snapshot, so none of the profile's slots starts.
740    Reseeding {
741        /// The profile.
742        profile: String,
743    },
744    /// Its profile's image is being pulled, so none of the profile's slots starts.
745    PullingImage {
746        /// The profile.
747        profile: String,
748    },
749    /// Every slot of its profile holds a runner.
750    NoSlot {
751        /// The profile.
752        profile: String,
753    },
754    /// What its runner would reserve does not fit the memory budget left. Under measured admission, `free` is 0
755    /// while the slice is at the high-water mark (`Budget::held`).
756    Memory {
757        /// What its runner would reserve, in bytes.
758        needs: u64,
759        /// What the budget has left, in bytes.
760        free: u64,
761    },
762    /// What its runner would reserve does not fit what the host's memory has to spare beside its other programs
763    /// (`HostBudget::spare`), though it fits the slice's budget. `free` is 0 while the host's tasks stall for memory
764    /// (`HostBudget::stalled`).
765    HostMemory {
766        /// What its runner would reserve, in bytes.
767        needs: u64,
768        /// What the host has to spare, in bytes.
769        free: u64,
770    },
771    /// What its runner would reserve does not fit the CPU budget left, in millicores.
772    Cpu {
773        /// What its runner would reserve.
774        needs: u64,
775        /// What the CPU budget has left.
776        free: u64,
777    },
778    /// An older job has waited long enough for CPU that no younger one takes CPU before it.
779    BehindCpu {
780        /// The older job's ID.
781        job: u64,
782    },
783    /// An older job has waited long enough for memory that no younger one starts before it (ADR 0096).
784    BehindMemory {
785        /// The older job's ID.
786        job: u64,
787    },
788    /// An older job is about to have a runner start for it, which takes the slot, memory or CPU this one would
789    /// otherwise start with: what holds it is that start, not what runs now.
790    BehindStart {
791        /// The older job's ID.
792        job: u64,
793    },
794    /// No pool of its repository has every label it asks for, so no runner of this daemon can take it (ADR 0058).
795    NoPool {
796        /// The labels it asks for.
797        labels: Vec<String>,
798    },
799    /// A host crash may have torn what runners wrote without syncing, and the daemon has not emptied it yet, so no
800    /// runner starts (ADR 0141, 0142).
801    Torn {
802        /// The volumes a container keeps it from removing, as of its last try: the owner's to remove once nothing
803        /// mounts them. Empty while it has not tried, or Docker failed.
804        volumes: Vec<String>,
805    },
806    /// A reason from a newer daemon than this client knows, so a reason added later does not keep the client from
807    /// reading the status.
808    #[serde(other)]
809    Unknown,
810}
811
812/// What a [`Waiting`] reason waits on, which clients draw it by.
813#[derive(Debug, Clone, Copy, PartialEq, Eq)]
814pub enum WaitKind {
815    /// Something the owner can act on: a pause, runners that failed to take a job, or labels no pool serves.
816    Act,
817    /// Something that clears by itself shortly: a runner starting, a reseed, an image pull, an older job's start.
818    Soon,
819    /// Room: a slot, memory or CPU that a running job holds.
820    Full,
821    /// A reason this client does not know (`Waiting::Unknown`), drawn neutral.
822    Other,
823}
824
825impl Waiting {
826    /// Whether runnerd cannot start the job with the config it runs with: its repository is paused, or no pool serves
827    /// its labels. Not that nothing ever will: a resume or a config change can, and another machine's runner may take
828    /// it. A backoff ends by itself, and a reason this client does not know says nothing (ADR 0132).
829    pub fn cannot_start(&self) -> bool {
830        matches!(self, Waiting::Paused | Waiting::NoPool { .. })
831    }
832
833    /// What it waits on.
834    pub fn kind(&self) -> WaitKind {
835        match self {
836            Waiting::Paused | Waiting::Backoff { .. } | Waiting::NoPool { .. } => WaitKind::Act,
837            Waiting::Torn { volumes } if !volumes.is_empty() => WaitKind::Act,
838            Waiting::Unknown => WaitKind::Other,
839            Waiting::Torn { .. }
840            | Waiting::RunnerStarting
841            | Waiting::Reseeding { .. }
842            | Waiting::PullingImage { .. }
843            | Waiting::BehindStart { .. } => WaitKind::Soon,
844            Waiting::NoSlot { .. }
845            | Waiting::Memory { .. }
846            | Waiting::HostMemory { .. }
847            | Waiting::Cpu { .. }
848            | Waiting::BehindCpu { .. }
849            | Waiting::BehindMemory { .. } => WaitKind::Full,
850        }
851    }
852
853    /// Why `job` waits, in the owner's words with the numbers that hold it, as every client says it: such as
854    /// `All 2 node slots are busy` or `Needs 9G of memory; 2.5G of the budget is free`. `status` is the one `job`
855    /// is queued in, which names the slot of a starting runner and the older job one waits behind.
856    pub fn reason(&self, job: &QueuedJob, status: &Status, now: u64) -> String {
857        let size = |bytes: u64| crate::config::ByteSize(bytes).to_string();
858        let older_name = |id: u64| {
859            status
860                .queued
861                .iter()
862                .find(|candidate| candidate.id == id)
863                .map_or_else(|| "an older job".to_owned(), |older| older.name.clone())
864        };
865        match self {
866            Waiting::Paused => format!("{} is paused", job.repository),
867            Waiting::Unknown => "Waits for a reason this client does not know yet".to_owned(),
868            Waiting::NoPool { labels } if labels.is_empty() => {
869                "It asks for no labels, so no profile serves it".to_owned()
870            }
871            Waiting::NoPool { labels } => format!(
872                "No profile serves the {} {}",
873                if labels.len() == 1 { "label" } else { "labels" },
874                labels.join(", ")
875            ),
876            Waiting::Backoff { until, cause } => format!(
877                "{}; retrying in {}",
878                match cause {
879                    HoldCause::NoJob => "Runners failed to take a job",
880                    HoldCause::IdleExits => "Runners ended without a job after waiting for one",
881                    HoldCause::Register => "Runners cannot register with GitHub",
882                    HoldCause::Start => "Runners cannot start",
883                },
884                duration(until.saturating_sub(now))
885            ),
886            Waiting::RunnerStarting => match starting_for(job, status) {
887                Some(slot) => format!("A runner is starting for it in {slot}"),
888                None => "A runner is starting for it".to_owned(),
889            },
890            Waiting::Reseeding { profile } => format!("{profile} is rebuilding its snapshot"),
891            Waiting::Torn { volumes } if volumes.is_empty() => {
892                "Emptying the caches a host crash may have torn".to_owned()
893            }
894            Waiting::Torn { volumes } => format!(
895                "A host crash may have torn caches a container holds; once nothing mounts them: docker volume rm {}",
896                volumes.join(" ")
897            ),
898            Waiting::PullingImage { profile } => format!("Pulling {profile}'s image"),
899            Waiting::NoSlot { profile } => {
900                let slots = status
901                    .profiles
902                    .iter()
903                    .find(|candidate| &candidate.name == profile)
904                    .map_or(0, |profile| profile.slots.len());
905                if slots == 1 {
906                    format!("The {profile} slot is busy")
907                } else {
908                    format!("All {slots} {profile} slots are busy")
909                }
910            }
911            Waiting::Memory { .. } if status.budget.held => {
912                "The slice is past 90% of its MemoryMax, so nothing starts".to_owned()
913            }
914            Waiting::Memory { needs, free } => format!(
915                "Needs {} of memory; {} of the budget is free",
916                size(*needs),
917                size(*free)
918            ),
919            Waiting::HostMemory { needs, free } => host_reason(
920                *needs,
921                *free,
922                status.budget.host.as_ref(),
923                status.budget.memory_in_use,
924            ),
925            Waiting::Cpu { needs, free } => {
926                format!(
927                    "Needs {}; {} are free",
928                    cpus(*needs),
929                    admission::free_cores(*free)
930                )
931            }
932            Waiting::BehindCpu { job: older } => {
933                format!("Behind {}, which waits for CPU", older_name(*older))
934            }
935            Waiting::BehindMemory { job: older } => {
936                format!("Behind {}, which waits for memory", older_name(*older))
937            }
938            Waiting::BehindStart { job: older } => {
939                format!(
940                    "Next after {}, whose runner starts first",
941                    older_name(*older)
942                )
943            }
944        }
945    }
946}
947
948/// `GET /v1/dashboard`: the dashboard's `[web]` settings, as the daemon read them when it started.
949#[derive(Debug, Clone, Default, PartialEq, Eq, Serialize, Deserialize)]
950pub struct Dashboard {
951    /// `[web] open`: the one of `[web] origins` that `runnerctl web` opens the dashboard on; the loopback address
952    /// when none.
953    #[serde(default, skip_serializing_if = "Option::is_none")]
954    pub open: Option<String>,
955}
956
957/// `POST /v1/dashboard/pairings`: which of `[web] origins` to make a link for.
958#[derive(Debug, Clone, Default, PartialEq, Eq, Serialize, Deserialize)]
959pub struct PairingQuery {
960    /// One of `[web] origins`; each of them when none.
961    #[serde(default, skip_serializing_if = "Option::is_none")]
962    pub origin: Option<String>,
963}
964
965/// `POST /v1/dashboard/pairings`: fresh one-time links that sign a device in to the dashboard.
966#[derive(Debug, Clone, Default, PartialEq, Eq, Serialize, Deserialize)]
967pub struct Pairings {
968    /// One per origin asked for.
969    pub links: Vec<Pairing>,
970}
971
972/// A link that signs one device in to the dashboard on `origin`, and nowhere else, within `lifetime`.
973#[derive(Clone, PartialEq, Eq, Serialize, Deserialize)]
974pub struct Pairing {
975    /// One of `[web] origins`.
976    pub origin: String,
977    /// The link with its one-time code: a credential, which `Debug` leaves out.
978    pub url: String,
979    /// Seconds it works for from when it was made.
980    pub lifetime: u64,
981}
982
983impl std::fmt::Debug for Pairing {
984    fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
985        formatter
986            .debug_struct("Pairing")
987            .field("origin", &self.origin)
988            .field("lifetime", &self.lifetime)
989            .finish_non_exhaustive()
990    }
991}
992
993/// `POST /v1/shutdown`.
994#[derive(Debug, Clone, Default, PartialEq, Eq, Serialize, Deserialize)]
995pub struct ShutdownRequest {
996    /// Kill running jobs and delete their runners instead of waiting for them.
997    #[serde(default)]
998    pub force: bool,
999}
1000
1001/// What `POST /v1/shutdown` answers: how the daemon stops now, which a stop asked for earlier may have made more
1002/// urgent than the request.
1003#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
1004pub struct ShutdownAnswer {
1005    /// How it stops.
1006    pub stopping: Stopping,
1007}
1008
1009/// How a stopping daemon treats running jobs (ADR 0091).
1010#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
1011#[serde(rename_all = "kebab-case")]
1012pub enum Stopping {
1013    /// `runnerd stop`: it waits for running jobs, then removes every runner.
1014    WaitsForJobs,
1015    /// A signal: it removes idle runners and leaves running jobs to the next start.
1016    LeavesJobs,
1017    /// `runnerd stop --force` or a second signal: it kills running jobs.
1018    KillsJobs,
1019    /// A kind of stop from a newer daemon than this client knows, so one added later does not keep the client from
1020    /// reading the status.
1021    #[serde(other)]
1022    Unknown,
1023}
1024
1025impl Stopping {
1026    /// What it does, as every client says it: `shutting down: waits for running jobs`.
1027    pub fn line(self) -> &'static str {
1028        match self {
1029            Stopping::WaitsForJobs => "shutting down: waits for running jobs",
1030            Stopping::LeavesJobs => "shutting down: leaves running jobs to the next start",
1031            Stopping::KillsJobs => "shutting down: kills running jobs",
1032            Stopping::Unknown => "shutting down",
1033        }
1034    }
1035}
1036
1037impl Status {
1038    /// What a stopping daemon does, as every client says it ([`Stopping::line`]); `shutting down` from a daemon
1039    /// that does not say how; `None` while it runs.
1040    pub fn stopping_line(&self) -> Option<&'static str> {
1041        match (self.stopping, self.shutting_down) {
1042            (Some(stopping), _) => Some(stopping.line()),
1043            (None, true) => Some("shutting down"),
1044            (None, false) => None,
1045        }
1046    }
1047
1048    /// Why no runner starts while GitHub's rate limit is used up, as every client says it; `None` while it is not.
1049    pub fn github_line(&self) -> Option<&'static str> {
1050        self.github_limited_until
1051            .map(|_| "GitHub's rate limit is used up: no runner starts until it resets")
1052    }
1053}
1054
1055/// The body of every error response.
1056#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
1057pub struct ErrorResponse {
1058    /// What went wrong, for people to read.
1059    pub message: String,
1060}