Skip to main content

runnerd_core/api/
attention.rs

1//! What needs the owner's attention now, as one list every client draws (ADR 0132): queued jobs runnerd cannot start
2//! with its config, a job held long for the host's memory, the doctor's failures and warnings, warning insights,
3//! GitHub refusing or failing the listings, and the rate limit. [`attention_items`] decides and words every item from
4//! what the daemon already serves; the daemon's attention worker calls it and owns the [`Attention`] snapshot. Plain
5//! data and pure functions: no I/O, and nothing inferred from history or occupancy where no evidence says it holds now.
6
7use serde::{Deserialize, Deserializer, Serialize};
8
9use super::{
10    CheckStatus, Doctor, DoctorState, Insight, InsightKind, Insights, QueuedJob, Severity, Status,
11    Tone, View, Waiting,
12};
13
14/// The most items an [`Attention`] lists; the rest are counted in `omitted`.
15pub const ATTENTION_MAX: usize = 50;
16
17/// `GET /v1/attention`: what needs attention now, most urgent first.
18#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
19pub struct Attention {
20    /// The daemon process that keeps the snapshot: new at each start, never the same as an earlier one's. Revisions
21    /// count within one instance; a client compares them only when the instances are the same.
22    pub instance: String,
23    /// From 1 at the daemon's start, one up whenever the content changes, also when one item replaces another.
24    /// `attention.changed` on `/v1/events` carries the new one. Two snapshots of one `instance` and `revision` say the
25    /// same.
26    pub revision: u64,
27    /// By tone, then known evidence times before unknown, newest first, then `key`; at most [`ATTENTION_MAX`].
28    pub items: Vec<AttentionItem>,
29    /// Items beyond [`ATTENTION_MAX`], left out.
30    #[serde(default, skip_serializing_if = "is_zero")]
31    pub omitted: u64,
32}
33
34#[expect(
35    clippy::trivially_copy_pass_by_ref,
36    reason = "serde's skip_serializing_if passes a reference"
37)]
38fn is_zero(count: &u64) -> bool {
39    *count == 0
40}
41
42/// One thing that needs attention.
43#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
44pub struct AttentionItem {
45    /// The same while the condition holds, such as `cannot-start:paused:halyard/ledger`: what a client keys it by.
46    pub key: String,
47    /// What source it comes from.
48    pub kind: AttentionKind,
49    /// How urgent; a tone this client does not know reads as [`Tone::Neutral`].
50    #[serde(deserialize_with = "tone_or_neutral")]
51    pub tone: Tone,
52    /// What holds, in one line.
53    pub title: String,
54    /// What it is about.
55    pub subject: Subject,
56    /// When the evidence for it began or was taken, in seconds since the Unix epoch; absent when unknown.
57    #[serde(default, skip_serializing_if = "Option::is_none")]
58    pub since: Option<u64>,
59    /// What clears it; absent when there is nothing to name, or the fix is one this client cannot read.
60    #[serde(
61        default,
62        skip_serializing_if = "Option::is_none",
63        deserialize_with = "lenient"
64    )]
65    pub fix: Option<Fix>,
66}
67
68/// Where an attention item comes from.
69#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
70#[serde(rename_all = "kebab-case")]
71pub enum AttentionKind {
72    /// Queued jobs runnerd cannot start with the config it runs with: their repository is paused, or no pool serves
73    /// their labels ([`Waiting::cannot_start`]).
74    CannotStart,
75    /// A job has waited 30 minutes or more for the host's memory, and one waits for it now.
76    HostMemory,
77    /// A doctor check failed or warns, in a run for the config the daemon runs with.
78    Doctor,
79    /// The doctor's checks are not current: never run yet, failed, or stale.
80    DoctorUnavailable,
81    /// A warning insight.
82    Insight,
83    /// A repository's queued jobs cannot be listed.
84    GithubListing,
85    /// GitHub refuses runnerd's token, or there is none: no repository is listed.
86    GithubAuth,
87    /// GitHub's rate limit is used up: no runner starts until it resets.
88    RateLimited,
89    /// A kind from a newer daemon than this client knows: draw it by its title.
90    #[serde(other)]
91    Unknown,
92}
93
94/// What an attention item is about, for a client to link.
95#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
96#[serde(rename_all = "kebab-case", tag = "kind")]
97pub enum Subject {
98    /// A job.
99    Job {
100        /// GitHub's job ID, the history's too.
101        id: u64,
102        /// `owner/name`.
103        repository: String,
104        /// The job's name.
105        name: String,
106    },
107    /// A repository.
108    Repository {
109        /// `owner/name`.
110        name: String,
111    },
112    /// One of the doctor's checks.
113    Check {
114        /// The check's name, such as `slice`.
115        name: String,
116    },
117    /// An insight.
118    Insight {
119        /// Its key in `GET /v1/insights`.
120        key: String,
121    },
122    /// The host as a whole.
123    Host,
124    /// GitHub, for every repository.
125    Github,
126    /// A subject from a newer daemon than this client knows.
127    #[serde(other)]
128    Unknown,
129}
130
131/// What clears an attention item. A client reads a fix of a kind or view it does not know as none
132/// ([`AttentionItem::fix`]).
133#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
134#[serde(rename_all = "kebab-case", tag = "kind")]
135pub enum Fix {
136    /// A command to run, such as `runnerctl resume halyard/ledger`.
137    Command {
138        /// The command line.
139        command: String,
140    },
141    /// A key of the config to change, such as `repositories.labels`.
142    Config {
143        /// The key, dotted.
144        key: String,
145    },
146    /// A view where the subject is shown with what to do.
147    Open {
148        /// The view.
149        view: View,
150    },
151    /// What to do, in words, such as a doctor check's fix.
152    Text {
153        /// What to do.
154        text: String,
155    },
156}
157
158impl Fix {
159    /// What to do, as every client says it: the command, `Change repositories.labels in the config`, `Open Settings`,
160    /// or the text.
161    pub fn line(&self) -> String {
162        match self {
163            Fix::Command { command } => command.clone(),
164            Fix::Config { key } => format!("Change {key} in the config"),
165            Fix::Open { view } => format!("Open {}", view.title()),
166            Fix::Text { text } => text.clone(),
167        }
168    }
169}
170
171impl Attention {
172    /// The snapshot of `items`, as [`attention_items`] orders them, at `revision` of `instance`: the first
173    /// [`ATTENTION_MAX`], and how many more.
174    pub fn new(instance: String, revision: u64, mut items: Vec<AttentionItem>) -> Attention {
175        let omitted = items.len().saturating_sub(ATTENTION_MAX);
176        items.truncate(ATTENTION_MAX);
177        Attention {
178            instance,
179            revision,
180            items,
181            omitted: omitted as u64,
182        }
183    }
184
185    /// Whether `other` says the same, whatever its instance and revision.
186    pub fn same_content(&self, other: &Attention) -> bool {
187        self.items == other.items && self.omitted == other.omitted
188    }
189
190    /// What a client that shows the first `shown` items says of the rest: `and 3 more`; `None` when it shows all.
191    pub fn more(&self, shown: usize) -> Option<String> {
192        let rest = self.items.len().saturating_sub(shown) as u64 + self.omitted;
193        (rest > 0).then(|| format!("and {rest} more"))
194    }
195}
196
197/// Everything that needs attention now, most urgent first: from `status`, the doctor's last run as `doctor` says it,
198/// and `insights` when they could be read, at `now` (seconds since the Unix epoch). Each item stands on evidence that
199/// holds now; a source without it is left out, and doctor checks that are not current are one item saying so.
200pub fn attention_items(
201    status: &Status,
202    doctor: &Doctor,
203    insights: Option<&Insights>,
204    now: u64,
205) -> Vec<AttentionItem> {
206    let mut items = cannot_start(status);
207    items.extend(host_memory(status, insights));
208    items.extend(doctor_items(doctor, now));
209    items.extend(
210        insights
211            .into_iter()
212            .flat_map(|insights| &insights.insights)
213            .filter_map(insight_item),
214    );
215    items.extend(github(status, now));
216    items.sort_by(|a, b| {
217        rank(a.tone)
218            .cmp(&rank(b.tone))
219            .then_with(|| b.since.is_some().cmp(&a.since.is_some()))
220            .then_with(|| b.since.cmp(&a.since))
221            .then_with(|| a.key.cmp(&b.key))
222    });
223    items
224}
225
226/// The most urgent tone first.
227fn rank(tone: Tone) -> u8 {
228    match tone {
229        Tone::Bad => 0,
230        Tone::Warn => 1,
231        Tone::Neutral => 2,
232        Tone::Quiet => 3,
233        Tone::Good => 4,
234    }
235}
236
237/// Queued jobs runnerd cannot start with its config, one item per repository and reason.
238fn cannot_start(status: &Status) -> Vec<AttentionItem> {
239    let mut groups: Vec<(String, Vec<&QueuedJob>)> = Vec::new();
240    for job in &status.queued {
241        let Some(waiting) = job
242            .waiting
243            .as_ref()
244            .filter(|waiting| waiting.cannot_start())
245        else {
246            continue;
247        };
248        let key = match waiting {
249            Waiting::NoPool { labels } => format!(
250                "cannot-start:no-pool:{}:{}",
251                job.repository,
252                labels.join(",")
253            ),
254            _ => format!("cannot-start:paused:{}", job.repository),
255        };
256        match groups.iter_mut().find(|(group, _)| *group == key) {
257            Some((_, jobs)) => jobs.push(job),
258            None => groups.push((key, vec![job])),
259        }
260    }
261    groups
262        .into_iter()
263        .filter_map(|(key, jobs)| {
264            let first = *jobs.first()?;
265            let repository = &first.repository;
266            let (what, subject) = match jobs.as_slice() {
267                [job] => (
268                    format!("{} of {repository}", job.name),
269                    Subject::Job {
270                        id: job.id,
271                        repository: repository.clone(),
272                        name: job.name.clone(),
273                    },
274                ),
275                _ => (
276                    format!("{} queued jobs of {repository}", jobs.len()),
277                    Subject::Repository {
278                        name: repository.clone(),
279                    },
280                ),
281            };
282            let (why, fix) = match first.waiting.as_ref()? {
283                Waiting::NoPool { labels } if labels.is_empty() => (
284                    "it asks for no labels, so no profile serves it".to_owned(),
285                    Fix::Config {
286                        key: "repositories.labels".to_owned(),
287                    },
288                ),
289                Waiting::NoPool { labels } => (
290                    format!(
291                        "no profile of {repository} serves the {} {}",
292                        if labels.len() == 1 { "label" } else { "labels" },
293                        labels.join(", ")
294                    ),
295                    Fix::Config {
296                        key: "repositories.labels".to_owned(),
297                    },
298                ),
299                _ => (
300                    format!("{repository} is paused"),
301                    Fix::Command {
302                        command: format!("runnerctl resume {repository}"),
303                    },
304                ),
305            };
306            Some(AttentionItem {
307                key,
308                kind: AttentionKind::CannotStart,
309                tone: Tone::Warn,
310                title: format!("runnerd cannot start {what} with the current configuration: {why}"),
311                subject,
312                since: jobs
313                    .iter()
314                    .map(|job| job.queued_at)
315                    .filter(|at| *at > 0)
316                    .min(),
317                fix: Some(fix),
318            })
319        })
320        .collect()
321}
322
323/// The host-held insight, while a queued job waits for the host's memory in `status` too: one item, which the
324/// insights leave out. It is about the host, which programs outside CI hold, not about one job: the insight names
325/// none, and the jobs waiting may have changed since it was read. Its evidence time is unknown for the same reason;
326/// the daemon reads the insights again when the jobs waiting for the host change.
327fn host_memory(status: &Status, insights: Option<&Insights>) -> Option<AttentionItem> {
328    let held = insights?
329        .insights
330        .iter()
331        .find(|insight| matches!(insight.kind, InsightKind::HostHeld { .. }))?;
332    status
333        .queued
334        .iter()
335        .any(|job| matches!(job.waiting, Some(Waiting::HostMemory { .. })))
336        .then(|| AttentionItem {
337            key: "host-memory".to_owned(),
338            kind: AttentionKind::HostMemory,
339            tone: Tone::Warn,
340            title: held.sentence.clone(),
341            subject: Subject::Host,
342            since: None,
343            fix: Some(Fix::Open {
344                view: View::Overview,
345            }),
346        })
347}
348
349/// The doctor's failures and warnings from a current run, or one item saying its checks are not current.
350fn doctor_items(doctor: &Doctor, now: u64) -> Vec<AttentionItem> {
351    let at = match &doctor.state {
352        // `runnerd doctor`'s own run, or one a daemon ran for the request: current.
353        None => None,
354        Some(DoctorState::Ready { at }) => Some(*at),
355        Some(
356            state @ (DoctorState::NotRun
357            | DoctorState::Stale { .. }
358            | DoctorState::Failed { .. }
359            | DoctorState::Unknown),
360        ) => {
361            let (tone, since) = match state {
362                DoctorState::Failed { at, .. } => (Tone::Warn, Some(*at)),
363                DoctorState::Stale { at } => (Tone::Quiet, Some(*at)),
364                _ => (Tone::Quiet, None),
365            };
366            return state
367                .line(now)
368                .map(|title| AttentionItem {
369                    key: "doctor-unavailable".to_owned(),
370                    kind: AttentionKind::DoctorUnavailable,
371                    tone,
372                    title,
373                    subject: Subject::Host,
374                    since,
375                    fix: Some(Fix::Open {
376                        view: View::Settings,
377                    }),
378                })
379                .into_iter()
380                .collect();
381        }
382    };
383    let shown: Vec<_> = doctor
384        .checks
385        .iter()
386        .filter(|check| matches!(check.status, CheckStatus::Fail | CheckStatus::Warn))
387        // Advisory whatever it says: only GitHub's answer to a real action is evidence (ADR 0134).
388        .filter(|check| check.name != super::JOB_ACTIONS_CHECK)
389        .collect();
390    let mut keys: Vec<String> = Vec::new();
391    shown
392        .iter()
393        .map(|check| {
394            // Checks of one name, such as each repository's visibility, are told apart by what they found, so a key
395            // stays with its check when another of the name comes or goes.
396            let repeated = shown
397                .iter()
398                .filter(|other| other.name == check.name)
399                .count()
400                > 1;
401            let mut key = if repeated {
402                format!("doctor:{}:{}", check.name, check.detail)
403            } else {
404                format!("doctor:{}", check.name)
405            };
406            let twins = keys.iter().filter(|seen| **seen == key).count();
407            keys.push(key.clone());
408            if twins > 0 {
409                key = format!("{key}#{}", twins + 1);
410            }
411            AttentionItem {
412                key,
413                kind: AttentionKind::Doctor,
414                tone: check.status.tone(),
415                title: format!("{}: {}", check.name, check.detail),
416                subject: Subject::Check {
417                    name: check.name.clone(),
418                },
419                since: at,
420                fix: Some(match &check.fix {
421                    Some(text) => Fix::Text { text: text.clone() },
422                    None => Fix::Open {
423                        view: View::Settings,
424                    },
425                }),
426            }
427        })
428        .collect()
429}
430
431/// A warning insight that says what holds now: not the host-held one, which [`host_memory`] stands for, nor a past
432/// day's low memory.
433fn insight_item(insight: &Insight) -> Option<AttentionItem> {
434    let current = !matches!(
435        insight.kind,
436        InsightKind::HostHeld { .. } | InsightKind::HostLow { .. } | InsightKind::Unknown
437    );
438    (insight.severity == Severity::Warning && current).then(|| AttentionItem {
439        key: format!("insight:{}", insight.key),
440        kind: AttentionKind::Insight,
441        tone: Tone::Warn,
442        title: insight.sentence.clone(),
443        subject: Subject::Insight {
444            key: insight.key.clone(),
445        },
446        since: insight.runs.first().map(|run| run.at),
447        fix: Some(Fix::Open {
448            view: View::Insights,
449        }),
450    })
451}
452
453/// The listings GitHub fails or refuses, and its rate limit.
454fn github(status: &Status, now: u64) -> Vec<AttentionItem> {
455    let failing: Vec<_> = status
456        .repositories
457        .iter()
458        .filter_map(|repository| Some((repository.name.as_str(), repository.listing.as_ref()?)))
459        .collect();
460    let token = failing
461        .iter()
462        .filter(|(_, failure)| failure.problem.is_token())
463        .min_by_key(|(_, failure)| failure.since);
464    let mut items: Vec<AttentionItem> = token
465        .map(|(_, failure)| AttentionItem {
466            key: "github-auth".to_owned(),
467            kind: AttentionKind::GithubAuth,
468            tone: Tone::Bad,
469            title: match failure.problem {
470                super::ListingProblem::NoToken => {
471                    "runnerd has no GitHub token, so it lists no repository's queued jobs"
472                        .to_owned()
473                }
474                _ => {
475                    "GitHub refuses runnerd's token (401), so it lists no repository's queued jobs"
476                        .to_owned()
477                }
478            },
479            subject: Subject::Github,
480            since: Some(failure.since),
481            fix: Some(Fix::Open {
482                view: View::Settings,
483            }),
484        })
485        .into_iter()
486        .collect();
487    items.extend(
488        failing
489            .iter()
490            .filter(|(_, failure)| !failure.problem.is_token())
491            .map(|(name, failure)| AttentionItem {
492                key: format!("github-listing:{name}"),
493                kind: AttentionKind::GithubListing,
494                tone: Tone::Warn,
495                title: failure.line(name),
496                subject: Subject::Repository {
497                    name: (*name).to_owned(),
498                },
499                since: Some(failure.since),
500                fix: matches!(failure.problem, super::ListingProblem::Refused { .. }).then_some(
501                    Fix::Open {
502                        view: View::Settings,
503                    },
504                ),
505            }),
506    );
507    // Gone once its reset time passed, whether or not the engine published since.
508    let limited = status.github_limited_until.is_some_and(|until| until > now);
509    items.extend(
510        status
511            .github_line()
512            .filter(|_| limited)
513            .map(|line| AttentionItem {
514                key: "rate-limited".to_owned(),
515                kind: AttentionKind::RateLimited,
516                tone: Tone::Warn,
517                title: line.to_owned(),
518                subject: Subject::Github,
519                since: None,
520                fix: None,
521            }),
522    );
523    items
524}
525
526/// A value of a type this client may not know all of, such as a [`Fix`] naming a view it lacks: `None` then, so the
527/// rest of the document still reads.
528fn lenient<'de, D: Deserializer<'de>, T: Deserialize<'de>>(
529    deserializer: D,
530) -> Result<Option<T>, D::Error> {
531    #[derive(Deserialize)]
532    #[serde(untagged)]
533    enum Read<T> {
534        Known(T),
535        Other(serde::de::IgnoredAny),
536    }
537    Ok(match Option::<Read<T>>::deserialize(deserializer)? {
538        Some(Read::Known(value)) => Some(value),
539        Some(Read::Other(_)) | None => None,
540    })
541}
542
543fn tone_or_neutral<'de, D: Deserializer<'de>>(deserializer: D) -> Result<Tone, D::Error> {
544    Ok(lenient(deserializer)?.unwrap_or(Tone::Neutral))
545}
546
547// The wire format is JSON, which only the host and client features read.
548#[cfg(all(test, any(feature = "host", feature = "client")))]
549mod tests;