Skip to main content

runnerd_core/api/
attention.rs

1//! What needs the owner's attention now, as one list every client draws (ADR 0132): queued jobs runnerd cannot start
2//! with its config, a job held long for the host's memory, the doctor's failures and warnings, warning insights,
3//! GitHub refusing or failing the listings, and the rate limit. [`attention_items`] decides and words every item from
4//! what the daemon already serves; the daemon's attention worker calls it and owns the [`Attention`] snapshot. Plain
5//! data and pure functions: no I/O, and nothing inferred from history or occupancy where no evidence says it holds now.
6
7use serde::{Deserialize, Deserializer, Serialize};
8
9use super::{
10    CheckStatus, Doctor, DoctorState, Insight, InsightKind, Insights, QueuedJob, Severity, Status,
11    Tone, View, Waiting,
12};
13
14/// The most items an [`Attention`] lists; the rest are counted in `omitted`.
15pub const ATTENTION_MAX: usize = 50;
16
17/// `GET /v1/attention`: what needs attention now, most urgent first.
18#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
19pub struct Attention {
20    /// The daemon process that keeps the snapshot: new at each start, never the same as an earlier one's. Revisions
21    /// count within one instance; a client compares them only when the instances are the same.
22    pub instance: String,
23    /// From 1 at the daemon's start, one up whenever the content changes, also when one item replaces another.
24    /// `attention.changed` on `/v1/events` carries the new one. Two snapshots of one `instance` and `revision` say the
25    /// same.
26    pub revision: u64,
27    /// By tone, then known evidence times before unknown, newest first, then `key`; at most [`ATTENTION_MAX`].
28    pub items: Vec<AttentionItem>,
29    /// Items beyond [`ATTENTION_MAX`], left out.
30    #[serde(default, skip_serializing_if = "is_zero")]
31    pub omitted: u64,
32}
33
34#[expect(
35    clippy::trivially_copy_pass_by_ref,
36    reason = "serde's skip_serializing_if passes a reference"
37)]
38fn is_zero(count: &u64) -> bool {
39    *count == 0
40}
41
42/// One thing that needs attention.
43#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
44pub struct AttentionItem {
45    /// The same while the condition holds, such as `cannot-start:paused:halyard/ledger`: what a client keys it by.
46    pub key: String,
47    /// What source it comes from.
48    pub kind: AttentionKind,
49    /// How urgent; a tone this client does not know reads as [`Tone::Neutral`].
50    #[serde(deserialize_with = "tone_or_neutral")]
51    pub tone: Tone,
52    /// What holds, in one line.
53    pub title: String,
54    /// What it is about.
55    pub subject: Subject,
56    /// When the evidence for it began or was taken, in seconds since the Unix epoch; absent when unknown.
57    #[serde(default, skip_serializing_if = "Option::is_none")]
58    pub since: Option<u64>,
59    /// What clears it; absent when there is nothing to name, or the fix is one this client cannot read.
60    #[serde(
61        default,
62        skip_serializing_if = "Option::is_none",
63        deserialize_with = "lenient"
64    )]
65    pub fix: Option<Fix>,
66}
67
68/// Where an attention item comes from.
69#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
70#[serde(rename_all = "kebab-case")]
71pub enum AttentionKind {
72    /// Queued jobs runnerd cannot start with the config it runs with: their repository is paused, or no pool serves
73    /// their labels ([`Waiting::cannot_start`]).
74    CannotStart,
75    /// A job has waited 30 minutes or more for the host's memory, and one waits for it now.
76    HostMemory,
77    /// A doctor check failed or warns, in a run for the config the daemon runs with.
78    Doctor,
79    /// The doctor's checks are not current: never run yet, failed, or stale.
80    DoctorUnavailable,
81    /// A warning insight.
82    Insight,
83    /// A repository's queued jobs cannot be listed.
84    GithubListing,
85    /// GitHub refuses runnerd's token, or there is none: no repository is listed.
86    GithubAuth,
87    /// GitHub's rate limit is used up: no runner starts until it resets.
88    RateLimited,
89    /// A kind from a newer daemon than this client knows: draw it by its title.
90    #[serde(other)]
91    Unknown,
92}
93
94/// What an attention item is about, for a client to link.
95#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
96#[serde(rename_all = "kebab-case", tag = "kind")]
97pub enum Subject {
98    /// A job.
99    Job {
100        /// GitHub's job ID, the history's too.
101        id: u64,
102        /// `owner/name`.
103        repository: String,
104        /// The job's name.
105        name: String,
106    },
107    /// A repository.
108    Repository {
109        /// `owner/name`.
110        name: String,
111    },
112    /// One of the doctor's checks.
113    Check {
114        /// The check's name, such as `slice`.
115        name: String,
116    },
117    /// An insight.
118    Insight {
119        /// Its key in `GET /v1/insights`.
120        key: String,
121    },
122    /// The host as a whole.
123    Host,
124    /// GitHub, for every repository.
125    Github,
126    /// A subject from a newer daemon than this client knows.
127    #[serde(other)]
128    Unknown,
129}
130
131/// What clears an attention item. A client reads a fix of a kind or view it does not know as none
132/// ([`AttentionItem::fix`]).
133#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
134#[serde(rename_all = "kebab-case", tag = "kind")]
135pub enum Fix {
136    /// A command to run, such as `runnerctl resume halyard/ledger`.
137    Command {
138        /// The command line.
139        command: String,
140    },
141    /// A key of the config to change, such as `repositories.labels`.
142    Config {
143        /// The key, dotted.
144        key: String,
145    },
146    /// A view where the subject is shown with what to do.
147    Open {
148        /// The view.
149        view: View,
150    },
151    /// What to do, in words, such as a doctor check's fix.
152    Text {
153        /// What to do.
154        text: String,
155    },
156}
157
158impl Fix {
159    /// What to do, as every client says it: the command, `Change repositories.labels in the config`, `Open Settings`,
160    /// or the text.
161    pub fn line(&self) -> String {
162        match self {
163            Fix::Command { command } => command.clone(),
164            Fix::Config { key } => format!("Change {key} in the config"),
165            Fix::Open { view } => format!("Open {}", view.title()),
166            Fix::Text { text } => text.clone(),
167        }
168    }
169}
170
171impl Attention {
172    /// The snapshot of `items`, as [`attention_items`] orders them, at `revision` of `instance`: the first
173    /// [`ATTENTION_MAX`], and how many more.
174    pub fn new(instance: String, revision: u64, mut items: Vec<AttentionItem>) -> Attention {
175        let omitted = items.len().saturating_sub(ATTENTION_MAX);
176        items.truncate(ATTENTION_MAX);
177        Attention {
178            instance,
179            revision,
180            items,
181            omitted: omitted as u64,
182        }
183    }
184
185    /// Whether `other` says the same, whatever its instance and revision.
186    pub fn same_content(&self, other: &Attention) -> bool {
187        self.items == other.items && self.omitted == other.omitted
188    }
189
190    /// How many items need attention, those left out included: what a client shows as the list's count.
191    pub fn count(&self) -> u64 {
192        self.items.len() as u64 + self.omitted
193    }
194
195    /// What a client that shows the first `shown` items says of the rest: `and 3 more`; `None` when it shows all.
196    pub fn more(&self, shown: usize) -> Option<String> {
197        let rest = self.items.len().saturating_sub(shown) as u64 + self.omitted;
198        (rest > 0).then(|| format!("and {rest} more"))
199    }
200}
201
202/// Everything that needs attention now, most urgent first: from `status`, the doctor's last run as `doctor` says it,
203/// and `insights` when they could be read, at `now` (seconds since the Unix epoch). Each item stands on evidence that
204/// holds now; a source without it is left out, and doctor checks that are not current are one item saying so.
205pub fn attention_items(
206    status: &Status,
207    doctor: &Doctor,
208    insights: Option<&Insights>,
209    now: u64,
210) -> Vec<AttentionItem> {
211    let mut items = cannot_start(status);
212    items.extend(host_memory(status, insights));
213    items.extend(doctor_items(doctor, now));
214    items.extend(
215        insights
216            .into_iter()
217            .flat_map(|insights| &insights.insights)
218            .filter_map(insight_item),
219    );
220    items.extend(github(status, now));
221    items.sort_by(|a, b| {
222        rank(a.tone)
223            .cmp(&rank(b.tone))
224            .then_with(|| b.since.is_some().cmp(&a.since.is_some()))
225            .then_with(|| b.since.cmp(&a.since))
226            .then_with(|| a.key.cmp(&b.key))
227    });
228    items
229}
230
231/// The most urgent tone first.
232fn rank(tone: Tone) -> u8 {
233    match tone {
234        Tone::Bad => 0,
235        Tone::Warn => 1,
236        Tone::Neutral => 2,
237        Tone::Quiet => 3,
238        Tone::Good => 4,
239    }
240}
241
242/// Queued jobs runnerd cannot start with its config, one item per repository and reason.
243fn cannot_start(status: &Status) -> Vec<AttentionItem> {
244    let mut groups: Vec<(String, Vec<&QueuedJob>)> = Vec::new();
245    for job in &status.queued {
246        let Some(waiting) = job
247            .waiting
248            .as_ref()
249            .filter(|waiting| waiting.cannot_start())
250        else {
251            continue;
252        };
253        let key = match waiting {
254            Waiting::NoPool { labels } => format!(
255                "cannot-start:no-pool:{}:{}",
256                job.repository,
257                labels.join(",")
258            ),
259            _ => format!("cannot-start:paused:{}", job.repository),
260        };
261        match groups.iter_mut().find(|(group, _)| *group == key) {
262            Some((_, jobs)) => jobs.push(job),
263            None => groups.push((key, vec![job])),
264        }
265    }
266    groups
267        .into_iter()
268        .filter_map(|(key, jobs)| {
269            let first = *jobs.first()?;
270            let repository = &first.repository;
271            let (what, subject) = match jobs.as_slice() {
272                [job] => (
273                    format!("{} of {repository}", job.name),
274                    Subject::Job {
275                        id: job.id,
276                        repository: repository.clone(),
277                        name: job.name.clone(),
278                    },
279                ),
280                _ => (
281                    format!("{} queued jobs of {repository}", jobs.len()),
282                    Subject::Repository {
283                        name: repository.clone(),
284                    },
285                ),
286            };
287            let (why, fix) = match first.waiting.as_ref()? {
288                Waiting::NoPool { labels } if labels.is_empty() => (
289                    "it asks for no labels, so no profile serves it".to_owned(),
290                    Fix::Config {
291                        key: "repositories.labels".to_owned(),
292                    },
293                ),
294                Waiting::NoPool { labels } => (
295                    format!(
296                        "no profile of {repository} serves the {} {}",
297                        if labels.len() == 1 { "label" } else { "labels" },
298                        labels.join(", ")
299                    ),
300                    Fix::Config {
301                        key: "repositories.labels".to_owned(),
302                    },
303                ),
304                _ => (
305                    format!("{repository} is paused"),
306                    Fix::Command {
307                        command: format!("runnerctl resume {repository}"),
308                    },
309                ),
310            };
311            Some(AttentionItem {
312                key,
313                kind: AttentionKind::CannotStart,
314                tone: Tone::Warn,
315                title: format!("runnerd cannot start {what} with the current configuration: {why}"),
316                subject,
317                since: jobs
318                    .iter()
319                    .map(|job| job.queued_at)
320                    .filter(|at| *at > 0)
321                    .min(),
322                fix: Some(fix),
323            })
324        })
325        .collect()
326}
327
328/// The host-held insight, while a queued job waits for the host's memory in `status` too: one item, which the
329/// insights leave out. It is about the host, which programs outside CI hold, not about one job: the insight names
330/// none, and the jobs waiting may have changed since it was read. Its evidence time is unknown for the same reason;
331/// the daemon reads the insights again when the jobs waiting for the host change.
332fn host_memory(status: &Status, insights: Option<&Insights>) -> Option<AttentionItem> {
333    let held = insights?
334        .insights
335        .iter()
336        .find(|insight| matches!(insight.kind, InsightKind::HostHeld { .. }))?;
337    status
338        .queued
339        .iter()
340        .any(|job| matches!(job.waiting, Some(Waiting::HostMemory { .. })))
341        .then(|| AttentionItem {
342            key: "host-memory".to_owned(),
343            kind: AttentionKind::HostMemory,
344            tone: Tone::Warn,
345            title: held.sentence.clone(),
346            subject: Subject::Host,
347            since: None,
348            fix: Some(Fix::Open {
349                view: View::Overview,
350            }),
351        })
352}
353
354/// The doctor's failures and warnings from a current run, or one item saying its checks are not current.
355fn doctor_items(doctor: &Doctor, now: u64) -> Vec<AttentionItem> {
356    let at = match &doctor.state {
357        // `runnerd doctor`'s own run, or one a daemon ran for the request: current.
358        None => None,
359        Some(DoctorState::Ready { at }) => Some(*at),
360        Some(
361            state @ (DoctorState::NotRun
362            | DoctorState::Stale { .. }
363            | DoctorState::Failed { .. }
364            | DoctorState::Unknown),
365        ) => {
366            let (tone, since) = match state {
367                DoctorState::Failed { at, .. } => (Tone::Warn, Some(*at)),
368                DoctorState::Stale { at } => (Tone::Quiet, Some(*at)),
369                _ => (Tone::Quiet, None),
370            };
371            return state
372                .line(now)
373                .map(|title| AttentionItem {
374                    key: "doctor-unavailable".to_owned(),
375                    kind: AttentionKind::DoctorUnavailable,
376                    tone,
377                    title,
378                    subject: Subject::Host,
379                    since,
380                    fix: Some(Fix::Open {
381                        view: View::Settings,
382                    }),
383                })
384                .into_iter()
385                .collect();
386        }
387    };
388    let shown: Vec<_> = doctor
389        .checks
390        .iter()
391        .filter(|check| matches!(check.status, CheckStatus::Fail | CheckStatus::Warn))
392        // Advisory whatever it says: only GitHub's answer to a real action is evidence (ADR 0134).
393        .filter(|check| check.name != super::JOB_ACTIONS_CHECK)
394        .collect();
395    let mut keys: Vec<String> = Vec::new();
396    shown
397        .iter()
398        .map(|check| {
399            // Checks of one name, such as each repository's visibility, are told apart by what they found, so a key
400            // stays with its check when another of the name comes or goes.
401            let repeated = shown
402                .iter()
403                .filter(|other| other.name == check.name)
404                .count()
405                > 1;
406            let mut key = if repeated {
407                format!("doctor:{}:{}", check.name, check.detail)
408            } else {
409                format!("doctor:{}", check.name)
410            };
411            let twins = keys.iter().filter(|seen| **seen == key).count();
412            keys.push(key.clone());
413            if twins > 0 {
414                key = format!("{key}#{}", twins + 1);
415            }
416            AttentionItem {
417                key,
418                kind: AttentionKind::Doctor,
419                tone: check.status.tone(),
420                title: format!("{}: {}", check.name, check.detail),
421                subject: Subject::Check {
422                    name: check.name.clone(),
423                },
424                since: at,
425                fix: Some(match &check.fix {
426                    Some(text) => Fix::Text { text: text.clone() },
427                    None => Fix::Open {
428                        view: View::Settings,
429                    },
430                }),
431            }
432        })
433        .collect()
434}
435
436/// A warning insight that says what holds now: not the host-held one, which [`host_memory`] stands for, nor a past
437/// day's low memory.
438fn insight_item(insight: &Insight) -> Option<AttentionItem> {
439    let current = !matches!(
440        insight.kind,
441        InsightKind::HostHeld { .. } | InsightKind::HostLow { .. } | InsightKind::Unknown
442    );
443    (insight.severity == Severity::Warning && current).then(|| AttentionItem {
444        key: format!("insight:{}", insight.key),
445        kind: AttentionKind::Insight,
446        tone: Tone::Warn,
447        title: insight.sentence.clone(),
448        subject: Subject::Insight {
449            key: insight.key.clone(),
450        },
451        since: insight.runs.first().map(|run| run.at),
452        fix: Some(Fix::Open {
453            view: View::Insights,
454        }),
455    })
456}
457
458/// The listings GitHub fails or refuses, and its rate limit.
459fn github(status: &Status, now: u64) -> Vec<AttentionItem> {
460    let failing: Vec<_> = status
461        .repositories
462        .iter()
463        .filter_map(|repository| Some((repository.name.as_str(), repository.listing.as_ref()?)))
464        .collect();
465    let token = failing
466        .iter()
467        .filter(|(_, failure)| failure.problem.is_token())
468        .min_by_key(|(_, failure)| failure.since);
469    let mut items: Vec<AttentionItem> = token
470        .map(|(_, failure)| AttentionItem {
471            key: "github-auth".to_owned(),
472            kind: AttentionKind::GithubAuth,
473            tone: Tone::Bad,
474            title: match failure.problem {
475                super::ListingProblem::NoToken => {
476                    "runnerd has no GitHub token, so it lists no repository's queued jobs"
477                        .to_owned()
478                }
479                _ => {
480                    "GitHub refuses runnerd's token (401), so it lists no repository's queued jobs"
481                        .to_owned()
482                }
483            },
484            subject: Subject::Github,
485            since: Some(failure.since),
486            fix: Some(Fix::Open {
487                view: View::Settings,
488            }),
489        })
490        .into_iter()
491        .collect();
492    items.extend(
493        failing
494            .iter()
495            .filter(|(_, failure)| !failure.problem.is_token())
496            .map(|(name, failure)| AttentionItem {
497                key: format!("github-listing:{name}"),
498                kind: AttentionKind::GithubListing,
499                tone: Tone::Warn,
500                title: failure.line(name),
501                subject: Subject::Repository {
502                    name: (*name).to_owned(),
503                },
504                since: Some(failure.since),
505                fix: matches!(failure.problem, super::ListingProblem::Refused { .. }).then_some(
506                    Fix::Open {
507                        view: View::Settings,
508                    },
509                ),
510            }),
511    );
512    // Gone once its reset time passed, whether or not the engine published since.
513    let limited = status.github_limited_until.is_some_and(|until| until > now);
514    items.extend(
515        status
516            .github_line()
517            .filter(|_| limited)
518            .map(|line| AttentionItem {
519                key: "rate-limited".to_owned(),
520                kind: AttentionKind::RateLimited,
521                tone: Tone::Warn,
522                title: line.to_owned(),
523                subject: Subject::Github,
524                since: None,
525                fix: None,
526            }),
527    );
528    items
529}
530
531/// A value of a type this client may not know all of, such as a [`Fix`] naming a view it lacks: `None` then, so the
532/// rest of the document still reads.
533fn lenient<'de, D: Deserializer<'de>, T: Deserialize<'de>>(
534    deserializer: D,
535) -> Result<Option<T>, D::Error> {
536    #[derive(Deserialize)]
537    #[serde(untagged)]
538    enum Read<T> {
539        Known(T),
540        Other(serde::de::IgnoredAny),
541    }
542    Ok(match Option::<Read<T>>::deserialize(deserializer)? {
543        Some(Read::Known(value)) => Some(value),
544        Some(Read::Other(_)) | None => None,
545    })
546}
547
548fn tone_or_neutral<'de, D: Deserializer<'de>>(deserializer: D) -> Result<Tone, D::Error> {
549    Ok(lenient(deserializer)?.unwrap_or(Tone::Neutral))
550}
551
552// The wire format is JSON, which only the host and client features read.
553#[cfg(all(test, any(feature = "host", feature = "client")))]
554mod tests;