Skip to main content

runnerd_core/api/
attention.rs

1//! What needs the owner's attention now, as one list every client draws (ADR 0132): queued jobs runnerd cannot start
2//! with its config, a job held long for the host's memory, the doctor's failures and warnings, warning insights,
3//! GitHub refusing or failing the listings, and the rate limit. [`attention_items`] decides and words every item from
4//! what the daemon already serves; the daemon's attention worker calls it and owns the [`Attention`] snapshot. Plain
5//! data and pure functions: no I/O, and nothing inferred from history or occupancy where no evidence says it holds now.
6
7use serde::{Deserialize, Deserializer, Serialize};
8
9use super::{
10    CheckStatus, Doctor, DoctorState, HostWait, Insight, InsightKind, Insights, QueuedJob,
11    Severity, Status, Tone, View, Waiting,
12};
13
14/// The most items an [`Attention`] lists; the rest are counted in `omitted`.
15pub const ATTENTION_MAX: usize = 50;
16
17/// `GET /v1/attention`: what needs attention now, most urgent first.
18#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
19pub struct Attention {
20    /// The daemon process that keeps the snapshot: new at each start, never the same as an earlier one's. Revisions
21    /// count within one instance; a client compares them only when the instances are the same.
22    pub instance: String,
23    /// From 1 at the daemon's start, one up whenever the content changes, also when one item replaces another.
24    /// `attention.changed` on `/v1/events` carries the new one. Two snapshots of one `instance` and `revision` say the
25    /// same.
26    pub revision: u64,
27    /// By tone, then known evidence times before unknown, newest first, then `key`; at most [`ATTENTION_MAX`].
28    pub items: Vec<AttentionItem>,
29    /// Items beyond [`ATTENTION_MAX`], left out.
30    #[serde(default, skip_serializing_if = "is_zero")]
31    pub omitted: u64,
32}
33
34#[expect(
35    clippy::trivially_copy_pass_by_ref,
36    reason = "serde's skip_serializing_if passes a reference"
37)]
38fn is_zero(count: &u64) -> bool {
39    *count == 0
40}
41
42/// One thing that needs attention.
43#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
44pub struct AttentionItem {
45    /// The same while the condition holds, such as `cannot-start:paused:halyard/ledger`: what a client keys it by.
46    pub key: String,
47    /// What source it comes from.
48    pub kind: AttentionKind,
49    /// How urgent; a tone this client does not know reads as [`Tone::Neutral`].
50    #[serde(deserialize_with = "tone_or_neutral")]
51    pub tone: Tone,
52    /// What holds, in one line.
53    pub title: String,
54    /// What it is about.
55    pub subject: Subject,
56    /// When the evidence for it began or was taken, in seconds since the Unix epoch; absent when unknown.
57    #[serde(default, skip_serializing_if = "Option::is_none")]
58    pub since: Option<u64>,
59    /// What clears it; absent when there is nothing to name, or the fix is one this client cannot read.
60    #[serde(
61        default,
62        skip_serializing_if = "Option::is_none",
63        deserialize_with = "lenient"
64    )]
65    pub fix: Option<Fix>,
66}
67
68/// Where an attention item comes from.
69#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
70#[serde(rename_all = "kebab-case")]
71pub enum AttentionKind {
72    /// Queued jobs runnerd cannot start with the config it runs with: their repository is paused, or no pool serves
73    /// their labels ([`Waiting::cannot_start`]).
74    CannotStart,
75    /// A job has waited 30 minutes or more for the host's memory, and one waits for it now.
76    HostMemory,
77    /// A doctor check failed or warns, in a run for the config the daemon runs with.
78    Doctor,
79    /// The doctor's checks are not current: never run yet, failed, or stale.
80    DoctorUnavailable,
81    /// A warning insight.
82    Insight,
83    /// A repository's queued jobs cannot be listed.
84    GithubListing,
85    /// GitHub refuses runnerd's token, or there is none: no repository is listed.
86    GithubAuth,
87    /// GitHub's rate limit is used up: no runner starts until it resets.
88    RateLimited,
89    /// Action archives runnerd should keep for its runners failed to be read, downloaded or kept (ADR 0147): jobs
90    /// download those actions themselves.
91    ActionArchives,
92    /// GitHub refused runnerd's automatic re-run of a run its runner failed (ADR 0153): it holds that repository's
93    /// re-runs for an hour.
94    RerunRefused,
95    /// A kind from a newer daemon than this client knows: draw it by its title.
96    #[serde(other)]
97    Unknown,
98}
99
100/// What an attention item is about, for a client to link.
101#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
102#[serde(rename_all = "kebab-case", tag = "kind")]
103pub enum Subject {
104    /// A job.
105    Job {
106        /// GitHub's job ID, the history's too.
107        id: u64,
108        /// `owner/name`.
109        repository: String,
110        /// The job's name.
111        name: String,
112    },
113    /// A repository.
114    Repository {
115        /// `owner/name`.
116        name: String,
117    },
118    /// One of the doctor's checks.
119    Check {
120        /// The check's name, such as `slice`.
121        name: String,
122    },
123    /// An insight.
124    Insight {
125        /// Its key in `GET /v1/insights`.
126        key: String,
127    },
128    /// The host as a whole.
129    Host,
130    /// GitHub, for every repository.
131    Github,
132    /// A subject from a newer daemon than this client knows.
133    #[serde(other)]
134    Unknown,
135}
136
137/// What clears an attention item. A client reads a fix of a kind or view it does not know as none
138/// ([`AttentionItem::fix`]).
139#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
140#[serde(rename_all = "kebab-case", tag = "kind")]
141pub enum Fix {
142    /// A command to run, such as `runnerctl resume halyard/ledger`.
143    Command {
144        /// The command line.
145        command: String,
146    },
147    /// A key of the config to change, such as `repositories.labels`.
148    Config {
149        /// The key, dotted.
150        key: String,
151    },
152    /// A view where the subject is shown with what to do.
153    Open {
154        /// The view.
155        view: View,
156    },
157    /// What to do, in words, such as a doctor check's fix.
158    Text {
159        /// What to do.
160        text: String,
161    },
162}
163
164impl Fix {
165    /// What to do, as every client says it: the command, `Change repositories.labels in the config`, `Open Settings`,
166    /// or the text.
167    pub fn line(&self) -> String {
168        match self {
169            Fix::Command { command } => command.clone(),
170            Fix::Config { key } => format!("Change {key} in the config"),
171            Fix::Open { view } => format!("Open {}", view.title()),
172            Fix::Text { text } => text.clone(),
173        }
174    }
175}
176
177impl Attention {
178    /// The snapshot of `items`, as [`attention_items`] orders them, at `revision` of `instance`: the first
179    /// [`ATTENTION_MAX`], and how many more.
180    pub fn new(instance: String, revision: u64, mut items: Vec<AttentionItem>) -> Attention {
181        let omitted = items.len().saturating_sub(ATTENTION_MAX);
182        items.truncate(ATTENTION_MAX);
183        Attention {
184            instance,
185            revision,
186            items,
187            omitted: omitted as u64,
188        }
189    }
190
191    /// Whether `other` says the same, whatever its instance and revision.
192    pub fn same_content(&self, other: &Attention) -> bool {
193        self.items == other.items && self.omitted == other.omitted
194    }
195
196    /// How many items need attention, those left out included: what a client shows as the list's count.
197    pub fn count(&self) -> u64 {
198        self.items.len() as u64 + self.omitted
199    }
200
201    /// What a client that shows the first `shown` items says of the rest: `and 3 more`; `None` when it shows all.
202    pub fn more(&self, shown: usize) -> Option<String> {
203        let rest = self.items.len().saturating_sub(shown) as u64 + self.omitted;
204        (rest > 0).then(|| format!("and {rest} more"))
205    }
206}
207
208/// Everything that needs attention now, most urgent first: from `status`, the doctor's last run as `doctor` says it,
209/// and `insights` when they could be read, at `now` (seconds since the Unix epoch). Each item stands on evidence that
210/// holds now; a source without it is left out, and doctor checks that are not current are one item saying so.
211pub fn attention_items(
212    status: &Status,
213    doctor: &Doctor,
214    insights: Option<&Insights>,
215    now: u64,
216) -> Vec<AttentionItem> {
217    let mut items = cannot_start(status);
218    items.extend(host_memory(status, insights));
219    items.extend(doctor_items(doctor, now));
220    items.extend(
221        insights
222            .into_iter()
223            .flat_map(|insights| &insights.insights)
224            .filter_map(insight_item),
225    );
226    items.extend(github(status, now));
227    items.extend(action_archives(status));
228    items.extend(rerun_refused(status));
229    items.sort_by(|a, b| {
230        rank(a.tone)
231            .cmp(&rank(b.tone))
232            .then_with(|| b.since.is_some().cmp(&a.since.is_some()))
233            .then_with(|| b.since.cmp(&a.since))
234            .then_with(|| a.key.cmp(&b.key))
235    });
236    items
237}
238
239/// The most urgent tone first.
240fn rank(tone: Tone) -> u8 {
241    match tone {
242        Tone::Bad => 0,
243        Tone::Warn => 1,
244        Tone::Neutral => 2,
245        Tone::Quiet => 3,
246        Tone::Good => 4,
247    }
248}
249
250/// Queued jobs runnerd cannot start with its config, one item per repository and reason.
251fn cannot_start(status: &Status) -> Vec<AttentionItem> {
252    let mut groups: Vec<(String, Vec<&QueuedJob>)> = Vec::new();
253    for job in &status.queued {
254        let Some(waiting) = job
255            .waiting
256            .as_ref()
257            .filter(|waiting| waiting.cannot_start())
258        else {
259            continue;
260        };
261        let key = match waiting {
262            Waiting::NoPool { labels } => format!(
263                "cannot-start:no-pool:{}:{}",
264                job.repository,
265                labels.join(",")
266            ),
267            _ => format!("cannot-start:paused:{}", job.repository),
268        };
269        match groups.iter_mut().find(|(group, _)| *group == key) {
270            Some((_, jobs)) => jobs.push(job),
271            None => groups.push((key, vec![job])),
272        }
273    }
274    groups
275        .into_iter()
276        .filter_map(|(key, jobs)| {
277            let first = *jobs.first()?;
278            let repository = &first.repository;
279            let (what, subject) = match jobs.as_slice() {
280                [job] => (
281                    format!("{} of {repository}", job.name),
282                    Subject::Job {
283                        id: job.id,
284                        repository: repository.clone(),
285                        name: job.name.clone(),
286                    },
287                ),
288                _ => (
289                    format!("{} queued jobs of {repository}", jobs.len()),
290                    Subject::Repository {
291                        name: repository.clone(),
292                    },
293                ),
294            };
295            let (why, fix) = match first.waiting.as_ref()? {
296                Waiting::NoPool { labels } if labels.is_empty() => (
297                    "it asks for no labels, so no profile serves it".to_owned(),
298                    Fix::Config {
299                        key: "repositories.labels".to_owned(),
300                    },
301                ),
302                Waiting::NoPool { labels } => (
303                    format!(
304                        "no profile of {repository} serves the {} {}",
305                        if labels.len() == 1 { "label" } else { "labels" },
306                        labels.join(", ")
307                    ),
308                    Fix::Config {
309                        key: "repositories.labels".to_owned(),
310                    },
311                ),
312                _ => (
313                    format!("{repository} is paused"),
314                    Fix::Command {
315                        command: format!("runnerctl resume {repository}"),
316                    },
317                ),
318            };
319            Some(AttentionItem {
320                key,
321                kind: AttentionKind::CannotStart,
322                tone: Tone::Warn,
323                title: format!("runnerd cannot start {what} with the current configuration: {why}"),
324                subject,
325                since: jobs
326                    .iter()
327                    .map(|job| job.queued_at)
328                    .filter(|at| *at > 0)
329                    .min(),
330                fix: Some(fix),
331            })
332        })
333        .collect()
334}
335
336/// The host-held insight, while a queued job waits for the host's memory in `status` too: one item, which the
337/// insights leave out. It is about the host, which programs outside CI hold, not about one job: the insight names
338/// none, and the jobs waiting may have changed since it was read. Its evidence time is unknown for the same reason;
339/// the daemon reads the insights again when the jobs waiting for the host change.
340fn host_memory(status: &Status, insights: Option<&Insights>) -> Option<AttentionItem> {
341    let held = insights?
342        .insights
343        .iter()
344        .find(|insight| matches!(insight.kind, InsightKind::HostHeld { .. }))?;
345    (status.host_wait(None) == HostWait::Waits).then(|| AttentionItem {
346        key: "host-memory".to_owned(),
347        kind: AttentionKind::HostMemory,
348        tone: Tone::Warn,
349        title: held.sentence.clone(),
350        subject: Subject::Host,
351        since: None,
352        fix: Some(Fix::Open {
353            view: View::Overview,
354        }),
355    })
356}
357
358/// The doctor's failures and warnings from a current run, or one item saying its checks are not current.
359fn doctor_items(doctor: &Doctor, now: u64) -> Vec<AttentionItem> {
360    let at = match &doctor.state {
361        // `runnerd doctor`'s own run, or one a daemon ran for the request: current.
362        None => None,
363        Some(DoctorState::Ready { at }) => Some(*at),
364        Some(
365            state @ (DoctorState::NotRun
366            | DoctorState::Stale { .. }
367            | DoctorState::Failed { .. }
368            | DoctorState::Unknown),
369        ) => {
370            let (tone, since) = match state {
371                DoctorState::Failed { at, .. } => (Tone::Warn, Some(*at)),
372                DoctorState::Stale { at } => (Tone::Quiet, Some(*at)),
373                _ => (Tone::Quiet, None),
374            };
375            return state
376                .line(now)
377                .map(|title| AttentionItem {
378                    key: "doctor-unavailable".to_owned(),
379                    kind: AttentionKind::DoctorUnavailable,
380                    tone,
381                    title,
382                    subject: Subject::Host,
383                    since,
384                    fix: Some(Fix::Open {
385                        view: View::Settings,
386                    }),
387                })
388                .into_iter()
389                .collect();
390        }
391    };
392    let shown: Vec<_> = doctor
393        .checks
394        .iter()
395        .filter(|check| matches!(check.status, CheckStatus::Fail | CheckStatus::Warn))
396        // Advisory whatever it says: only GitHub's answer to a real action is evidence (ADR 0134).
397        .filter(|check| check.name != super::JOB_ACTIONS_CHECK)
398        .collect();
399    let mut keys: Vec<String> = Vec::new();
400    shown
401        .iter()
402        .map(|check| {
403            // Checks of one name, such as each repository's visibility, are told apart by what they found, so a key
404            // stays with its check when another of the name comes or goes.
405            let repeated = shown
406                .iter()
407                .filter(|other| other.name == check.name)
408                .count()
409                > 1;
410            let mut key = if repeated {
411                format!("doctor:{}:{}", check.name, check.detail)
412            } else {
413                format!("doctor:{}", check.name)
414            };
415            let twins = keys.iter().filter(|seen| **seen == key).count();
416            keys.push(key.clone());
417            if twins > 0 {
418                key = format!("{key}#{}", twins + 1);
419            }
420            AttentionItem {
421                key,
422                kind: AttentionKind::Doctor,
423                tone: check.status.tone(),
424                title: format!("{}: {}", check.name, check.detail),
425                subject: Subject::Check {
426                    name: check.name.clone(),
427                },
428                since: at,
429                fix: Some(match &check.fix {
430                    Some(text) => Fix::Text { text: text.clone() },
431                    None => Fix::Open {
432                        view: View::Settings,
433                    },
434                }),
435            }
436        })
437        .collect()
438}
439
440/// A warning insight that says what holds now: not the host-held one, which [`host_memory`] stands for, nor a past
441/// day's low memory.
442fn insight_item(insight: &Insight) -> Option<AttentionItem> {
443    let current = !matches!(
444        insight.kind,
445        InsightKind::HostHeld { .. } | InsightKind::HostLow { .. } | InsightKind::Unknown
446    );
447    (insight.severity == Severity::Warning && current).then(|| AttentionItem {
448        key: format!("insight:{}", insight.key),
449        kind: AttentionKind::Insight,
450        tone: Tone::Warn,
451        title: insight.sentence.clone(),
452        subject: Subject::Insight {
453            key: insight.key.clone(),
454        },
455        since: insight.runs.first().map(|run| run.at),
456        fix: Some(Fix::Open {
457            view: View::Insights,
458        }),
459    })
460}
461
462/// The listings GitHub fails or refuses, and its rate limit.
463/// The action archives' failures, as one warning: jobs still run, downloading what is missing.
464fn action_archives(status: &Status) -> Option<AttentionItem> {
465    let archives = status.action_archives.as_ref()?;
466    Some(AttentionItem {
467        key: "action-archives".to_owned(),
468        kind: AttentionKind::ActionArchives,
469        tone: archives.tone(),
470        title: archives.attention_title()?,
471        subject: Subject::Github,
472        since: archives.failing_since(),
473        fix: None,
474    })
475}
476
477/// One item per repository whose automatic re-runs GitHub refused.
478fn rerun_refused(status: &Status) -> impl Iterator<Item = AttentionItem> + '_ {
479    status.repositories.iter().filter_map(|repository| {
480        let refused = repository.rerun_refused.as_ref()?;
481        Some(AttentionItem {
482            key: format!("rerun-refused:{}", repository.name),
483            kind: AttentionKind::RerunRefused,
484            tone: Tone::Warn,
485            title: refused.title(&repository.name),
486            subject: Subject::Repository {
487                name: repository.name.clone(),
488            },
489            since: Some(refused.since),
490            fix: Some(Fix::Text {
491                text: refused.fix().to_owned(),
492            }),
493        })
494    })
495}
496
497fn github(status: &Status, now: u64) -> Vec<AttentionItem> {
498    let failing: Vec<_> = status
499        .repositories
500        .iter()
501        .filter_map(|repository| Some((repository.name.as_str(), repository.listing.as_ref()?)))
502        .collect();
503    let token = failing
504        .iter()
505        .filter(|(_, failure)| failure.problem.is_token())
506        .min_by_key(|(_, failure)| failure.since);
507    let mut items: Vec<AttentionItem> = token
508        .map(|(_, failure)| AttentionItem {
509            key: "github-auth".to_owned(),
510            kind: AttentionKind::GithubAuth,
511            tone: Tone::Bad,
512            title: match failure.problem {
513                super::ListingProblem::NoToken => {
514                    "runnerd has no GitHub token, so it lists no repository's queued jobs"
515                        .to_owned()
516                }
517                _ => {
518                    "GitHub refuses runnerd's token (401), so it lists no repository's queued jobs"
519                        .to_owned()
520                }
521            },
522            subject: Subject::Github,
523            since: Some(failure.since),
524            fix: Some(Fix::Open {
525                view: View::Settings,
526            }),
527        })
528        .into_iter()
529        .collect();
530    items.extend(
531        failing
532            .iter()
533            .filter(|(_, failure)| !failure.problem.is_token())
534            .map(|(name, failure)| AttentionItem {
535                key: format!("github-listing:{name}"),
536                kind: AttentionKind::GithubListing,
537                tone: Tone::Warn,
538                title: failure.line(name),
539                subject: Subject::Repository {
540                    name: (*name).to_owned(),
541                },
542                since: Some(failure.since),
543                fix: matches!(failure.problem, super::ListingProblem::Refused { .. }).then_some(
544                    Fix::Open {
545                        view: View::Settings,
546                    },
547                ),
548            }),
549    );
550    // Gone once its reset time passed, whether or not the engine published since.
551    let limited = status.github_limited_until.is_some_and(|until| until > now);
552    items.extend(
553        status
554            .github_line()
555            .filter(|_| limited)
556            .map(|line| AttentionItem {
557                key: "rate-limited".to_owned(),
558                kind: AttentionKind::RateLimited,
559                tone: Tone::Warn,
560                title: line.to_owned(),
561                subject: Subject::Github,
562                since: None,
563                fix: None,
564            }),
565    );
566    items
567}
568
569/// A value of a type this client may not know all of, such as a [`Fix`] naming a view it lacks: `None` then, so the
570/// rest of the document still reads.
571fn lenient<'de, D: Deserializer<'de>, T: Deserialize<'de>>(
572    deserializer: D,
573) -> Result<Option<T>, D::Error> {
574    #[derive(Deserialize)]
575    #[serde(untagged)]
576    enum Read<T> {
577        Known(T),
578        Other(serde::de::IgnoredAny),
579    }
580    Ok(match Option::<Read<T>>::deserialize(deserializer)? {
581        Some(Read::Known(value)) => Some(value),
582        Some(Read::Other(_)) | None => None,
583    })
584}
585
586fn tone_or_neutral<'de, D: Deserializer<'de>>(deserializer: D) -> Result<Tone, D::Error> {
587    Ok(lenient(deserializer)?.unwrap_or(Tone::Neutral))
588}
589
590// The wire format is JSON, which only the host and client features read.
591#[cfg(all(test, any(feature = "host", feature = "client")))]
592mod tests;