Skip to main content

magi/
notices.rs

1//! Durable attention-report outbox, reconciled into task/run chats by
2//! [`crate::notice_delivery`]. Reports are facts to investigate, never decisions
3//! or owner authority. The compatibility list remains for deferred delivery.
4//!
5//! Keys name kind + subject, never prose. Repeats update evidence without
6//! another page; only a newer cause or severity escalation advances delivery.
7//! Chat drafts persist their receipt key atomically before outbox acknowledgement.
8//! Read/dismissed legacy history is never replayed, and pending unread records
9//! are never evicted by the history cap.
10
11use std::path::{Path, PathBuf};
12
13use anyhow::{Context, Result, bail};
14use jiff::Timestamp;
15use serde::{Deserialize, Serialize};
16
17/// On-disk format. A file is refused only when its own schema is *greater*
18/// than this one; every field added later must carry `#[serde(default)]`.
19pub const SCHEMA: u32 = 1;
20
21/// The history cap. Pending unread deliveries are exempt.
22pub const CAP: usize = 200;
23
24/// How serious a notice is. Ordered so a rise is a plain comparison.
25#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Serialize, Deserialize)]
26#[serde(rename_all = "lowercase")]
27pub enum Severity {
28    /// Worth knowing, nothing to do.
29    Info,
30    /// Something needs a look soon.
31    Warn,
32    /// Something failed and stays failed until someone acts.
33    Error,
34}
35
36/// Where a notice points, when it points anywhere.
37#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
38#[serde(tag = "kind", rename_all = "lowercase")]
39pub enum Link {
40    /// A run, by id.
41    Run {
42        /// The run id.
43        id: String,
44    },
45    /// A queued task, by id.
46    Task {
47        /// The task id.
48        id: String,
49    },
50    /// An external page, such as a pull request.
51    Url {
52        /// The address.
53        url: String,
54    },
55}
56
57/// One thing the operator should know.
58#[derive(Debug, Clone, Serialize, Deserialize)]
59pub struct Notice {
60    /// Stable file-name-safe id derived from [`Notice::key`].
61    pub id: String,
62    /// Deduplication key: kind and subject, never prose.
63    pub key: String,
64    /// How serious.
65    pub severity: Severity,
66    /// What happened.
67    pub message: String,
68    /// Where it points, when it points anywhere.
69    #[serde(default)]
70    pub link: Option<Link>,
71    /// First raised.
72    pub first_at: Timestamp,
73    /// Last raised.
74    pub last_at: Timestamp,
75    /// How many times this key was raised.
76    #[serde(default = "one")]
77    pub count: u32,
78    /// When it was marked read.
79    #[serde(default)]
80    pub read_at: Option<Timestamp>,
81    /// When it was dismissed; the tombstone.
82    #[serde(default)]
83    pub dismissed_at: Option<Timestamp>,
84    /// The task and run ids this notice is about. A question whose `run`
85    /// names one of them already pages the operator for the same cause.
86    #[serde(default)]
87    pub subjects: Vec<String>,
88    /// Run association when the display link points elsewhere (for example a PR).
89    #[serde(default)]
90    pub related_run: Option<String>,
91    /// The question that carries this notice as context, when it was filed
92    /// already read for that reason. Never a tombstone: a recurrence with a
93    /// newer cause or higher severity is unread again.
94    #[serde(default)]
95    pub covered_by: Option<String>,
96    /// When the cause this notice reports began (a task's `held_at`). A
97    /// question filed before it is about an older cause and does not cover
98    /// it; `None` keeps the broad coverage of notices without one.
99    #[serde(default)]
100    pub since: Option<Timestamp>,
101    /// Producer-defined cause identity (phase and immutable target), never prose.
102    #[serde(default)]
103    pub cause: Option<String>,
104    /// On-disk format version.
105    #[serde(default = "schema")]
106    pub schema: u32,
107    /// Persistent incarnation, so pruning history cannot reuse a chat receipt.
108    /// Legacy records keep their original receipt key.
109    #[serde(default)]
110    pub delivery_id: Option<String>,
111    /// Delivery generation, advanced only by the shared page decision.
112    #[serde(default = "one")]
113    pub generation: u32,
114    /// Last generation durably accepted by a chat.
115    #[serde(default)]
116    pub delivered: u32,
117}
118
119fn one() -> u32 {
120    1
121}
122
123fn schema() -> u32 {
124    SCHEMA
125}
126
127impl Notice {
128    /// A notice of `severity` for `key`.
129    pub fn new(severity: Severity, key: &str, message: impl Into<String>) -> Self {
130        let now = Timestamp::now();
131        Self {
132            id: id_of(key),
133            key: key.to_owned(),
134            severity,
135            message: message.into(),
136            link: None,
137            first_at: now,
138            last_at: now,
139            count: 1,
140            read_at: None,
141            dismissed_at: None,
142            subjects: Vec::new(),
143            related_run: None,
144            covered_by: None,
145            since: None,
146            cause: None,
147            schema: SCHEMA,
148            delivery_id: Some(format!("{now}-{:016x}", crate::rng::entropy())),
149            generation: 1,
150            delivered: 0,
151        }
152    }
153
154    /// Name the task / run ids this notice is about.
155    pub fn about<I, S>(mut self, subjects: I) -> Self
156    where
157        I: IntoIterator<Item = S>,
158        S: Into<String>,
159    {
160        self.subjects = subjects.into_iter().map(Into::into).collect();
161        self
162    }
163
164    /// Associate a run independently of the display link.
165    pub fn for_run(mut self, run: Option<String>) -> Self {
166        self.related_run = run;
167        self
168    }
169
170    /// An [`Severity::Info`] notice.
171    pub fn info(key: &str, message: impl Into<String>) -> Self {
172        Self::new(Severity::Info, key, message)
173    }
174
175    /// A [`Severity::Warn`] notice.
176    pub fn warn(key: &str, message: impl Into<String>) -> Self {
177        Self::new(Severity::Warn, key, message)
178    }
179
180    /// An [`Severity::Error`] notice.
181    pub fn error(key: &str, message: impl Into<String>) -> Self {
182        Self::new(Severity::Error, key, message)
183    }
184
185    /// Record when the reported cause began.
186    pub fn since(mut self, at: Option<Timestamp>) -> Self {
187        self.since = at;
188        self
189    }
190
191    /// Distinguish material producer state transitions without keying on wording.
192    pub fn cause(mut self, cause: impl Into<String>) -> Self {
193        self.cause = Some(cause.into());
194        self
195    }
196
197    /// Attach a link.
198    pub fn link(mut self, link: Link) -> Self {
199        self.link = Some(link);
200        self
201    }
202
203    /// Neither read nor dismissed.
204    pub fn unread(&self) -> bool {
205        self.read_at.is_none() && self.dismissed_at.is_none()
206    }
207
208    /// Fold a repeat into this notice. Wording updates evidence; a newer cause
209    /// or higher severity makes it unread and advances delivery.
210    ///
211    /// Returns whether this raise should page the operator: it relit the
212    /// notice (higher severity or newer cause) and no open question
213    /// already carries it. This is the only place that decides.
214    pub fn raise_again(&mut self, again: &Notice, now: Timestamp) -> bool {
215        self.count = self.count.saturating_add(1);
216        let previous_last_at = self.last_at;
217        self.last_at = now;
218        if again.link.is_some() {
219            self.link = again.link.clone();
220        }
221        if again.related_run.is_some() {
222            self.related_run = again.related_run.clone();
223        }
224        if !again.subjects.is_empty() {
225            self.subjects = again.subjects.clone();
226        }
227        let escalated = again.severity > self.severity;
228        // Legacy history lacks a cause timestamp. Only a cause that began
229        // after the saved raise is news; replaying that old failure is not.
230        // A previously covering question can also prove a new uncovered cause.
231        let newer_cause = match (self.since, again.since) {
232            (Some(old), Some(new)) => new > old,
233            (None, Some(new)) => {
234                new > previous_last_at || (self.covered_by.is_some() && again.covered_by.is_none())
235            }
236            _ => false,
237        };
238        self.since = self.since.max(again.since);
239        // Wording is evidence, never a reason to spend another agent turn.
240        let changed_cause = again
241            .cause
242            .as_ref()
243            .is_some_and(|cause| self.cause.as_ref() != Some(cause));
244        if again.cause.is_some() {
245            self.cause = again.cause.clone();
246        }
247        let relit = escalated || newer_cause || changed_cause;
248        self.message = again.message.clone();
249        if relit {
250            self.generation = self.generation.saturating_add(1);
251            self.severity = self.severity.max(again.severity);
252            self.read_at = None;
253            self.dismissed_at = None;
254            self.covered_by = None;
255        }
256        // A question already pages for this cause: keep the record, skip the
257        // second page. A higher severity is news the question did not carry.
258        if let Some(q) = &again.covered_by
259            && !escalated
260            && self.unread()
261        {
262            self.read_at = Some(now);
263            self.covered_by = Some(q.clone());
264        }
265        relit && self.covered_by.is_none()
266    }
267
268    /// Mark read, keeping the first read time.
269    pub fn mark_read(&mut self, now: Timestamp) {
270        if self.read_at.is_none() {
271            self.read_at = Some(now);
272        }
273    }
274
275    /// Tombstone: read and hidden from the list.
276    pub fn dismiss(&mut self, now: Timestamp) {
277        self.mark_read(now);
278        if self.dismissed_at.is_none() {
279            self.dismissed_at = Some(now);
280        }
281    }
282
283    /// Legacy exhausted-deputy warnings belong on the question card now.
284    /// Match only this retired namespace, not `deputy-turn:` failures.
285    fn obsolete(&self) -> bool {
286        self.key.starts_with("deputy:")
287    }
288
289    /// Whether this notice belongs in the notification centre and bell.
290    fn visible(&self) -> bool {
291        !self.obsolete() && self.dismissed_at.is_none()
292    }
293
294    /// Pruning order: obsolete, dismissed, read, then unread.
295    fn keep_rank(&self) -> u8 {
296        if self.obsolete() {
297            0
298        } else if self.dismissed_at.is_some() {
299            1
300        } else if self.read_at.is_some() {
301            2
302        } else {
303            3
304        }
305    }
306}
307
308/// A stable, file-name-safe id for a key: a readable slug plus FNV-1a of the
309/// whole key. Not `DefaultHasher`, whose output may change between builds and
310/// would strand every existing file's dedupe.
311pub fn id_of(key: &str) -> String {
312    let mut hash: u64 = 0xcbf2_9ce4_8422_2325;
313    for b in key.bytes() {
314        hash ^= u64::from(b);
315        hash = hash.wrapping_mul(0x0100_0000_01b3);
316    }
317    let slug: String = key
318        .chars()
319        .map(|c| {
320            if c.is_ascii_alphanumeric() {
321                c.to_ascii_lowercase()
322            } else {
323                '-'
324            }
325        })
326        .take(32)
327        .collect();
328    format!("{slug}-{hash:016x}")
329}
330
331/// Whether `id` could have come from [`id_of`]. Checked before any path is
332/// built from a request.
333fn valid_id(id: &str) -> bool {
334    !id.is_empty()
335        && id.len() <= 64
336        && id
337            .bytes()
338            .all(|b| b.is_ascii_lowercase() || b.is_ascii_digit() || b == b'-')
339}
340
341/// How long a caller waits for a notice's lock before proceeding without it.
342const LOCK_WAIT: std::time::Duration = std::time::Duration::from_secs(5);
343
344/// Age after which a lock file is taken to belong to a dead process.
345const LOCK_STALE: std::time::Duration = std::time::Duration::from_secs(10);
346
347/// Distinguishes temp files written by threads of one process.
348static TMP_SEQ: std::sync::atomic::AtomicU64 = std::sync::atomic::AtomicU64::new(0);
349
350/// A notice store on disk.
351#[derive(Debug, Clone)]
352pub struct Notices {
353    root: PathBuf,
354}
355
356impl Notices {
357    /// The operator's notices, `<home>/notifications`.
358    pub fn open() -> Self {
359        Self::at(crate::run::home().join("notifications"))
360    }
361
362    /// A store at an explicit root; tests use this.
363    pub fn at(root: PathBuf) -> Self {
364        Self { root }
365    }
366
367    /// Directory holding the notice files.
368    pub fn root(&self) -> &Path {
369        &self.root
370    }
371
372    fn path_of(&self, id: &str) -> PathBuf {
373        self.root.join(format!("{id}.json"))
374    }
375
376    fn put(&self, n: &Notice) -> Result<()> {
377        std::fs::create_dir_all(&self.root)
378            .with_context(|| format!("create {}", self.root.display()))?;
379        let body = serde_json::to_string_pretty(n).context("serialize notice")?;
380        let path = self.path_of(&n.id);
381        // Unique per write: two processes raising or marking the same notice
382        // must not share a temp file, or the second rename finds it gone.
383        let tmp = path.with_extension(format!(
384            "json.{}.{}.tmp",
385            std::process::id(),
386            TMP_SEQ.fetch_add(1, std::sync::atomic::Ordering::Relaxed)
387        ));
388        std::fs::write(&tmp, body).with_context(|| format!("write {}", tmp.display()))?;
389        if let Err(e) = std::fs::rename(&tmp, &path) {
390            let _ = std::fs::remove_file(&tmp);
391            return Err(e).with_context(|| format!("replace {}", path.display()));
392        }
393        Ok(())
394    }
395
396    /// Load one notice by id.
397    pub fn get(&self, id: &str) -> Result<Notice> {
398        if !valid_id(id) {
399            bail!("`{id}` is not a notification id");
400        }
401        read_path(&self.path_of(id))
402    }
403
404    /// Run a read-modify-write of one notice under its lock file.
405    ///
406    /// `magi serve`, `magi web` and the CLI all update the same files, and a
407    /// unique temp file only makes each write atomic - it does not stop one
408    /// process saving a stale read over another's newer raise. The lock is an
409    /// exclusive-create file beside the notice; a holder that died is broken
410    /// after [`LOCK_STALE`], and a caller that cannot get it in
411    /// [`LOCK_WAIT`] proceeds anyway rather than lose the notice.
412    fn locked<T>(&self, id: &str, f: impl FnOnce() -> Result<T>) -> Result<T> {
413        std::fs::create_dir_all(&self.root)
414            .with_context(|| format!("create {}", self.root.display()))?;
415        let lock = self.root.join(format!("{id}.lock"));
416        let start = std::time::Instant::now();
417        let retry = crate::lock_file::Retry::new(Some(start + LOCK_WAIT));
418        let mut held = false;
419        while start.elapsed() < LOCK_WAIT {
420            match retry.create(&lock) {
421                Ok(_) => {
422                    held = true;
423                    break;
424                }
425                Err(_) => {
426                    let stale = std::fs::metadata(&lock)
427                        .and_then(|m| m.modified())
428                        .ok()
429                        .and_then(|t| t.elapsed().ok())
430                        .is_some_and(|age| age > LOCK_STALE);
431                    if stale {
432                        let _ = std::fs::remove_file(&lock);
433                    } else {
434                        std::thread::sleep(std::time::Duration::from_millis(5));
435                    }
436                }
437            }
438        }
439        let out = f();
440        if held {
441            let _ = std::fs::remove_file(&lock);
442        }
443        out
444    }
445
446    /// File a notice, folding it into an existing one with the same key.
447    pub fn raise(&self, incoming: Notice) -> Result<Notice> {
448        self.raise_paged(incoming).map(|(n, _)| n)
449    }
450
451    /// File through the common page decision, including explicit-store producers.
452    pub fn raise_with(
453        &self,
454        mut incoming: Notice,
455        notify: &crate::config::Notify,
456    ) -> Result<Notice> {
457        if let Some(home) = self.root.parent()
458            && let Some(q) = covering_question(home, &incoming)
459        {
460            incoming.covered_by = Some(q);
461        }
462        let (notice, page) = self.raise_paged(incoming)?;
463        if page {
464            send_page(
465                Page {
466                    run: notice
467                        .related_run
468                        .clone()
469                        .or_else(|| match &notice.link {
470                            Some(Link::Run { id }) => Some(id.clone()),
471                            _ => None,
472                        })
473                        .unwrap_or_default(),
474                    summary: notice.message.clone(),
475                },
476                Some(notify.clone()),
477            );
478        }
479        Ok(notice)
480    }
481
482    /// [`Notices::raise`], also saying whether the raise should page.
483    pub fn raise_paged(&self, incoming: Notice) -> Result<(Notice, bool)> {
484        let out = self.locked(&incoming.id.clone(), || {
485            let now = Timestamp::now();
486            let (stored, page) = match read_path(&self.path_of(&incoming.id)) {
487                Ok(mut existing) => {
488                    let page = existing.raise_again(&incoming, now);
489                    (existing, page)
490                }
491                Err(_) => {
492                    let mut fresh = incoming;
493                    if fresh.covered_by.is_some() {
494                        fresh.read_at = Some(now);
495                    }
496                    let page = fresh.covered_by.is_none();
497                    (fresh, page)
498                }
499            };
500            self.put(&stored)?;
501            Ok((stored, page))
502        })?;
503        self.prune();
504        Ok(out)
505    }
506
507    /// Mark every unread notice about `subject` read, recording `question` as
508    /// the card that carries it. Best-effort per notice.
509    pub fn cover(&self, question: &crate::ask::Question) {
510        for n in self.all() {
511            if n.unread() && covers(question, &n) {
512                let _ = self.update(&n.id, |n| {
513                    // Re-checked under the lock: a newer cause may have
514                    // landed since `all()` read it.
515                    if n.unread() && covers(question, n) {
516                        n.mark_read(Timestamp::now());
517                        n.covered_by = Some(question.id.clone());
518                    }
519                });
520            }
521        }
522    }
523
524    /// Load, change and save one notice under its lock.
525    pub(crate) fn update(&self, id: &str, change: impl FnOnce(&mut Notice)) -> Result<Notice> {
526        if !valid_id(id) {
527            bail!("`{id}` is not a notification id");
528        }
529        self.locked(id, || {
530            let mut n = read_path(&self.path_of(id))?;
531            change(&mut n);
532            self.put(&n)?;
533            Ok(n)
534        })
535    }
536
537    /// Every notice on disk, unreadable files skipped, newest first.
538    pub(crate) fn all(&self) -> Vec<Notice> {
539        let mut all: Vec<Notice> = std::fs::read_dir(&self.root)
540            .into_iter()
541            .flatten()
542            .flatten()
543            .map(|e| e.path())
544            .filter(|p| p.extension().is_some_and(|x| x == "json"))
545            .filter_map(|p| read_path(&p).ok())
546            .collect();
547        all.sort_by(|a, b| b.last_at.cmp(&a.last_at).then_with(|| a.id.cmp(&b.id)));
548        all
549    }
550
551    /// What the bell lists: visible, current notices, newest first.
552    pub fn list(&self) -> Vec<Notice> {
553        self.all().into_iter().filter(Notice::visible).collect()
554    }
555
556    /// How many notices are unread.
557    pub fn count_unread(&self) -> usize {
558        self.list().iter().filter(|n| n.unread()).count()
559    }
560
561    /// Mark read, keeping the first read time.
562    pub fn mark_read(&self, id: &str) -> Result<Notice> {
563        let now = Timestamp::now();
564        self.update(id, |n| n.mark_read(now))
565    }
566
567    /// Mark every unread notice read; returns how many changed.
568    pub fn mark_all_read(&self) -> Result<usize> {
569        let now = Timestamp::now();
570        let mut changed = 0;
571        for n in self.list().into_iter().filter(Notice::unread) {
572            // Re-read under the lock: a raise since the scan may have made it
573            // a different, still-unread notice, which stays as it is.
574            let done = self.update(&n.id, |n| n.mark_read(now));
575            if done.is_ok() {
576                changed += 1;
577            }
578        }
579        Ok(changed)
580    }
581
582    /// Tombstone: read and hidden from the list.
583    pub fn dismiss(&self, id: &str) -> Result<Notice> {
584        let now = Timestamp::now();
585        self.update(id, |n| n.dismiss(now))
586    }
587
588    /// Keep at most [`CAP`] files. Best-effort.
589    fn prune(&self) {
590        let mut all = self.all();
591        if all.len() <= CAP {
592            return;
593        }
594        // Most worth keeping first; drop the tail.
595        all.sort_by(|a, b| {
596            b.keep_rank()
597                .cmp(&a.keep_rank())
598                .then_with(|| b.last_at.cmp(&a.last_at))
599        });
600        for n in all
601            .split_off(CAP)
602            .into_iter()
603            .filter(|n| n.obsolete() || !n.unread() || n.delivered >= n.generation)
604        {
605            let _ = std::fs::remove_file(self.path_of(&n.id));
606        }
607    }
608
609    /// Change token for the stream. Hashes every file's name and mtime, so a
610    /// prune or delete moves it as surely as a write does (a max-mtime would
611    /// not move when the newest file is not the one removed).
612    pub fn revision(&self) -> u64 {
613        use std::hash::{Hash as _, Hasher as _};
614        let mut entries: Vec<(std::ffi::OsString, u128)> = std::fs::read_dir(&self.root)
615            .into_iter()
616            .flatten()
617            .flatten()
618            .filter(|e| e.path().extension().is_some_and(|x| x == "json"))
619            .map(|e| {
620                let at = e
621                    .metadata()
622                    .and_then(|m| m.modified())
623                    .ok()
624                    .and_then(|t| t.duration_since(std::time::UNIX_EPOCH).ok())
625                    .map_or(0, |d| d.as_nanos());
626                (e.file_name(), at)
627            })
628            .collect();
629        if entries.is_empty() {
630            return 0;
631        }
632        entries.sort();
633        let mut h = std::collections::hash_map::DefaultHasher::new();
634        entries.hash(&mut h);
635        h.finish()
636    }
637}
638
639fn read_path(path: &Path) -> Result<Notice> {
640    let body = std::fs::read_to_string(path).with_context(|| format!("read {}", path.display()))?;
641    let n: Notice =
642        serde_json::from_str(&body).with_context(|| format!("parse {}", path.display()))?;
643    if n.schema > SCHEMA {
644        bail!(
645            "notice {} was written by a newer magi (schema {}, this build speaks up to {SCHEMA})",
646            n.id,
647            n.schema
648        );
649    }
650    Ok(n)
651}
652
653/// A non-empty optional detail, trimmed for display. Never rewrites saved prose.
654pub(crate) fn detail(value: Option<&str>) -> Option<&str> {
655    clean_text(value)
656}
657
658/// Short explanation of a CLI turn that produced no usable response.
659pub(crate) fn turn_status(code: Option<i32>, timed_out: bool, quota_exhausted: bool) -> String {
660    let mut parts = vec![exit_status(code)];
661    if timed_out {
662        parts.push("time limit reached".to_owned());
663    }
664    if quota_exhausted {
665        parts.push("quota exhausted".to_owned());
666    }
667    parts.join("; ")
668}
669
670/// A currently open registered question associated with this notice, if any.
671/// `covered_by` records historical deduplication, not current answerability.
672pub(crate) fn answerable_question(n: &Notice, questions: &crate::ask::Questions) -> Option<String> {
673    let direct = n.covered_by.as_deref().or_else(|| {
674        ["question:", "deputy-turn:", "repair-owner:", "action:"]
675            .iter()
676            .find_map(|prefix| n.key.strip_prefix(prefix))
677    });
678    let release = if direct.is_none() && n.key.starts_with("release-pr:") {
679        questions
680            .root()
681            .parent()
682            .and_then(|home| crate::release_watch::question_for_notice(home, &n.key))
683    } else {
684        None
685    };
686    let id = direct.or(release.as_deref())?;
687    questions
688        .get(id)
689        .ok()
690        .filter(|q| q.status.open())
691        .map(|q| q.id)
692}
693
694/// The notice for a run that ended Blocked, Stalled or Failed, if it did.
695///
696/// A parked run was only asked to stop, so it is not news. A run that left a
697/// pull request behind still is: the PR is waiting on someone. Keyed on the
698/// run with a message free of anything that varies between attempts.
699pub fn run_ended(state: &crate::run::RunState) -> Option<Notice> {
700    use crate::run::RunStatus;
701    let failed = matches!(
702        state.status,
703        RunStatus::Blocked | RunStatus::Stalled | RunStatus::Failed
704    );
705    (failed && !state.parked).then(|| {
706        Notice::error(
707            &format!("run:{}", state.id),
708            format!(
709                "Run {} ended {}. {}",
710                state.id,
711                state.status.as_str(),
712                run_cause(state)
713            ),
714        )
715        .link(Link::Run {
716            id: state.id.clone(),
717        })
718        .about([state.id.clone()])
719        .since(Some(
720            state
721                .base_sync_recovery
722                .stop
723                .as_ref()
724                .filter(|_| state.base_sync_blocked())
725                .map(|s| s.at)
726                .or_else(|| state.passes.last().map(|p| p.started_at))
727                .unwrap_or(state.created_at),
728        ))
729    })
730}
731
732/// The notice for a pull request that merged while checks were red.
733///
734/// Keyed on the run. `summary` names the repository, pull request and failing
735/// checks: with no notify command configured this is the only place the
736/// operator learns them. A run merges once, so the wording cannot churn.
737pub fn merged_red(run_id: &str, summary: &str) -> Notice {
738    Notice::warn(&format!("merged-red:{run_id}"), summary).link(Link::Run {
739        id: run_id.to_owned(),
740    })
741}
742
743/// Human-readable process status for operator-facing failure details.
744pub fn exit_status(code: Option<i32>) -> String {
745    match code {
746        Some(code) => format!("exit code {code}"),
747        None => "exit code unavailable".to_owned(),
748    }
749}
750
751/// Trims text and drops empty values or meaningless placeholder phrases like `"(no reason given)"`.
752pub fn clean_text(s: Option<&str>) -> Option<&str> {
753    s.map(str::trim)
754        .filter(|t| !t.is_empty() && *t != "(no reason given)" && *t != "(none)")
755}
756
757/// Formats a list of identifiers or items into human-readable comma-separated prose.
758/// Returns `None` if the list is empty or contains only blank entries.
759pub fn format_list<T: AsRef<str>>(items: &[T]) -> Option<String> {
760    let non_empty: Vec<&str> = items
761        .iter()
762        .map(|s| s.as_ref().trim())
763        .filter(|s| !s.is_empty())
764        .collect();
765    if non_empty.is_empty() {
766        None
767    } else {
768        Some(non_empty.join(", "))
769    }
770}
771
772/// Ensures a sentence ends with a single period and has no trailing whitespace.
773pub fn ensure_sentence(text: &str) -> String {
774    let trimmed = text.trim();
775    if trimmed.is_empty() {
776        return String::new();
777    }
778    let trimmed_dot = trimmed.trim_end_matches('.');
779    format!("{trimmed_dot}.")
780}
781
782/// The notice for a task the machine or its attempt budget has held, if it is.
783///
784/// Called from [`crate::queue::Queue::put`], which every task transition goes
785/// through, so a hold made anywhere - the loop, the conductor, triage, a
786/// dependency removed from under a blocked task - is announced. A hold the
787/// operator placed by hand is their own action and is not news. Keyed on the
788/// task, with wording free of anything that varies between retries.
789///
790/// Falls back to [`crate::queue::Task::last_error`] when `hold_reason` is
791/// empty, the same fallback `crate::triage`'s question detail uses: a record
792/// written before `Task::fail`/`Task::handed_off` started copying `why` into
793/// `hold_reason` too would otherwise still read "no reason recorded" even
794/// though the run said exactly why it stopped.
795pub fn task_held(task: &crate::queue::Task) -> Option<Notice> {
796    use crate::queue::{HoldSource, TaskStatus};
797    if task.status != TaskStatus::Held || task.hold_source != Some(HoldSource::Machine) {
798        return None;
799    }
800    if task.has_disk_capacity_cause() {
801        return Some(Notice::warn(&format!("task:{}", task.id),
802            format!("Task {} is waiting for disk capacity and will resume automatically after admission succeeds.", task.short()))
803            .link(Link::Task { id: task.id.clone() }).about([task.id.clone()]).cause("disk-capacity"));
804    }
805    let why = clean_text(task.hold_reason.as_deref())
806        .or_else(|| clean_text(task.last_error.as_deref()))
807        .unwrap_or("no reason recorded");
808    let why_sentence = why.trim_end_matches('.');
809    Some(
810        Notice::warn(
811            &format!("task:{}", task.id),
812            format!("Task {} is held: {why_sentence}.", task.short()),
813        )
814        .link(Link::Task {
815            id: task.id.clone(),
816        })
817        .about([task.id.clone()])
818        .since(task.held_at),
819    )
820}
821
822/// The notice for a conductor concern raised after explicit resumption without
823/// verified new blocking evidence.
824///
825/// Informational: preserves the task state without re-blocking on old evidence,
826/// explains that no state changed and no reply is required, and attaches a link
827/// to the task. Any proposed question text is omitted to avoid soliciting an
828/// answer to an unregistered question.
829pub fn conductor_resumption(
830    task_id: &str,
831    task_short: &str,
832    reason: Option<&str>,
833    blocked_by: &[String],
834    evidence: &[String],
835    has_question: bool,
836) -> Notice {
837    let mut parts: Vec<String> = Vec::new();
838    parts.push(format!(
839        "Conductor raised a concern for task {task_short} after explicit resumption, but its state was not changed because no verified new blocking evidence was found."
840    ));
841
842    if let Some(r) = clean_text(reason) {
843        parts.push(format!("Concern: {}.", r.trim_end_matches('.')));
844    }
845
846    if has_question {
847        parts.push("An unregistered question was proposed; no reply is required.".to_owned());
848    }
849
850    if let Some(deps) = format_list(blocked_by) {
851        parts.push(format!("Proposed dependencies: {deps}."));
852    }
853
854    if let Some(ev) = format_list(evidence) {
855        parts.push(format!("Cited evidence: {ev}."));
856    }
857
858    parts.push("Existing holds and questions still apply.".to_owned());
859    parts.push(
860        "No reply to this notice is needed. To hold this task, use manual hold via the queue."
861            .to_owned(),
862    );
863
864    Notice::warn(&format!("conductor-resumption:{task_id}"), parts.join(" "))
865        .link(Link::Task {
866            id: task_id.to_owned(),
867        })
868        .about([task_id.to_owned()])
869}
870
871/// The notice for a run whose graph returned an error before it settled.
872pub fn run_stopped(id: &str, state: &crate::run::RunState) -> Notice {
873    Notice::error(
874        &format!("run:{id}"),
875        format!("Run {} stopped with an error. {}", id, run_cause(state)),
876    )
877    .link(Link::Run { id: id.to_owned() })
878    .about([id.to_owned()])
879    .since(Some(
880        state
881            .base_sync_recovery
882            .stop
883            .as_ref()
884            .filter(|_| state.base_sync_blocked())
885            .map(|s| s.at)
886            .or_else(|| state.passes.last().map(|p| p.started_at))
887            .unwrap_or(state.created_at),
888    ))
889}
890
891/// Last recorded graph evidence; immediate errors are attached by the caller.
892fn run_cause(state: &crate::run::RunState) -> String {
893    let verification_cause = state.verification_blocking_cause();
894    let cause = state
895        .prep_failure
896        .as_deref()
897        .or_else(|| state.resource_block.as_ref().map(|b| b.detail.as_str()))
898        .or_else(|| {
899            state
900                .candidate_preservation
901                .values()
902                .next()
903                .map(String::as_str)
904        })
905        .or(verification_cause.as_deref())
906        .or_else(|| state.events.last().map(|e| e.message.as_str()));
907    if state.base_sync_blocked() || state.resource_block.is_some() {
908        return format!(
909            "{} Next: {}",
910            cause.unwrap_or("Synchronization stopped."),
911            state.base_sync_next_action()
912        );
913    }
914    cause
915        .and_then(|s| s.lines().find(|line| !line.trim().is_empty()))
916        .unwrap_or("No cause recorded; inspect the run.")
917        .to_owned()
918}
919
920/// One page for the operator's `[notify]` command.
921#[derive(Debug, Clone, PartialEq, Eq)]
922pub struct Page {
923    /// The run id when the notice links to a run, else empty.
924    pub run: String,
925    /// The notice message.
926    pub summary: String,
927}
928
929/// Set once by `main`: until then no notice fires the command, so a unit or
930/// integration test that pins a home never reaches the operator's real config.
931static PAGER: std::sync::OnceLock<()> = std::sync::OnceLock::new();
932
933/// Let notices fire the `[notify]` command in this process.
934pub fn install_pager() {
935    let _ = PAGER.set(());
936}
937
938/// The production sender. Fire and forget on its own thread with its own
939/// runtime, so it works with or without an ambient one and never blocks the
940/// producer; `ask::notify_text` bounds the command by `NOTIFY_TIMEOUT`.
941/// `notify` is the producer's own config, else the machine layer's.
942fn send_page(page: Page, notify: Option<crate::config::Notify>) {
943    if PAGER.get().is_none() {
944        return;
945    }
946    let spawned = std::thread::Builder::new()
947        .name("notice-pager".into())
948        .spawn(move || {
949            let notify = notify.unwrap_or_else(machine_notify);
950            if notify.command.is_empty() {
951                return;
952            }
953            let rt = match tokio::runtime::Builder::new_current_thread()
954                .enable_all()
955                .build()
956            {
957                Ok(rt) => rt,
958                Err(e) => return tracing::warn!("could not page about a notification: {e:#}"),
959            };
960            if let Err(e) = rt.block_on(crate::ask::notify_text(&notify, &page.run, &page.summary))
961            {
962                tracing::warn!("could not page about a notification: {e:#}");
963            }
964        });
965    if let Err(e) = spawned {
966        tracing::warn!("could not page about a notification: {e:#}");
967    }
968}
969
970/// `[notify]` from the machine layer alone; a producer with no repo config
971/// cannot name a better one. Unreadable is no command, with a warning.
972fn machine_notify() -> crate::config::Notify {
973    let layers: Vec<PathBuf> = crate::config::Config::machine_layer()
974        .into_iter()
975        .filter(|p| p.is_file())
976        .collect();
977    match crate::config::Config::load_layers(&layers) {
978        Ok(c) => c.notify,
979        Err(e) => {
980            tracing::warn!("could not read [notify] for a notification: {e:#}");
981            Default::default()
982        }
983    }
984}
985
986/// The one function producers call. Best-effort by construction: a notice that
987/// cannot be filed is a `tracing::warn`, never a reason to fail the run, the
988/// loop or the request that wanted to mention it.
989pub fn raise(notice: Notice) {
990    if let Some(home) = crate::run::try_home() {
991        raise_in(&home, notice);
992    }
993}
994
995/// [`raise`] for a producer that already holds the repo's `[notify]`.
996pub fn raise_with(notice: Notice, notify: &crate::config::Notify) {
997    if let Some(home) = crate::run::try_home() {
998        raise_in_with(&home, notice, &|page| send_page(page, Some(notify.clone())));
999    }
1000}
1001
1002/// Does open question `q` already page the operator for what `n` reports?
1003///
1004/// The cause is decided by what each side is about, not by when it happened:
1005/// a conductor / triage question exists *because* its task is held, so it
1006/// covers that task's hold and handover notices whenever it was filed; a
1007/// question raised from inside a run (any other seat node) covers that run's
1008/// ended / stopped notices and never a task hold. Exact match on
1009/// `Question::run` (a task id for the former, a run id for the latter). The
1010/// land approval and release questions are to-dos, not duplicates, and cover
1011/// nothing.
1012///
1013/// A notice with a `since` (a task hold's start) is covered only by a question
1014/// filed at or after it: one filed earlier is about an older cause, and
1015/// letting it silence a new hold would hide a fresh failure behind a stale
1016/// card. Missing a duplicate costs one extra page; hiding a failure costs the
1017/// failure. A notice without `since` keeps the broad rule.
1018pub fn covers(q: &crate::ask::Question, n: &Notice) -> bool {
1019    // Delivery alone proves no relationship to a later failure in the same
1020    // run pass. Retired questions only retain coverage already recorded while
1021    // they were open; never infer a cause match from the run ID or pass start.
1022    let recorded_coverage =
1023        q.routed_chat.is_some() && n.covered_by.as_deref() == Some(q.id.as_str());
1024    if !(q.status.open() || recorded_coverage) || !n.subjects.contains(&q.run) {
1025        return false;
1026    }
1027    if n.since.is_some_and(|since| q.asked_at < since) {
1028        return false;
1029    }
1030    let about_task = n.key.starts_with("task:") || n.key.starts_with("handover:");
1031    let about_run = n.key.starts_with("run:");
1032    match q.node.as_str() {
1033        crate::bump::NOTICE_NODE | crate::land::APPROVAL_NODE => false,
1034        _ if q.run_names_task() => about_task,
1035        _ => about_run,
1036    }
1037}
1038
1039fn covering_question(home: &Path, n: &Notice) -> Option<String> {
1040    if n.subjects.is_empty() {
1041        return None;
1042    }
1043    crate::ask::Questions::at(home.join("questions"))
1044        .list()
1045        .into_iter()
1046        .find(|q| covers(q, n))
1047        .map(|q| q.id)
1048}
1049
1050/// A question was just filed for `run`: quiet the unread notices about it.
1051pub fn quiet_for(home: &Path, question: &crate::ask::Question) {
1052    Notices::at(home.join("notifications")).cover(question);
1053}
1054
1055/// [`raise`] into an explicit magi home, for callers that already carry one
1056/// (the janitor) and so must not reach for the process-global.
1057pub fn raise_in(home: &Path, notice: Notice) {
1058    raise_in_with(home, notice, &|page| send_page(page, None));
1059}
1060
1061/// [`raise_in`] with the pager injected. `send` is called only when the raise
1062/// should page (see [`Notice::raise_again`]); it must not block.
1063pub fn raise_in_with(home: &Path, mut notice: Notice, send: &dyn Fn(Page)) {
1064    if let Some(q) = covering_question(home, &notice) {
1065        notice.covered_by = Some(q);
1066    }
1067    match Notices::at(home.join("notifications")).raise_paged(notice) {
1068        Ok((n, true)) => send(Page {
1069            run: n
1070                .related_run
1071                .clone()
1072                .or_else(|| match &n.link {
1073                    Some(Link::Run { id }) => Some(id.clone()),
1074                    _ => None,
1075                })
1076                .unwrap_or_default(),
1077            summary: n.message.clone(),
1078        }),
1079        Ok(_) => {}
1080        Err(e) => tracing::warn!("could not file a notification: {e:#}"),
1081    }
1082}
1083
1084#[cfg(test)]
1085mod tests {
1086    use super::*;
1087
1088    #[test]
1089    fn notification_details_omit_blanks_and_explain_process_results() {
1090        assert_eq!(detail(None), None);
1091        assert_eq!(detail(Some(" \n\t")), None);
1092        assert_eq!(detail(Some(" concern ")), Some("concern"));
1093        assert_eq!(exit_status(Some(7)), "exit code 7");
1094        assert_eq!(exit_status(None), "exit code unavailable");
1095        assert_eq!(
1096            turn_status(None, true, true),
1097            "exit code unavailable; time limit reached; quota exhausted"
1098        );
1099        let mut task = held_task();
1100        task.hold_reason = Some("   ".into());
1101        task.last_error = Some(" failure recorded ".into());
1102        assert!(
1103            task_held(&task)
1104                .unwrap()
1105                .message
1106                .ends_with("failure recorded.")
1107        );
1108    }
1109
1110    #[test]
1111    fn resumption_notice_and_external_page_are_informational() {
1112        let dir = tempfile::tempdir().unwrap();
1113        let n = conductor_resumption(
1114            "t1",
1115            "t1",
1116            Some("Inspection remains a concern."),
1117            &["dep1".into(), " ".into()],
1118            &[],
1119            false,
1120        );
1121        assert!(n.message.contains("Concern: Inspection remains a concern."));
1122        assert!(n.message.contains("Proposed dependencies: dep1."));
1123        assert!(n.message.contains("its state was not changed"));
1124        assert!(
1125            n.message
1126                .contains("Existing holds and questions still apply")
1127        );
1128        assert!(n.message.contains("No reply to this notice is needed"));
1129        assert_eq!(n.link, Some(Link::Task { id: "t1".into() }));
1130        let sent = pages(dir.path(), n.clone());
1131        assert_eq!(sent[0].summary, n.message);
1132        assert!(pages(dir.path(), n).is_empty(), "repeat remains silent");
1133        let empty = conductor_resumption("t2", "t2", Some(" "), &[], &[], false);
1134        for absent in [
1135            "Concern:",
1136            "Proposed dependencies:",
1137            "Some(",
1138            "None",
1139            "[]",
1140            "question:",
1141            "evidence:",
1142            "?",
1143        ] {
1144            assert!(!empty.message.contains(absent), "{absent}");
1145        }
1146    }
1147
1148    #[test]
1149    fn only_currently_open_registered_questions_have_response_links() {
1150        let dir = tempfile::tempdir().unwrap();
1151        let q = ask(dir.path(), "t1", "conduct");
1152        let questions = crate::ask::Questions::at(dir.path().join("questions"));
1153        let mut n = Notice::warn("task:t1", "held");
1154        n.covered_by = Some(q.id.clone());
1155        assert_eq!(answerable_question(&n, &questions), Some(q.id.clone()));
1156        questions
1157            .update(&q.id, |q| {
1158                q.answer(crate::ask::Answer::Text("continue".into()))
1159            })
1160            .unwrap();
1161        assert_eq!(answerable_question(&n, &questions), None);
1162        let abandoned = ask(dir.path(), "t2", "conduct");
1163        let failure = Notice::warn(&format!("question:{}", abandoned.id), "delivery failed");
1164        assert_eq!(
1165            answerable_question(&failure, &questions),
1166            Some(abandoned.id.clone())
1167        );
1168        questions
1169            .update(&abandoned.id, |q| {
1170                q.abandon("run gone");
1171                Ok(())
1172            })
1173            .unwrap();
1174        assert_eq!(answerable_question(&failure, &questions), None);
1175        n.covered_by = Some("missing".into());
1176        assert_eq!(answerable_question(&n, &questions), None);
1177        assert_eq!(
1178            answerable_question(
1179                &conductor_resumption("t1", "t1", None, &[], &[], false),
1180                &questions
1181            ),
1182            None
1183        );
1184    }
1185
1186    #[test]
1187    fn legacy_deputy_warnings_are_hidden_without_hiding_other_namespaces() {
1188        let (_dir, s) = store();
1189        // No question lookup: these may belong to deleted or settled questions.
1190        for key in ["deputy:open", "deputy:answered", "deputy:deleted"] {
1191            s.raise(Notice::warn(key, "old exhausted warning")).unwrap();
1192        }
1193        let keys = ["deputy-turn:open", "deputyish:open", "deputy", "task:open"];
1194        for key in keys {
1195            s.raise(Notice::warn(key, "still relevant")).unwrap();
1196        }
1197        assert_eq!(s.list().len(), keys.len());
1198        assert_eq!(s.count_unread(), keys.len());
1199        assert!(s.list().iter().all(|n| keys.contains(&n.key.as_str())));
1200        assert_eq!(s.mark_all_read().unwrap(), keys.len());
1201        assert_eq!(s.count_unread(), 0);
1202        assert!(s.get(&id_of("deputy:open")).unwrap().unread());
1203    }
1204
1205    #[test]
1206    fn obsolete_deputy_warnings_are_pruned_before_current_notices() {
1207        let (_dir, s) = store();
1208        for i in 0..CAP {
1209            let id = s
1210                .raise(Notice::info(&format!("current:{i}"), "keep"))
1211                .unwrap()
1212                .id;
1213            s.dismiss(&id).unwrap();
1214        }
1215        // Even a newer unread legacy warning cannot evict a current tombstone.
1216        s.raise(Notice::warn("deputy:old", "obsolete")).unwrap();
1217        assert!(s.get(&id_of("deputy:old")).is_err());
1218        assert_eq!(s.all().len(), CAP);
1219        assert!(s.all().iter().all(|n| n.key.starts_with("current:")));
1220    }
1221
1222    #[test]
1223    fn process_exit_status_is_readable_even_without_a_code() {
1224        assert_eq!(exit_status(Some(1)), "exit code 1");
1225        assert_eq!(exit_status(None), "exit code unavailable");
1226    }
1227
1228    fn store() -> (tempfile::TempDir, Notices) {
1229        let dir = tempfile::tempdir().unwrap();
1230        let s = Notices::at(dir.path().join("notifications"));
1231        (dir, s)
1232    }
1233
1234    #[test]
1235    fn a_red_merge_notice_is_keyed_on_the_run_and_raised_once() {
1236        let (_d, s) = store();
1237        let n = merged_red("run-1", "Merged a/b PR #1 with red checks: x (u)");
1238        assert_eq!(n.key, "merged-red:run-1");
1239        assert_eq!(
1240            n.link,
1241            Some(Link::Run {
1242                id: "run-1".to_owned()
1243            })
1244        );
1245        s.raise(merged_red(
1246            "run-1",
1247            "Merged a/b PR #1 with red checks: x (u)",
1248        ))
1249        .unwrap();
1250        s.raise(merged_red(
1251            "run-1",
1252            "Merged a/b PR #1 with red checks: x (u)",
1253        ))
1254        .unwrap();
1255        assert_eq!(s.list().len(), 1);
1256    }
1257
1258    #[test]
1259    fn producer_cause_transitions_page_once_without_using_message_prose() {
1260        let (dir, store) = store();
1261        let sent = std::cell::RefCell::new(Vec::new());
1262        let raise = |cause: &str, message: &str| {
1263            raise_in_with(
1264                dir.path(),
1265                Notice::warn("release-pr:pr", message).cause(cause),
1266                &|p| sent.borrow_mut().push(p),
1267            )
1268        };
1269        raise("approval:h1", "approval is needed");
1270        let id = id_of("release-pr:pr");
1271        store.mark_read(&id).unwrap();
1272        raise("approval:h1", "approval wording updated");
1273        assert!(!store.get(&id).unwrap().unread());
1274        raise("merge-failed:h1", "merge refused");
1275        assert!(store.get(&id).unwrap().unread());
1276        assert_eq!(sent.borrow().len(), 2);
1277        assert_eq!(store.get(&id).unwrap().generation, 2);
1278        store.mark_read(&id).unwrap();
1279        raise("merge-failed:h1", "merge refusal wording updated");
1280        assert_eq!(sent.borrow().len(), 2);
1281        assert!(!store.get(&id).unwrap().unread());
1282    }
1283
1284    #[test]
1285    fn a_repeat_updates_evidence_without_relighting_it() {
1286        let now = Timestamp::now();
1287        let mut n = Notice::warn("task:1", "held");
1288        n.mark_read(now);
1289        n.raise_again(&Notice::warn("task:1", "held"), now);
1290        assert_eq!(n.count, 2);
1291        assert!(n.read_at.is_some(), "identical repeat stays read");
1292        n.raise_again(&Notice::warn("task:1", "held differently"), now);
1293        assert!(!n.unread());
1294        assert_eq!(n.message, "held differently");
1295    }
1296
1297    #[test]
1298    fn a_rise_in_severity_resurrects_a_dismissed_notice_but_a_repeat_does_not() {
1299        let now = Timestamp::now();
1300        let mut n = Notice::warn("task:x", "m");
1301        n.dismiss(now);
1302        n.raise_again(&Notice::warn("task:x", "m"), now);
1303        assert!(n.dismissed_at.is_some(), "tombstone holds");
1304        n.raise_again(&Notice::error("k", "m"), now);
1305        assert!(n.unread());
1306        assert_eq!(n.severity, Severity::Error);
1307        // Severity never falls on a repeat.
1308        n.raise_again(&Notice::info("k", "other"), now);
1309        assert_eq!(n.severity, Severity::Error);
1310    }
1311
1312    #[test]
1313    fn adding_link_on_repeat_updates_link_without_relighting_read_or_dismissed() {
1314        let now = Timestamp::now();
1315        // 1. Read notice
1316        let mut read_notice = Notice::warn("task:1", "held");
1317        read_notice.mark_read(now);
1318        let link_to_add = Link::Task { id: "1".to_owned() };
1319        let again = Notice::warn("task:1", "held").link(link_to_add.clone());
1320        let relit = read_notice.raise_again(&again, now);
1321        assert!(!relit, "link-only update must not page/relight");
1322        assert!(read_notice.read_at.is_some(), "read notice stays read");
1323        assert_eq!(read_notice.link, Some(link_to_add.clone()));
1324
1325        // 2. Dismissed notice
1326        let mut dismissed_notice = Notice::warn("task:2", "held");
1327        dismissed_notice.dismiss(now);
1328        let link_to_add_2 = Link::Task { id: "2".to_owned() };
1329        let again_2 = Notice::warn("task:2", "held").link(link_to_add_2.clone());
1330        let relit_2 = dismissed_notice.raise_again(&again_2, now);
1331        assert!(!relit_2, "link-only update must not page/relight");
1332        assert!(dismissed_notice.dismissed_at.is_some(), "tombstone holds");
1333        assert_eq!(dismissed_notice.link, Some(link_to_add_2));
1334    }
1335
1336    #[test]
1337    fn identical_raises_share_one_file() {
1338        let (_d, s) = store();
1339        for _ in 0..5 {
1340            s.raise(Notice::error("run:abc", "blocked")).unwrap();
1341        }
1342        let all = s.list();
1343        assert_eq!(all.len(), 1);
1344        assert_eq!(all[0].count, 5);
1345        assert_eq!(s.count_unread(), 1);
1346    }
1347
1348    #[test]
1349    fn transitions_persist_and_dismissed_leave_the_list() {
1350        let (_d, s) = store();
1351        let a = s.raise(Notice::info("a", "one")).unwrap();
1352        let b = s.raise(Notice::warn("b", "two")).unwrap();
1353        assert_eq!(s.count_unread(), 2);
1354        s.mark_read(&a.id).unwrap();
1355        assert_eq!(s.count_unread(), 1);
1356        s.dismiss(&b.id).unwrap();
1357        assert_eq!(s.count_unread(), 0);
1358        assert_eq!(s.list().len(), 1);
1359        s.raise(Notice::warn("b", "two")).unwrap();
1360        assert_eq!(
1361            s.list().len(),
1362            1,
1363            "a tombstone survives a same-message raise"
1364        );
1365        s.raise(Notice::info("c", "three")).unwrap();
1366        assert_eq!(s.mark_all_read().unwrap(), 1);
1367        assert_eq!(s.count_unread(), 0);
1368    }
1369
1370    #[test]
1371    fn the_list_is_newest_first() {
1372        let (_d, s) = store();
1373        let mut old = Notice::info("old", "old");
1374        old.last_at = "2020-01-01T00:00:00Z".parse().unwrap();
1375        s.put(&old).unwrap();
1376        s.raise(Notice::info("new", "new")).unwrap();
1377        let keys: Vec<_> = s.list().into_iter().map(|n| n.key).collect();
1378        assert_eq!(keys, ["new", "old"]);
1379    }
1380
1381    #[test]
1382    fn the_cap_prunes_dismissed_then_read_then_oldest() {
1383        let (_d, s) = store();
1384        let keep = s.raise(Notice::error("keep", "unread")).unwrap();
1385        let gone = s.raise(Notice::info("gone", "dismissed")).unwrap();
1386        s.dismiss(&gone.id).unwrap();
1387        for i in 0..CAP - 1 {
1388            s.raise(Notice::info(&format!("k{i}"), "x")).unwrap();
1389        }
1390        let files = std::fs::read_dir(s.root()).unwrap().count();
1391        assert_eq!(files, CAP);
1392        assert!(
1393            s.get(&keep.id).is_ok(),
1394            "an unread notice outlives a dismissed one"
1395        );
1396        assert!(s.get(&gone.id).is_err(), "the dismissed one went first");
1397    }
1398
1399    #[test]
1400    fn revision_moves_on_write_and_on_removal() {
1401        let (_d, s) = store();
1402        assert_eq!(s.revision(), 0);
1403        let a = s.raise(Notice::info("a", "m")).unwrap();
1404        let r1 = s.revision();
1405        assert_ne!(r1, 0);
1406        s.raise(Notice::info("b", "m")).unwrap();
1407        let r2 = s.revision();
1408        assert_ne!(r1, r2);
1409        std::fs::remove_file(s.path_of(&a.id)).unwrap();
1410        assert_ne!(s.revision(), r2);
1411    }
1412
1413    #[test]
1414    fn ids_are_stable_and_untrusted_ids_are_refused() {
1415        assert_eq!(id_of("run:1"), id_of("run:1"));
1416        assert_ne!(id_of("run:1"), id_of("run-1"));
1417        assert!(valid_id(&id_of("release-bump:20260101-abc")));
1418        let (_d, s) = store();
1419        for bad in ["", "../x", "a/b", "A", "x.json"] {
1420            assert!(s.get(bad).is_err(), "{bad}");
1421        }
1422    }
1423
1424    #[test]
1425    fn a_newer_schema_is_refused_and_an_older_reads() {
1426        let (_d, s) = store();
1427        let mut n = Notice::info("k", "m");
1428        n.schema = SCHEMA + 1;
1429        s.put(&n).unwrap();
1430        assert!(s.get(&n.id).is_err());
1431        let old = r#"{"id":"x-1","key":"x","severity":"warn","message":"m",
1432            "first_at":"2020-01-01T00:00:00Z","last_at":"2020-01-01T00:00:00Z"}"#;
1433        std::fs::write(s.path_of("x-1"), old).unwrap();
1434        assert_eq!(s.get("x-1").unwrap().count, 1);
1435    }
1436
1437    fn state(status: crate::run::RunStatus) -> crate::run::RunState {
1438        let mut st = crate::run::RunState::new(
1439            std::path::PathBuf::from("/repo"),
1440            "main".to_owned(),
1441            "abc1234def".to_owned(),
1442            "task".to_owned(),
1443            crate::config::Config::default(),
1444        );
1445        st.status = status;
1446        st
1447    }
1448
1449    #[test]
1450    fn a_run_that_ended_badly_is_news_unless_it_only_parked() {
1451        use crate::run::RunStatus;
1452        for bad in [RunStatus::Blocked, RunStatus::Stalled, RunStatus::Failed] {
1453            let st = state(bad);
1454            let n = run_ended(&st).expect("news");
1455            assert_eq!(n.key, format!("run:{}", st.id));
1456            assert_eq!(n.severity, Severity::Error);
1457        }
1458        assert!(run_ended(&state(RunStatus::Merged)).is_none());
1459        let mut parked = state(RunStatus::Stalled);
1460        parked.parked = true;
1461        assert!(run_ended(&parked).is_none());
1462    }
1463
1464    #[test]
1465    fn concurrent_writers_of_one_key_all_succeed() {
1466        let (_d, s) = store();
1467        let handles: Vec<_> = (0..8)
1468            .map(|_| {
1469                let s = s.clone();
1470                std::thread::spawn(move || {
1471                    for _ in 0..20 {
1472                        s.raise(Notice::warn("same", "m")).unwrap();
1473                    }
1474                })
1475            })
1476            .collect();
1477        for h in handles {
1478            h.join().unwrap();
1479        }
1480        assert_eq!(s.list().len(), 1);
1481        let stray = std::fs::read_dir(s.root())
1482            .unwrap()
1483            .flatten()
1484            .filter(|e| e.path().extension().is_some_and(|x| x == "tmp"))
1485            .count();
1486        assert_eq!(stray, 0);
1487    }
1488
1489    #[test]
1490    fn a_dead_holders_lock_is_broken_and_no_lock_is_left_behind() {
1491        let (_d, s) = store();
1492        let n = s.raise(Notice::info("k", "m")).unwrap();
1493        let lock = s.root().join(format!("{}.lock", n.id));
1494        std::fs::write(&lock, "").unwrap();
1495        let old = std::time::SystemTime::now() - std::time::Duration::from_secs(60);
1496        std::fs::File::options()
1497            .write(true)
1498            .open(&lock)
1499            .unwrap()
1500            .set_modified(old)
1501            .unwrap();
1502        s.mark_read(&n.id).unwrap();
1503        assert!(!lock.exists());
1504        assert!(s.get(&n.id).unwrap().read_at.is_some());
1505    }
1506
1507    #[test]
1508    fn only_a_machine_hold_is_a_task_notice() {
1509        let mut t = crate::queue::Task::new(
1510            "t".to_owned(),
1511            "do it".to_owned(),
1512            std::path::PathBuf::from("/repo"),
1513            crate::queue::Source::Human,
1514        );
1515        assert!(task_held(&t).is_none());
1516        t.hold_manual(None);
1517        assert!(task_held(&t).is_none(), "the operator's own hold");
1518        t.hold_machine(Some("missing blocker".to_owned()));
1519        let n = task_held(&t).expect("machine hold");
1520        assert_eq!(n.key, format!("task:{}", t.id));
1521        assert!(n.message.contains("missing blocker"));
1522    }
1523
1524    fn held_task() -> crate::queue::Task {
1525        let mut t = crate::queue::Task::new(
1526            "t".to_owned(),
1527            "do it".to_owned(),
1528            std::path::PathBuf::from("/repo"),
1529            crate::queue::Source::Human,
1530        );
1531        t.hold_machine(Some("branch b is checked out".to_owned()));
1532        t
1533    }
1534
1535    fn ask(home: &Path, run: &str, node: &str) -> crate::ask::Question {
1536        let mut q = crate::ask::Question::new(
1537            run.to_owned(),
1538            node.to_owned(),
1539            "conductor".to_owned(),
1540            "cannot resume".to_owned(),
1541            String::new(),
1542            Vec::new(),
1543        );
1544        crate::ask::Questions::at(home.join("questions"))
1545            .put(&mut q)
1546            .unwrap();
1547        q
1548    }
1549
1550    fn unread(home: &Path) -> usize {
1551        Notices::at(home.join("notifications")).count_unread()
1552    }
1553
1554    #[test]
1555    fn a_hold_with_an_open_question_for_the_task_pages_once() {
1556        let dir = tempfile::tempdir().unwrap();
1557        let mut t = held_task();
1558        let q = ask(dir.path(), &t.id, "conduct");
1559        crate::queue::Queue::at(dir.path().join("queue"))
1560            .put(&mut t)
1561            .unwrap();
1562        let s = Notices::at(dir.path().join("notifications"));
1563        assert_eq!(s.count_unread(), 0);
1564        let n = s.list().pop().expect("the record is kept");
1565        assert_eq!(n.covered_by.as_deref(), Some(q.id.as_str()));
1566        assert!(n.message.contains("checked out"));
1567    }
1568
1569    #[test]
1570    fn a_refused_handover_without_a_question_pages_once() {
1571        let dir = tempfile::tempdir().unwrap();
1572        let q = crate::queue::Queue::at(dir.path().join("queue"));
1573        let mut t = held_task();
1574        t.hold_for_handover(Some("magi/x/A".to_owned()), "checked out".to_owned());
1575        q.put(&mut t).unwrap();
1576        q.put(&mut t).unwrap();
1577        let s = Notices::at(dir.path().join("notifications"));
1578        assert_eq!(s.list().len(), 1);
1579        assert_eq!(s.list()[0].key, format!("task:{}", t.id));
1580        assert_eq!(s.count_unread(), 1);
1581    }
1582
1583    #[test]
1584    fn an_older_question_does_not_silence_a_new_hold() {
1585        for node in ["deps", "conduct", "triage"] {
1586            let dir = tempfile::tempdir().unwrap();
1587            let mut t = crate::queue::Task::new(
1588                "t".to_owned(),
1589                "do it".to_owned(),
1590                std::path::PathBuf::from("/repo"),
1591                crate::queue::Source::Human,
1592            );
1593            let old = ask(dir.path(), &t.id, node);
1594            std::thread::sleep(std::time::Duration::from_millis(5));
1595            t.hold_machine(Some("new failure".to_owned()));
1596            crate::queue::Queue::at(dir.path().join("queue"))
1597                .put(&mut t)
1598                .unwrap();
1599            let s = Notices::at(dir.path().join("notifications"));
1600            let n = s.list().pop().unwrap();
1601            assert_eq!(n.covered_by, None, "{node} question {}", old.id);
1602            assert_eq!(s.count_unread(), 1);
1603        }
1604    }
1605
1606    #[test]
1607    fn a_legacy_covered_notice_pages_for_a_new_uncovered_hold() {
1608        let (_d, s) = store();
1609        let mut legacy = Notice::warn("task:x", "held");
1610        legacy.covered_by = Some("q1".to_owned());
1611        let id = legacy.id.clone();
1612        s.raise(legacy).unwrap();
1613        assert!(!s.get(&id).unwrap().unread());
1614        s.raise(Notice::warn("task:x", "held").since(Some(Timestamp::now())))
1615            .unwrap();
1616        let n = s.get(&id).unwrap();
1617        assert!(n.unread());
1618        assert_eq!(n.covered_by, None);
1619    }
1620
1621    #[test]
1622    fn a_newer_cause_relights_a_dismissed_notice() {
1623        let (_d, s) = store();
1624        let t0 = Timestamp::now();
1625        let first = Notice::warn("task:x", "held").since(Some(t0));
1626        let id = first.id.clone();
1627        s.raise(first).unwrap();
1628        s.dismiss(&id).unwrap();
1629        s.raise(Notice::warn("task:x", "held").since(Some(t0)))
1630            .unwrap();
1631        assert!(!s.get(&id).unwrap().unread(), "same cause stays dismissed");
1632        let later = t0 + std::time::Duration::from_secs(1);
1633        s.raise(Notice::warn("task:x", "held").since(Some(later)))
1634            .unwrap();
1635        assert!(s.get(&id).unwrap().unread());
1636    }
1637
1638    #[test]
1639    fn a_hold_with_no_question_still_notifies() {
1640        let dir = tempfile::tempdir().unwrap();
1641        let mut t = held_task();
1642        crate::queue::Queue::at(dir.path().join("queue"))
1643            .put(&mut t)
1644            .unwrap();
1645        assert_eq!(unread(dir.path()), 1);
1646    }
1647
1648    #[test]
1649    fn a_question_for_another_task_does_not_suppress() {
1650        let dir = tempfile::tempdir().unwrap();
1651        let mut t = held_task();
1652        ask(dir.path(), "some-other-task", "conduct");
1653        crate::queue::Queue::at(dir.path().join("queue"))
1654            .put(&mut t)
1655            .unwrap();
1656        assert_eq!(unread(dir.path()), 1);
1657    }
1658
1659    #[test]
1660    fn a_question_filed_after_the_hold_quiets_it_once() {
1661        let dir = tempfile::tempdir().unwrap();
1662        let mut t = held_task();
1663        crate::queue::Queue::at(dir.path().join("queue"))
1664            .put(&mut t)
1665            .unwrap();
1666        assert_eq!(unread(dir.path()), 1);
1667        let q = ask(dir.path(), &t.id, "conduct");
1668        assert_eq!(unread(dir.path()), 0);
1669        // A later update of the same question does not run the hook again.
1670        let s = Notices::at(dir.path().join("notifications"));
1671        let id = s.list()[0].id.clone();
1672        s.update(&id, |n| n.read_at = None).unwrap();
1673        let mut again = crate::ask::Questions::at(dir.path().join("questions"))
1674            .get(&q.id)
1675            .unwrap();
1676        crate::ask::Questions::at(dir.path().join("questions"))
1677            .put(&mut again)
1678            .unwrap();
1679        assert_eq!(unread(dir.path()), 1);
1680    }
1681
1682    #[test]
1683    fn a_blocked_run_with_an_open_question_is_quiet() {
1684        let dir = tempfile::tempdir().unwrap();
1685        ask(dir.path(), "run-1", "implement");
1686        raise_in(
1687            dir.path(),
1688            Notice::error("run:run-1", "Run r ended blocked.").about(["run-1"]),
1689        );
1690        assert_eq!(unread(dir.path()), 0);
1691        raise_in(
1692            dir.path(),
1693            Notice::error("run:run-2", "Run r ended blocked.").about(["run-2"]),
1694        );
1695        assert_eq!(unread(dir.path()), 1);
1696    }
1697
1698    #[test]
1699    fn escalation_makes_a_covered_notice_unread_again() {
1700        let dir = tempfile::tempdir().unwrap();
1701        ask(dir.path(), "t1", "conduct");
1702        raise_in(dir.path(), Notice::warn("task:t1", "held").about(["t1"]));
1703        assert_eq!(unread(dir.path()), 0);
1704        raise_in(dir.path(), Notice::error("task:t1", "held").about(["t1"]));
1705        assert_eq!(unread(dir.path()), 1);
1706    }
1707
1708    #[test]
1709    fn covers_matches_run_exactly_and_ignores_release_questions() {
1710        let dir = tempfile::tempdir().unwrap();
1711        let n = Notice::warn("task:x", "m").about(["t1"]);
1712        let q = ask(dir.path(), "t1", "conduct");
1713        assert!(covers(&q, &n));
1714        assert!(!covers(&q, &Notice::warn("task:x", "m")));
1715        assert!(!covers(&q, &Notice::warn("task:x", "m").about(["t"])));
1716        let release = ask(dir.path(), "t1", crate::bump::NOTICE_NODE);
1717        assert!(!covers(&release, &n));
1718    }
1719
1720    #[test]
1721    fn a_run_question_does_not_swallow_a_task_hold_and_vice_versa() {
1722        let dir = tempfile::tempdir().unwrap();
1723        ask(dir.path(), "t1", "implement");
1724        raise_in(dir.path(), Notice::warn("task:t1", "held").about(["t1"]));
1725        assert_eq!(unread(dir.path()), 1);
1726        ask(dir.path(), "r1", "conduct");
1727        raise_in(dir.path(), Notice::error("run:r1", "ended").about(["r1"]));
1728        assert_eq!(unread(dir.path()), 2);
1729    }
1730
1731    #[test]
1732    fn a_conduct_question_covers_the_hold_however_late_it_was_filed() {
1733        let dir = tempfile::tempdir().unwrap();
1734        let mut q = crate::ask::Question::new(
1735            "t1".to_owned(),
1736            "conduct".to_owned(),
1737            "conductor".to_owned(),
1738            "s".to_owned(),
1739            String::new(),
1740            Vec::new(),
1741        );
1742        q.asked_at = Timestamp::from_second(Timestamp::now().as_second() - 3600).unwrap();
1743        crate::ask::Questions::at(dir.path().join("questions"))
1744            .put(&mut q)
1745            .unwrap();
1746        raise_in(dir.path(), Notice::warn("task:t1", "held").about(["t1"]));
1747        assert_eq!(unread(dir.path()), 0);
1748    }
1749
1750    fn pages(home: &Path, n: Notice) -> Vec<Page> {
1751        let sent = std::cell::RefCell::new(Vec::new());
1752        raise_in_with(home, n, &|p| sent.borrow_mut().push(p));
1753        sent.into_inner()
1754    }
1755
1756    #[test]
1757    fn a_page_fires_on_new_causes_and_escalation_only() {
1758        let dir = tempfile::tempdir().unwrap();
1759        let h = dir.path();
1760        let n = || Notice::warn("run:p1", "stopped").link(Link::Run { id: "p1".into() });
1761        let first = pages(h, n());
1762        assert_eq!(
1763            first,
1764            vec![Page {
1765                run: "p1".into(),
1766                summary: "stopped".into()
1767            }]
1768        );
1769        assert!(pages(h, n()).is_empty(), "identical re-raise is silent");
1770        assert!(pages(h, Notice::warn("run:p1", "stopped again")).is_empty());
1771        assert_eq!(pages(h, Notice::error("run:p1", "stopped again")).len(), 1);
1772        assert!(pages(h, Notice::error("run:p1", "stopped again")).is_empty());
1773    }
1774
1775    #[test]
1776    fn a_non_run_link_leaves_run_empty() {
1777        let dir = tempfile::tempdir().unwrap();
1778        let n = Notice::warn("task:t1", "held").link(Link::Task { id: "t1".into() });
1779        assert_eq!(pages(dir.path(), n)[0].run, "");
1780    }
1781
1782    #[test]
1783    fn a_dismissed_tombstone_ignores_wording_changes() {
1784        let dir = tempfile::tempdir().unwrap();
1785        let h = dir.path();
1786        pages(h, Notice::warn("disk:r", "low"));
1787        Notices::at(h.join("notifications"))
1788            .dismiss(&id_of("disk:r"))
1789            .unwrap();
1790        assert!(pages(h, Notice::warn("disk:r", "low")).is_empty());
1791        assert!(pages(h, Notice::warn("disk:r", "lower")).is_empty());
1792    }
1793
1794    #[test]
1795    fn a_newer_cause_pages_again() {
1796        let dir = tempfile::tempdir().unwrap();
1797        let h = dir.path();
1798        let at = |s: i64| Some(Timestamp::from_second(s).unwrap());
1799        let n = |s| Notice::warn("task:t2", "held").since(at(s));
1800        assert_eq!(pages(h, n(100)).len(), 1);
1801        assert!(pages(h, n(100)).is_empty());
1802        assert_eq!(pages(h, n(200)).len(), 1);
1803    }
1804
1805    #[test]
1806    fn a_covered_notice_does_not_page() {
1807        let dir = tempfile::tempdir().unwrap();
1808        let h = dir.path();
1809        let q = ask(h, "r9", "implement");
1810        let n = || Notice::error("run:r9", "ended").about(["r9".to_owned()]);
1811        assert!(pages(h, n()).is_empty());
1812        let stored = Notices::at(h.join("notifications"))
1813            .get(&id_of("run:r9"))
1814            .unwrap();
1815        assert_eq!(stored.covered_by.as_deref(), Some(q.id.as_str()));
1816        assert!(pages(h, n()).is_empty());
1817    }
1818
1819    #[test]
1820    fn a_failing_sender_path_never_propagates() {
1821        // A pager that cannot run its command only warns.
1822        let notify = crate::config::Notify {
1823            command: vec!["magi-no-such-program-xyz".into()],
1824        };
1825        let rt = tokio::runtime::Builder::new_current_thread()
1826            .enable_all()
1827            .build()
1828            .unwrap();
1829        let r = rt.block_on(crate::ask::notify_text(&notify, "", "x"));
1830        assert!(r.is_err());
1831        // `send_page` without an installed pager is a no-op and cannot panic.
1832        send_page(
1833            Page {
1834                run: String::new(),
1835                summary: "x".into(),
1836            },
1837            Some(notify),
1838        );
1839    }
1840
1841    #[test]
1842    fn formatting_helpers_clean_text_and_lists() {
1843        assert_eq!(clean_text(None), None);
1844        assert_eq!(clean_text(Some("")), None);
1845        assert_eq!(clean_text(Some("   ")), None);
1846        assert_eq!(clean_text(Some("(no reason given)")), None);
1847        assert_eq!(clean_text(Some("(none)")), None);
1848        assert_eq!(clean_text(Some("  real reason  ")), Some("real reason"));
1849
1850        let empty: Vec<String> = Vec::new();
1851        assert_eq!(format_list(&empty), None);
1852        assert_eq!(format_list(&["", "  "]), None);
1853        assert_eq!(format_list(&["task-1"]), Some("task-1".into()));
1854        assert_eq!(format_list(&["a", "b", "c"]), Some("a, b, c".into()));
1855
1856        assert_eq!(ensure_sentence(""), "");
1857        assert_eq!(ensure_sentence("hello"), "hello.");
1858        assert_eq!(ensure_sentence("hello."), "hello.");
1859        assert_eq!(ensure_sentence("hello.."), "hello.");
1860    }
1861
1862    #[test]
1863    fn conductor_resumption_notice_is_readable_and_non_soliciting() {
1864        let task_id = "20261017-000000-abcd";
1865        let task_short = "abcd";
1866        let reason = "awaiting manual review";
1867        let blocked_by = vec!["dep-1".to_owned(), "dep-2".to_owned()];
1868        let evidence = vec!["check-failed".to_owned()];
1869        let has_question = true;
1870
1871        let notice = conductor_resumption(
1872            task_id,
1873            task_short,
1874            Some(reason),
1875            &blocked_by,
1876            &evidence,
1877            has_question,
1878        );
1879
1880        assert_eq!(notice.key, format!("conductor-resumption:{task_id}"));
1881        assert_eq!(notice.severity, Severity::Warn);
1882        assert_eq!(
1883            notice.link,
1884            Some(Link::Task {
1885                id: task_id.to_owned()
1886            })
1887        );
1888        assert_eq!(notice.subjects, vec![task_id.to_owned()]);
1889
1890        let msg = &notice.message;
1891        // Contains concise understandable prose explaining what happened
1892        assert!(msg.contains("Conductor raised a concern for task abcd after explicit resumption"));
1893        assert!(msg.contains("Concern: awaiting manual review."));
1894        assert!(msg.contains("Proposed dependencies: dep-1, dep-2."));
1895        assert!(msg.contains("Cited evidence: check-failed."));
1896        // Explicitly clarifies that no reply is required
1897        assert!(msg.contains("No reply to this notice is needed."));
1898        // No raw Debug formatting or dumped structures
1899        assert!(!msg.contains("Some("));
1900        assert!(!msg.contains("None"));
1901        assert!(!msg.contains("[]"));
1902        assert!(!msg.contains("{:?}"));
1903    }
1904
1905    #[test]
1906    fn conductor_resumption_notice_omits_empty_details() {
1907        let task_id = "20261017-000000-abcd";
1908        let notice = conductor_resumption(task_id, "abcd", None, &[], &[], false);
1909
1910        let msg = &notice.message;
1911        assert!(!msg.contains("Concern:"));
1912        assert!(!msg.contains("Proposed dependencies:"));
1913        assert!(!msg.contains("Cited evidence:"));
1914        assert!(!msg.contains("unregistered question"));
1915        assert!(!msg.contains("None"));
1916        assert!(!msg.contains("[]"));
1917        assert!(msg.contains("No reply to this notice is needed."));
1918    }
1919}