Skip to main content

second_brain_api/
health.rs

1//! The daemon's own view of whether it is capturing and indexing. The
2//! watcher, the outbox drain and the repair loop report what they see; each
3//! report re-assesses the state, so `GET /health` only clones the result.
4
5use std::path::PathBuf;
6use std::sync::Mutex;
7
8use chrono::{DateTime, Duration, Utc};
9use second_brain_core::config::AlertConfig;
10use second_brain_core::keys::{ASIDE_RETENTION_MICROS, MovedAside};
11use second_brain_core::sqlite_store::{FailedCounts, FloorHealth, VacuumSpace};
12use second_brain_sync::keyring::IdentityStatus;
13use serde::Serialize;
14use tokio::sync::mpsc::UnboundedSender;
15use uuid::Uuid;
16
17use crate::snapshot_merge::MergeStatus;
18use crate::value_metrics::{
19    PeerOutcome, RecallEvent, ResidentSize, StoreSize, ValueFacts, ValueProblems, ValueReport,
20    WINDOW,
21};
22
23const CAPTURE_LAG_DEGRADED: Duration = Duration::minutes(15);
24const CAPTURE_LAG_FAILING: Duration = Duration::hours(2);
25const BACKLOG_DEGRADED: Duration = Duration::minutes(15);
26const BACKLOG_FAILING: Duration = Duration::hours(2);
27/// Three missed transcript scans (see `watcher::SCAN_INTERVAL`).
28const WATCHER_SILENT: Duration = Duration::minutes(15);
29/// A scheduled repair holds the drain thread for minutes on a large store.
30const DRAIN_SILENT: Duration = Duration::hours(1);
31/// An ingest or drain error must outlast this before it degrades, so one
32/// `database is locked` that the next attempt clears sends no webhook.
33const ERROR_GRACE: Duration = Duration::minutes(10);
34/// The replayer runs every 30 s, so an entry this old is not being sent.
35const QUEUE_STUCK: Duration = Duration::minutes(10);
36/// A refused peer that never handshakes again leaves its refusal shown this
37/// long.
38const CLOCK_REJECTION_SHOWN: Duration = Duration::days(1);
39/// A boot that waits a few seconds for NTP holds writes for less than this
40/// and sends no alert.
41pub const CLOCK_HELD_GRACE: std::time::Duration = std::time::Duration::from_secs(120);
42
43#[derive(Debug, Clone, Copy, Default, PartialEq, Eq, PartialOrd, Ord, Serialize)]
44#[serde(rename_all = "lowercase")]
45pub enum State {
46    #[default]
47    Ok,
48    Degraded,
49    Failing,
50}
51
52#[derive(Debug, Clone, PartialEq, Serialize)]
53pub struct Transition {
54    pub state: State,
55    pub reason: Option<String>,
56    pub at: DateTime<Utc>,
57}
58
59#[derive(Debug, Clone, Serialize)]
60pub struct ErrorRecord {
61    pub source: &'static str,
62    pub message: String,
63    pub at: DateTime<Utc>,
64}
65
66#[derive(Debug, Clone, Default, Serialize)]
67pub struct CaptureReport {
68    pub checked_at: Option<DateTime<Utc>>,
69    pub newest_transcript_at: Option<DateTime<Utc>>,
70    /// Modification time of the newest transcript this daemon ingested.
71    pub captured_through: Option<DateTime<Utc>>,
72    pub lag_secs: Option<i64>,
73}
74
75#[derive(Debug, Clone, Default, Serialize)]
76pub struct IndexReport {
77    pub checked_at: Option<DateTime<Utc>>,
78    pub outbox_pending: Option<u64>,
79    pub backlog_since: Option<DateTime<Utc>>,
80}
81
82/// Only the tool, status and time: the daemon's message can echo the
83/// write's text, and `/health` is unauthenticated.
84#[derive(Debug, Clone, PartialEq, Serialize)]
85pub struct RejectedWrite {
86    pub tool: String,
87    pub status: u16,
88    pub at: DateTime<Utc>,
89}
90
91#[derive(Debug, Clone, Default, Serialize)]
92pub struct QueueReport {
93    pub depth: u64,
94    pub oldest_at: Option<DateTime<Utc>>,
95    pub oldest_age_secs: Option<i64>,
96    pub last_rejected: Option<RejectedWrite>,
97}
98
99/// Where the one-time compaction after the stored-content redaction pass
100/// stands: a VACUUM of the store and a merge of the lexical index, which
101/// clear the plaintext the pass left in free pages and old segments.
102#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize)]
103#[serde(rename_all = "snake_case")]
104pub enum MaintenanceState {
105    Pending,
106    WaitingForSpace,
107    MergingIndex,
108    /// Agent writes are queued until it ends.
109    Vacuuming,
110    Done,
111}
112
113#[derive(Debug, Clone, Serialize)]
114pub struct MaintenanceReport {
115    pub state: MaintenanceState,
116    pub since: Option<DateTime<Utc>>,
117    pub space: Option<VacuumSpace>,
118    pub last_error: Option<String>,
119}
120
121impl Default for MaintenanceReport {
122    fn default() -> Self {
123        Self {
124            state: MaintenanceState::Pending,
125            since: None,
126            space: None,
127            last_error: None,
128        }
129    }
130}
131
132#[derive(Debug, Clone, Copy, Default, PartialEq, Eq, Serialize)]
133#[serde(rename_all = "snake_case")]
134pub enum EncryptionState {
135    /// Outside the data dir, or a build without SQLCipher; never migrated.
136    #[default]
137    Plaintext,
138    /// Waiting for a minute with no write.
139    Pending,
140    WaitingForSpace,
141    /// The one-time rewrite under the store key; agent writes are queued.
142    Encrypting,
143    /// A rewrite under a new store key after a revocation.
144    Rotating,
145    Encrypted,
146}
147
148#[derive(Debug, Clone, Default, Serialize)]
149pub struct EncryptionReport {
150    pub state: EncryptionState,
151    pub since: Option<DateTime<Utc>>,
152    pub space: Option<VacuumSpace>,
153    /// How long writes waited for the one-time migration.
154    pub migration_pause_ms: Option<u64>,
155    pub notice: Option<String>,
156    pub last_error: Option<String>,
157}
158
159pub const PLAINTEXT_LEFT_NOTICE: &str = "the store is encrypted now, but deleting the plaintext \
160     file does not erase it from disk or flash blocks; only full-disk encryption or a wipe does";
161
162#[derive(Debug, Clone, Copy, Default, PartialEq, Eq, Serialize)]
163#[serde(rename_all = "snake_case")]
164pub enum BackfillState {
165    #[default]
166    NotStarted,
167    Running,
168    Done,
169}
170
171#[derive(Debug, Clone, Default, PartialEq, Eq, Serialize)]
172pub struct BackfillReport {
173    pub state: BackfillState,
174    pub total: usize,
175    pub done: usize,
176    pub failed: usize,
177    pub updated_at: Option<DateTime<Utc>>,
178}
179
180/// A data dir a restore or a pairing moved aside, which the daemon deletes
181/// 90 days after `moved_at`.
182#[derive(Debug, Clone, Serialize)]
183pub struct MovedAsideReport {
184    pub name: String,
185    pub path: PathBuf,
186    pub moved_at: DateTime<Utc>,
187    /// Rounded up, and 0 once the 90 days have passed.
188    pub deleted_in_days: i64,
189}
190
191fn moved_aside_report(asides: &[MovedAside], now: DateTime<Utc>) -> Vec<MovedAsideReport> {
192    const DAY_MICROS: i64 = 86_400 * 1_000_000;
193    asides
194        .iter()
195        .map(|aside| {
196            let left = (aside.moved_at_micros + ASIDE_RETENTION_MICROS)
197                .saturating_sub(now.timestamp_micros())
198                .max(0);
199            MovedAsideReport {
200                name: aside
201                    .path
202                    .file_name()
203                    .map(|n| n.to_string_lossy().into_owned())
204                    .unwrap_or_default(),
205                path: aside.path.clone(),
206                moved_at: DateTime::from_timestamp_micros(aside.moved_at_micros)
207                    .unwrap_or_default(),
208                deleted_in_days: (left + DAY_MICROS - 1) / DAY_MICROS,
209            }
210        })
211        .collect()
212}
213
214#[derive(Debug, Clone, Default, Serialize)]
215pub struct SyncReport {
216    pub checked_at: Option<DateTime<Utc>>,
217    /// How many paired devices and unmatched `sync-peers` lines this device
218    /// syncs with; None until a check has counted them.
219    pub peers: Option<usize>,
220    /// A sync cursor for some peer is on record.
221    pub synced_before: bool,
222    pub last_success_at: Option<DateTime<Utc>>,
223    /// The newest `sync_state` time: when records last moved, which bounds
224    /// the last success when no stamp was written yet.
225    pub last_cursor_at: Option<DateTime<Utc>>,
226    /// None until the first floor check.
227    pub floor: Option<FloorReport>,
228    /// The spooled snapshot merge that has not finished, if any.
229    pub snapshot_merge: Option<MergeStatus>,
230}
231
232#[derive(Debug, Clone, Default, PartialEq, Eq, Serialize)]
233pub struct FloorReport {
234    pub floor: u64,
235    pub log_floor: u64,
236    pub head: u64,
237    pub top: u64,
238    pub silent: Vec<String>,
239    pub stuck: Vec<String>,
240    pub failing: u64,
241    pub settled: u64,
242    /// Settled records dropped over the cap, ever.
243    pub settled_dropped: u64,
244}
245
246#[derive(Debug, Clone, Default, Serialize)]
247pub struct IdentityReport {
248    pub checked_at: Option<DateTime<Utc>>,
249    /// Where the device key is kept: "file" or "tpm".
250    pub key_storage: Option<&'static str>,
251    /// How the device keys compared with this hardware at start: "bound",
252    /// "recorded", "no-keys", "unreadable" (the check is off on this
253    /// machine), "reminted" (the data dir came from other hardware) or
254    /// "failed".
255    pub hardware: Option<&'static str>,
256    /// None until the store was read.
257    #[serde(flatten)]
258    pub status: Option<IdentityStatus>,
259}
260
261pub const NO_GPU_NOTE: &str = "no LLM endpoint configured and no LLM distillation in 7 days: \
262     raws covered by an agent note are retired, the rest stay searchable";
263
264/// `NoGpu` when no LLM endpoint is configured and no LLM distilled lately
265/// (`note_cover::distill_mode`), so waiting raws are expected rather than
266/// overdue.
267#[derive(Debug, Clone, Copy, Default, PartialEq, Eq, Serialize)]
268pub enum DistillMode {
269    #[default]
270    #[serde(rename = "llm")]
271    Llm,
272    #[serde(rename = "no-gpu")]
273    NoGpu,
274}
275
276impl DistillMode {
277    fn note(self) -> Option<&'static str> {
278        match self {
279            DistillMode::Llm => None,
280            DistillMode::NoGpu => Some(NO_GPU_NOTE),
281        }
282    }
283}
284
285#[derive(Debug, Clone, Default, Serialize)]
286pub struct DistillReport {
287    pub checked_at: Option<DateTime<Utc>>,
288    pub mode: DistillMode,
289    pub pending: u64,
290    pub oldest_at: Option<DateTime<Utc>>,
291    #[serde(skip_serializing_if = "Option::is_none")]
292    pub note: Option<&'static str>,
293}
294
295/// How long a sync or a raw's distillation may be overdue before it
296/// degrades; None turns that check off.
297#[derive(Debug, Clone, Copy, PartialEq)]
298pub struct Windows {
299    pub sync_overdue: Option<Duration>,
300    pub undistilled: Option<Duration>,
301}
302
303impl Windows {
304    pub fn from_config(config: &AlertConfig) -> Self {
305        let hours = |h: u64| {
306            i64::try_from(h)
307                .ok()
308                .filter(|&h| h > 0)
309                .and_then(Duration::try_hours)
310        };
311        Self {
312            sync_overdue: hours(config.sync_overdue_hours),
313            undistilled: hours(config.undistilled_overdue_hours),
314        }
315    }
316}
317
318impl Default for Windows {
319    fn default() -> Self {
320        Self::from_config(&AlertConfig::default())
321    }
322}
323
324/// What one progress pass read. None where the read failed keeps the
325/// previous reading; `error` says what failed.
326#[derive(Debug, Clone, Default)]
327pub struct ProgressSeen {
328    pub sync: Option<SyncReport>,
329    pub distill: Option<DistillReport>,
330    /// Claude Code sessions the ingest log shows captured since
331    /// `Health::ingest_window_start`; None when unknown.
332    pub ingested_sessions: Option<Vec<String>>,
333    pub error: Option<String>,
334}
335
336#[derive(Debug, Clone, PartialEq, Serialize)]
337pub struct Verdict {
338    pub state: State,
339    pub reason: Option<String>,
340}
341
342#[derive(Debug, Clone, Serialize)]
343pub struct HealthReport {
344    pub state: State,
345    pub reason: Option<String>,
346    pub since: DateTime<Utc>,
347    /// Capture and recall alone, leaving out sync and distillation: what an
348    /// agent using second-brain needs to be told about.
349    pub serving: Verdict,
350    pub capture: CaptureReport,
351    pub index: IndexReport,
352    pub queue: QueueReport,
353    pub maintenance: MaintenanceReport,
354    pub encryption: EncryptionReport,
355    pub backfill: BackfillReport,
356    pub sync: SyncReport,
357    pub identity: IdentityReport,
358    pub distill: DistillReport,
359    pub moved_aside: Vec<MovedAsideReport>,
360    #[serde(flatten)]
361    pub value: ValueReport,
362    pub last_error: Option<ErrorRecord>,
363}
364
365/// What the reporters last said; only `assess` turns it into a state.
366struct Facts {
367    started: DateTime<Utc>,
368    capture: CaptureReport,
369    watcher_error: Option<String>,
370    ingest_error: Option<Streak>,
371    index: IndexReport,
372    drain_error: Option<Streak>,
373    indexes_error: Option<String>,
374    repair_error: Option<String>,
375    queue: QueueReport,
376    maintenance: MaintenanceReport,
377    encryption: EncryptionReport,
378    backfill: BackfillReport,
379    sync: SyncReport,
380    identity: IdentityReport,
381    distill: DistillReport,
382    progress_error: Option<Streak>,
383    value: ValueFacts,
384    windows: Windows,
385    last_error: Option<ErrorRecord>,
386    clock_held_for: Option<std::time::Duration>,
387    clock_rejected: Option<ClockRejection>,
388    clock_basis_ahead: bool,
389    floor: Option<FloorReport>,
390    snapshot_merge: Option<MergeStatus>,
391    moved_aside: Vec<MovedAside>,
392}
393
394struct ClockRejection {
395    machine_id: String,
396    reason: String,
397    at: DateTime<Utc>,
398}
399
400/// An error repeated since `since` with no success in between.
401struct Streak {
402    message: String,
403    since: DateTime<Utc>,
404}
405
406impl Streak {
407    fn extend(previous: Option<Streak>, message: String, now: DateTime<Utc>) -> Option<Streak> {
408        let since = previous.map_or(now, |p| p.since);
409        Some(Streak { message, since })
410    }
411
412    fn outlasted_grace(&self, now: DateTime<Utc>) -> bool {
413        now - self.since >= ERROR_GRACE
414    }
415}
416
417struct Inner {
418    facts: Facts,
419    report: HealthReport,
420    /// The state this process last sent, if it sent one.
421    sent: Option<State>,
422}
423
424pub struct Health {
425    inner: Mutex<Inner>,
426    transitions: Option<UnboundedSender<Transition>>,
427}
428
429impl Health {
430    /// `transitions` receives one message per state change, never one per
431    /// report; None when nothing listens.
432    pub fn new(now: DateTime<Utc>, transitions: Option<UnboundedSender<Transition>>) -> Self {
433        let mut facts = Facts {
434            started: now,
435            capture: CaptureReport::default(),
436            watcher_error: None,
437            ingest_error: None,
438            index: IndexReport::default(),
439            drain_error: None,
440            indexes_error: None,
441            repair_error: None,
442            queue: QueueReport::default(),
443            maintenance: MaintenanceReport::default(),
444            encryption: EncryptionReport::default(),
445            backfill: BackfillReport::default(),
446            sync: SyncReport::default(),
447            identity: IdentityReport::default(),
448            distill: DistillReport::default(),
449            progress_error: None,
450            value: ValueFacts::default(),
451            windows: Windows::default(),
452            last_error: None,
453            clock_held_for: None,
454            clock_rejected: None,
455            clock_basis_ahead: false,
456            floor: None,
457            snapshot_merge: None,
458            moved_aside: Vec::new(),
459        };
460        let report = HealthReport {
461            state: State::Ok,
462            reason: None,
463            since: now,
464            serving: Verdict {
465                state: State::Ok,
466                reason: None,
467            },
468            capture: CaptureReport::default(),
469            index: IndexReport::default(),
470            queue: QueueReport::default(),
471            maintenance: MaintenanceReport::default(),
472            encryption: EncryptionReport::default(),
473            backfill: BackfillReport::default(),
474            sync: SyncReport::default(),
475            identity: IdentityReport::default(),
476            distill: DistillReport::default(),
477            moved_aside: Vec::new(),
478            value: facts.value.refresh(now, now, facts.windows.sync_overdue).0,
479            last_error: None,
480        };
481        Self {
482            inner: Mutex::new(Inner {
483                facts,
484                report,
485                sent: None,
486            }),
487            transitions,
488        }
489    }
490
491    pub fn with_windows(self, windows: Windows) -> Self {
492        self.lock().facts.windows = windows;
493        self
494    }
495
496    pub fn progress_checked(&self, seen: ProgressSeen, now: DateTime<Utc>) {
497        self.update(now, |f| {
498            if let Some(sync) = seen.sync {
499                f.sync = SyncReport {
500                    checked_at: Some(now),
501                    peers: sync.peers.or(f.sync.peers),
502                    ..sync
503                };
504            }
505            if let Some(distill) = seen.distill {
506                f.distill = DistillReport {
507                    checked_at: Some(now),
508                    note: distill.mode.note(),
509                    ..distill
510                };
511            }
512            f.value.sessions_ingested(seen.ingested_sessions);
513            match seen.error {
514                None => f.progress_error = None,
515                Some(error) => {
516                    f.record_error("progress", &error, now);
517                    f.progress_error = Streak::extend(f.progress_error.take(), error, now);
518                }
519            }
520        });
521    }
522
523    /// Sends the current state unless this process already sent it, so a
524    /// receiver that missed transitions while the daemon was down learns it.
525    pub fn announce(&self, now: DateTime<Utc>) {
526        let mut inner = self.lock();
527        self.reassess(&mut inner, now, true);
528    }
529
530    pub fn report(&self) -> HealthReport {
531        self.lock().report.clone()
532    }
533
534    pub fn transcripts_scanned(&self, newest: Option<DateTime<Utc>>, now: DateTime<Utc>) {
535        self.update(now, |f| {
536            f.capture.checked_at = Some(now);
537            f.capture.newest_transcript_at = newest;
538        });
539    }
540
541    pub fn watcher_started(&self, now: DateTime<Utc>) {
542        self.update(now, |f| {
543            f.watcher_error = None;
544            f.capture.checked_at = Some(now);
545        });
546    }
547
548    pub fn watcher_failed(&self, error: &str, now: DateTime<Utc>) {
549        self.update(now, |f| {
550            f.watcher_error = Some(error.to_string());
551            f.record_error("watcher", error, now);
552        });
553    }
554
555    /// `transcript_mtime` is None for sources that are not transcripts,
556    /// which clear an ingest error without advancing capture.
557    pub fn ingested(&self, transcript_mtime: Option<DateTime<Utc>>, now: DateTime<Utc>) {
558        self.update(now, |f| {
559            f.ingest_error = None;
560            f.capture.checked_at = Some(now);
561            if let Some(mtime) = transcript_mtime {
562                let through = f.capture.captured_through.map_or(mtime, |t| t.max(mtime));
563                f.capture.captured_through = Some(through);
564            }
565        });
566    }
567
568    pub fn ingest_failed(&self, error: &str, now: DateTime<Utc>) {
569        self.update(now, |f| {
570            f.ingest_error = Streak::extend(f.ingest_error.take(), error.to_string(), now);
571            f.capture.checked_at = Some(now);
572            f.record_error("ingest", error, now);
573        });
574    }
575
576    pub fn drained(&self, outbox_pending: u64, now: DateTime<Utc>) {
577        self.update(now, |f| {
578            f.drain_error = None;
579            f.index.checked_at = Some(now);
580            f.index.outbox_pending = Some(outbox_pending);
581            f.index.backlog_since = if outbox_pending == 0 {
582                None
583            } else {
584                Some(f.index.backlog_since.unwrap_or(now))
585            };
586        });
587    }
588
589    pub fn drain_failed(&self, error: &str, now: DateTime<Utc>) {
590        self.update(now, |f| {
591            f.index.checked_at = Some(now);
592            f.drain_error = Streak::extend(f.drain_error.take(), error.to_string(), now);
593            f.record_error("drain", error, now);
594        });
595    }
596
597    /// No drain thread will ever report, so the error stays until restart.
598    pub fn indexes_unavailable(&self, error: &str, now: DateTime<Utc>) {
599        self.update(now, |f| {
600            f.indexes_error = Some(error.to_string());
601            f.record_error("index", error, now);
602        });
603    }
604
605    pub fn repaired(&self, now: DateTime<Utc>) {
606        self.update(now, |f| f.repair_error = None);
607    }
608
609    pub fn repair_failed(&self, error: &str, now: DateTime<Utc>) {
610        self.update(now, |f| {
611            f.repair_error = Some(error.to_string());
612            f.record_error("repair", error, now);
613        });
614    }
615
616    pub fn queue_checked(&self, depth: u64, oldest_at: Option<DateTime<Utc>>, now: DateTime<Utc>) {
617        self.update(now, |f| {
618            f.queue.depth = depth;
619            f.queue.oldest_at = oldest_at;
620        });
621    }
622
623    pub fn write_rejected(&self, tool: &str, status: u16, now: DateTime<Utc>) {
624        self.update(now, |f| {
625            f.queue.last_rejected = Some(RejectedWrite {
626                tool: tool.to_string(),
627                status,
628                at: now,
629            });
630        });
631    }
632
633    /// `space` is what the latest free-space check found, if one ran.
634    pub fn maintenance(
635        &self,
636        state: MaintenanceState,
637        space: Option<VacuumSpace>,
638        now: DateTime<Utc>,
639    ) {
640        self.update(now, |f| {
641            if f.maintenance.state != state {
642                f.maintenance.since = Some(now);
643            }
644            f.maintenance.state = state;
645            if space.is_some() {
646                f.maintenance.space = space;
647            }
648            if state == MaintenanceState::Done {
649                f.maintenance.last_error = None;
650            }
651        });
652    }
653
654    pub fn encryption(
655        &self,
656        state: EncryptionState,
657        space: Option<VacuumSpace>,
658        now: DateTime<Utc>,
659    ) {
660        self.update(now, |f| {
661            if f.encryption.state != state {
662                f.encryption.since = Some(now);
663            }
664            f.encryption.state = state;
665            if space.is_some() {
666                f.encryption.space = space;
667            }
668            if state == EncryptionState::Encrypted {
669                f.encryption.last_error = None;
670            }
671        });
672    }
673
674    /// Shown for the days after the one-time migration, so the user
675    /// learns that the old plaintext file's blocks may still be readable.
676    pub fn encryption_migrated(&self, pause_ms: u64, notice: bool, now: DateTime<Utc>) {
677        self.update(now, |f| {
678            f.encryption.migration_pause_ms = Some(pause_ms);
679            f.encryption.notice = notice.then(|| PLAINTEXT_LEFT_NOTICE.to_string());
680        });
681    }
682
683    /// Stays until the next rewrite under the store key completes.
684    pub fn encryption_failed(&self, error: &str, now: DateTime<Utc>) {
685        self.update(now, |f| {
686            f.encryption.last_error = Some(error.to_string());
687            f.record_error("encryption", error, now);
688        });
689    }
690
691    /// Stays until the compaction next completes.
692    pub fn maintenance_failed(&self, error: &str, now: DateTime<Utc>) {
693        self.update(now, |f| {
694            f.maintenance.last_error = Some(error.to_string());
695            f.record_error("maintenance", error, now);
696        });
697    }
698
699    /// `held_for` is how long the gate has been closed, on a monotonic clock
700    /// so a wall clock step cannot stretch it; None while open.
701    pub fn clock_gate(&self, held_for: Option<std::time::Duration>, now: DateTime<Utc>) {
702        self.update(now, |f| f.clock_held_for = held_for);
703    }
704
705    pub fn clock_handshake_rejected(&self, machine_id: &str, reason: String, now: DateTime<Utc>) {
706        self.update(now, |f| {
707            f.record_error("clock", &reason, now);
708            f.clock_rejected = Some(ClockRejection {
709                machine_id: machine_id.to_string(),
710                reason,
711                at: now,
712            });
713        });
714    }
715
716    /// Clears a refusal of the same peer; another peer's handshake says
717    /// nothing about it.
718    pub fn clock_handshake_accepted(&self, machine_id: &str, now: DateTime<Utc>) {
719        self.update(now, |f| {
720            if f.clock_rejected
721                .as_ref()
722                .is_some_and(|r| r.machine_id == machine_id)
723            {
724                f.clock_rejected = None;
725            }
726        });
727    }
728
729    pub fn clock_basis_ahead(&self, ahead: bool, now: DateTime<Utc>) {
730        self.update(now, |f| f.clock_basis_ahead = ahead);
731    }
732
733    pub fn snapshot_merge_checked(&self, merge: Option<MergeStatus>, now: DateTime<Utc>) {
734        self.update(now, |f| f.snapshot_merge = merge);
735    }
736
737    pub fn floor_checked(&self, floor: FloorHealth, failed: FailedCounts, now: DateTime<Utc>) {
738        self.update(now, |f| {
739            f.floor = Some(FloorReport {
740                floor: floor.floor,
741                log_floor: floor.log_floor,
742                head: floor.head,
743                top: floor.top,
744                silent: floor.silent,
745                stuck: floor.stuck,
746                failing: failed.failing,
747                settled: failed.settled,
748                settled_dropped: failed.dropped,
749            });
750        });
751    }
752
753    pub fn key_storage_detected(&self, name: &'static str, now: DateTime<Utc>) {
754        self.update(now, |f| f.identity.key_storage = Some(name));
755    }
756
757    pub fn hardware_checked(&self, state: &'static str, now: DateTime<Utc>) {
758        self.update(now, |f| f.identity.hardware = Some(state));
759    }
760
761    pub fn identity_checked(&self, status: IdentityStatus, now: DateTime<Utc>) {
762        self.update(now, |f| {
763            f.identity.checked_at = Some(now);
764            f.identity.status = Some(status);
765        });
766    }
767
768    pub fn recalled(&self, event: RecallEvent, now: DateTime<Utc>) {
769        self.update(now, |f| f.value.recalled(event, now));
770    }
771
772    pub fn memory_corrected(&self, id: Uuid, now: DateTime<Utc>) {
773        self.update(now, |f| f.value.corrected(id));
774    }
775
776    pub fn memories_used(&self, ids: &[Uuid], now: DateTime<Utc>) {
777        self.update(now, |f| f.value.used(ids));
778    }
779
780    pub fn sync_peers_listed(&self, peers: &[String], now: DateTime<Utc>) {
781        self.update(now, |f| f.value.peers_listed(peers, now));
782    }
783
784    pub fn sync_attempted(&self, peer: &str, outcome: PeerOutcome, now: DateTime<Utc>) {
785        self.update(now, |f| f.value.sync_attempted(peer, outcome, now));
786    }
787
788    /// None when the measurement failed, which drops the old reading.
789    pub fn store_measured(&self, size: Option<StoreSize>, now: DateTime<Utc>) {
790        self.update(now, |f| f.value.store_measured(size));
791    }
792
793    /// None when the measurement failed, which drops the old reading.
794    pub fn resident_measured(&self, size: Option<ResidentSize>, now: DateTime<Utc>) {
795        self.update(now, |f| f.value.resident_measured(size));
796    }
797
798    /// Where a count of recently ingested sessions can start so that it
799    /// compares with recall sessions, which this process saw only since it
800    /// started: the window, this process's start and the end of a backfill,
801    /// whose old sessions look freshly ingested. None while one runs.
802    pub fn ingest_window_start(&self, now: DateTime<Utc>) -> Option<DateTime<Utc>> {
803        let inner = self.lock();
804        let f = &inner.facts;
805        let mut start = (now - WINDOW).max(f.started);
806        match f.backfill.state {
807            BackfillState::Running => return None,
808            BackfillState::Done => {
809                if let Some(end) = f.backfill.updated_at {
810                    start = start.max(end);
811                }
812            }
813            BackfillState::NotStarted => {}
814        }
815        Some(start)
816    }
817
818    pub fn moved_aside_listed(&self, asides: Vec<MovedAside>, now: DateTime<Utc>) {
819        self.update(now, |f| f.moved_aside = asides);
820    }
821
822    /// Progress only: skipping the reassessment keeps the state, reason and
823    /// transitions exactly as the last real report left them.
824    pub fn backfill_progress(&self, report: BackfillReport) {
825        let mut inner = self.lock();
826        inner.facts.backfill = report.clone();
827        inner.report.backfill = report;
828    }
829
830    fn lock(&self) -> std::sync::MutexGuard<'_, Inner> {
831        match self.inner.lock() {
832            Ok(g) => g,
833            Err(poisoned) => poisoned.into_inner(),
834        }
835    }
836
837    fn update(&self, now: DateTime<Utc>, change: impl FnOnce(&mut Facts)) {
838        let mut inner = self.lock();
839        change(&mut inner.facts);
840        self.reassess(&mut inner, now, false);
841    }
842
843    /// `announcing` sends an unchanged state that this process has not sent.
844    fn reassess(&self, inner: &mut Inner, now: DateTime<Utc>, announcing: bool) {
845        let (started, sync_overdue) = (inner.facts.started, inner.facts.windows.sync_overdue);
846        let (value, value_problems) = inner.facts.value.refresh(now, started, sync_overdue);
847        let mut serving = Assessment::default();
848        assess_serving(&inner.facts, now, &mut serving);
849        let mut overall = serving.clone();
850        assess_upkeep(&inner.facts, now, &mut overall, value_problems);
851        let (state, reason) = overall.verdict();
852        let changed = state != inner.report.state;
853        let since = if changed { now } else { inner.report.since };
854        let mut capture = inner.facts.capture.clone();
855        capture.lag_secs = capture_lag(&inner.facts).map(|d| d.num_seconds());
856        let mut queue = inner.facts.queue.clone();
857        queue.oldest_age_secs = queue.oldest_at.map(|t| (now - t).num_seconds().max(0));
858        inner.report = HealthReport {
859            state,
860            reason: reason.clone(),
861            since,
862            serving: Verdict {
863                state: serving.state,
864                reason: serving.verdict().1,
865            },
866            capture,
867            index: inner.facts.index.clone(),
868            queue,
869            maintenance: inner.facts.maintenance.clone(),
870            encryption: inner.facts.encryption.clone(),
871            backfill: inner.facts.backfill.clone(),
872            sync: SyncReport {
873                floor: inner.facts.floor.clone(),
874                snapshot_merge: inner.facts.snapshot_merge.clone(),
875                ..inner.facts.sync.clone()
876            },
877            identity: inner.facts.identity.clone(),
878            distill: inner.facts.distill.clone(),
879            moved_aside: moved_aside_report(&inner.facts.moved_aside, now),
880            value,
881            last_error: inner.facts.last_error.clone(),
882        };
883        // Sent under the lock so concurrent reporters cannot deliver
884        // transitions out of order; the send never blocks.
885        let unsent = inner.sent != Some(state);
886        if (changed || (announcing && unsent))
887            && let Some(tx) = &self.transitions
888        {
889            // The listener only goes away when the runtime shuts down.
890            let _ = tx.send(Transition {
891                state,
892                reason,
893                at: now,
894            });
895            inner.sent = Some(state);
896        }
897    }
898}
899
900impl Facts {
901    fn record_error(&mut self, source: &'static str, message: &str, at: DateTime<Utc>) {
902        self.last_error = Some(ErrorRecord {
903            source,
904            message: message.to_string(),
905            at,
906        });
907    }
908}
909
910/// Transcripts written before the daemon started are not its lag: it
911/// watches for changes and never saw them.
912fn capture_lag(f: &Facts) -> Option<Duration> {
913    let newest = f.capture.newest_transcript_at?;
914    let through = f
915        .capture
916        .captured_through
917        .map_or(f.started, |t| t.max(f.started));
918    Some((newest - through).max(Duration::zero()))
919}
920
921#[derive(Clone, Default)]
922struct Assessment {
923    state: State,
924    problems: Vec<String>,
925}
926
927impl Assessment {
928    fn flag(&mut self, level: State, problem: String) {
929        self.state = self.state.max(level);
930        self.problems.push(problem);
931    }
932
933    fn verdict(&self) -> (State, Option<String>) {
934        let reason = (!self.problems.is_empty()).then(|| self.problems.join("; "));
935        (self.state, reason)
936    }
937}
938
939fn assess_serving(f: &Facts, now: DateTime<Utc>, a: &mut Assessment) {
940    let mut flag = |level: State, problem: String| a.flag(level, problem);
941
942    if f.clock_held_for
943        .is_some_and(|held| held >= CLOCK_HELD_GRACE)
944    {
945        flag(
946            State::Degraded,
947            "clock not yet trusted: writes held".to_string(),
948        );
949    }
950
951    match &f.watcher_error {
952        Some(e) => flag(
953            State::Failing,
954            format!("transcript watcher is not running: {e}"),
955        ),
956        None => {
957            let last = f.capture.checked_at.unwrap_or(f.started);
958            if now - last >= WATCHER_SILENT {
959                flag(
960                    State::Degraded,
961                    format!(
962                        "transcript watcher has not reported for {}",
963                        human(now - last)
964                    ),
965                );
966            }
967        }
968    }
969    if let Some(lag) = capture_lag(f)
970        && lag >= CAPTURE_LAG_DEGRADED
971    {
972        let level = if lag >= CAPTURE_LAG_FAILING {
973            State::Failing
974        } else {
975            State::Degraded
976        };
977        flag(
978            level,
979            format!("capture is {} behind the newest transcript", human(lag)),
980        );
981    }
982    if let Some(e) = &f.ingest_error
983        && e.outlasted_grace(now)
984    {
985        flag(
986            State::Degraded,
987            format!(
988                "ingest has failed for {}: {}",
989                human(now - e.since),
990                e.message
991            ),
992        );
993    }
994
995    if let Some(e) = &f.drain_error
996        && e.outlasted_grace(now)
997    {
998        flag(
999            State::Degraded,
1000            format!(
1001                "index drain failed for {}: {}",
1002                human(now - e.since),
1003                e.message
1004            ),
1005        );
1006    }
1007    match &f.indexes_error {
1008        Some(e) => flag(State::Degraded, format!("search indexes unavailable: {e}")),
1009        None => {
1010            let last = f.index.checked_at.unwrap_or(f.started);
1011            if now - last >= DRAIN_SILENT {
1012                flag(
1013                    State::Degraded,
1014                    format!("index drain has not reported for {}", human(now - last)),
1015                );
1016            }
1017        }
1018    }
1019    if let (Some(since), Some(pending)) = (f.index.backlog_since, f.index.outbox_pending) {
1020        let age = now - since;
1021        if age >= BACKLOG_DEGRADED {
1022            let level = if age >= BACKLOG_FAILING {
1023                State::Failing
1024            } else {
1025                State::Degraded
1026            };
1027            flag(
1028                level,
1029                format!("{pending} changes have waited {} to be indexed", human(age)),
1030            );
1031        }
1032    }
1033    if let Some(e) = &f.repair_error {
1034        flag(State::Degraded, format!("last store repair failed: {e}"));
1035    }
1036    if let Some(oldest) = f.queue.oldest_at
1037        && now - oldest >= QUEUE_STUCK
1038    {
1039        flag(
1040            State::Degraded,
1041            format!(
1042                "write queue not draining ({} queued, oldest {})",
1043                f.queue.depth,
1044                human(now - oldest)
1045            ),
1046        );
1047    }
1048
1049    match (&f.maintenance.state, &f.maintenance.space) {
1050        (MaintenanceState::Vacuuming, _) => flag(
1051            State::Degraded,
1052            "store maintenance is running; writes are queued until it finishes".to_string(),
1053        ),
1054        (MaintenanceState::WaitingForSpace, Some(space)) => flag(
1055            State::Degraded,
1056            format!(
1057                "store compaction waits for disk space: it needs {} MB free at {} and has {} MB",
1058                space.needed_bytes / 1_000_000,
1059                space.path.display(),
1060                space.free_bytes / 1_000_000
1061            ),
1062        ),
1063        _ => {}
1064    }
1065    if let Some(e) = &f.maintenance.last_error {
1066        flag(State::Degraded, format!("store compaction failed: {e}"));
1067    }
1068    match (&f.encryption.state, &f.encryption.space) {
1069        (EncryptionState::Encrypting | EncryptionState::Rotating, _) => flag(
1070            State::Degraded,
1071            "the store is being rewritten under its key; writes are queued until it finishes"
1072                .to_string(),
1073        ),
1074        (EncryptionState::WaitingForSpace, Some(space)) => flag(
1075            State::Degraded,
1076            format!(
1077                "encryption pending, needs {} MB free at {} and has {} MB",
1078                space.needed_bytes / 1_000_000,
1079                space.path.display(),
1080                space.free_bytes / 1_000_000
1081            ),
1082        ),
1083        _ => {}
1084    }
1085    if let Some(e) = &f.encryption.last_error {
1086        flag(State::Degraded, format!("store encryption failed: {e}"));
1087    }
1088}
1089
1090fn identity_problems(
1091    identity: &IdentityStatus,
1092    peers: Option<usize>,
1093    reminted: bool,
1094    flag: &mut impl FnMut(State, String),
1095) {
1096    if identity.identity.is_none() {
1097        if peers.is_some_and(|n| n > 0) {
1098            let why = if reminted {
1099                "this data dir was copied from other hardware, so this device took a new machine \
1100                 id and new device keys and holds no sync identity key"
1101            } else {
1102                "this device holds no sync identity key, so it cannot sync with its peers"
1103            };
1104            flag(
1105                State::Degraded,
1106                format!(
1107                    "{why}; run `sb pair` on a device that syncs, then `sb pair <code>` on this \
1108                     device"
1109                ),
1110            );
1111        }
1112        return;
1113    }
1114    if identity.removed {
1115        flag(
1116            State::Degraded,
1117            "this device was removed from its sync identity, so its peers refuse it; join \
1118             another device's identity with `sb pair <code>` on this device"
1119                .to_string(),
1120        );
1121    } else if identity.possibly_revoked {
1122        let withheld: Vec<String> = identity
1123            .peers
1124            .iter()
1125            .filter(|p| p.holds_own_row && !p.lacking.is_empty())
1126            .map(|p| {
1127                let numbers: Vec<String> = p.lacking.iter().map(u32::to_string).collect();
1128                let noun = if p.lacking.len() == 1 {
1129                    "epoch"
1130                } else {
1131                    "epochs"
1132                };
1133                format!(
1134                    "{} holds identity key {noun} {}",
1135                    p.peer,
1136                    numbers.join(", ")
1137                )
1138            })
1139            .collect();
1140        flag(
1141            State::Degraded,
1142            format!(
1143                "{} but did not wrap it to this device, which may have been removed from its \
1144                 sync identity; run `sb device list` on a device that syncs",
1145                withheld.join("; ")
1146            ),
1147        );
1148    }
1149    if identity.row_accepted == Some(false) {
1150        flag(
1151            State::Degraded,
1152            "no peer has accepted this device's row, so none wraps it a newer identity key; \
1153             pair it with `sb pair` from a device that syncs"
1154                .to_string(),
1155        );
1156    }
1157}
1158
1159/// Sync, distillation and the value metrics: overdue work and poor value
1160/// that leave capture and recall running.
1161fn assess_upkeep(f: &Facts, now: DateTime<Utc>, a: &mut Assessment, value: ValueProblems) {
1162    let mut flag = |level: State, problem: String| a.flag(level, problem);
1163    let mut no_sync_at_all = false;
1164    let sync = &f.sync;
1165    if (sync.peers.is_some_and(|n| n > 0) || sync.synced_before)
1166        && let Some(window) = f.windows.sync_overdue
1167    {
1168        let since = sync
1169            .last_success_at
1170            .or(sync.last_cursor_at)
1171            .unwrap_or(f.started);
1172        let overdue = now - since;
1173        if overdue >= window {
1174            let problem = match sync.peers {
1175                Some(0) => format!(
1176                    "no sync has succeeded in {}, and this device has no paired device to sync \
1177                     with although it has synced before; pair one with `sb pair`, \
1178                     or set alert.sync_overdue_hours = 0 if this device no longer syncs",
1179                    human(overdue)
1180                ),
1181                Some(n) => format!(
1182                    "no sync has succeeded in {} with the {} this device syncs with; \
1183                     run `sb doctor` for what to check",
1184                    human(overdue),
1185                    counted(n as u64, "device", "devices")
1186                ),
1187                None => format!(
1188                    "no sync has succeeded in {} although this device has synced before; \
1189                     run `sb sync <user@host>` (or check the job that runs it), \
1190                     or set alert.sync_overdue_hours = 0 if this device no longer syncs",
1191                    human(overdue)
1192                ),
1193            };
1194            flag(State::Degraded, problem);
1195            no_sync_at_all = true;
1196        }
1197    }
1198    if !no_sync_at_all {
1199        for problem in value.peers {
1200            flag(State::Degraded, problem);
1201        }
1202    }
1203    for problem in value.other {
1204        flag(State::Degraded, problem);
1205    }
1206    if f.distill.mode == DistillMode::Llm
1207        && let (Some(oldest), Some(window)) = (f.distill.oldest_at, f.windows.undistilled)
1208        && now - oldest >= window
1209    {
1210        flag(
1211            State::Degraded,
1212            format!(
1213                "{} waited {} to be distilled; run `sb consolidate --llm-url <endpoint>` or add an [llm] section, or set \
1214                 alert.undistilled_overdue_hours = 0 if this device does not distill",
1215                if f.distill.pending == 1 {
1216                    "1 raw memory has".to_string()
1217                } else {
1218                    format!("{} raw memories have", f.distill.pending)
1219                },
1220                human(now - oldest)
1221            ),
1222        );
1223    }
1224    if let Some(e) = &f.progress_error
1225        && e.outlasted_grace(now)
1226    {
1227        flag(
1228            State::Degraded,
1229            format!(
1230                "cannot check sync and distillation for {}: {}",
1231                human(now - e.since),
1232                e.message
1233            ),
1234        );
1235    }
1236    if let Some(rejection) = &f.clock_rejected
1237        && now - rejection.at < CLOCK_REJECTION_SHOWN
1238    {
1239        flag(
1240            State::Degraded,
1241            format!(
1242                "a sync handshake was refused {} ago: {}",
1243                human(now - rejection.at),
1244                rejection.reason
1245            ),
1246        );
1247    }
1248    if f.clock_basis_ahead {
1249        flag(
1250            State::Degraded,
1251            "this store's clock basis is ahead of synchronised time; new records count up \
1252             from the last one until real time catches up"
1253                .to_string(),
1254        );
1255    }
1256    if let Some(identity) = &f.identity.status {
1257        let reminted = f.identity.hardware == Some("reminted");
1258        identity_problems(identity, f.sync.peers, reminted, &mut flag);
1259    }
1260    if let Some(floor) = &f.floor {
1261        for id in &floor.silent {
1262            flag(
1263                State::Degraded,
1264                format!(
1265                    "sync neighbour {id} has been silent for 30 days and holds the log floor back"
1266                ),
1267            );
1268        }
1269        for id in &floor.stuck {
1270            flag(
1271                State::Degraded,
1272                format!("sync neighbour {id} keeps acking but its ack has not moved in 7 days"),
1273            );
1274        }
1275    }
1276    if let Some(MergeStatus {
1277        merging,
1278        failing: Some(failing),
1279    }) = &f.snapshot_merge
1280    {
1281        flag(
1282            State::Degraded,
1283            format!(
1284                "the snapshot merge from {} through seq {} has failed {} times: {}",
1285                merging.exporter, merging.seq, failing.attempts, failing.error
1286            ),
1287        );
1288    }
1289}
1290
1291pub(crate) fn counted(n: u64, one: &str, many: &str) -> String {
1292    if n == 1 {
1293        format!("1 {one}")
1294    } else {
1295        format!("{n} {many}")
1296    }
1297}
1298
1299pub(crate) fn human(d: Duration) -> String {
1300    let minutes = d.num_minutes();
1301    if minutes < 60 {
1302        format!("{minutes}m")
1303    } else if minutes < 48 * 60 {
1304        format!("{}h {}m", minutes / 60, minutes % 60)
1305    } else {
1306        format!("{}d", minutes / (24 * 60))
1307    }
1308}
1309
1310#[cfg(test)]
1311mod tests {
1312    use tokio::sync::mpsc::{UnboundedReceiver, unbounded_channel};
1313
1314    use second_brain_sync::keyring::PeerStanding;
1315
1316    use super::*;
1317
1318    fn t0() -> DateTime<Utc> {
1319        DateTime::parse_from_rfc3339("2026-10-31T12:00:00Z")
1320            .unwrap()
1321            .with_timezone(&Utc)
1322    }
1323
1324    fn mins(m: i64) -> DateTime<Utc> {
1325        t0() + Duration::minutes(m)
1326    }
1327
1328    fn monitored() -> (Health, UnboundedReceiver<Transition>) {
1329        let (tx, rx) = unbounded_channel();
1330        (Health::new(t0(), Some(tx)), rx)
1331    }
1332
1333    fn drain(rx: &mut UnboundedReceiver<Transition>) -> Vec<State> {
1334        std::iter::from_fn(|| rx.try_recv().ok())
1335            .map(|t| t.state)
1336            .collect()
1337    }
1338
1339    #[test]
1340    fn a_fresh_daemon_that_keeps_up_is_ok() {
1341        let (health, mut rx) = monitored();
1342        health.watcher_started(mins(0));
1343        health.transcripts_scanned(Some(mins(1)), mins(1));
1344        health.ingested(Some(mins(1)), mins(1));
1345        health.drained(0, mins(1));
1346        let report = health.report();
1347        assert_eq!(report.state, State::Ok);
1348        assert_eq!(report.reason, None);
1349        assert_eq!(report.capture.lag_secs, Some(0));
1350        assert!(drain(&mut rx).is_empty());
1351    }
1352
1353    #[test]
1354    fn capture_lag_degrades_then_fails_then_recovers() {
1355        let (health, mut rx) = monitored();
1356        health.drained(0, mins(0));
1357        health.ingested(Some(mins(1)), mins(1));
1358        health.transcripts_scanned(Some(mins(10)), mins(10));
1359        assert_eq!(health.report().state, State::Ok);
1360
1361        health.transcripts_scanned(Some(mins(20)), mins(20));
1362        let report = health.report();
1363        assert_eq!(report.state, State::Degraded);
1364        assert_eq!(report.capture.lag_secs, Some(19 * 60));
1365        assert!(report.reason.unwrap().contains("capture is 19m behind"));
1366
1367        health.drained(0, mins(125));
1368        health.transcripts_scanned(Some(mins(125)), mins(125));
1369        assert_eq!(health.report().state, State::Failing);
1370
1371        health.ingested(Some(mins(125)), mins(126));
1372        assert_eq!(health.report().state, State::Ok);
1373        assert_eq!(
1374            drain(&mut rx),
1375            vec![State::Degraded, State::Failing, State::Ok]
1376        );
1377    }
1378
1379    #[test]
1380    fn transcripts_from_before_startup_are_not_lag() {
1381        let (health, _rx) = monitored();
1382        health.transcripts_scanned(Some(t0() - Duration::days(3)), mins(1));
1383        assert_eq!(health.report().capture.lag_secs, Some(0));
1384        assert_eq!(health.report().state, State::Ok);
1385    }
1386
1387    #[test]
1388    fn repeated_reports_in_one_state_send_one_transition() {
1389        let (health, mut rx) = monitored();
1390        for m in 0..=12 {
1391            health.ingest_failed("database is locked", mins(m));
1392        }
1393        health.ingested(None, mins(13));
1394        health.ingested(None, mins(14));
1395        assert_eq!(drain(&mut rx), vec![State::Degraded, State::Ok]);
1396        let report = health.report();
1397        assert_eq!(report.since, mins(13));
1398        let last = report.last_error.expect("the error is kept after recovery");
1399        assert_eq!(last.source, "ingest");
1400        assert_eq!(last.at, mins(12));
1401    }
1402
1403    #[test]
1404    fn an_error_that_clears_within_the_grace_sends_nothing() {
1405        let (health, mut rx) = monitored();
1406        health.watcher_started(mins(0));
1407        health.ingest_failed("database is locked", mins(0));
1408        health.drain_failed("database is locked", mins(1));
1409        health.drained(0, mins(9));
1410        health.ingested(None, mins(9));
1411        assert_eq!(health.report().state, State::Ok);
1412        assert!(drain(&mut rx).is_empty());
1413        assert_eq!(health.report().last_error.unwrap().source, "drain");
1414    }
1415
1416    #[test]
1417    fn a_drain_error_degrades_once_it_outlasts_the_grace() {
1418        let (health, _rx) = monitored();
1419        health.watcher_started(mins(0));
1420        for m in 0..10 {
1421            health.drain_failed("disk I/O error", mins(m));
1422        }
1423        assert_eq!(health.report().state, State::Ok);
1424        health.drain_failed("disk I/O error", mins(10));
1425        let report = health.report();
1426        assert_eq!(report.state, State::Degraded);
1427        assert!(report.reason.unwrap().contains("index drain failed"));
1428        health.drained(0, mins(11));
1429        assert_eq!(health.report().state, State::Ok);
1430    }
1431
1432    #[test]
1433    fn a_long_ingest_batch_counts_as_the_watcher_reporting() {
1434        let (health, _rx) = monitored();
1435        health.watcher_started(mins(0));
1436        health.ingested(None, mins(10));
1437        health.ingest_failed("unreadable line", mins(19));
1438        health.drained(0, mins(20));
1439        let report = health.report();
1440        assert_eq!(report.state, State::Ok, "{:?}", report.reason);
1441    }
1442
1443    #[test]
1444    fn an_outbox_backlog_only_counts_once_it_persists() {
1445        let (health, _rx) = monitored();
1446        health.drained(500, mins(0));
1447        health.drained(800, mins(14));
1448        health.watcher_started(mins(14));
1449        assert_eq!(health.report().state, State::Ok);
1450        health.drained(800, mins(15));
1451        let report = health.report();
1452        assert_eq!(report.state, State::Degraded);
1453        assert_eq!(report.index.backlog_since, Some(mins(0)));
1454        health.drained(0, mins(16));
1455        assert_eq!(health.report().state, State::Ok);
1456        assert_eq!(health.report().index.backlog_since, None);
1457    }
1458
1459    #[test]
1460    fn a_dead_watcher_fails_and_a_silent_one_degrades() {
1461        let (health, _rx) = monitored();
1462        health.watcher_failed("inotify watch limit reached", mins(0));
1463        assert_eq!(health.report().state, State::Failing);
1464        health.watcher_started(mins(1));
1465        health.drained(0, mins(1));
1466        assert_eq!(health.report().state, State::Ok);
1467        health.drained(0, mins(16));
1468        let report = health.report();
1469        assert_eq!(report.state, State::Degraded);
1470        assert!(report.reason.unwrap().contains("watcher has not reported"));
1471    }
1472
1473    #[test]
1474    fn index_and_repair_errors_degrade_until_the_next_success() {
1475        let (health, _rx) = monitored();
1476        health.watcher_started(mins(0));
1477        health.drain_failed("disk I/O error", mins(0));
1478        health.repair_failed("disk I/O error", mins(0));
1479        let reason = health.report().reason.unwrap();
1480        assert!(reason.contains("store repair failed"), "{reason}");
1481        health.drain_failed("disk I/O error", mins(10));
1482        let reason = health.report().reason.unwrap();
1483        assert!(reason.contains("index drain failed"), "{reason}");
1484        health.drained(0, mins(11));
1485        assert_eq!(health.report().state, State::Degraded);
1486        health.repaired(mins(12));
1487        assert_eq!(health.report().state, State::Ok);
1488    }
1489
1490    #[test]
1491    fn missing_indexes_stay_degraded_without_a_drain() {
1492        let (health, _rx) = monitored();
1493        health.indexes_unavailable("tantivy meta.json missing", mins(0));
1494        health.watcher_started(mins(90));
1495        let report = health.report();
1496        assert_eq!(report.state, State::Degraded);
1497        let reason = report.reason.unwrap();
1498        assert!(reason.contains("search indexes unavailable"), "{reason}");
1499        assert!(!reason.contains("has not reported"), "{reason}");
1500    }
1501
1502    #[test]
1503    fn the_report_serializes_without_content() {
1504        let (health, _rx) = monitored();
1505        health.ingest_failed("database is locked", mins(0));
1506        health.ingest_failed("database is locked", mins(10));
1507        let json = serde_json::to_value(health.report()).unwrap();
1508        assert_eq!(json["state"], "degraded");
1509        assert_eq!(json["last_error"]["source"], "ingest");
1510        for key in ["reason", "since", "capture", "index"] {
1511            assert!(json.get(key).is_some(), "{key} missing: {json}");
1512        }
1513    }
1514
1515    #[test]
1516    fn a_queue_entry_older_than_ten_minutes_degrades_until_it_drains() {
1517        let (health, mut rx) = monitored();
1518        health.watcher_started(mins(0));
1519        health.drained(0, mins(0));
1520        health.queue_checked(2, Some(mins(0)), mins(9));
1521        assert_eq!(health.report().state, State::Ok);
1522        assert_eq!(health.report().queue.depth, 2);
1523        health.queue_checked(2, Some(mins(0)), mins(10));
1524        let report = health.report();
1525        assert_eq!(report.state, State::Degraded);
1526        assert!(
1527            report
1528                .reason
1529                .as_deref()
1530                .unwrap()
1531                .contains("write queue not draining"),
1532            "{report:?}"
1533        );
1534        assert_eq!(report.queue.oldest_age_secs, Some(600));
1535        health.queue_checked(0, None, mins(11));
1536        assert_eq!(health.report().state, State::Ok);
1537        assert_eq!(drain(&mut rx), vec![State::Degraded, State::Ok]);
1538    }
1539
1540    #[test]
1541    fn a_rejected_write_is_reported_by_tool_status_and_time_only() {
1542        let (health, _rx) = monitored();
1543        health.write_rejected("associate", 404, mins(3));
1544        let json = serde_json::to_value(health.report()).unwrap();
1545        assert_eq!(
1546            json["queue"]["last_rejected"],
1547            serde_json::json!({"tool": "associate", "status": 404, "at": "2026-10-31T12:03:00Z"})
1548        );
1549        assert_eq!(json["state"], "ok");
1550    }
1551
1552    fn hours(h: i64) -> DateTime<Utc> {
1553        t0() + Duration::hours(h)
1554    }
1555
1556    fn alive(health: &Health, at: DateTime<Utc>) {
1557        health.update(at, |f| {
1558            f.capture.checked_at = Some(at);
1559            f.index.checked_at = Some(at);
1560        });
1561    }
1562
1563    fn sync_seen(health: &Health, sync: SyncReport, at: DateTime<Utc>) {
1564        let seen = ProgressSeen {
1565            sync: Some(sync),
1566            ..ProgressSeen::default()
1567        };
1568        health.progress_checked(seen, at);
1569    }
1570
1571    fn distill_seen(
1572        health: &Health,
1573        pending: u64,
1574        oldest_at: Option<DateTime<Utc>>,
1575        at: DateTime<Utc>,
1576    ) {
1577        distill_seen_in(health, DistillMode::Llm, pending, oldest_at, at);
1578    }
1579
1580    fn distill_seen_in(
1581        health: &Health,
1582        mode: DistillMode,
1583        pending: u64,
1584        oldest_at: Option<DateTime<Utc>>,
1585        at: DateTime<Utc>,
1586    ) {
1587        let distill = DistillReport {
1588            mode,
1589            pending,
1590            oldest_at,
1591            ..DistillReport::default()
1592        };
1593        let seen = ProgressSeen {
1594            distill: Some(distill),
1595            ..ProgressSeen::default()
1596        };
1597        health.progress_checked(seen, at);
1598    }
1599
1600    fn synced(peers: usize, synced_before: bool, last: Option<DateTime<Utc>>) -> SyncReport {
1601        SyncReport {
1602            checked_at: None,
1603            peers: Some(peers),
1604            synced_before,
1605            last_success_at: last,
1606            last_cursor_at: None,
1607            floor: None,
1608            snapshot_merge: None,
1609        }
1610    }
1611
1612    #[test]
1613    fn without_a_stamp_the_window_counts_from_when_records_last_moved() {
1614        let (health, _rx) = monitored();
1615        let seen = SyncReport {
1616            last_cursor_at: Some(hours(-48)),
1617            ..synced(0, true, None)
1618        };
1619        sync_seen(&health, seen.clone(), mins(0));
1620        let report = health.report();
1621        assert_eq!(report.state, State::Degraded);
1622        let reason = report.reason.unwrap();
1623        assert!(reason.contains("no sync has succeeded in 2d"), "{reason}");
1624
1625        let stamped = SyncReport {
1626            last_success_at: Some(mins(0)),
1627            ..seen
1628        };
1629        sync_seen(&health, stamped, mins(0));
1630        assert_eq!(health.report().state, State::Ok);
1631    }
1632
1633    #[test]
1634    fn reasons_count_one_peer_and_one_raw_in_the_singular() {
1635        let (health, _rx) = monitored();
1636        sync_seen(&health, synced(1, true, Some(hours(-30))), mins(0));
1637        distill_seen(&health, 1, Some(hours(-30)), mins(0));
1638        let reason = health.report().reason.unwrap();
1639        assert!(
1640            reason.contains("with the 1 device this device syncs with"),
1641            "{reason}"
1642        );
1643        assert!(reason.contains("1 raw memory has waited"), "{reason}");
1644    }
1645
1646    #[test]
1647    fn a_device_that_synced_before_and_lost_its_peers_degrades_with_the_fix() {
1648        let (health, mut rx) = monitored();
1649        sync_seen(&health, synced(0, true, Some(hours(-25))), mins(0));
1650        let report = health.report();
1651        assert_eq!(report.state, State::Degraded);
1652        let reason = report.reason.unwrap();
1653        assert!(
1654            reason.contains("no sync has succeeded in 25h 0m"),
1655            "{reason}"
1656        );
1657        assert!(reason.contains("no paired device"), "{reason}");
1658        assert!(reason.contains("`sb pair`"), "{reason}");
1659        assert!(reason.contains("sync_overdue_hours = 0"), "{reason}");
1660        assert_eq!(report.sync.peers, Some(0));
1661        assert_eq!(report.sync.checked_at, Some(mins(0)));
1662        assert_eq!(drain(&mut rx), vec![State::Degraded]);
1663
1664        sync_seen(&health, synced(1, true, Some(mins(1))), mins(1));
1665        assert_eq!(health.report().state, State::Ok);
1666    }
1667
1668    #[test]
1669    fn a_device_synced_by_hand_without_sync_peers_stays_ok_while_it_syncs() {
1670        let (health, mut rx) = monitored();
1671        let by_hand = SyncReport {
1672            peers: None,
1673            ..synced(0, true, Some(hours(10)))
1674        };
1675        alive(&health, hours(30));
1676        sync_seen(&health, by_hand.clone(), hours(30));
1677        assert_eq!(health.report().state, State::Ok);
1678        alive(&health, hours(34));
1679        sync_seen(&health, by_hand, hours(34));
1680        let report = health.report();
1681        assert_eq!(report.state, State::Degraded);
1682        let reason = report.reason.unwrap();
1683        assert!(
1684            reason.contains("no sync has succeeded in 24h 0m"),
1685            "{reason}"
1686        );
1687        assert!(reason.contains("sb sync <user@host>"), "{reason}");
1688        assert!(reason.contains("sync_overdue_hours = 0"), "{reason}");
1689        assert!(!reason.contains("sync-peers"), "{reason}");
1690        assert_eq!(drain(&mut rx), vec![State::Degraded]);
1691    }
1692
1693    #[test]
1694    fn a_solo_device_that_never_synced_stays_ok() {
1695        let (health, mut rx) = monitored();
1696        for h in [0, 25, 100] {
1697            alive(&health, hours(h));
1698            sync_seen(&health, synced(0, false, None), hours(h));
1699        }
1700        let report = health.report();
1701        assert_eq!(report.state, State::Ok, "{:?}", report.reason);
1702        assert!(drain(&mut rx).is_empty());
1703    }
1704
1705    #[test]
1706    fn a_sync_overdue_past_the_window_degrades_only_on_a_device_that_syncs() {
1707        let (health, _rx) = monitored();
1708        alive(&health, hours(0));
1709        sync_seen(&health, synced(2, true, Some(hours(-23))), hours(0));
1710        assert_eq!(health.report().state, State::Ok);
1711
1712        alive(&health, hours(1));
1713        sync_seen(&health, synced(2, true, Some(hours(-23))), hours(1));
1714        let report = health.report();
1715        assert_eq!(report.state, State::Degraded);
1716        let reason = report.reason.unwrap();
1717        assert!(
1718            reason.contains("no sync has succeeded in 24h 0m"),
1719            "{reason}"
1720        );
1721        assert!(reason.contains("2 devices"), "{reason}");
1722
1723        sync_seen(&health, synced(2, true, Some(hours(1))), hours(1));
1724        assert_eq!(health.report().state, State::Ok);
1725
1726        sync_seen(&health, synced(0, false, Some(hours(-500))), hours(1));
1727        assert_eq!(health.report().state, State::Ok);
1728    }
1729
1730    #[test]
1731    fn with_peers_and_no_recorded_success_the_window_starts_at_startup() {
1732        let (health, _rx) = monitored();
1733        alive(&health, hours(23));
1734        sync_seen(&health, synced(1, false, None), hours(23));
1735        assert_eq!(health.report().state, State::Ok);
1736        alive(&health, hours(24));
1737        sync_seen(&health, synced(1, false, None), hours(24));
1738        let reason = health.report().reason.unwrap();
1739        assert!(
1740            reason.contains("no sync has succeeded in 24h 0m"),
1741            "{reason}"
1742        );
1743    }
1744
1745    #[test]
1746    fn raws_left_undistilled_past_the_window_degrade_with_the_count() {
1747        let (health, mut rx) = monitored();
1748        alive(&health, hours(0));
1749        distill_seen(&health, 520, Some(hours(-23)), hours(0));
1750        assert_eq!(health.report().state, State::Ok);
1751        assert_eq!(health.report().distill.pending, 520);
1752
1753        alive(&health, hours(2));
1754        distill_seen(&health, 520, Some(hours(-23)), hours(2));
1755        let report = health.report();
1756        assert_eq!(report.state, State::Degraded);
1757        let reason = report.reason.unwrap();
1758        assert!(
1759            reason.contains("520 raw memories have waited 25h 0m to be distilled"),
1760            "{reason}"
1761        );
1762        assert!(reason.contains("undistilled_overdue_hours = 0"), "{reason}");
1763
1764        alive(&health, hours(3));
1765        distill_seen(&health, 0, None, hours(3));
1766        assert_eq!(health.report().state, State::Ok);
1767        assert_eq!(drain(&mut rx), vec![State::Degraded, State::Ok]);
1768    }
1769
1770    #[test]
1771    fn without_an_llm_endpoint_waiting_raws_report_the_mode_instead_of_degrading() {
1772        let (health, mut rx) = monitored();
1773        alive(&health, hours(100));
1774        distill_seen_in(
1775            &health,
1776            DistillMode::NoGpu,
1777            520,
1778            Some(hours(-100)),
1779            hours(100),
1780        );
1781        let report = health.report();
1782        assert_eq!(report.state, State::Ok, "{:?}", report.reason);
1783        assert_eq!(report.reason, None);
1784        assert_eq!(report.distill.pending, 520);
1785        let json = serde_json::to_value(&report).unwrap();
1786        assert_eq!(json["distill"]["mode"], "no-gpu");
1787        assert_eq!(json["distill"]["note"], NO_GPU_NOTE);
1788        assert!(drain(&mut rx).is_empty());
1789    }
1790
1791    #[test]
1792    fn with_an_llm_endpoint_the_report_names_the_mode_and_carries_no_note() {
1793        let (health, _rx) = monitored();
1794        alive(&health, hours(2));
1795        distill_seen(&health, 3, Some(hours(-23)), hours(2));
1796        let report = health.report();
1797        assert_eq!(report.state, State::Degraded);
1798        let json = serde_json::to_value(&report).unwrap();
1799        assert_eq!(json["distill"]["mode"], "llm");
1800        assert!(json["distill"].get("note").is_none(), "{json}");
1801    }
1802
1803    #[test]
1804    fn a_window_turned_off_never_degrades() {
1805        let (tx, _rx) = unbounded_channel();
1806        let windows = Windows::from_config(&AlertConfig {
1807            sync_overdue_hours: 0,
1808            undistilled_overdue_hours: 0,
1809            ..AlertConfig::default()
1810        });
1811        assert_eq!(
1812            windows,
1813            Windows {
1814                sync_overdue: None,
1815                undistilled: None
1816            }
1817        );
1818        let health = Health::new(t0(), Some(tx)).with_windows(windows);
1819        alive(&health, hours(100));
1820        sync_seen(&health, synced(1, true, Some(hours(-100))), hours(100));
1821        distill_seen(&health, 9, Some(hours(-100)), hours(100));
1822        assert_eq!(health.report().state, State::Ok);
1823        sync_seen(&health, synced(0, true, Some(hours(-100))), hours(100));
1824        assert_eq!(health.report().state, State::Ok);
1825    }
1826
1827    #[test]
1828    fn the_windows_come_from_the_alert_config() {
1829        let windows = Windows::from_config(&AlertConfig {
1830            sync_overdue_hours: 6,
1831            undistilled_overdue_hours: 48,
1832            ..AlertConfig::default()
1833        });
1834        assert_eq!(windows.sync_overdue, Some(Duration::hours(6)));
1835        assert_eq!(windows.undistilled, Some(Duration::hours(48)));
1836        assert_eq!(Windows::default().sync_overdue, Some(Duration::hours(24)));
1837        assert_eq!(Windows::default().undistilled, Some(Duration::hours(24)));
1838    }
1839
1840    #[test]
1841    fn announce_sends_the_current_state_once_unless_it_was_already_sent() {
1842        let (health, mut rx) = monitored();
1843        health.announce(mins(0));
1844        let sent: Vec<Transition> = std::iter::from_fn(|| rx.try_recv().ok()).collect();
1845        assert_eq!(
1846            sent,
1847            vec![Transition {
1848                state: State::Ok,
1849                reason: None,
1850                at: mins(0),
1851            }]
1852        );
1853
1854        health.announce(mins(1));
1855        assert!(drain(&mut rx).is_empty(), "ok was already sent");
1856
1857        sync_seen(&health, synced(0, true, Some(hours(-30))), mins(2));
1858        health.announce(mins(3));
1859        assert_eq!(drain(&mut rx), vec![State::Degraded]);
1860    }
1861
1862    #[test]
1863    fn announce_reassesses_and_sends_one_message_for_a_pending_change() {
1864        let (health, mut rx) = monitored();
1865        alive(&health, hours(0));
1866        distill_seen(&health, 3, Some(hours(-23)), hours(0));
1867        assert!(drain(&mut rx).is_empty());
1868        health.announce(hours(2));
1869        assert_eq!(drain(&mut rx), vec![State::Degraded]);
1870    }
1871
1872    #[test]
1873    fn serving_leaves_out_sync_and_distillation() {
1874        let (health, _rx) = monitored();
1875        alive(&health, hours(0));
1876        let seen = ProgressSeen {
1877            sync: Some(synced(1, true, Some(hours(-30)))),
1878            distill: Some(DistillReport {
1879                pending: 4,
1880                oldest_at: Some(hours(-30)),
1881                ..DistillReport::default()
1882            }),
1883            ingested_sessions: None,
1884            error: None,
1885        };
1886        health.progress_checked(seen, hours(0));
1887        let report = health.report();
1888        assert_eq!(report.state, State::Degraded);
1889        assert_eq!(
1890            report.serving,
1891            Verdict {
1892                state: State::Ok,
1893                reason: None
1894            }
1895        );
1896
1897        health.watcher_failed("inotify watch limit reached", hours(0));
1898        let report = health.report();
1899        assert_eq!(report.state, State::Failing);
1900        assert_eq!(report.serving.state, State::Failing);
1901        let serving = report.serving.reason.unwrap();
1902        assert!(serving.contains("watcher is not running"), "{serving}");
1903        assert!(!serving.contains("sync"), "{serving}");
1904
1905        let json = serde_json::to_value(health.report()).unwrap();
1906        assert_eq!(json["serving"]["state"], "failing");
1907    }
1908
1909    #[test]
1910    fn one_pass_that_clears_sync_and_finds_distillation_overdue_sends_nothing() {
1911        let (health, mut rx) = monitored();
1912        alive(&health, hours(0));
1913        sync_seen(&health, synced(1, true, Some(hours(-30))), hours(0));
1914        assert_eq!(drain(&mut rx), vec![State::Degraded]);
1915        let seen = ProgressSeen {
1916            sync: Some(synced(1, true, Some(hours(0)))),
1917            distill: Some(DistillReport {
1918                pending: 4,
1919                oldest_at: Some(hours(-30)),
1920                ..DistillReport::default()
1921            }),
1922            ingested_sessions: None,
1923            error: None,
1924        };
1925        health.progress_checked(seen, hours(0));
1926        assert_eq!(health.report().state, State::Degraded);
1927        assert!(drain(&mut rx).is_empty());
1928    }
1929
1930    fn backfill_running(done: usize, failed: usize, at: DateTime<Utc>) -> BackfillReport {
1931        BackfillReport {
1932            state: BackfillState::Running,
1933            total: 40,
1934            done,
1935            failed,
1936            updated_at: Some(at),
1937        }
1938    }
1939
1940    #[test]
1941    fn a_new_daemon_reports_the_backfill_not_started() {
1942        let (health, _rx) = monitored();
1943        let report = health.report();
1944        assert_eq!(report.backfill, BackfillReport::default());
1945        assert_eq!(report.backfill.state, BackfillState::NotStarted);
1946        assert_eq!(report.backfill.total, 0);
1947        assert_eq!(report.backfill.done, 0);
1948        assert_eq!(report.backfill.failed, 0);
1949        let json = serde_json::to_value(report).unwrap();
1950        assert_eq!(json["backfill"]["state"], "not_started");
1951    }
1952
1953    #[test]
1954    fn backfill_progress_is_reported_and_survives_later_reports() {
1955        let (health, _rx) = monitored();
1956        health.backfill_progress(backfill_running(3, 1, mins(1)));
1957        assert_eq!(health.report().backfill, backfill_running(3, 1, mins(1)));
1958
1959        health.ingested(None, mins(2));
1960        assert_eq!(health.report().backfill, backfill_running(3, 1, mins(1)));
1961
1962        let done = BackfillReport {
1963            state: BackfillState::Done,
1964            total: 40,
1965            done: 40,
1966            failed: 1,
1967            updated_at: Some(mins(3)),
1968        };
1969        health.backfill_progress(done.clone());
1970        assert_eq!(health.report().backfill, done);
1971        let json = serde_json::to_value(health.report()).unwrap();
1972        assert_eq!(json["backfill"]["state"], "done");
1973        assert_eq!(json["backfill"]["done"], 40);
1974    }
1975
1976    #[test]
1977    fn backfill_progress_leaves_the_state_and_reason_alone() {
1978        let (health, mut rx) = monitored();
1979        for m in 0..=12 {
1980            health.ingest_failed("database is locked", mins(m));
1981        }
1982        assert_eq!(drain(&mut rx), vec![State::Degraded]);
1983        let before = health.report();
1984
1985        health.backfill_progress(backfill_running(5, 2, mins(13)));
1986        let after = health.report();
1987        assert_eq!(after.state, before.state);
1988        assert_eq!(after.reason, before.reason);
1989        assert_eq!(after.since, before.since);
1990        assert_eq!(after.serving, before.serving);
1991        assert!(drain(&mut rx).is_empty());
1992    }
1993
1994    fn held(secs: u64) -> Option<std::time::Duration> {
1995        Some(std::time::Duration::from_secs(secs))
1996    }
1997
1998    #[test]
1999    fn a_clock_gate_closed_past_the_grace_degrades_serving_until_it_opens() {
2000        let (health, mut rx) = monitored();
2001        alive(&health, mins(0));
2002        health.clock_gate(held(0), mins(0));
2003        health.clock_gate(held(90), mins(0) + Duration::seconds(90));
2004        assert_eq!(
2005            health.report().state,
2006            State::Ok,
2007            "a brief hold sends nothing"
2008        );
2009
2010        health.clock_gate(held(150), mins(3));
2011        let report = health.report();
2012        assert_eq!(report.state, State::Degraded);
2013        assert_eq!(report.serving.state, State::Degraded);
2014        let reason = report.reason.unwrap();
2015        assert!(
2016            reason.contains("clock not yet trusted: writes held"),
2017            "{reason}"
2018        );
2019
2020        health.clock_gate(None, mins(4));
2021        let report = health.report();
2022        assert_eq!(report.state, State::Ok);
2023        assert_eq!(report.reason, None);
2024        assert_eq!(drain(&mut rx), vec![State::Degraded, State::Ok]);
2025    }
2026
2027    #[test]
2028    fn a_short_monotonic_hold_stays_ok_when_the_wall_clock_steps_forward() {
2029        let (health, mut rx) = monitored();
2030        alive(&health, mins(0));
2031        health.clock_gate(held(0), mins(0));
2032        alive(&health, mins(50));
2033        health.clock_gate(held(3), mins(50));
2034        alive(&health, mins(50) + Duration::seconds(1));
2035        assert_eq!(health.report().state, State::Ok);
2036        assert!(drain(&mut rx).is_empty());
2037    }
2038
2039    #[test]
2040    fn a_rejected_handshake_degrades_naming_the_reason_for_a_day() {
2041        let (health, _rx) = monitored();
2042        alive(&health, hours(0));
2043        health.clock_handshake_rejected(
2044            "peer-b",
2045            "from peer-b: peer time is more than a day ahead".to_string(),
2046            hours(0),
2047        );
2048        let report = health.report();
2049        assert_eq!(report.state, State::Degraded);
2050        assert_eq!(report.serving.state, State::Ok);
2051        let reason = report.reason.unwrap();
2052        assert!(reason.contains("more than a day ahead"), "{reason}");
2053        assert_eq!(report.last_error.unwrap().source, "clock");
2054
2055        alive(&health, hours(25));
2056        health.clock_gate(None, hours(25));
2057        assert_eq!(health.report().state, State::Ok);
2058    }
2059
2060    #[test]
2061    fn a_basis_ahead_of_real_time_degrades_until_time_catches_up() {
2062        let (health, _rx) = monitored();
2063        alive(&health, mins(0));
2064        health.clock_basis_ahead(true, mins(0));
2065        let report = health.report();
2066        assert_eq!(report.state, State::Degraded);
2067        assert_eq!(report.serving.state, State::Ok);
2068        let reason = report.reason.unwrap();
2069        assert!(reason.contains("ahead of synchronised time"), "{reason}");
2070
2071        health.clock_basis_ahead(false, mins(1));
2072        assert_eq!(health.report().state, State::Ok);
2073    }
2074
2075    fn floor_health(silent: &[&str], stuck: &[&str]) -> FloorHealth {
2076        FloorHealth {
2077            floor: 5,
2078            log_floor: 2,
2079            head: 9,
2080            top: 9,
2081            silent: silent.iter().map(|s| s.to_string()).collect(),
2082            stuck: stuck.iter().map(|s| s.to_string()).collect(),
2083        }
2084    }
2085
2086    #[test]
2087    fn before_the_first_floor_check_there_is_no_floor_section() {
2088        let (health, _rx) = monitored();
2089        alive(&health, mins(0));
2090        let report = health.report();
2091        assert_eq!(report.state, State::Ok);
2092        assert!(report.sync.floor.is_none());
2093    }
2094
2095    #[test]
2096    fn a_silent_neighbour_degrades_and_is_named() {
2097        let (health, _rx) = monitored();
2098        alive(&health, mins(0));
2099        health.floor_checked(
2100            floor_health(&["peer-a"], &[]),
2101            FailedCounts::default(),
2102            mins(0),
2103        );
2104        let report = health.report();
2105        assert_eq!(report.state, State::Degraded);
2106        let reason = report.reason.unwrap();
2107        assert!(
2108            reason.contains("peer-a") && reason.contains("silent"),
2109            "{reason}"
2110        );
2111        assert_eq!(report.sync.floor.unwrap().silent, vec!["peer-a"]);
2112
2113        health.floor_checked(floor_health(&[], &[]), FailedCounts::default(), mins(1));
2114        assert_eq!(health.report().state, State::Ok);
2115    }
2116
2117    fn standing(peer: &str, holds_own_row: bool, lacking: &[u32]) -> PeerStanding {
2118        PeerStanding {
2119            peer: peer.to_string(),
2120            heard_at_micros: 1,
2121            holds_own_row,
2122            lacking: lacking.to_vec(),
2123        }
2124    }
2125
2126    fn identity(peers: Vec<PeerStanding>, removed: bool) -> IdentityStatus {
2127        let row_accepted = (!peers.is_empty()).then(|| peers.iter().any(|p| p.holds_own_row));
2128        let possibly_revoked = removed
2129            || peers
2130                .iter()
2131                .any(|p| p.holds_own_row && !p.lacking.is_empty());
2132        IdentityStatus {
2133            identity: Some("m-0001".to_string()),
2134            epochs: vec![0, 1],
2135            removed,
2136            possibly_revoked,
2137            row_accepted,
2138            peers,
2139            retired_recovery: Vec::new(),
2140        }
2141    }
2142
2143    #[test]
2144    fn identity_state_is_reported_and_judged_only_once_peers_were_heard() {
2145        let (health, _rx) = monitored();
2146        alive(&health, mins(0));
2147        health.key_storage_detected("tpm", mins(0));
2148        assert!(health.report().identity.status.is_none());
2149        assert_eq!(health.report().identity.key_storage, Some("tpm"));
2150
2151        health.identity_checked(identity(Vec::new(), false), mins(1));
2152        let report = health.report();
2153        assert_eq!(report.state, State::Ok, "{:?}", report.reason);
2154        assert_eq!(report.identity.checked_at, Some(mins(1)));
2155        let status = report.identity.status.unwrap();
2156        assert_eq!(status.epochs, vec![0, 1]);
2157        assert_eq!(status.row_accepted, None);
2158
2159        health.identity_checked(identity(vec![standing("pi", true, &[])], false), mins(2));
2160        assert_eq!(health.report().state, State::Ok);
2161    }
2162
2163    #[test]
2164    fn a_device_with_peers_but_no_identity_degrades_until_it_is_paired() {
2165        let (health, _rx) = monitored();
2166        alive(&health, mins(0));
2167        health.identity_checked(IdentityStatus::default(), mins(1));
2168        assert_eq!(
2169            health.report().state,
2170            State::Ok,
2171            "no peers, nothing to sync"
2172        );
2173
2174        sync_seen(
2175            &health,
2176            SyncReport {
2177                checked_at: Some(mins(2)),
2178                peers: Some(1),
2179                ..SyncReport::default()
2180            },
2181            mins(2),
2182        );
2183        let report = health.report();
2184        assert_eq!(report.state, State::Degraded);
2185        let reason = report.reason.unwrap();
2186        assert!(reason.contains("holds no sync identity key"), "{reason}");
2187        assert!(
2188            reason.contains("`sb pair <code>` on this device"),
2189            "{reason}"
2190        );
2191
2192        health.identity_checked(identity(vec![standing("pi", true, &[])], false), mins(3));
2193        let reason = health.report().reason.unwrap_or_default();
2194        assert!(!reason.contains("sync identity"), "{reason}");
2195    }
2196
2197    #[test]
2198    fn an_unreadable_hardware_id_is_reported_and_never_degrades() {
2199        let (health, _rx) = monitored();
2200        alive(&health, mins(0));
2201        assert_eq!(health.report().identity.hardware, None);
2202        health.hardware_checked("unreadable", mins(0));
2203        health.identity_checked(identity(vec![standing("pi", true, &[])], false), mins(1));
2204        let report = health.report();
2205        assert_eq!(report.identity.hardware, Some("unreadable"));
2206        assert_eq!(report.state, State::Ok, "{:?}", report.reason);
2207        let json = serde_json::to_value(&report.identity).unwrap();
2208        assert_eq!(json["hardware"], "unreadable");
2209    }
2210
2211    #[test]
2212    fn a_reminted_copy_with_peers_is_told_it_came_from_other_hardware_and_to_pair() {
2213        let (health, _rx) = monitored();
2214        alive(&health, mins(0));
2215        health.hardware_checked("reminted", mins(0));
2216        health.identity_checked(IdentityStatus::default(), mins(1));
2217        sync_seen(
2218            &health,
2219            SyncReport {
2220                checked_at: Some(mins(2)),
2221                peers: Some(1),
2222                ..SyncReport::default()
2223            },
2224            mins(2),
2225        );
2226        let report = health.report();
2227        assert_eq!(report.state, State::Degraded);
2228        let reason = report.reason.unwrap();
2229        assert!(reason.contains("copied from other hardware"), "{reason}");
2230        assert!(reason.contains("new machine id"), "{reason}");
2231        assert!(
2232            reason.contains("`sb pair <code>` on this device"),
2233            "{reason}"
2234        );
2235
2236        health.identity_checked(identity(vec![standing("pi", true, &[])], false), mins(3));
2237        let reason = health.report().reason.unwrap_or_default();
2238        assert!(!reason.contains("other hardware"), "{reason}");
2239    }
2240
2241    #[test]
2242    fn a_row_no_peer_accepted_degrades_naming_pairing() {
2243        let (health, _rx) = monitored();
2244        alive(&health, mins(0));
2245        health.identity_checked(
2246            identity(
2247                vec![standing("pi", false, &[]), standing("mac", false, &[])],
2248                false,
2249            ),
2250            mins(1),
2251        );
2252        let report = health.report();
2253        assert_eq!(report.state, State::Degraded);
2254        let reason = report.reason.unwrap();
2255        assert!(reason.contains("no peer has accepted"), "{reason}");
2256        assert!(reason.contains("sb pair"), "{reason}");
2257
2258        health.identity_checked(
2259            identity(
2260                vec![standing("pi", false, &[]), standing("mac", true, &[])],
2261                false,
2262            ),
2263            mins(2),
2264        );
2265        assert_eq!(health.report().state, State::Ok);
2266    }
2267
2268    #[test]
2269    fn a_withheld_epoch_or_a_removal_degrades_as_possibly_revoked() {
2270        let (health, _rx) = monitored();
2271        alive(&health, mins(0));
2272        health.identity_checked(identity(vec![standing("pi", true, &[2])], false), mins(1));
2273        let reason = health.report().reason.unwrap();
2274        assert!(reason.contains("pi holds identity key epoch 2"), "{reason}");
2275        assert!(reason.contains("may have been removed"), "{reason}");
2276
2277        health.identity_checked(identity(vec![standing("pi", true, &[])], true), mins(2));
2278        let reason = health.report().reason.unwrap();
2279        assert!(
2280            reason.contains("removed from its sync identity"),
2281            "{reason}"
2282        );
2283        assert!(reason.contains("sb pair <code>"), "{reason}");
2284        assert!(!reason.contains("may have been"), "{reason}");
2285    }
2286
2287    #[test]
2288    fn a_stuck_neighbour_degrades_and_is_named() {
2289        let (health, _rx) = monitored();
2290        alive(&health, mins(0));
2291        health.floor_checked(
2292            floor_health(&[], &["peer-b"]),
2293            FailedCounts::default(),
2294            mins(0),
2295        );
2296        let report = health.report();
2297        assert_eq!(report.state, State::Degraded);
2298        let reason = report.reason.unwrap();
2299        assert!(
2300            reason.contains("peer-b") && reason.contains("not moved"),
2301            "{reason}"
2302        );
2303    }
2304
2305    #[test]
2306    fn a_failing_snapshot_merge_degrades_and_a_running_one_does_not() {
2307        use crate::snapshot_merge::MergeFailing;
2308        use second_brain_sync::protocol::Merging;
2309        let (health, _rx) = monitored();
2310        alive(&health, mins(0));
2311        let merging = Merging {
2312            snapshot_id: "s1".to_string(),
2313            exporter: "peer-a".to_string(),
2314            seq: 42,
2315        };
2316        health.snapshot_merge_checked(
2317            Some(MergeStatus {
2318                merging: merging.clone(),
2319                failing: None,
2320            }),
2321            mins(0),
2322        );
2323        assert_eq!(health.report().state, State::Ok);
2324        assert!(health.report().sync.snapshot_merge.is_some());
2325
2326        health.snapshot_merge_checked(
2327            Some(MergeStatus {
2328                merging,
2329                failing: Some(MergeFailing {
2330                    since: 0,
2331                    attempts: 3,
2332                    error: "disk full".to_string(),
2333                }),
2334            }),
2335            mins(1),
2336        );
2337        let report = health.report();
2338        assert_eq!(report.state, State::Degraded);
2339        let reason = report.reason.unwrap();
2340        assert!(
2341            reason.contains("peer-a") && reason.contains("disk full"),
2342            "{reason}"
2343        );
2344
2345        health.snapshot_merge_checked(None, mins(2));
2346        assert_eq!(health.report().state, State::Ok);
2347    }
2348
2349    #[test]
2350    fn settled_records_are_reported_without_degrading() {
2351        let (health, _rx) = monitored();
2352        alive(&health, mins(0));
2353        health.floor_checked(
2354            floor_health(&[], &[]),
2355            FailedCounts {
2356                failing: 2,
2357                settled: 3,
2358                dropped: 4,
2359            },
2360            mins(0),
2361        );
2362        let report = health.report();
2363        assert_eq!(report.state, State::Ok);
2364        let floor = report.sync.floor.unwrap();
2365        assert_eq!(
2366            (floor.failing, floor.settled, floor.settled_dropped),
2367            (2, 3, 4)
2368        );
2369    }
2370
2371    fn empty_recall(session: &str) -> RecallEvent {
2372        RecallEvent {
2373            session: Some(session.to_string()),
2374            verdict: crate::value_metrics::RecallVerdict::Empty,
2375            memory_ids: Vec::new(),
2376        }
2377    }
2378
2379    #[test]
2380    fn a_value_metric_crossing_its_threshold_sends_exactly_one_transition() {
2381        let (health, mut rx) = monitored();
2382        alive(&health, mins(0));
2383        for _ in 0..49 {
2384            health.recalled(empty_recall("s"), mins(1));
2385        }
2386        assert!(drain(&mut rx).is_empty());
2387        for _ in 0..30 {
2388            health.recalled(empty_recall("s"), mins(2));
2389        }
2390        let report = health.report();
2391        assert_eq!(report.state, State::Degraded);
2392        assert_eq!(report.serving.state, State::Ok);
2393        let reason = report.reason.unwrap();
2394        assert!(reason.contains("found nothing that cleared"), "{reason}");
2395        assert_eq!(drain(&mut rx), vec![State::Degraded]);
2396    }
2397
2398    #[test]
2399    fn a_share_hovering_at_its_threshold_sends_one_transition_each_way() {
2400        let (health, mut rx) = monitored();
2401        let hit = || RecallEvent {
2402            session: None,
2403            verdict: crate::value_metrics::RecallVerdict::Hit,
2404            memory_ids: Vec::new(),
2405        };
2406        for m in 0..50 {
2407            alive(&health, mins(m));
2408            let event = if m < 37 { empty_recall("s") } else { hit() };
2409            health.recalled(event, mins(m));
2410        }
2411        assert_eq!(health.report().state, State::Ok, "37 of 50 is 74%");
2412        for _ in 0..3 {
2413            health.recalled(empty_recall("s"), mins(50));
2414        }
2415        assert_eq!(drain(&mut rx), vec![State::Degraded], "40 of 53 is 75.5%");
2416        for m in 51..56 {
2417            alive(&health, mins(m));
2418            health.recalled(hit(), mins(m));
2419            health.recalled(empty_recall("s"), mins(m));
2420        }
2421        assert!(drain(&mut rx).is_empty(), "74% then 71% stays degraded");
2422        for m in 56..66 {
2423            alive(&health, mins(m));
2424            health.recalled(hit(), mins(m));
2425        }
2426        assert_eq!(health.report().value.recall.recalls, 73);
2427        assert_eq!(drain(&mut rx), vec![State::Ok], "45 of 73 is 62%");
2428    }
2429
2430    #[test]
2431    fn peer_lag_is_left_out_when_no_sync_succeeded_at_all() {
2432        let (health, _rx) = monitored();
2433        health.sync_peers_listed(&["u@pi".to_string()], hours(0));
2434        alive(&health, hours(30));
2435        let reason = health.report().reason.unwrap();
2436        assert!(reason.contains("no sync with u@pi"), "{reason}");
2437
2438        sync_seen(&health, synced(1, true, Some(hours(-30))), hours(30));
2439        let reason = health.report().reason.unwrap();
2440        assert!(reason.contains("no sync has succeeded in"), "{reason}");
2441        assert!(!reason.contains("no sync with u@pi"), "{reason}");
2442        assert_eq!(health.report().value.sync_peers.len(), 1);
2443    }
2444
2445    #[test]
2446    fn the_ingest_window_starts_after_startup_and_any_backfill() {
2447        let (health, _rx) = monitored();
2448        assert_eq!(health.ingest_window_start(mins(5)), Some(t0()));
2449        assert_eq!(
2450            health.ingest_window_start(hours(30)),
2451            Some(hours(30) - WINDOW)
2452        );
2453        health.backfill_progress(backfill_running(1, 0, mins(1)));
2454        assert_eq!(health.ingest_window_start(mins(5)), None);
2455        health.backfill_progress(BackfillReport {
2456            state: BackfillState::Done,
2457            updated_at: Some(mins(20)),
2458            ..BackfillReport::default()
2459        });
2460        assert_eq!(health.ingest_window_start(mins(30)), Some(mins(20)));
2461    }
2462
2463    #[test]
2464    fn progress_sets_the_ingested_sessions_and_a_failed_read_clears_them() {
2465        let (health, _rx) = monitored();
2466        health.recalled(empty_recall("a"), mins(1));
2467        let seen = ProgressSeen {
2468            ingested_sessions: Some(vec!["a".to_string(), "b".to_string()]),
2469            ..ProgressSeen::default()
2470        };
2471        health.progress_checked(seen, mins(2));
2472        let recall = health.report().value.recall;
2473        assert_eq!(recall.sessions_ingested, Some(2));
2474        assert_eq!(recall.session_recall_rate, Some(0.5));
2475        health.progress_checked(ProgressSeen::default(), mins(3));
2476        assert_eq!(health.report().value.recall.sessions_ingested, None);
2477    }
2478
2479    #[test]
2480    fn every_value_metric_is_in_the_report_json() {
2481        let (health, _rx) = monitored();
2482        health.sync_peers_listed(&["u@pi".to_string()], mins(0));
2483        health.store_measured(Some(StoreSize::default()), mins(0));
2484        health.resident_measured(Some(ResidentSize::default()), mins(0));
2485        let json = serde_json::to_value(health.report()).unwrap();
2486        let keys = |value: &serde_json::Value| -> Vec<String> {
2487            let mut keys: Vec<String> = value.as_object().unwrap().keys().cloned().collect();
2488            keys.sort();
2489            keys
2490        };
2491        assert_eq!(
2492            keys(&json["sync_peers"][0]),
2493            [
2494                "consecutive_failures",
2495                "lag_secs",
2496                "last_attempt_at",
2497                "last_error",
2498                "last_success_at",
2499                "peer"
2500            ]
2501        );
2502        assert_eq!(
2503            keys(&json["store"]),
2504            [
2505                "archive_bytes",
2506                "checked_at",
2507                "cold_bytes",
2508                "db_bytes",
2509                "free_bytes",
2510                "index_bytes",
2511                "raw_index_bytes",
2512                "total_bytes",
2513                "wal_bytes"
2514            ]
2515        );
2516        assert_eq!(
2517            keys(&json["resident"]),
2518            [
2519                "checked_at",
2520                "components",
2521                "limit_bytes",
2522                "raw_index_bytes",
2523                "rss_anon_bytes",
2524                "unattributed_bytes"
2525            ]
2526        );
2527        assert_eq!(
2528            keys(&json["resident"]["components"]),
2529            [
2530                "allocator_free",
2531                "embedder",
2532                "lexical",
2533                "reranker",
2534                "sqlite",
2535                "vector_reader",
2536                "vector_writer"
2537            ]
2538        );
2539        for key in ["recall", "recalled", "sync_peers", "store", "resident"] {
2540            assert!(json.get(key).is_some(), "{key} missing: {json}");
2541        }
2542        for key in [
2543            "window_secs",
2544            "recalls",
2545            "hits",
2546            "low_score",
2547            "unjudged",
2548            "empty",
2549            "empty_share",
2550            "low_score_share",
2551            "unjudged_share",
2552            "sessions_with_recall",
2553            "sessions_ingested",
2554            "session_recall_rate",
2555        ] {
2556            assert!(json["recall"].get(key).is_some(), "recall.{key}: {json}");
2557        }
2558        for key in [
2559            "window_secs",
2560            "memories",
2561            "corrected",
2562            "used",
2563            "corrected_share",
2564            "used_share",
2565        ] {
2566            assert!(
2567                json["recalled"].get(key).is_some(),
2568                "recalled.{key}: {json}"
2569            );
2570        }
2571    }
2572}