Skip to main content

second_brain_api/
health.rs

1//! The daemon's own view of whether it is capturing and indexing. The
2//! watcher, the outbox drain and the repair loop report what they see; each
3//! report re-assesses the state, so `GET /health` only clones the result.
4
5use std::sync::Mutex;
6
7use chrono::{DateTime, Duration, Utc};
8use second_brain_core::config::AlertConfig;
9use second_brain_core::sqlite_store::VacuumSpace;
10use serde::Serialize;
11use tokio::sync::mpsc::UnboundedSender;
12use uuid::Uuid;
13
14use crate::value_metrics::{
15    PeerOutcome, RecallEvent, ResidentSize, StoreSize, ValueFacts, ValueProblems, ValueReport,
16    WINDOW,
17};
18
19const CAPTURE_LAG_DEGRADED: Duration = Duration::minutes(15);
20const CAPTURE_LAG_FAILING: Duration = Duration::hours(2);
21const BACKLOG_DEGRADED: Duration = Duration::minutes(15);
22const BACKLOG_FAILING: Duration = Duration::hours(2);
23/// Three missed transcript scans (see `watcher::SCAN_INTERVAL`).
24const WATCHER_SILENT: Duration = Duration::minutes(15);
25/// A scheduled repair holds the drain thread for minutes on a large store.
26const DRAIN_SILENT: Duration = Duration::hours(1);
27/// An ingest or drain error must outlast this before it degrades, so one
28/// `database is locked` that the next attempt clears sends no webhook.
29const ERROR_GRACE: Duration = Duration::minutes(10);
30/// The replayer runs every 30 s, so an entry this old is not being sent.
31const QUEUE_STUCK: Duration = Duration::minutes(10);
32
33#[derive(Debug, Clone, Copy, Default, PartialEq, Eq, PartialOrd, Ord, Serialize)]
34#[serde(rename_all = "lowercase")]
35pub enum State {
36    #[default]
37    Ok,
38    Degraded,
39    Failing,
40}
41
42#[derive(Debug, Clone, PartialEq, Serialize)]
43pub struct Transition {
44    pub state: State,
45    pub reason: Option<String>,
46    pub at: DateTime<Utc>,
47}
48
49#[derive(Debug, Clone, Serialize)]
50pub struct ErrorRecord {
51    pub source: &'static str,
52    pub message: String,
53    pub at: DateTime<Utc>,
54}
55
56#[derive(Debug, Clone, Default, Serialize)]
57pub struct CaptureReport {
58    pub checked_at: Option<DateTime<Utc>>,
59    pub newest_transcript_at: Option<DateTime<Utc>>,
60    /// Modification time of the newest transcript this daemon ingested.
61    pub captured_through: Option<DateTime<Utc>>,
62    pub lag_secs: Option<i64>,
63}
64
65#[derive(Debug, Clone, Default, Serialize)]
66pub struct IndexReport {
67    pub checked_at: Option<DateTime<Utc>>,
68    pub outbox_pending: Option<u64>,
69    pub backlog_since: Option<DateTime<Utc>>,
70}
71
72/// Only the tool, status and time: the daemon's message can echo the
73/// write's text, and `/health` is unauthenticated.
74#[derive(Debug, Clone, PartialEq, Serialize)]
75pub struct RejectedWrite {
76    pub tool: String,
77    pub status: u16,
78    pub at: DateTime<Utc>,
79}
80
81#[derive(Debug, Clone, Default, Serialize)]
82pub struct QueueReport {
83    pub depth: u64,
84    pub oldest_at: Option<DateTime<Utc>>,
85    pub oldest_age_secs: Option<i64>,
86    pub last_rejected: Option<RejectedWrite>,
87}
88
89/// Where the one-time compaction after the stored-content redaction pass
90/// stands: a VACUUM of the store and a merge of the lexical index, which
91/// clear the plaintext the pass left in free pages and old segments.
92#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize)]
93#[serde(rename_all = "snake_case")]
94pub enum MaintenanceState {
95    Pending,
96    WaitingForSpace,
97    MergingIndex,
98    /// Agent writes are queued until it ends.
99    Vacuuming,
100    Done,
101}
102
103#[derive(Debug, Clone, Serialize)]
104pub struct MaintenanceReport {
105    pub state: MaintenanceState,
106    pub since: Option<DateTime<Utc>>,
107    pub space: Option<VacuumSpace>,
108    pub last_error: Option<String>,
109}
110
111impl Default for MaintenanceReport {
112    fn default() -> Self {
113        Self {
114            state: MaintenanceState::Pending,
115            since: None,
116            space: None,
117            last_error: None,
118        }
119    }
120}
121
122#[derive(Debug, Clone, Copy, Default, PartialEq, Eq, Serialize)]
123#[serde(rename_all = "snake_case")]
124pub enum BackfillState {
125    #[default]
126    NotStarted,
127    Running,
128    Done,
129}
130
131#[derive(Debug, Clone, Default, PartialEq, Eq, Serialize)]
132pub struct BackfillReport {
133    pub state: BackfillState,
134    pub total: usize,
135    pub done: usize,
136    pub failed: usize,
137    pub updated_at: Option<DateTime<Utc>>,
138}
139
140#[derive(Debug, Clone, Default, Serialize)]
141pub struct SyncReport {
142    pub checked_at: Option<DateTime<Utc>>,
143    /// None when there is no `sync-peers` file.
144    pub peers: Option<usize>,
145    /// A sync cursor for some peer is on record.
146    pub synced_before: bool,
147    pub last_success_at: Option<DateTime<Utc>>,
148    /// The newest `sync_state` time: when records last moved, which bounds
149    /// the last success when no stamp was written yet.
150    pub last_cursor_at: Option<DateTime<Utc>>,
151}
152
153pub const NO_GPU_NOTE: &str = "no LLM endpoint configured and no LLM distillation in 7 days: \
154     raws covered by an agent note are retired, the rest stay searchable";
155
156/// `NoGpu` when no LLM endpoint is configured and no LLM distilled lately
157/// (`note_cover::distill_mode`), so waiting raws are expected rather than
158/// overdue.
159#[derive(Debug, Clone, Copy, Default, PartialEq, Eq, Serialize)]
160pub enum DistillMode {
161    #[default]
162    #[serde(rename = "llm")]
163    Llm,
164    #[serde(rename = "no-gpu")]
165    NoGpu,
166}
167
168impl DistillMode {
169    fn note(self) -> Option<&'static str> {
170        match self {
171            DistillMode::Llm => None,
172            DistillMode::NoGpu => Some(NO_GPU_NOTE),
173        }
174    }
175}
176
177#[derive(Debug, Clone, Default, Serialize)]
178pub struct DistillReport {
179    pub checked_at: Option<DateTime<Utc>>,
180    pub mode: DistillMode,
181    pub pending: u64,
182    pub oldest_at: Option<DateTime<Utc>>,
183    #[serde(skip_serializing_if = "Option::is_none")]
184    pub note: Option<&'static str>,
185}
186
187/// How long a sync or a raw's distillation may be overdue before it
188/// degrades; None turns that check off.
189#[derive(Debug, Clone, Copy, PartialEq)]
190pub struct Windows {
191    pub sync_overdue: Option<Duration>,
192    pub undistilled: Option<Duration>,
193}
194
195impl Windows {
196    pub fn from_config(config: &AlertConfig) -> Self {
197        let hours = |h: u64| {
198            i64::try_from(h)
199                .ok()
200                .filter(|&h| h > 0)
201                .and_then(Duration::try_hours)
202        };
203        Self {
204            sync_overdue: hours(config.sync_overdue_hours),
205            undistilled: hours(config.undistilled_overdue_hours),
206        }
207    }
208}
209
210impl Default for Windows {
211    fn default() -> Self {
212        Self::from_config(&AlertConfig::default())
213    }
214}
215
216/// What one progress pass read. None where the read failed keeps the
217/// previous reading; `error` says what failed.
218#[derive(Debug, Clone, Default)]
219pub struct ProgressSeen {
220    pub sync: Option<SyncReport>,
221    pub distill: Option<DistillReport>,
222    /// Claude Code sessions the ingest log shows captured since
223    /// `Health::ingest_window_start`; None when unknown.
224    pub ingested_sessions: Option<Vec<String>>,
225    pub error: Option<String>,
226}
227
228#[derive(Debug, Clone, PartialEq, Serialize)]
229pub struct Verdict {
230    pub state: State,
231    pub reason: Option<String>,
232}
233
234#[derive(Debug, Clone, Serialize)]
235pub struct HealthReport {
236    pub state: State,
237    pub reason: Option<String>,
238    pub since: DateTime<Utc>,
239    /// Capture and recall alone, leaving out sync and distillation: what an
240    /// agent using second-brain needs to be told about.
241    pub serving: Verdict,
242    pub capture: CaptureReport,
243    pub index: IndexReport,
244    pub queue: QueueReport,
245    pub maintenance: MaintenanceReport,
246    pub backfill: BackfillReport,
247    pub sync: SyncReport,
248    pub distill: DistillReport,
249    #[serde(flatten)]
250    pub value: ValueReport,
251    pub last_error: Option<ErrorRecord>,
252}
253
254/// What the reporters last said; only `assess` turns it into a state.
255struct Facts {
256    started: DateTime<Utc>,
257    capture: CaptureReport,
258    watcher_error: Option<String>,
259    ingest_error: Option<Streak>,
260    index: IndexReport,
261    drain_error: Option<Streak>,
262    indexes_error: Option<String>,
263    repair_error: Option<String>,
264    queue: QueueReport,
265    maintenance: MaintenanceReport,
266    backfill: BackfillReport,
267    sync: SyncReport,
268    distill: DistillReport,
269    progress_error: Option<Streak>,
270    value: ValueFacts,
271    windows: Windows,
272    last_error: Option<ErrorRecord>,
273}
274
275/// An error repeated since `since` with no success in between.
276struct Streak {
277    message: String,
278    since: DateTime<Utc>,
279}
280
281impl Streak {
282    fn extend(previous: Option<Streak>, message: String, now: DateTime<Utc>) -> Option<Streak> {
283        let since = previous.map_or(now, |p| p.since);
284        Some(Streak { message, since })
285    }
286
287    fn outlasted_grace(&self, now: DateTime<Utc>) -> bool {
288        now - self.since >= ERROR_GRACE
289    }
290}
291
292struct Inner {
293    facts: Facts,
294    report: HealthReport,
295    /// The state this process last sent, if it sent one.
296    sent: Option<State>,
297}
298
299pub struct Health {
300    inner: Mutex<Inner>,
301    transitions: Option<UnboundedSender<Transition>>,
302}
303
304impl Health {
305    /// `transitions` receives one message per state change, never one per
306    /// report; None when nothing listens.
307    pub fn new(now: DateTime<Utc>, transitions: Option<UnboundedSender<Transition>>) -> Self {
308        let mut facts = Facts {
309            started: now,
310            capture: CaptureReport::default(),
311            watcher_error: None,
312            ingest_error: None,
313            index: IndexReport::default(),
314            drain_error: None,
315            indexes_error: None,
316            repair_error: None,
317            queue: QueueReport::default(),
318            maintenance: MaintenanceReport::default(),
319            backfill: BackfillReport::default(),
320            sync: SyncReport::default(),
321            distill: DistillReport::default(),
322            progress_error: None,
323            value: ValueFacts::default(),
324            windows: Windows::default(),
325            last_error: None,
326        };
327        let report = HealthReport {
328            state: State::Ok,
329            reason: None,
330            since: now,
331            serving: Verdict {
332                state: State::Ok,
333                reason: None,
334            },
335            capture: CaptureReport::default(),
336            index: IndexReport::default(),
337            queue: QueueReport::default(),
338            maintenance: MaintenanceReport::default(),
339            backfill: BackfillReport::default(),
340            sync: SyncReport::default(),
341            distill: DistillReport::default(),
342            value: facts.value.refresh(now, now, facts.windows.sync_overdue).0,
343            last_error: None,
344        };
345        Self {
346            inner: Mutex::new(Inner {
347                facts,
348                report,
349                sent: None,
350            }),
351            transitions,
352        }
353    }
354
355    pub fn with_windows(self, windows: Windows) -> Self {
356        self.lock().facts.windows = windows;
357        self
358    }
359
360    pub fn progress_checked(&self, seen: ProgressSeen, now: DateTime<Utc>) {
361        self.update(now, |f| {
362            if let Some(sync) = seen.sync {
363                f.sync = SyncReport {
364                    checked_at: Some(now),
365                    ..sync
366                };
367            }
368            if let Some(distill) = seen.distill {
369                f.distill = DistillReport {
370                    checked_at: Some(now),
371                    note: distill.mode.note(),
372                    ..distill
373                };
374            }
375            f.value.sessions_ingested(seen.ingested_sessions);
376            match seen.error {
377                None => f.progress_error = None,
378                Some(error) => {
379                    f.record_error("progress", &error, now);
380                    f.progress_error = Streak::extend(f.progress_error.take(), error, now);
381                }
382            }
383        });
384    }
385
386    /// Sends the current state unless this process already sent it, so a
387    /// receiver that missed transitions while the daemon was down learns it.
388    pub fn announce(&self, now: DateTime<Utc>) {
389        let mut inner = self.lock();
390        self.reassess(&mut inner, now, true);
391    }
392
393    pub fn report(&self) -> HealthReport {
394        self.lock().report.clone()
395    }
396
397    pub fn transcripts_scanned(&self, newest: Option<DateTime<Utc>>, now: DateTime<Utc>) {
398        self.update(now, |f| {
399            f.capture.checked_at = Some(now);
400            f.capture.newest_transcript_at = newest;
401        });
402    }
403
404    pub fn watcher_started(&self, now: DateTime<Utc>) {
405        self.update(now, |f| {
406            f.watcher_error = None;
407            f.capture.checked_at = Some(now);
408        });
409    }
410
411    pub fn watcher_failed(&self, error: &str, now: DateTime<Utc>) {
412        self.update(now, |f| {
413            f.watcher_error = Some(error.to_string());
414            f.record_error("watcher", error, now);
415        });
416    }
417
418    /// `transcript_mtime` is None for sources that are not transcripts,
419    /// which clear an ingest error without advancing capture.
420    pub fn ingested(&self, transcript_mtime: Option<DateTime<Utc>>, now: DateTime<Utc>) {
421        self.update(now, |f| {
422            f.ingest_error = None;
423            f.capture.checked_at = Some(now);
424            if let Some(mtime) = transcript_mtime {
425                let through = f.capture.captured_through.map_or(mtime, |t| t.max(mtime));
426                f.capture.captured_through = Some(through);
427            }
428        });
429    }
430
431    pub fn ingest_failed(&self, error: &str, now: DateTime<Utc>) {
432        self.update(now, |f| {
433            f.ingest_error = Streak::extend(f.ingest_error.take(), error.to_string(), now);
434            f.capture.checked_at = Some(now);
435            f.record_error("ingest", error, now);
436        });
437    }
438
439    pub fn drained(&self, outbox_pending: u64, now: DateTime<Utc>) {
440        self.update(now, |f| {
441            f.drain_error = None;
442            f.index.checked_at = Some(now);
443            f.index.outbox_pending = Some(outbox_pending);
444            f.index.backlog_since = if outbox_pending == 0 {
445                None
446            } else {
447                Some(f.index.backlog_since.unwrap_or(now))
448            };
449        });
450    }
451
452    pub fn drain_failed(&self, error: &str, now: DateTime<Utc>) {
453        self.update(now, |f| {
454            f.index.checked_at = Some(now);
455            f.drain_error = Streak::extend(f.drain_error.take(), error.to_string(), now);
456            f.record_error("drain", error, now);
457        });
458    }
459
460    /// No drain thread will ever report, so the error stays until restart.
461    pub fn indexes_unavailable(&self, error: &str, now: DateTime<Utc>) {
462        self.update(now, |f| {
463            f.indexes_error = Some(error.to_string());
464            f.record_error("index", error, now);
465        });
466    }
467
468    pub fn repaired(&self, now: DateTime<Utc>) {
469        self.update(now, |f| f.repair_error = None);
470    }
471
472    pub fn repair_failed(&self, error: &str, now: DateTime<Utc>) {
473        self.update(now, |f| {
474            f.repair_error = Some(error.to_string());
475            f.record_error("repair", error, now);
476        });
477    }
478
479    pub fn queue_checked(&self, depth: u64, oldest_at: Option<DateTime<Utc>>, now: DateTime<Utc>) {
480        self.update(now, |f| {
481            f.queue.depth = depth;
482            f.queue.oldest_at = oldest_at;
483        });
484    }
485
486    pub fn write_rejected(&self, tool: &str, status: u16, now: DateTime<Utc>) {
487        self.update(now, |f| {
488            f.queue.last_rejected = Some(RejectedWrite {
489                tool: tool.to_string(),
490                status,
491                at: now,
492            });
493        });
494    }
495
496    /// `space` is what the latest free-space check found, if one ran.
497    pub fn maintenance(
498        &self,
499        state: MaintenanceState,
500        space: Option<VacuumSpace>,
501        now: DateTime<Utc>,
502    ) {
503        self.update(now, |f| {
504            if f.maintenance.state != state {
505                f.maintenance.since = Some(now);
506            }
507            f.maintenance.state = state;
508            if space.is_some() {
509                f.maintenance.space = space;
510            }
511            if state == MaintenanceState::Done {
512                f.maintenance.last_error = None;
513            }
514        });
515    }
516
517    /// Stays until the compaction next completes.
518    pub fn maintenance_failed(&self, error: &str, now: DateTime<Utc>) {
519        self.update(now, |f| {
520            f.maintenance.last_error = Some(error.to_string());
521            f.record_error("maintenance", error, now);
522        });
523    }
524
525    pub fn recalled(&self, event: RecallEvent, now: DateTime<Utc>) {
526        self.update(now, |f| f.value.recalled(event, now));
527    }
528
529    pub fn memory_corrected(&self, id: Uuid, now: DateTime<Utc>) {
530        self.update(now, |f| f.value.corrected(id));
531    }
532
533    pub fn memories_used(&self, ids: &[Uuid], now: DateTime<Utc>) {
534        self.update(now, |f| f.value.used(ids));
535    }
536
537    pub fn sync_peers_listed(&self, peers: &[String], now: DateTime<Utc>) {
538        self.update(now, |f| f.value.peers_listed(peers, now));
539    }
540
541    pub fn sync_attempted(&self, peer: &str, outcome: PeerOutcome, now: DateTime<Utc>) {
542        self.update(now, |f| f.value.sync_attempted(peer, outcome, now));
543    }
544
545    /// None when the measurement failed, which drops the old reading.
546    pub fn store_measured(&self, size: Option<StoreSize>, now: DateTime<Utc>) {
547        self.update(now, |f| f.value.store_measured(size));
548    }
549
550    /// None when the measurement failed, which drops the old reading.
551    pub fn resident_measured(&self, size: Option<ResidentSize>, now: DateTime<Utc>) {
552        self.update(now, |f| f.value.resident_measured(size));
553    }
554
555    /// Where a count of recently ingested sessions can start so that it
556    /// compares with recall sessions, which this process saw only since it
557    /// started: the window, this process's start and the end of a backfill,
558    /// whose old sessions look freshly ingested. None while one runs.
559    pub fn ingest_window_start(&self, now: DateTime<Utc>) -> Option<DateTime<Utc>> {
560        let inner = self.lock();
561        let f = &inner.facts;
562        let mut start = (now - WINDOW).max(f.started);
563        match f.backfill.state {
564            BackfillState::Running => return None,
565            BackfillState::Done => {
566                if let Some(end) = f.backfill.updated_at {
567                    start = start.max(end);
568                }
569            }
570            BackfillState::NotStarted => {}
571        }
572        Some(start)
573    }
574
575    /// Progress only: skipping the reassessment keeps the state, reason and
576    /// transitions exactly as the last real report left them.
577    pub fn backfill_progress(&self, report: BackfillReport) {
578        let mut inner = self.lock();
579        inner.facts.backfill = report.clone();
580        inner.report.backfill = report;
581    }
582
583    fn lock(&self) -> std::sync::MutexGuard<'_, Inner> {
584        match self.inner.lock() {
585            Ok(g) => g,
586            Err(poisoned) => poisoned.into_inner(),
587        }
588    }
589
590    fn update(&self, now: DateTime<Utc>, change: impl FnOnce(&mut Facts)) {
591        let mut inner = self.lock();
592        change(&mut inner.facts);
593        self.reassess(&mut inner, now, false);
594    }
595
596    /// `announcing` sends an unchanged state that this process has not sent.
597    fn reassess(&self, inner: &mut Inner, now: DateTime<Utc>, announcing: bool) {
598        let (started, sync_overdue) = (inner.facts.started, inner.facts.windows.sync_overdue);
599        let (value, value_problems) = inner.facts.value.refresh(now, started, sync_overdue);
600        let mut serving = Assessment::default();
601        assess_serving(&inner.facts, now, &mut serving);
602        let mut overall = serving.clone();
603        assess_upkeep(&inner.facts, now, &mut overall, value_problems);
604        let (state, reason) = overall.verdict();
605        let changed = state != inner.report.state;
606        let since = if changed { now } else { inner.report.since };
607        let mut capture = inner.facts.capture.clone();
608        capture.lag_secs = capture_lag(&inner.facts).map(|d| d.num_seconds());
609        let mut queue = inner.facts.queue.clone();
610        queue.oldest_age_secs = queue.oldest_at.map(|t| (now - t).num_seconds().max(0));
611        inner.report = HealthReport {
612            state,
613            reason: reason.clone(),
614            since,
615            serving: Verdict {
616                state: serving.state,
617                reason: serving.verdict().1,
618            },
619            capture,
620            index: inner.facts.index.clone(),
621            queue,
622            maintenance: inner.facts.maintenance.clone(),
623            backfill: inner.facts.backfill.clone(),
624            sync: inner.facts.sync.clone(),
625            distill: inner.facts.distill.clone(),
626            value,
627            last_error: inner.facts.last_error.clone(),
628        };
629        // Sent under the lock so concurrent reporters cannot deliver
630        // transitions out of order; the send never blocks.
631        let unsent = inner.sent != Some(state);
632        if (changed || (announcing && unsent))
633            && let Some(tx) = &self.transitions
634        {
635            // The listener only goes away when the runtime shuts down.
636            let _ = tx.send(Transition {
637                state,
638                reason,
639                at: now,
640            });
641            inner.sent = Some(state);
642        }
643    }
644}
645
646impl Facts {
647    fn record_error(&mut self, source: &'static str, message: &str, at: DateTime<Utc>) {
648        self.last_error = Some(ErrorRecord {
649            source,
650            message: message.to_string(),
651            at,
652        });
653    }
654}
655
656/// Transcripts written before the daemon started are not its lag: it
657/// watches for changes and never saw them.
658fn capture_lag(f: &Facts) -> Option<Duration> {
659    let newest = f.capture.newest_transcript_at?;
660    let through = f
661        .capture
662        .captured_through
663        .map_or(f.started, |t| t.max(f.started));
664    Some((newest - through).max(Duration::zero()))
665}
666
667#[derive(Clone, Default)]
668struct Assessment {
669    state: State,
670    problems: Vec<String>,
671}
672
673impl Assessment {
674    fn flag(&mut self, level: State, problem: String) {
675        self.state = self.state.max(level);
676        self.problems.push(problem);
677    }
678
679    fn verdict(&self) -> (State, Option<String>) {
680        let reason = (!self.problems.is_empty()).then(|| self.problems.join("; "));
681        (self.state, reason)
682    }
683}
684
685fn assess_serving(f: &Facts, now: DateTime<Utc>, a: &mut Assessment) {
686    let mut flag = |level: State, problem: String| a.flag(level, problem);
687
688    match &f.watcher_error {
689        Some(e) => flag(
690            State::Failing,
691            format!("transcript watcher is not running: {e}"),
692        ),
693        None => {
694            let last = f.capture.checked_at.unwrap_or(f.started);
695            if now - last >= WATCHER_SILENT {
696                flag(
697                    State::Degraded,
698                    format!(
699                        "transcript watcher has not reported for {}",
700                        human(now - last)
701                    ),
702                );
703            }
704        }
705    }
706    if let Some(lag) = capture_lag(f)
707        && lag >= CAPTURE_LAG_DEGRADED
708    {
709        let level = if lag >= CAPTURE_LAG_FAILING {
710            State::Failing
711        } else {
712            State::Degraded
713        };
714        flag(
715            level,
716            format!("capture is {} behind the newest transcript", human(lag)),
717        );
718    }
719    if let Some(e) = &f.ingest_error
720        && e.outlasted_grace(now)
721    {
722        flag(
723            State::Degraded,
724            format!(
725                "ingest has failed for {}: {}",
726                human(now - e.since),
727                e.message
728            ),
729        );
730    }
731
732    if let Some(e) = &f.drain_error
733        && e.outlasted_grace(now)
734    {
735        flag(
736            State::Degraded,
737            format!(
738                "index drain failed for {}: {}",
739                human(now - e.since),
740                e.message
741            ),
742        );
743    }
744    match &f.indexes_error {
745        Some(e) => flag(State::Degraded, format!("search indexes unavailable: {e}")),
746        None => {
747            let last = f.index.checked_at.unwrap_or(f.started);
748            if now - last >= DRAIN_SILENT {
749                flag(
750                    State::Degraded,
751                    format!("index drain has not reported for {}", human(now - last)),
752                );
753            }
754        }
755    }
756    if let (Some(since), Some(pending)) = (f.index.backlog_since, f.index.outbox_pending) {
757        let age = now - since;
758        if age >= BACKLOG_DEGRADED {
759            let level = if age >= BACKLOG_FAILING {
760                State::Failing
761            } else {
762                State::Degraded
763            };
764            flag(
765                level,
766                format!("{pending} changes have waited {} to be indexed", human(age)),
767            );
768        }
769    }
770    if let Some(e) = &f.repair_error {
771        flag(State::Degraded, format!("last store repair failed: {e}"));
772    }
773    if let Some(oldest) = f.queue.oldest_at
774        && now - oldest >= QUEUE_STUCK
775    {
776        flag(
777            State::Degraded,
778            format!(
779                "write queue not draining ({} queued, oldest {})",
780                f.queue.depth,
781                human(now - oldest)
782            ),
783        );
784    }
785
786    match (&f.maintenance.state, &f.maintenance.space) {
787        (MaintenanceState::Vacuuming, _) => flag(
788            State::Degraded,
789            "store maintenance is running; writes are queued until it finishes".to_string(),
790        ),
791        (MaintenanceState::WaitingForSpace, Some(space)) => flag(
792            State::Degraded,
793            format!(
794                "store compaction waits for disk space: it needs {} MB free at {} and has {} MB",
795                space.needed_bytes / 1_000_000,
796                space.path.display(),
797                space.free_bytes / 1_000_000
798            ),
799        ),
800        _ => {}
801    }
802    if let Some(e) = &f.maintenance.last_error {
803        flag(State::Degraded, format!("store compaction failed: {e}"));
804    }
805}
806
807/// Sync, distillation and the value metrics: overdue work and poor value
808/// that leave capture and recall running.
809fn assess_upkeep(f: &Facts, now: DateTime<Utc>, a: &mut Assessment, value: ValueProblems) {
810    let mut flag = |level: State, problem: String| a.flag(level, problem);
811    let mut no_sync_at_all = false;
812    let sync = &f.sync;
813    if (sync.peers.is_some_and(|n| n > 0) || sync.synced_before)
814        && let Some(window) = f.windows.sync_overdue
815    {
816        let since = sync
817            .last_success_at
818            .or(sync.last_cursor_at)
819            .unwrap_or(f.started);
820        let overdue = now - since;
821        if overdue >= window {
822            let problem = match sync.peers {
823                Some(0) => format!(
824                    "no sync has succeeded in {}, and ~/.second-brain/sync-peers lists no peers \
825                     although this device has synced before; add each peer to it as user@host, \
826                     one per line, or set alert.sync_overdue_hours = 0 if this device no longer syncs",
827                    human(overdue)
828                ),
829                Some(n) => format!(
830                    "no sync has succeeded in {} with {} in sync-peers; \
831                     run `sb sync <peer>` to see why",
832                    human(overdue),
833                    counted(n as u64, "peer", "peers")
834                ),
835                None => format!(
836                    "no sync has succeeded in {} although this device has synced before; \
837                     run `sb sync <user@host>` (or check the job that runs it), \
838                     or set alert.sync_overdue_hours = 0 if this device no longer syncs",
839                    human(overdue)
840                ),
841            };
842            flag(State::Degraded, problem);
843            no_sync_at_all = true;
844        }
845    }
846    if !no_sync_at_all {
847        for problem in value.peers {
848            flag(State::Degraded, problem);
849        }
850    }
851    for problem in value.other {
852        flag(State::Degraded, problem);
853    }
854    if f.distill.mode == DistillMode::Llm
855        && let (Some(oldest), Some(window)) = (f.distill.oldest_at, f.windows.undistilled)
856        && now - oldest >= window
857    {
858        flag(
859            State::Degraded,
860            format!(
861                "{} waited {} to be distilled; run `sb consolidate --llm-url <endpoint>` or add an [llm] section, or set \
862                 alert.undistilled_overdue_hours = 0 if this device does not distill",
863                if f.distill.pending == 1 {
864                    "1 raw memory has".to_string()
865                } else {
866                    format!("{} raw memories have", f.distill.pending)
867                },
868                human(now - oldest)
869            ),
870        );
871    }
872    if let Some(e) = &f.progress_error
873        && e.outlasted_grace(now)
874    {
875        flag(
876            State::Degraded,
877            format!(
878                "cannot check sync and distillation for {}: {}",
879                human(now - e.since),
880                e.message
881            ),
882        );
883    }
884}
885
886pub(crate) fn counted(n: u64, one: &str, many: &str) -> String {
887    if n == 1 {
888        format!("1 {one}")
889    } else {
890        format!("{n} {many}")
891    }
892}
893
894pub(crate) fn human(d: Duration) -> String {
895    let minutes = d.num_minutes();
896    if minutes < 60 {
897        format!("{minutes}m")
898    } else if minutes < 48 * 60 {
899        format!("{}h {}m", minutes / 60, minutes % 60)
900    } else {
901        format!("{}d", minutes / (24 * 60))
902    }
903}
904
905#[cfg(test)]
906mod tests {
907    use tokio::sync::mpsc::{UnboundedReceiver, unbounded_channel};
908
909    use super::*;
910
911    fn t0() -> DateTime<Utc> {
912        DateTime::parse_from_rfc3339("2026-10-31T12:00:00Z")
913            .unwrap()
914            .with_timezone(&Utc)
915    }
916
917    fn mins(m: i64) -> DateTime<Utc> {
918        t0() + Duration::minutes(m)
919    }
920
921    fn monitored() -> (Health, UnboundedReceiver<Transition>) {
922        let (tx, rx) = unbounded_channel();
923        (Health::new(t0(), Some(tx)), rx)
924    }
925
926    fn drain(rx: &mut UnboundedReceiver<Transition>) -> Vec<State> {
927        std::iter::from_fn(|| rx.try_recv().ok())
928            .map(|t| t.state)
929            .collect()
930    }
931
932    #[test]
933    fn a_fresh_daemon_that_keeps_up_is_ok() {
934        let (health, mut rx) = monitored();
935        health.watcher_started(mins(0));
936        health.transcripts_scanned(Some(mins(1)), mins(1));
937        health.ingested(Some(mins(1)), mins(1));
938        health.drained(0, mins(1));
939        let report = health.report();
940        assert_eq!(report.state, State::Ok);
941        assert_eq!(report.reason, None);
942        assert_eq!(report.capture.lag_secs, Some(0));
943        assert!(drain(&mut rx).is_empty());
944    }
945
946    #[test]
947    fn capture_lag_degrades_then_fails_then_recovers() {
948        let (health, mut rx) = monitored();
949        health.drained(0, mins(0));
950        health.ingested(Some(mins(1)), mins(1));
951        health.transcripts_scanned(Some(mins(10)), mins(10));
952        assert_eq!(health.report().state, State::Ok);
953
954        health.transcripts_scanned(Some(mins(20)), mins(20));
955        let report = health.report();
956        assert_eq!(report.state, State::Degraded);
957        assert_eq!(report.capture.lag_secs, Some(19 * 60));
958        assert!(report.reason.unwrap().contains("capture is 19m behind"));
959
960        health.drained(0, mins(125));
961        health.transcripts_scanned(Some(mins(125)), mins(125));
962        assert_eq!(health.report().state, State::Failing);
963
964        health.ingested(Some(mins(125)), mins(126));
965        assert_eq!(health.report().state, State::Ok);
966        assert_eq!(
967            drain(&mut rx),
968            vec![State::Degraded, State::Failing, State::Ok]
969        );
970    }
971
972    #[test]
973    fn transcripts_from_before_startup_are_not_lag() {
974        let (health, _rx) = monitored();
975        health.transcripts_scanned(Some(t0() - Duration::days(3)), mins(1));
976        assert_eq!(health.report().capture.lag_secs, Some(0));
977        assert_eq!(health.report().state, State::Ok);
978    }
979
980    #[test]
981    fn repeated_reports_in_one_state_send_one_transition() {
982        let (health, mut rx) = monitored();
983        for m in 0..=12 {
984            health.ingest_failed("database is locked", mins(m));
985        }
986        health.ingested(None, mins(13));
987        health.ingested(None, mins(14));
988        assert_eq!(drain(&mut rx), vec![State::Degraded, State::Ok]);
989        let report = health.report();
990        assert_eq!(report.since, mins(13));
991        let last = report.last_error.expect("the error is kept after recovery");
992        assert_eq!(last.source, "ingest");
993        assert_eq!(last.at, mins(12));
994    }
995
996    #[test]
997    fn an_error_that_clears_within_the_grace_sends_nothing() {
998        let (health, mut rx) = monitored();
999        health.watcher_started(mins(0));
1000        health.ingest_failed("database is locked", mins(0));
1001        health.drain_failed("database is locked", mins(1));
1002        health.drained(0, mins(9));
1003        health.ingested(None, mins(9));
1004        assert_eq!(health.report().state, State::Ok);
1005        assert!(drain(&mut rx).is_empty());
1006        assert_eq!(health.report().last_error.unwrap().source, "drain");
1007    }
1008
1009    #[test]
1010    fn a_drain_error_degrades_once_it_outlasts_the_grace() {
1011        let (health, _rx) = monitored();
1012        health.watcher_started(mins(0));
1013        for m in 0..10 {
1014            health.drain_failed("disk I/O error", mins(m));
1015        }
1016        assert_eq!(health.report().state, State::Ok);
1017        health.drain_failed("disk I/O error", mins(10));
1018        let report = health.report();
1019        assert_eq!(report.state, State::Degraded);
1020        assert!(report.reason.unwrap().contains("index drain failed"));
1021        health.drained(0, mins(11));
1022        assert_eq!(health.report().state, State::Ok);
1023    }
1024
1025    #[test]
1026    fn a_long_ingest_batch_counts_as_the_watcher_reporting() {
1027        let (health, _rx) = monitored();
1028        health.watcher_started(mins(0));
1029        health.ingested(None, mins(10));
1030        health.ingest_failed("unreadable line", mins(19));
1031        health.drained(0, mins(20));
1032        let report = health.report();
1033        assert_eq!(report.state, State::Ok, "{:?}", report.reason);
1034    }
1035
1036    #[test]
1037    fn an_outbox_backlog_only_counts_once_it_persists() {
1038        let (health, _rx) = monitored();
1039        health.drained(500, mins(0));
1040        health.drained(800, mins(14));
1041        health.watcher_started(mins(14));
1042        assert_eq!(health.report().state, State::Ok);
1043        health.drained(800, mins(15));
1044        let report = health.report();
1045        assert_eq!(report.state, State::Degraded);
1046        assert_eq!(report.index.backlog_since, Some(mins(0)));
1047        health.drained(0, mins(16));
1048        assert_eq!(health.report().state, State::Ok);
1049        assert_eq!(health.report().index.backlog_since, None);
1050    }
1051
1052    #[test]
1053    fn a_dead_watcher_fails_and_a_silent_one_degrades() {
1054        let (health, _rx) = monitored();
1055        health.watcher_failed("inotify watch limit reached", mins(0));
1056        assert_eq!(health.report().state, State::Failing);
1057        health.watcher_started(mins(1));
1058        health.drained(0, mins(1));
1059        assert_eq!(health.report().state, State::Ok);
1060        health.drained(0, mins(16));
1061        let report = health.report();
1062        assert_eq!(report.state, State::Degraded);
1063        assert!(report.reason.unwrap().contains("watcher has not reported"));
1064    }
1065
1066    #[test]
1067    fn index_and_repair_errors_degrade_until_the_next_success() {
1068        let (health, _rx) = monitored();
1069        health.watcher_started(mins(0));
1070        health.drain_failed("disk I/O error", mins(0));
1071        health.repair_failed("disk I/O error", mins(0));
1072        let reason = health.report().reason.unwrap();
1073        assert!(reason.contains("store repair failed"), "{reason}");
1074        health.drain_failed("disk I/O error", mins(10));
1075        let reason = health.report().reason.unwrap();
1076        assert!(reason.contains("index drain failed"), "{reason}");
1077        health.drained(0, mins(11));
1078        assert_eq!(health.report().state, State::Degraded);
1079        health.repaired(mins(12));
1080        assert_eq!(health.report().state, State::Ok);
1081    }
1082
1083    #[test]
1084    fn missing_indexes_stay_degraded_without_a_drain() {
1085        let (health, _rx) = monitored();
1086        health.indexes_unavailable("tantivy meta.json missing", mins(0));
1087        health.watcher_started(mins(90));
1088        let report = health.report();
1089        assert_eq!(report.state, State::Degraded);
1090        let reason = report.reason.unwrap();
1091        assert!(reason.contains("search indexes unavailable"), "{reason}");
1092        assert!(!reason.contains("has not reported"), "{reason}");
1093    }
1094
1095    #[test]
1096    fn the_report_serializes_without_content() {
1097        let (health, _rx) = monitored();
1098        health.ingest_failed("database is locked", mins(0));
1099        health.ingest_failed("database is locked", mins(10));
1100        let json = serde_json::to_value(health.report()).unwrap();
1101        assert_eq!(json["state"], "degraded");
1102        assert_eq!(json["last_error"]["source"], "ingest");
1103        for key in ["reason", "since", "capture", "index"] {
1104            assert!(json.get(key).is_some(), "{key} missing: {json}");
1105        }
1106    }
1107
1108    #[test]
1109    fn a_queue_entry_older_than_ten_minutes_degrades_until_it_drains() {
1110        let (health, mut rx) = monitored();
1111        health.watcher_started(mins(0));
1112        health.drained(0, mins(0));
1113        health.queue_checked(2, Some(mins(0)), mins(9));
1114        assert_eq!(health.report().state, State::Ok);
1115        assert_eq!(health.report().queue.depth, 2);
1116        health.queue_checked(2, Some(mins(0)), mins(10));
1117        let report = health.report();
1118        assert_eq!(report.state, State::Degraded);
1119        assert!(
1120            report
1121                .reason
1122                .as_deref()
1123                .unwrap()
1124                .contains("write queue not draining"),
1125            "{report:?}"
1126        );
1127        assert_eq!(report.queue.oldest_age_secs, Some(600));
1128        health.queue_checked(0, None, mins(11));
1129        assert_eq!(health.report().state, State::Ok);
1130        assert_eq!(drain(&mut rx), vec![State::Degraded, State::Ok]);
1131    }
1132
1133    #[test]
1134    fn a_rejected_write_is_reported_by_tool_status_and_time_only() {
1135        let (health, _rx) = monitored();
1136        health.write_rejected("associate", 404, mins(3));
1137        let json = serde_json::to_value(health.report()).unwrap();
1138        assert_eq!(
1139            json["queue"]["last_rejected"],
1140            serde_json::json!({"tool": "associate", "status": 404, "at": "2026-10-31T12:03:00Z"})
1141        );
1142        assert_eq!(json["state"], "ok");
1143    }
1144
1145    fn hours(h: i64) -> DateTime<Utc> {
1146        t0() + Duration::hours(h)
1147    }
1148
1149    fn alive(health: &Health, at: DateTime<Utc>) {
1150        health.update(at, |f| {
1151            f.capture.checked_at = Some(at);
1152            f.index.checked_at = Some(at);
1153        });
1154    }
1155
1156    fn sync_seen(health: &Health, sync: SyncReport, at: DateTime<Utc>) {
1157        let seen = ProgressSeen {
1158            sync: Some(sync),
1159            ..ProgressSeen::default()
1160        };
1161        health.progress_checked(seen, at);
1162    }
1163
1164    fn distill_seen(
1165        health: &Health,
1166        pending: u64,
1167        oldest_at: Option<DateTime<Utc>>,
1168        at: DateTime<Utc>,
1169    ) {
1170        distill_seen_in(health, DistillMode::Llm, pending, oldest_at, at);
1171    }
1172
1173    fn distill_seen_in(
1174        health: &Health,
1175        mode: DistillMode,
1176        pending: u64,
1177        oldest_at: Option<DateTime<Utc>>,
1178        at: DateTime<Utc>,
1179    ) {
1180        let distill = DistillReport {
1181            mode,
1182            pending,
1183            oldest_at,
1184            ..DistillReport::default()
1185        };
1186        let seen = ProgressSeen {
1187            distill: Some(distill),
1188            ..ProgressSeen::default()
1189        };
1190        health.progress_checked(seen, at);
1191    }
1192
1193    fn synced(peers: usize, synced_before: bool, last: Option<DateTime<Utc>>) -> SyncReport {
1194        SyncReport {
1195            checked_at: None,
1196            peers: Some(peers),
1197            synced_before,
1198            last_success_at: last,
1199            last_cursor_at: None,
1200        }
1201    }
1202
1203    #[test]
1204    fn without_a_stamp_the_window_counts_from_when_records_last_moved() {
1205        let (health, _rx) = monitored();
1206        let seen = SyncReport {
1207            last_cursor_at: Some(hours(-48)),
1208            ..synced(0, true, None)
1209        };
1210        sync_seen(&health, seen.clone(), mins(0));
1211        let report = health.report();
1212        assert_eq!(report.state, State::Degraded);
1213        let reason = report.reason.unwrap();
1214        assert!(reason.contains("no sync has succeeded in 2d"), "{reason}");
1215
1216        let stamped = SyncReport {
1217            last_success_at: Some(mins(0)),
1218            ..seen
1219        };
1220        sync_seen(&health, stamped, mins(0));
1221        assert_eq!(health.report().state, State::Ok);
1222    }
1223
1224    #[test]
1225    fn reasons_count_one_peer_and_one_raw_in_the_singular() {
1226        let (health, _rx) = monitored();
1227        sync_seen(&health, synced(1, true, Some(hours(-30))), mins(0));
1228        distill_seen(&health, 1, Some(hours(-30)), mins(0));
1229        let reason = health.report().reason.unwrap();
1230        assert!(reason.contains("with 1 peer in sync-peers"), "{reason}");
1231        assert!(reason.contains("1 raw memory has waited"), "{reason}");
1232    }
1233
1234    #[test]
1235    fn a_device_that_synced_before_and_lost_its_peers_degrades_with_the_fix() {
1236        let (health, mut rx) = monitored();
1237        sync_seen(&health, synced(0, true, Some(hours(-25))), mins(0));
1238        let report = health.report();
1239        assert_eq!(report.state, State::Degraded);
1240        let reason = report.reason.unwrap();
1241        assert!(
1242            reason.contains("no sync has succeeded in 25h 0m"),
1243            "{reason}"
1244        );
1245        assert!(reason.contains("lists no peers"), "{reason}");
1246        assert!(reason.contains("~/.second-brain/sync-peers"), "{reason}");
1247        assert!(reason.contains("sync_overdue_hours = 0"), "{reason}");
1248        assert_eq!(report.sync.peers, Some(0));
1249        assert_eq!(report.sync.checked_at, Some(mins(0)));
1250        assert_eq!(drain(&mut rx), vec![State::Degraded]);
1251
1252        sync_seen(&health, synced(1, true, Some(mins(1))), mins(1));
1253        assert_eq!(health.report().state, State::Ok);
1254    }
1255
1256    #[test]
1257    fn a_device_synced_by_hand_without_sync_peers_stays_ok_while_it_syncs() {
1258        let (health, mut rx) = monitored();
1259        let by_hand = SyncReport {
1260            peers: None,
1261            ..synced(0, true, Some(hours(10)))
1262        };
1263        alive(&health, hours(30));
1264        sync_seen(&health, by_hand.clone(), hours(30));
1265        assert_eq!(health.report().state, State::Ok);
1266        alive(&health, hours(34));
1267        sync_seen(&health, by_hand, hours(34));
1268        let report = health.report();
1269        assert_eq!(report.state, State::Degraded);
1270        let reason = report.reason.unwrap();
1271        assert!(
1272            reason.contains("no sync has succeeded in 24h 0m"),
1273            "{reason}"
1274        );
1275        assert!(reason.contains("sb sync <user@host>"), "{reason}");
1276        assert!(reason.contains("sync_overdue_hours = 0"), "{reason}");
1277        assert!(!reason.contains("sync-peers"), "{reason}");
1278        assert_eq!(drain(&mut rx), vec![State::Degraded]);
1279    }
1280
1281    #[test]
1282    fn a_solo_device_that_never_synced_stays_ok() {
1283        let (health, mut rx) = monitored();
1284        for h in [0, 25, 100] {
1285            alive(&health, hours(h));
1286            sync_seen(&health, synced(0, false, None), hours(h));
1287        }
1288        let report = health.report();
1289        assert_eq!(report.state, State::Ok, "{:?}", report.reason);
1290        assert!(drain(&mut rx).is_empty());
1291    }
1292
1293    #[test]
1294    fn a_sync_overdue_past_the_window_degrades_only_on_a_device_that_syncs() {
1295        let (health, _rx) = monitored();
1296        alive(&health, hours(0));
1297        sync_seen(&health, synced(2, true, Some(hours(-23))), hours(0));
1298        assert_eq!(health.report().state, State::Ok);
1299
1300        alive(&health, hours(1));
1301        sync_seen(&health, synced(2, true, Some(hours(-23))), hours(1));
1302        let report = health.report();
1303        assert_eq!(report.state, State::Degraded);
1304        let reason = report.reason.unwrap();
1305        assert!(
1306            reason.contains("no sync has succeeded in 24h 0m"),
1307            "{reason}"
1308        );
1309        assert!(reason.contains("2 peers"), "{reason}");
1310
1311        sync_seen(&health, synced(2, true, Some(hours(1))), hours(1));
1312        assert_eq!(health.report().state, State::Ok);
1313
1314        sync_seen(&health, synced(0, false, Some(hours(-500))), hours(1));
1315        assert_eq!(health.report().state, State::Ok);
1316    }
1317
1318    #[test]
1319    fn with_peers_and_no_recorded_success_the_window_starts_at_startup() {
1320        let (health, _rx) = monitored();
1321        alive(&health, hours(23));
1322        sync_seen(&health, synced(1, false, None), hours(23));
1323        assert_eq!(health.report().state, State::Ok);
1324        alive(&health, hours(24));
1325        sync_seen(&health, synced(1, false, None), hours(24));
1326        let reason = health.report().reason.unwrap();
1327        assert!(
1328            reason.contains("no sync has succeeded in 24h 0m"),
1329            "{reason}"
1330        );
1331    }
1332
1333    #[test]
1334    fn raws_left_undistilled_past_the_window_degrade_with_the_count() {
1335        let (health, mut rx) = monitored();
1336        alive(&health, hours(0));
1337        distill_seen(&health, 520, Some(hours(-23)), hours(0));
1338        assert_eq!(health.report().state, State::Ok);
1339        assert_eq!(health.report().distill.pending, 520);
1340
1341        alive(&health, hours(2));
1342        distill_seen(&health, 520, Some(hours(-23)), hours(2));
1343        let report = health.report();
1344        assert_eq!(report.state, State::Degraded);
1345        let reason = report.reason.unwrap();
1346        assert!(
1347            reason.contains("520 raw memories have waited 25h 0m to be distilled"),
1348            "{reason}"
1349        );
1350        assert!(reason.contains("undistilled_overdue_hours = 0"), "{reason}");
1351
1352        alive(&health, hours(3));
1353        distill_seen(&health, 0, None, hours(3));
1354        assert_eq!(health.report().state, State::Ok);
1355        assert_eq!(drain(&mut rx), vec![State::Degraded, State::Ok]);
1356    }
1357
1358    #[test]
1359    fn without_an_llm_endpoint_waiting_raws_report_the_mode_instead_of_degrading() {
1360        let (health, mut rx) = monitored();
1361        alive(&health, hours(100));
1362        distill_seen_in(
1363            &health,
1364            DistillMode::NoGpu,
1365            520,
1366            Some(hours(-100)),
1367            hours(100),
1368        );
1369        let report = health.report();
1370        assert_eq!(report.state, State::Ok, "{:?}", report.reason);
1371        assert_eq!(report.reason, None);
1372        assert_eq!(report.distill.pending, 520);
1373        let json = serde_json::to_value(&report).unwrap();
1374        assert_eq!(json["distill"]["mode"], "no-gpu");
1375        assert_eq!(json["distill"]["note"], NO_GPU_NOTE);
1376        assert!(drain(&mut rx).is_empty());
1377    }
1378
1379    #[test]
1380    fn with_an_llm_endpoint_the_report_names_the_mode_and_carries_no_note() {
1381        let (health, _rx) = monitored();
1382        alive(&health, hours(2));
1383        distill_seen(&health, 3, Some(hours(-23)), hours(2));
1384        let report = health.report();
1385        assert_eq!(report.state, State::Degraded);
1386        let json = serde_json::to_value(&report).unwrap();
1387        assert_eq!(json["distill"]["mode"], "llm");
1388        assert!(json["distill"].get("note").is_none(), "{json}");
1389    }
1390
1391    #[test]
1392    fn a_window_turned_off_never_degrades() {
1393        let (tx, _rx) = unbounded_channel();
1394        let windows = Windows::from_config(&AlertConfig {
1395            sync_overdue_hours: 0,
1396            undistilled_overdue_hours: 0,
1397            ..AlertConfig::default()
1398        });
1399        assert_eq!(
1400            windows,
1401            Windows {
1402                sync_overdue: None,
1403                undistilled: None
1404            }
1405        );
1406        let health = Health::new(t0(), Some(tx)).with_windows(windows);
1407        alive(&health, hours(100));
1408        sync_seen(&health, synced(1, true, Some(hours(-100))), hours(100));
1409        distill_seen(&health, 9, Some(hours(-100)), hours(100));
1410        assert_eq!(health.report().state, State::Ok);
1411        sync_seen(&health, synced(0, true, Some(hours(-100))), hours(100));
1412        assert_eq!(health.report().state, State::Ok);
1413    }
1414
1415    #[test]
1416    fn the_windows_come_from_the_alert_config() {
1417        let windows = Windows::from_config(&AlertConfig {
1418            sync_overdue_hours: 6,
1419            undistilled_overdue_hours: 48,
1420            ..AlertConfig::default()
1421        });
1422        assert_eq!(windows.sync_overdue, Some(Duration::hours(6)));
1423        assert_eq!(windows.undistilled, Some(Duration::hours(48)));
1424        assert_eq!(Windows::default().sync_overdue, Some(Duration::hours(24)));
1425        assert_eq!(Windows::default().undistilled, Some(Duration::hours(24)));
1426    }
1427
1428    #[test]
1429    fn announce_sends_the_current_state_once_unless_it_was_already_sent() {
1430        let (health, mut rx) = monitored();
1431        health.announce(mins(0));
1432        let sent: Vec<Transition> = std::iter::from_fn(|| rx.try_recv().ok()).collect();
1433        assert_eq!(
1434            sent,
1435            vec![Transition {
1436                state: State::Ok,
1437                reason: None,
1438                at: mins(0),
1439            }]
1440        );
1441
1442        health.announce(mins(1));
1443        assert!(drain(&mut rx).is_empty(), "ok was already sent");
1444
1445        sync_seen(&health, synced(0, true, Some(hours(-30))), mins(2));
1446        health.announce(mins(3));
1447        assert_eq!(drain(&mut rx), vec![State::Degraded]);
1448    }
1449
1450    #[test]
1451    fn announce_reassesses_and_sends_one_message_for_a_pending_change() {
1452        let (health, mut rx) = monitored();
1453        alive(&health, hours(0));
1454        distill_seen(&health, 3, Some(hours(-23)), hours(0));
1455        assert!(drain(&mut rx).is_empty());
1456        health.announce(hours(2));
1457        assert_eq!(drain(&mut rx), vec![State::Degraded]);
1458    }
1459
1460    #[test]
1461    fn serving_leaves_out_sync_and_distillation() {
1462        let (health, _rx) = monitored();
1463        alive(&health, hours(0));
1464        let seen = ProgressSeen {
1465            sync: Some(synced(1, true, Some(hours(-30)))),
1466            distill: Some(DistillReport {
1467                pending: 4,
1468                oldest_at: Some(hours(-30)),
1469                ..DistillReport::default()
1470            }),
1471            ingested_sessions: None,
1472            error: None,
1473        };
1474        health.progress_checked(seen, hours(0));
1475        let report = health.report();
1476        assert_eq!(report.state, State::Degraded);
1477        assert_eq!(
1478            report.serving,
1479            Verdict {
1480                state: State::Ok,
1481                reason: None
1482            }
1483        );
1484
1485        health.watcher_failed("inotify watch limit reached", hours(0));
1486        let report = health.report();
1487        assert_eq!(report.state, State::Failing);
1488        assert_eq!(report.serving.state, State::Failing);
1489        let serving = report.serving.reason.unwrap();
1490        assert!(serving.contains("watcher is not running"), "{serving}");
1491        assert!(!serving.contains("sync"), "{serving}");
1492
1493        let json = serde_json::to_value(health.report()).unwrap();
1494        assert_eq!(json["serving"]["state"], "failing");
1495    }
1496
1497    #[test]
1498    fn one_pass_that_clears_sync_and_finds_distillation_overdue_sends_nothing() {
1499        let (health, mut rx) = monitored();
1500        alive(&health, hours(0));
1501        sync_seen(&health, synced(1, true, Some(hours(-30))), hours(0));
1502        assert_eq!(drain(&mut rx), vec![State::Degraded]);
1503        let seen = ProgressSeen {
1504            sync: Some(synced(1, true, Some(hours(0)))),
1505            distill: Some(DistillReport {
1506                pending: 4,
1507                oldest_at: Some(hours(-30)),
1508                ..DistillReport::default()
1509            }),
1510            ingested_sessions: None,
1511            error: None,
1512        };
1513        health.progress_checked(seen, hours(0));
1514        assert_eq!(health.report().state, State::Degraded);
1515        assert!(drain(&mut rx).is_empty());
1516    }
1517
1518    fn backfill_running(done: usize, failed: usize, at: DateTime<Utc>) -> BackfillReport {
1519        BackfillReport {
1520            state: BackfillState::Running,
1521            total: 40,
1522            done,
1523            failed,
1524            updated_at: Some(at),
1525        }
1526    }
1527
1528    #[test]
1529    fn a_new_daemon_reports_the_backfill_not_started() {
1530        let (health, _rx) = monitored();
1531        let report = health.report();
1532        assert_eq!(report.backfill, BackfillReport::default());
1533        assert_eq!(report.backfill.state, BackfillState::NotStarted);
1534        assert_eq!(report.backfill.total, 0);
1535        assert_eq!(report.backfill.done, 0);
1536        assert_eq!(report.backfill.failed, 0);
1537        let json = serde_json::to_value(report).unwrap();
1538        assert_eq!(json["backfill"]["state"], "not_started");
1539    }
1540
1541    #[test]
1542    fn backfill_progress_is_reported_and_survives_later_reports() {
1543        let (health, _rx) = monitored();
1544        health.backfill_progress(backfill_running(3, 1, mins(1)));
1545        assert_eq!(health.report().backfill, backfill_running(3, 1, mins(1)));
1546
1547        health.ingested(None, mins(2));
1548        assert_eq!(health.report().backfill, backfill_running(3, 1, mins(1)));
1549
1550        let done = BackfillReport {
1551            state: BackfillState::Done,
1552            total: 40,
1553            done: 40,
1554            failed: 1,
1555            updated_at: Some(mins(3)),
1556        };
1557        health.backfill_progress(done.clone());
1558        assert_eq!(health.report().backfill, done);
1559        let json = serde_json::to_value(health.report()).unwrap();
1560        assert_eq!(json["backfill"]["state"], "done");
1561        assert_eq!(json["backfill"]["done"], 40);
1562    }
1563
1564    #[test]
1565    fn backfill_progress_leaves_the_state_and_reason_alone() {
1566        let (health, mut rx) = monitored();
1567        for m in 0..=12 {
1568            health.ingest_failed("database is locked", mins(m));
1569        }
1570        assert_eq!(drain(&mut rx), vec![State::Degraded]);
1571        let before = health.report();
1572
1573        health.backfill_progress(backfill_running(5, 2, mins(13)));
1574        let after = health.report();
1575        assert_eq!(after.state, before.state);
1576        assert_eq!(after.reason, before.reason);
1577        assert_eq!(after.since, before.since);
1578        assert_eq!(after.serving, before.serving);
1579        assert!(drain(&mut rx).is_empty());
1580    }
1581
1582    fn empty_recall(session: &str) -> RecallEvent {
1583        RecallEvent {
1584            session: Some(session.to_string()),
1585            verdict: crate::value_metrics::RecallVerdict::Empty,
1586            memory_ids: Vec::new(),
1587        }
1588    }
1589
1590    #[test]
1591    fn a_value_metric_crossing_its_threshold_sends_exactly_one_transition() {
1592        let (health, mut rx) = monitored();
1593        alive(&health, mins(0));
1594        for _ in 0..49 {
1595            health.recalled(empty_recall("s"), mins(1));
1596        }
1597        assert!(drain(&mut rx).is_empty());
1598        for _ in 0..30 {
1599            health.recalled(empty_recall("s"), mins(2));
1600        }
1601        let report = health.report();
1602        assert_eq!(report.state, State::Degraded);
1603        assert_eq!(report.serving.state, State::Ok);
1604        let reason = report.reason.unwrap();
1605        assert!(reason.contains("found nothing that cleared"), "{reason}");
1606        assert_eq!(drain(&mut rx), vec![State::Degraded]);
1607    }
1608
1609    #[test]
1610    fn a_share_hovering_at_its_threshold_sends_one_transition_each_way() {
1611        let (health, mut rx) = monitored();
1612        let hit = || RecallEvent {
1613            session: None,
1614            verdict: crate::value_metrics::RecallVerdict::Hit,
1615            memory_ids: Vec::new(),
1616        };
1617        for m in 0..50 {
1618            alive(&health, mins(m));
1619            let event = if m < 37 { empty_recall("s") } else { hit() };
1620            health.recalled(event, mins(m));
1621        }
1622        assert_eq!(health.report().state, State::Ok, "37 of 50 is 74%");
1623        for _ in 0..3 {
1624            health.recalled(empty_recall("s"), mins(50));
1625        }
1626        assert_eq!(drain(&mut rx), vec![State::Degraded], "40 of 53 is 75.5%");
1627        for m in 51..56 {
1628            alive(&health, mins(m));
1629            health.recalled(hit(), mins(m));
1630            health.recalled(empty_recall("s"), mins(m));
1631        }
1632        assert!(drain(&mut rx).is_empty(), "74% then 71% stays degraded");
1633        for m in 56..66 {
1634            alive(&health, mins(m));
1635            health.recalled(hit(), mins(m));
1636        }
1637        assert_eq!(health.report().value.recall.recalls, 73);
1638        assert_eq!(drain(&mut rx), vec![State::Ok], "45 of 73 is 62%");
1639    }
1640
1641    #[test]
1642    fn peer_lag_is_left_out_when_no_sync_succeeded_at_all() {
1643        let (health, _rx) = monitored();
1644        health.sync_peers_listed(&["u@pi".to_string()], hours(0));
1645        alive(&health, hours(30));
1646        let reason = health.report().reason.unwrap();
1647        assert!(reason.contains("no sync with u@pi"), "{reason}");
1648
1649        sync_seen(&health, synced(1, true, Some(hours(-30))), hours(30));
1650        let reason = health.report().reason.unwrap();
1651        assert!(reason.contains("no sync has succeeded in"), "{reason}");
1652        assert!(!reason.contains("no sync with u@pi"), "{reason}");
1653        assert_eq!(health.report().value.sync_peers.len(), 1);
1654    }
1655
1656    #[test]
1657    fn the_ingest_window_starts_after_startup_and_any_backfill() {
1658        let (health, _rx) = monitored();
1659        assert_eq!(health.ingest_window_start(mins(5)), Some(t0()));
1660        assert_eq!(
1661            health.ingest_window_start(hours(30)),
1662            Some(hours(30) - WINDOW)
1663        );
1664        health.backfill_progress(backfill_running(1, 0, mins(1)));
1665        assert_eq!(health.ingest_window_start(mins(5)), None);
1666        health.backfill_progress(BackfillReport {
1667            state: BackfillState::Done,
1668            updated_at: Some(mins(20)),
1669            ..BackfillReport::default()
1670        });
1671        assert_eq!(health.ingest_window_start(mins(30)), Some(mins(20)));
1672    }
1673
1674    #[test]
1675    fn progress_sets_the_ingested_sessions_and_a_failed_read_clears_them() {
1676        let (health, _rx) = monitored();
1677        health.recalled(empty_recall("a"), mins(1));
1678        let seen = ProgressSeen {
1679            ingested_sessions: Some(vec!["a".to_string(), "b".to_string()]),
1680            ..ProgressSeen::default()
1681        };
1682        health.progress_checked(seen, mins(2));
1683        let recall = health.report().value.recall;
1684        assert_eq!(recall.sessions_ingested, Some(2));
1685        assert_eq!(recall.session_recall_rate, Some(0.5));
1686        health.progress_checked(ProgressSeen::default(), mins(3));
1687        assert_eq!(health.report().value.recall.sessions_ingested, None);
1688    }
1689
1690    #[test]
1691    fn every_value_metric_is_in_the_report_json() {
1692        let (health, _rx) = monitored();
1693        health.sync_peers_listed(&["u@pi".to_string()], mins(0));
1694        health.store_measured(Some(StoreSize::default()), mins(0));
1695        health.resident_measured(Some(ResidentSize::default()), mins(0));
1696        let json = serde_json::to_value(health.report()).unwrap();
1697        let keys = |value: &serde_json::Value| -> Vec<String> {
1698            let mut keys: Vec<String> = value.as_object().unwrap().keys().cloned().collect();
1699            keys.sort();
1700            keys
1701        };
1702        assert_eq!(
1703            keys(&json["sync_peers"][0]),
1704            [
1705                "consecutive_failures",
1706                "lag_secs",
1707                "last_attempt_at",
1708                "last_success_at",
1709                "peer"
1710            ]
1711        );
1712        assert_eq!(
1713            keys(&json["store"]),
1714            [
1715                "archive_bytes",
1716                "checked_at",
1717                "db_bytes",
1718                "free_bytes",
1719                "index_bytes",
1720                "raw_index_bytes",
1721                "total_bytes",
1722                "wal_bytes"
1723            ]
1724        );
1725        assert_eq!(
1726            keys(&json["resident"]),
1727            [
1728                "checked_at",
1729                "components",
1730                "limit_bytes",
1731                "raw_index_bytes",
1732                "rss_anon_bytes",
1733                "unattributed_bytes"
1734            ]
1735        );
1736        assert_eq!(
1737            keys(&json["resident"]["components"]),
1738            [
1739                "allocator_free",
1740                "embedder",
1741                "lexical",
1742                "reranker",
1743                "sqlite",
1744                "vector_reader",
1745                "vector_writer"
1746            ]
1747        );
1748        for key in ["recall", "recalled", "sync_peers", "store", "resident"] {
1749            assert!(json.get(key).is_some(), "{key} missing: {json}");
1750        }
1751        for key in [
1752            "window_secs",
1753            "recalls",
1754            "hits",
1755            "low_score",
1756            "unjudged",
1757            "empty",
1758            "empty_share",
1759            "low_score_share",
1760            "unjudged_share",
1761            "sessions_with_recall",
1762            "sessions_ingested",
1763            "session_recall_rate",
1764        ] {
1765            assert!(json["recall"].get(key).is_some(), "recall.{key}: {json}");
1766        }
1767        for key in [
1768            "window_secs",
1769            "memories",
1770            "corrected",
1771            "used",
1772            "corrected_share",
1773            "used_share",
1774        ] {
1775            assert!(
1776                json["recalled"].get(key).is_some(),
1777                "recalled.{key}: {json}"
1778            );
1779        }
1780    }
1781}