Skip to main content

second_brain_api/
health.rs

1//! The daemon's own view of whether it is capturing and indexing. The
2//! watcher, the outbox drain and the repair loop report what they see; each
3//! report re-assesses the state, so `GET /health` only clones the result.
4
5use std::sync::Mutex;
6
7use chrono::{DateTime, Duration, Utc};
8use second_brain_core::config::AlertConfig;
9use second_brain_core::sqlite_store::VacuumSpace;
10use serde::Serialize;
11use tokio::sync::mpsc::UnboundedSender;
12
13const CAPTURE_LAG_DEGRADED: Duration = Duration::minutes(15);
14const CAPTURE_LAG_FAILING: Duration = Duration::hours(2);
15const BACKLOG_DEGRADED: Duration = Duration::minutes(15);
16const BACKLOG_FAILING: Duration = Duration::hours(2);
17/// Three missed transcript scans (see `watcher::SCAN_INTERVAL`).
18const WATCHER_SILENT: Duration = Duration::minutes(15);
19/// A scheduled repair holds the drain thread for minutes on a large store.
20const DRAIN_SILENT: Duration = Duration::hours(1);
21/// An ingest or drain error must outlast this before it degrades, so one
22/// `database is locked` that the next attempt clears sends no webhook.
23const ERROR_GRACE: Duration = Duration::minutes(10);
24/// The replayer runs every 30 s, so an entry this old is not being sent.
25const QUEUE_STUCK: Duration = Duration::minutes(10);
26
27#[derive(Debug, Clone, Copy, Default, PartialEq, Eq, PartialOrd, Ord, Serialize)]
28#[serde(rename_all = "lowercase")]
29pub enum State {
30    #[default]
31    Ok,
32    Degraded,
33    Failing,
34}
35
36#[derive(Debug, Clone, PartialEq, Serialize)]
37pub struct Transition {
38    pub state: State,
39    pub reason: Option<String>,
40    pub at: DateTime<Utc>,
41}
42
43#[derive(Debug, Clone, Serialize)]
44pub struct ErrorRecord {
45    pub source: &'static str,
46    pub message: String,
47    pub at: DateTime<Utc>,
48}
49
50#[derive(Debug, Clone, Default, Serialize)]
51pub struct CaptureReport {
52    pub checked_at: Option<DateTime<Utc>>,
53    pub newest_transcript_at: Option<DateTime<Utc>>,
54    /// Modification time of the newest transcript this daemon ingested.
55    pub captured_through: Option<DateTime<Utc>>,
56    pub lag_secs: Option<i64>,
57}
58
59#[derive(Debug, Clone, Default, Serialize)]
60pub struct IndexReport {
61    pub checked_at: Option<DateTime<Utc>>,
62    pub outbox_pending: Option<u64>,
63    pub backlog_since: Option<DateTime<Utc>>,
64}
65
66/// Only the tool, status and time: the daemon's message can echo the
67/// write's text, and `/health` is unauthenticated.
68#[derive(Debug, Clone, PartialEq, Serialize)]
69pub struct RejectedWrite {
70    pub tool: String,
71    pub status: u16,
72    pub at: DateTime<Utc>,
73}
74
75#[derive(Debug, Clone, Default, Serialize)]
76pub struct QueueReport {
77    pub depth: u64,
78    pub oldest_at: Option<DateTime<Utc>>,
79    pub oldest_age_secs: Option<i64>,
80    pub last_rejected: Option<RejectedWrite>,
81}
82
83/// Where the one-time compaction after the stored-content redaction pass
84/// stands: a VACUUM of the store and a merge of the lexical index, which
85/// clear the plaintext the pass left in free pages and old segments.
86#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize)]
87#[serde(rename_all = "snake_case")]
88pub enum MaintenanceState {
89    Pending,
90    WaitingForSpace,
91    MergingIndex,
92    /// Agent writes are queued until it ends.
93    Vacuuming,
94    Done,
95}
96
97#[derive(Debug, Clone, Serialize)]
98pub struct MaintenanceReport {
99    pub state: MaintenanceState,
100    pub since: Option<DateTime<Utc>>,
101    pub space: Option<VacuumSpace>,
102    pub last_error: Option<String>,
103}
104
105impl Default for MaintenanceReport {
106    fn default() -> Self {
107        Self {
108            state: MaintenanceState::Pending,
109            since: None,
110            space: None,
111            last_error: None,
112        }
113    }
114}
115
116#[derive(Debug, Clone, Default, Serialize)]
117pub struct SyncReport {
118    pub checked_at: Option<DateTime<Utc>>,
119    /// None when there is no `sync-peers` file.
120    pub peers: Option<usize>,
121    /// A sync cursor for some peer is on record.
122    pub synced_before: bool,
123    pub last_success_at: Option<DateTime<Utc>>,
124    /// The newest `sync_state` time: when records last moved, which bounds
125    /// the last success when no stamp was written yet.
126    pub last_cursor_at: Option<DateTime<Utc>>,
127}
128
129#[derive(Debug, Clone, Default, Serialize)]
130pub struct DistillReport {
131    pub checked_at: Option<DateTime<Utc>>,
132    pub pending: u64,
133    pub oldest_at: Option<DateTime<Utc>>,
134}
135
136/// How long a sync or a raw's distillation may be overdue before it
137/// degrades; None turns that check off.
138#[derive(Debug, Clone, Copy, PartialEq)]
139pub struct Windows {
140    pub sync_overdue: Option<Duration>,
141    pub undistilled: Option<Duration>,
142}
143
144impl Windows {
145    pub fn from_config(config: &AlertConfig) -> Self {
146        let hours = |h: u64| {
147            i64::try_from(h)
148                .ok()
149                .filter(|&h| h > 0)
150                .and_then(Duration::try_hours)
151        };
152        Self {
153            sync_overdue: hours(config.sync_overdue_hours),
154            undistilled: hours(config.undistilled_overdue_hours),
155        }
156    }
157}
158
159impl Default for Windows {
160    fn default() -> Self {
161        Self::from_config(&AlertConfig::default())
162    }
163}
164
165/// What one progress pass read. None where the read failed keeps the
166/// previous reading; `error` says what failed.
167#[derive(Debug, Clone, Default)]
168pub struct ProgressSeen {
169    pub sync: Option<SyncReport>,
170    pub distill: Option<DistillReport>,
171    pub error: Option<String>,
172}
173
174#[derive(Debug, Clone, PartialEq, Serialize)]
175pub struct Verdict {
176    pub state: State,
177    pub reason: Option<String>,
178}
179
180#[derive(Debug, Clone, Serialize)]
181pub struct HealthReport {
182    pub state: State,
183    pub reason: Option<String>,
184    pub since: DateTime<Utc>,
185    /// Capture and recall alone, leaving out sync and distillation: what an
186    /// agent using second-brain needs to be told about.
187    pub serving: Verdict,
188    pub capture: CaptureReport,
189    pub index: IndexReport,
190    pub queue: QueueReport,
191    pub maintenance: MaintenanceReport,
192    pub sync: SyncReport,
193    pub distill: DistillReport,
194    pub last_error: Option<ErrorRecord>,
195}
196
197/// What the reporters last said; only `assess` turns it into a state.
198struct Facts {
199    started: DateTime<Utc>,
200    capture: CaptureReport,
201    watcher_error: Option<String>,
202    ingest_error: Option<Streak>,
203    index: IndexReport,
204    drain_error: Option<Streak>,
205    indexes_error: Option<String>,
206    repair_error: Option<String>,
207    queue: QueueReport,
208    maintenance: MaintenanceReport,
209    sync: SyncReport,
210    distill: DistillReport,
211    progress_error: Option<Streak>,
212    windows: Windows,
213    last_error: Option<ErrorRecord>,
214}
215
216/// An error repeated since `since` with no success in between.
217struct Streak {
218    message: String,
219    since: DateTime<Utc>,
220}
221
222impl Streak {
223    fn extend(previous: Option<Streak>, message: String, now: DateTime<Utc>) -> Option<Streak> {
224        let since = previous.map_or(now, |p| p.since);
225        Some(Streak { message, since })
226    }
227
228    fn outlasted_grace(&self, now: DateTime<Utc>) -> bool {
229        now - self.since >= ERROR_GRACE
230    }
231}
232
233struct Inner {
234    facts: Facts,
235    report: HealthReport,
236    /// The state this process last sent, if it sent one.
237    sent: Option<State>,
238}
239
240pub struct Health {
241    inner: Mutex<Inner>,
242    transitions: Option<UnboundedSender<Transition>>,
243}
244
245impl Health {
246    /// `transitions` receives one message per state change, never one per
247    /// report; None when nothing listens.
248    pub fn new(now: DateTime<Utc>, transitions: Option<UnboundedSender<Transition>>) -> Self {
249        let facts = Facts {
250            started: now,
251            capture: CaptureReport::default(),
252            watcher_error: None,
253            ingest_error: None,
254            index: IndexReport::default(),
255            drain_error: None,
256            indexes_error: None,
257            repair_error: None,
258            queue: QueueReport::default(),
259            maintenance: MaintenanceReport::default(),
260            sync: SyncReport::default(),
261            distill: DistillReport::default(),
262            progress_error: None,
263            windows: Windows::default(),
264            last_error: None,
265        };
266        let report = HealthReport {
267            state: State::Ok,
268            reason: None,
269            since: now,
270            serving: Verdict {
271                state: State::Ok,
272                reason: None,
273            },
274            capture: CaptureReport::default(),
275            index: IndexReport::default(),
276            queue: QueueReport::default(),
277            maintenance: MaintenanceReport::default(),
278            sync: SyncReport::default(),
279            distill: DistillReport::default(),
280            last_error: None,
281        };
282        Self {
283            inner: Mutex::new(Inner {
284                facts,
285                report,
286                sent: None,
287            }),
288            transitions,
289        }
290    }
291
292    pub fn with_windows(self, windows: Windows) -> Self {
293        self.lock().facts.windows = windows;
294        self
295    }
296
297    pub fn progress_checked(&self, seen: ProgressSeen, now: DateTime<Utc>) {
298        self.update(now, |f| {
299            if let Some(sync) = seen.sync {
300                f.sync = SyncReport {
301                    checked_at: Some(now),
302                    ..sync
303                };
304            }
305            if let Some(distill) = seen.distill {
306                f.distill = DistillReport {
307                    checked_at: Some(now),
308                    ..distill
309                };
310            }
311            match seen.error {
312                None => f.progress_error = None,
313                Some(error) => {
314                    f.record_error("progress", &error, now);
315                    f.progress_error = Streak::extend(f.progress_error.take(), error, now);
316                }
317            }
318        });
319    }
320
321    /// Sends the current state unless this process already sent it, so a
322    /// receiver that missed transitions while the daemon was down learns it.
323    pub fn announce(&self, now: DateTime<Utc>) {
324        let mut inner = self.lock();
325        self.reassess(&mut inner, now, true);
326    }
327
328    pub fn report(&self) -> HealthReport {
329        self.lock().report.clone()
330    }
331
332    pub fn transcripts_scanned(&self, newest: Option<DateTime<Utc>>, now: DateTime<Utc>) {
333        self.update(now, |f| {
334            f.capture.checked_at = Some(now);
335            f.capture.newest_transcript_at = newest;
336        });
337    }
338
339    pub fn watcher_started(&self, now: DateTime<Utc>) {
340        self.update(now, |f| {
341            f.watcher_error = None;
342            f.capture.checked_at = Some(now);
343        });
344    }
345
346    pub fn watcher_failed(&self, error: &str, now: DateTime<Utc>) {
347        self.update(now, |f| {
348            f.watcher_error = Some(error.to_string());
349            f.record_error("watcher", error, now);
350        });
351    }
352
353    /// `transcript_mtime` is None for sources that are not transcripts,
354    /// which clear an ingest error without advancing capture.
355    pub fn ingested(&self, transcript_mtime: Option<DateTime<Utc>>, now: DateTime<Utc>) {
356        self.update(now, |f| {
357            f.ingest_error = None;
358            f.capture.checked_at = Some(now);
359            if let Some(mtime) = transcript_mtime {
360                let through = f.capture.captured_through.map_or(mtime, |t| t.max(mtime));
361                f.capture.captured_through = Some(through);
362            }
363        });
364    }
365
366    pub fn ingest_failed(&self, error: &str, now: DateTime<Utc>) {
367        self.update(now, |f| {
368            f.ingest_error = Streak::extend(f.ingest_error.take(), error.to_string(), now);
369            f.capture.checked_at = Some(now);
370            f.record_error("ingest", error, now);
371        });
372    }
373
374    pub fn drained(&self, outbox_pending: u64, now: DateTime<Utc>) {
375        self.update(now, |f| {
376            f.drain_error = None;
377            f.index.checked_at = Some(now);
378            f.index.outbox_pending = Some(outbox_pending);
379            f.index.backlog_since = if outbox_pending == 0 {
380                None
381            } else {
382                Some(f.index.backlog_since.unwrap_or(now))
383            };
384        });
385    }
386
387    pub fn drain_failed(&self, error: &str, now: DateTime<Utc>) {
388        self.update(now, |f| {
389            f.index.checked_at = Some(now);
390            f.drain_error = Streak::extend(f.drain_error.take(), error.to_string(), now);
391            f.record_error("drain", error, now);
392        });
393    }
394
395    /// No drain thread will ever report, so the error stays until restart.
396    pub fn indexes_unavailable(&self, error: &str, now: DateTime<Utc>) {
397        self.update(now, |f| {
398            f.indexes_error = Some(error.to_string());
399            f.record_error("index", error, now);
400        });
401    }
402
403    pub fn repaired(&self, now: DateTime<Utc>) {
404        self.update(now, |f| f.repair_error = None);
405    }
406
407    pub fn repair_failed(&self, error: &str, now: DateTime<Utc>) {
408        self.update(now, |f| {
409            f.repair_error = Some(error.to_string());
410            f.record_error("repair", error, now);
411        });
412    }
413
414    pub fn queue_checked(&self, depth: u64, oldest_at: Option<DateTime<Utc>>, now: DateTime<Utc>) {
415        self.update(now, |f| {
416            f.queue.depth = depth;
417            f.queue.oldest_at = oldest_at;
418        });
419    }
420
421    pub fn write_rejected(&self, tool: &str, status: u16, now: DateTime<Utc>) {
422        self.update(now, |f| {
423            f.queue.last_rejected = Some(RejectedWrite {
424                tool: tool.to_string(),
425                status,
426                at: now,
427            });
428        });
429    }
430
431    /// `space` is what the latest free-space check found, if one ran.
432    pub fn maintenance(
433        &self,
434        state: MaintenanceState,
435        space: Option<VacuumSpace>,
436        now: DateTime<Utc>,
437    ) {
438        self.update(now, |f| {
439            if f.maintenance.state != state {
440                f.maintenance.since = Some(now);
441            }
442            f.maintenance.state = state;
443            if space.is_some() {
444                f.maintenance.space = space;
445            }
446            if state == MaintenanceState::Done {
447                f.maintenance.last_error = None;
448            }
449        });
450    }
451
452    /// Stays until the compaction next completes.
453    pub fn maintenance_failed(&self, error: &str, now: DateTime<Utc>) {
454        self.update(now, |f| {
455            f.maintenance.last_error = Some(error.to_string());
456            f.record_error("maintenance", error, now);
457        });
458    }
459
460    fn lock(&self) -> std::sync::MutexGuard<'_, Inner> {
461        match self.inner.lock() {
462            Ok(g) => g,
463            Err(poisoned) => poisoned.into_inner(),
464        }
465    }
466
467    fn update(&self, now: DateTime<Utc>, change: impl FnOnce(&mut Facts)) {
468        let mut inner = self.lock();
469        change(&mut inner.facts);
470        self.reassess(&mut inner, now, false);
471    }
472
473    /// `announcing` sends an unchanged state that this process has not sent.
474    fn reassess(&self, inner: &mut Inner, now: DateTime<Utc>, announcing: bool) {
475        let mut serving = Assessment::default();
476        assess_serving(&inner.facts, now, &mut serving);
477        let mut overall = serving.clone();
478        assess_upkeep(&inner.facts, now, &mut overall);
479        let (state, reason) = overall.verdict();
480        let changed = state != inner.report.state;
481        let since = if changed { now } else { inner.report.since };
482        let mut capture = inner.facts.capture.clone();
483        capture.lag_secs = capture_lag(&inner.facts).map(|d| d.num_seconds());
484        let mut queue = inner.facts.queue.clone();
485        queue.oldest_age_secs = queue.oldest_at.map(|t| (now - t).num_seconds().max(0));
486        inner.report = HealthReport {
487            state,
488            reason: reason.clone(),
489            since,
490            serving: Verdict {
491                state: serving.state,
492                reason: serving.verdict().1,
493            },
494            capture,
495            index: inner.facts.index.clone(),
496            queue,
497            maintenance: inner.facts.maintenance.clone(),
498            sync: inner.facts.sync.clone(),
499            distill: inner.facts.distill.clone(),
500            last_error: inner.facts.last_error.clone(),
501        };
502        // Sent under the lock so concurrent reporters cannot deliver
503        // transitions out of order; the send never blocks.
504        let unsent = inner.sent != Some(state);
505        if (changed || (announcing && unsent))
506            && let Some(tx) = &self.transitions
507        {
508            // The listener only goes away when the runtime shuts down.
509            let _ = tx.send(Transition {
510                state,
511                reason,
512                at: now,
513            });
514            inner.sent = Some(state);
515        }
516    }
517}
518
519impl Facts {
520    fn record_error(&mut self, source: &'static str, message: &str, at: DateTime<Utc>) {
521        self.last_error = Some(ErrorRecord {
522            source,
523            message: message.to_string(),
524            at,
525        });
526    }
527}
528
529/// Transcripts written before the daemon started are not its lag: it
530/// watches for changes and never saw them.
531fn capture_lag(f: &Facts) -> Option<Duration> {
532    let newest = f.capture.newest_transcript_at?;
533    let through = f
534        .capture
535        .captured_through
536        .map_or(f.started, |t| t.max(f.started));
537    Some((newest - through).max(Duration::zero()))
538}
539
540#[derive(Clone, Default)]
541struct Assessment {
542    state: State,
543    problems: Vec<String>,
544}
545
546impl Assessment {
547    fn flag(&mut self, level: State, problem: String) {
548        self.state = self.state.max(level);
549        self.problems.push(problem);
550    }
551
552    fn verdict(&self) -> (State, Option<String>) {
553        let reason = (!self.problems.is_empty()).then(|| self.problems.join("; "));
554        (self.state, reason)
555    }
556}
557
558fn assess_serving(f: &Facts, now: DateTime<Utc>, a: &mut Assessment) {
559    let mut flag = |level: State, problem: String| a.flag(level, problem);
560
561    match &f.watcher_error {
562        Some(e) => flag(
563            State::Failing,
564            format!("transcript watcher is not running: {e}"),
565        ),
566        None => {
567            let last = f.capture.checked_at.unwrap_or(f.started);
568            if now - last >= WATCHER_SILENT {
569                flag(
570                    State::Degraded,
571                    format!(
572                        "transcript watcher has not reported for {}",
573                        human(now - last)
574                    ),
575                );
576            }
577        }
578    }
579    if let Some(lag) = capture_lag(f)
580        && lag >= CAPTURE_LAG_DEGRADED
581    {
582        let level = if lag >= CAPTURE_LAG_FAILING {
583            State::Failing
584        } else {
585            State::Degraded
586        };
587        flag(
588            level,
589            format!("capture is {} behind the newest transcript", human(lag)),
590        );
591    }
592    if let Some(e) = &f.ingest_error
593        && e.outlasted_grace(now)
594    {
595        flag(
596            State::Degraded,
597            format!(
598                "ingest has failed for {}: {}",
599                human(now - e.since),
600                e.message
601            ),
602        );
603    }
604
605    if let Some(e) = &f.drain_error
606        && e.outlasted_grace(now)
607    {
608        flag(
609            State::Degraded,
610            format!(
611                "index drain failed for {}: {}",
612                human(now - e.since),
613                e.message
614            ),
615        );
616    }
617    match &f.indexes_error {
618        Some(e) => flag(State::Degraded, format!("search indexes unavailable: {e}")),
619        None => {
620            let last = f.index.checked_at.unwrap_or(f.started);
621            if now - last >= DRAIN_SILENT {
622                flag(
623                    State::Degraded,
624                    format!("index drain has not reported for {}", human(now - last)),
625                );
626            }
627        }
628    }
629    if let (Some(since), Some(pending)) = (f.index.backlog_since, f.index.outbox_pending) {
630        let age = now - since;
631        if age >= BACKLOG_DEGRADED {
632            let level = if age >= BACKLOG_FAILING {
633                State::Failing
634            } else {
635                State::Degraded
636            };
637            flag(
638                level,
639                format!("{pending} changes have waited {} to be indexed", human(age)),
640            );
641        }
642    }
643    if let Some(e) = &f.repair_error {
644        flag(State::Degraded, format!("last store repair failed: {e}"));
645    }
646    if let Some(oldest) = f.queue.oldest_at
647        && now - oldest >= QUEUE_STUCK
648    {
649        flag(
650            State::Degraded,
651            format!(
652                "write queue not draining ({} queued, oldest {})",
653                f.queue.depth,
654                human(now - oldest)
655            ),
656        );
657    }
658
659    match (&f.maintenance.state, &f.maintenance.space) {
660        (MaintenanceState::Vacuuming, _) => flag(
661            State::Degraded,
662            "store maintenance is running; writes are queued until it finishes".to_string(),
663        ),
664        (MaintenanceState::WaitingForSpace, Some(space)) => flag(
665            State::Degraded,
666            format!(
667                "store compaction waits for disk space: it needs {} MB free at {} and has {} MB",
668                space.needed_bytes / 1_000_000,
669                space.path.display(),
670                space.free_bytes / 1_000_000
671            ),
672        ),
673        _ => {}
674    }
675    if let Some(e) = &f.maintenance.last_error {
676        flag(State::Degraded, format!("store compaction failed: {e}"));
677    }
678}
679
680/// Sync and distillation: overdue work that leaves capture and recall
681/// running.
682fn assess_upkeep(f: &Facts, now: DateTime<Utc>, a: &mut Assessment) {
683    let mut flag = |level: State, problem: String| a.flag(level, problem);
684    let sync = &f.sync;
685    if (sync.peers.is_some_and(|n| n > 0) || sync.synced_before)
686        && let Some(window) = f.windows.sync_overdue
687    {
688        let since = sync
689            .last_success_at
690            .or(sync.last_cursor_at)
691            .unwrap_or(f.started);
692        let overdue = now - since;
693        if overdue >= window {
694            let problem = match sync.peers {
695                Some(0) => format!(
696                    "no sync has succeeded in {}, and ~/.second-brain/sync-peers lists no peers \
697                     although this device has synced before; add each peer to it as user@host, \
698                     one per line, or set alert.sync_overdue_hours = 0 if this device no longer syncs",
699                    human(overdue)
700                ),
701                Some(n) => format!(
702                    "no sync has succeeded in {} with {} in sync-peers; \
703                     run `sb sync <peer>` to see why",
704                    human(overdue),
705                    counted(n as u64, "peer", "peers")
706                ),
707                None => format!(
708                    "no sync has succeeded in {} although this device has synced before; \
709                     run `sb sync <user@host>` (or check the job that runs it), \
710                     or set alert.sync_overdue_hours = 0 if this device no longer syncs",
711                    human(overdue)
712                ),
713            };
714            flag(State::Degraded, problem);
715        }
716    }
717    if let (Some(oldest), Some(window)) = (f.distill.oldest_at, f.windows.undistilled)
718        && now - oldest >= window
719    {
720        flag(
721            State::Degraded,
722            format!(
723                "{} waited {} to be distilled; run `sb consolidate`, or set \
724                 alert.undistilled_overdue_hours = 0 if this device does not distill",
725                if f.distill.pending == 1 {
726                    "1 raw memory has".to_string()
727                } else {
728                    format!("{} raw memories have", f.distill.pending)
729                },
730                human(now - oldest)
731            ),
732        );
733    }
734    if let Some(e) = &f.progress_error
735        && e.outlasted_grace(now)
736    {
737        flag(
738            State::Degraded,
739            format!(
740                "cannot check sync and distillation for {}: {}",
741                human(now - e.since),
742                e.message
743            ),
744        );
745    }
746}
747
748fn counted(n: u64, one: &str, many: &str) -> String {
749    if n == 1 {
750        format!("1 {one}")
751    } else {
752        format!("{n} {many}")
753    }
754}
755
756fn human(d: Duration) -> String {
757    let minutes = d.num_minutes();
758    if minutes < 60 {
759        format!("{minutes}m")
760    } else if minutes < 48 * 60 {
761        format!("{}h {}m", minutes / 60, minutes % 60)
762    } else {
763        format!("{}d", minutes / (24 * 60))
764    }
765}
766
767#[cfg(test)]
768mod tests {
769    use tokio::sync::mpsc::{UnboundedReceiver, unbounded_channel};
770
771    use super::*;
772
773    fn t0() -> DateTime<Utc> {
774        DateTime::parse_from_rfc3339("2026-10-31T12:00:00Z")
775            .unwrap()
776            .with_timezone(&Utc)
777    }
778
779    fn mins(m: i64) -> DateTime<Utc> {
780        t0() + Duration::minutes(m)
781    }
782
783    fn monitored() -> (Health, UnboundedReceiver<Transition>) {
784        let (tx, rx) = unbounded_channel();
785        (Health::new(t0(), Some(tx)), rx)
786    }
787
788    fn drain(rx: &mut UnboundedReceiver<Transition>) -> Vec<State> {
789        std::iter::from_fn(|| rx.try_recv().ok())
790            .map(|t| t.state)
791            .collect()
792    }
793
794    #[test]
795    fn a_fresh_daemon_that_keeps_up_is_ok() {
796        let (health, mut rx) = monitored();
797        health.watcher_started(mins(0));
798        health.transcripts_scanned(Some(mins(1)), mins(1));
799        health.ingested(Some(mins(1)), mins(1));
800        health.drained(0, mins(1));
801        let report = health.report();
802        assert_eq!(report.state, State::Ok);
803        assert_eq!(report.reason, None);
804        assert_eq!(report.capture.lag_secs, Some(0));
805        assert!(drain(&mut rx).is_empty());
806    }
807
808    #[test]
809    fn capture_lag_degrades_then_fails_then_recovers() {
810        let (health, mut rx) = monitored();
811        health.drained(0, mins(0));
812        health.ingested(Some(mins(1)), mins(1));
813        health.transcripts_scanned(Some(mins(10)), mins(10));
814        assert_eq!(health.report().state, State::Ok);
815
816        health.transcripts_scanned(Some(mins(20)), mins(20));
817        let report = health.report();
818        assert_eq!(report.state, State::Degraded);
819        assert_eq!(report.capture.lag_secs, Some(19 * 60));
820        assert!(report.reason.unwrap().contains("capture is 19m behind"));
821
822        health.drained(0, mins(125));
823        health.transcripts_scanned(Some(mins(125)), mins(125));
824        assert_eq!(health.report().state, State::Failing);
825
826        health.ingested(Some(mins(125)), mins(126));
827        assert_eq!(health.report().state, State::Ok);
828        assert_eq!(
829            drain(&mut rx),
830            vec![State::Degraded, State::Failing, State::Ok]
831        );
832    }
833
834    #[test]
835    fn transcripts_from_before_startup_are_not_lag() {
836        let (health, _rx) = monitored();
837        health.transcripts_scanned(Some(t0() - Duration::days(3)), mins(1));
838        assert_eq!(health.report().capture.lag_secs, Some(0));
839        assert_eq!(health.report().state, State::Ok);
840    }
841
842    #[test]
843    fn repeated_reports_in_one_state_send_one_transition() {
844        let (health, mut rx) = monitored();
845        for m in 0..=12 {
846            health.ingest_failed("database is locked", mins(m));
847        }
848        health.ingested(None, mins(13));
849        health.ingested(None, mins(14));
850        assert_eq!(drain(&mut rx), vec![State::Degraded, State::Ok]);
851        let report = health.report();
852        assert_eq!(report.since, mins(13));
853        let last = report.last_error.expect("the error is kept after recovery");
854        assert_eq!(last.source, "ingest");
855        assert_eq!(last.at, mins(12));
856    }
857
858    #[test]
859    fn an_error_that_clears_within_the_grace_sends_nothing() {
860        let (health, mut rx) = monitored();
861        health.watcher_started(mins(0));
862        health.ingest_failed("database is locked", mins(0));
863        health.drain_failed("database is locked", mins(1));
864        health.drained(0, mins(9));
865        health.ingested(None, mins(9));
866        assert_eq!(health.report().state, State::Ok);
867        assert!(drain(&mut rx).is_empty());
868        assert_eq!(health.report().last_error.unwrap().source, "drain");
869    }
870
871    #[test]
872    fn a_drain_error_degrades_once_it_outlasts_the_grace() {
873        let (health, _rx) = monitored();
874        health.watcher_started(mins(0));
875        for m in 0..10 {
876            health.drain_failed("disk I/O error", mins(m));
877        }
878        assert_eq!(health.report().state, State::Ok);
879        health.drain_failed("disk I/O error", mins(10));
880        let report = health.report();
881        assert_eq!(report.state, State::Degraded);
882        assert!(report.reason.unwrap().contains("index drain failed"));
883        health.drained(0, mins(11));
884        assert_eq!(health.report().state, State::Ok);
885    }
886
887    #[test]
888    fn a_long_ingest_batch_counts_as_the_watcher_reporting() {
889        let (health, _rx) = monitored();
890        health.watcher_started(mins(0));
891        health.ingested(None, mins(10));
892        health.ingest_failed("unreadable line", mins(19));
893        health.drained(0, mins(20));
894        let report = health.report();
895        assert_eq!(report.state, State::Ok, "{:?}", report.reason);
896    }
897
898    #[test]
899    fn an_outbox_backlog_only_counts_once_it_persists() {
900        let (health, _rx) = monitored();
901        health.drained(500, mins(0));
902        health.drained(800, mins(14));
903        health.watcher_started(mins(14));
904        assert_eq!(health.report().state, State::Ok);
905        health.drained(800, mins(15));
906        let report = health.report();
907        assert_eq!(report.state, State::Degraded);
908        assert_eq!(report.index.backlog_since, Some(mins(0)));
909        health.drained(0, mins(16));
910        assert_eq!(health.report().state, State::Ok);
911        assert_eq!(health.report().index.backlog_since, None);
912    }
913
914    #[test]
915    fn a_dead_watcher_fails_and_a_silent_one_degrades() {
916        let (health, _rx) = monitored();
917        health.watcher_failed("inotify watch limit reached", mins(0));
918        assert_eq!(health.report().state, State::Failing);
919        health.watcher_started(mins(1));
920        health.drained(0, mins(1));
921        assert_eq!(health.report().state, State::Ok);
922        health.drained(0, mins(16));
923        let report = health.report();
924        assert_eq!(report.state, State::Degraded);
925        assert!(report.reason.unwrap().contains("watcher has not reported"));
926    }
927
928    #[test]
929    fn index_and_repair_errors_degrade_until_the_next_success() {
930        let (health, _rx) = monitored();
931        health.watcher_started(mins(0));
932        health.drain_failed("disk I/O error", mins(0));
933        health.repair_failed("disk I/O error", mins(0));
934        let reason = health.report().reason.unwrap();
935        assert!(reason.contains("store repair failed"), "{reason}");
936        health.drain_failed("disk I/O error", mins(10));
937        let reason = health.report().reason.unwrap();
938        assert!(reason.contains("index drain failed"), "{reason}");
939        health.drained(0, mins(11));
940        assert_eq!(health.report().state, State::Degraded);
941        health.repaired(mins(12));
942        assert_eq!(health.report().state, State::Ok);
943    }
944
945    #[test]
946    fn missing_indexes_stay_degraded_without_a_drain() {
947        let (health, _rx) = monitored();
948        health.indexes_unavailable("tantivy meta.json missing", mins(0));
949        health.watcher_started(mins(90));
950        let report = health.report();
951        assert_eq!(report.state, State::Degraded);
952        let reason = report.reason.unwrap();
953        assert!(reason.contains("search indexes unavailable"), "{reason}");
954        assert!(!reason.contains("has not reported"), "{reason}");
955    }
956
957    #[test]
958    fn the_report_serializes_without_content() {
959        let (health, _rx) = monitored();
960        health.ingest_failed("database is locked", mins(0));
961        health.ingest_failed("database is locked", mins(10));
962        let json = serde_json::to_value(health.report()).unwrap();
963        assert_eq!(json["state"], "degraded");
964        assert_eq!(json["last_error"]["source"], "ingest");
965        for key in ["reason", "since", "capture", "index"] {
966            assert!(json.get(key).is_some(), "{key} missing: {json}");
967        }
968    }
969
970    #[test]
971    fn a_queue_entry_older_than_ten_minutes_degrades_until_it_drains() {
972        let (health, mut rx) = monitored();
973        health.watcher_started(mins(0));
974        health.drained(0, mins(0));
975        health.queue_checked(2, Some(mins(0)), mins(9));
976        assert_eq!(health.report().state, State::Ok);
977        assert_eq!(health.report().queue.depth, 2);
978        health.queue_checked(2, Some(mins(0)), mins(10));
979        let report = health.report();
980        assert_eq!(report.state, State::Degraded);
981        assert!(
982            report
983                .reason
984                .as_deref()
985                .unwrap()
986                .contains("write queue not draining"),
987            "{report:?}"
988        );
989        assert_eq!(report.queue.oldest_age_secs, Some(600));
990        health.queue_checked(0, None, mins(11));
991        assert_eq!(health.report().state, State::Ok);
992        assert_eq!(drain(&mut rx), vec![State::Degraded, State::Ok]);
993    }
994
995    #[test]
996    fn a_rejected_write_is_reported_by_tool_status_and_time_only() {
997        let (health, _rx) = monitored();
998        health.write_rejected("associate", 404, mins(3));
999        let json = serde_json::to_value(health.report()).unwrap();
1000        assert_eq!(
1001            json["queue"]["last_rejected"],
1002            serde_json::json!({"tool": "associate", "status": 404, "at": "2026-10-31T12:03:00Z"})
1003        );
1004        assert_eq!(json["state"], "ok");
1005    }
1006
1007    fn hours(h: i64) -> DateTime<Utc> {
1008        t0() + Duration::hours(h)
1009    }
1010
1011    fn alive(health: &Health, at: DateTime<Utc>) {
1012        health.update(at, |f| {
1013            f.capture.checked_at = Some(at);
1014            f.index.checked_at = Some(at);
1015        });
1016    }
1017
1018    fn sync_seen(health: &Health, sync: SyncReport, at: DateTime<Utc>) {
1019        let seen = ProgressSeen {
1020            sync: Some(sync),
1021            ..ProgressSeen::default()
1022        };
1023        health.progress_checked(seen, at);
1024    }
1025
1026    fn distill_seen(
1027        health: &Health,
1028        pending: u64,
1029        oldest_at: Option<DateTime<Utc>>,
1030        at: DateTime<Utc>,
1031    ) {
1032        let distill = DistillReport {
1033            checked_at: None,
1034            pending,
1035            oldest_at,
1036        };
1037        let seen = ProgressSeen {
1038            distill: Some(distill),
1039            ..ProgressSeen::default()
1040        };
1041        health.progress_checked(seen, at);
1042    }
1043
1044    fn synced(peers: usize, synced_before: bool, last: Option<DateTime<Utc>>) -> SyncReport {
1045        SyncReport {
1046            checked_at: None,
1047            peers: Some(peers),
1048            synced_before,
1049            last_success_at: last,
1050            last_cursor_at: None,
1051        }
1052    }
1053
1054    #[test]
1055    fn without_a_stamp_the_window_counts_from_when_records_last_moved() {
1056        let (health, _rx) = monitored();
1057        let seen = SyncReport {
1058            last_cursor_at: Some(hours(-48)),
1059            ..synced(0, true, None)
1060        };
1061        sync_seen(&health, seen.clone(), mins(0));
1062        let report = health.report();
1063        assert_eq!(report.state, State::Degraded);
1064        let reason = report.reason.unwrap();
1065        assert!(reason.contains("no sync has succeeded in 2d"), "{reason}");
1066
1067        let stamped = SyncReport {
1068            last_success_at: Some(mins(0)),
1069            ..seen
1070        };
1071        sync_seen(&health, stamped, mins(0));
1072        assert_eq!(health.report().state, State::Ok);
1073    }
1074
1075    #[test]
1076    fn reasons_count_one_peer_and_one_raw_in_the_singular() {
1077        let (health, _rx) = monitored();
1078        sync_seen(&health, synced(1, true, Some(hours(-30))), mins(0));
1079        distill_seen(&health, 1, Some(hours(-30)), mins(0));
1080        let reason = health.report().reason.unwrap();
1081        assert!(reason.contains("with 1 peer in sync-peers"), "{reason}");
1082        assert!(reason.contains("1 raw memory has waited"), "{reason}");
1083    }
1084
1085    #[test]
1086    fn a_device_that_synced_before_and_lost_its_peers_degrades_with_the_fix() {
1087        let (health, mut rx) = monitored();
1088        sync_seen(&health, synced(0, true, Some(hours(-25))), mins(0));
1089        let report = health.report();
1090        assert_eq!(report.state, State::Degraded);
1091        let reason = report.reason.unwrap();
1092        assert!(
1093            reason.contains("no sync has succeeded in 25h 0m"),
1094            "{reason}"
1095        );
1096        assert!(reason.contains("lists no peers"), "{reason}");
1097        assert!(reason.contains("~/.second-brain/sync-peers"), "{reason}");
1098        assert!(reason.contains("sync_overdue_hours = 0"), "{reason}");
1099        assert_eq!(report.sync.peers, Some(0));
1100        assert_eq!(report.sync.checked_at, Some(mins(0)));
1101        assert_eq!(drain(&mut rx), vec![State::Degraded]);
1102
1103        sync_seen(&health, synced(1, true, Some(mins(1))), mins(1));
1104        assert_eq!(health.report().state, State::Ok);
1105    }
1106
1107    #[test]
1108    fn a_device_synced_by_hand_without_sync_peers_stays_ok_while_it_syncs() {
1109        let (health, mut rx) = monitored();
1110        let by_hand = SyncReport {
1111            peers: None,
1112            ..synced(0, true, Some(hours(10)))
1113        };
1114        alive(&health, hours(30));
1115        sync_seen(&health, by_hand.clone(), hours(30));
1116        assert_eq!(health.report().state, State::Ok);
1117        alive(&health, hours(34));
1118        sync_seen(&health, by_hand, hours(34));
1119        let report = health.report();
1120        assert_eq!(report.state, State::Degraded);
1121        let reason = report.reason.unwrap();
1122        assert!(
1123            reason.contains("no sync has succeeded in 24h 0m"),
1124            "{reason}"
1125        );
1126        assert!(reason.contains("sb sync <user@host>"), "{reason}");
1127        assert!(reason.contains("sync_overdue_hours = 0"), "{reason}");
1128        assert!(!reason.contains("sync-peers"), "{reason}");
1129        assert_eq!(drain(&mut rx), vec![State::Degraded]);
1130    }
1131
1132    #[test]
1133    fn a_solo_device_that_never_synced_stays_ok() {
1134        let (health, mut rx) = monitored();
1135        for h in [0, 25, 100] {
1136            alive(&health, hours(h));
1137            sync_seen(&health, synced(0, false, None), hours(h));
1138        }
1139        let report = health.report();
1140        assert_eq!(report.state, State::Ok, "{:?}", report.reason);
1141        assert!(drain(&mut rx).is_empty());
1142    }
1143
1144    #[test]
1145    fn a_sync_overdue_past_the_window_degrades_only_on_a_device_that_syncs() {
1146        let (health, _rx) = monitored();
1147        alive(&health, hours(0));
1148        sync_seen(&health, synced(2, true, Some(hours(-23))), hours(0));
1149        assert_eq!(health.report().state, State::Ok);
1150
1151        alive(&health, hours(1));
1152        sync_seen(&health, synced(2, true, Some(hours(-23))), hours(1));
1153        let report = health.report();
1154        assert_eq!(report.state, State::Degraded);
1155        let reason = report.reason.unwrap();
1156        assert!(
1157            reason.contains("no sync has succeeded in 24h 0m"),
1158            "{reason}"
1159        );
1160        assert!(reason.contains("2 peers"), "{reason}");
1161
1162        sync_seen(&health, synced(2, true, Some(hours(1))), hours(1));
1163        assert_eq!(health.report().state, State::Ok);
1164
1165        sync_seen(&health, synced(0, false, Some(hours(-500))), hours(1));
1166        assert_eq!(health.report().state, State::Ok);
1167    }
1168
1169    #[test]
1170    fn with_peers_and_no_recorded_success_the_window_starts_at_startup() {
1171        let (health, _rx) = monitored();
1172        alive(&health, hours(23));
1173        sync_seen(&health, synced(1, false, None), hours(23));
1174        assert_eq!(health.report().state, State::Ok);
1175        alive(&health, hours(24));
1176        sync_seen(&health, synced(1, false, None), hours(24));
1177        let reason = health.report().reason.unwrap();
1178        assert!(
1179            reason.contains("no sync has succeeded in 24h 0m"),
1180            "{reason}"
1181        );
1182    }
1183
1184    #[test]
1185    fn raws_left_undistilled_past_the_window_degrade_with_the_count() {
1186        let (health, mut rx) = monitored();
1187        alive(&health, hours(0));
1188        distill_seen(&health, 520, Some(hours(-23)), hours(0));
1189        assert_eq!(health.report().state, State::Ok);
1190        assert_eq!(health.report().distill.pending, 520);
1191
1192        alive(&health, hours(2));
1193        distill_seen(&health, 520, Some(hours(-23)), hours(2));
1194        let report = health.report();
1195        assert_eq!(report.state, State::Degraded);
1196        let reason = report.reason.unwrap();
1197        assert!(
1198            reason.contains("520 raw memories have waited 25h 0m to be distilled"),
1199            "{reason}"
1200        );
1201        assert!(reason.contains("undistilled_overdue_hours = 0"), "{reason}");
1202
1203        alive(&health, hours(3));
1204        distill_seen(&health, 0, None, hours(3));
1205        assert_eq!(health.report().state, State::Ok);
1206        assert_eq!(drain(&mut rx), vec![State::Degraded, State::Ok]);
1207    }
1208
1209    #[test]
1210    fn a_window_turned_off_never_degrades() {
1211        let (tx, _rx) = unbounded_channel();
1212        let windows = Windows::from_config(&AlertConfig {
1213            sync_overdue_hours: 0,
1214            undistilled_overdue_hours: 0,
1215            ..AlertConfig::default()
1216        });
1217        assert_eq!(
1218            windows,
1219            Windows {
1220                sync_overdue: None,
1221                undistilled: None
1222            }
1223        );
1224        let health = Health::new(t0(), Some(tx)).with_windows(windows);
1225        alive(&health, hours(100));
1226        sync_seen(&health, synced(1, true, Some(hours(-100))), hours(100));
1227        distill_seen(&health, 9, Some(hours(-100)), hours(100));
1228        assert_eq!(health.report().state, State::Ok);
1229        sync_seen(&health, synced(0, true, Some(hours(-100))), hours(100));
1230        assert_eq!(health.report().state, State::Ok);
1231    }
1232
1233    #[test]
1234    fn the_windows_come_from_the_alert_config() {
1235        let windows = Windows::from_config(&AlertConfig {
1236            sync_overdue_hours: 6,
1237            undistilled_overdue_hours: 48,
1238            ..AlertConfig::default()
1239        });
1240        assert_eq!(windows.sync_overdue, Some(Duration::hours(6)));
1241        assert_eq!(windows.undistilled, Some(Duration::hours(48)));
1242        assert_eq!(Windows::default().sync_overdue, Some(Duration::hours(24)));
1243        assert_eq!(Windows::default().undistilled, Some(Duration::hours(24)));
1244    }
1245
1246    #[test]
1247    fn announce_sends_the_current_state_once_unless_it_was_already_sent() {
1248        let (health, mut rx) = monitored();
1249        health.announce(mins(0));
1250        let sent: Vec<Transition> = std::iter::from_fn(|| rx.try_recv().ok()).collect();
1251        assert_eq!(
1252            sent,
1253            vec![Transition {
1254                state: State::Ok,
1255                reason: None,
1256                at: mins(0),
1257            }]
1258        );
1259
1260        health.announce(mins(1));
1261        assert!(drain(&mut rx).is_empty(), "ok was already sent");
1262
1263        sync_seen(&health, synced(0, true, Some(hours(-30))), mins(2));
1264        health.announce(mins(3));
1265        assert_eq!(drain(&mut rx), vec![State::Degraded]);
1266    }
1267
1268    #[test]
1269    fn announce_reassesses_and_sends_one_message_for_a_pending_change() {
1270        let (health, mut rx) = monitored();
1271        alive(&health, hours(0));
1272        distill_seen(&health, 3, Some(hours(-23)), hours(0));
1273        assert!(drain(&mut rx).is_empty());
1274        health.announce(hours(2));
1275        assert_eq!(drain(&mut rx), vec![State::Degraded]);
1276    }
1277
1278    #[test]
1279    fn serving_leaves_out_sync_and_distillation() {
1280        let (health, _rx) = monitored();
1281        alive(&health, hours(0));
1282        let seen = ProgressSeen {
1283            sync: Some(synced(1, true, Some(hours(-30)))),
1284            distill: Some(DistillReport {
1285                checked_at: None,
1286                pending: 4,
1287                oldest_at: Some(hours(-30)),
1288            }),
1289            error: None,
1290        };
1291        health.progress_checked(seen, hours(0));
1292        let report = health.report();
1293        assert_eq!(report.state, State::Degraded);
1294        assert_eq!(
1295            report.serving,
1296            Verdict {
1297                state: State::Ok,
1298                reason: None
1299            }
1300        );
1301
1302        health.watcher_failed("inotify watch limit reached", hours(0));
1303        let report = health.report();
1304        assert_eq!(report.state, State::Failing);
1305        assert_eq!(report.serving.state, State::Failing);
1306        let serving = report.serving.reason.unwrap();
1307        assert!(serving.contains("watcher is not running"), "{serving}");
1308        assert!(!serving.contains("sync"), "{serving}");
1309
1310        let json = serde_json::to_value(health.report()).unwrap();
1311        assert_eq!(json["serving"]["state"], "failing");
1312    }
1313
1314    #[test]
1315    fn one_pass_that_clears_sync_and_finds_distillation_overdue_sends_nothing() {
1316        let (health, mut rx) = monitored();
1317        alive(&health, hours(0));
1318        sync_seen(&health, synced(1, true, Some(hours(-30))), hours(0));
1319        assert_eq!(drain(&mut rx), vec![State::Degraded]);
1320        let seen = ProgressSeen {
1321            sync: Some(synced(1, true, Some(hours(0)))),
1322            distill: Some(DistillReport {
1323                checked_at: None,
1324                pending: 4,
1325                oldest_at: Some(hours(-30)),
1326            }),
1327            error: None,
1328        };
1329        health.progress_checked(seen, hours(0));
1330        assert_eq!(health.report().state, State::Degraded);
1331        assert!(drain(&mut rx).is_empty());
1332    }
1333}