1use std::sync::Mutex;
6
7use chrono::{DateTime, Duration, Utc};
8use second_brain_core::config::AlertConfig;
9use second_brain_core::sqlite_store::VacuumSpace;
10use serde::Serialize;
11use tokio::sync::mpsc::UnboundedSender;
12
13const CAPTURE_LAG_DEGRADED: Duration = Duration::minutes(15);
14const CAPTURE_LAG_FAILING: Duration = Duration::hours(2);
15const BACKLOG_DEGRADED: Duration = Duration::minutes(15);
16const BACKLOG_FAILING: Duration = Duration::hours(2);
17const WATCHER_SILENT: Duration = Duration::minutes(15);
19const DRAIN_SILENT: Duration = Duration::hours(1);
21const ERROR_GRACE: Duration = Duration::minutes(10);
24const QUEUE_STUCK: Duration = Duration::minutes(10);
26
27#[derive(Debug, Clone, Copy, Default, PartialEq, Eq, PartialOrd, Ord, Serialize)]
28#[serde(rename_all = "lowercase")]
29pub enum State {
30 #[default]
31 Ok,
32 Degraded,
33 Failing,
34}
35
36#[derive(Debug, Clone, PartialEq, Serialize)]
37pub struct Transition {
38 pub state: State,
39 pub reason: Option<String>,
40 pub at: DateTime<Utc>,
41}
42
43#[derive(Debug, Clone, Serialize)]
44pub struct ErrorRecord {
45 pub source: &'static str,
46 pub message: String,
47 pub at: DateTime<Utc>,
48}
49
50#[derive(Debug, Clone, Default, Serialize)]
51pub struct CaptureReport {
52 pub checked_at: Option<DateTime<Utc>>,
53 pub newest_transcript_at: Option<DateTime<Utc>>,
54 pub captured_through: Option<DateTime<Utc>>,
56 pub lag_secs: Option<i64>,
57}
58
59#[derive(Debug, Clone, Default, Serialize)]
60pub struct IndexReport {
61 pub checked_at: Option<DateTime<Utc>>,
62 pub outbox_pending: Option<u64>,
63 pub backlog_since: Option<DateTime<Utc>>,
64}
65
66#[derive(Debug, Clone, PartialEq, Serialize)]
69pub struct RejectedWrite {
70 pub tool: String,
71 pub status: u16,
72 pub at: DateTime<Utc>,
73}
74
75#[derive(Debug, Clone, Default, Serialize)]
76pub struct QueueReport {
77 pub depth: u64,
78 pub oldest_at: Option<DateTime<Utc>>,
79 pub oldest_age_secs: Option<i64>,
80 pub last_rejected: Option<RejectedWrite>,
81}
82
83#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize)]
87#[serde(rename_all = "snake_case")]
88pub enum MaintenanceState {
89 Pending,
90 WaitingForSpace,
91 MergingIndex,
92 Vacuuming,
94 Done,
95}
96
97#[derive(Debug, Clone, Serialize)]
98pub struct MaintenanceReport {
99 pub state: MaintenanceState,
100 pub since: Option<DateTime<Utc>>,
101 pub space: Option<VacuumSpace>,
102 pub last_error: Option<String>,
103}
104
105impl Default for MaintenanceReport {
106 fn default() -> Self {
107 Self {
108 state: MaintenanceState::Pending,
109 since: None,
110 space: None,
111 last_error: None,
112 }
113 }
114}
115
116#[derive(Debug, Clone, Default, Serialize)]
117pub struct SyncReport {
118 pub checked_at: Option<DateTime<Utc>>,
119 pub peers: Option<usize>,
121 pub synced_before: bool,
123 pub last_success_at: Option<DateTime<Utc>>,
124 pub last_cursor_at: Option<DateTime<Utc>>,
127}
128
129#[derive(Debug, Clone, Default, Serialize)]
130pub struct DistillReport {
131 pub checked_at: Option<DateTime<Utc>>,
132 pub pending: u64,
133 pub oldest_at: Option<DateTime<Utc>>,
134}
135
136#[derive(Debug, Clone, Copy, PartialEq)]
139pub struct Windows {
140 pub sync_overdue: Option<Duration>,
141 pub undistilled: Option<Duration>,
142}
143
144impl Windows {
145 pub fn from_config(config: &AlertConfig) -> Self {
146 let hours = |h: u64| {
147 i64::try_from(h)
148 .ok()
149 .filter(|&h| h > 0)
150 .and_then(Duration::try_hours)
151 };
152 Self {
153 sync_overdue: hours(config.sync_overdue_hours),
154 undistilled: hours(config.undistilled_overdue_hours),
155 }
156 }
157}
158
159impl Default for Windows {
160 fn default() -> Self {
161 Self::from_config(&AlertConfig::default())
162 }
163}
164
165#[derive(Debug, Clone, Default)]
168pub struct ProgressSeen {
169 pub sync: Option<SyncReport>,
170 pub distill: Option<DistillReport>,
171 pub error: Option<String>,
172}
173
174#[derive(Debug, Clone, PartialEq, Serialize)]
175pub struct Verdict {
176 pub state: State,
177 pub reason: Option<String>,
178}
179
180#[derive(Debug, Clone, Serialize)]
181pub struct HealthReport {
182 pub state: State,
183 pub reason: Option<String>,
184 pub since: DateTime<Utc>,
185 pub serving: Verdict,
188 pub capture: CaptureReport,
189 pub index: IndexReport,
190 pub queue: QueueReport,
191 pub maintenance: MaintenanceReport,
192 pub sync: SyncReport,
193 pub distill: DistillReport,
194 pub last_error: Option<ErrorRecord>,
195}
196
197struct Facts {
199 started: DateTime<Utc>,
200 capture: CaptureReport,
201 watcher_error: Option<String>,
202 ingest_error: Option<Streak>,
203 index: IndexReport,
204 drain_error: Option<Streak>,
205 indexes_error: Option<String>,
206 repair_error: Option<String>,
207 queue: QueueReport,
208 maintenance: MaintenanceReport,
209 sync: SyncReport,
210 distill: DistillReport,
211 progress_error: Option<Streak>,
212 windows: Windows,
213 last_error: Option<ErrorRecord>,
214}
215
216struct Streak {
218 message: String,
219 since: DateTime<Utc>,
220}
221
222impl Streak {
223 fn extend(previous: Option<Streak>, message: String, now: DateTime<Utc>) -> Option<Streak> {
224 let since = previous.map_or(now, |p| p.since);
225 Some(Streak { message, since })
226 }
227
228 fn outlasted_grace(&self, now: DateTime<Utc>) -> bool {
229 now - self.since >= ERROR_GRACE
230 }
231}
232
233struct Inner {
234 facts: Facts,
235 report: HealthReport,
236 sent: Option<State>,
238}
239
240pub struct Health {
241 inner: Mutex<Inner>,
242 transitions: Option<UnboundedSender<Transition>>,
243}
244
245impl Health {
246 pub fn new(now: DateTime<Utc>, transitions: Option<UnboundedSender<Transition>>) -> Self {
249 let facts = Facts {
250 started: now,
251 capture: CaptureReport::default(),
252 watcher_error: None,
253 ingest_error: None,
254 index: IndexReport::default(),
255 drain_error: None,
256 indexes_error: None,
257 repair_error: None,
258 queue: QueueReport::default(),
259 maintenance: MaintenanceReport::default(),
260 sync: SyncReport::default(),
261 distill: DistillReport::default(),
262 progress_error: None,
263 windows: Windows::default(),
264 last_error: None,
265 };
266 let report = HealthReport {
267 state: State::Ok,
268 reason: None,
269 since: now,
270 serving: Verdict {
271 state: State::Ok,
272 reason: None,
273 },
274 capture: CaptureReport::default(),
275 index: IndexReport::default(),
276 queue: QueueReport::default(),
277 maintenance: MaintenanceReport::default(),
278 sync: SyncReport::default(),
279 distill: DistillReport::default(),
280 last_error: None,
281 };
282 Self {
283 inner: Mutex::new(Inner {
284 facts,
285 report,
286 sent: None,
287 }),
288 transitions,
289 }
290 }
291
292 pub fn with_windows(self, windows: Windows) -> Self {
293 self.lock().facts.windows = windows;
294 self
295 }
296
297 pub fn progress_checked(&self, seen: ProgressSeen, now: DateTime<Utc>) {
298 self.update(now, |f| {
299 if let Some(sync) = seen.sync {
300 f.sync = SyncReport {
301 checked_at: Some(now),
302 ..sync
303 };
304 }
305 if let Some(distill) = seen.distill {
306 f.distill = DistillReport {
307 checked_at: Some(now),
308 ..distill
309 };
310 }
311 match seen.error {
312 None => f.progress_error = None,
313 Some(error) => {
314 f.record_error("progress", &error, now);
315 f.progress_error = Streak::extend(f.progress_error.take(), error, now);
316 }
317 }
318 });
319 }
320
321 pub fn announce(&self, now: DateTime<Utc>) {
324 let mut inner = self.lock();
325 self.reassess(&mut inner, now, true);
326 }
327
328 pub fn report(&self) -> HealthReport {
329 self.lock().report.clone()
330 }
331
332 pub fn transcripts_scanned(&self, newest: Option<DateTime<Utc>>, now: DateTime<Utc>) {
333 self.update(now, |f| {
334 f.capture.checked_at = Some(now);
335 f.capture.newest_transcript_at = newest;
336 });
337 }
338
339 pub fn watcher_started(&self, now: DateTime<Utc>) {
340 self.update(now, |f| {
341 f.watcher_error = None;
342 f.capture.checked_at = Some(now);
343 });
344 }
345
346 pub fn watcher_failed(&self, error: &str, now: DateTime<Utc>) {
347 self.update(now, |f| {
348 f.watcher_error = Some(error.to_string());
349 f.record_error("watcher", error, now);
350 });
351 }
352
353 pub fn ingested(&self, transcript_mtime: Option<DateTime<Utc>>, now: DateTime<Utc>) {
356 self.update(now, |f| {
357 f.ingest_error = None;
358 f.capture.checked_at = Some(now);
359 if let Some(mtime) = transcript_mtime {
360 let through = f.capture.captured_through.map_or(mtime, |t| t.max(mtime));
361 f.capture.captured_through = Some(through);
362 }
363 });
364 }
365
366 pub fn ingest_failed(&self, error: &str, now: DateTime<Utc>) {
367 self.update(now, |f| {
368 f.ingest_error = Streak::extend(f.ingest_error.take(), error.to_string(), now);
369 f.capture.checked_at = Some(now);
370 f.record_error("ingest", error, now);
371 });
372 }
373
374 pub fn drained(&self, outbox_pending: u64, now: DateTime<Utc>) {
375 self.update(now, |f| {
376 f.drain_error = None;
377 f.index.checked_at = Some(now);
378 f.index.outbox_pending = Some(outbox_pending);
379 f.index.backlog_since = if outbox_pending == 0 {
380 None
381 } else {
382 Some(f.index.backlog_since.unwrap_or(now))
383 };
384 });
385 }
386
387 pub fn drain_failed(&self, error: &str, now: DateTime<Utc>) {
388 self.update(now, |f| {
389 f.index.checked_at = Some(now);
390 f.drain_error = Streak::extend(f.drain_error.take(), error.to_string(), now);
391 f.record_error("drain", error, now);
392 });
393 }
394
395 pub fn indexes_unavailable(&self, error: &str, now: DateTime<Utc>) {
397 self.update(now, |f| {
398 f.indexes_error = Some(error.to_string());
399 f.record_error("index", error, now);
400 });
401 }
402
403 pub fn repaired(&self, now: DateTime<Utc>) {
404 self.update(now, |f| f.repair_error = None);
405 }
406
407 pub fn repair_failed(&self, error: &str, now: DateTime<Utc>) {
408 self.update(now, |f| {
409 f.repair_error = Some(error.to_string());
410 f.record_error("repair", error, now);
411 });
412 }
413
414 pub fn queue_checked(&self, depth: u64, oldest_at: Option<DateTime<Utc>>, now: DateTime<Utc>) {
415 self.update(now, |f| {
416 f.queue.depth = depth;
417 f.queue.oldest_at = oldest_at;
418 });
419 }
420
421 pub fn write_rejected(&self, tool: &str, status: u16, now: DateTime<Utc>) {
422 self.update(now, |f| {
423 f.queue.last_rejected = Some(RejectedWrite {
424 tool: tool.to_string(),
425 status,
426 at: now,
427 });
428 });
429 }
430
431 pub fn maintenance(
433 &self,
434 state: MaintenanceState,
435 space: Option<VacuumSpace>,
436 now: DateTime<Utc>,
437 ) {
438 self.update(now, |f| {
439 if f.maintenance.state != state {
440 f.maintenance.since = Some(now);
441 }
442 f.maintenance.state = state;
443 if space.is_some() {
444 f.maintenance.space = space;
445 }
446 if state == MaintenanceState::Done {
447 f.maintenance.last_error = None;
448 }
449 });
450 }
451
452 pub fn maintenance_failed(&self, error: &str, now: DateTime<Utc>) {
454 self.update(now, |f| {
455 f.maintenance.last_error = Some(error.to_string());
456 f.record_error("maintenance", error, now);
457 });
458 }
459
460 fn lock(&self) -> std::sync::MutexGuard<'_, Inner> {
461 match self.inner.lock() {
462 Ok(g) => g,
463 Err(poisoned) => poisoned.into_inner(),
464 }
465 }
466
467 fn update(&self, now: DateTime<Utc>, change: impl FnOnce(&mut Facts)) {
468 let mut inner = self.lock();
469 change(&mut inner.facts);
470 self.reassess(&mut inner, now, false);
471 }
472
473 fn reassess(&self, inner: &mut Inner, now: DateTime<Utc>, announcing: bool) {
475 let mut serving = Assessment::default();
476 assess_serving(&inner.facts, now, &mut serving);
477 let mut overall = serving.clone();
478 assess_upkeep(&inner.facts, now, &mut overall);
479 let (state, reason) = overall.verdict();
480 let changed = state != inner.report.state;
481 let since = if changed { now } else { inner.report.since };
482 let mut capture = inner.facts.capture.clone();
483 capture.lag_secs = capture_lag(&inner.facts).map(|d| d.num_seconds());
484 let mut queue = inner.facts.queue.clone();
485 queue.oldest_age_secs = queue.oldest_at.map(|t| (now - t).num_seconds().max(0));
486 inner.report = HealthReport {
487 state,
488 reason: reason.clone(),
489 since,
490 serving: Verdict {
491 state: serving.state,
492 reason: serving.verdict().1,
493 },
494 capture,
495 index: inner.facts.index.clone(),
496 queue,
497 maintenance: inner.facts.maintenance.clone(),
498 sync: inner.facts.sync.clone(),
499 distill: inner.facts.distill.clone(),
500 last_error: inner.facts.last_error.clone(),
501 };
502 let unsent = inner.sent != Some(state);
505 if (changed || (announcing && unsent))
506 && let Some(tx) = &self.transitions
507 {
508 let _ = tx.send(Transition {
510 state,
511 reason,
512 at: now,
513 });
514 inner.sent = Some(state);
515 }
516 }
517}
518
519impl Facts {
520 fn record_error(&mut self, source: &'static str, message: &str, at: DateTime<Utc>) {
521 self.last_error = Some(ErrorRecord {
522 source,
523 message: message.to_string(),
524 at,
525 });
526 }
527}
528
529fn capture_lag(f: &Facts) -> Option<Duration> {
532 let newest = f.capture.newest_transcript_at?;
533 let through = f
534 .capture
535 .captured_through
536 .map_or(f.started, |t| t.max(f.started));
537 Some((newest - through).max(Duration::zero()))
538}
539
540#[derive(Clone, Default)]
541struct Assessment {
542 state: State,
543 problems: Vec<String>,
544}
545
546impl Assessment {
547 fn flag(&mut self, level: State, problem: String) {
548 self.state = self.state.max(level);
549 self.problems.push(problem);
550 }
551
552 fn verdict(&self) -> (State, Option<String>) {
553 let reason = (!self.problems.is_empty()).then(|| self.problems.join("; "));
554 (self.state, reason)
555 }
556}
557
558fn assess_serving(f: &Facts, now: DateTime<Utc>, a: &mut Assessment) {
559 let mut flag = |level: State, problem: String| a.flag(level, problem);
560
561 match &f.watcher_error {
562 Some(e) => flag(
563 State::Failing,
564 format!("transcript watcher is not running: {e}"),
565 ),
566 None => {
567 let last = f.capture.checked_at.unwrap_or(f.started);
568 if now - last >= WATCHER_SILENT {
569 flag(
570 State::Degraded,
571 format!(
572 "transcript watcher has not reported for {}",
573 human(now - last)
574 ),
575 );
576 }
577 }
578 }
579 if let Some(lag) = capture_lag(f)
580 && lag >= CAPTURE_LAG_DEGRADED
581 {
582 let level = if lag >= CAPTURE_LAG_FAILING {
583 State::Failing
584 } else {
585 State::Degraded
586 };
587 flag(
588 level,
589 format!("capture is {} behind the newest transcript", human(lag)),
590 );
591 }
592 if let Some(e) = &f.ingest_error
593 && e.outlasted_grace(now)
594 {
595 flag(
596 State::Degraded,
597 format!(
598 "ingest has failed for {}: {}",
599 human(now - e.since),
600 e.message
601 ),
602 );
603 }
604
605 if let Some(e) = &f.drain_error
606 && e.outlasted_grace(now)
607 {
608 flag(
609 State::Degraded,
610 format!(
611 "index drain failed for {}: {}",
612 human(now - e.since),
613 e.message
614 ),
615 );
616 }
617 match &f.indexes_error {
618 Some(e) => flag(State::Degraded, format!("search indexes unavailable: {e}")),
619 None => {
620 let last = f.index.checked_at.unwrap_or(f.started);
621 if now - last >= DRAIN_SILENT {
622 flag(
623 State::Degraded,
624 format!("index drain has not reported for {}", human(now - last)),
625 );
626 }
627 }
628 }
629 if let (Some(since), Some(pending)) = (f.index.backlog_since, f.index.outbox_pending) {
630 let age = now - since;
631 if age >= BACKLOG_DEGRADED {
632 let level = if age >= BACKLOG_FAILING {
633 State::Failing
634 } else {
635 State::Degraded
636 };
637 flag(
638 level,
639 format!("{pending} changes have waited {} to be indexed", human(age)),
640 );
641 }
642 }
643 if let Some(e) = &f.repair_error {
644 flag(State::Degraded, format!("last store repair failed: {e}"));
645 }
646 if let Some(oldest) = f.queue.oldest_at
647 && now - oldest >= QUEUE_STUCK
648 {
649 flag(
650 State::Degraded,
651 format!(
652 "write queue not draining ({} queued, oldest {})",
653 f.queue.depth,
654 human(now - oldest)
655 ),
656 );
657 }
658
659 match (&f.maintenance.state, &f.maintenance.space) {
660 (MaintenanceState::Vacuuming, _) => flag(
661 State::Degraded,
662 "store maintenance is running; writes are queued until it finishes".to_string(),
663 ),
664 (MaintenanceState::WaitingForSpace, Some(space)) => flag(
665 State::Degraded,
666 format!(
667 "store compaction waits for disk space: it needs {} MB free at {} and has {} MB",
668 space.needed_bytes / 1_000_000,
669 space.path.display(),
670 space.free_bytes / 1_000_000
671 ),
672 ),
673 _ => {}
674 }
675 if let Some(e) = &f.maintenance.last_error {
676 flag(State::Degraded, format!("store compaction failed: {e}"));
677 }
678}
679
680fn assess_upkeep(f: &Facts, now: DateTime<Utc>, a: &mut Assessment) {
683 let mut flag = |level: State, problem: String| a.flag(level, problem);
684 let sync = &f.sync;
685 if (sync.peers.is_some_and(|n| n > 0) || sync.synced_before)
686 && let Some(window) = f.windows.sync_overdue
687 {
688 let since = sync
689 .last_success_at
690 .or(sync.last_cursor_at)
691 .unwrap_or(f.started);
692 let overdue = now - since;
693 if overdue >= window {
694 let problem = match sync.peers {
695 Some(0) => format!(
696 "no sync has succeeded in {}, and ~/.second-brain/sync-peers lists no peers \
697 although this device has synced before; add each peer to it as user@host, \
698 one per line, or set alert.sync_overdue_hours = 0 if this device no longer syncs",
699 human(overdue)
700 ),
701 Some(n) => format!(
702 "no sync has succeeded in {} with {} in sync-peers; \
703 run `sb sync <peer>` to see why",
704 human(overdue),
705 counted(n as u64, "peer", "peers")
706 ),
707 None => format!(
708 "no sync has succeeded in {} although this device has synced before; \
709 run `sb sync <user@host>` (or check the job that runs it), \
710 or set alert.sync_overdue_hours = 0 if this device no longer syncs",
711 human(overdue)
712 ),
713 };
714 flag(State::Degraded, problem);
715 }
716 }
717 if let (Some(oldest), Some(window)) = (f.distill.oldest_at, f.windows.undistilled)
718 && now - oldest >= window
719 {
720 flag(
721 State::Degraded,
722 format!(
723 "{} waited {} to be distilled; run `sb consolidate`, or set \
724 alert.undistilled_overdue_hours = 0 if this device does not distill",
725 if f.distill.pending == 1 {
726 "1 raw memory has".to_string()
727 } else {
728 format!("{} raw memories have", f.distill.pending)
729 },
730 human(now - oldest)
731 ),
732 );
733 }
734 if let Some(e) = &f.progress_error
735 && e.outlasted_grace(now)
736 {
737 flag(
738 State::Degraded,
739 format!(
740 "cannot check sync and distillation for {}: {}",
741 human(now - e.since),
742 e.message
743 ),
744 );
745 }
746}
747
748fn counted(n: u64, one: &str, many: &str) -> String {
749 if n == 1 {
750 format!("1 {one}")
751 } else {
752 format!("{n} {many}")
753 }
754}
755
756fn human(d: Duration) -> String {
757 let minutes = d.num_minutes();
758 if minutes < 60 {
759 format!("{minutes}m")
760 } else if minutes < 48 * 60 {
761 format!("{}h {}m", minutes / 60, minutes % 60)
762 } else {
763 format!("{}d", minutes / (24 * 60))
764 }
765}
766
767#[cfg(test)]
768mod tests {
769 use tokio::sync::mpsc::{UnboundedReceiver, unbounded_channel};
770
771 use super::*;
772
773 fn t0() -> DateTime<Utc> {
774 DateTime::parse_from_rfc3339("2026-10-31T12:00:00Z")
775 .unwrap()
776 .with_timezone(&Utc)
777 }
778
779 fn mins(m: i64) -> DateTime<Utc> {
780 t0() + Duration::minutes(m)
781 }
782
783 fn monitored() -> (Health, UnboundedReceiver<Transition>) {
784 let (tx, rx) = unbounded_channel();
785 (Health::new(t0(), Some(tx)), rx)
786 }
787
788 fn drain(rx: &mut UnboundedReceiver<Transition>) -> Vec<State> {
789 std::iter::from_fn(|| rx.try_recv().ok())
790 .map(|t| t.state)
791 .collect()
792 }
793
794 #[test]
795 fn a_fresh_daemon_that_keeps_up_is_ok() {
796 let (health, mut rx) = monitored();
797 health.watcher_started(mins(0));
798 health.transcripts_scanned(Some(mins(1)), mins(1));
799 health.ingested(Some(mins(1)), mins(1));
800 health.drained(0, mins(1));
801 let report = health.report();
802 assert_eq!(report.state, State::Ok);
803 assert_eq!(report.reason, None);
804 assert_eq!(report.capture.lag_secs, Some(0));
805 assert!(drain(&mut rx).is_empty());
806 }
807
808 #[test]
809 fn capture_lag_degrades_then_fails_then_recovers() {
810 let (health, mut rx) = monitored();
811 health.drained(0, mins(0));
812 health.ingested(Some(mins(1)), mins(1));
813 health.transcripts_scanned(Some(mins(10)), mins(10));
814 assert_eq!(health.report().state, State::Ok);
815
816 health.transcripts_scanned(Some(mins(20)), mins(20));
817 let report = health.report();
818 assert_eq!(report.state, State::Degraded);
819 assert_eq!(report.capture.lag_secs, Some(19 * 60));
820 assert!(report.reason.unwrap().contains("capture is 19m behind"));
821
822 health.drained(0, mins(125));
823 health.transcripts_scanned(Some(mins(125)), mins(125));
824 assert_eq!(health.report().state, State::Failing);
825
826 health.ingested(Some(mins(125)), mins(126));
827 assert_eq!(health.report().state, State::Ok);
828 assert_eq!(
829 drain(&mut rx),
830 vec![State::Degraded, State::Failing, State::Ok]
831 );
832 }
833
834 #[test]
835 fn transcripts_from_before_startup_are_not_lag() {
836 let (health, _rx) = monitored();
837 health.transcripts_scanned(Some(t0() - Duration::days(3)), mins(1));
838 assert_eq!(health.report().capture.lag_secs, Some(0));
839 assert_eq!(health.report().state, State::Ok);
840 }
841
842 #[test]
843 fn repeated_reports_in_one_state_send_one_transition() {
844 let (health, mut rx) = monitored();
845 for m in 0..=12 {
846 health.ingest_failed("database is locked", mins(m));
847 }
848 health.ingested(None, mins(13));
849 health.ingested(None, mins(14));
850 assert_eq!(drain(&mut rx), vec![State::Degraded, State::Ok]);
851 let report = health.report();
852 assert_eq!(report.since, mins(13));
853 let last = report.last_error.expect("the error is kept after recovery");
854 assert_eq!(last.source, "ingest");
855 assert_eq!(last.at, mins(12));
856 }
857
858 #[test]
859 fn an_error_that_clears_within_the_grace_sends_nothing() {
860 let (health, mut rx) = monitored();
861 health.watcher_started(mins(0));
862 health.ingest_failed("database is locked", mins(0));
863 health.drain_failed("database is locked", mins(1));
864 health.drained(0, mins(9));
865 health.ingested(None, mins(9));
866 assert_eq!(health.report().state, State::Ok);
867 assert!(drain(&mut rx).is_empty());
868 assert_eq!(health.report().last_error.unwrap().source, "drain");
869 }
870
871 #[test]
872 fn a_drain_error_degrades_once_it_outlasts_the_grace() {
873 let (health, _rx) = monitored();
874 health.watcher_started(mins(0));
875 for m in 0..10 {
876 health.drain_failed("disk I/O error", mins(m));
877 }
878 assert_eq!(health.report().state, State::Ok);
879 health.drain_failed("disk I/O error", mins(10));
880 let report = health.report();
881 assert_eq!(report.state, State::Degraded);
882 assert!(report.reason.unwrap().contains("index drain failed"));
883 health.drained(0, mins(11));
884 assert_eq!(health.report().state, State::Ok);
885 }
886
887 #[test]
888 fn a_long_ingest_batch_counts_as_the_watcher_reporting() {
889 let (health, _rx) = monitored();
890 health.watcher_started(mins(0));
891 health.ingested(None, mins(10));
892 health.ingest_failed("unreadable line", mins(19));
893 health.drained(0, mins(20));
894 let report = health.report();
895 assert_eq!(report.state, State::Ok, "{:?}", report.reason);
896 }
897
898 #[test]
899 fn an_outbox_backlog_only_counts_once_it_persists() {
900 let (health, _rx) = monitored();
901 health.drained(500, mins(0));
902 health.drained(800, mins(14));
903 health.watcher_started(mins(14));
904 assert_eq!(health.report().state, State::Ok);
905 health.drained(800, mins(15));
906 let report = health.report();
907 assert_eq!(report.state, State::Degraded);
908 assert_eq!(report.index.backlog_since, Some(mins(0)));
909 health.drained(0, mins(16));
910 assert_eq!(health.report().state, State::Ok);
911 assert_eq!(health.report().index.backlog_since, None);
912 }
913
914 #[test]
915 fn a_dead_watcher_fails_and_a_silent_one_degrades() {
916 let (health, _rx) = monitored();
917 health.watcher_failed("inotify watch limit reached", mins(0));
918 assert_eq!(health.report().state, State::Failing);
919 health.watcher_started(mins(1));
920 health.drained(0, mins(1));
921 assert_eq!(health.report().state, State::Ok);
922 health.drained(0, mins(16));
923 let report = health.report();
924 assert_eq!(report.state, State::Degraded);
925 assert!(report.reason.unwrap().contains("watcher has not reported"));
926 }
927
928 #[test]
929 fn index_and_repair_errors_degrade_until_the_next_success() {
930 let (health, _rx) = monitored();
931 health.watcher_started(mins(0));
932 health.drain_failed("disk I/O error", mins(0));
933 health.repair_failed("disk I/O error", mins(0));
934 let reason = health.report().reason.unwrap();
935 assert!(reason.contains("store repair failed"), "{reason}");
936 health.drain_failed("disk I/O error", mins(10));
937 let reason = health.report().reason.unwrap();
938 assert!(reason.contains("index drain failed"), "{reason}");
939 health.drained(0, mins(11));
940 assert_eq!(health.report().state, State::Degraded);
941 health.repaired(mins(12));
942 assert_eq!(health.report().state, State::Ok);
943 }
944
945 #[test]
946 fn missing_indexes_stay_degraded_without_a_drain() {
947 let (health, _rx) = monitored();
948 health.indexes_unavailable("tantivy meta.json missing", mins(0));
949 health.watcher_started(mins(90));
950 let report = health.report();
951 assert_eq!(report.state, State::Degraded);
952 let reason = report.reason.unwrap();
953 assert!(reason.contains("search indexes unavailable"), "{reason}");
954 assert!(!reason.contains("has not reported"), "{reason}");
955 }
956
957 #[test]
958 fn the_report_serializes_without_content() {
959 let (health, _rx) = monitored();
960 health.ingest_failed("database is locked", mins(0));
961 health.ingest_failed("database is locked", mins(10));
962 let json = serde_json::to_value(health.report()).unwrap();
963 assert_eq!(json["state"], "degraded");
964 assert_eq!(json["last_error"]["source"], "ingest");
965 for key in ["reason", "since", "capture", "index"] {
966 assert!(json.get(key).is_some(), "{key} missing: {json}");
967 }
968 }
969
970 #[test]
971 fn a_queue_entry_older_than_ten_minutes_degrades_until_it_drains() {
972 let (health, mut rx) = monitored();
973 health.watcher_started(mins(0));
974 health.drained(0, mins(0));
975 health.queue_checked(2, Some(mins(0)), mins(9));
976 assert_eq!(health.report().state, State::Ok);
977 assert_eq!(health.report().queue.depth, 2);
978 health.queue_checked(2, Some(mins(0)), mins(10));
979 let report = health.report();
980 assert_eq!(report.state, State::Degraded);
981 assert!(
982 report
983 .reason
984 .as_deref()
985 .unwrap()
986 .contains("write queue not draining"),
987 "{report:?}"
988 );
989 assert_eq!(report.queue.oldest_age_secs, Some(600));
990 health.queue_checked(0, None, mins(11));
991 assert_eq!(health.report().state, State::Ok);
992 assert_eq!(drain(&mut rx), vec![State::Degraded, State::Ok]);
993 }
994
995 #[test]
996 fn a_rejected_write_is_reported_by_tool_status_and_time_only() {
997 let (health, _rx) = monitored();
998 health.write_rejected("associate", 404, mins(3));
999 let json = serde_json::to_value(health.report()).unwrap();
1000 assert_eq!(
1001 json["queue"]["last_rejected"],
1002 serde_json::json!({"tool": "associate", "status": 404, "at": "2026-10-31T12:03:00Z"})
1003 );
1004 assert_eq!(json["state"], "ok");
1005 }
1006
1007 fn hours(h: i64) -> DateTime<Utc> {
1008 t0() + Duration::hours(h)
1009 }
1010
1011 fn alive(health: &Health, at: DateTime<Utc>) {
1012 health.update(at, |f| {
1013 f.capture.checked_at = Some(at);
1014 f.index.checked_at = Some(at);
1015 });
1016 }
1017
1018 fn sync_seen(health: &Health, sync: SyncReport, at: DateTime<Utc>) {
1019 let seen = ProgressSeen {
1020 sync: Some(sync),
1021 ..ProgressSeen::default()
1022 };
1023 health.progress_checked(seen, at);
1024 }
1025
1026 fn distill_seen(
1027 health: &Health,
1028 pending: u64,
1029 oldest_at: Option<DateTime<Utc>>,
1030 at: DateTime<Utc>,
1031 ) {
1032 let distill = DistillReport {
1033 checked_at: None,
1034 pending,
1035 oldest_at,
1036 };
1037 let seen = ProgressSeen {
1038 distill: Some(distill),
1039 ..ProgressSeen::default()
1040 };
1041 health.progress_checked(seen, at);
1042 }
1043
1044 fn synced(peers: usize, synced_before: bool, last: Option<DateTime<Utc>>) -> SyncReport {
1045 SyncReport {
1046 checked_at: None,
1047 peers: Some(peers),
1048 synced_before,
1049 last_success_at: last,
1050 last_cursor_at: None,
1051 }
1052 }
1053
1054 #[test]
1055 fn without_a_stamp_the_window_counts_from_when_records_last_moved() {
1056 let (health, _rx) = monitored();
1057 let seen = SyncReport {
1058 last_cursor_at: Some(hours(-48)),
1059 ..synced(0, true, None)
1060 };
1061 sync_seen(&health, seen.clone(), mins(0));
1062 let report = health.report();
1063 assert_eq!(report.state, State::Degraded);
1064 let reason = report.reason.unwrap();
1065 assert!(reason.contains("no sync has succeeded in 2d"), "{reason}");
1066
1067 let stamped = SyncReport {
1068 last_success_at: Some(mins(0)),
1069 ..seen
1070 };
1071 sync_seen(&health, stamped, mins(0));
1072 assert_eq!(health.report().state, State::Ok);
1073 }
1074
1075 #[test]
1076 fn reasons_count_one_peer_and_one_raw_in_the_singular() {
1077 let (health, _rx) = monitored();
1078 sync_seen(&health, synced(1, true, Some(hours(-30))), mins(0));
1079 distill_seen(&health, 1, Some(hours(-30)), mins(0));
1080 let reason = health.report().reason.unwrap();
1081 assert!(reason.contains("with 1 peer in sync-peers"), "{reason}");
1082 assert!(reason.contains("1 raw memory has waited"), "{reason}");
1083 }
1084
1085 #[test]
1086 fn a_device_that_synced_before_and_lost_its_peers_degrades_with_the_fix() {
1087 let (health, mut rx) = monitored();
1088 sync_seen(&health, synced(0, true, Some(hours(-25))), mins(0));
1089 let report = health.report();
1090 assert_eq!(report.state, State::Degraded);
1091 let reason = report.reason.unwrap();
1092 assert!(
1093 reason.contains("no sync has succeeded in 25h 0m"),
1094 "{reason}"
1095 );
1096 assert!(reason.contains("lists no peers"), "{reason}");
1097 assert!(reason.contains("~/.second-brain/sync-peers"), "{reason}");
1098 assert!(reason.contains("sync_overdue_hours = 0"), "{reason}");
1099 assert_eq!(report.sync.peers, Some(0));
1100 assert_eq!(report.sync.checked_at, Some(mins(0)));
1101 assert_eq!(drain(&mut rx), vec![State::Degraded]);
1102
1103 sync_seen(&health, synced(1, true, Some(mins(1))), mins(1));
1104 assert_eq!(health.report().state, State::Ok);
1105 }
1106
1107 #[test]
1108 fn a_device_synced_by_hand_without_sync_peers_stays_ok_while_it_syncs() {
1109 let (health, mut rx) = monitored();
1110 let by_hand = SyncReport {
1111 peers: None,
1112 ..synced(0, true, Some(hours(10)))
1113 };
1114 alive(&health, hours(30));
1115 sync_seen(&health, by_hand.clone(), hours(30));
1116 assert_eq!(health.report().state, State::Ok);
1117 alive(&health, hours(34));
1118 sync_seen(&health, by_hand, hours(34));
1119 let report = health.report();
1120 assert_eq!(report.state, State::Degraded);
1121 let reason = report.reason.unwrap();
1122 assert!(
1123 reason.contains("no sync has succeeded in 24h 0m"),
1124 "{reason}"
1125 );
1126 assert!(reason.contains("sb sync <user@host>"), "{reason}");
1127 assert!(reason.contains("sync_overdue_hours = 0"), "{reason}");
1128 assert!(!reason.contains("sync-peers"), "{reason}");
1129 assert_eq!(drain(&mut rx), vec![State::Degraded]);
1130 }
1131
1132 #[test]
1133 fn a_solo_device_that_never_synced_stays_ok() {
1134 let (health, mut rx) = monitored();
1135 for h in [0, 25, 100] {
1136 alive(&health, hours(h));
1137 sync_seen(&health, synced(0, false, None), hours(h));
1138 }
1139 let report = health.report();
1140 assert_eq!(report.state, State::Ok, "{:?}", report.reason);
1141 assert!(drain(&mut rx).is_empty());
1142 }
1143
1144 #[test]
1145 fn a_sync_overdue_past_the_window_degrades_only_on_a_device_that_syncs() {
1146 let (health, _rx) = monitored();
1147 alive(&health, hours(0));
1148 sync_seen(&health, synced(2, true, Some(hours(-23))), hours(0));
1149 assert_eq!(health.report().state, State::Ok);
1150
1151 alive(&health, hours(1));
1152 sync_seen(&health, synced(2, true, Some(hours(-23))), hours(1));
1153 let report = health.report();
1154 assert_eq!(report.state, State::Degraded);
1155 let reason = report.reason.unwrap();
1156 assert!(
1157 reason.contains("no sync has succeeded in 24h 0m"),
1158 "{reason}"
1159 );
1160 assert!(reason.contains("2 peers"), "{reason}");
1161
1162 sync_seen(&health, synced(2, true, Some(hours(1))), hours(1));
1163 assert_eq!(health.report().state, State::Ok);
1164
1165 sync_seen(&health, synced(0, false, Some(hours(-500))), hours(1));
1166 assert_eq!(health.report().state, State::Ok);
1167 }
1168
1169 #[test]
1170 fn with_peers_and_no_recorded_success_the_window_starts_at_startup() {
1171 let (health, _rx) = monitored();
1172 alive(&health, hours(23));
1173 sync_seen(&health, synced(1, false, None), hours(23));
1174 assert_eq!(health.report().state, State::Ok);
1175 alive(&health, hours(24));
1176 sync_seen(&health, synced(1, false, None), hours(24));
1177 let reason = health.report().reason.unwrap();
1178 assert!(
1179 reason.contains("no sync has succeeded in 24h 0m"),
1180 "{reason}"
1181 );
1182 }
1183
1184 #[test]
1185 fn raws_left_undistilled_past_the_window_degrade_with_the_count() {
1186 let (health, mut rx) = monitored();
1187 alive(&health, hours(0));
1188 distill_seen(&health, 520, Some(hours(-23)), hours(0));
1189 assert_eq!(health.report().state, State::Ok);
1190 assert_eq!(health.report().distill.pending, 520);
1191
1192 alive(&health, hours(2));
1193 distill_seen(&health, 520, Some(hours(-23)), hours(2));
1194 let report = health.report();
1195 assert_eq!(report.state, State::Degraded);
1196 let reason = report.reason.unwrap();
1197 assert!(
1198 reason.contains("520 raw memories have waited 25h 0m to be distilled"),
1199 "{reason}"
1200 );
1201 assert!(reason.contains("undistilled_overdue_hours = 0"), "{reason}");
1202
1203 alive(&health, hours(3));
1204 distill_seen(&health, 0, None, hours(3));
1205 assert_eq!(health.report().state, State::Ok);
1206 assert_eq!(drain(&mut rx), vec![State::Degraded, State::Ok]);
1207 }
1208
1209 #[test]
1210 fn a_window_turned_off_never_degrades() {
1211 let (tx, _rx) = unbounded_channel();
1212 let windows = Windows::from_config(&AlertConfig {
1213 sync_overdue_hours: 0,
1214 undistilled_overdue_hours: 0,
1215 ..AlertConfig::default()
1216 });
1217 assert_eq!(
1218 windows,
1219 Windows {
1220 sync_overdue: None,
1221 undistilled: None
1222 }
1223 );
1224 let health = Health::new(t0(), Some(tx)).with_windows(windows);
1225 alive(&health, hours(100));
1226 sync_seen(&health, synced(1, true, Some(hours(-100))), hours(100));
1227 distill_seen(&health, 9, Some(hours(-100)), hours(100));
1228 assert_eq!(health.report().state, State::Ok);
1229 sync_seen(&health, synced(0, true, Some(hours(-100))), hours(100));
1230 assert_eq!(health.report().state, State::Ok);
1231 }
1232
1233 #[test]
1234 fn the_windows_come_from_the_alert_config() {
1235 let windows = Windows::from_config(&AlertConfig {
1236 sync_overdue_hours: 6,
1237 undistilled_overdue_hours: 48,
1238 ..AlertConfig::default()
1239 });
1240 assert_eq!(windows.sync_overdue, Some(Duration::hours(6)));
1241 assert_eq!(windows.undistilled, Some(Duration::hours(48)));
1242 assert_eq!(Windows::default().sync_overdue, Some(Duration::hours(24)));
1243 assert_eq!(Windows::default().undistilled, Some(Duration::hours(24)));
1244 }
1245
1246 #[test]
1247 fn announce_sends_the_current_state_once_unless_it_was_already_sent() {
1248 let (health, mut rx) = monitored();
1249 health.announce(mins(0));
1250 let sent: Vec<Transition> = std::iter::from_fn(|| rx.try_recv().ok()).collect();
1251 assert_eq!(
1252 sent,
1253 vec![Transition {
1254 state: State::Ok,
1255 reason: None,
1256 at: mins(0),
1257 }]
1258 );
1259
1260 health.announce(mins(1));
1261 assert!(drain(&mut rx).is_empty(), "ok was already sent");
1262
1263 sync_seen(&health, synced(0, true, Some(hours(-30))), mins(2));
1264 health.announce(mins(3));
1265 assert_eq!(drain(&mut rx), vec![State::Degraded]);
1266 }
1267
1268 #[test]
1269 fn announce_reassesses_and_sends_one_message_for_a_pending_change() {
1270 let (health, mut rx) = monitored();
1271 alive(&health, hours(0));
1272 distill_seen(&health, 3, Some(hours(-23)), hours(0));
1273 assert!(drain(&mut rx).is_empty());
1274 health.announce(hours(2));
1275 assert_eq!(drain(&mut rx), vec![State::Degraded]);
1276 }
1277
1278 #[test]
1279 fn serving_leaves_out_sync_and_distillation() {
1280 let (health, _rx) = monitored();
1281 alive(&health, hours(0));
1282 let seen = ProgressSeen {
1283 sync: Some(synced(1, true, Some(hours(-30)))),
1284 distill: Some(DistillReport {
1285 checked_at: None,
1286 pending: 4,
1287 oldest_at: Some(hours(-30)),
1288 }),
1289 error: None,
1290 };
1291 health.progress_checked(seen, hours(0));
1292 let report = health.report();
1293 assert_eq!(report.state, State::Degraded);
1294 assert_eq!(
1295 report.serving,
1296 Verdict {
1297 state: State::Ok,
1298 reason: None
1299 }
1300 );
1301
1302 health.watcher_failed("inotify watch limit reached", hours(0));
1303 let report = health.report();
1304 assert_eq!(report.state, State::Failing);
1305 assert_eq!(report.serving.state, State::Failing);
1306 let serving = report.serving.reason.unwrap();
1307 assert!(serving.contains("watcher is not running"), "{serving}");
1308 assert!(!serving.contains("sync"), "{serving}");
1309
1310 let json = serde_json::to_value(health.report()).unwrap();
1311 assert_eq!(json["serving"]["state"], "failing");
1312 }
1313
1314 #[test]
1315 fn one_pass_that_clears_sync_and_finds_distillation_overdue_sends_nothing() {
1316 let (health, mut rx) = monitored();
1317 alive(&health, hours(0));
1318 sync_seen(&health, synced(1, true, Some(hours(-30))), hours(0));
1319 assert_eq!(drain(&mut rx), vec![State::Degraded]);
1320 let seen = ProgressSeen {
1321 sync: Some(synced(1, true, Some(hours(0)))),
1322 distill: Some(DistillReport {
1323 checked_at: None,
1324 pending: 4,
1325 oldest_at: Some(hours(-30)),
1326 }),
1327 error: None,
1328 };
1329 health.progress_checked(seen, hours(0));
1330 assert_eq!(health.report().state, State::Degraded);
1331 assert!(drain(&mut rx).is_empty());
1332 }
1333}