1use std::sync::Mutex;
6
7use chrono::{DateTime, Duration, Utc};
8use second_brain_core::config::AlertConfig;
9use second_brain_core::sqlite_store::VacuumSpace;
10use serde::Serialize;
11use tokio::sync::mpsc::UnboundedSender;
12use uuid::Uuid;
13
14use crate::value_metrics::{
15 PeerOutcome, RecallEvent, ResidentSize, StoreSize, ValueFacts, ValueProblems, ValueReport,
16 WINDOW,
17};
18
19const CAPTURE_LAG_DEGRADED: Duration = Duration::minutes(15);
20const CAPTURE_LAG_FAILING: Duration = Duration::hours(2);
21const BACKLOG_DEGRADED: Duration = Duration::minutes(15);
22const BACKLOG_FAILING: Duration = Duration::hours(2);
23const WATCHER_SILENT: Duration = Duration::minutes(15);
25const DRAIN_SILENT: Duration = Duration::hours(1);
27const ERROR_GRACE: Duration = Duration::minutes(10);
30const QUEUE_STUCK: Duration = Duration::minutes(10);
32
33#[derive(Debug, Clone, Copy, Default, PartialEq, Eq, PartialOrd, Ord, Serialize)]
34#[serde(rename_all = "lowercase")]
35pub enum State {
36 #[default]
37 Ok,
38 Degraded,
39 Failing,
40}
41
42#[derive(Debug, Clone, PartialEq, Serialize)]
43pub struct Transition {
44 pub state: State,
45 pub reason: Option<String>,
46 pub at: DateTime<Utc>,
47}
48
49#[derive(Debug, Clone, Serialize)]
50pub struct ErrorRecord {
51 pub source: &'static str,
52 pub message: String,
53 pub at: DateTime<Utc>,
54}
55
56#[derive(Debug, Clone, Default, Serialize)]
57pub struct CaptureReport {
58 pub checked_at: Option<DateTime<Utc>>,
59 pub newest_transcript_at: Option<DateTime<Utc>>,
60 pub captured_through: Option<DateTime<Utc>>,
62 pub lag_secs: Option<i64>,
63}
64
65#[derive(Debug, Clone, Default, Serialize)]
66pub struct IndexReport {
67 pub checked_at: Option<DateTime<Utc>>,
68 pub outbox_pending: Option<u64>,
69 pub backlog_since: Option<DateTime<Utc>>,
70}
71
72#[derive(Debug, Clone, PartialEq, Serialize)]
75pub struct RejectedWrite {
76 pub tool: String,
77 pub status: u16,
78 pub at: DateTime<Utc>,
79}
80
81#[derive(Debug, Clone, Default, Serialize)]
82pub struct QueueReport {
83 pub depth: u64,
84 pub oldest_at: Option<DateTime<Utc>>,
85 pub oldest_age_secs: Option<i64>,
86 pub last_rejected: Option<RejectedWrite>,
87}
88
89#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize)]
93#[serde(rename_all = "snake_case")]
94pub enum MaintenanceState {
95 Pending,
96 WaitingForSpace,
97 MergingIndex,
98 Vacuuming,
100 Done,
101}
102
103#[derive(Debug, Clone, Serialize)]
104pub struct MaintenanceReport {
105 pub state: MaintenanceState,
106 pub since: Option<DateTime<Utc>>,
107 pub space: Option<VacuumSpace>,
108 pub last_error: Option<String>,
109}
110
111impl Default for MaintenanceReport {
112 fn default() -> Self {
113 Self {
114 state: MaintenanceState::Pending,
115 since: None,
116 space: None,
117 last_error: None,
118 }
119 }
120}
121
122#[derive(Debug, Clone, Copy, Default, PartialEq, Eq, Serialize)]
123#[serde(rename_all = "snake_case")]
124pub enum BackfillState {
125 #[default]
126 NotStarted,
127 Running,
128 Done,
129}
130
131#[derive(Debug, Clone, Default, PartialEq, Eq, Serialize)]
132pub struct BackfillReport {
133 pub state: BackfillState,
134 pub total: usize,
135 pub done: usize,
136 pub failed: usize,
137 pub updated_at: Option<DateTime<Utc>>,
138}
139
140#[derive(Debug, Clone, Default, Serialize)]
141pub struct SyncReport {
142 pub checked_at: Option<DateTime<Utc>>,
143 pub peers: Option<usize>,
145 pub synced_before: bool,
147 pub last_success_at: Option<DateTime<Utc>>,
148 pub last_cursor_at: Option<DateTime<Utc>>,
151}
152
153pub const NO_GPU_NOTE: &str = "no LLM endpoint configured and no LLM distillation in 7 days: \
154 raws covered by an agent note are retired, the rest stay searchable";
155
156#[derive(Debug, Clone, Copy, Default, PartialEq, Eq, Serialize)]
160pub enum DistillMode {
161 #[default]
162 #[serde(rename = "llm")]
163 Llm,
164 #[serde(rename = "no-gpu")]
165 NoGpu,
166}
167
168impl DistillMode {
169 fn note(self) -> Option<&'static str> {
170 match self {
171 DistillMode::Llm => None,
172 DistillMode::NoGpu => Some(NO_GPU_NOTE),
173 }
174 }
175}
176
177#[derive(Debug, Clone, Default, Serialize)]
178pub struct DistillReport {
179 pub checked_at: Option<DateTime<Utc>>,
180 pub mode: DistillMode,
181 pub pending: u64,
182 pub oldest_at: Option<DateTime<Utc>>,
183 #[serde(skip_serializing_if = "Option::is_none")]
184 pub note: Option<&'static str>,
185}
186
187#[derive(Debug, Clone, Copy, PartialEq)]
190pub struct Windows {
191 pub sync_overdue: Option<Duration>,
192 pub undistilled: Option<Duration>,
193}
194
195impl Windows {
196 pub fn from_config(config: &AlertConfig) -> Self {
197 let hours = |h: u64| {
198 i64::try_from(h)
199 .ok()
200 .filter(|&h| h > 0)
201 .and_then(Duration::try_hours)
202 };
203 Self {
204 sync_overdue: hours(config.sync_overdue_hours),
205 undistilled: hours(config.undistilled_overdue_hours),
206 }
207 }
208}
209
210impl Default for Windows {
211 fn default() -> Self {
212 Self::from_config(&AlertConfig::default())
213 }
214}
215
216#[derive(Debug, Clone, Default)]
219pub struct ProgressSeen {
220 pub sync: Option<SyncReport>,
221 pub distill: Option<DistillReport>,
222 pub ingested_sessions: Option<Vec<String>>,
225 pub error: Option<String>,
226}
227
228#[derive(Debug, Clone, PartialEq, Serialize)]
229pub struct Verdict {
230 pub state: State,
231 pub reason: Option<String>,
232}
233
234#[derive(Debug, Clone, Serialize)]
235pub struct HealthReport {
236 pub state: State,
237 pub reason: Option<String>,
238 pub since: DateTime<Utc>,
239 pub serving: Verdict,
242 pub capture: CaptureReport,
243 pub index: IndexReport,
244 pub queue: QueueReport,
245 pub maintenance: MaintenanceReport,
246 pub backfill: BackfillReport,
247 pub sync: SyncReport,
248 pub distill: DistillReport,
249 #[serde(flatten)]
250 pub value: ValueReport,
251 pub last_error: Option<ErrorRecord>,
252}
253
254struct Facts {
256 started: DateTime<Utc>,
257 capture: CaptureReport,
258 watcher_error: Option<String>,
259 ingest_error: Option<Streak>,
260 index: IndexReport,
261 drain_error: Option<Streak>,
262 indexes_error: Option<String>,
263 repair_error: Option<String>,
264 queue: QueueReport,
265 maintenance: MaintenanceReport,
266 backfill: BackfillReport,
267 sync: SyncReport,
268 distill: DistillReport,
269 progress_error: Option<Streak>,
270 value: ValueFacts,
271 windows: Windows,
272 last_error: Option<ErrorRecord>,
273}
274
275struct Streak {
277 message: String,
278 since: DateTime<Utc>,
279}
280
281impl Streak {
282 fn extend(previous: Option<Streak>, message: String, now: DateTime<Utc>) -> Option<Streak> {
283 let since = previous.map_or(now, |p| p.since);
284 Some(Streak { message, since })
285 }
286
287 fn outlasted_grace(&self, now: DateTime<Utc>) -> bool {
288 now - self.since >= ERROR_GRACE
289 }
290}
291
292struct Inner {
293 facts: Facts,
294 report: HealthReport,
295 sent: Option<State>,
297}
298
299pub struct Health {
300 inner: Mutex<Inner>,
301 transitions: Option<UnboundedSender<Transition>>,
302}
303
304impl Health {
305 pub fn new(now: DateTime<Utc>, transitions: Option<UnboundedSender<Transition>>) -> Self {
308 let mut facts = Facts {
309 started: now,
310 capture: CaptureReport::default(),
311 watcher_error: None,
312 ingest_error: None,
313 index: IndexReport::default(),
314 drain_error: None,
315 indexes_error: None,
316 repair_error: None,
317 queue: QueueReport::default(),
318 maintenance: MaintenanceReport::default(),
319 backfill: BackfillReport::default(),
320 sync: SyncReport::default(),
321 distill: DistillReport::default(),
322 progress_error: None,
323 value: ValueFacts::default(),
324 windows: Windows::default(),
325 last_error: None,
326 };
327 let report = HealthReport {
328 state: State::Ok,
329 reason: None,
330 since: now,
331 serving: Verdict {
332 state: State::Ok,
333 reason: None,
334 },
335 capture: CaptureReport::default(),
336 index: IndexReport::default(),
337 queue: QueueReport::default(),
338 maintenance: MaintenanceReport::default(),
339 backfill: BackfillReport::default(),
340 sync: SyncReport::default(),
341 distill: DistillReport::default(),
342 value: facts.value.refresh(now, now, facts.windows.sync_overdue).0,
343 last_error: None,
344 };
345 Self {
346 inner: Mutex::new(Inner {
347 facts,
348 report,
349 sent: None,
350 }),
351 transitions,
352 }
353 }
354
355 pub fn with_windows(self, windows: Windows) -> Self {
356 self.lock().facts.windows = windows;
357 self
358 }
359
360 pub fn progress_checked(&self, seen: ProgressSeen, now: DateTime<Utc>) {
361 self.update(now, |f| {
362 if let Some(sync) = seen.sync {
363 f.sync = SyncReport {
364 checked_at: Some(now),
365 ..sync
366 };
367 }
368 if let Some(distill) = seen.distill {
369 f.distill = DistillReport {
370 checked_at: Some(now),
371 note: distill.mode.note(),
372 ..distill
373 };
374 }
375 f.value.sessions_ingested(seen.ingested_sessions);
376 match seen.error {
377 None => f.progress_error = None,
378 Some(error) => {
379 f.record_error("progress", &error, now);
380 f.progress_error = Streak::extend(f.progress_error.take(), error, now);
381 }
382 }
383 });
384 }
385
386 pub fn announce(&self, now: DateTime<Utc>) {
389 let mut inner = self.lock();
390 self.reassess(&mut inner, now, true);
391 }
392
393 pub fn report(&self) -> HealthReport {
394 self.lock().report.clone()
395 }
396
397 pub fn transcripts_scanned(&self, newest: Option<DateTime<Utc>>, now: DateTime<Utc>) {
398 self.update(now, |f| {
399 f.capture.checked_at = Some(now);
400 f.capture.newest_transcript_at = newest;
401 });
402 }
403
404 pub fn watcher_started(&self, now: DateTime<Utc>) {
405 self.update(now, |f| {
406 f.watcher_error = None;
407 f.capture.checked_at = Some(now);
408 });
409 }
410
411 pub fn watcher_failed(&self, error: &str, now: DateTime<Utc>) {
412 self.update(now, |f| {
413 f.watcher_error = Some(error.to_string());
414 f.record_error("watcher", error, now);
415 });
416 }
417
418 pub fn ingested(&self, transcript_mtime: Option<DateTime<Utc>>, now: DateTime<Utc>) {
421 self.update(now, |f| {
422 f.ingest_error = None;
423 f.capture.checked_at = Some(now);
424 if let Some(mtime) = transcript_mtime {
425 let through = f.capture.captured_through.map_or(mtime, |t| t.max(mtime));
426 f.capture.captured_through = Some(through);
427 }
428 });
429 }
430
431 pub fn ingest_failed(&self, error: &str, now: DateTime<Utc>) {
432 self.update(now, |f| {
433 f.ingest_error = Streak::extend(f.ingest_error.take(), error.to_string(), now);
434 f.capture.checked_at = Some(now);
435 f.record_error("ingest", error, now);
436 });
437 }
438
439 pub fn drained(&self, outbox_pending: u64, now: DateTime<Utc>) {
440 self.update(now, |f| {
441 f.drain_error = None;
442 f.index.checked_at = Some(now);
443 f.index.outbox_pending = Some(outbox_pending);
444 f.index.backlog_since = if outbox_pending == 0 {
445 None
446 } else {
447 Some(f.index.backlog_since.unwrap_or(now))
448 };
449 });
450 }
451
452 pub fn drain_failed(&self, error: &str, now: DateTime<Utc>) {
453 self.update(now, |f| {
454 f.index.checked_at = Some(now);
455 f.drain_error = Streak::extend(f.drain_error.take(), error.to_string(), now);
456 f.record_error("drain", error, now);
457 });
458 }
459
460 pub fn indexes_unavailable(&self, error: &str, now: DateTime<Utc>) {
462 self.update(now, |f| {
463 f.indexes_error = Some(error.to_string());
464 f.record_error("index", error, now);
465 });
466 }
467
468 pub fn repaired(&self, now: DateTime<Utc>) {
469 self.update(now, |f| f.repair_error = None);
470 }
471
472 pub fn repair_failed(&self, error: &str, now: DateTime<Utc>) {
473 self.update(now, |f| {
474 f.repair_error = Some(error.to_string());
475 f.record_error("repair", error, now);
476 });
477 }
478
479 pub fn queue_checked(&self, depth: u64, oldest_at: Option<DateTime<Utc>>, now: DateTime<Utc>) {
480 self.update(now, |f| {
481 f.queue.depth = depth;
482 f.queue.oldest_at = oldest_at;
483 });
484 }
485
486 pub fn write_rejected(&self, tool: &str, status: u16, now: DateTime<Utc>) {
487 self.update(now, |f| {
488 f.queue.last_rejected = Some(RejectedWrite {
489 tool: tool.to_string(),
490 status,
491 at: now,
492 });
493 });
494 }
495
496 pub fn maintenance(
498 &self,
499 state: MaintenanceState,
500 space: Option<VacuumSpace>,
501 now: DateTime<Utc>,
502 ) {
503 self.update(now, |f| {
504 if f.maintenance.state != state {
505 f.maintenance.since = Some(now);
506 }
507 f.maintenance.state = state;
508 if space.is_some() {
509 f.maintenance.space = space;
510 }
511 if state == MaintenanceState::Done {
512 f.maintenance.last_error = None;
513 }
514 });
515 }
516
517 pub fn maintenance_failed(&self, error: &str, now: DateTime<Utc>) {
519 self.update(now, |f| {
520 f.maintenance.last_error = Some(error.to_string());
521 f.record_error("maintenance", error, now);
522 });
523 }
524
525 pub fn recalled(&self, event: RecallEvent, now: DateTime<Utc>) {
526 self.update(now, |f| f.value.recalled(event, now));
527 }
528
529 pub fn memory_corrected(&self, id: Uuid, now: DateTime<Utc>) {
530 self.update(now, |f| f.value.corrected(id));
531 }
532
533 pub fn memories_used(&self, ids: &[Uuid], now: DateTime<Utc>) {
534 self.update(now, |f| f.value.used(ids));
535 }
536
537 pub fn sync_peers_listed(&self, peers: &[String], now: DateTime<Utc>) {
538 self.update(now, |f| f.value.peers_listed(peers, now));
539 }
540
541 pub fn sync_attempted(&self, peer: &str, outcome: PeerOutcome, now: DateTime<Utc>) {
542 self.update(now, |f| f.value.sync_attempted(peer, outcome, now));
543 }
544
545 pub fn store_measured(&self, size: Option<StoreSize>, now: DateTime<Utc>) {
547 self.update(now, |f| f.value.store_measured(size));
548 }
549
550 pub fn resident_measured(&self, size: Option<ResidentSize>, now: DateTime<Utc>) {
552 self.update(now, |f| f.value.resident_measured(size));
553 }
554
555 pub fn ingest_window_start(&self, now: DateTime<Utc>) -> Option<DateTime<Utc>> {
560 let inner = self.lock();
561 let f = &inner.facts;
562 let mut start = (now - WINDOW).max(f.started);
563 match f.backfill.state {
564 BackfillState::Running => return None,
565 BackfillState::Done => {
566 if let Some(end) = f.backfill.updated_at {
567 start = start.max(end);
568 }
569 }
570 BackfillState::NotStarted => {}
571 }
572 Some(start)
573 }
574
575 pub fn backfill_progress(&self, report: BackfillReport) {
578 let mut inner = self.lock();
579 inner.facts.backfill = report.clone();
580 inner.report.backfill = report;
581 }
582
583 fn lock(&self) -> std::sync::MutexGuard<'_, Inner> {
584 match self.inner.lock() {
585 Ok(g) => g,
586 Err(poisoned) => poisoned.into_inner(),
587 }
588 }
589
590 fn update(&self, now: DateTime<Utc>, change: impl FnOnce(&mut Facts)) {
591 let mut inner = self.lock();
592 change(&mut inner.facts);
593 self.reassess(&mut inner, now, false);
594 }
595
596 fn reassess(&self, inner: &mut Inner, now: DateTime<Utc>, announcing: bool) {
598 let (started, sync_overdue) = (inner.facts.started, inner.facts.windows.sync_overdue);
599 let (value, value_problems) = inner.facts.value.refresh(now, started, sync_overdue);
600 let mut serving = Assessment::default();
601 assess_serving(&inner.facts, now, &mut serving);
602 let mut overall = serving.clone();
603 assess_upkeep(&inner.facts, now, &mut overall, value_problems);
604 let (state, reason) = overall.verdict();
605 let changed = state != inner.report.state;
606 let since = if changed { now } else { inner.report.since };
607 let mut capture = inner.facts.capture.clone();
608 capture.lag_secs = capture_lag(&inner.facts).map(|d| d.num_seconds());
609 let mut queue = inner.facts.queue.clone();
610 queue.oldest_age_secs = queue.oldest_at.map(|t| (now - t).num_seconds().max(0));
611 inner.report = HealthReport {
612 state,
613 reason: reason.clone(),
614 since,
615 serving: Verdict {
616 state: serving.state,
617 reason: serving.verdict().1,
618 },
619 capture,
620 index: inner.facts.index.clone(),
621 queue,
622 maintenance: inner.facts.maintenance.clone(),
623 backfill: inner.facts.backfill.clone(),
624 sync: inner.facts.sync.clone(),
625 distill: inner.facts.distill.clone(),
626 value,
627 last_error: inner.facts.last_error.clone(),
628 };
629 let unsent = inner.sent != Some(state);
632 if (changed || (announcing && unsent))
633 && let Some(tx) = &self.transitions
634 {
635 let _ = tx.send(Transition {
637 state,
638 reason,
639 at: now,
640 });
641 inner.sent = Some(state);
642 }
643 }
644}
645
646impl Facts {
647 fn record_error(&mut self, source: &'static str, message: &str, at: DateTime<Utc>) {
648 self.last_error = Some(ErrorRecord {
649 source,
650 message: message.to_string(),
651 at,
652 });
653 }
654}
655
656fn capture_lag(f: &Facts) -> Option<Duration> {
659 let newest = f.capture.newest_transcript_at?;
660 let through = f
661 .capture
662 .captured_through
663 .map_or(f.started, |t| t.max(f.started));
664 Some((newest - through).max(Duration::zero()))
665}
666
667#[derive(Clone, Default)]
668struct Assessment {
669 state: State,
670 problems: Vec<String>,
671}
672
673impl Assessment {
674 fn flag(&mut self, level: State, problem: String) {
675 self.state = self.state.max(level);
676 self.problems.push(problem);
677 }
678
679 fn verdict(&self) -> (State, Option<String>) {
680 let reason = (!self.problems.is_empty()).then(|| self.problems.join("; "));
681 (self.state, reason)
682 }
683}
684
685fn assess_serving(f: &Facts, now: DateTime<Utc>, a: &mut Assessment) {
686 let mut flag = |level: State, problem: String| a.flag(level, problem);
687
688 match &f.watcher_error {
689 Some(e) => flag(
690 State::Failing,
691 format!("transcript watcher is not running: {e}"),
692 ),
693 None => {
694 let last = f.capture.checked_at.unwrap_or(f.started);
695 if now - last >= WATCHER_SILENT {
696 flag(
697 State::Degraded,
698 format!(
699 "transcript watcher has not reported for {}",
700 human(now - last)
701 ),
702 );
703 }
704 }
705 }
706 if let Some(lag) = capture_lag(f)
707 && lag >= CAPTURE_LAG_DEGRADED
708 {
709 let level = if lag >= CAPTURE_LAG_FAILING {
710 State::Failing
711 } else {
712 State::Degraded
713 };
714 flag(
715 level,
716 format!("capture is {} behind the newest transcript", human(lag)),
717 );
718 }
719 if let Some(e) = &f.ingest_error
720 && e.outlasted_grace(now)
721 {
722 flag(
723 State::Degraded,
724 format!(
725 "ingest has failed for {}: {}",
726 human(now - e.since),
727 e.message
728 ),
729 );
730 }
731
732 if let Some(e) = &f.drain_error
733 && e.outlasted_grace(now)
734 {
735 flag(
736 State::Degraded,
737 format!(
738 "index drain failed for {}: {}",
739 human(now - e.since),
740 e.message
741 ),
742 );
743 }
744 match &f.indexes_error {
745 Some(e) => flag(State::Degraded, format!("search indexes unavailable: {e}")),
746 None => {
747 let last = f.index.checked_at.unwrap_or(f.started);
748 if now - last >= DRAIN_SILENT {
749 flag(
750 State::Degraded,
751 format!("index drain has not reported for {}", human(now - last)),
752 );
753 }
754 }
755 }
756 if let (Some(since), Some(pending)) = (f.index.backlog_since, f.index.outbox_pending) {
757 let age = now - since;
758 if age >= BACKLOG_DEGRADED {
759 let level = if age >= BACKLOG_FAILING {
760 State::Failing
761 } else {
762 State::Degraded
763 };
764 flag(
765 level,
766 format!("{pending} changes have waited {} to be indexed", human(age)),
767 );
768 }
769 }
770 if let Some(e) = &f.repair_error {
771 flag(State::Degraded, format!("last store repair failed: {e}"));
772 }
773 if let Some(oldest) = f.queue.oldest_at
774 && now - oldest >= QUEUE_STUCK
775 {
776 flag(
777 State::Degraded,
778 format!(
779 "write queue not draining ({} queued, oldest {})",
780 f.queue.depth,
781 human(now - oldest)
782 ),
783 );
784 }
785
786 match (&f.maintenance.state, &f.maintenance.space) {
787 (MaintenanceState::Vacuuming, _) => flag(
788 State::Degraded,
789 "store maintenance is running; writes are queued until it finishes".to_string(),
790 ),
791 (MaintenanceState::WaitingForSpace, Some(space)) => flag(
792 State::Degraded,
793 format!(
794 "store compaction waits for disk space: it needs {} MB free at {} and has {} MB",
795 space.needed_bytes / 1_000_000,
796 space.path.display(),
797 space.free_bytes / 1_000_000
798 ),
799 ),
800 _ => {}
801 }
802 if let Some(e) = &f.maintenance.last_error {
803 flag(State::Degraded, format!("store compaction failed: {e}"));
804 }
805}
806
807fn assess_upkeep(f: &Facts, now: DateTime<Utc>, a: &mut Assessment, value: ValueProblems) {
810 let mut flag = |level: State, problem: String| a.flag(level, problem);
811 let mut no_sync_at_all = false;
812 let sync = &f.sync;
813 if (sync.peers.is_some_and(|n| n > 0) || sync.synced_before)
814 && let Some(window) = f.windows.sync_overdue
815 {
816 let since = sync
817 .last_success_at
818 .or(sync.last_cursor_at)
819 .unwrap_or(f.started);
820 let overdue = now - since;
821 if overdue >= window {
822 let problem = match sync.peers {
823 Some(0) => format!(
824 "no sync has succeeded in {}, and ~/.second-brain/sync-peers lists no peers \
825 although this device has synced before; add each peer to it as user@host, \
826 one per line, or set alert.sync_overdue_hours = 0 if this device no longer syncs",
827 human(overdue)
828 ),
829 Some(n) => format!(
830 "no sync has succeeded in {} with {} in sync-peers; \
831 run `sb sync <peer>` to see why",
832 human(overdue),
833 counted(n as u64, "peer", "peers")
834 ),
835 None => format!(
836 "no sync has succeeded in {} although this device has synced before; \
837 run `sb sync <user@host>` (or check the job that runs it), \
838 or set alert.sync_overdue_hours = 0 if this device no longer syncs",
839 human(overdue)
840 ),
841 };
842 flag(State::Degraded, problem);
843 no_sync_at_all = true;
844 }
845 }
846 if !no_sync_at_all {
847 for problem in value.peers {
848 flag(State::Degraded, problem);
849 }
850 }
851 for problem in value.other {
852 flag(State::Degraded, problem);
853 }
854 if f.distill.mode == DistillMode::Llm
855 && let (Some(oldest), Some(window)) = (f.distill.oldest_at, f.windows.undistilled)
856 && now - oldest >= window
857 {
858 flag(
859 State::Degraded,
860 format!(
861 "{} waited {} to be distilled; run `sb consolidate --llm-url <endpoint>` or add an [llm] section, or set \
862 alert.undistilled_overdue_hours = 0 if this device does not distill",
863 if f.distill.pending == 1 {
864 "1 raw memory has".to_string()
865 } else {
866 format!("{} raw memories have", f.distill.pending)
867 },
868 human(now - oldest)
869 ),
870 );
871 }
872 if let Some(e) = &f.progress_error
873 && e.outlasted_grace(now)
874 {
875 flag(
876 State::Degraded,
877 format!(
878 "cannot check sync and distillation for {}: {}",
879 human(now - e.since),
880 e.message
881 ),
882 );
883 }
884}
885
886pub(crate) fn counted(n: u64, one: &str, many: &str) -> String {
887 if n == 1 {
888 format!("1 {one}")
889 } else {
890 format!("{n} {many}")
891 }
892}
893
894pub(crate) fn human(d: Duration) -> String {
895 let minutes = d.num_minutes();
896 if minutes < 60 {
897 format!("{minutes}m")
898 } else if minutes < 48 * 60 {
899 format!("{}h {}m", minutes / 60, minutes % 60)
900 } else {
901 format!("{}d", minutes / (24 * 60))
902 }
903}
904
905#[cfg(test)]
906mod tests {
907 use tokio::sync::mpsc::{UnboundedReceiver, unbounded_channel};
908
909 use super::*;
910
911 fn t0() -> DateTime<Utc> {
912 DateTime::parse_from_rfc3339("2026-10-31T12:00:00Z")
913 .unwrap()
914 .with_timezone(&Utc)
915 }
916
917 fn mins(m: i64) -> DateTime<Utc> {
918 t0() + Duration::minutes(m)
919 }
920
921 fn monitored() -> (Health, UnboundedReceiver<Transition>) {
922 let (tx, rx) = unbounded_channel();
923 (Health::new(t0(), Some(tx)), rx)
924 }
925
926 fn drain(rx: &mut UnboundedReceiver<Transition>) -> Vec<State> {
927 std::iter::from_fn(|| rx.try_recv().ok())
928 .map(|t| t.state)
929 .collect()
930 }
931
932 #[test]
933 fn a_fresh_daemon_that_keeps_up_is_ok() {
934 let (health, mut rx) = monitored();
935 health.watcher_started(mins(0));
936 health.transcripts_scanned(Some(mins(1)), mins(1));
937 health.ingested(Some(mins(1)), mins(1));
938 health.drained(0, mins(1));
939 let report = health.report();
940 assert_eq!(report.state, State::Ok);
941 assert_eq!(report.reason, None);
942 assert_eq!(report.capture.lag_secs, Some(0));
943 assert!(drain(&mut rx).is_empty());
944 }
945
946 #[test]
947 fn capture_lag_degrades_then_fails_then_recovers() {
948 let (health, mut rx) = monitored();
949 health.drained(0, mins(0));
950 health.ingested(Some(mins(1)), mins(1));
951 health.transcripts_scanned(Some(mins(10)), mins(10));
952 assert_eq!(health.report().state, State::Ok);
953
954 health.transcripts_scanned(Some(mins(20)), mins(20));
955 let report = health.report();
956 assert_eq!(report.state, State::Degraded);
957 assert_eq!(report.capture.lag_secs, Some(19 * 60));
958 assert!(report.reason.unwrap().contains("capture is 19m behind"));
959
960 health.drained(0, mins(125));
961 health.transcripts_scanned(Some(mins(125)), mins(125));
962 assert_eq!(health.report().state, State::Failing);
963
964 health.ingested(Some(mins(125)), mins(126));
965 assert_eq!(health.report().state, State::Ok);
966 assert_eq!(
967 drain(&mut rx),
968 vec![State::Degraded, State::Failing, State::Ok]
969 );
970 }
971
972 #[test]
973 fn transcripts_from_before_startup_are_not_lag() {
974 let (health, _rx) = monitored();
975 health.transcripts_scanned(Some(t0() - Duration::days(3)), mins(1));
976 assert_eq!(health.report().capture.lag_secs, Some(0));
977 assert_eq!(health.report().state, State::Ok);
978 }
979
980 #[test]
981 fn repeated_reports_in_one_state_send_one_transition() {
982 let (health, mut rx) = monitored();
983 for m in 0..=12 {
984 health.ingest_failed("database is locked", mins(m));
985 }
986 health.ingested(None, mins(13));
987 health.ingested(None, mins(14));
988 assert_eq!(drain(&mut rx), vec![State::Degraded, State::Ok]);
989 let report = health.report();
990 assert_eq!(report.since, mins(13));
991 let last = report.last_error.expect("the error is kept after recovery");
992 assert_eq!(last.source, "ingest");
993 assert_eq!(last.at, mins(12));
994 }
995
996 #[test]
997 fn an_error_that_clears_within_the_grace_sends_nothing() {
998 let (health, mut rx) = monitored();
999 health.watcher_started(mins(0));
1000 health.ingest_failed("database is locked", mins(0));
1001 health.drain_failed("database is locked", mins(1));
1002 health.drained(0, mins(9));
1003 health.ingested(None, mins(9));
1004 assert_eq!(health.report().state, State::Ok);
1005 assert!(drain(&mut rx).is_empty());
1006 assert_eq!(health.report().last_error.unwrap().source, "drain");
1007 }
1008
1009 #[test]
1010 fn a_drain_error_degrades_once_it_outlasts_the_grace() {
1011 let (health, _rx) = monitored();
1012 health.watcher_started(mins(0));
1013 for m in 0..10 {
1014 health.drain_failed("disk I/O error", mins(m));
1015 }
1016 assert_eq!(health.report().state, State::Ok);
1017 health.drain_failed("disk I/O error", mins(10));
1018 let report = health.report();
1019 assert_eq!(report.state, State::Degraded);
1020 assert!(report.reason.unwrap().contains("index drain failed"));
1021 health.drained(0, mins(11));
1022 assert_eq!(health.report().state, State::Ok);
1023 }
1024
1025 #[test]
1026 fn a_long_ingest_batch_counts_as_the_watcher_reporting() {
1027 let (health, _rx) = monitored();
1028 health.watcher_started(mins(0));
1029 health.ingested(None, mins(10));
1030 health.ingest_failed("unreadable line", mins(19));
1031 health.drained(0, mins(20));
1032 let report = health.report();
1033 assert_eq!(report.state, State::Ok, "{:?}", report.reason);
1034 }
1035
1036 #[test]
1037 fn an_outbox_backlog_only_counts_once_it_persists() {
1038 let (health, _rx) = monitored();
1039 health.drained(500, mins(0));
1040 health.drained(800, mins(14));
1041 health.watcher_started(mins(14));
1042 assert_eq!(health.report().state, State::Ok);
1043 health.drained(800, mins(15));
1044 let report = health.report();
1045 assert_eq!(report.state, State::Degraded);
1046 assert_eq!(report.index.backlog_since, Some(mins(0)));
1047 health.drained(0, mins(16));
1048 assert_eq!(health.report().state, State::Ok);
1049 assert_eq!(health.report().index.backlog_since, None);
1050 }
1051
1052 #[test]
1053 fn a_dead_watcher_fails_and_a_silent_one_degrades() {
1054 let (health, _rx) = monitored();
1055 health.watcher_failed("inotify watch limit reached", mins(0));
1056 assert_eq!(health.report().state, State::Failing);
1057 health.watcher_started(mins(1));
1058 health.drained(0, mins(1));
1059 assert_eq!(health.report().state, State::Ok);
1060 health.drained(0, mins(16));
1061 let report = health.report();
1062 assert_eq!(report.state, State::Degraded);
1063 assert!(report.reason.unwrap().contains("watcher has not reported"));
1064 }
1065
1066 #[test]
1067 fn index_and_repair_errors_degrade_until_the_next_success() {
1068 let (health, _rx) = monitored();
1069 health.watcher_started(mins(0));
1070 health.drain_failed("disk I/O error", mins(0));
1071 health.repair_failed("disk I/O error", mins(0));
1072 let reason = health.report().reason.unwrap();
1073 assert!(reason.contains("store repair failed"), "{reason}");
1074 health.drain_failed("disk I/O error", mins(10));
1075 let reason = health.report().reason.unwrap();
1076 assert!(reason.contains("index drain failed"), "{reason}");
1077 health.drained(0, mins(11));
1078 assert_eq!(health.report().state, State::Degraded);
1079 health.repaired(mins(12));
1080 assert_eq!(health.report().state, State::Ok);
1081 }
1082
1083 #[test]
1084 fn missing_indexes_stay_degraded_without_a_drain() {
1085 let (health, _rx) = monitored();
1086 health.indexes_unavailable("tantivy meta.json missing", mins(0));
1087 health.watcher_started(mins(90));
1088 let report = health.report();
1089 assert_eq!(report.state, State::Degraded);
1090 let reason = report.reason.unwrap();
1091 assert!(reason.contains("search indexes unavailable"), "{reason}");
1092 assert!(!reason.contains("has not reported"), "{reason}");
1093 }
1094
1095 #[test]
1096 fn the_report_serializes_without_content() {
1097 let (health, _rx) = monitored();
1098 health.ingest_failed("database is locked", mins(0));
1099 health.ingest_failed("database is locked", mins(10));
1100 let json = serde_json::to_value(health.report()).unwrap();
1101 assert_eq!(json["state"], "degraded");
1102 assert_eq!(json["last_error"]["source"], "ingest");
1103 for key in ["reason", "since", "capture", "index"] {
1104 assert!(json.get(key).is_some(), "{key} missing: {json}");
1105 }
1106 }
1107
1108 #[test]
1109 fn a_queue_entry_older_than_ten_minutes_degrades_until_it_drains() {
1110 let (health, mut rx) = monitored();
1111 health.watcher_started(mins(0));
1112 health.drained(0, mins(0));
1113 health.queue_checked(2, Some(mins(0)), mins(9));
1114 assert_eq!(health.report().state, State::Ok);
1115 assert_eq!(health.report().queue.depth, 2);
1116 health.queue_checked(2, Some(mins(0)), mins(10));
1117 let report = health.report();
1118 assert_eq!(report.state, State::Degraded);
1119 assert!(
1120 report
1121 .reason
1122 .as_deref()
1123 .unwrap()
1124 .contains("write queue not draining"),
1125 "{report:?}"
1126 );
1127 assert_eq!(report.queue.oldest_age_secs, Some(600));
1128 health.queue_checked(0, None, mins(11));
1129 assert_eq!(health.report().state, State::Ok);
1130 assert_eq!(drain(&mut rx), vec![State::Degraded, State::Ok]);
1131 }
1132
1133 #[test]
1134 fn a_rejected_write_is_reported_by_tool_status_and_time_only() {
1135 let (health, _rx) = monitored();
1136 health.write_rejected("associate", 404, mins(3));
1137 let json = serde_json::to_value(health.report()).unwrap();
1138 assert_eq!(
1139 json["queue"]["last_rejected"],
1140 serde_json::json!({"tool": "associate", "status": 404, "at": "2026-10-31T12:03:00Z"})
1141 );
1142 assert_eq!(json["state"], "ok");
1143 }
1144
1145 fn hours(h: i64) -> DateTime<Utc> {
1146 t0() + Duration::hours(h)
1147 }
1148
1149 fn alive(health: &Health, at: DateTime<Utc>) {
1150 health.update(at, |f| {
1151 f.capture.checked_at = Some(at);
1152 f.index.checked_at = Some(at);
1153 });
1154 }
1155
1156 fn sync_seen(health: &Health, sync: SyncReport, at: DateTime<Utc>) {
1157 let seen = ProgressSeen {
1158 sync: Some(sync),
1159 ..ProgressSeen::default()
1160 };
1161 health.progress_checked(seen, at);
1162 }
1163
1164 fn distill_seen(
1165 health: &Health,
1166 pending: u64,
1167 oldest_at: Option<DateTime<Utc>>,
1168 at: DateTime<Utc>,
1169 ) {
1170 distill_seen_in(health, DistillMode::Llm, pending, oldest_at, at);
1171 }
1172
1173 fn distill_seen_in(
1174 health: &Health,
1175 mode: DistillMode,
1176 pending: u64,
1177 oldest_at: Option<DateTime<Utc>>,
1178 at: DateTime<Utc>,
1179 ) {
1180 let distill = DistillReport {
1181 mode,
1182 pending,
1183 oldest_at,
1184 ..DistillReport::default()
1185 };
1186 let seen = ProgressSeen {
1187 distill: Some(distill),
1188 ..ProgressSeen::default()
1189 };
1190 health.progress_checked(seen, at);
1191 }
1192
1193 fn synced(peers: usize, synced_before: bool, last: Option<DateTime<Utc>>) -> SyncReport {
1194 SyncReport {
1195 checked_at: None,
1196 peers: Some(peers),
1197 synced_before,
1198 last_success_at: last,
1199 last_cursor_at: None,
1200 }
1201 }
1202
1203 #[test]
1204 fn without_a_stamp_the_window_counts_from_when_records_last_moved() {
1205 let (health, _rx) = monitored();
1206 let seen = SyncReport {
1207 last_cursor_at: Some(hours(-48)),
1208 ..synced(0, true, None)
1209 };
1210 sync_seen(&health, seen.clone(), mins(0));
1211 let report = health.report();
1212 assert_eq!(report.state, State::Degraded);
1213 let reason = report.reason.unwrap();
1214 assert!(reason.contains("no sync has succeeded in 2d"), "{reason}");
1215
1216 let stamped = SyncReport {
1217 last_success_at: Some(mins(0)),
1218 ..seen
1219 };
1220 sync_seen(&health, stamped, mins(0));
1221 assert_eq!(health.report().state, State::Ok);
1222 }
1223
1224 #[test]
1225 fn reasons_count_one_peer_and_one_raw_in_the_singular() {
1226 let (health, _rx) = monitored();
1227 sync_seen(&health, synced(1, true, Some(hours(-30))), mins(0));
1228 distill_seen(&health, 1, Some(hours(-30)), mins(0));
1229 let reason = health.report().reason.unwrap();
1230 assert!(reason.contains("with 1 peer in sync-peers"), "{reason}");
1231 assert!(reason.contains("1 raw memory has waited"), "{reason}");
1232 }
1233
1234 #[test]
1235 fn a_device_that_synced_before_and_lost_its_peers_degrades_with_the_fix() {
1236 let (health, mut rx) = monitored();
1237 sync_seen(&health, synced(0, true, Some(hours(-25))), mins(0));
1238 let report = health.report();
1239 assert_eq!(report.state, State::Degraded);
1240 let reason = report.reason.unwrap();
1241 assert!(
1242 reason.contains("no sync has succeeded in 25h 0m"),
1243 "{reason}"
1244 );
1245 assert!(reason.contains("lists no peers"), "{reason}");
1246 assert!(reason.contains("~/.second-brain/sync-peers"), "{reason}");
1247 assert!(reason.contains("sync_overdue_hours = 0"), "{reason}");
1248 assert_eq!(report.sync.peers, Some(0));
1249 assert_eq!(report.sync.checked_at, Some(mins(0)));
1250 assert_eq!(drain(&mut rx), vec![State::Degraded]);
1251
1252 sync_seen(&health, synced(1, true, Some(mins(1))), mins(1));
1253 assert_eq!(health.report().state, State::Ok);
1254 }
1255
1256 #[test]
1257 fn a_device_synced_by_hand_without_sync_peers_stays_ok_while_it_syncs() {
1258 let (health, mut rx) = monitored();
1259 let by_hand = SyncReport {
1260 peers: None,
1261 ..synced(0, true, Some(hours(10)))
1262 };
1263 alive(&health, hours(30));
1264 sync_seen(&health, by_hand.clone(), hours(30));
1265 assert_eq!(health.report().state, State::Ok);
1266 alive(&health, hours(34));
1267 sync_seen(&health, by_hand, hours(34));
1268 let report = health.report();
1269 assert_eq!(report.state, State::Degraded);
1270 let reason = report.reason.unwrap();
1271 assert!(
1272 reason.contains("no sync has succeeded in 24h 0m"),
1273 "{reason}"
1274 );
1275 assert!(reason.contains("sb sync <user@host>"), "{reason}");
1276 assert!(reason.contains("sync_overdue_hours = 0"), "{reason}");
1277 assert!(!reason.contains("sync-peers"), "{reason}");
1278 assert_eq!(drain(&mut rx), vec![State::Degraded]);
1279 }
1280
1281 #[test]
1282 fn a_solo_device_that_never_synced_stays_ok() {
1283 let (health, mut rx) = monitored();
1284 for h in [0, 25, 100] {
1285 alive(&health, hours(h));
1286 sync_seen(&health, synced(0, false, None), hours(h));
1287 }
1288 let report = health.report();
1289 assert_eq!(report.state, State::Ok, "{:?}", report.reason);
1290 assert!(drain(&mut rx).is_empty());
1291 }
1292
1293 #[test]
1294 fn a_sync_overdue_past_the_window_degrades_only_on_a_device_that_syncs() {
1295 let (health, _rx) = monitored();
1296 alive(&health, hours(0));
1297 sync_seen(&health, synced(2, true, Some(hours(-23))), hours(0));
1298 assert_eq!(health.report().state, State::Ok);
1299
1300 alive(&health, hours(1));
1301 sync_seen(&health, synced(2, true, Some(hours(-23))), hours(1));
1302 let report = health.report();
1303 assert_eq!(report.state, State::Degraded);
1304 let reason = report.reason.unwrap();
1305 assert!(
1306 reason.contains("no sync has succeeded in 24h 0m"),
1307 "{reason}"
1308 );
1309 assert!(reason.contains("2 peers"), "{reason}");
1310
1311 sync_seen(&health, synced(2, true, Some(hours(1))), hours(1));
1312 assert_eq!(health.report().state, State::Ok);
1313
1314 sync_seen(&health, synced(0, false, Some(hours(-500))), hours(1));
1315 assert_eq!(health.report().state, State::Ok);
1316 }
1317
1318 #[test]
1319 fn with_peers_and_no_recorded_success_the_window_starts_at_startup() {
1320 let (health, _rx) = monitored();
1321 alive(&health, hours(23));
1322 sync_seen(&health, synced(1, false, None), hours(23));
1323 assert_eq!(health.report().state, State::Ok);
1324 alive(&health, hours(24));
1325 sync_seen(&health, synced(1, false, None), hours(24));
1326 let reason = health.report().reason.unwrap();
1327 assert!(
1328 reason.contains("no sync has succeeded in 24h 0m"),
1329 "{reason}"
1330 );
1331 }
1332
1333 #[test]
1334 fn raws_left_undistilled_past_the_window_degrade_with_the_count() {
1335 let (health, mut rx) = monitored();
1336 alive(&health, hours(0));
1337 distill_seen(&health, 520, Some(hours(-23)), hours(0));
1338 assert_eq!(health.report().state, State::Ok);
1339 assert_eq!(health.report().distill.pending, 520);
1340
1341 alive(&health, hours(2));
1342 distill_seen(&health, 520, Some(hours(-23)), hours(2));
1343 let report = health.report();
1344 assert_eq!(report.state, State::Degraded);
1345 let reason = report.reason.unwrap();
1346 assert!(
1347 reason.contains("520 raw memories have waited 25h 0m to be distilled"),
1348 "{reason}"
1349 );
1350 assert!(reason.contains("undistilled_overdue_hours = 0"), "{reason}");
1351
1352 alive(&health, hours(3));
1353 distill_seen(&health, 0, None, hours(3));
1354 assert_eq!(health.report().state, State::Ok);
1355 assert_eq!(drain(&mut rx), vec![State::Degraded, State::Ok]);
1356 }
1357
1358 #[test]
1359 fn without_an_llm_endpoint_waiting_raws_report_the_mode_instead_of_degrading() {
1360 let (health, mut rx) = monitored();
1361 alive(&health, hours(100));
1362 distill_seen_in(
1363 &health,
1364 DistillMode::NoGpu,
1365 520,
1366 Some(hours(-100)),
1367 hours(100),
1368 );
1369 let report = health.report();
1370 assert_eq!(report.state, State::Ok, "{:?}", report.reason);
1371 assert_eq!(report.reason, None);
1372 assert_eq!(report.distill.pending, 520);
1373 let json = serde_json::to_value(&report).unwrap();
1374 assert_eq!(json["distill"]["mode"], "no-gpu");
1375 assert_eq!(json["distill"]["note"], NO_GPU_NOTE);
1376 assert!(drain(&mut rx).is_empty());
1377 }
1378
1379 #[test]
1380 fn with_an_llm_endpoint_the_report_names_the_mode_and_carries_no_note() {
1381 let (health, _rx) = monitored();
1382 alive(&health, hours(2));
1383 distill_seen(&health, 3, Some(hours(-23)), hours(2));
1384 let report = health.report();
1385 assert_eq!(report.state, State::Degraded);
1386 let json = serde_json::to_value(&report).unwrap();
1387 assert_eq!(json["distill"]["mode"], "llm");
1388 assert!(json["distill"].get("note").is_none(), "{json}");
1389 }
1390
1391 #[test]
1392 fn a_window_turned_off_never_degrades() {
1393 let (tx, _rx) = unbounded_channel();
1394 let windows = Windows::from_config(&AlertConfig {
1395 sync_overdue_hours: 0,
1396 undistilled_overdue_hours: 0,
1397 ..AlertConfig::default()
1398 });
1399 assert_eq!(
1400 windows,
1401 Windows {
1402 sync_overdue: None,
1403 undistilled: None
1404 }
1405 );
1406 let health = Health::new(t0(), Some(tx)).with_windows(windows);
1407 alive(&health, hours(100));
1408 sync_seen(&health, synced(1, true, Some(hours(-100))), hours(100));
1409 distill_seen(&health, 9, Some(hours(-100)), hours(100));
1410 assert_eq!(health.report().state, State::Ok);
1411 sync_seen(&health, synced(0, true, Some(hours(-100))), hours(100));
1412 assert_eq!(health.report().state, State::Ok);
1413 }
1414
1415 #[test]
1416 fn the_windows_come_from_the_alert_config() {
1417 let windows = Windows::from_config(&AlertConfig {
1418 sync_overdue_hours: 6,
1419 undistilled_overdue_hours: 48,
1420 ..AlertConfig::default()
1421 });
1422 assert_eq!(windows.sync_overdue, Some(Duration::hours(6)));
1423 assert_eq!(windows.undistilled, Some(Duration::hours(48)));
1424 assert_eq!(Windows::default().sync_overdue, Some(Duration::hours(24)));
1425 assert_eq!(Windows::default().undistilled, Some(Duration::hours(24)));
1426 }
1427
1428 #[test]
1429 fn announce_sends_the_current_state_once_unless_it_was_already_sent() {
1430 let (health, mut rx) = monitored();
1431 health.announce(mins(0));
1432 let sent: Vec<Transition> = std::iter::from_fn(|| rx.try_recv().ok()).collect();
1433 assert_eq!(
1434 sent,
1435 vec![Transition {
1436 state: State::Ok,
1437 reason: None,
1438 at: mins(0),
1439 }]
1440 );
1441
1442 health.announce(mins(1));
1443 assert!(drain(&mut rx).is_empty(), "ok was already sent");
1444
1445 sync_seen(&health, synced(0, true, Some(hours(-30))), mins(2));
1446 health.announce(mins(3));
1447 assert_eq!(drain(&mut rx), vec![State::Degraded]);
1448 }
1449
1450 #[test]
1451 fn announce_reassesses_and_sends_one_message_for_a_pending_change() {
1452 let (health, mut rx) = monitored();
1453 alive(&health, hours(0));
1454 distill_seen(&health, 3, Some(hours(-23)), hours(0));
1455 assert!(drain(&mut rx).is_empty());
1456 health.announce(hours(2));
1457 assert_eq!(drain(&mut rx), vec![State::Degraded]);
1458 }
1459
1460 #[test]
1461 fn serving_leaves_out_sync_and_distillation() {
1462 let (health, _rx) = monitored();
1463 alive(&health, hours(0));
1464 let seen = ProgressSeen {
1465 sync: Some(synced(1, true, Some(hours(-30)))),
1466 distill: Some(DistillReport {
1467 pending: 4,
1468 oldest_at: Some(hours(-30)),
1469 ..DistillReport::default()
1470 }),
1471 ingested_sessions: None,
1472 error: None,
1473 };
1474 health.progress_checked(seen, hours(0));
1475 let report = health.report();
1476 assert_eq!(report.state, State::Degraded);
1477 assert_eq!(
1478 report.serving,
1479 Verdict {
1480 state: State::Ok,
1481 reason: None
1482 }
1483 );
1484
1485 health.watcher_failed("inotify watch limit reached", hours(0));
1486 let report = health.report();
1487 assert_eq!(report.state, State::Failing);
1488 assert_eq!(report.serving.state, State::Failing);
1489 let serving = report.serving.reason.unwrap();
1490 assert!(serving.contains("watcher is not running"), "{serving}");
1491 assert!(!serving.contains("sync"), "{serving}");
1492
1493 let json = serde_json::to_value(health.report()).unwrap();
1494 assert_eq!(json["serving"]["state"], "failing");
1495 }
1496
1497 #[test]
1498 fn one_pass_that_clears_sync_and_finds_distillation_overdue_sends_nothing() {
1499 let (health, mut rx) = monitored();
1500 alive(&health, hours(0));
1501 sync_seen(&health, synced(1, true, Some(hours(-30))), hours(0));
1502 assert_eq!(drain(&mut rx), vec![State::Degraded]);
1503 let seen = ProgressSeen {
1504 sync: Some(synced(1, true, Some(hours(0)))),
1505 distill: Some(DistillReport {
1506 pending: 4,
1507 oldest_at: Some(hours(-30)),
1508 ..DistillReport::default()
1509 }),
1510 ingested_sessions: None,
1511 error: None,
1512 };
1513 health.progress_checked(seen, hours(0));
1514 assert_eq!(health.report().state, State::Degraded);
1515 assert!(drain(&mut rx).is_empty());
1516 }
1517
1518 fn backfill_running(done: usize, failed: usize, at: DateTime<Utc>) -> BackfillReport {
1519 BackfillReport {
1520 state: BackfillState::Running,
1521 total: 40,
1522 done,
1523 failed,
1524 updated_at: Some(at),
1525 }
1526 }
1527
1528 #[test]
1529 fn a_new_daemon_reports_the_backfill_not_started() {
1530 let (health, _rx) = monitored();
1531 let report = health.report();
1532 assert_eq!(report.backfill, BackfillReport::default());
1533 assert_eq!(report.backfill.state, BackfillState::NotStarted);
1534 assert_eq!(report.backfill.total, 0);
1535 assert_eq!(report.backfill.done, 0);
1536 assert_eq!(report.backfill.failed, 0);
1537 let json = serde_json::to_value(report).unwrap();
1538 assert_eq!(json["backfill"]["state"], "not_started");
1539 }
1540
1541 #[test]
1542 fn backfill_progress_is_reported_and_survives_later_reports() {
1543 let (health, _rx) = monitored();
1544 health.backfill_progress(backfill_running(3, 1, mins(1)));
1545 assert_eq!(health.report().backfill, backfill_running(3, 1, mins(1)));
1546
1547 health.ingested(None, mins(2));
1548 assert_eq!(health.report().backfill, backfill_running(3, 1, mins(1)));
1549
1550 let done = BackfillReport {
1551 state: BackfillState::Done,
1552 total: 40,
1553 done: 40,
1554 failed: 1,
1555 updated_at: Some(mins(3)),
1556 };
1557 health.backfill_progress(done.clone());
1558 assert_eq!(health.report().backfill, done);
1559 let json = serde_json::to_value(health.report()).unwrap();
1560 assert_eq!(json["backfill"]["state"], "done");
1561 assert_eq!(json["backfill"]["done"], 40);
1562 }
1563
1564 #[test]
1565 fn backfill_progress_leaves_the_state_and_reason_alone() {
1566 let (health, mut rx) = monitored();
1567 for m in 0..=12 {
1568 health.ingest_failed("database is locked", mins(m));
1569 }
1570 assert_eq!(drain(&mut rx), vec![State::Degraded]);
1571 let before = health.report();
1572
1573 health.backfill_progress(backfill_running(5, 2, mins(13)));
1574 let after = health.report();
1575 assert_eq!(after.state, before.state);
1576 assert_eq!(after.reason, before.reason);
1577 assert_eq!(after.since, before.since);
1578 assert_eq!(after.serving, before.serving);
1579 assert!(drain(&mut rx).is_empty());
1580 }
1581
1582 fn empty_recall(session: &str) -> RecallEvent {
1583 RecallEvent {
1584 session: Some(session.to_string()),
1585 verdict: crate::value_metrics::RecallVerdict::Empty,
1586 memory_ids: Vec::new(),
1587 }
1588 }
1589
1590 #[test]
1591 fn a_value_metric_crossing_its_threshold_sends_exactly_one_transition() {
1592 let (health, mut rx) = monitored();
1593 alive(&health, mins(0));
1594 for _ in 0..49 {
1595 health.recalled(empty_recall("s"), mins(1));
1596 }
1597 assert!(drain(&mut rx).is_empty());
1598 for _ in 0..30 {
1599 health.recalled(empty_recall("s"), mins(2));
1600 }
1601 let report = health.report();
1602 assert_eq!(report.state, State::Degraded);
1603 assert_eq!(report.serving.state, State::Ok);
1604 let reason = report.reason.unwrap();
1605 assert!(reason.contains("found nothing that cleared"), "{reason}");
1606 assert_eq!(drain(&mut rx), vec![State::Degraded]);
1607 }
1608
1609 #[test]
1610 fn a_share_hovering_at_its_threshold_sends_one_transition_each_way() {
1611 let (health, mut rx) = monitored();
1612 let hit = || RecallEvent {
1613 session: None,
1614 verdict: crate::value_metrics::RecallVerdict::Hit,
1615 memory_ids: Vec::new(),
1616 };
1617 for m in 0..50 {
1618 alive(&health, mins(m));
1619 let event = if m < 37 { empty_recall("s") } else { hit() };
1620 health.recalled(event, mins(m));
1621 }
1622 assert_eq!(health.report().state, State::Ok, "37 of 50 is 74%");
1623 for _ in 0..3 {
1624 health.recalled(empty_recall("s"), mins(50));
1625 }
1626 assert_eq!(drain(&mut rx), vec![State::Degraded], "40 of 53 is 75.5%");
1627 for m in 51..56 {
1628 alive(&health, mins(m));
1629 health.recalled(hit(), mins(m));
1630 health.recalled(empty_recall("s"), mins(m));
1631 }
1632 assert!(drain(&mut rx).is_empty(), "74% then 71% stays degraded");
1633 for m in 56..66 {
1634 alive(&health, mins(m));
1635 health.recalled(hit(), mins(m));
1636 }
1637 assert_eq!(health.report().value.recall.recalls, 73);
1638 assert_eq!(drain(&mut rx), vec![State::Ok], "45 of 73 is 62%");
1639 }
1640
1641 #[test]
1642 fn peer_lag_is_left_out_when_no_sync_succeeded_at_all() {
1643 let (health, _rx) = monitored();
1644 health.sync_peers_listed(&["u@pi".to_string()], hours(0));
1645 alive(&health, hours(30));
1646 let reason = health.report().reason.unwrap();
1647 assert!(reason.contains("no sync with u@pi"), "{reason}");
1648
1649 sync_seen(&health, synced(1, true, Some(hours(-30))), hours(30));
1650 let reason = health.report().reason.unwrap();
1651 assert!(reason.contains("no sync has succeeded in"), "{reason}");
1652 assert!(!reason.contains("no sync with u@pi"), "{reason}");
1653 assert_eq!(health.report().value.sync_peers.len(), 1);
1654 }
1655
1656 #[test]
1657 fn the_ingest_window_starts_after_startup_and_any_backfill() {
1658 let (health, _rx) = monitored();
1659 assert_eq!(health.ingest_window_start(mins(5)), Some(t0()));
1660 assert_eq!(
1661 health.ingest_window_start(hours(30)),
1662 Some(hours(30) - WINDOW)
1663 );
1664 health.backfill_progress(backfill_running(1, 0, mins(1)));
1665 assert_eq!(health.ingest_window_start(mins(5)), None);
1666 health.backfill_progress(BackfillReport {
1667 state: BackfillState::Done,
1668 updated_at: Some(mins(20)),
1669 ..BackfillReport::default()
1670 });
1671 assert_eq!(health.ingest_window_start(mins(30)), Some(mins(20)));
1672 }
1673
1674 #[test]
1675 fn progress_sets_the_ingested_sessions_and_a_failed_read_clears_them() {
1676 let (health, _rx) = monitored();
1677 health.recalled(empty_recall("a"), mins(1));
1678 let seen = ProgressSeen {
1679 ingested_sessions: Some(vec!["a".to_string(), "b".to_string()]),
1680 ..ProgressSeen::default()
1681 };
1682 health.progress_checked(seen, mins(2));
1683 let recall = health.report().value.recall;
1684 assert_eq!(recall.sessions_ingested, Some(2));
1685 assert_eq!(recall.session_recall_rate, Some(0.5));
1686 health.progress_checked(ProgressSeen::default(), mins(3));
1687 assert_eq!(health.report().value.recall.sessions_ingested, None);
1688 }
1689
1690 #[test]
1691 fn every_value_metric_is_in_the_report_json() {
1692 let (health, _rx) = monitored();
1693 health.sync_peers_listed(&["u@pi".to_string()], mins(0));
1694 health.store_measured(Some(StoreSize::default()), mins(0));
1695 health.resident_measured(Some(ResidentSize::default()), mins(0));
1696 let json = serde_json::to_value(health.report()).unwrap();
1697 let keys = |value: &serde_json::Value| -> Vec<String> {
1698 let mut keys: Vec<String> = value.as_object().unwrap().keys().cloned().collect();
1699 keys.sort();
1700 keys
1701 };
1702 assert_eq!(
1703 keys(&json["sync_peers"][0]),
1704 [
1705 "consecutive_failures",
1706 "lag_secs",
1707 "last_attempt_at",
1708 "last_success_at",
1709 "peer"
1710 ]
1711 );
1712 assert_eq!(
1713 keys(&json["store"]),
1714 [
1715 "archive_bytes",
1716 "checked_at",
1717 "db_bytes",
1718 "free_bytes",
1719 "index_bytes",
1720 "raw_index_bytes",
1721 "total_bytes",
1722 "wal_bytes"
1723 ]
1724 );
1725 assert_eq!(
1726 keys(&json["resident"]),
1727 [
1728 "checked_at",
1729 "components",
1730 "limit_bytes",
1731 "raw_index_bytes",
1732 "rss_anon_bytes",
1733 "unattributed_bytes"
1734 ]
1735 );
1736 assert_eq!(
1737 keys(&json["resident"]["components"]),
1738 [
1739 "allocator_free",
1740 "embedder",
1741 "lexical",
1742 "reranker",
1743 "sqlite",
1744 "vector_reader",
1745 "vector_writer"
1746 ]
1747 );
1748 for key in ["recall", "recalled", "sync_peers", "store", "resident"] {
1749 assert!(json.get(key).is_some(), "{key} missing: {json}");
1750 }
1751 for key in [
1752 "window_secs",
1753 "recalls",
1754 "hits",
1755 "low_score",
1756 "unjudged",
1757 "empty",
1758 "empty_share",
1759 "low_score_share",
1760 "unjudged_share",
1761 "sessions_with_recall",
1762 "sessions_ingested",
1763 "session_recall_rate",
1764 ] {
1765 assert!(json["recall"].get(key).is_some(), "recall.{key}: {json}");
1766 }
1767 for key in [
1768 "window_secs",
1769 "memories",
1770 "corrected",
1771 "used",
1772 "corrected_share",
1773 "used_share",
1774 ] {
1775 assert!(
1776 json["recalled"].get(key).is_some(),
1777 "recalled.{key}: {json}"
1778 );
1779 }
1780 }
1781}