1use std::path::PathBuf;
6use std::sync::Mutex;
7
8use chrono::{DateTime, Duration, Utc};
9use second_brain_core::config::AlertConfig;
10use second_brain_core::keys::{ASIDE_RETENTION_MICROS, MovedAside};
11use second_brain_core::sqlite_store::{FailedCounts, FloorHealth, VacuumSpace};
12use second_brain_sync::keyring::IdentityStatus;
13use serde::Serialize;
14use tokio::sync::mpsc::UnboundedSender;
15use uuid::Uuid;
16
17use crate::snapshot_merge::MergeStatus;
18use crate::value_metrics::{
19 PeerOutcome, RecallEvent, ResidentSize, StoreSize, ValueFacts, ValueProblems, ValueReport,
20 WINDOW,
21};
22
23const CAPTURE_LAG_DEGRADED: Duration = Duration::minutes(15);
24const CAPTURE_LAG_FAILING: Duration = Duration::hours(2);
25const BACKLOG_DEGRADED: Duration = Duration::minutes(15);
26const BACKLOG_FAILING: Duration = Duration::hours(2);
27const WATCHER_SILENT: Duration = Duration::minutes(15);
29const DRAIN_SILENT: Duration = Duration::hours(1);
31const ERROR_GRACE: Duration = Duration::minutes(10);
34const QUEUE_STUCK: Duration = Duration::minutes(10);
36const CLOCK_REJECTION_SHOWN: Duration = Duration::days(1);
39pub const CLOCK_HELD_GRACE: std::time::Duration = std::time::Duration::from_secs(120);
42
43#[derive(Debug, Clone, Copy, Default, PartialEq, Eq, PartialOrd, Ord, Serialize)]
44#[serde(rename_all = "lowercase")]
45pub enum State {
46 #[default]
47 Ok,
48 Degraded,
49 Failing,
50}
51
52#[derive(Debug, Clone, PartialEq, Serialize)]
53pub struct Transition {
54 pub state: State,
55 pub reason: Option<String>,
56 pub at: DateTime<Utc>,
57}
58
59#[derive(Debug, Clone, Serialize)]
60pub struct ErrorRecord {
61 pub source: &'static str,
62 pub message: String,
63 pub at: DateTime<Utc>,
64}
65
66#[derive(Debug, Clone, Default, Serialize)]
67pub struct CaptureReport {
68 pub checked_at: Option<DateTime<Utc>>,
69 pub newest_transcript_at: Option<DateTime<Utc>>,
70 pub captured_through: Option<DateTime<Utc>>,
72 pub lag_secs: Option<i64>,
73}
74
75#[derive(Debug, Clone, Default, Serialize)]
76pub struct IndexReport {
77 pub checked_at: Option<DateTime<Utc>>,
78 pub outbox_pending: Option<u64>,
79 pub backlog_since: Option<DateTime<Utc>>,
80}
81
82#[derive(Debug, Clone, PartialEq, Serialize)]
85pub struct RejectedWrite {
86 pub tool: String,
87 pub status: u16,
88 pub at: DateTime<Utc>,
89}
90
91#[derive(Debug, Clone, Default, Serialize)]
92pub struct QueueReport {
93 pub depth: u64,
94 pub oldest_at: Option<DateTime<Utc>>,
95 pub oldest_age_secs: Option<i64>,
96 pub last_rejected: Option<RejectedWrite>,
97}
98
99#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize)]
103#[serde(rename_all = "snake_case")]
104pub enum MaintenanceState {
105 Pending,
106 WaitingForSpace,
107 MergingIndex,
108 Vacuuming,
110 Done,
111}
112
113#[derive(Debug, Clone, Serialize)]
114pub struct MaintenanceReport {
115 pub state: MaintenanceState,
116 pub since: Option<DateTime<Utc>>,
117 pub space: Option<VacuumSpace>,
118 pub last_error: Option<String>,
119}
120
121impl Default for MaintenanceReport {
122 fn default() -> Self {
123 Self {
124 state: MaintenanceState::Pending,
125 since: None,
126 space: None,
127 last_error: None,
128 }
129 }
130}
131
132#[derive(Debug, Clone, Copy, Default, PartialEq, Eq, Serialize)]
133#[serde(rename_all = "snake_case")]
134pub enum EncryptionState {
135 #[default]
137 Plaintext,
138 Pending,
140 WaitingForSpace,
141 Encrypting,
143 Rotating,
145 Encrypted,
146}
147
148#[derive(Debug, Clone, Default, Serialize)]
149pub struct EncryptionReport {
150 pub state: EncryptionState,
151 pub since: Option<DateTime<Utc>>,
152 pub space: Option<VacuumSpace>,
153 pub migration_pause_ms: Option<u64>,
155 pub notice: Option<String>,
156 pub last_error: Option<String>,
157}
158
159pub const PLAINTEXT_LEFT_NOTICE: &str = "the store is encrypted now, but deleting the plaintext \
160 file does not erase it from disk or flash blocks; only full-disk encryption or a wipe does";
161
162#[derive(Debug, Clone, Copy, Default, PartialEq, Eq, Serialize)]
163#[serde(rename_all = "snake_case")]
164pub enum BackfillState {
165 #[default]
166 NotStarted,
167 Running,
168 Done,
169}
170
171#[derive(Debug, Clone, Default, PartialEq, Eq, Serialize)]
172pub struct BackfillReport {
173 pub state: BackfillState,
174 pub total: usize,
175 pub done: usize,
176 pub failed: usize,
177 pub updated_at: Option<DateTime<Utc>>,
178}
179
180#[derive(Debug, Clone, Serialize)]
183pub struct MovedAsideReport {
184 pub name: String,
185 pub path: PathBuf,
186 pub moved_at: DateTime<Utc>,
187 pub deleted_in_days: i64,
189}
190
191fn moved_aside_report(asides: &[MovedAside], now: DateTime<Utc>) -> Vec<MovedAsideReport> {
192 const DAY_MICROS: i64 = 86_400 * 1_000_000;
193 asides
194 .iter()
195 .map(|aside| {
196 let left = (aside.moved_at_micros + ASIDE_RETENTION_MICROS)
197 .saturating_sub(now.timestamp_micros())
198 .max(0);
199 MovedAsideReport {
200 name: aside
201 .path
202 .file_name()
203 .map(|n| n.to_string_lossy().into_owned())
204 .unwrap_or_default(),
205 path: aside.path.clone(),
206 moved_at: DateTime::from_timestamp_micros(aside.moved_at_micros)
207 .unwrap_or_default(),
208 deleted_in_days: (left + DAY_MICROS - 1) / DAY_MICROS,
209 }
210 })
211 .collect()
212}
213
214#[derive(Debug, Clone, Default, Serialize)]
215pub struct SyncReport {
216 pub checked_at: Option<DateTime<Utc>>,
217 pub peers: Option<usize>,
220 pub synced_before: bool,
222 pub last_success_at: Option<DateTime<Utc>>,
223 pub last_cursor_at: Option<DateTime<Utc>>,
226 pub floor: Option<FloorReport>,
228 pub snapshot_merge: Option<MergeStatus>,
230}
231
232#[derive(Debug, Clone, Default, PartialEq, Eq, Serialize)]
233pub struct FloorReport {
234 pub floor: u64,
235 pub log_floor: u64,
236 pub head: u64,
237 pub top: u64,
238 pub silent: Vec<String>,
239 pub stuck: Vec<String>,
240 pub failing: u64,
241 pub settled: u64,
242 pub settled_dropped: u64,
244}
245
246#[derive(Debug, Clone, Default, Serialize)]
247pub struct IdentityReport {
248 pub checked_at: Option<DateTime<Utc>>,
249 pub key_storage: Option<&'static str>,
251 pub hardware: Option<&'static str>,
256 #[serde(flatten)]
258 pub status: Option<IdentityStatus>,
259}
260
261pub const NO_GPU_NOTE: &str = "no LLM endpoint configured and no LLM distillation in 7 days: \
262 raws covered by an agent note are retired, the rest stay searchable";
263
264#[derive(Debug, Clone, Copy, Default, PartialEq, Eq, Serialize)]
268pub enum DistillMode {
269 #[default]
270 #[serde(rename = "llm")]
271 Llm,
272 #[serde(rename = "no-gpu")]
273 NoGpu,
274}
275
276impl DistillMode {
277 fn note(self) -> Option<&'static str> {
278 match self {
279 DistillMode::Llm => None,
280 DistillMode::NoGpu => Some(NO_GPU_NOTE),
281 }
282 }
283}
284
285#[derive(Debug, Clone, Default, Serialize)]
286pub struct DistillReport {
287 pub checked_at: Option<DateTime<Utc>>,
288 pub mode: DistillMode,
289 pub pending: u64,
290 pub oldest_at: Option<DateTime<Utc>>,
291 #[serde(skip_serializing_if = "Option::is_none")]
292 pub note: Option<&'static str>,
293}
294
295#[derive(Debug, Clone, Copy, PartialEq)]
298pub struct Windows {
299 pub sync_overdue: Option<Duration>,
300 pub undistilled: Option<Duration>,
301}
302
303impl Windows {
304 pub fn from_config(config: &AlertConfig) -> Self {
305 let hours = |h: u64| {
306 i64::try_from(h)
307 .ok()
308 .filter(|&h| h > 0)
309 .and_then(Duration::try_hours)
310 };
311 Self {
312 sync_overdue: hours(config.sync_overdue_hours),
313 undistilled: hours(config.undistilled_overdue_hours),
314 }
315 }
316}
317
318impl Default for Windows {
319 fn default() -> Self {
320 Self::from_config(&AlertConfig::default())
321 }
322}
323
324#[derive(Debug, Clone, Default)]
327pub struct ProgressSeen {
328 pub sync: Option<SyncReport>,
329 pub distill: Option<DistillReport>,
330 pub ingested_sessions: Option<Vec<String>>,
333 pub error: Option<String>,
334}
335
336#[derive(Debug, Clone, PartialEq, Serialize)]
337pub struct Verdict {
338 pub state: State,
339 pub reason: Option<String>,
340}
341
342#[derive(Debug, Clone, Serialize)]
343pub struct HealthReport {
344 pub state: State,
345 pub reason: Option<String>,
346 pub since: DateTime<Utc>,
347 pub serving: Verdict,
350 pub capture: CaptureReport,
351 pub index: IndexReport,
352 pub queue: QueueReport,
353 pub maintenance: MaintenanceReport,
354 pub encryption: EncryptionReport,
355 pub backfill: BackfillReport,
356 pub sync: SyncReport,
357 pub identity: IdentityReport,
358 pub distill: DistillReport,
359 pub moved_aside: Vec<MovedAsideReport>,
360 #[serde(flatten)]
361 pub value: ValueReport,
362 pub last_error: Option<ErrorRecord>,
363}
364
365struct Facts {
367 started: DateTime<Utc>,
368 capture: CaptureReport,
369 watcher_error: Option<String>,
370 ingest_error: Option<Streak>,
371 index: IndexReport,
372 drain_error: Option<Streak>,
373 indexes_error: Option<String>,
374 repair_error: Option<String>,
375 queue: QueueReport,
376 maintenance: MaintenanceReport,
377 encryption: EncryptionReport,
378 backfill: BackfillReport,
379 sync: SyncReport,
380 identity: IdentityReport,
381 distill: DistillReport,
382 progress_error: Option<Streak>,
383 value: ValueFacts,
384 windows: Windows,
385 last_error: Option<ErrorRecord>,
386 clock_held_for: Option<std::time::Duration>,
387 clock_rejected: Option<ClockRejection>,
388 clock_basis_ahead: bool,
389 floor: Option<FloorReport>,
390 snapshot_merge: Option<MergeStatus>,
391 moved_aside: Vec<MovedAside>,
392}
393
394struct ClockRejection {
395 machine_id: String,
396 reason: String,
397 at: DateTime<Utc>,
398}
399
400struct Streak {
402 message: String,
403 since: DateTime<Utc>,
404}
405
406impl Streak {
407 fn extend(previous: Option<Streak>, message: String, now: DateTime<Utc>) -> Option<Streak> {
408 let since = previous.map_or(now, |p| p.since);
409 Some(Streak { message, since })
410 }
411
412 fn outlasted_grace(&self, now: DateTime<Utc>) -> bool {
413 now - self.since >= ERROR_GRACE
414 }
415}
416
417struct Inner {
418 facts: Facts,
419 report: HealthReport,
420 sent: Option<State>,
422}
423
424pub struct Health {
425 inner: Mutex<Inner>,
426 transitions: Option<UnboundedSender<Transition>>,
427}
428
429impl Health {
430 pub fn new(now: DateTime<Utc>, transitions: Option<UnboundedSender<Transition>>) -> Self {
433 let mut facts = Facts {
434 started: now,
435 capture: CaptureReport::default(),
436 watcher_error: None,
437 ingest_error: None,
438 index: IndexReport::default(),
439 drain_error: None,
440 indexes_error: None,
441 repair_error: None,
442 queue: QueueReport::default(),
443 maintenance: MaintenanceReport::default(),
444 encryption: EncryptionReport::default(),
445 backfill: BackfillReport::default(),
446 sync: SyncReport::default(),
447 identity: IdentityReport::default(),
448 distill: DistillReport::default(),
449 progress_error: None,
450 value: ValueFacts::default(),
451 windows: Windows::default(),
452 last_error: None,
453 clock_held_for: None,
454 clock_rejected: None,
455 clock_basis_ahead: false,
456 floor: None,
457 snapshot_merge: None,
458 moved_aside: Vec::new(),
459 };
460 let report = HealthReport {
461 state: State::Ok,
462 reason: None,
463 since: now,
464 serving: Verdict {
465 state: State::Ok,
466 reason: None,
467 },
468 capture: CaptureReport::default(),
469 index: IndexReport::default(),
470 queue: QueueReport::default(),
471 maintenance: MaintenanceReport::default(),
472 encryption: EncryptionReport::default(),
473 backfill: BackfillReport::default(),
474 sync: SyncReport::default(),
475 identity: IdentityReport::default(),
476 distill: DistillReport::default(),
477 moved_aside: Vec::new(),
478 value: facts.value.refresh(now, now, facts.windows.sync_overdue).0,
479 last_error: None,
480 };
481 Self {
482 inner: Mutex::new(Inner {
483 facts,
484 report,
485 sent: None,
486 }),
487 transitions,
488 }
489 }
490
491 pub fn with_windows(self, windows: Windows) -> Self {
492 self.lock().facts.windows = windows;
493 self
494 }
495
496 pub fn progress_checked(&self, seen: ProgressSeen, now: DateTime<Utc>) {
497 self.update(now, |f| {
498 if let Some(sync) = seen.sync {
499 f.sync = SyncReport {
500 checked_at: Some(now),
501 peers: sync.peers.or(f.sync.peers),
502 ..sync
503 };
504 }
505 if let Some(distill) = seen.distill {
506 f.distill = DistillReport {
507 checked_at: Some(now),
508 note: distill.mode.note(),
509 ..distill
510 };
511 }
512 f.value.sessions_ingested(seen.ingested_sessions);
513 match seen.error {
514 None => f.progress_error = None,
515 Some(error) => {
516 f.record_error("progress", &error, now);
517 f.progress_error = Streak::extend(f.progress_error.take(), error, now);
518 }
519 }
520 });
521 }
522
523 pub fn announce(&self, now: DateTime<Utc>) {
526 let mut inner = self.lock();
527 self.reassess(&mut inner, now, true);
528 }
529
530 pub fn report(&self) -> HealthReport {
531 self.lock().report.clone()
532 }
533
534 pub fn transcripts_scanned(&self, newest: Option<DateTime<Utc>>, now: DateTime<Utc>) {
535 self.update(now, |f| {
536 f.capture.checked_at = Some(now);
537 f.capture.newest_transcript_at = newest;
538 });
539 }
540
541 pub fn watcher_started(&self, now: DateTime<Utc>) {
542 self.update(now, |f| {
543 f.watcher_error = None;
544 f.capture.checked_at = Some(now);
545 });
546 }
547
548 pub fn watcher_failed(&self, error: &str, now: DateTime<Utc>) {
549 self.update(now, |f| {
550 f.watcher_error = Some(error.to_string());
551 f.record_error("watcher", error, now);
552 });
553 }
554
555 pub fn ingested(&self, transcript_mtime: Option<DateTime<Utc>>, now: DateTime<Utc>) {
558 self.update(now, |f| {
559 f.ingest_error = None;
560 f.capture.checked_at = Some(now);
561 if let Some(mtime) = transcript_mtime {
562 let through = f.capture.captured_through.map_or(mtime, |t| t.max(mtime));
563 f.capture.captured_through = Some(through);
564 }
565 });
566 }
567
568 pub fn ingest_failed(&self, error: &str, now: DateTime<Utc>) {
569 self.update(now, |f| {
570 f.ingest_error = Streak::extend(f.ingest_error.take(), error.to_string(), now);
571 f.capture.checked_at = Some(now);
572 f.record_error("ingest", error, now);
573 });
574 }
575
576 pub fn drained(&self, outbox_pending: u64, now: DateTime<Utc>) {
577 self.update(now, |f| {
578 f.drain_error = None;
579 f.index.checked_at = Some(now);
580 f.index.outbox_pending = Some(outbox_pending);
581 f.index.backlog_since = if outbox_pending == 0 {
582 None
583 } else {
584 Some(f.index.backlog_since.unwrap_or(now))
585 };
586 });
587 }
588
589 pub fn drain_failed(&self, error: &str, now: DateTime<Utc>) {
590 self.update(now, |f| {
591 f.index.checked_at = Some(now);
592 f.drain_error = Streak::extend(f.drain_error.take(), error.to_string(), now);
593 f.record_error("drain", error, now);
594 });
595 }
596
597 pub fn indexes_unavailable(&self, error: &str, now: DateTime<Utc>) {
599 self.update(now, |f| {
600 f.indexes_error = Some(error.to_string());
601 f.record_error("index", error, now);
602 });
603 }
604
605 pub fn repaired(&self, now: DateTime<Utc>) {
606 self.update(now, |f| f.repair_error = None);
607 }
608
609 pub fn repair_failed(&self, error: &str, now: DateTime<Utc>) {
610 self.update(now, |f| {
611 f.repair_error = Some(error.to_string());
612 f.record_error("repair", error, now);
613 });
614 }
615
616 pub fn queue_checked(&self, depth: u64, oldest_at: Option<DateTime<Utc>>, now: DateTime<Utc>) {
617 self.update(now, |f| {
618 f.queue.depth = depth;
619 f.queue.oldest_at = oldest_at;
620 });
621 }
622
623 pub fn write_rejected(&self, tool: &str, status: u16, now: DateTime<Utc>) {
624 self.update(now, |f| {
625 f.queue.last_rejected = Some(RejectedWrite {
626 tool: tool.to_string(),
627 status,
628 at: now,
629 });
630 });
631 }
632
633 pub fn maintenance(
635 &self,
636 state: MaintenanceState,
637 space: Option<VacuumSpace>,
638 now: DateTime<Utc>,
639 ) {
640 self.update(now, |f| {
641 if f.maintenance.state != state {
642 f.maintenance.since = Some(now);
643 }
644 f.maintenance.state = state;
645 if space.is_some() {
646 f.maintenance.space = space;
647 }
648 if state == MaintenanceState::Done {
649 f.maintenance.last_error = None;
650 }
651 });
652 }
653
654 pub fn encryption(
655 &self,
656 state: EncryptionState,
657 space: Option<VacuumSpace>,
658 now: DateTime<Utc>,
659 ) {
660 self.update(now, |f| {
661 if f.encryption.state != state {
662 f.encryption.since = Some(now);
663 }
664 f.encryption.state = state;
665 if space.is_some() {
666 f.encryption.space = space;
667 }
668 if state == EncryptionState::Encrypted {
669 f.encryption.last_error = None;
670 }
671 });
672 }
673
674 pub fn encryption_migrated(&self, pause_ms: u64, notice: bool, now: DateTime<Utc>) {
677 self.update(now, |f| {
678 f.encryption.migration_pause_ms = Some(pause_ms);
679 f.encryption.notice = notice.then(|| PLAINTEXT_LEFT_NOTICE.to_string());
680 });
681 }
682
683 pub fn encryption_failed(&self, error: &str, now: DateTime<Utc>) {
685 self.update(now, |f| {
686 f.encryption.last_error = Some(error.to_string());
687 f.record_error("encryption", error, now);
688 });
689 }
690
691 pub fn maintenance_failed(&self, error: &str, now: DateTime<Utc>) {
693 self.update(now, |f| {
694 f.maintenance.last_error = Some(error.to_string());
695 f.record_error("maintenance", error, now);
696 });
697 }
698
699 pub fn clock_gate(&self, held_for: Option<std::time::Duration>, now: DateTime<Utc>) {
702 self.update(now, |f| f.clock_held_for = held_for);
703 }
704
705 pub fn clock_handshake_rejected(&self, machine_id: &str, reason: String, now: DateTime<Utc>) {
706 self.update(now, |f| {
707 f.record_error("clock", &reason, now);
708 f.clock_rejected = Some(ClockRejection {
709 machine_id: machine_id.to_string(),
710 reason,
711 at: now,
712 });
713 });
714 }
715
716 pub fn clock_handshake_accepted(&self, machine_id: &str, now: DateTime<Utc>) {
719 self.update(now, |f| {
720 if f.clock_rejected
721 .as_ref()
722 .is_some_and(|r| r.machine_id == machine_id)
723 {
724 f.clock_rejected = None;
725 }
726 });
727 }
728
729 pub fn clock_basis_ahead(&self, ahead: bool, now: DateTime<Utc>) {
730 self.update(now, |f| f.clock_basis_ahead = ahead);
731 }
732
733 pub fn snapshot_merge_checked(&self, merge: Option<MergeStatus>, now: DateTime<Utc>) {
734 self.update(now, |f| f.snapshot_merge = merge);
735 }
736
737 pub fn floor_checked(&self, floor: FloorHealth, failed: FailedCounts, now: DateTime<Utc>) {
738 self.update(now, |f| {
739 f.floor = Some(FloorReport {
740 floor: floor.floor,
741 log_floor: floor.log_floor,
742 head: floor.head,
743 top: floor.top,
744 silent: floor.silent,
745 stuck: floor.stuck,
746 failing: failed.failing,
747 settled: failed.settled,
748 settled_dropped: failed.dropped,
749 });
750 });
751 }
752
753 pub fn key_storage_detected(&self, name: &'static str, now: DateTime<Utc>) {
754 self.update(now, |f| f.identity.key_storage = Some(name));
755 }
756
757 pub fn hardware_checked(&self, state: &'static str, now: DateTime<Utc>) {
758 self.update(now, |f| f.identity.hardware = Some(state));
759 }
760
761 pub fn identity_checked(&self, status: IdentityStatus, now: DateTime<Utc>) {
762 self.update(now, |f| {
763 f.identity.checked_at = Some(now);
764 f.identity.status = Some(status);
765 });
766 }
767
768 pub fn recalled(&self, event: RecallEvent, now: DateTime<Utc>) {
769 self.update(now, |f| f.value.recalled(event, now));
770 }
771
772 pub fn memory_corrected(&self, id: Uuid, now: DateTime<Utc>) {
773 self.update(now, |f| f.value.corrected(id));
774 }
775
776 pub fn memories_used(&self, ids: &[Uuid], now: DateTime<Utc>) {
777 self.update(now, |f| f.value.used(ids));
778 }
779
780 pub fn sync_peers_listed(&self, peers: &[String], now: DateTime<Utc>) {
781 self.update(now, |f| f.value.peers_listed(peers, now));
782 }
783
784 pub fn sync_attempted(&self, peer: &str, outcome: PeerOutcome, now: DateTime<Utc>) {
785 self.update(now, |f| f.value.sync_attempted(peer, outcome, now));
786 }
787
788 pub fn store_measured(&self, size: Option<StoreSize>, now: DateTime<Utc>) {
790 self.update(now, |f| f.value.store_measured(size));
791 }
792
793 pub fn resident_measured(&self, size: Option<ResidentSize>, now: DateTime<Utc>) {
795 self.update(now, |f| f.value.resident_measured(size));
796 }
797
798 pub fn ingest_window_start(&self, now: DateTime<Utc>) -> Option<DateTime<Utc>> {
803 let inner = self.lock();
804 let f = &inner.facts;
805 let mut start = (now - WINDOW).max(f.started);
806 match f.backfill.state {
807 BackfillState::Running => return None,
808 BackfillState::Done => {
809 if let Some(end) = f.backfill.updated_at {
810 start = start.max(end);
811 }
812 }
813 BackfillState::NotStarted => {}
814 }
815 Some(start)
816 }
817
818 pub fn moved_aside_listed(&self, asides: Vec<MovedAside>, now: DateTime<Utc>) {
819 self.update(now, |f| f.moved_aside = asides);
820 }
821
822 pub fn backfill_progress(&self, report: BackfillReport) {
825 let mut inner = self.lock();
826 inner.facts.backfill = report.clone();
827 inner.report.backfill = report;
828 }
829
830 fn lock(&self) -> std::sync::MutexGuard<'_, Inner> {
831 match self.inner.lock() {
832 Ok(g) => g,
833 Err(poisoned) => poisoned.into_inner(),
834 }
835 }
836
837 fn update(&self, now: DateTime<Utc>, change: impl FnOnce(&mut Facts)) {
838 let mut inner = self.lock();
839 change(&mut inner.facts);
840 self.reassess(&mut inner, now, false);
841 }
842
843 fn reassess(&self, inner: &mut Inner, now: DateTime<Utc>, announcing: bool) {
845 let (started, sync_overdue) = (inner.facts.started, inner.facts.windows.sync_overdue);
846 let (value, value_problems) = inner.facts.value.refresh(now, started, sync_overdue);
847 let mut serving = Assessment::default();
848 assess_serving(&inner.facts, now, &mut serving);
849 let mut overall = serving.clone();
850 assess_upkeep(&inner.facts, now, &mut overall, value_problems);
851 let (state, reason) = overall.verdict();
852 let changed = state != inner.report.state;
853 let since = if changed { now } else { inner.report.since };
854 let mut capture = inner.facts.capture.clone();
855 capture.lag_secs = capture_lag(&inner.facts).map(|d| d.num_seconds());
856 let mut queue = inner.facts.queue.clone();
857 queue.oldest_age_secs = queue.oldest_at.map(|t| (now - t).num_seconds().max(0));
858 inner.report = HealthReport {
859 state,
860 reason: reason.clone(),
861 since,
862 serving: Verdict {
863 state: serving.state,
864 reason: serving.verdict().1,
865 },
866 capture,
867 index: inner.facts.index.clone(),
868 queue,
869 maintenance: inner.facts.maintenance.clone(),
870 encryption: inner.facts.encryption.clone(),
871 backfill: inner.facts.backfill.clone(),
872 sync: SyncReport {
873 floor: inner.facts.floor.clone(),
874 snapshot_merge: inner.facts.snapshot_merge.clone(),
875 ..inner.facts.sync.clone()
876 },
877 identity: inner.facts.identity.clone(),
878 distill: inner.facts.distill.clone(),
879 moved_aside: moved_aside_report(&inner.facts.moved_aside, now),
880 value,
881 last_error: inner.facts.last_error.clone(),
882 };
883 let unsent = inner.sent != Some(state);
886 if (changed || (announcing && unsent))
887 && let Some(tx) = &self.transitions
888 {
889 let _ = tx.send(Transition {
891 state,
892 reason,
893 at: now,
894 });
895 inner.sent = Some(state);
896 }
897 }
898}
899
900impl Facts {
901 fn record_error(&mut self, source: &'static str, message: &str, at: DateTime<Utc>) {
902 self.last_error = Some(ErrorRecord {
903 source,
904 message: message.to_string(),
905 at,
906 });
907 }
908}
909
910fn capture_lag(f: &Facts) -> Option<Duration> {
913 let newest = f.capture.newest_transcript_at?;
914 let through = f
915 .capture
916 .captured_through
917 .map_or(f.started, |t| t.max(f.started));
918 Some((newest - through).max(Duration::zero()))
919}
920
921#[derive(Clone, Default)]
922struct Assessment {
923 state: State,
924 problems: Vec<String>,
925}
926
927impl Assessment {
928 fn flag(&mut self, level: State, problem: String) {
929 self.state = self.state.max(level);
930 self.problems.push(problem);
931 }
932
933 fn verdict(&self) -> (State, Option<String>) {
934 let reason = (!self.problems.is_empty()).then(|| self.problems.join("; "));
935 (self.state, reason)
936 }
937}
938
939fn assess_serving(f: &Facts, now: DateTime<Utc>, a: &mut Assessment) {
940 let mut flag = |level: State, problem: String| a.flag(level, problem);
941
942 if f.clock_held_for
943 .is_some_and(|held| held >= CLOCK_HELD_GRACE)
944 {
945 flag(
946 State::Degraded,
947 "clock not yet trusted: writes held".to_string(),
948 );
949 }
950
951 match &f.watcher_error {
952 Some(e) => flag(
953 State::Failing,
954 format!("transcript watcher is not running: {e}"),
955 ),
956 None => {
957 let last = f.capture.checked_at.unwrap_or(f.started);
958 if now - last >= WATCHER_SILENT {
959 flag(
960 State::Degraded,
961 format!(
962 "transcript watcher has not reported for {}",
963 human(now - last)
964 ),
965 );
966 }
967 }
968 }
969 if let Some(lag) = capture_lag(f)
970 && lag >= CAPTURE_LAG_DEGRADED
971 {
972 let level = if lag >= CAPTURE_LAG_FAILING {
973 State::Failing
974 } else {
975 State::Degraded
976 };
977 flag(
978 level,
979 format!("capture is {} behind the newest transcript", human(lag)),
980 );
981 }
982 if let Some(e) = &f.ingest_error
983 && e.outlasted_grace(now)
984 {
985 flag(
986 State::Degraded,
987 format!(
988 "ingest has failed for {}: {}",
989 human(now - e.since),
990 e.message
991 ),
992 );
993 }
994
995 if let Some(e) = &f.drain_error
996 && e.outlasted_grace(now)
997 {
998 flag(
999 State::Degraded,
1000 format!(
1001 "index drain failed for {}: {}",
1002 human(now - e.since),
1003 e.message
1004 ),
1005 );
1006 }
1007 match &f.indexes_error {
1008 Some(e) => flag(State::Degraded, format!("search indexes unavailable: {e}")),
1009 None => {
1010 let last = f.index.checked_at.unwrap_or(f.started);
1011 if now - last >= DRAIN_SILENT {
1012 flag(
1013 State::Degraded,
1014 format!("index drain has not reported for {}", human(now - last)),
1015 );
1016 }
1017 }
1018 }
1019 if let (Some(since), Some(pending)) = (f.index.backlog_since, f.index.outbox_pending) {
1020 let age = now - since;
1021 if age >= BACKLOG_DEGRADED {
1022 let level = if age >= BACKLOG_FAILING {
1023 State::Failing
1024 } else {
1025 State::Degraded
1026 };
1027 flag(
1028 level,
1029 format!("{pending} changes have waited {} to be indexed", human(age)),
1030 );
1031 }
1032 }
1033 if let Some(e) = &f.repair_error {
1034 flag(State::Degraded, format!("last store repair failed: {e}"));
1035 }
1036 if let Some(oldest) = f.queue.oldest_at
1037 && now - oldest >= QUEUE_STUCK
1038 {
1039 flag(
1040 State::Degraded,
1041 format!(
1042 "write queue not draining ({} queued, oldest {})",
1043 f.queue.depth,
1044 human(now - oldest)
1045 ),
1046 );
1047 }
1048
1049 match (&f.maintenance.state, &f.maintenance.space) {
1050 (MaintenanceState::Vacuuming, _) => flag(
1051 State::Degraded,
1052 "store maintenance is running; writes are queued until it finishes".to_string(),
1053 ),
1054 (MaintenanceState::WaitingForSpace, Some(space)) => flag(
1055 State::Degraded,
1056 format!(
1057 "store compaction waits for disk space: it needs {} MB free at {} and has {} MB",
1058 space.needed_bytes / 1_000_000,
1059 space.path.display(),
1060 space.free_bytes / 1_000_000
1061 ),
1062 ),
1063 _ => {}
1064 }
1065 if let Some(e) = &f.maintenance.last_error {
1066 flag(State::Degraded, format!("store compaction failed: {e}"));
1067 }
1068 match (&f.encryption.state, &f.encryption.space) {
1069 (EncryptionState::Encrypting | EncryptionState::Rotating, _) => flag(
1070 State::Degraded,
1071 "the store is being rewritten under its key; writes are queued until it finishes"
1072 .to_string(),
1073 ),
1074 (EncryptionState::WaitingForSpace, Some(space)) => flag(
1075 State::Degraded,
1076 format!(
1077 "encryption pending, needs {} MB free at {} and has {} MB",
1078 space.needed_bytes / 1_000_000,
1079 space.path.display(),
1080 space.free_bytes / 1_000_000
1081 ),
1082 ),
1083 _ => {}
1084 }
1085 if let Some(e) = &f.encryption.last_error {
1086 flag(State::Degraded, format!("store encryption failed: {e}"));
1087 }
1088}
1089
1090fn identity_problems(
1091 identity: &IdentityStatus,
1092 peers: Option<usize>,
1093 reminted: bool,
1094 flag: &mut impl FnMut(State, String),
1095) {
1096 if identity.identity.is_none() {
1097 if peers.is_some_and(|n| n > 0) {
1098 let why = if reminted {
1099 "this data dir was copied from other hardware, so this device took a new machine \
1100 id and new device keys and holds no sync identity key"
1101 } else {
1102 "this device holds no sync identity key, so it cannot sync with its peers"
1103 };
1104 flag(
1105 State::Degraded,
1106 format!(
1107 "{why}; run `sb pair` on a device that syncs, then `sb pair <code>` on this \
1108 device"
1109 ),
1110 );
1111 }
1112 return;
1113 }
1114 if identity.removed {
1115 flag(
1116 State::Degraded,
1117 "this device was removed from its sync identity, so its peers refuse it; join \
1118 another device's identity with `sb pair <code>` on this device"
1119 .to_string(),
1120 );
1121 } else if identity.possibly_revoked {
1122 let withheld: Vec<String> = identity
1123 .peers
1124 .iter()
1125 .filter(|p| p.holds_own_row && !p.lacking.is_empty())
1126 .map(|p| {
1127 let numbers: Vec<String> = p.lacking.iter().map(u32::to_string).collect();
1128 let noun = if p.lacking.len() == 1 {
1129 "epoch"
1130 } else {
1131 "epochs"
1132 };
1133 format!(
1134 "{} holds identity key {noun} {}",
1135 p.peer,
1136 numbers.join(", ")
1137 )
1138 })
1139 .collect();
1140 flag(
1141 State::Degraded,
1142 format!(
1143 "{} but did not wrap it to this device, which may have been removed from its \
1144 sync identity; run `sb device list` on a device that syncs",
1145 withheld.join("; ")
1146 ),
1147 );
1148 }
1149 if identity.row_accepted == Some(false) {
1150 flag(
1151 State::Degraded,
1152 "no peer has accepted this device's row, so none wraps it a newer identity key; \
1153 pair it with `sb pair` from a device that syncs"
1154 .to_string(),
1155 );
1156 }
1157}
1158
1159fn assess_upkeep(f: &Facts, now: DateTime<Utc>, a: &mut Assessment, value: ValueProblems) {
1162 let mut flag = |level: State, problem: String| a.flag(level, problem);
1163 let mut no_sync_at_all = false;
1164 let sync = &f.sync;
1165 if (sync.peers.is_some_and(|n| n > 0) || sync.synced_before)
1166 && let Some(window) = f.windows.sync_overdue
1167 {
1168 let since = sync
1169 .last_success_at
1170 .or(sync.last_cursor_at)
1171 .unwrap_or(f.started);
1172 let overdue = now - since;
1173 if overdue >= window {
1174 let problem = match sync.peers {
1175 Some(0) => format!(
1176 "no sync has succeeded in {}, and this device has no paired device to sync \
1177 with although it has synced before; pair one with `sb pair`, \
1178 or set alert.sync_overdue_hours = 0 if this device no longer syncs",
1179 human(overdue)
1180 ),
1181 Some(n) => format!(
1182 "no sync has succeeded in {} with the {} this device syncs with; \
1183 run `sb doctor` for what to check",
1184 human(overdue),
1185 counted(n as u64, "device", "devices")
1186 ),
1187 None => format!(
1188 "no sync has succeeded in {} although this device has synced before; \
1189 run `sb sync <user@host>` (or check the job that runs it), \
1190 or set alert.sync_overdue_hours = 0 if this device no longer syncs",
1191 human(overdue)
1192 ),
1193 };
1194 flag(State::Degraded, problem);
1195 no_sync_at_all = true;
1196 }
1197 }
1198 if !no_sync_at_all {
1199 for problem in value.peers {
1200 flag(State::Degraded, problem);
1201 }
1202 }
1203 for problem in value.other {
1204 flag(State::Degraded, problem);
1205 }
1206 if f.distill.mode == DistillMode::Llm
1207 && let (Some(oldest), Some(window)) = (f.distill.oldest_at, f.windows.undistilled)
1208 && now - oldest >= window
1209 {
1210 flag(
1211 State::Degraded,
1212 format!(
1213 "{} waited {} to be distilled; run `sb consolidate --llm-url <endpoint>` or add an [llm] section, or set \
1214 alert.undistilled_overdue_hours = 0 if this device does not distill",
1215 if f.distill.pending == 1 {
1216 "1 raw memory has".to_string()
1217 } else {
1218 format!("{} raw memories have", f.distill.pending)
1219 },
1220 human(now - oldest)
1221 ),
1222 );
1223 }
1224 if let Some(e) = &f.progress_error
1225 && e.outlasted_grace(now)
1226 {
1227 flag(
1228 State::Degraded,
1229 format!(
1230 "cannot check sync and distillation for {}: {}",
1231 human(now - e.since),
1232 e.message
1233 ),
1234 );
1235 }
1236 if let Some(rejection) = &f.clock_rejected
1237 && now - rejection.at < CLOCK_REJECTION_SHOWN
1238 {
1239 flag(
1240 State::Degraded,
1241 format!(
1242 "a sync handshake was refused {} ago: {}",
1243 human(now - rejection.at),
1244 rejection.reason
1245 ),
1246 );
1247 }
1248 if f.clock_basis_ahead {
1249 flag(
1250 State::Degraded,
1251 "this store's clock basis is ahead of synchronised time; new records count up \
1252 from the last one until real time catches up"
1253 .to_string(),
1254 );
1255 }
1256 if let Some(identity) = &f.identity.status {
1257 let reminted = f.identity.hardware == Some("reminted");
1258 identity_problems(identity, f.sync.peers, reminted, &mut flag);
1259 }
1260 if let Some(floor) = &f.floor {
1261 for id in &floor.silent {
1262 flag(
1263 State::Degraded,
1264 format!(
1265 "sync neighbour {id} has been silent for 30 days and holds the log floor back"
1266 ),
1267 );
1268 }
1269 for id in &floor.stuck {
1270 flag(
1271 State::Degraded,
1272 format!("sync neighbour {id} keeps acking but its ack has not moved in 7 days"),
1273 );
1274 }
1275 }
1276 if let Some(MergeStatus {
1277 merging,
1278 failing: Some(failing),
1279 }) = &f.snapshot_merge
1280 {
1281 flag(
1282 State::Degraded,
1283 format!(
1284 "the snapshot merge from {} through seq {} has failed {} times: {}",
1285 merging.exporter, merging.seq, failing.attempts, failing.error
1286 ),
1287 );
1288 }
1289}
1290
1291pub(crate) fn counted(n: u64, one: &str, many: &str) -> String {
1292 if n == 1 {
1293 format!("1 {one}")
1294 } else {
1295 format!("{n} {many}")
1296 }
1297}
1298
1299pub(crate) fn human(d: Duration) -> String {
1300 let minutes = d.num_minutes();
1301 if minutes < 60 {
1302 format!("{minutes}m")
1303 } else if minutes < 48 * 60 {
1304 format!("{}h {}m", minutes / 60, minutes % 60)
1305 } else {
1306 format!("{}d", minutes / (24 * 60))
1307 }
1308}
1309
1310#[cfg(test)]
1311mod tests {
1312 use tokio::sync::mpsc::{UnboundedReceiver, unbounded_channel};
1313
1314 use second_brain_sync::keyring::PeerStanding;
1315
1316 use super::*;
1317
1318 fn t0() -> DateTime<Utc> {
1319 DateTime::parse_from_rfc3339("2026-10-31T12:00:00Z")
1320 .unwrap()
1321 .with_timezone(&Utc)
1322 }
1323
1324 fn mins(m: i64) -> DateTime<Utc> {
1325 t0() + Duration::minutes(m)
1326 }
1327
1328 fn monitored() -> (Health, UnboundedReceiver<Transition>) {
1329 let (tx, rx) = unbounded_channel();
1330 (Health::new(t0(), Some(tx)), rx)
1331 }
1332
1333 fn drain(rx: &mut UnboundedReceiver<Transition>) -> Vec<State> {
1334 std::iter::from_fn(|| rx.try_recv().ok())
1335 .map(|t| t.state)
1336 .collect()
1337 }
1338
1339 #[test]
1340 fn a_fresh_daemon_that_keeps_up_is_ok() {
1341 let (health, mut rx) = monitored();
1342 health.watcher_started(mins(0));
1343 health.transcripts_scanned(Some(mins(1)), mins(1));
1344 health.ingested(Some(mins(1)), mins(1));
1345 health.drained(0, mins(1));
1346 let report = health.report();
1347 assert_eq!(report.state, State::Ok);
1348 assert_eq!(report.reason, None);
1349 assert_eq!(report.capture.lag_secs, Some(0));
1350 assert!(drain(&mut rx).is_empty());
1351 }
1352
1353 #[test]
1354 fn capture_lag_degrades_then_fails_then_recovers() {
1355 let (health, mut rx) = monitored();
1356 health.drained(0, mins(0));
1357 health.ingested(Some(mins(1)), mins(1));
1358 health.transcripts_scanned(Some(mins(10)), mins(10));
1359 assert_eq!(health.report().state, State::Ok);
1360
1361 health.transcripts_scanned(Some(mins(20)), mins(20));
1362 let report = health.report();
1363 assert_eq!(report.state, State::Degraded);
1364 assert_eq!(report.capture.lag_secs, Some(19 * 60));
1365 assert!(report.reason.unwrap().contains("capture is 19m behind"));
1366
1367 health.drained(0, mins(125));
1368 health.transcripts_scanned(Some(mins(125)), mins(125));
1369 assert_eq!(health.report().state, State::Failing);
1370
1371 health.ingested(Some(mins(125)), mins(126));
1372 assert_eq!(health.report().state, State::Ok);
1373 assert_eq!(
1374 drain(&mut rx),
1375 vec![State::Degraded, State::Failing, State::Ok]
1376 );
1377 }
1378
1379 #[test]
1380 fn transcripts_from_before_startup_are_not_lag() {
1381 let (health, _rx) = monitored();
1382 health.transcripts_scanned(Some(t0() - Duration::days(3)), mins(1));
1383 assert_eq!(health.report().capture.lag_secs, Some(0));
1384 assert_eq!(health.report().state, State::Ok);
1385 }
1386
1387 #[test]
1388 fn repeated_reports_in_one_state_send_one_transition() {
1389 let (health, mut rx) = monitored();
1390 for m in 0..=12 {
1391 health.ingest_failed("database is locked", mins(m));
1392 }
1393 health.ingested(None, mins(13));
1394 health.ingested(None, mins(14));
1395 assert_eq!(drain(&mut rx), vec![State::Degraded, State::Ok]);
1396 let report = health.report();
1397 assert_eq!(report.since, mins(13));
1398 let last = report.last_error.expect("the error is kept after recovery");
1399 assert_eq!(last.source, "ingest");
1400 assert_eq!(last.at, mins(12));
1401 }
1402
1403 #[test]
1404 fn an_error_that_clears_within_the_grace_sends_nothing() {
1405 let (health, mut rx) = monitored();
1406 health.watcher_started(mins(0));
1407 health.ingest_failed("database is locked", mins(0));
1408 health.drain_failed("database is locked", mins(1));
1409 health.drained(0, mins(9));
1410 health.ingested(None, mins(9));
1411 assert_eq!(health.report().state, State::Ok);
1412 assert!(drain(&mut rx).is_empty());
1413 assert_eq!(health.report().last_error.unwrap().source, "drain");
1414 }
1415
1416 #[test]
1417 fn a_drain_error_degrades_once_it_outlasts_the_grace() {
1418 let (health, _rx) = monitored();
1419 health.watcher_started(mins(0));
1420 for m in 0..10 {
1421 health.drain_failed("disk I/O error", mins(m));
1422 }
1423 assert_eq!(health.report().state, State::Ok);
1424 health.drain_failed("disk I/O error", mins(10));
1425 let report = health.report();
1426 assert_eq!(report.state, State::Degraded);
1427 assert!(report.reason.unwrap().contains("index drain failed"));
1428 health.drained(0, mins(11));
1429 assert_eq!(health.report().state, State::Ok);
1430 }
1431
1432 #[test]
1433 fn a_long_ingest_batch_counts_as_the_watcher_reporting() {
1434 let (health, _rx) = monitored();
1435 health.watcher_started(mins(0));
1436 health.ingested(None, mins(10));
1437 health.ingest_failed("unreadable line", mins(19));
1438 health.drained(0, mins(20));
1439 let report = health.report();
1440 assert_eq!(report.state, State::Ok, "{:?}", report.reason);
1441 }
1442
1443 #[test]
1444 fn an_outbox_backlog_only_counts_once_it_persists() {
1445 let (health, _rx) = monitored();
1446 health.drained(500, mins(0));
1447 health.drained(800, mins(14));
1448 health.watcher_started(mins(14));
1449 assert_eq!(health.report().state, State::Ok);
1450 health.drained(800, mins(15));
1451 let report = health.report();
1452 assert_eq!(report.state, State::Degraded);
1453 assert_eq!(report.index.backlog_since, Some(mins(0)));
1454 health.drained(0, mins(16));
1455 assert_eq!(health.report().state, State::Ok);
1456 assert_eq!(health.report().index.backlog_since, None);
1457 }
1458
1459 #[test]
1460 fn a_dead_watcher_fails_and_a_silent_one_degrades() {
1461 let (health, _rx) = monitored();
1462 health.watcher_failed("inotify watch limit reached", mins(0));
1463 assert_eq!(health.report().state, State::Failing);
1464 health.watcher_started(mins(1));
1465 health.drained(0, mins(1));
1466 assert_eq!(health.report().state, State::Ok);
1467 health.drained(0, mins(16));
1468 let report = health.report();
1469 assert_eq!(report.state, State::Degraded);
1470 assert!(report.reason.unwrap().contains("watcher has not reported"));
1471 }
1472
1473 #[test]
1474 fn index_and_repair_errors_degrade_until_the_next_success() {
1475 let (health, _rx) = monitored();
1476 health.watcher_started(mins(0));
1477 health.drain_failed("disk I/O error", mins(0));
1478 health.repair_failed("disk I/O error", mins(0));
1479 let reason = health.report().reason.unwrap();
1480 assert!(reason.contains("store repair failed"), "{reason}");
1481 health.drain_failed("disk I/O error", mins(10));
1482 let reason = health.report().reason.unwrap();
1483 assert!(reason.contains("index drain failed"), "{reason}");
1484 health.drained(0, mins(11));
1485 assert_eq!(health.report().state, State::Degraded);
1486 health.repaired(mins(12));
1487 assert_eq!(health.report().state, State::Ok);
1488 }
1489
1490 #[test]
1491 fn missing_indexes_stay_degraded_without_a_drain() {
1492 let (health, _rx) = monitored();
1493 health.indexes_unavailable("tantivy meta.json missing", mins(0));
1494 health.watcher_started(mins(90));
1495 let report = health.report();
1496 assert_eq!(report.state, State::Degraded);
1497 let reason = report.reason.unwrap();
1498 assert!(reason.contains("search indexes unavailable"), "{reason}");
1499 assert!(!reason.contains("has not reported"), "{reason}");
1500 }
1501
1502 #[test]
1503 fn the_report_serializes_without_content() {
1504 let (health, _rx) = monitored();
1505 health.ingest_failed("database is locked", mins(0));
1506 health.ingest_failed("database is locked", mins(10));
1507 let json = serde_json::to_value(health.report()).unwrap();
1508 assert_eq!(json["state"], "degraded");
1509 assert_eq!(json["last_error"]["source"], "ingest");
1510 for key in ["reason", "since", "capture", "index"] {
1511 assert!(json.get(key).is_some(), "{key} missing: {json}");
1512 }
1513 }
1514
1515 #[test]
1516 fn a_queue_entry_older_than_ten_minutes_degrades_until_it_drains() {
1517 let (health, mut rx) = monitored();
1518 health.watcher_started(mins(0));
1519 health.drained(0, mins(0));
1520 health.queue_checked(2, Some(mins(0)), mins(9));
1521 assert_eq!(health.report().state, State::Ok);
1522 assert_eq!(health.report().queue.depth, 2);
1523 health.queue_checked(2, Some(mins(0)), mins(10));
1524 let report = health.report();
1525 assert_eq!(report.state, State::Degraded);
1526 assert!(
1527 report
1528 .reason
1529 .as_deref()
1530 .unwrap()
1531 .contains("write queue not draining"),
1532 "{report:?}"
1533 );
1534 assert_eq!(report.queue.oldest_age_secs, Some(600));
1535 health.queue_checked(0, None, mins(11));
1536 assert_eq!(health.report().state, State::Ok);
1537 assert_eq!(drain(&mut rx), vec![State::Degraded, State::Ok]);
1538 }
1539
1540 #[test]
1541 fn a_rejected_write_is_reported_by_tool_status_and_time_only() {
1542 let (health, _rx) = monitored();
1543 health.write_rejected("associate", 404, mins(3));
1544 let json = serde_json::to_value(health.report()).unwrap();
1545 assert_eq!(
1546 json["queue"]["last_rejected"],
1547 serde_json::json!({"tool": "associate", "status": 404, "at": "2026-10-31T12:03:00Z"})
1548 );
1549 assert_eq!(json["state"], "ok");
1550 }
1551
1552 fn hours(h: i64) -> DateTime<Utc> {
1553 t0() + Duration::hours(h)
1554 }
1555
1556 fn alive(health: &Health, at: DateTime<Utc>) {
1557 health.update(at, |f| {
1558 f.capture.checked_at = Some(at);
1559 f.index.checked_at = Some(at);
1560 });
1561 }
1562
1563 fn sync_seen(health: &Health, sync: SyncReport, at: DateTime<Utc>) {
1564 let seen = ProgressSeen {
1565 sync: Some(sync),
1566 ..ProgressSeen::default()
1567 };
1568 health.progress_checked(seen, at);
1569 }
1570
1571 fn distill_seen(
1572 health: &Health,
1573 pending: u64,
1574 oldest_at: Option<DateTime<Utc>>,
1575 at: DateTime<Utc>,
1576 ) {
1577 distill_seen_in(health, DistillMode::Llm, pending, oldest_at, at);
1578 }
1579
1580 fn distill_seen_in(
1581 health: &Health,
1582 mode: DistillMode,
1583 pending: u64,
1584 oldest_at: Option<DateTime<Utc>>,
1585 at: DateTime<Utc>,
1586 ) {
1587 let distill = DistillReport {
1588 mode,
1589 pending,
1590 oldest_at,
1591 ..DistillReport::default()
1592 };
1593 let seen = ProgressSeen {
1594 distill: Some(distill),
1595 ..ProgressSeen::default()
1596 };
1597 health.progress_checked(seen, at);
1598 }
1599
1600 fn synced(peers: usize, synced_before: bool, last: Option<DateTime<Utc>>) -> SyncReport {
1601 SyncReport {
1602 checked_at: None,
1603 peers: Some(peers),
1604 synced_before,
1605 last_success_at: last,
1606 last_cursor_at: None,
1607 floor: None,
1608 snapshot_merge: None,
1609 }
1610 }
1611
1612 #[test]
1613 fn without_a_stamp_the_window_counts_from_when_records_last_moved() {
1614 let (health, _rx) = monitored();
1615 let seen = SyncReport {
1616 last_cursor_at: Some(hours(-48)),
1617 ..synced(0, true, None)
1618 };
1619 sync_seen(&health, seen.clone(), mins(0));
1620 let report = health.report();
1621 assert_eq!(report.state, State::Degraded);
1622 let reason = report.reason.unwrap();
1623 assert!(reason.contains("no sync has succeeded in 2d"), "{reason}");
1624
1625 let stamped = SyncReport {
1626 last_success_at: Some(mins(0)),
1627 ..seen
1628 };
1629 sync_seen(&health, stamped, mins(0));
1630 assert_eq!(health.report().state, State::Ok);
1631 }
1632
1633 #[test]
1634 fn reasons_count_one_peer_and_one_raw_in_the_singular() {
1635 let (health, _rx) = monitored();
1636 sync_seen(&health, synced(1, true, Some(hours(-30))), mins(0));
1637 distill_seen(&health, 1, Some(hours(-30)), mins(0));
1638 let reason = health.report().reason.unwrap();
1639 assert!(
1640 reason.contains("with the 1 device this device syncs with"),
1641 "{reason}"
1642 );
1643 assert!(reason.contains("1 raw memory has waited"), "{reason}");
1644 }
1645
1646 #[test]
1647 fn a_device_that_synced_before_and_lost_its_peers_degrades_with_the_fix() {
1648 let (health, mut rx) = monitored();
1649 sync_seen(&health, synced(0, true, Some(hours(-25))), mins(0));
1650 let report = health.report();
1651 assert_eq!(report.state, State::Degraded);
1652 let reason = report.reason.unwrap();
1653 assert!(
1654 reason.contains("no sync has succeeded in 25h 0m"),
1655 "{reason}"
1656 );
1657 assert!(reason.contains("no paired device"), "{reason}");
1658 assert!(reason.contains("`sb pair`"), "{reason}");
1659 assert!(reason.contains("sync_overdue_hours = 0"), "{reason}");
1660 assert_eq!(report.sync.peers, Some(0));
1661 assert_eq!(report.sync.checked_at, Some(mins(0)));
1662 assert_eq!(drain(&mut rx), vec![State::Degraded]);
1663
1664 sync_seen(&health, synced(1, true, Some(mins(1))), mins(1));
1665 assert_eq!(health.report().state, State::Ok);
1666 }
1667
1668 #[test]
1669 fn a_device_synced_by_hand_without_sync_peers_stays_ok_while_it_syncs() {
1670 let (health, mut rx) = monitored();
1671 let by_hand = SyncReport {
1672 peers: None,
1673 ..synced(0, true, Some(hours(10)))
1674 };
1675 alive(&health, hours(30));
1676 sync_seen(&health, by_hand.clone(), hours(30));
1677 assert_eq!(health.report().state, State::Ok);
1678 alive(&health, hours(34));
1679 sync_seen(&health, by_hand, hours(34));
1680 let report = health.report();
1681 assert_eq!(report.state, State::Degraded);
1682 let reason = report.reason.unwrap();
1683 assert!(
1684 reason.contains("no sync has succeeded in 24h 0m"),
1685 "{reason}"
1686 );
1687 assert!(reason.contains("sb sync <user@host>"), "{reason}");
1688 assert!(reason.contains("sync_overdue_hours = 0"), "{reason}");
1689 assert!(!reason.contains("sync-peers"), "{reason}");
1690 assert_eq!(drain(&mut rx), vec![State::Degraded]);
1691 }
1692
1693 #[test]
1694 fn a_solo_device_that_never_synced_stays_ok() {
1695 let (health, mut rx) = monitored();
1696 for h in [0, 25, 100] {
1697 alive(&health, hours(h));
1698 sync_seen(&health, synced(0, false, None), hours(h));
1699 }
1700 let report = health.report();
1701 assert_eq!(report.state, State::Ok, "{:?}", report.reason);
1702 assert!(drain(&mut rx).is_empty());
1703 }
1704
1705 #[test]
1706 fn a_sync_overdue_past_the_window_degrades_only_on_a_device_that_syncs() {
1707 let (health, _rx) = monitored();
1708 alive(&health, hours(0));
1709 sync_seen(&health, synced(2, true, Some(hours(-23))), hours(0));
1710 assert_eq!(health.report().state, State::Ok);
1711
1712 alive(&health, hours(1));
1713 sync_seen(&health, synced(2, true, Some(hours(-23))), hours(1));
1714 let report = health.report();
1715 assert_eq!(report.state, State::Degraded);
1716 let reason = report.reason.unwrap();
1717 assert!(
1718 reason.contains("no sync has succeeded in 24h 0m"),
1719 "{reason}"
1720 );
1721 assert!(reason.contains("2 devices"), "{reason}");
1722
1723 sync_seen(&health, synced(2, true, Some(hours(1))), hours(1));
1724 assert_eq!(health.report().state, State::Ok);
1725
1726 sync_seen(&health, synced(0, false, Some(hours(-500))), hours(1));
1727 assert_eq!(health.report().state, State::Ok);
1728 }
1729
1730 #[test]
1731 fn with_peers_and_no_recorded_success_the_window_starts_at_startup() {
1732 let (health, _rx) = monitored();
1733 alive(&health, hours(23));
1734 sync_seen(&health, synced(1, false, None), hours(23));
1735 assert_eq!(health.report().state, State::Ok);
1736 alive(&health, hours(24));
1737 sync_seen(&health, synced(1, false, None), hours(24));
1738 let reason = health.report().reason.unwrap();
1739 assert!(
1740 reason.contains("no sync has succeeded in 24h 0m"),
1741 "{reason}"
1742 );
1743 }
1744
1745 #[test]
1746 fn raws_left_undistilled_past_the_window_degrade_with_the_count() {
1747 let (health, mut rx) = monitored();
1748 alive(&health, hours(0));
1749 distill_seen(&health, 520, Some(hours(-23)), hours(0));
1750 assert_eq!(health.report().state, State::Ok);
1751 assert_eq!(health.report().distill.pending, 520);
1752
1753 alive(&health, hours(2));
1754 distill_seen(&health, 520, Some(hours(-23)), hours(2));
1755 let report = health.report();
1756 assert_eq!(report.state, State::Degraded);
1757 let reason = report.reason.unwrap();
1758 assert!(
1759 reason.contains("520 raw memories have waited 25h 0m to be distilled"),
1760 "{reason}"
1761 );
1762 assert!(reason.contains("undistilled_overdue_hours = 0"), "{reason}");
1763
1764 alive(&health, hours(3));
1765 distill_seen(&health, 0, None, hours(3));
1766 assert_eq!(health.report().state, State::Ok);
1767 assert_eq!(drain(&mut rx), vec![State::Degraded, State::Ok]);
1768 }
1769
1770 #[test]
1771 fn without_an_llm_endpoint_waiting_raws_report_the_mode_instead_of_degrading() {
1772 let (health, mut rx) = monitored();
1773 alive(&health, hours(100));
1774 distill_seen_in(
1775 &health,
1776 DistillMode::NoGpu,
1777 520,
1778 Some(hours(-100)),
1779 hours(100),
1780 );
1781 let report = health.report();
1782 assert_eq!(report.state, State::Ok, "{:?}", report.reason);
1783 assert_eq!(report.reason, None);
1784 assert_eq!(report.distill.pending, 520);
1785 let json = serde_json::to_value(&report).unwrap();
1786 assert_eq!(json["distill"]["mode"], "no-gpu");
1787 assert_eq!(json["distill"]["note"], NO_GPU_NOTE);
1788 assert!(drain(&mut rx).is_empty());
1789 }
1790
1791 #[test]
1792 fn with_an_llm_endpoint_the_report_names_the_mode_and_carries_no_note() {
1793 let (health, _rx) = monitored();
1794 alive(&health, hours(2));
1795 distill_seen(&health, 3, Some(hours(-23)), hours(2));
1796 let report = health.report();
1797 assert_eq!(report.state, State::Degraded);
1798 let json = serde_json::to_value(&report).unwrap();
1799 assert_eq!(json["distill"]["mode"], "llm");
1800 assert!(json["distill"].get("note").is_none(), "{json}");
1801 }
1802
1803 #[test]
1804 fn a_window_turned_off_never_degrades() {
1805 let (tx, _rx) = unbounded_channel();
1806 let windows = Windows::from_config(&AlertConfig {
1807 sync_overdue_hours: 0,
1808 undistilled_overdue_hours: 0,
1809 ..AlertConfig::default()
1810 });
1811 assert_eq!(
1812 windows,
1813 Windows {
1814 sync_overdue: None,
1815 undistilled: None
1816 }
1817 );
1818 let health = Health::new(t0(), Some(tx)).with_windows(windows);
1819 alive(&health, hours(100));
1820 sync_seen(&health, synced(1, true, Some(hours(-100))), hours(100));
1821 distill_seen(&health, 9, Some(hours(-100)), hours(100));
1822 assert_eq!(health.report().state, State::Ok);
1823 sync_seen(&health, synced(0, true, Some(hours(-100))), hours(100));
1824 assert_eq!(health.report().state, State::Ok);
1825 }
1826
1827 #[test]
1828 fn the_windows_come_from_the_alert_config() {
1829 let windows = Windows::from_config(&AlertConfig {
1830 sync_overdue_hours: 6,
1831 undistilled_overdue_hours: 48,
1832 ..AlertConfig::default()
1833 });
1834 assert_eq!(windows.sync_overdue, Some(Duration::hours(6)));
1835 assert_eq!(windows.undistilled, Some(Duration::hours(48)));
1836 assert_eq!(Windows::default().sync_overdue, Some(Duration::hours(24)));
1837 assert_eq!(Windows::default().undistilled, Some(Duration::hours(24)));
1838 }
1839
1840 #[test]
1841 fn announce_sends_the_current_state_once_unless_it_was_already_sent() {
1842 let (health, mut rx) = monitored();
1843 health.announce(mins(0));
1844 let sent: Vec<Transition> = std::iter::from_fn(|| rx.try_recv().ok()).collect();
1845 assert_eq!(
1846 sent,
1847 vec![Transition {
1848 state: State::Ok,
1849 reason: None,
1850 at: mins(0),
1851 }]
1852 );
1853
1854 health.announce(mins(1));
1855 assert!(drain(&mut rx).is_empty(), "ok was already sent");
1856
1857 sync_seen(&health, synced(0, true, Some(hours(-30))), mins(2));
1858 health.announce(mins(3));
1859 assert_eq!(drain(&mut rx), vec![State::Degraded]);
1860 }
1861
1862 #[test]
1863 fn announce_reassesses_and_sends_one_message_for_a_pending_change() {
1864 let (health, mut rx) = monitored();
1865 alive(&health, hours(0));
1866 distill_seen(&health, 3, Some(hours(-23)), hours(0));
1867 assert!(drain(&mut rx).is_empty());
1868 health.announce(hours(2));
1869 assert_eq!(drain(&mut rx), vec![State::Degraded]);
1870 }
1871
1872 #[test]
1873 fn serving_leaves_out_sync_and_distillation() {
1874 let (health, _rx) = monitored();
1875 alive(&health, hours(0));
1876 let seen = ProgressSeen {
1877 sync: Some(synced(1, true, Some(hours(-30)))),
1878 distill: Some(DistillReport {
1879 pending: 4,
1880 oldest_at: Some(hours(-30)),
1881 ..DistillReport::default()
1882 }),
1883 ingested_sessions: None,
1884 error: None,
1885 };
1886 health.progress_checked(seen, hours(0));
1887 let report = health.report();
1888 assert_eq!(report.state, State::Degraded);
1889 assert_eq!(
1890 report.serving,
1891 Verdict {
1892 state: State::Ok,
1893 reason: None
1894 }
1895 );
1896
1897 health.watcher_failed("inotify watch limit reached", hours(0));
1898 let report = health.report();
1899 assert_eq!(report.state, State::Failing);
1900 assert_eq!(report.serving.state, State::Failing);
1901 let serving = report.serving.reason.unwrap();
1902 assert!(serving.contains("watcher is not running"), "{serving}");
1903 assert!(!serving.contains("sync"), "{serving}");
1904
1905 let json = serde_json::to_value(health.report()).unwrap();
1906 assert_eq!(json["serving"]["state"], "failing");
1907 }
1908
1909 #[test]
1910 fn one_pass_that_clears_sync_and_finds_distillation_overdue_sends_nothing() {
1911 let (health, mut rx) = monitored();
1912 alive(&health, hours(0));
1913 sync_seen(&health, synced(1, true, Some(hours(-30))), hours(0));
1914 assert_eq!(drain(&mut rx), vec![State::Degraded]);
1915 let seen = ProgressSeen {
1916 sync: Some(synced(1, true, Some(hours(0)))),
1917 distill: Some(DistillReport {
1918 pending: 4,
1919 oldest_at: Some(hours(-30)),
1920 ..DistillReport::default()
1921 }),
1922 ingested_sessions: None,
1923 error: None,
1924 };
1925 health.progress_checked(seen, hours(0));
1926 assert_eq!(health.report().state, State::Degraded);
1927 assert!(drain(&mut rx).is_empty());
1928 }
1929
1930 fn backfill_running(done: usize, failed: usize, at: DateTime<Utc>) -> BackfillReport {
1931 BackfillReport {
1932 state: BackfillState::Running,
1933 total: 40,
1934 done,
1935 failed,
1936 updated_at: Some(at),
1937 }
1938 }
1939
1940 #[test]
1941 fn a_new_daemon_reports_the_backfill_not_started() {
1942 let (health, _rx) = monitored();
1943 let report = health.report();
1944 assert_eq!(report.backfill, BackfillReport::default());
1945 assert_eq!(report.backfill.state, BackfillState::NotStarted);
1946 assert_eq!(report.backfill.total, 0);
1947 assert_eq!(report.backfill.done, 0);
1948 assert_eq!(report.backfill.failed, 0);
1949 let json = serde_json::to_value(report).unwrap();
1950 assert_eq!(json["backfill"]["state"], "not_started");
1951 }
1952
1953 #[test]
1954 fn backfill_progress_is_reported_and_survives_later_reports() {
1955 let (health, _rx) = monitored();
1956 health.backfill_progress(backfill_running(3, 1, mins(1)));
1957 assert_eq!(health.report().backfill, backfill_running(3, 1, mins(1)));
1958
1959 health.ingested(None, mins(2));
1960 assert_eq!(health.report().backfill, backfill_running(3, 1, mins(1)));
1961
1962 let done = BackfillReport {
1963 state: BackfillState::Done,
1964 total: 40,
1965 done: 40,
1966 failed: 1,
1967 updated_at: Some(mins(3)),
1968 };
1969 health.backfill_progress(done.clone());
1970 assert_eq!(health.report().backfill, done);
1971 let json = serde_json::to_value(health.report()).unwrap();
1972 assert_eq!(json["backfill"]["state"], "done");
1973 assert_eq!(json["backfill"]["done"], 40);
1974 }
1975
1976 #[test]
1977 fn backfill_progress_leaves_the_state_and_reason_alone() {
1978 let (health, mut rx) = monitored();
1979 for m in 0..=12 {
1980 health.ingest_failed("database is locked", mins(m));
1981 }
1982 assert_eq!(drain(&mut rx), vec![State::Degraded]);
1983 let before = health.report();
1984
1985 health.backfill_progress(backfill_running(5, 2, mins(13)));
1986 let after = health.report();
1987 assert_eq!(after.state, before.state);
1988 assert_eq!(after.reason, before.reason);
1989 assert_eq!(after.since, before.since);
1990 assert_eq!(after.serving, before.serving);
1991 assert!(drain(&mut rx).is_empty());
1992 }
1993
1994 fn held(secs: u64) -> Option<std::time::Duration> {
1995 Some(std::time::Duration::from_secs(secs))
1996 }
1997
1998 #[test]
1999 fn a_clock_gate_closed_past_the_grace_degrades_serving_until_it_opens() {
2000 let (health, mut rx) = monitored();
2001 alive(&health, mins(0));
2002 health.clock_gate(held(0), mins(0));
2003 health.clock_gate(held(90), mins(0) + Duration::seconds(90));
2004 assert_eq!(
2005 health.report().state,
2006 State::Ok,
2007 "a brief hold sends nothing"
2008 );
2009
2010 health.clock_gate(held(150), mins(3));
2011 let report = health.report();
2012 assert_eq!(report.state, State::Degraded);
2013 assert_eq!(report.serving.state, State::Degraded);
2014 let reason = report.reason.unwrap();
2015 assert!(
2016 reason.contains("clock not yet trusted: writes held"),
2017 "{reason}"
2018 );
2019
2020 health.clock_gate(None, mins(4));
2021 let report = health.report();
2022 assert_eq!(report.state, State::Ok);
2023 assert_eq!(report.reason, None);
2024 assert_eq!(drain(&mut rx), vec![State::Degraded, State::Ok]);
2025 }
2026
2027 #[test]
2028 fn a_short_monotonic_hold_stays_ok_when_the_wall_clock_steps_forward() {
2029 let (health, mut rx) = monitored();
2030 alive(&health, mins(0));
2031 health.clock_gate(held(0), mins(0));
2032 alive(&health, mins(50));
2033 health.clock_gate(held(3), mins(50));
2034 alive(&health, mins(50) + Duration::seconds(1));
2035 assert_eq!(health.report().state, State::Ok);
2036 assert!(drain(&mut rx).is_empty());
2037 }
2038
2039 #[test]
2040 fn a_rejected_handshake_degrades_naming_the_reason_for_a_day() {
2041 let (health, _rx) = monitored();
2042 alive(&health, hours(0));
2043 health.clock_handshake_rejected(
2044 "peer-b",
2045 "from peer-b: peer time is more than a day ahead".to_string(),
2046 hours(0),
2047 );
2048 let report = health.report();
2049 assert_eq!(report.state, State::Degraded);
2050 assert_eq!(report.serving.state, State::Ok);
2051 let reason = report.reason.unwrap();
2052 assert!(reason.contains("more than a day ahead"), "{reason}");
2053 assert_eq!(report.last_error.unwrap().source, "clock");
2054
2055 alive(&health, hours(25));
2056 health.clock_gate(None, hours(25));
2057 assert_eq!(health.report().state, State::Ok);
2058 }
2059
2060 #[test]
2061 fn a_basis_ahead_of_real_time_degrades_until_time_catches_up() {
2062 let (health, _rx) = monitored();
2063 alive(&health, mins(0));
2064 health.clock_basis_ahead(true, mins(0));
2065 let report = health.report();
2066 assert_eq!(report.state, State::Degraded);
2067 assert_eq!(report.serving.state, State::Ok);
2068 let reason = report.reason.unwrap();
2069 assert!(reason.contains("ahead of synchronised time"), "{reason}");
2070
2071 health.clock_basis_ahead(false, mins(1));
2072 assert_eq!(health.report().state, State::Ok);
2073 }
2074
2075 fn floor_health(silent: &[&str], stuck: &[&str]) -> FloorHealth {
2076 FloorHealth {
2077 floor: 5,
2078 log_floor: 2,
2079 head: 9,
2080 top: 9,
2081 silent: silent.iter().map(|s| s.to_string()).collect(),
2082 stuck: stuck.iter().map(|s| s.to_string()).collect(),
2083 }
2084 }
2085
2086 #[test]
2087 fn before_the_first_floor_check_there_is_no_floor_section() {
2088 let (health, _rx) = monitored();
2089 alive(&health, mins(0));
2090 let report = health.report();
2091 assert_eq!(report.state, State::Ok);
2092 assert!(report.sync.floor.is_none());
2093 }
2094
2095 #[test]
2096 fn a_silent_neighbour_degrades_and_is_named() {
2097 let (health, _rx) = monitored();
2098 alive(&health, mins(0));
2099 health.floor_checked(
2100 floor_health(&["peer-a"], &[]),
2101 FailedCounts::default(),
2102 mins(0),
2103 );
2104 let report = health.report();
2105 assert_eq!(report.state, State::Degraded);
2106 let reason = report.reason.unwrap();
2107 assert!(
2108 reason.contains("peer-a") && reason.contains("silent"),
2109 "{reason}"
2110 );
2111 assert_eq!(report.sync.floor.unwrap().silent, vec!["peer-a"]);
2112
2113 health.floor_checked(floor_health(&[], &[]), FailedCounts::default(), mins(1));
2114 assert_eq!(health.report().state, State::Ok);
2115 }
2116
2117 fn standing(peer: &str, holds_own_row: bool, lacking: &[u32]) -> PeerStanding {
2118 PeerStanding {
2119 peer: peer.to_string(),
2120 heard_at_micros: 1,
2121 holds_own_row,
2122 lacking: lacking.to_vec(),
2123 }
2124 }
2125
2126 fn identity(peers: Vec<PeerStanding>, removed: bool) -> IdentityStatus {
2127 let row_accepted = (!peers.is_empty()).then(|| peers.iter().any(|p| p.holds_own_row));
2128 let possibly_revoked = removed
2129 || peers
2130 .iter()
2131 .any(|p| p.holds_own_row && !p.lacking.is_empty());
2132 IdentityStatus {
2133 identity: Some("m-0001".to_string()),
2134 epochs: vec![0, 1],
2135 removed,
2136 possibly_revoked,
2137 row_accepted,
2138 peers,
2139 retired_recovery: Vec::new(),
2140 }
2141 }
2142
2143 #[test]
2144 fn identity_state_is_reported_and_judged_only_once_peers_were_heard() {
2145 let (health, _rx) = monitored();
2146 alive(&health, mins(0));
2147 health.key_storage_detected("tpm", mins(0));
2148 assert!(health.report().identity.status.is_none());
2149 assert_eq!(health.report().identity.key_storage, Some("tpm"));
2150
2151 health.identity_checked(identity(Vec::new(), false), mins(1));
2152 let report = health.report();
2153 assert_eq!(report.state, State::Ok, "{:?}", report.reason);
2154 assert_eq!(report.identity.checked_at, Some(mins(1)));
2155 let status = report.identity.status.unwrap();
2156 assert_eq!(status.epochs, vec![0, 1]);
2157 assert_eq!(status.row_accepted, None);
2158
2159 health.identity_checked(identity(vec![standing("pi", true, &[])], false), mins(2));
2160 assert_eq!(health.report().state, State::Ok);
2161 }
2162
2163 #[test]
2164 fn a_device_with_peers_but_no_identity_degrades_until_it_is_paired() {
2165 let (health, _rx) = monitored();
2166 alive(&health, mins(0));
2167 health.identity_checked(IdentityStatus::default(), mins(1));
2168 assert_eq!(
2169 health.report().state,
2170 State::Ok,
2171 "no peers, nothing to sync"
2172 );
2173
2174 sync_seen(
2175 &health,
2176 SyncReport {
2177 checked_at: Some(mins(2)),
2178 peers: Some(1),
2179 ..SyncReport::default()
2180 },
2181 mins(2),
2182 );
2183 let report = health.report();
2184 assert_eq!(report.state, State::Degraded);
2185 let reason = report.reason.unwrap();
2186 assert!(reason.contains("holds no sync identity key"), "{reason}");
2187 assert!(
2188 reason.contains("`sb pair <code>` on this device"),
2189 "{reason}"
2190 );
2191
2192 health.identity_checked(identity(vec![standing("pi", true, &[])], false), mins(3));
2193 let reason = health.report().reason.unwrap_or_default();
2194 assert!(!reason.contains("sync identity"), "{reason}");
2195 }
2196
2197 #[test]
2198 fn an_unreadable_hardware_id_is_reported_and_never_degrades() {
2199 let (health, _rx) = monitored();
2200 alive(&health, mins(0));
2201 assert_eq!(health.report().identity.hardware, None);
2202 health.hardware_checked("unreadable", mins(0));
2203 health.identity_checked(identity(vec![standing("pi", true, &[])], false), mins(1));
2204 let report = health.report();
2205 assert_eq!(report.identity.hardware, Some("unreadable"));
2206 assert_eq!(report.state, State::Ok, "{:?}", report.reason);
2207 let json = serde_json::to_value(&report.identity).unwrap();
2208 assert_eq!(json["hardware"], "unreadable");
2209 }
2210
2211 #[test]
2212 fn a_reminted_copy_with_peers_is_told_it_came_from_other_hardware_and_to_pair() {
2213 let (health, _rx) = monitored();
2214 alive(&health, mins(0));
2215 health.hardware_checked("reminted", mins(0));
2216 health.identity_checked(IdentityStatus::default(), mins(1));
2217 sync_seen(
2218 &health,
2219 SyncReport {
2220 checked_at: Some(mins(2)),
2221 peers: Some(1),
2222 ..SyncReport::default()
2223 },
2224 mins(2),
2225 );
2226 let report = health.report();
2227 assert_eq!(report.state, State::Degraded);
2228 let reason = report.reason.unwrap();
2229 assert!(reason.contains("copied from other hardware"), "{reason}");
2230 assert!(reason.contains("new machine id"), "{reason}");
2231 assert!(
2232 reason.contains("`sb pair <code>` on this device"),
2233 "{reason}"
2234 );
2235
2236 health.identity_checked(identity(vec![standing("pi", true, &[])], false), mins(3));
2237 let reason = health.report().reason.unwrap_or_default();
2238 assert!(!reason.contains("other hardware"), "{reason}");
2239 }
2240
2241 #[test]
2242 fn a_row_no_peer_accepted_degrades_naming_pairing() {
2243 let (health, _rx) = monitored();
2244 alive(&health, mins(0));
2245 health.identity_checked(
2246 identity(
2247 vec![standing("pi", false, &[]), standing("mac", false, &[])],
2248 false,
2249 ),
2250 mins(1),
2251 );
2252 let report = health.report();
2253 assert_eq!(report.state, State::Degraded);
2254 let reason = report.reason.unwrap();
2255 assert!(reason.contains("no peer has accepted"), "{reason}");
2256 assert!(reason.contains("sb pair"), "{reason}");
2257
2258 health.identity_checked(
2259 identity(
2260 vec![standing("pi", false, &[]), standing("mac", true, &[])],
2261 false,
2262 ),
2263 mins(2),
2264 );
2265 assert_eq!(health.report().state, State::Ok);
2266 }
2267
2268 #[test]
2269 fn a_withheld_epoch_or_a_removal_degrades_as_possibly_revoked() {
2270 let (health, _rx) = monitored();
2271 alive(&health, mins(0));
2272 health.identity_checked(identity(vec![standing("pi", true, &[2])], false), mins(1));
2273 let reason = health.report().reason.unwrap();
2274 assert!(reason.contains("pi holds identity key epoch 2"), "{reason}");
2275 assert!(reason.contains("may have been removed"), "{reason}");
2276
2277 health.identity_checked(identity(vec![standing("pi", true, &[])], true), mins(2));
2278 let reason = health.report().reason.unwrap();
2279 assert!(
2280 reason.contains("removed from its sync identity"),
2281 "{reason}"
2282 );
2283 assert!(reason.contains("sb pair <code>"), "{reason}");
2284 assert!(!reason.contains("may have been"), "{reason}");
2285 }
2286
2287 #[test]
2288 fn a_stuck_neighbour_degrades_and_is_named() {
2289 let (health, _rx) = monitored();
2290 alive(&health, mins(0));
2291 health.floor_checked(
2292 floor_health(&[], &["peer-b"]),
2293 FailedCounts::default(),
2294 mins(0),
2295 );
2296 let report = health.report();
2297 assert_eq!(report.state, State::Degraded);
2298 let reason = report.reason.unwrap();
2299 assert!(
2300 reason.contains("peer-b") && reason.contains("not moved"),
2301 "{reason}"
2302 );
2303 }
2304
2305 #[test]
2306 fn a_failing_snapshot_merge_degrades_and_a_running_one_does_not() {
2307 use crate::snapshot_merge::MergeFailing;
2308 use second_brain_sync::protocol::Merging;
2309 let (health, _rx) = monitored();
2310 alive(&health, mins(0));
2311 let merging = Merging {
2312 snapshot_id: "s1".to_string(),
2313 exporter: "peer-a".to_string(),
2314 seq: 42,
2315 };
2316 health.snapshot_merge_checked(
2317 Some(MergeStatus {
2318 merging: merging.clone(),
2319 failing: None,
2320 }),
2321 mins(0),
2322 );
2323 assert_eq!(health.report().state, State::Ok);
2324 assert!(health.report().sync.snapshot_merge.is_some());
2325
2326 health.snapshot_merge_checked(
2327 Some(MergeStatus {
2328 merging,
2329 failing: Some(MergeFailing {
2330 since: 0,
2331 attempts: 3,
2332 error: "disk full".to_string(),
2333 }),
2334 }),
2335 mins(1),
2336 );
2337 let report = health.report();
2338 assert_eq!(report.state, State::Degraded);
2339 let reason = report.reason.unwrap();
2340 assert!(
2341 reason.contains("peer-a") && reason.contains("disk full"),
2342 "{reason}"
2343 );
2344
2345 health.snapshot_merge_checked(None, mins(2));
2346 assert_eq!(health.report().state, State::Ok);
2347 }
2348
2349 #[test]
2350 fn settled_records_are_reported_without_degrading() {
2351 let (health, _rx) = monitored();
2352 alive(&health, mins(0));
2353 health.floor_checked(
2354 floor_health(&[], &[]),
2355 FailedCounts {
2356 failing: 2,
2357 settled: 3,
2358 dropped: 4,
2359 },
2360 mins(0),
2361 );
2362 let report = health.report();
2363 assert_eq!(report.state, State::Ok);
2364 let floor = report.sync.floor.unwrap();
2365 assert_eq!(
2366 (floor.failing, floor.settled, floor.settled_dropped),
2367 (2, 3, 4)
2368 );
2369 }
2370
2371 fn empty_recall(session: &str) -> RecallEvent {
2372 RecallEvent {
2373 session: Some(session.to_string()),
2374 verdict: crate::value_metrics::RecallVerdict::Empty,
2375 memory_ids: Vec::new(),
2376 }
2377 }
2378
2379 #[test]
2380 fn a_value_metric_crossing_its_threshold_sends_exactly_one_transition() {
2381 let (health, mut rx) = monitored();
2382 alive(&health, mins(0));
2383 for _ in 0..49 {
2384 health.recalled(empty_recall("s"), mins(1));
2385 }
2386 assert!(drain(&mut rx).is_empty());
2387 for _ in 0..30 {
2388 health.recalled(empty_recall("s"), mins(2));
2389 }
2390 let report = health.report();
2391 assert_eq!(report.state, State::Degraded);
2392 assert_eq!(report.serving.state, State::Ok);
2393 let reason = report.reason.unwrap();
2394 assert!(reason.contains("found nothing that cleared"), "{reason}");
2395 assert_eq!(drain(&mut rx), vec![State::Degraded]);
2396 }
2397
2398 #[test]
2399 fn a_share_hovering_at_its_threshold_sends_one_transition_each_way() {
2400 let (health, mut rx) = monitored();
2401 let hit = || RecallEvent {
2402 session: None,
2403 verdict: crate::value_metrics::RecallVerdict::Hit,
2404 memory_ids: Vec::new(),
2405 };
2406 for m in 0..50 {
2407 alive(&health, mins(m));
2408 let event = if m < 37 { empty_recall("s") } else { hit() };
2409 health.recalled(event, mins(m));
2410 }
2411 assert_eq!(health.report().state, State::Ok, "37 of 50 is 74%");
2412 for _ in 0..3 {
2413 health.recalled(empty_recall("s"), mins(50));
2414 }
2415 assert_eq!(drain(&mut rx), vec![State::Degraded], "40 of 53 is 75.5%");
2416 for m in 51..56 {
2417 alive(&health, mins(m));
2418 health.recalled(hit(), mins(m));
2419 health.recalled(empty_recall("s"), mins(m));
2420 }
2421 assert!(drain(&mut rx).is_empty(), "74% then 71% stays degraded");
2422 for m in 56..66 {
2423 alive(&health, mins(m));
2424 health.recalled(hit(), mins(m));
2425 }
2426 assert_eq!(health.report().value.recall.recalls, 73);
2427 assert_eq!(drain(&mut rx), vec![State::Ok], "45 of 73 is 62%");
2428 }
2429
2430 #[test]
2431 fn peer_lag_is_left_out_when_no_sync_succeeded_at_all() {
2432 let (health, _rx) = monitored();
2433 health.sync_peers_listed(&["u@pi".to_string()], hours(0));
2434 alive(&health, hours(30));
2435 let reason = health.report().reason.unwrap();
2436 assert!(reason.contains("no sync with u@pi"), "{reason}");
2437
2438 sync_seen(&health, synced(1, true, Some(hours(-30))), hours(30));
2439 let reason = health.report().reason.unwrap();
2440 assert!(reason.contains("no sync has succeeded in"), "{reason}");
2441 assert!(!reason.contains("no sync with u@pi"), "{reason}");
2442 assert_eq!(health.report().value.sync_peers.len(), 1);
2443 }
2444
2445 #[test]
2446 fn the_ingest_window_starts_after_startup_and_any_backfill() {
2447 let (health, _rx) = monitored();
2448 assert_eq!(health.ingest_window_start(mins(5)), Some(t0()));
2449 assert_eq!(
2450 health.ingest_window_start(hours(30)),
2451 Some(hours(30) - WINDOW)
2452 );
2453 health.backfill_progress(backfill_running(1, 0, mins(1)));
2454 assert_eq!(health.ingest_window_start(mins(5)), None);
2455 health.backfill_progress(BackfillReport {
2456 state: BackfillState::Done,
2457 updated_at: Some(mins(20)),
2458 ..BackfillReport::default()
2459 });
2460 assert_eq!(health.ingest_window_start(mins(30)), Some(mins(20)));
2461 }
2462
2463 #[test]
2464 fn progress_sets_the_ingested_sessions_and_a_failed_read_clears_them() {
2465 let (health, _rx) = monitored();
2466 health.recalled(empty_recall("a"), mins(1));
2467 let seen = ProgressSeen {
2468 ingested_sessions: Some(vec!["a".to_string(), "b".to_string()]),
2469 ..ProgressSeen::default()
2470 };
2471 health.progress_checked(seen, mins(2));
2472 let recall = health.report().value.recall;
2473 assert_eq!(recall.sessions_ingested, Some(2));
2474 assert_eq!(recall.session_recall_rate, Some(0.5));
2475 health.progress_checked(ProgressSeen::default(), mins(3));
2476 assert_eq!(health.report().value.recall.sessions_ingested, None);
2477 }
2478
2479 #[test]
2480 fn every_value_metric_is_in_the_report_json() {
2481 let (health, _rx) = monitored();
2482 health.sync_peers_listed(&["u@pi".to_string()], mins(0));
2483 health.store_measured(Some(StoreSize::default()), mins(0));
2484 health.resident_measured(Some(ResidentSize::default()), mins(0));
2485 let json = serde_json::to_value(health.report()).unwrap();
2486 let keys = |value: &serde_json::Value| -> Vec<String> {
2487 let mut keys: Vec<String> = value.as_object().unwrap().keys().cloned().collect();
2488 keys.sort();
2489 keys
2490 };
2491 assert_eq!(
2492 keys(&json["sync_peers"][0]),
2493 [
2494 "consecutive_failures",
2495 "lag_secs",
2496 "last_attempt_at",
2497 "last_error",
2498 "last_success_at",
2499 "peer"
2500 ]
2501 );
2502 assert_eq!(
2503 keys(&json["store"]),
2504 [
2505 "archive_bytes",
2506 "checked_at",
2507 "cold_bytes",
2508 "db_bytes",
2509 "free_bytes",
2510 "index_bytes",
2511 "raw_index_bytes",
2512 "total_bytes",
2513 "wal_bytes"
2514 ]
2515 );
2516 assert_eq!(
2517 keys(&json["resident"]),
2518 [
2519 "checked_at",
2520 "components",
2521 "limit_bytes",
2522 "raw_index_bytes",
2523 "rss_anon_bytes",
2524 "unattributed_bytes"
2525 ]
2526 );
2527 assert_eq!(
2528 keys(&json["resident"]["components"]),
2529 [
2530 "allocator_free",
2531 "embedder",
2532 "lexical",
2533 "reranker",
2534 "sqlite",
2535 "vector_reader",
2536 "vector_writer"
2537 ]
2538 );
2539 for key in ["recall", "recalled", "sync_peers", "store", "resident"] {
2540 assert!(json.get(key).is_some(), "{key} missing: {json}");
2541 }
2542 for key in [
2543 "window_secs",
2544 "recalls",
2545 "hits",
2546 "low_score",
2547 "unjudged",
2548 "empty",
2549 "empty_share",
2550 "low_score_share",
2551 "unjudged_share",
2552 "sessions_with_recall",
2553 "sessions_ingested",
2554 "session_recall_rate",
2555 ] {
2556 assert!(json["recall"].get(key).is_some(), "recall.{key}: {json}");
2557 }
2558 for key in [
2559 "window_secs",
2560 "memories",
2561 "corrected",
2562 "used",
2563 "corrected_share",
2564 "used_share",
2565 ] {
2566 assert!(
2567 json["recalled"].get(key).is_some(),
2568 "recalled.{key}: {json}"
2569 );
2570 }
2571 }
2572}