use std::sync::Mutex;
use chrono::{DateTime, Duration, Utc};
use second_brain_core::config::AlertConfig;
use second_brain_core::sqlite_store::VacuumSpace;
use serde::Serialize;
use tokio::sync::mpsc::UnboundedSender;
use uuid::Uuid;
use crate::value_metrics::{
PeerOutcome, RecallEvent, ResidentSize, StoreSize, ValueFacts, ValueProblems, ValueReport,
WINDOW,
};
const CAPTURE_LAG_DEGRADED: Duration = Duration::minutes(15);
const CAPTURE_LAG_FAILING: Duration = Duration::hours(2);
const BACKLOG_DEGRADED: Duration = Duration::minutes(15);
const BACKLOG_FAILING: Duration = Duration::hours(2);
const WATCHER_SILENT: Duration = Duration::minutes(15);
const DRAIN_SILENT: Duration = Duration::hours(1);
const ERROR_GRACE: Duration = Duration::minutes(10);
const QUEUE_STUCK: Duration = Duration::minutes(10);
#[derive(Debug, Clone, Copy, Default, PartialEq, Eq, PartialOrd, Ord, Serialize)]
#[serde(rename_all = "lowercase")]
pub enum State {
#[default]
Ok,
Degraded,
Failing,
}
#[derive(Debug, Clone, PartialEq, Serialize)]
pub struct Transition {
pub state: State,
pub reason: Option<String>,
pub at: DateTime<Utc>,
}
#[derive(Debug, Clone, Serialize)]
pub struct ErrorRecord {
pub source: &'static str,
pub message: String,
pub at: DateTime<Utc>,
}
#[derive(Debug, Clone, Default, Serialize)]
pub struct CaptureReport {
pub checked_at: Option<DateTime<Utc>>,
pub newest_transcript_at: Option<DateTime<Utc>>,
pub captured_through: Option<DateTime<Utc>>,
pub lag_secs: Option<i64>,
}
#[derive(Debug, Clone, Default, Serialize)]
pub struct IndexReport {
pub checked_at: Option<DateTime<Utc>>,
pub outbox_pending: Option<u64>,
pub backlog_since: Option<DateTime<Utc>>,
}
#[derive(Debug, Clone, PartialEq, Serialize)]
pub struct RejectedWrite {
pub tool: String,
pub status: u16,
pub at: DateTime<Utc>,
}
#[derive(Debug, Clone, Default, Serialize)]
pub struct QueueReport {
pub depth: u64,
pub oldest_at: Option<DateTime<Utc>>,
pub oldest_age_secs: Option<i64>,
pub last_rejected: Option<RejectedWrite>,
}
#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize)]
#[serde(rename_all = "snake_case")]
pub enum MaintenanceState {
Pending,
WaitingForSpace,
MergingIndex,
Vacuuming,
Done,
}
#[derive(Debug, Clone, Serialize)]
pub struct MaintenanceReport {
pub state: MaintenanceState,
pub since: Option<DateTime<Utc>>,
pub space: Option<VacuumSpace>,
pub last_error: Option<String>,
}
impl Default for MaintenanceReport {
fn default() -> Self {
Self {
state: MaintenanceState::Pending,
since: None,
space: None,
last_error: None,
}
}
}
#[derive(Debug, Clone, Copy, Default, PartialEq, Eq, Serialize)]
#[serde(rename_all = "snake_case")]
pub enum BackfillState {
#[default]
NotStarted,
Running,
Done,
}
#[derive(Debug, Clone, Default, PartialEq, Eq, Serialize)]
pub struct BackfillReport {
pub state: BackfillState,
pub total: usize,
pub done: usize,
pub failed: usize,
pub updated_at: Option<DateTime<Utc>>,
}
#[derive(Debug, Clone, Default, Serialize)]
pub struct SyncReport {
pub checked_at: Option<DateTime<Utc>>,
pub peers: Option<usize>,
pub synced_before: bool,
pub last_success_at: Option<DateTime<Utc>>,
pub last_cursor_at: Option<DateTime<Utc>>,
}
pub const NO_GPU_NOTE: &str = "no LLM endpoint configured and no LLM distillation in 7 days: \
raws covered by an agent note are retired, the rest stay searchable";
#[derive(Debug, Clone, Copy, Default, PartialEq, Eq, Serialize)]
pub enum DistillMode {
#[default]
#[serde(rename = "llm")]
Llm,
#[serde(rename = "no-gpu")]
NoGpu,
}
impl DistillMode {
fn note(self) -> Option<&'static str> {
match self {
DistillMode::Llm => None,
DistillMode::NoGpu => Some(NO_GPU_NOTE),
}
}
}
#[derive(Debug, Clone, Default, Serialize)]
pub struct DistillReport {
pub checked_at: Option<DateTime<Utc>>,
pub mode: DistillMode,
pub pending: u64,
pub oldest_at: Option<DateTime<Utc>>,
#[serde(skip_serializing_if = "Option::is_none")]
pub note: Option<&'static str>,
}
#[derive(Debug, Clone, Copy, PartialEq)]
pub struct Windows {
pub sync_overdue: Option<Duration>,
pub undistilled: Option<Duration>,
}
impl Windows {
pub fn from_config(config: &AlertConfig) -> Self {
let hours = |h: u64| {
i64::try_from(h)
.ok()
.filter(|&h| h > 0)
.and_then(Duration::try_hours)
};
Self {
sync_overdue: hours(config.sync_overdue_hours),
undistilled: hours(config.undistilled_overdue_hours),
}
}
}
impl Default for Windows {
fn default() -> Self {
Self::from_config(&AlertConfig::default())
}
}
#[derive(Debug, Clone, Default)]
pub struct ProgressSeen {
pub sync: Option<SyncReport>,
pub distill: Option<DistillReport>,
pub ingested_sessions: Option<Vec<String>>,
pub error: Option<String>,
}
#[derive(Debug, Clone, PartialEq, Serialize)]
pub struct Verdict {
pub state: State,
pub reason: Option<String>,
}
#[derive(Debug, Clone, Serialize)]
pub struct HealthReport {
pub state: State,
pub reason: Option<String>,
pub since: DateTime<Utc>,
pub serving: Verdict,
pub capture: CaptureReport,
pub index: IndexReport,
pub queue: QueueReport,
pub maintenance: MaintenanceReport,
pub backfill: BackfillReport,
pub sync: SyncReport,
pub distill: DistillReport,
#[serde(flatten)]
pub value: ValueReport,
pub last_error: Option<ErrorRecord>,
}
struct Facts {
started: DateTime<Utc>,
capture: CaptureReport,
watcher_error: Option<String>,
ingest_error: Option<Streak>,
index: IndexReport,
drain_error: Option<Streak>,
indexes_error: Option<String>,
repair_error: Option<String>,
queue: QueueReport,
maintenance: MaintenanceReport,
backfill: BackfillReport,
sync: SyncReport,
distill: DistillReport,
progress_error: Option<Streak>,
value: ValueFacts,
windows: Windows,
last_error: Option<ErrorRecord>,
}
struct Streak {
message: String,
since: DateTime<Utc>,
}
impl Streak {
fn extend(previous: Option<Streak>, message: String, now: DateTime<Utc>) -> Option<Streak> {
let since = previous.map_or(now, |p| p.since);
Some(Streak { message, since })
}
fn outlasted_grace(&self, now: DateTime<Utc>) -> bool {
now - self.since >= ERROR_GRACE
}
}
struct Inner {
facts: Facts,
report: HealthReport,
sent: Option<State>,
}
pub struct Health {
inner: Mutex<Inner>,
transitions: Option<UnboundedSender<Transition>>,
}
impl Health {
pub fn new(now: DateTime<Utc>, transitions: Option<UnboundedSender<Transition>>) -> Self {
let mut facts = Facts {
started: now,
capture: CaptureReport::default(),
watcher_error: None,
ingest_error: None,
index: IndexReport::default(),
drain_error: None,
indexes_error: None,
repair_error: None,
queue: QueueReport::default(),
maintenance: MaintenanceReport::default(),
backfill: BackfillReport::default(),
sync: SyncReport::default(),
distill: DistillReport::default(),
progress_error: None,
value: ValueFacts::default(),
windows: Windows::default(),
last_error: None,
};
let report = HealthReport {
state: State::Ok,
reason: None,
since: now,
serving: Verdict {
state: State::Ok,
reason: None,
},
capture: CaptureReport::default(),
index: IndexReport::default(),
queue: QueueReport::default(),
maintenance: MaintenanceReport::default(),
backfill: BackfillReport::default(),
sync: SyncReport::default(),
distill: DistillReport::default(),
value: facts.value.refresh(now, now, facts.windows.sync_overdue).0,
last_error: None,
};
Self {
inner: Mutex::new(Inner {
facts,
report,
sent: None,
}),
transitions,
}
}
pub fn with_windows(self, windows: Windows) -> Self {
self.lock().facts.windows = windows;
self
}
pub fn progress_checked(&self, seen: ProgressSeen, now: DateTime<Utc>) {
self.update(now, |f| {
if let Some(sync) = seen.sync {
f.sync = SyncReport {
checked_at: Some(now),
..sync
};
}
if let Some(distill) = seen.distill {
f.distill = DistillReport {
checked_at: Some(now),
note: distill.mode.note(),
..distill
};
}
f.value.sessions_ingested(seen.ingested_sessions);
match seen.error {
None => f.progress_error = None,
Some(error) => {
f.record_error("progress", &error, now);
f.progress_error = Streak::extend(f.progress_error.take(), error, now);
}
}
});
}
pub fn announce(&self, now: DateTime<Utc>) {
let mut inner = self.lock();
self.reassess(&mut inner, now, true);
}
pub fn report(&self) -> HealthReport {
self.lock().report.clone()
}
pub fn transcripts_scanned(&self, newest: Option<DateTime<Utc>>, now: DateTime<Utc>) {
self.update(now, |f| {
f.capture.checked_at = Some(now);
f.capture.newest_transcript_at = newest;
});
}
pub fn watcher_started(&self, now: DateTime<Utc>) {
self.update(now, |f| {
f.watcher_error = None;
f.capture.checked_at = Some(now);
});
}
pub fn watcher_failed(&self, error: &str, now: DateTime<Utc>) {
self.update(now, |f| {
f.watcher_error = Some(error.to_string());
f.record_error("watcher", error, now);
});
}
pub fn ingested(&self, transcript_mtime: Option<DateTime<Utc>>, now: DateTime<Utc>) {
self.update(now, |f| {
f.ingest_error = None;
f.capture.checked_at = Some(now);
if let Some(mtime) = transcript_mtime {
let through = f.capture.captured_through.map_or(mtime, |t| t.max(mtime));
f.capture.captured_through = Some(through);
}
});
}
pub fn ingest_failed(&self, error: &str, now: DateTime<Utc>) {
self.update(now, |f| {
f.ingest_error = Streak::extend(f.ingest_error.take(), error.to_string(), now);
f.capture.checked_at = Some(now);
f.record_error("ingest", error, now);
});
}
pub fn drained(&self, outbox_pending: u64, now: DateTime<Utc>) {
self.update(now, |f| {
f.drain_error = None;
f.index.checked_at = Some(now);
f.index.outbox_pending = Some(outbox_pending);
f.index.backlog_since = if outbox_pending == 0 {
None
} else {
Some(f.index.backlog_since.unwrap_or(now))
};
});
}
pub fn drain_failed(&self, error: &str, now: DateTime<Utc>) {
self.update(now, |f| {
f.index.checked_at = Some(now);
f.drain_error = Streak::extend(f.drain_error.take(), error.to_string(), now);
f.record_error("drain", error, now);
});
}
pub fn indexes_unavailable(&self, error: &str, now: DateTime<Utc>) {
self.update(now, |f| {
f.indexes_error = Some(error.to_string());
f.record_error("index", error, now);
});
}
pub fn repaired(&self, now: DateTime<Utc>) {
self.update(now, |f| f.repair_error = None);
}
pub fn repair_failed(&self, error: &str, now: DateTime<Utc>) {
self.update(now, |f| {
f.repair_error = Some(error.to_string());
f.record_error("repair", error, now);
});
}
pub fn queue_checked(&self, depth: u64, oldest_at: Option<DateTime<Utc>>, now: DateTime<Utc>) {
self.update(now, |f| {
f.queue.depth = depth;
f.queue.oldest_at = oldest_at;
});
}
pub fn write_rejected(&self, tool: &str, status: u16, now: DateTime<Utc>) {
self.update(now, |f| {
f.queue.last_rejected = Some(RejectedWrite {
tool: tool.to_string(),
status,
at: now,
});
});
}
pub fn maintenance(
&self,
state: MaintenanceState,
space: Option<VacuumSpace>,
now: DateTime<Utc>,
) {
self.update(now, |f| {
if f.maintenance.state != state {
f.maintenance.since = Some(now);
}
f.maintenance.state = state;
if space.is_some() {
f.maintenance.space = space;
}
if state == MaintenanceState::Done {
f.maintenance.last_error = None;
}
});
}
pub fn maintenance_failed(&self, error: &str, now: DateTime<Utc>) {
self.update(now, |f| {
f.maintenance.last_error = Some(error.to_string());
f.record_error("maintenance", error, now);
});
}
pub fn recalled(&self, event: RecallEvent, now: DateTime<Utc>) {
self.update(now, |f| f.value.recalled(event, now));
}
pub fn memory_corrected(&self, id: Uuid, now: DateTime<Utc>) {
self.update(now, |f| f.value.corrected(id));
}
pub fn memories_used(&self, ids: &[Uuid], now: DateTime<Utc>) {
self.update(now, |f| f.value.used(ids));
}
pub fn sync_peers_listed(&self, peers: &[String], now: DateTime<Utc>) {
self.update(now, |f| f.value.peers_listed(peers, now));
}
pub fn sync_attempted(&self, peer: &str, outcome: PeerOutcome, now: DateTime<Utc>) {
self.update(now, |f| f.value.sync_attempted(peer, outcome, now));
}
pub fn store_measured(&self, size: Option<StoreSize>, now: DateTime<Utc>) {
self.update(now, |f| f.value.store_measured(size));
}
pub fn resident_measured(&self, size: Option<ResidentSize>, now: DateTime<Utc>) {
self.update(now, |f| f.value.resident_measured(size));
}
pub fn ingest_window_start(&self, now: DateTime<Utc>) -> Option<DateTime<Utc>> {
let inner = self.lock();
let f = &inner.facts;
let mut start = (now - WINDOW).max(f.started);
match f.backfill.state {
BackfillState::Running => return None,
BackfillState::Done => {
if let Some(end) = f.backfill.updated_at {
start = start.max(end);
}
}
BackfillState::NotStarted => {}
}
Some(start)
}
pub fn backfill_progress(&self, report: BackfillReport) {
let mut inner = self.lock();
inner.facts.backfill = report.clone();
inner.report.backfill = report;
}
fn lock(&self) -> std::sync::MutexGuard<'_, Inner> {
match self.inner.lock() {
Ok(g) => g,
Err(poisoned) => poisoned.into_inner(),
}
}
fn update(&self, now: DateTime<Utc>, change: impl FnOnce(&mut Facts)) {
let mut inner = self.lock();
change(&mut inner.facts);
self.reassess(&mut inner, now, false);
}
fn reassess(&self, inner: &mut Inner, now: DateTime<Utc>, announcing: bool) {
let (started, sync_overdue) = (inner.facts.started, inner.facts.windows.sync_overdue);
let (value, value_problems) = inner.facts.value.refresh(now, started, sync_overdue);
let mut serving = Assessment::default();
assess_serving(&inner.facts, now, &mut serving);
let mut overall = serving.clone();
assess_upkeep(&inner.facts, now, &mut overall, value_problems);
let (state, reason) = overall.verdict();
let changed = state != inner.report.state;
let since = if changed { now } else { inner.report.since };
let mut capture = inner.facts.capture.clone();
capture.lag_secs = capture_lag(&inner.facts).map(|d| d.num_seconds());
let mut queue = inner.facts.queue.clone();
queue.oldest_age_secs = queue.oldest_at.map(|t| (now - t).num_seconds().max(0));
inner.report = HealthReport {
state,
reason: reason.clone(),
since,
serving: Verdict {
state: serving.state,
reason: serving.verdict().1,
},
capture,
index: inner.facts.index.clone(),
queue,
maintenance: inner.facts.maintenance.clone(),
backfill: inner.facts.backfill.clone(),
sync: inner.facts.sync.clone(),
distill: inner.facts.distill.clone(),
value,
last_error: inner.facts.last_error.clone(),
};
let unsent = inner.sent != Some(state);
if (changed || (announcing && unsent))
&& let Some(tx) = &self.transitions
{
let _ = tx.send(Transition {
state,
reason,
at: now,
});
inner.sent = Some(state);
}
}
}
impl Facts {
fn record_error(&mut self, source: &'static str, message: &str, at: DateTime<Utc>) {
self.last_error = Some(ErrorRecord {
source,
message: message.to_string(),
at,
});
}
}
fn capture_lag(f: &Facts) -> Option<Duration> {
let newest = f.capture.newest_transcript_at?;
let through = f
.capture
.captured_through
.map_or(f.started, |t| t.max(f.started));
Some((newest - through).max(Duration::zero()))
}
#[derive(Clone, Default)]
struct Assessment {
state: State,
problems: Vec<String>,
}
impl Assessment {
fn flag(&mut self, level: State, problem: String) {
self.state = self.state.max(level);
self.problems.push(problem);
}
fn verdict(&self) -> (State, Option<String>) {
let reason = (!self.problems.is_empty()).then(|| self.problems.join("; "));
(self.state, reason)
}
}
fn assess_serving(f: &Facts, now: DateTime<Utc>, a: &mut Assessment) {
let mut flag = |level: State, problem: String| a.flag(level, problem);
match &f.watcher_error {
Some(e) => flag(
State::Failing,
format!("transcript watcher is not running: {e}"),
),
None => {
let last = f.capture.checked_at.unwrap_or(f.started);
if now - last >= WATCHER_SILENT {
flag(
State::Degraded,
format!(
"transcript watcher has not reported for {}",
human(now - last)
),
);
}
}
}
if let Some(lag) = capture_lag(f)
&& lag >= CAPTURE_LAG_DEGRADED
{
let level = if lag >= CAPTURE_LAG_FAILING {
State::Failing
} else {
State::Degraded
};
flag(
level,
format!("capture is {} behind the newest transcript", human(lag)),
);
}
if let Some(e) = &f.ingest_error
&& e.outlasted_grace(now)
{
flag(
State::Degraded,
format!(
"ingest has failed for {}: {}",
human(now - e.since),
e.message
),
);
}
if let Some(e) = &f.drain_error
&& e.outlasted_grace(now)
{
flag(
State::Degraded,
format!(
"index drain failed for {}: {}",
human(now - e.since),
e.message
),
);
}
match &f.indexes_error {
Some(e) => flag(State::Degraded, format!("search indexes unavailable: {e}")),
None => {
let last = f.index.checked_at.unwrap_or(f.started);
if now - last >= DRAIN_SILENT {
flag(
State::Degraded,
format!("index drain has not reported for {}", human(now - last)),
);
}
}
}
if let (Some(since), Some(pending)) = (f.index.backlog_since, f.index.outbox_pending) {
let age = now - since;
if age >= BACKLOG_DEGRADED {
let level = if age >= BACKLOG_FAILING {
State::Failing
} else {
State::Degraded
};
flag(
level,
format!("{pending} changes have waited {} to be indexed", human(age)),
);
}
}
if let Some(e) = &f.repair_error {
flag(State::Degraded, format!("last store repair failed: {e}"));
}
if let Some(oldest) = f.queue.oldest_at
&& now - oldest >= QUEUE_STUCK
{
flag(
State::Degraded,
format!(
"write queue not draining ({} queued, oldest {})",
f.queue.depth,
human(now - oldest)
),
);
}
match (&f.maintenance.state, &f.maintenance.space) {
(MaintenanceState::Vacuuming, _) => flag(
State::Degraded,
"store maintenance is running; writes are queued until it finishes".to_string(),
),
(MaintenanceState::WaitingForSpace, Some(space)) => flag(
State::Degraded,
format!(
"store compaction waits for disk space: it needs {} MB free at {} and has {} MB",
space.needed_bytes / 1_000_000,
space.path.display(),
space.free_bytes / 1_000_000
),
),
_ => {}
}
if let Some(e) = &f.maintenance.last_error {
flag(State::Degraded, format!("store compaction failed: {e}"));
}
}
fn assess_upkeep(f: &Facts, now: DateTime<Utc>, a: &mut Assessment, value: ValueProblems) {
let mut flag = |level: State, problem: String| a.flag(level, problem);
let mut no_sync_at_all = false;
let sync = &f.sync;
if (sync.peers.is_some_and(|n| n > 0) || sync.synced_before)
&& let Some(window) = f.windows.sync_overdue
{
let since = sync
.last_success_at
.or(sync.last_cursor_at)
.unwrap_or(f.started);
let overdue = now - since;
if overdue >= window {
let problem = match sync.peers {
Some(0) => format!(
"no sync has succeeded in {}, and ~/.second-brain/sync-peers lists no peers \
although this device has synced before; add each peer to it as user@host, \
one per line, or set alert.sync_overdue_hours = 0 if this device no longer syncs",
human(overdue)
),
Some(n) => format!(
"no sync has succeeded in {} with {} in sync-peers; \
run `sb sync <peer>` to see why",
human(overdue),
counted(n as u64, "peer", "peers")
),
None => format!(
"no sync has succeeded in {} although this device has synced before; \
run `sb sync <user@host>` (or check the job that runs it), \
or set alert.sync_overdue_hours = 0 if this device no longer syncs",
human(overdue)
),
};
flag(State::Degraded, problem);
no_sync_at_all = true;
}
}
if !no_sync_at_all {
for problem in value.peers {
flag(State::Degraded, problem);
}
}
for problem in value.other {
flag(State::Degraded, problem);
}
if f.distill.mode == DistillMode::Llm
&& let (Some(oldest), Some(window)) = (f.distill.oldest_at, f.windows.undistilled)
&& now - oldest >= window
{
flag(
State::Degraded,
format!(
"{} waited {} to be distilled; run `sb consolidate --llm-url <endpoint>` or add an [llm] section, or set \
alert.undistilled_overdue_hours = 0 if this device does not distill",
if f.distill.pending == 1 {
"1 raw memory has".to_string()
} else {
format!("{} raw memories have", f.distill.pending)
},
human(now - oldest)
),
);
}
if let Some(e) = &f.progress_error
&& e.outlasted_grace(now)
{
flag(
State::Degraded,
format!(
"cannot check sync and distillation for {}: {}",
human(now - e.since),
e.message
),
);
}
}
pub(crate) fn counted(n: u64, one: &str, many: &str) -> String {
if n == 1 {
format!("1 {one}")
} else {
format!("{n} {many}")
}
}
pub(crate) fn human(d: Duration) -> String {
let minutes = d.num_minutes();
if minutes < 60 {
format!("{minutes}m")
} else if minutes < 48 * 60 {
format!("{}h {}m", minutes / 60, minutes % 60)
} else {
format!("{}d", minutes / (24 * 60))
}
}
#[cfg(test)]
mod tests {
use tokio::sync::mpsc::{UnboundedReceiver, unbounded_channel};
use super::*;
fn t0() -> DateTime<Utc> {
DateTime::parse_from_rfc3339("2026-10-31T12:00:00Z")
.unwrap()
.with_timezone(&Utc)
}
fn mins(m: i64) -> DateTime<Utc> {
t0() + Duration::minutes(m)
}
fn monitored() -> (Health, UnboundedReceiver<Transition>) {
let (tx, rx) = unbounded_channel();
(Health::new(t0(), Some(tx)), rx)
}
fn drain(rx: &mut UnboundedReceiver<Transition>) -> Vec<State> {
std::iter::from_fn(|| rx.try_recv().ok())
.map(|t| t.state)
.collect()
}
#[test]
fn a_fresh_daemon_that_keeps_up_is_ok() {
let (health, mut rx) = monitored();
health.watcher_started(mins(0));
health.transcripts_scanned(Some(mins(1)), mins(1));
health.ingested(Some(mins(1)), mins(1));
health.drained(0, mins(1));
let report = health.report();
assert_eq!(report.state, State::Ok);
assert_eq!(report.reason, None);
assert_eq!(report.capture.lag_secs, Some(0));
assert!(drain(&mut rx).is_empty());
}
#[test]
fn capture_lag_degrades_then_fails_then_recovers() {
let (health, mut rx) = monitored();
health.drained(0, mins(0));
health.ingested(Some(mins(1)), mins(1));
health.transcripts_scanned(Some(mins(10)), mins(10));
assert_eq!(health.report().state, State::Ok);
health.transcripts_scanned(Some(mins(20)), mins(20));
let report = health.report();
assert_eq!(report.state, State::Degraded);
assert_eq!(report.capture.lag_secs, Some(19 * 60));
assert!(report.reason.unwrap().contains("capture is 19m behind"));
health.drained(0, mins(125));
health.transcripts_scanned(Some(mins(125)), mins(125));
assert_eq!(health.report().state, State::Failing);
health.ingested(Some(mins(125)), mins(126));
assert_eq!(health.report().state, State::Ok);
assert_eq!(
drain(&mut rx),
vec![State::Degraded, State::Failing, State::Ok]
);
}
#[test]
fn transcripts_from_before_startup_are_not_lag() {
let (health, _rx) = monitored();
health.transcripts_scanned(Some(t0() - Duration::days(3)), mins(1));
assert_eq!(health.report().capture.lag_secs, Some(0));
assert_eq!(health.report().state, State::Ok);
}
#[test]
fn repeated_reports_in_one_state_send_one_transition() {
let (health, mut rx) = monitored();
for m in 0..=12 {
health.ingest_failed("database is locked", mins(m));
}
health.ingested(None, mins(13));
health.ingested(None, mins(14));
assert_eq!(drain(&mut rx), vec![State::Degraded, State::Ok]);
let report = health.report();
assert_eq!(report.since, mins(13));
let last = report.last_error.expect("the error is kept after recovery");
assert_eq!(last.source, "ingest");
assert_eq!(last.at, mins(12));
}
#[test]
fn an_error_that_clears_within_the_grace_sends_nothing() {
let (health, mut rx) = monitored();
health.watcher_started(mins(0));
health.ingest_failed("database is locked", mins(0));
health.drain_failed("database is locked", mins(1));
health.drained(0, mins(9));
health.ingested(None, mins(9));
assert_eq!(health.report().state, State::Ok);
assert!(drain(&mut rx).is_empty());
assert_eq!(health.report().last_error.unwrap().source, "drain");
}
#[test]
fn a_drain_error_degrades_once_it_outlasts_the_grace() {
let (health, _rx) = monitored();
health.watcher_started(mins(0));
for m in 0..10 {
health.drain_failed("disk I/O error", mins(m));
}
assert_eq!(health.report().state, State::Ok);
health.drain_failed("disk I/O error", mins(10));
let report = health.report();
assert_eq!(report.state, State::Degraded);
assert!(report.reason.unwrap().contains("index drain failed"));
health.drained(0, mins(11));
assert_eq!(health.report().state, State::Ok);
}
#[test]
fn a_long_ingest_batch_counts_as_the_watcher_reporting() {
let (health, _rx) = monitored();
health.watcher_started(mins(0));
health.ingested(None, mins(10));
health.ingest_failed("unreadable line", mins(19));
health.drained(0, mins(20));
let report = health.report();
assert_eq!(report.state, State::Ok, "{:?}", report.reason);
}
#[test]
fn an_outbox_backlog_only_counts_once_it_persists() {
let (health, _rx) = monitored();
health.drained(500, mins(0));
health.drained(800, mins(14));
health.watcher_started(mins(14));
assert_eq!(health.report().state, State::Ok);
health.drained(800, mins(15));
let report = health.report();
assert_eq!(report.state, State::Degraded);
assert_eq!(report.index.backlog_since, Some(mins(0)));
health.drained(0, mins(16));
assert_eq!(health.report().state, State::Ok);
assert_eq!(health.report().index.backlog_since, None);
}
#[test]
fn a_dead_watcher_fails_and_a_silent_one_degrades() {
let (health, _rx) = monitored();
health.watcher_failed("inotify watch limit reached", mins(0));
assert_eq!(health.report().state, State::Failing);
health.watcher_started(mins(1));
health.drained(0, mins(1));
assert_eq!(health.report().state, State::Ok);
health.drained(0, mins(16));
let report = health.report();
assert_eq!(report.state, State::Degraded);
assert!(report.reason.unwrap().contains("watcher has not reported"));
}
#[test]
fn index_and_repair_errors_degrade_until_the_next_success() {
let (health, _rx) = monitored();
health.watcher_started(mins(0));
health.drain_failed("disk I/O error", mins(0));
health.repair_failed("disk I/O error", mins(0));
let reason = health.report().reason.unwrap();
assert!(reason.contains("store repair failed"), "{reason}");
health.drain_failed("disk I/O error", mins(10));
let reason = health.report().reason.unwrap();
assert!(reason.contains("index drain failed"), "{reason}");
health.drained(0, mins(11));
assert_eq!(health.report().state, State::Degraded);
health.repaired(mins(12));
assert_eq!(health.report().state, State::Ok);
}
#[test]
fn missing_indexes_stay_degraded_without_a_drain() {
let (health, _rx) = monitored();
health.indexes_unavailable("tantivy meta.json missing", mins(0));
health.watcher_started(mins(90));
let report = health.report();
assert_eq!(report.state, State::Degraded);
let reason = report.reason.unwrap();
assert!(reason.contains("search indexes unavailable"), "{reason}");
assert!(!reason.contains("has not reported"), "{reason}");
}
#[test]
fn the_report_serializes_without_content() {
let (health, _rx) = monitored();
health.ingest_failed("database is locked", mins(0));
health.ingest_failed("database is locked", mins(10));
let json = serde_json::to_value(health.report()).unwrap();
assert_eq!(json["state"], "degraded");
assert_eq!(json["last_error"]["source"], "ingest");
for key in ["reason", "since", "capture", "index"] {
assert!(json.get(key).is_some(), "{key} missing: {json}");
}
}
#[test]
fn a_queue_entry_older_than_ten_minutes_degrades_until_it_drains() {
let (health, mut rx) = monitored();
health.watcher_started(mins(0));
health.drained(0, mins(0));
health.queue_checked(2, Some(mins(0)), mins(9));
assert_eq!(health.report().state, State::Ok);
assert_eq!(health.report().queue.depth, 2);
health.queue_checked(2, Some(mins(0)), mins(10));
let report = health.report();
assert_eq!(report.state, State::Degraded);
assert!(
report
.reason
.as_deref()
.unwrap()
.contains("write queue not draining"),
"{report:?}"
);
assert_eq!(report.queue.oldest_age_secs, Some(600));
health.queue_checked(0, None, mins(11));
assert_eq!(health.report().state, State::Ok);
assert_eq!(drain(&mut rx), vec![State::Degraded, State::Ok]);
}
#[test]
fn a_rejected_write_is_reported_by_tool_status_and_time_only() {
let (health, _rx) = monitored();
health.write_rejected("associate", 404, mins(3));
let json = serde_json::to_value(health.report()).unwrap();
assert_eq!(
json["queue"]["last_rejected"],
serde_json::json!({"tool": "associate", "status": 404, "at": "2026-10-31T12:03:00Z"})
);
assert_eq!(json["state"], "ok");
}
fn hours(h: i64) -> DateTime<Utc> {
t0() + Duration::hours(h)
}
fn alive(health: &Health, at: DateTime<Utc>) {
health.update(at, |f| {
f.capture.checked_at = Some(at);
f.index.checked_at = Some(at);
});
}
fn sync_seen(health: &Health, sync: SyncReport, at: DateTime<Utc>) {
let seen = ProgressSeen {
sync: Some(sync),
..ProgressSeen::default()
};
health.progress_checked(seen, at);
}
fn distill_seen(
health: &Health,
pending: u64,
oldest_at: Option<DateTime<Utc>>,
at: DateTime<Utc>,
) {
distill_seen_in(health, DistillMode::Llm, pending, oldest_at, at);
}
fn distill_seen_in(
health: &Health,
mode: DistillMode,
pending: u64,
oldest_at: Option<DateTime<Utc>>,
at: DateTime<Utc>,
) {
let distill = DistillReport {
mode,
pending,
oldest_at,
..DistillReport::default()
};
let seen = ProgressSeen {
distill: Some(distill),
..ProgressSeen::default()
};
health.progress_checked(seen, at);
}
fn synced(peers: usize, synced_before: bool, last: Option<DateTime<Utc>>) -> SyncReport {
SyncReport {
checked_at: None,
peers: Some(peers),
synced_before,
last_success_at: last,
last_cursor_at: None,
}
}
#[test]
fn without_a_stamp_the_window_counts_from_when_records_last_moved() {
let (health, _rx) = monitored();
let seen = SyncReport {
last_cursor_at: Some(hours(-48)),
..synced(0, true, None)
};
sync_seen(&health, seen.clone(), mins(0));
let report = health.report();
assert_eq!(report.state, State::Degraded);
let reason = report.reason.unwrap();
assert!(reason.contains("no sync has succeeded in 2d"), "{reason}");
let stamped = SyncReport {
last_success_at: Some(mins(0)),
..seen
};
sync_seen(&health, stamped, mins(0));
assert_eq!(health.report().state, State::Ok);
}
#[test]
fn reasons_count_one_peer_and_one_raw_in_the_singular() {
let (health, _rx) = monitored();
sync_seen(&health, synced(1, true, Some(hours(-30))), mins(0));
distill_seen(&health, 1, Some(hours(-30)), mins(0));
let reason = health.report().reason.unwrap();
assert!(reason.contains("with 1 peer in sync-peers"), "{reason}");
assert!(reason.contains("1 raw memory has waited"), "{reason}");
}
#[test]
fn a_device_that_synced_before_and_lost_its_peers_degrades_with_the_fix() {
let (health, mut rx) = monitored();
sync_seen(&health, synced(0, true, Some(hours(-25))), mins(0));
let report = health.report();
assert_eq!(report.state, State::Degraded);
let reason = report.reason.unwrap();
assert!(
reason.contains("no sync has succeeded in 25h 0m"),
"{reason}"
);
assert!(reason.contains("lists no peers"), "{reason}");
assert!(reason.contains("~/.second-brain/sync-peers"), "{reason}");
assert!(reason.contains("sync_overdue_hours = 0"), "{reason}");
assert_eq!(report.sync.peers, Some(0));
assert_eq!(report.sync.checked_at, Some(mins(0)));
assert_eq!(drain(&mut rx), vec![State::Degraded]);
sync_seen(&health, synced(1, true, Some(mins(1))), mins(1));
assert_eq!(health.report().state, State::Ok);
}
#[test]
fn a_device_synced_by_hand_without_sync_peers_stays_ok_while_it_syncs() {
let (health, mut rx) = monitored();
let by_hand = SyncReport {
peers: None,
..synced(0, true, Some(hours(10)))
};
alive(&health, hours(30));
sync_seen(&health, by_hand.clone(), hours(30));
assert_eq!(health.report().state, State::Ok);
alive(&health, hours(34));
sync_seen(&health, by_hand, hours(34));
let report = health.report();
assert_eq!(report.state, State::Degraded);
let reason = report.reason.unwrap();
assert!(
reason.contains("no sync has succeeded in 24h 0m"),
"{reason}"
);
assert!(reason.contains("sb sync <user@host>"), "{reason}");
assert!(reason.contains("sync_overdue_hours = 0"), "{reason}");
assert!(!reason.contains("sync-peers"), "{reason}");
assert_eq!(drain(&mut rx), vec![State::Degraded]);
}
#[test]
fn a_solo_device_that_never_synced_stays_ok() {
let (health, mut rx) = monitored();
for h in [0, 25, 100] {
alive(&health, hours(h));
sync_seen(&health, synced(0, false, None), hours(h));
}
let report = health.report();
assert_eq!(report.state, State::Ok, "{:?}", report.reason);
assert!(drain(&mut rx).is_empty());
}
#[test]
fn a_sync_overdue_past_the_window_degrades_only_on_a_device_that_syncs() {
let (health, _rx) = monitored();
alive(&health, hours(0));
sync_seen(&health, synced(2, true, Some(hours(-23))), hours(0));
assert_eq!(health.report().state, State::Ok);
alive(&health, hours(1));
sync_seen(&health, synced(2, true, Some(hours(-23))), hours(1));
let report = health.report();
assert_eq!(report.state, State::Degraded);
let reason = report.reason.unwrap();
assert!(
reason.contains("no sync has succeeded in 24h 0m"),
"{reason}"
);
assert!(reason.contains("2 peers"), "{reason}");
sync_seen(&health, synced(2, true, Some(hours(1))), hours(1));
assert_eq!(health.report().state, State::Ok);
sync_seen(&health, synced(0, false, Some(hours(-500))), hours(1));
assert_eq!(health.report().state, State::Ok);
}
#[test]
fn with_peers_and_no_recorded_success_the_window_starts_at_startup() {
let (health, _rx) = monitored();
alive(&health, hours(23));
sync_seen(&health, synced(1, false, None), hours(23));
assert_eq!(health.report().state, State::Ok);
alive(&health, hours(24));
sync_seen(&health, synced(1, false, None), hours(24));
let reason = health.report().reason.unwrap();
assert!(
reason.contains("no sync has succeeded in 24h 0m"),
"{reason}"
);
}
#[test]
fn raws_left_undistilled_past_the_window_degrade_with_the_count() {
let (health, mut rx) = monitored();
alive(&health, hours(0));
distill_seen(&health, 520, Some(hours(-23)), hours(0));
assert_eq!(health.report().state, State::Ok);
assert_eq!(health.report().distill.pending, 520);
alive(&health, hours(2));
distill_seen(&health, 520, Some(hours(-23)), hours(2));
let report = health.report();
assert_eq!(report.state, State::Degraded);
let reason = report.reason.unwrap();
assert!(
reason.contains("520 raw memories have waited 25h 0m to be distilled"),
"{reason}"
);
assert!(reason.contains("undistilled_overdue_hours = 0"), "{reason}");
alive(&health, hours(3));
distill_seen(&health, 0, None, hours(3));
assert_eq!(health.report().state, State::Ok);
assert_eq!(drain(&mut rx), vec![State::Degraded, State::Ok]);
}
#[test]
fn without_an_llm_endpoint_waiting_raws_report_the_mode_instead_of_degrading() {
let (health, mut rx) = monitored();
alive(&health, hours(100));
distill_seen_in(
&health,
DistillMode::NoGpu,
520,
Some(hours(-100)),
hours(100),
);
let report = health.report();
assert_eq!(report.state, State::Ok, "{:?}", report.reason);
assert_eq!(report.reason, None);
assert_eq!(report.distill.pending, 520);
let json = serde_json::to_value(&report).unwrap();
assert_eq!(json["distill"]["mode"], "no-gpu");
assert_eq!(json["distill"]["note"], NO_GPU_NOTE);
assert!(drain(&mut rx).is_empty());
}
#[test]
fn with_an_llm_endpoint_the_report_names_the_mode_and_carries_no_note() {
let (health, _rx) = monitored();
alive(&health, hours(2));
distill_seen(&health, 3, Some(hours(-23)), hours(2));
let report = health.report();
assert_eq!(report.state, State::Degraded);
let json = serde_json::to_value(&report).unwrap();
assert_eq!(json["distill"]["mode"], "llm");
assert!(json["distill"].get("note").is_none(), "{json}");
}
#[test]
fn a_window_turned_off_never_degrades() {
let (tx, _rx) = unbounded_channel();
let windows = Windows::from_config(&AlertConfig {
sync_overdue_hours: 0,
undistilled_overdue_hours: 0,
..AlertConfig::default()
});
assert_eq!(
windows,
Windows {
sync_overdue: None,
undistilled: None
}
);
let health = Health::new(t0(), Some(tx)).with_windows(windows);
alive(&health, hours(100));
sync_seen(&health, synced(1, true, Some(hours(-100))), hours(100));
distill_seen(&health, 9, Some(hours(-100)), hours(100));
assert_eq!(health.report().state, State::Ok);
sync_seen(&health, synced(0, true, Some(hours(-100))), hours(100));
assert_eq!(health.report().state, State::Ok);
}
#[test]
fn the_windows_come_from_the_alert_config() {
let windows = Windows::from_config(&AlertConfig {
sync_overdue_hours: 6,
undistilled_overdue_hours: 48,
..AlertConfig::default()
});
assert_eq!(windows.sync_overdue, Some(Duration::hours(6)));
assert_eq!(windows.undistilled, Some(Duration::hours(48)));
assert_eq!(Windows::default().sync_overdue, Some(Duration::hours(24)));
assert_eq!(Windows::default().undistilled, Some(Duration::hours(24)));
}
#[test]
fn announce_sends_the_current_state_once_unless_it_was_already_sent() {
let (health, mut rx) = monitored();
health.announce(mins(0));
let sent: Vec<Transition> = std::iter::from_fn(|| rx.try_recv().ok()).collect();
assert_eq!(
sent,
vec![Transition {
state: State::Ok,
reason: None,
at: mins(0),
}]
);
health.announce(mins(1));
assert!(drain(&mut rx).is_empty(), "ok was already sent");
sync_seen(&health, synced(0, true, Some(hours(-30))), mins(2));
health.announce(mins(3));
assert_eq!(drain(&mut rx), vec![State::Degraded]);
}
#[test]
fn announce_reassesses_and_sends_one_message_for_a_pending_change() {
let (health, mut rx) = monitored();
alive(&health, hours(0));
distill_seen(&health, 3, Some(hours(-23)), hours(0));
assert!(drain(&mut rx).is_empty());
health.announce(hours(2));
assert_eq!(drain(&mut rx), vec![State::Degraded]);
}
#[test]
fn serving_leaves_out_sync_and_distillation() {
let (health, _rx) = monitored();
alive(&health, hours(0));
let seen = ProgressSeen {
sync: Some(synced(1, true, Some(hours(-30)))),
distill: Some(DistillReport {
pending: 4,
oldest_at: Some(hours(-30)),
..DistillReport::default()
}),
ingested_sessions: None,
error: None,
};
health.progress_checked(seen, hours(0));
let report = health.report();
assert_eq!(report.state, State::Degraded);
assert_eq!(
report.serving,
Verdict {
state: State::Ok,
reason: None
}
);
health.watcher_failed("inotify watch limit reached", hours(0));
let report = health.report();
assert_eq!(report.state, State::Failing);
assert_eq!(report.serving.state, State::Failing);
let serving = report.serving.reason.unwrap();
assert!(serving.contains("watcher is not running"), "{serving}");
assert!(!serving.contains("sync"), "{serving}");
let json = serde_json::to_value(health.report()).unwrap();
assert_eq!(json["serving"]["state"], "failing");
}
#[test]
fn one_pass_that_clears_sync_and_finds_distillation_overdue_sends_nothing() {
let (health, mut rx) = monitored();
alive(&health, hours(0));
sync_seen(&health, synced(1, true, Some(hours(-30))), hours(0));
assert_eq!(drain(&mut rx), vec![State::Degraded]);
let seen = ProgressSeen {
sync: Some(synced(1, true, Some(hours(0)))),
distill: Some(DistillReport {
pending: 4,
oldest_at: Some(hours(-30)),
..DistillReport::default()
}),
ingested_sessions: None,
error: None,
};
health.progress_checked(seen, hours(0));
assert_eq!(health.report().state, State::Degraded);
assert!(drain(&mut rx).is_empty());
}
fn backfill_running(done: usize, failed: usize, at: DateTime<Utc>) -> BackfillReport {
BackfillReport {
state: BackfillState::Running,
total: 40,
done,
failed,
updated_at: Some(at),
}
}
#[test]
fn a_new_daemon_reports_the_backfill_not_started() {
let (health, _rx) = monitored();
let report = health.report();
assert_eq!(report.backfill, BackfillReport::default());
assert_eq!(report.backfill.state, BackfillState::NotStarted);
assert_eq!(report.backfill.total, 0);
assert_eq!(report.backfill.done, 0);
assert_eq!(report.backfill.failed, 0);
let json = serde_json::to_value(report).unwrap();
assert_eq!(json["backfill"]["state"], "not_started");
}
#[test]
fn backfill_progress_is_reported_and_survives_later_reports() {
let (health, _rx) = monitored();
health.backfill_progress(backfill_running(3, 1, mins(1)));
assert_eq!(health.report().backfill, backfill_running(3, 1, mins(1)));
health.ingested(None, mins(2));
assert_eq!(health.report().backfill, backfill_running(3, 1, mins(1)));
let done = BackfillReport {
state: BackfillState::Done,
total: 40,
done: 40,
failed: 1,
updated_at: Some(mins(3)),
};
health.backfill_progress(done.clone());
assert_eq!(health.report().backfill, done);
let json = serde_json::to_value(health.report()).unwrap();
assert_eq!(json["backfill"]["state"], "done");
assert_eq!(json["backfill"]["done"], 40);
}
#[test]
fn backfill_progress_leaves_the_state_and_reason_alone() {
let (health, mut rx) = monitored();
for m in 0..=12 {
health.ingest_failed("database is locked", mins(m));
}
assert_eq!(drain(&mut rx), vec![State::Degraded]);
let before = health.report();
health.backfill_progress(backfill_running(5, 2, mins(13)));
let after = health.report();
assert_eq!(after.state, before.state);
assert_eq!(after.reason, before.reason);
assert_eq!(after.since, before.since);
assert_eq!(after.serving, before.serving);
assert!(drain(&mut rx).is_empty());
}
fn empty_recall(session: &str) -> RecallEvent {
RecallEvent {
session: Some(session.to_string()),
verdict: crate::value_metrics::RecallVerdict::Empty,
memory_ids: Vec::new(),
}
}
#[test]
fn a_value_metric_crossing_its_threshold_sends_exactly_one_transition() {
let (health, mut rx) = monitored();
alive(&health, mins(0));
for _ in 0..49 {
health.recalled(empty_recall("s"), mins(1));
}
assert!(drain(&mut rx).is_empty());
for _ in 0..30 {
health.recalled(empty_recall("s"), mins(2));
}
let report = health.report();
assert_eq!(report.state, State::Degraded);
assert_eq!(report.serving.state, State::Ok);
let reason = report.reason.unwrap();
assert!(reason.contains("found nothing that cleared"), "{reason}");
assert_eq!(drain(&mut rx), vec![State::Degraded]);
}
#[test]
fn a_share_hovering_at_its_threshold_sends_one_transition_each_way() {
let (health, mut rx) = monitored();
let hit = || RecallEvent {
session: None,
verdict: crate::value_metrics::RecallVerdict::Hit,
memory_ids: Vec::new(),
};
for m in 0..50 {
alive(&health, mins(m));
let event = if m < 37 { empty_recall("s") } else { hit() };
health.recalled(event, mins(m));
}
assert_eq!(health.report().state, State::Ok, "37 of 50 is 74%");
for _ in 0..3 {
health.recalled(empty_recall("s"), mins(50));
}
assert_eq!(drain(&mut rx), vec![State::Degraded], "40 of 53 is 75.5%");
for m in 51..56 {
alive(&health, mins(m));
health.recalled(hit(), mins(m));
health.recalled(empty_recall("s"), mins(m));
}
assert!(drain(&mut rx).is_empty(), "74% then 71% stays degraded");
for m in 56..66 {
alive(&health, mins(m));
health.recalled(hit(), mins(m));
}
assert_eq!(health.report().value.recall.recalls, 73);
assert_eq!(drain(&mut rx), vec![State::Ok], "45 of 73 is 62%");
}
#[test]
fn peer_lag_is_left_out_when_no_sync_succeeded_at_all() {
let (health, _rx) = monitored();
health.sync_peers_listed(&["u@pi".to_string()], hours(0));
alive(&health, hours(30));
let reason = health.report().reason.unwrap();
assert!(reason.contains("no sync with u@pi"), "{reason}");
sync_seen(&health, synced(1, true, Some(hours(-30))), hours(30));
let reason = health.report().reason.unwrap();
assert!(reason.contains("no sync has succeeded in"), "{reason}");
assert!(!reason.contains("no sync with u@pi"), "{reason}");
assert_eq!(health.report().value.sync_peers.len(), 1);
}
#[test]
fn the_ingest_window_starts_after_startup_and_any_backfill() {
let (health, _rx) = monitored();
assert_eq!(health.ingest_window_start(mins(5)), Some(t0()));
assert_eq!(
health.ingest_window_start(hours(30)),
Some(hours(30) - WINDOW)
);
health.backfill_progress(backfill_running(1, 0, mins(1)));
assert_eq!(health.ingest_window_start(mins(5)), None);
health.backfill_progress(BackfillReport {
state: BackfillState::Done,
updated_at: Some(mins(20)),
..BackfillReport::default()
});
assert_eq!(health.ingest_window_start(mins(30)), Some(mins(20)));
}
#[test]
fn progress_sets_the_ingested_sessions_and_a_failed_read_clears_them() {
let (health, _rx) = monitored();
health.recalled(empty_recall("a"), mins(1));
let seen = ProgressSeen {
ingested_sessions: Some(vec!["a".to_string(), "b".to_string()]),
..ProgressSeen::default()
};
health.progress_checked(seen, mins(2));
let recall = health.report().value.recall;
assert_eq!(recall.sessions_ingested, Some(2));
assert_eq!(recall.session_recall_rate, Some(0.5));
health.progress_checked(ProgressSeen::default(), mins(3));
assert_eq!(health.report().value.recall.sessions_ingested, None);
}
#[test]
fn every_value_metric_is_in_the_report_json() {
let (health, _rx) = monitored();
health.sync_peers_listed(&["u@pi".to_string()], mins(0));
health.store_measured(Some(StoreSize::default()), mins(0));
health.resident_measured(Some(ResidentSize::default()), mins(0));
let json = serde_json::to_value(health.report()).unwrap();
let keys = |value: &serde_json::Value| -> Vec<String> {
let mut keys: Vec<String> = value.as_object().unwrap().keys().cloned().collect();
keys.sort();
keys
};
assert_eq!(
keys(&json["sync_peers"][0]),
[
"consecutive_failures",
"lag_secs",
"last_attempt_at",
"last_success_at",
"peer"
]
);
assert_eq!(
keys(&json["store"]),
[
"archive_bytes",
"checked_at",
"db_bytes",
"free_bytes",
"index_bytes",
"raw_index_bytes",
"total_bytes",
"wal_bytes"
]
);
assert_eq!(
keys(&json["resident"]),
[
"checked_at",
"components",
"limit_bytes",
"raw_index_bytes",
"rss_anon_bytes",
"unattributed_bytes"
]
);
assert_eq!(
keys(&json["resident"]["components"]),
[
"allocator_free",
"embedder",
"lexical",
"reranker",
"sqlite",
"vector_reader",
"vector_writer"
]
);
for key in ["recall", "recalled", "sync_peers", "store", "resident"] {
assert!(json.get(key).is_some(), "{key} missing: {json}");
}
for key in [
"window_secs",
"recalls",
"hits",
"low_score",
"unjudged",
"empty",
"empty_share",
"low_score_share",
"unjudged_share",
"sessions_with_recall",
"sessions_ingested",
"session_recall_rate",
] {
assert!(json["recall"].get(key).is_some(), "recall.{key}: {json}");
}
for key in [
"window_secs",
"memories",
"corrected",
"used",
"corrected_share",
"used_share",
] {
assert!(
json["recalled"].get(key).is_some(),
"recalled.{key}: {json}"
);
}
}
}