use std::sync::Mutex;
use chrono::{DateTime, Duration, Utc};
use second_brain_core::config::AlertConfig;
use second_brain_core::sqlite_store::VacuumSpace;
use serde::Serialize;
use tokio::sync::mpsc::UnboundedSender;
const CAPTURE_LAG_DEGRADED: Duration = Duration::minutes(15);
const CAPTURE_LAG_FAILING: Duration = Duration::hours(2);
const BACKLOG_DEGRADED: Duration = Duration::minutes(15);
const BACKLOG_FAILING: Duration = Duration::hours(2);
const WATCHER_SILENT: Duration = Duration::minutes(15);
const DRAIN_SILENT: Duration = Duration::hours(1);
const ERROR_GRACE: Duration = Duration::minutes(10);
const QUEUE_STUCK: Duration = Duration::minutes(10);
#[derive(Debug, Clone, Copy, Default, PartialEq, Eq, PartialOrd, Ord, Serialize)]
#[serde(rename_all = "lowercase")]
pub enum State {
#[default]
Ok,
Degraded,
Failing,
}
#[derive(Debug, Clone, PartialEq, Serialize)]
pub struct Transition {
pub state: State,
pub reason: Option<String>,
pub at: DateTime<Utc>,
}
#[derive(Debug, Clone, Serialize)]
pub struct ErrorRecord {
pub source: &'static str,
pub message: String,
pub at: DateTime<Utc>,
}
#[derive(Debug, Clone, Default, Serialize)]
pub struct CaptureReport {
pub checked_at: Option<DateTime<Utc>>,
pub newest_transcript_at: Option<DateTime<Utc>>,
pub captured_through: Option<DateTime<Utc>>,
pub lag_secs: Option<i64>,
}
#[derive(Debug, Clone, Default, Serialize)]
pub struct IndexReport {
pub checked_at: Option<DateTime<Utc>>,
pub outbox_pending: Option<u64>,
pub backlog_since: Option<DateTime<Utc>>,
}
#[derive(Debug, Clone, PartialEq, Serialize)]
pub struct RejectedWrite {
pub tool: String,
pub status: u16,
pub at: DateTime<Utc>,
}
#[derive(Debug, Clone, Default, Serialize)]
pub struct QueueReport {
pub depth: u64,
pub oldest_at: Option<DateTime<Utc>>,
pub oldest_age_secs: Option<i64>,
pub last_rejected: Option<RejectedWrite>,
}
#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize)]
#[serde(rename_all = "snake_case")]
pub enum MaintenanceState {
Pending,
WaitingForSpace,
MergingIndex,
Vacuuming,
Done,
}
#[derive(Debug, Clone, Serialize)]
pub struct MaintenanceReport {
pub state: MaintenanceState,
pub since: Option<DateTime<Utc>>,
pub space: Option<VacuumSpace>,
pub last_error: Option<String>,
}
impl Default for MaintenanceReport {
fn default() -> Self {
Self {
state: MaintenanceState::Pending,
since: None,
space: None,
last_error: None,
}
}
}
#[derive(Debug, Clone, Default, Serialize)]
pub struct SyncReport {
pub checked_at: Option<DateTime<Utc>>,
pub peers: Option<usize>,
pub synced_before: bool,
pub last_success_at: Option<DateTime<Utc>>,
pub last_cursor_at: Option<DateTime<Utc>>,
}
#[derive(Debug, Clone, Default, Serialize)]
pub struct DistillReport {
pub checked_at: Option<DateTime<Utc>>,
pub pending: u64,
pub oldest_at: Option<DateTime<Utc>>,
}
#[derive(Debug, Clone, Copy, PartialEq)]
pub struct Windows {
pub sync_overdue: Option<Duration>,
pub undistilled: Option<Duration>,
}
impl Windows {
pub fn from_config(config: &AlertConfig) -> Self {
let hours = |h: u64| {
i64::try_from(h)
.ok()
.filter(|&h| h > 0)
.and_then(Duration::try_hours)
};
Self {
sync_overdue: hours(config.sync_overdue_hours),
undistilled: hours(config.undistilled_overdue_hours),
}
}
}
impl Default for Windows {
fn default() -> Self {
Self::from_config(&AlertConfig::default())
}
}
#[derive(Debug, Clone, Default)]
pub struct ProgressSeen {
pub sync: Option<SyncReport>,
pub distill: Option<DistillReport>,
pub error: Option<String>,
}
#[derive(Debug, Clone, PartialEq, Serialize)]
pub struct Verdict {
pub state: State,
pub reason: Option<String>,
}
#[derive(Debug, Clone, Serialize)]
pub struct HealthReport {
pub state: State,
pub reason: Option<String>,
pub since: DateTime<Utc>,
pub serving: Verdict,
pub capture: CaptureReport,
pub index: IndexReport,
pub queue: QueueReport,
pub maintenance: MaintenanceReport,
pub sync: SyncReport,
pub distill: DistillReport,
pub last_error: Option<ErrorRecord>,
}
struct Facts {
started: DateTime<Utc>,
capture: CaptureReport,
watcher_error: Option<String>,
ingest_error: Option<Streak>,
index: IndexReport,
drain_error: Option<Streak>,
indexes_error: Option<String>,
repair_error: Option<String>,
queue: QueueReport,
maintenance: MaintenanceReport,
sync: SyncReport,
distill: DistillReport,
progress_error: Option<Streak>,
windows: Windows,
last_error: Option<ErrorRecord>,
}
struct Streak {
message: String,
since: DateTime<Utc>,
}
impl Streak {
fn extend(previous: Option<Streak>, message: String, now: DateTime<Utc>) -> Option<Streak> {
let since = previous.map_or(now, |p| p.since);
Some(Streak { message, since })
}
fn outlasted_grace(&self, now: DateTime<Utc>) -> bool {
now - self.since >= ERROR_GRACE
}
}
struct Inner {
facts: Facts,
report: HealthReport,
sent: Option<State>,
}
pub struct Health {
inner: Mutex<Inner>,
transitions: Option<UnboundedSender<Transition>>,
}
impl Health {
pub fn new(now: DateTime<Utc>, transitions: Option<UnboundedSender<Transition>>) -> Self {
let facts = Facts {
started: now,
capture: CaptureReport::default(),
watcher_error: None,
ingest_error: None,
index: IndexReport::default(),
drain_error: None,
indexes_error: None,
repair_error: None,
queue: QueueReport::default(),
maintenance: MaintenanceReport::default(),
sync: SyncReport::default(),
distill: DistillReport::default(),
progress_error: None,
windows: Windows::default(),
last_error: None,
};
let report = HealthReport {
state: State::Ok,
reason: None,
since: now,
serving: Verdict {
state: State::Ok,
reason: None,
},
capture: CaptureReport::default(),
index: IndexReport::default(),
queue: QueueReport::default(),
maintenance: MaintenanceReport::default(),
sync: SyncReport::default(),
distill: DistillReport::default(),
last_error: None,
};
Self {
inner: Mutex::new(Inner {
facts,
report,
sent: None,
}),
transitions,
}
}
pub fn with_windows(self, windows: Windows) -> Self {
self.lock().facts.windows = windows;
self
}
pub fn progress_checked(&self, seen: ProgressSeen, now: DateTime<Utc>) {
self.update(now, |f| {
if let Some(sync) = seen.sync {
f.sync = SyncReport {
checked_at: Some(now),
..sync
};
}
if let Some(distill) = seen.distill {
f.distill = DistillReport {
checked_at: Some(now),
..distill
};
}
match seen.error {
None => f.progress_error = None,
Some(error) => {
f.record_error("progress", &error, now);
f.progress_error = Streak::extend(f.progress_error.take(), error, now);
}
}
});
}
pub fn announce(&self, now: DateTime<Utc>) {
let mut inner = self.lock();
self.reassess(&mut inner, now, true);
}
pub fn report(&self) -> HealthReport {
self.lock().report.clone()
}
pub fn transcripts_scanned(&self, newest: Option<DateTime<Utc>>, now: DateTime<Utc>) {
self.update(now, |f| {
f.capture.checked_at = Some(now);
f.capture.newest_transcript_at = newest;
});
}
pub fn watcher_started(&self, now: DateTime<Utc>) {
self.update(now, |f| {
f.watcher_error = None;
f.capture.checked_at = Some(now);
});
}
pub fn watcher_failed(&self, error: &str, now: DateTime<Utc>) {
self.update(now, |f| {
f.watcher_error = Some(error.to_string());
f.record_error("watcher", error, now);
});
}
pub fn ingested(&self, transcript_mtime: Option<DateTime<Utc>>, now: DateTime<Utc>) {
self.update(now, |f| {
f.ingest_error = None;
f.capture.checked_at = Some(now);
if let Some(mtime) = transcript_mtime {
let through = f.capture.captured_through.map_or(mtime, |t| t.max(mtime));
f.capture.captured_through = Some(through);
}
});
}
pub fn ingest_failed(&self, error: &str, now: DateTime<Utc>) {
self.update(now, |f| {
f.ingest_error = Streak::extend(f.ingest_error.take(), error.to_string(), now);
f.capture.checked_at = Some(now);
f.record_error("ingest", error, now);
});
}
pub fn drained(&self, outbox_pending: u64, now: DateTime<Utc>) {
self.update(now, |f| {
f.drain_error = None;
f.index.checked_at = Some(now);
f.index.outbox_pending = Some(outbox_pending);
f.index.backlog_since = if outbox_pending == 0 {
None
} else {
Some(f.index.backlog_since.unwrap_or(now))
};
});
}
pub fn drain_failed(&self, error: &str, now: DateTime<Utc>) {
self.update(now, |f| {
f.index.checked_at = Some(now);
f.drain_error = Streak::extend(f.drain_error.take(), error.to_string(), now);
f.record_error("drain", error, now);
});
}
pub fn indexes_unavailable(&self, error: &str, now: DateTime<Utc>) {
self.update(now, |f| {
f.indexes_error = Some(error.to_string());
f.record_error("index", error, now);
});
}
pub fn repaired(&self, now: DateTime<Utc>) {
self.update(now, |f| f.repair_error = None);
}
pub fn repair_failed(&self, error: &str, now: DateTime<Utc>) {
self.update(now, |f| {
f.repair_error = Some(error.to_string());
f.record_error("repair", error, now);
});
}
pub fn queue_checked(&self, depth: u64, oldest_at: Option<DateTime<Utc>>, now: DateTime<Utc>) {
self.update(now, |f| {
f.queue.depth = depth;
f.queue.oldest_at = oldest_at;
});
}
pub fn write_rejected(&self, tool: &str, status: u16, now: DateTime<Utc>) {
self.update(now, |f| {
f.queue.last_rejected = Some(RejectedWrite {
tool: tool.to_string(),
status,
at: now,
});
});
}
pub fn maintenance(
&self,
state: MaintenanceState,
space: Option<VacuumSpace>,
now: DateTime<Utc>,
) {
self.update(now, |f| {
if f.maintenance.state != state {
f.maintenance.since = Some(now);
}
f.maintenance.state = state;
if space.is_some() {
f.maintenance.space = space;
}
if state == MaintenanceState::Done {
f.maintenance.last_error = None;
}
});
}
pub fn maintenance_failed(&self, error: &str, now: DateTime<Utc>) {
self.update(now, |f| {
f.maintenance.last_error = Some(error.to_string());
f.record_error("maintenance", error, now);
});
}
fn lock(&self) -> std::sync::MutexGuard<'_, Inner> {
match self.inner.lock() {
Ok(g) => g,
Err(poisoned) => poisoned.into_inner(),
}
}
fn update(&self, now: DateTime<Utc>, change: impl FnOnce(&mut Facts)) {
let mut inner = self.lock();
change(&mut inner.facts);
self.reassess(&mut inner, now, false);
}
fn reassess(&self, inner: &mut Inner, now: DateTime<Utc>, announcing: bool) {
let mut serving = Assessment::default();
assess_serving(&inner.facts, now, &mut serving);
let mut overall = serving.clone();
assess_upkeep(&inner.facts, now, &mut overall);
let (state, reason) = overall.verdict();
let changed = state != inner.report.state;
let since = if changed { now } else { inner.report.since };
let mut capture = inner.facts.capture.clone();
capture.lag_secs = capture_lag(&inner.facts).map(|d| d.num_seconds());
let mut queue = inner.facts.queue.clone();
queue.oldest_age_secs = queue.oldest_at.map(|t| (now - t).num_seconds().max(0));
inner.report = HealthReport {
state,
reason: reason.clone(),
since,
serving: Verdict {
state: serving.state,
reason: serving.verdict().1,
},
capture,
index: inner.facts.index.clone(),
queue,
maintenance: inner.facts.maintenance.clone(),
sync: inner.facts.sync.clone(),
distill: inner.facts.distill.clone(),
last_error: inner.facts.last_error.clone(),
};
let unsent = inner.sent != Some(state);
if (changed || (announcing && unsent))
&& let Some(tx) = &self.transitions
{
let _ = tx.send(Transition {
state,
reason,
at: now,
});
inner.sent = Some(state);
}
}
}
impl Facts {
fn record_error(&mut self, source: &'static str, message: &str, at: DateTime<Utc>) {
self.last_error = Some(ErrorRecord {
source,
message: message.to_string(),
at,
});
}
}
fn capture_lag(f: &Facts) -> Option<Duration> {
let newest = f.capture.newest_transcript_at?;
let through = f
.capture
.captured_through
.map_or(f.started, |t| t.max(f.started));
Some((newest - through).max(Duration::zero()))
}
#[derive(Clone, Default)]
struct Assessment {
state: State,
problems: Vec<String>,
}
impl Assessment {
fn flag(&mut self, level: State, problem: String) {
self.state = self.state.max(level);
self.problems.push(problem);
}
fn verdict(&self) -> (State, Option<String>) {
let reason = (!self.problems.is_empty()).then(|| self.problems.join("; "));
(self.state, reason)
}
}
fn assess_serving(f: &Facts, now: DateTime<Utc>, a: &mut Assessment) {
let mut flag = |level: State, problem: String| a.flag(level, problem);
match &f.watcher_error {
Some(e) => flag(
State::Failing,
format!("transcript watcher is not running: {e}"),
),
None => {
let last = f.capture.checked_at.unwrap_or(f.started);
if now - last >= WATCHER_SILENT {
flag(
State::Degraded,
format!(
"transcript watcher has not reported for {}",
human(now - last)
),
);
}
}
}
if let Some(lag) = capture_lag(f)
&& lag >= CAPTURE_LAG_DEGRADED
{
let level = if lag >= CAPTURE_LAG_FAILING {
State::Failing
} else {
State::Degraded
};
flag(
level,
format!("capture is {} behind the newest transcript", human(lag)),
);
}
if let Some(e) = &f.ingest_error
&& e.outlasted_grace(now)
{
flag(
State::Degraded,
format!(
"ingest has failed for {}: {}",
human(now - e.since),
e.message
),
);
}
if let Some(e) = &f.drain_error
&& e.outlasted_grace(now)
{
flag(
State::Degraded,
format!(
"index drain failed for {}: {}",
human(now - e.since),
e.message
),
);
}
match &f.indexes_error {
Some(e) => flag(State::Degraded, format!("search indexes unavailable: {e}")),
None => {
let last = f.index.checked_at.unwrap_or(f.started);
if now - last >= DRAIN_SILENT {
flag(
State::Degraded,
format!("index drain has not reported for {}", human(now - last)),
);
}
}
}
if let (Some(since), Some(pending)) = (f.index.backlog_since, f.index.outbox_pending) {
let age = now - since;
if age >= BACKLOG_DEGRADED {
let level = if age >= BACKLOG_FAILING {
State::Failing
} else {
State::Degraded
};
flag(
level,
format!("{pending} changes have waited {} to be indexed", human(age)),
);
}
}
if let Some(e) = &f.repair_error {
flag(State::Degraded, format!("last store repair failed: {e}"));
}
if let Some(oldest) = f.queue.oldest_at
&& now - oldest >= QUEUE_STUCK
{
flag(
State::Degraded,
format!(
"write queue not draining ({} queued, oldest {})",
f.queue.depth,
human(now - oldest)
),
);
}
match (&f.maintenance.state, &f.maintenance.space) {
(MaintenanceState::Vacuuming, _) => flag(
State::Degraded,
"store maintenance is running; writes are queued until it finishes".to_string(),
),
(MaintenanceState::WaitingForSpace, Some(space)) => flag(
State::Degraded,
format!(
"store compaction waits for disk space: it needs {} MB free at {} and has {} MB",
space.needed_bytes / 1_000_000,
space.path.display(),
space.free_bytes / 1_000_000
),
),
_ => {}
}
if let Some(e) = &f.maintenance.last_error {
flag(State::Degraded, format!("store compaction failed: {e}"));
}
}
fn assess_upkeep(f: &Facts, now: DateTime<Utc>, a: &mut Assessment) {
let mut flag = |level: State, problem: String| a.flag(level, problem);
let sync = &f.sync;
if (sync.peers.is_some_and(|n| n > 0) || sync.synced_before)
&& let Some(window) = f.windows.sync_overdue
{
let since = sync
.last_success_at
.or(sync.last_cursor_at)
.unwrap_or(f.started);
let overdue = now - since;
if overdue >= window {
let problem = match sync.peers {
Some(0) => format!(
"no sync has succeeded in {}, and ~/.second-brain/sync-peers lists no peers \
although this device has synced before; add each peer to it as user@host, \
one per line, or set alert.sync_overdue_hours = 0 if this device no longer syncs",
human(overdue)
),
Some(n) => format!(
"no sync has succeeded in {} with {} in sync-peers; \
run `sb sync <peer>` to see why",
human(overdue),
counted(n as u64, "peer", "peers")
),
None => format!(
"no sync has succeeded in {} although this device has synced before; \
run `sb sync <user@host>` (or check the job that runs it), \
or set alert.sync_overdue_hours = 0 if this device no longer syncs",
human(overdue)
),
};
flag(State::Degraded, problem);
}
}
if let (Some(oldest), Some(window)) = (f.distill.oldest_at, f.windows.undistilled)
&& now - oldest >= window
{
flag(
State::Degraded,
format!(
"{} waited {} to be distilled; run `sb consolidate`, or set \
alert.undistilled_overdue_hours = 0 if this device does not distill",
if f.distill.pending == 1 {
"1 raw memory has".to_string()
} else {
format!("{} raw memories have", f.distill.pending)
},
human(now - oldest)
),
);
}
if let Some(e) = &f.progress_error
&& e.outlasted_grace(now)
{
flag(
State::Degraded,
format!(
"cannot check sync and distillation for {}: {}",
human(now - e.since),
e.message
),
);
}
}
fn counted(n: u64, one: &str, many: &str) -> String {
if n == 1 {
format!("1 {one}")
} else {
format!("{n} {many}")
}
}
fn human(d: Duration) -> String {
let minutes = d.num_minutes();
if minutes < 60 {
format!("{minutes}m")
} else if minutes < 48 * 60 {
format!("{}h {}m", minutes / 60, minutes % 60)
} else {
format!("{}d", minutes / (24 * 60))
}
}
#[cfg(test)]
mod tests {
use tokio::sync::mpsc::{UnboundedReceiver, unbounded_channel};
use super::*;
fn t0() -> DateTime<Utc> {
DateTime::parse_from_rfc3339("2026-10-31T12:00:00Z")
.unwrap()
.with_timezone(&Utc)
}
fn mins(m: i64) -> DateTime<Utc> {
t0() + Duration::minutes(m)
}
fn monitored() -> (Health, UnboundedReceiver<Transition>) {
let (tx, rx) = unbounded_channel();
(Health::new(t0(), Some(tx)), rx)
}
fn drain(rx: &mut UnboundedReceiver<Transition>) -> Vec<State> {
std::iter::from_fn(|| rx.try_recv().ok())
.map(|t| t.state)
.collect()
}
#[test]
fn a_fresh_daemon_that_keeps_up_is_ok() {
let (health, mut rx) = monitored();
health.watcher_started(mins(0));
health.transcripts_scanned(Some(mins(1)), mins(1));
health.ingested(Some(mins(1)), mins(1));
health.drained(0, mins(1));
let report = health.report();
assert_eq!(report.state, State::Ok);
assert_eq!(report.reason, None);
assert_eq!(report.capture.lag_secs, Some(0));
assert!(drain(&mut rx).is_empty());
}
#[test]
fn capture_lag_degrades_then_fails_then_recovers() {
let (health, mut rx) = monitored();
health.drained(0, mins(0));
health.ingested(Some(mins(1)), mins(1));
health.transcripts_scanned(Some(mins(10)), mins(10));
assert_eq!(health.report().state, State::Ok);
health.transcripts_scanned(Some(mins(20)), mins(20));
let report = health.report();
assert_eq!(report.state, State::Degraded);
assert_eq!(report.capture.lag_secs, Some(19 * 60));
assert!(report.reason.unwrap().contains("capture is 19m behind"));
health.drained(0, mins(125));
health.transcripts_scanned(Some(mins(125)), mins(125));
assert_eq!(health.report().state, State::Failing);
health.ingested(Some(mins(125)), mins(126));
assert_eq!(health.report().state, State::Ok);
assert_eq!(
drain(&mut rx),
vec![State::Degraded, State::Failing, State::Ok]
);
}
#[test]
fn transcripts_from_before_startup_are_not_lag() {
let (health, _rx) = monitored();
health.transcripts_scanned(Some(t0() - Duration::days(3)), mins(1));
assert_eq!(health.report().capture.lag_secs, Some(0));
assert_eq!(health.report().state, State::Ok);
}
#[test]
fn repeated_reports_in_one_state_send_one_transition() {
let (health, mut rx) = monitored();
for m in 0..=12 {
health.ingest_failed("database is locked", mins(m));
}
health.ingested(None, mins(13));
health.ingested(None, mins(14));
assert_eq!(drain(&mut rx), vec![State::Degraded, State::Ok]);
let report = health.report();
assert_eq!(report.since, mins(13));
let last = report.last_error.expect("the error is kept after recovery");
assert_eq!(last.source, "ingest");
assert_eq!(last.at, mins(12));
}
#[test]
fn an_error_that_clears_within_the_grace_sends_nothing() {
let (health, mut rx) = monitored();
health.watcher_started(mins(0));
health.ingest_failed("database is locked", mins(0));
health.drain_failed("database is locked", mins(1));
health.drained(0, mins(9));
health.ingested(None, mins(9));
assert_eq!(health.report().state, State::Ok);
assert!(drain(&mut rx).is_empty());
assert_eq!(health.report().last_error.unwrap().source, "drain");
}
#[test]
fn a_drain_error_degrades_once_it_outlasts_the_grace() {
let (health, _rx) = monitored();
health.watcher_started(mins(0));
for m in 0..10 {
health.drain_failed("disk I/O error", mins(m));
}
assert_eq!(health.report().state, State::Ok);
health.drain_failed("disk I/O error", mins(10));
let report = health.report();
assert_eq!(report.state, State::Degraded);
assert!(report.reason.unwrap().contains("index drain failed"));
health.drained(0, mins(11));
assert_eq!(health.report().state, State::Ok);
}
#[test]
fn a_long_ingest_batch_counts_as_the_watcher_reporting() {
let (health, _rx) = monitored();
health.watcher_started(mins(0));
health.ingested(None, mins(10));
health.ingest_failed("unreadable line", mins(19));
health.drained(0, mins(20));
let report = health.report();
assert_eq!(report.state, State::Ok, "{:?}", report.reason);
}
#[test]
fn an_outbox_backlog_only_counts_once_it_persists() {
let (health, _rx) = monitored();
health.drained(500, mins(0));
health.drained(800, mins(14));
health.watcher_started(mins(14));
assert_eq!(health.report().state, State::Ok);
health.drained(800, mins(15));
let report = health.report();
assert_eq!(report.state, State::Degraded);
assert_eq!(report.index.backlog_since, Some(mins(0)));
health.drained(0, mins(16));
assert_eq!(health.report().state, State::Ok);
assert_eq!(health.report().index.backlog_since, None);
}
#[test]
fn a_dead_watcher_fails_and_a_silent_one_degrades() {
let (health, _rx) = monitored();
health.watcher_failed("inotify watch limit reached", mins(0));
assert_eq!(health.report().state, State::Failing);
health.watcher_started(mins(1));
health.drained(0, mins(1));
assert_eq!(health.report().state, State::Ok);
health.drained(0, mins(16));
let report = health.report();
assert_eq!(report.state, State::Degraded);
assert!(report.reason.unwrap().contains("watcher has not reported"));
}
#[test]
fn index_and_repair_errors_degrade_until_the_next_success() {
let (health, _rx) = monitored();
health.watcher_started(mins(0));
health.drain_failed("disk I/O error", mins(0));
health.repair_failed("disk I/O error", mins(0));
let reason = health.report().reason.unwrap();
assert!(reason.contains("store repair failed"), "{reason}");
health.drain_failed("disk I/O error", mins(10));
let reason = health.report().reason.unwrap();
assert!(reason.contains("index drain failed"), "{reason}");
health.drained(0, mins(11));
assert_eq!(health.report().state, State::Degraded);
health.repaired(mins(12));
assert_eq!(health.report().state, State::Ok);
}
#[test]
fn missing_indexes_stay_degraded_without_a_drain() {
let (health, _rx) = monitored();
health.indexes_unavailable("tantivy meta.json missing", mins(0));
health.watcher_started(mins(90));
let report = health.report();
assert_eq!(report.state, State::Degraded);
let reason = report.reason.unwrap();
assert!(reason.contains("search indexes unavailable"), "{reason}");
assert!(!reason.contains("has not reported"), "{reason}");
}
#[test]
fn the_report_serializes_without_content() {
let (health, _rx) = monitored();
health.ingest_failed("database is locked", mins(0));
health.ingest_failed("database is locked", mins(10));
let json = serde_json::to_value(health.report()).unwrap();
assert_eq!(json["state"], "degraded");
assert_eq!(json["last_error"]["source"], "ingest");
for key in ["reason", "since", "capture", "index"] {
assert!(json.get(key).is_some(), "{key} missing: {json}");
}
}
#[test]
fn a_queue_entry_older_than_ten_minutes_degrades_until_it_drains() {
let (health, mut rx) = monitored();
health.watcher_started(mins(0));
health.drained(0, mins(0));
health.queue_checked(2, Some(mins(0)), mins(9));
assert_eq!(health.report().state, State::Ok);
assert_eq!(health.report().queue.depth, 2);
health.queue_checked(2, Some(mins(0)), mins(10));
let report = health.report();
assert_eq!(report.state, State::Degraded);
assert!(
report
.reason
.as_deref()
.unwrap()
.contains("write queue not draining"),
"{report:?}"
);
assert_eq!(report.queue.oldest_age_secs, Some(600));
health.queue_checked(0, None, mins(11));
assert_eq!(health.report().state, State::Ok);
assert_eq!(drain(&mut rx), vec![State::Degraded, State::Ok]);
}
#[test]
fn a_rejected_write_is_reported_by_tool_status_and_time_only() {
let (health, _rx) = monitored();
health.write_rejected("associate", 404, mins(3));
let json = serde_json::to_value(health.report()).unwrap();
assert_eq!(
json["queue"]["last_rejected"],
serde_json::json!({"tool": "associate", "status": 404, "at": "2026-10-31T12:03:00Z"})
);
assert_eq!(json["state"], "ok");
}
fn hours(h: i64) -> DateTime<Utc> {
t0() + Duration::hours(h)
}
fn alive(health: &Health, at: DateTime<Utc>) {
health.update(at, |f| {
f.capture.checked_at = Some(at);
f.index.checked_at = Some(at);
});
}
fn sync_seen(health: &Health, sync: SyncReport, at: DateTime<Utc>) {
let seen = ProgressSeen {
sync: Some(sync),
..ProgressSeen::default()
};
health.progress_checked(seen, at);
}
fn distill_seen(
health: &Health,
pending: u64,
oldest_at: Option<DateTime<Utc>>,
at: DateTime<Utc>,
) {
let distill = DistillReport {
checked_at: None,
pending,
oldest_at,
};
let seen = ProgressSeen {
distill: Some(distill),
..ProgressSeen::default()
};
health.progress_checked(seen, at);
}
fn synced(peers: usize, synced_before: bool, last: Option<DateTime<Utc>>) -> SyncReport {
SyncReport {
checked_at: None,
peers: Some(peers),
synced_before,
last_success_at: last,
last_cursor_at: None,
}
}
#[test]
fn without_a_stamp_the_window_counts_from_when_records_last_moved() {
let (health, _rx) = monitored();
let seen = SyncReport {
last_cursor_at: Some(hours(-48)),
..synced(0, true, None)
};
sync_seen(&health, seen.clone(), mins(0));
let report = health.report();
assert_eq!(report.state, State::Degraded);
let reason = report.reason.unwrap();
assert!(reason.contains("no sync has succeeded in 2d"), "{reason}");
let stamped = SyncReport {
last_success_at: Some(mins(0)),
..seen
};
sync_seen(&health, stamped, mins(0));
assert_eq!(health.report().state, State::Ok);
}
#[test]
fn reasons_count_one_peer_and_one_raw_in_the_singular() {
let (health, _rx) = monitored();
sync_seen(&health, synced(1, true, Some(hours(-30))), mins(0));
distill_seen(&health, 1, Some(hours(-30)), mins(0));
let reason = health.report().reason.unwrap();
assert!(reason.contains("with 1 peer in sync-peers"), "{reason}");
assert!(reason.contains("1 raw memory has waited"), "{reason}");
}
#[test]
fn a_device_that_synced_before_and_lost_its_peers_degrades_with_the_fix() {
let (health, mut rx) = monitored();
sync_seen(&health, synced(0, true, Some(hours(-25))), mins(0));
let report = health.report();
assert_eq!(report.state, State::Degraded);
let reason = report.reason.unwrap();
assert!(
reason.contains("no sync has succeeded in 25h 0m"),
"{reason}"
);
assert!(reason.contains("lists no peers"), "{reason}");
assert!(reason.contains("~/.second-brain/sync-peers"), "{reason}");
assert!(reason.contains("sync_overdue_hours = 0"), "{reason}");
assert_eq!(report.sync.peers, Some(0));
assert_eq!(report.sync.checked_at, Some(mins(0)));
assert_eq!(drain(&mut rx), vec![State::Degraded]);
sync_seen(&health, synced(1, true, Some(mins(1))), mins(1));
assert_eq!(health.report().state, State::Ok);
}
#[test]
fn a_device_synced_by_hand_without_sync_peers_stays_ok_while_it_syncs() {
let (health, mut rx) = monitored();
let by_hand = SyncReport {
peers: None,
..synced(0, true, Some(hours(10)))
};
alive(&health, hours(30));
sync_seen(&health, by_hand.clone(), hours(30));
assert_eq!(health.report().state, State::Ok);
alive(&health, hours(34));
sync_seen(&health, by_hand, hours(34));
let report = health.report();
assert_eq!(report.state, State::Degraded);
let reason = report.reason.unwrap();
assert!(
reason.contains("no sync has succeeded in 24h 0m"),
"{reason}"
);
assert!(reason.contains("sb sync <user@host>"), "{reason}");
assert!(reason.contains("sync_overdue_hours = 0"), "{reason}");
assert!(!reason.contains("sync-peers"), "{reason}");
assert_eq!(drain(&mut rx), vec![State::Degraded]);
}
#[test]
fn a_solo_device_that_never_synced_stays_ok() {
let (health, mut rx) = monitored();
for h in [0, 25, 100] {
alive(&health, hours(h));
sync_seen(&health, synced(0, false, None), hours(h));
}
let report = health.report();
assert_eq!(report.state, State::Ok, "{:?}", report.reason);
assert!(drain(&mut rx).is_empty());
}
#[test]
fn a_sync_overdue_past_the_window_degrades_only_on_a_device_that_syncs() {
let (health, _rx) = monitored();
alive(&health, hours(0));
sync_seen(&health, synced(2, true, Some(hours(-23))), hours(0));
assert_eq!(health.report().state, State::Ok);
alive(&health, hours(1));
sync_seen(&health, synced(2, true, Some(hours(-23))), hours(1));
let report = health.report();
assert_eq!(report.state, State::Degraded);
let reason = report.reason.unwrap();
assert!(
reason.contains("no sync has succeeded in 24h 0m"),
"{reason}"
);
assert!(reason.contains("2 peers"), "{reason}");
sync_seen(&health, synced(2, true, Some(hours(1))), hours(1));
assert_eq!(health.report().state, State::Ok);
sync_seen(&health, synced(0, false, Some(hours(-500))), hours(1));
assert_eq!(health.report().state, State::Ok);
}
#[test]
fn with_peers_and_no_recorded_success_the_window_starts_at_startup() {
let (health, _rx) = monitored();
alive(&health, hours(23));
sync_seen(&health, synced(1, false, None), hours(23));
assert_eq!(health.report().state, State::Ok);
alive(&health, hours(24));
sync_seen(&health, synced(1, false, None), hours(24));
let reason = health.report().reason.unwrap();
assert!(
reason.contains("no sync has succeeded in 24h 0m"),
"{reason}"
);
}
#[test]
fn raws_left_undistilled_past_the_window_degrade_with_the_count() {
let (health, mut rx) = monitored();
alive(&health, hours(0));
distill_seen(&health, 520, Some(hours(-23)), hours(0));
assert_eq!(health.report().state, State::Ok);
assert_eq!(health.report().distill.pending, 520);
alive(&health, hours(2));
distill_seen(&health, 520, Some(hours(-23)), hours(2));
let report = health.report();
assert_eq!(report.state, State::Degraded);
let reason = report.reason.unwrap();
assert!(
reason.contains("520 raw memories have waited 25h 0m to be distilled"),
"{reason}"
);
assert!(reason.contains("undistilled_overdue_hours = 0"), "{reason}");
alive(&health, hours(3));
distill_seen(&health, 0, None, hours(3));
assert_eq!(health.report().state, State::Ok);
assert_eq!(drain(&mut rx), vec![State::Degraded, State::Ok]);
}
#[test]
fn a_window_turned_off_never_degrades() {
let (tx, _rx) = unbounded_channel();
let windows = Windows::from_config(&AlertConfig {
sync_overdue_hours: 0,
undistilled_overdue_hours: 0,
..AlertConfig::default()
});
assert_eq!(
windows,
Windows {
sync_overdue: None,
undistilled: None
}
);
let health = Health::new(t0(), Some(tx)).with_windows(windows);
alive(&health, hours(100));
sync_seen(&health, synced(1, true, Some(hours(-100))), hours(100));
distill_seen(&health, 9, Some(hours(-100)), hours(100));
assert_eq!(health.report().state, State::Ok);
sync_seen(&health, synced(0, true, Some(hours(-100))), hours(100));
assert_eq!(health.report().state, State::Ok);
}
#[test]
fn the_windows_come_from_the_alert_config() {
let windows = Windows::from_config(&AlertConfig {
sync_overdue_hours: 6,
undistilled_overdue_hours: 48,
..AlertConfig::default()
});
assert_eq!(windows.sync_overdue, Some(Duration::hours(6)));
assert_eq!(windows.undistilled, Some(Duration::hours(48)));
assert_eq!(Windows::default().sync_overdue, Some(Duration::hours(24)));
assert_eq!(Windows::default().undistilled, Some(Duration::hours(24)));
}
#[test]
fn announce_sends_the_current_state_once_unless_it_was_already_sent() {
let (health, mut rx) = monitored();
health.announce(mins(0));
let sent: Vec<Transition> = std::iter::from_fn(|| rx.try_recv().ok()).collect();
assert_eq!(
sent,
vec![Transition {
state: State::Ok,
reason: None,
at: mins(0),
}]
);
health.announce(mins(1));
assert!(drain(&mut rx).is_empty(), "ok was already sent");
sync_seen(&health, synced(0, true, Some(hours(-30))), mins(2));
health.announce(mins(3));
assert_eq!(drain(&mut rx), vec![State::Degraded]);
}
#[test]
fn announce_reassesses_and_sends_one_message_for_a_pending_change() {
let (health, mut rx) = monitored();
alive(&health, hours(0));
distill_seen(&health, 3, Some(hours(-23)), hours(0));
assert!(drain(&mut rx).is_empty());
health.announce(hours(2));
assert_eq!(drain(&mut rx), vec![State::Degraded]);
}
#[test]
fn serving_leaves_out_sync_and_distillation() {
let (health, _rx) = monitored();
alive(&health, hours(0));
let seen = ProgressSeen {
sync: Some(synced(1, true, Some(hours(-30)))),
distill: Some(DistillReport {
checked_at: None,
pending: 4,
oldest_at: Some(hours(-30)),
}),
error: None,
};
health.progress_checked(seen, hours(0));
let report = health.report();
assert_eq!(report.state, State::Degraded);
assert_eq!(
report.serving,
Verdict {
state: State::Ok,
reason: None
}
);
health.watcher_failed("inotify watch limit reached", hours(0));
let report = health.report();
assert_eq!(report.state, State::Failing);
assert_eq!(report.serving.state, State::Failing);
let serving = report.serving.reason.unwrap();
assert!(serving.contains("watcher is not running"), "{serving}");
assert!(!serving.contains("sync"), "{serving}");
let json = serde_json::to_value(health.report()).unwrap();
assert_eq!(json["serving"]["state"], "failing");
}
#[test]
fn one_pass_that_clears_sync_and_finds_distillation_overdue_sends_nothing() {
let (health, mut rx) = monitored();
alive(&health, hours(0));
sync_seen(&health, synced(1, true, Some(hours(-30))), hours(0));
assert_eq!(drain(&mut rx), vec![State::Degraded]);
let seen = ProgressSeen {
sync: Some(synced(1, true, Some(hours(0)))),
distill: Some(DistillReport {
checked_at: None,
pending: 4,
oldest_at: Some(hours(-30)),
}),
error: None,
};
health.progress_checked(seen, hours(0));
assert_eq!(health.report().state, State::Degraded);
assert!(drain(&mut rx).is_empty());
}
}