use std::collections::{BTreeMap, HashMap, HashSet, VecDeque};
use std::hash::Hash;
use crate::health::{counted, human};
use chrono::{DateTime, Duration, Utc};
use second_brain_core::residency::ComponentResidency;
use serde::Serialize;
use uuid::Uuid;
pub const WINDOW: Duration = Duration::hours(24);
const MAX_EVENTS: usize = 50_000;
const SESSION_RATE: Band = Band {
min_samples: 10,
degrade: 0.25,
recover: 0.35,
};
const EMPTY_OR_LOW: Band = Band {
min_samples: 50,
degrade: 0.75,
recover: 0.65,
};
const CORRECTED: Band = Band {
min_samples: 20,
degrade: 0.20,
recover: 0.15,
};
const RESIDENT_DEGRADE: f64 = 0.85;
const RESIDENT_RECOVER: f64 = 0.75;
const STORE_RECOVER: f64 = 1.1;
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub enum RecallVerdict {
Hit,
LowScore,
Unjudged,
Empty,
}
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub enum PeerOutcome {
Synced,
Pushed,
Failed,
}
#[derive(Debug, Clone, PartialEq)]
pub struct RecallEvent {
pub session: Option<String>,
pub verdict: RecallVerdict,
pub memory_ids: Vec<Uuid>,
}
#[derive(Debug, Clone, Default, PartialEq, Serialize)]
pub struct RecallReport {
pub window_secs: i64,
pub recalls: u64,
pub hits: u64,
pub low_score: u64,
pub unjudged: u64,
pub empty: u64,
pub empty_share: Option<f64>,
pub low_score_share: Option<f64>,
pub unjudged_share: Option<f64>,
pub sessions_with_recall: u64,
pub sessions_ingested: Option<u64>,
pub session_recall_rate: Option<f64>,
}
#[derive(Debug, Clone, Default, PartialEq, Serialize)]
pub struct RecalledReport {
pub window_secs: i64,
pub memories: u64,
pub corrected: u64,
pub used: u64,
pub corrected_share: Option<f64>,
pub used_share: Option<f64>,
}
#[derive(Debug, Clone, PartialEq, Serialize)]
pub struct PeerSync {
pub peer: String,
pub last_success_at: Option<DateTime<Utc>>,
pub last_attempt_at: Option<DateTime<Utc>>,
pub consecutive_failures: u32,
pub lag_secs: i64,
}
#[derive(Debug, Clone, Default, PartialEq, Eq, Serialize)]
pub struct StoreSize {
pub checked_at: Option<DateTime<Utc>>,
pub db_bytes: u64,
pub wal_bytes: u64,
pub index_bytes: u64,
pub archive_bytes: u64,
pub raw_index_bytes: u64,
pub total_bytes: u64,
pub free_bytes: Option<u64>,
}
#[derive(Debug, Clone, Default, PartialEq, Eq, Serialize)]
pub struct ResidentSize {
pub checked_at: Option<DateTime<Utc>>,
pub rss_anon_bytes: u64,
pub components: ComponentResidency,
pub raw_index_bytes: u64,
pub unattributed_bytes: i64,
pub limit_bytes: Option<u64>,
}
struct Band {
min_samples: u64,
degrade: f64,
recover: f64,
}
#[derive(Debug, Default)]
struct Latches {
session_rate: bool,
empty_or_low: bool,
corrected: bool,
store: bool,
resident: bool,
}
#[derive(Debug, Clone, Copy, Default)]
struct Flags {
corrected: bool,
used: bool,
}
#[derive(Debug)]
struct Recent<K, V> {
entries: HashMap<K, (DateTime<Utc>, V)>,
order: VecDeque<(DateTime<Utc>, K)>,
}
impl<K, V> Default for Recent<K, V> {
fn default() -> Self {
Self {
entries: HashMap::new(),
order: VecDeque::new(),
}
}
}
impl Band {
fn above(&self, latched: bool, count: u64, of: u64) -> bool {
if of == 0 || (!latched && of < self.min_samples) {
return false;
}
let share = count as f64 / of as f64;
share > if latched { self.recover } else { self.degrade }
}
fn below(&self, latched: bool, count: u64, of: u64) -> bool {
if of == 0 || (!latched && of < self.min_samples) {
return false;
}
let share = count as f64 / of as f64;
share < if latched { self.recover } else { self.degrade }
}
}
impl<K: Hash + Eq + Clone, V: Default> Recent<K, V> {
fn touch(&mut self, key: K, at: DateTime<Utc>) {
match self.entries.get_mut(&key) {
Some(entry) => entry.0 = at,
None => {
self.entries.insert(key.clone(), (at, V::default()));
}
}
self.order.push_back((at, key));
if self.order.len() > 4 * MAX_EVENTS {
let entries = &self.entries;
self.order
.retain(|(at, key)| entries.get(key).is_some_and(|e| e.0 == *at));
}
}
fn get_mut(&mut self, key: &K) -> Option<&mut V> {
self.entries.get_mut(key).map(|e| &mut e.1)
}
fn prune(&mut self, cutoff: DateTime<Utc>, mut evicted: impl FnMut(&V)) {
while let Some((at, key)) = self.order.front() {
let live = self.entries.get(key).is_some_and(|e| e.0 == *at);
if live && *at >= cutoff && self.entries.len() <= MAX_EVENTS {
break;
}
if let Some((_, key)) = self.order.pop_front()
&& live
&& let Some((_, value)) = self.entries.remove(&key)
{
evicted(&value);
}
}
}
fn len(&self) -> usize {
self.entries.len()
}
fn count_since(&self, cutoff: DateTime<Utc>) -> usize {
self.entries
.values()
.filter(|(at, _)| *at >= cutoff)
.count()
}
fn contains(&self, key: &K) -> bool {
self.entries.contains_key(key)
}
}
#[derive(Debug, Default)]
pub struct ValueFacts {
recalls: VecDeque<(DateTime<Utc>, RecallVerdict)>,
hits: u64,
low_score: u64,
unjudged: u64,
empty: u64,
recall_sessions: Recent<String, ()>,
recalled: Recent<Uuid, Flags>,
corrected: u64,
used: u64,
ingested_sessions: Option<HashSet<String>>,
session_problem: Option<String>,
peers: BTreeMap<String, PeerState>,
store: Option<StoreSize>,
resident: Option<ResidentSize>,
latches: Latches,
}
#[derive(Debug, Clone, Default)]
struct PeerState {
listed_at: Option<DateTime<Utc>>,
last_success_at: Option<DateTime<Utc>>,
last_attempt_at: Option<DateTime<Utc>>,
consecutive_failures: u32,
}
#[derive(Debug, Default, PartialEq)]
pub struct ValueProblems {
pub peers: Vec<String>,
pub other: Vec<String>,
}
#[derive(Debug, Clone, Default, PartialEq, Serialize)]
pub struct ValueReport {
pub recall: RecallReport,
pub recalled: RecalledReport,
pub sync_peers: Vec<PeerSync>,
pub store: Option<StoreSize>,
pub resident: Option<ResidentSize>,
}
pub fn session_of(key: &str) -> &str {
key.split_once('/').map_or(key, |(session, _)| session)
}
impl ValueFacts {
pub fn recalled(&mut self, event: RecallEvent, now: DateTime<Utc>) {
self.recalls.push_back((now, event.verdict));
*self.count_of(event.verdict) += 1;
if let Some(session) = &event.session {
self.recall_sessions
.touch(session_of(session).to_string(), now);
}
for id in event.memory_ids {
self.recalled.touch(id, now);
}
}
fn count_of(&mut self, verdict: RecallVerdict) -> &mut u64 {
match verdict {
RecallVerdict::Hit => &mut self.hits,
RecallVerdict::LowScore => &mut self.low_score,
RecallVerdict::Unjudged => &mut self.unjudged,
RecallVerdict::Empty => &mut self.empty,
}
}
pub fn corrected(&mut self, id: Uuid) {
if let Some(flags) = self.recalled.get_mut(&id)
&& !flags.corrected
{
flags.corrected = true;
self.corrected += 1;
}
}
pub fn used(&mut self, ids: &[Uuid]) {
for id in ids {
if let Some(flags) = self.recalled.get_mut(id)
&& !flags.used
{
flags.used = true;
self.used += 1;
}
}
}
pub fn sessions_ingested(&mut self, keys: Option<Vec<String>>) {
self.ingested_sessions =
keys.map(|keys| keys.iter().map(|key| session_of(key).to_string()).collect());
}
pub fn peers_listed(&mut self, peers: &[String], now: DateTime<Utc>) {
self.peers.retain(|peer, _| peers.contains(peer));
for peer in peers {
self.peers.entry(peer.clone()).or_insert_with(|| PeerState {
listed_at: Some(now),
..PeerState::default()
});
}
}
pub fn sync_attempted(&mut self, peer: &str, outcome: PeerOutcome, now: DateTime<Utc>) {
let Some(state) = self.peers.get_mut(peer) else {
return;
};
match outcome {
PeerOutcome::Synced => {
state.last_attempt_at = Some(now);
state.last_success_at = Some(now);
state.consecutive_failures = 0;
}
PeerOutcome::Pushed => {
state.last_attempt_at = Some(now);
state.consecutive_failures = 0;
}
PeerOutcome::Failed => {
state.last_attempt_at = Some(now);
state.consecutive_failures += 1;
}
}
}
pub fn store_measured(&mut self, size: Option<StoreSize>) {
self.store = size;
}
pub fn resident_measured(&mut self, size: Option<ResidentSize>) {
self.resident = size;
}
pub fn refresh(
&mut self,
now: DateTime<Utc>,
started: DateTime<Utc>,
sync_overdue: Option<Duration>,
) -> (ValueReport, ValueProblems) {
self.prune(now - WINDOW);
let mut problems = ValueProblems::default();
let recall = self.recall_report(now - WINDOW, &mut problems);
let recalled = self.recalled_report(&mut problems);
let sync_peers = self.peer_report(now, started, sync_overdue, &mut problems);
self.check_store(&mut problems);
self.check_resident(&mut problems);
let report = ValueReport {
recall,
recalled,
sync_peers,
store: self.store.clone(),
resident: self.resident.clone(),
};
(report, problems)
}
fn prune(&mut self, cutoff: DateTime<Utc>) {
while let Some(&(at, verdict)) = self.recalls.front() {
if at >= cutoff && self.recalls.len() <= MAX_EVENTS {
break;
}
self.recalls.pop_front();
*self.count_of(verdict) -= 1;
}
self.recall_sessions.prune(cutoff - session_grace(), |_| {});
let (corrected, used) = (&mut self.corrected, &mut self.used);
self.recalled.prune(cutoff, |flags| {
*corrected -= u64::from(flags.corrected);
*used -= u64::from(flags.used);
});
}
fn recall_report(
&mut self,
cutoff: DateTime<Utc>,
problems: &mut ValueProblems,
) -> RecallReport {
let recalls = self.recalls.len() as u64;
let ingested = self.ingested_sessions.as_ref();
let joined = ingested.map(|keys| {
keys.iter()
.filter(|key| self.recall_sessions.contains(*key))
.count() as u64
});
let sessions_ingested = ingested.map(|keys| keys.len() as u64);
let judged = recalls - self.unjudged;
let bad = self.empty + self.low_score;
self.latches.empty_or_low = EMPTY_OR_LOW.above(self.latches.empty_or_low, bad, judged);
if self.latches.empty_or_low {
problems.other.push(format!(
"{} of {judged} judged recalls in the last 24h found nothing that cleared the \
relevance bar ({} empty, {} below it)",
percent(bad, judged),
self.empty,
self.low_score
));
}
if let (Some(joined), Some(of)) = (joined, sessions_ingested) {
self.latches.session_rate = SESSION_RATE.below(self.latches.session_rate, joined, of);
self.session_problem = self.latches.session_rate.then(|| {
format!(
"only {joined} of {of} sessions in the last 24h recalled from second-brain; \
check its hooks and MCP server with `sb doctor`"
)
});
}
if let Some(problem) = &self.session_problem {
problems.other.push(problem.clone());
}
RecallReport {
window_secs: WINDOW.num_seconds(),
recalls,
hits: self.hits,
low_score: self.low_score,
unjudged: self.unjudged,
empty: self.empty,
empty_share: share(self.empty, judged),
low_score_share: share(self.low_score, judged),
unjudged_share: share(self.unjudged, recalls),
sessions_with_recall: self.recall_sessions.count_since(cutoff) as u64,
sessions_ingested,
session_recall_rate: joined
.zip(sessions_ingested)
.and_then(|(j, of)| share(j, of)),
}
}
fn recalled_report(&mut self, problems: &mut ValueProblems) -> RecalledReport {
let memories = self.recalled.len() as u64;
self.latches.corrected = CORRECTED.above(self.latches.corrected, self.corrected, memories);
if self.latches.corrected {
problems.other.push(format!(
"{} of {memories} memories recalled in the last 24h were corrected",
self.corrected
));
}
RecalledReport {
window_secs: WINDOW.num_seconds(),
memories,
corrected: self.corrected,
used: self.used,
corrected_share: share(self.corrected, memories),
used_share: share(self.used, memories),
}
}
fn peer_report(
&self,
now: DateTime<Utc>,
started: DateTime<Utc>,
sync_overdue: Option<Duration>,
problems: &mut ValueProblems,
) -> Vec<PeerSync> {
self.peers
.iter()
.map(|(peer, state)| {
let since = state.last_success_at.or(state.listed_at).unwrap_or(started);
let lag = (now - since).max(Duration::zero());
if sync_overdue.is_some_and(|window| lag >= window) {
let failures = match state.consecutive_failures {
0 => String::new(),
n => format!(
" ({} in a row)",
counted(u64::from(n), "failure", "failures")
),
};
problems.peers.push(format!(
"no sync with {peer} has succeeded in {}{failures}",
human(lag)
));
}
PeerSync {
peer: peer.clone(),
last_success_at: state.last_success_at,
last_attempt_at: state.last_attempt_at,
consecutive_failures: state.consecutive_failures,
lag_secs: lag.num_seconds(),
}
})
.collect()
}
fn check_store(&mut self, problems: &mut ValueProblems) {
let Some((rewritten, free)) = self.store.as_ref().and_then(|s| {
let rewritten =
s.db_bytes.saturating_sub(s.wal_bytes) + s.index_bytes + s.raw_index_bytes;
s.free_bytes.map(|free| (rewritten, free))
}) else {
self.latches.store = false;
return;
};
let needed = if self.latches.store {
rewritten as f64 * STORE_RECOVER
} else {
rewritten as f64
};
self.latches.store = (free as f64) < needed;
if self.latches.store {
problems.other.push(format!(
"compacting or rebuilding the store needs {} MB and its disk has {} MB free",
rewritten / 1_000_000,
free / 1_000_000
));
}
}
fn check_resident(&mut self, problems: &mut ValueProblems) {
let Some((rss, limit)) = self.resident.as_ref().and_then(|r| {
r.limit_bytes
.filter(|&l| l > 0)
.map(|l| (r.rss_anon_bytes, l))
}) else {
self.latches.resident = false;
return;
};
let ratio = rss as f64 / limit as f64;
let line = if self.latches.resident {
RESIDENT_RECOVER
} else {
RESIDENT_DEGRADE
};
self.latches.resident = ratio >= line;
if self.latches.resident {
problems.other.push(format!(
"the daemon holds {} MB resident, {} of its {} MB limit",
rss / 1_000_000,
percent(rss, limit),
limit / 1_000_000
));
}
}
}
fn session_grace() -> Duration {
Duration::from_std(crate::progress::INTERVAL).unwrap_or(Duration::zero())
}
fn share(count: u64, of: u64) -> Option<f64> {
(of > 0).then(|| count as f64 / of as f64)
}
fn percent(count: u64, of: u64) -> String {
format!("{:.0}%", count as f64 * 100.0 / of as f64)
}
#[cfg(test)]
mod tests {
use super::*;
fn t0() -> DateTime<Utc> {
DateTime::parse_from_rfc3339("2026-10-31T12:00:00Z")
.unwrap()
.with_timezone(&Utc)
}
fn mins(m: i64) -> DateTime<Utc> {
t0() + Duration::minutes(m)
}
fn id(n: u128) -> Uuid {
Uuid::from_u128(n)
}
fn recall(session: &str, verdict: RecallVerdict, ids: &[u128]) -> RecallEvent {
RecallEvent {
session: Some(session.to_string()),
verdict,
memory_ids: ids.iter().map(|&n| id(n)).collect(),
}
}
fn refresh(v: &mut ValueFacts, now: DateTime<Utc>) -> (ValueReport, ValueProblems) {
v.refresh(now, t0(), Some(Duration::hours(24)))
}
#[test]
fn subagent_keys_count_as_their_parent_session() {
assert_eq!(session_of("abc"), "abc");
assert_eq!(session_of("abc/agent-1"), "abc");
}
#[test]
fn recall_shares_count_empty_and_low_score_recalls() {
let mut v = ValueFacts::default();
v.recalled(recall("s1", RecallVerdict::Hit, &[1]), mins(0));
v.recalled(recall("s1", RecallVerdict::Empty, &[]), mins(1));
v.recalled(recall("s2", RecallVerdict::LowScore, &[2]), mins(2));
v.recalled(recall("s2", RecallVerdict::Empty, &[]), mins(3));
v.recalled(recall("s2", RecallVerdict::Unjudged, &[]), mins(3));
let (report, problems) = refresh(&mut v, mins(4));
let r = report.recall;
assert_eq!(r.window_secs, WINDOW.num_seconds());
assert_eq!((r.recalls, r.hits, r.low_score, r.empty), (5, 1, 1, 2));
assert_eq!(r.empty_share, Some(0.5), "over judged recalls");
assert_eq!(r.low_score_share, Some(0.25), "over judged recalls");
assert_eq!(r.unjudged_share, Some(0.2), "over all recalls");
assert_eq!(r.sessions_with_recall, 2);
assert_eq!(problems, ValueProblems::default());
}
#[test]
fn mostly_empty_recalls_degrade_once_and_recover_past_the_band() {
let mut v = ValueFacts::default();
for m in 0..49 {
v.recalled(recall("s", RecallVerdict::Empty, &[]), mins(m));
}
assert!(refresh(&mut v, mins(49)).1.other.is_empty(), "too few");
v.recalled(recall("s", RecallVerdict::LowScore, &[1]), mins(49));
let (_, problems) = refresh(&mut v, mins(50));
assert_eq!(problems.other.len(), 1, "{problems:?}");
assert!(
problems.other[0].contains("of 50 judged recalls in the last 24h"),
"{problems:?}"
);
for m in 50..71 {
v.recalled(recall("s", RecallVerdict::Hit, &[1]), mins(m));
}
let (report, problems) = refresh(&mut v, mins(71));
assert_eq!(
report.recall.recalls, 71,
"50 of 71 is 70%, inside the band"
);
assert_eq!(problems.other.len(), 1);
for m in 71..77 {
v.recalled(recall("s", RecallVerdict::Hit, &[1]), mins(m));
}
assert!(
refresh(&mut v, mins(77)).1.other.is_empty(),
"50 of 77 is 65%"
);
}
#[test]
fn a_latched_share_holds_below_the_minimum_until_the_window_empties() {
let mut v = ValueFacts::default();
for m in 0..50 {
v.recalled(recall("s", RecallVerdict::Empty, &[]), mins(m));
}
assert_eq!(refresh(&mut v, mins(50)).1.other.len(), 1);
let first_gone = mins(0) + WINDOW + Duration::seconds(30);
let (report, problems) = refresh(&mut v, first_gone);
assert_eq!(report.recall.recalls, 49);
assert_eq!(problems.other.len(), 1, "below the minimum, still latched");
let all_gone = mins(50) + WINDOW;
assert!(refresh(&mut v, all_gone).1.other.is_empty());
}
#[test]
fn unjudged_recalls_are_left_out_of_the_empty_or_low_share() {
let mut v = ValueFacts::default();
for m in 0..50 {
v.recalled(recall("s", RecallVerdict::Unjudged, &[]), mins(m));
}
let (report, problems) = refresh(&mut v, mins(50));
assert_eq!(report.recall.unjudged, 50);
assert_eq!(report.recall.unjudged_share, Some(1.0));
assert!(problems.other.is_empty(), "{problems:?}");
for m in 50..99 {
v.recalled(recall("s", RecallVerdict::Empty, &[]), mins(m));
}
assert!(refresh(&mut v, mins(99)).1.other.is_empty(), "49 judged");
v.recalled(recall("s", RecallVerdict::LowScore, &[]), mins(99));
let (_, problems) = refresh(&mut v, mins(100));
assert_eq!(problems.other.len(), 1);
assert!(
problems.other[0].contains("100% of 50 judged recalls in the last 24h"),
"{problems:?}"
);
assert!(
problems.other[0].contains("(49 empty, 1 below it)"),
"{problems:?}"
);
}
#[test]
fn a_session_recalled_just_before_the_window_edge_counts_until_the_next_ingest_read() {
let mut v = ValueFacts::default();
v.recalled(recall("a", RecallVerdict::Hit, &[]), mins(0));
v.sessions_ingested(Some(vec!["a".to_string()]));
let edge = mins(0) + WINDOW + Duration::minutes(4);
let recall = refresh(&mut v, edge).0.recall;
assert_eq!(recall.session_recall_rate, Some(1.0));
assert_eq!(
recall.sessions_with_recall, 0,
"outside the reported window"
);
let later = mins(0) + WINDOW + Duration::minutes(6);
assert_eq!(
refresh(&mut v, later).0.recall.session_recall_rate,
Some(0.0)
);
}
#[test]
fn recalls_older_than_the_window_drop_out() {
let mut v = ValueFacts::default();
v.recalled(recall("s1", RecallVerdict::Empty, &[1]), mins(0));
v.recalled(recall("s2", RecallVerdict::Hit, &[2]), mins(60));
let (report, _) = refresh(&mut v, mins(0) + WINDOW + Duration::minutes(10));
assert_eq!(report.recall.recalls, 1);
assert_eq!(report.recall.empty, 0);
assert_eq!(report.recall.sessions_with_recall, 1);
assert_eq!(report.recalled.memories, 1);
}
#[test]
fn the_event_window_is_bounded() {
let mut v = ValueFacts::default();
for n in 0..(MAX_EVENTS as u128 + 10) {
v.recalled(recall(&format!("s{n}"), RecallVerdict::Hit, &[n]), mins(0));
}
let (report, _) = refresh(&mut v, mins(1));
assert_eq!(report.recall.recalls, MAX_EVENTS as u64);
assert_eq!(report.recall.sessions_with_recall, MAX_EVENTS as u64);
assert_eq!(report.recalled.memories, MAX_EVENTS as u64);
}
#[test]
fn session_recall_rate_joins_recalling_sessions_with_ingested_ones() {
let mut v = ValueFacts::default();
let (report, _) = refresh(&mut v, mins(0));
assert_eq!(report.recall.sessions_ingested, None);
assert_eq!(report.recall.session_recall_rate, None);
v.recalled(recall("a", RecallVerdict::Hit, &[]), mins(0));
v.recalled(recall("b/agent-1", RecallVerdict::Hit, &[]), mins(0));
v.recalled(recall("not-ingested", RecallVerdict::Hit, &[]), mins(0));
let keys = ["a", "a/agent-2", "b", "c", "d"].map(String::from).to_vec();
v.sessions_ingested(Some(keys));
let (report, problems) = refresh(&mut v, mins(1));
assert_eq!(report.recall.sessions_ingested, Some(4));
assert_eq!(report.recall.session_recall_rate, Some(0.5));
assert!(problems.other.is_empty(), "under the minimum sample");
}
#[test]
fn sessions_that_never_recall_degrade_once_until_they_recover_past_the_band() {
let mut v = ValueFacts::default();
let keys: Vec<String> = (0..10).map(|n| format!("s{n}")).collect();
v.recalled(recall("s0", RecallVerdict::Hit, &[]), mins(0));
v.recalled(recall("s1", RecallVerdict::Hit, &[]), mins(0));
v.sessions_ingested(Some(keys.clone()));
let (_, problems) = refresh(&mut v, mins(1));
assert_eq!(problems.other.len(), 1);
assert!(
problems.other[0].contains("2 of 10 sessions in the last 24h recalled"),
"{problems:?}"
);
v.recalled(recall("s2", RecallVerdict::Hit, &[]), mins(2));
assert_eq!(refresh(&mut v, mins(3)).1.other.len(), 1, "inside band");
v.recalled(recall("s3", RecallVerdict::Hit, &[]), mins(4));
assert!(refresh(&mut v, mins(5)).1.other.is_empty());
v.sessions_ingested(None);
let (report, _) = refresh(&mut v, mins(8));
assert_eq!(report.recall.session_recall_rate, None);
}
#[test]
fn an_unknown_ingest_count_keeps_the_session_alert_until_a_count_clears_it() {
let mut v = ValueFacts::default();
let keys: Vec<String> = (0..10).map(|n| format!("s{n}")).collect();
v.recalled(recall("s0", RecallVerdict::Hit, &[]), mins(0));
v.sessions_ingested(Some(keys.clone()));
assert_eq!(refresh(&mut v, mins(1)).1.other.len(), 1);
v.sessions_ingested(None);
let (report, problems) = refresh(&mut v, mins(2));
assert_eq!(report.recall.session_recall_rate, None);
assert_eq!(problems.other.len(), 1, "{problems:?}");
for n in 1..4 {
v.recalled(recall(&format!("s{n}"), RecallVerdict::Hit, &[]), mins(3));
}
v.sessions_ingested(Some(keys));
assert!(refresh(&mut v, mins(4)).1.other.is_empty());
}
#[test]
fn corrections_and_uses_count_only_recalled_memories_once_each() {
let mut v = ValueFacts::default();
v.recalled(recall("s", RecallVerdict::Hit, &[1, 2, 3, 4]), mins(0));
v.corrected(id(1));
v.corrected(id(1));
v.corrected(id(99));
v.used(&[id(2), id(3), id(2), id(98)]);
let (report, _) = refresh(&mut v, mins(1));
let r = report.recalled;
assert_eq!((r.memories, r.corrected, r.used), (4, 1, 2));
assert_eq!(r.corrected_share, Some(0.25));
assert_eq!(r.used_share, Some(0.5));
let later = mins(0) + WINDOW + Duration::minutes(1);
let (report, _) = refresh(&mut v, later);
assert_eq!(
report.recalled,
RecalledReport {
window_secs: WINDOW.num_seconds(),
..RecalledReport::default()
}
);
}
#[test]
fn a_memory_corrected_before_it_was_recalled_is_not_counted() {
let mut v = ValueFacts::default();
v.corrected(id(1));
v.recalled(recall("s", RecallVerdict::Hit, &[1]), mins(0));
let (report, _) = refresh(&mut v, mins(1));
assert_eq!(report.recalled.corrected, 0);
}
#[test]
fn many_corrected_recalls_degrade_once_and_recover_past_the_band() {
let mut v = ValueFacts::default();
let ids: Vec<u128> = (0..20).collect();
v.recalled(recall("s", RecallVerdict::Hit, &ids), mins(0));
for n in 0..4 {
v.corrected(id(n));
}
assert!(
refresh(&mut v, mins(1)).1.other.is_empty(),
"20% is the line"
);
v.corrected(id(4));
let (_, problems) = refresh(&mut v, mins(2));
assert_eq!(problems.other.len(), 1);
assert!(
problems.other[0].contains("5 of 20 memories recalled in the last 24h were corrected"),
"{problems:?}"
);
let more: Vec<u128> = (100..110).collect();
v.recalled(recall("s", RecallVerdict::Hit, &more), mins(3));
assert_eq!(refresh(&mut v, mins(4)).1.other.len(), 1, "inside band");
let more: Vec<u128> = (110..120).collect();
v.recalled(recall("s", RecallVerdict::Hit, &more), mins(5));
assert!(refresh(&mut v, mins(6)).1.other.is_empty());
}
#[test]
fn a_peer_lagging_past_the_window_degrades_until_it_syncs() {
let mut v = ValueFacts::default();
v.peers_listed(&["u@pi".to_string(), "u@mac".to_string()], mins(0));
v.sync_attempted("u@pi", PeerOutcome::Synced, mins(0));
v.sync_attempted("u@mac", PeerOutcome::Failed, mins(0));
v.sync_attempted("u@mac", PeerOutcome::Failed, mins(5));
let (report, problems) = refresh(&mut v, mins(10));
assert_eq!(report.sync_peers.len(), 2);
let mac = &report.sync_peers[0];
assert_eq!(mac.peer, "u@mac");
assert_eq!(mac.last_success_at, None);
assert_eq!(mac.last_attempt_at, Some(mins(5)));
assert_eq!(mac.consecutive_failures, 2);
assert_eq!(mac.lag_secs, 600, "from startup");
assert_eq!(report.sync_peers[1].lag_secs, 600);
assert!(problems.peers.is_empty());
let late = t0() + Duration::hours(24);
v.sync_attempted("u@pi", PeerOutcome::Synced, late);
let (_, problems) = refresh(&mut v, late);
assert_eq!(problems.peers.len(), 1);
assert!(
problems.peers[0].contains("no sync with u@mac has succeeded in 24h 0m"),
"{problems:?}"
);
assert!(
problems.peers[0].contains("2 failures in a row"),
"{problems:?}"
);
v.sync_attempted("u@mac", PeerOutcome::Synced, late);
assert!(refresh(&mut v, late).1.peers.is_empty());
v.peers_listed(&["u@pi".to_string()], late);
assert_eq!(refresh(&mut v, late).0.sync_peers.len(), 1);
assert!(
v.refresh(late + Duration::hours(30), t0(), None)
.1
.peers
.is_empty(),
"sync checks off"
);
}
#[test]
fn a_push_ends_a_failure_streak_but_not_the_lag() {
let mut v = ValueFacts::default();
v.peers_listed(&["u@pi".to_string()], mins(0));
v.sync_attempted("u@pi", PeerOutcome::Failed, mins(1));
v.sync_attempted("u@pi", PeerOutcome::Pushed, mins(2));
let (report, _) = refresh(&mut v, mins(3));
let pi = &report.sync_peers[0];
assert_eq!(pi.consecutive_failures, 0);
assert_eq!(pi.last_attempt_at, Some(mins(2)));
assert_eq!(pi.last_success_at, None);
assert_eq!(pi.lag_secs, 180);
}
#[test]
fn a_peer_listed_late_lags_from_when_it_was_listed() {
let mut v = ValueFacts::default();
let late = t0() + Duration::hours(30);
v.peers_listed(&["u@mac".to_string()], late);
let (report, problems) = refresh(&mut v, late + Duration::minutes(5));
assert_eq!(report.sync_peers[0].lag_secs, 300);
assert!(problems.peers.is_empty());
v.peers_listed(&["u@mac".to_string()], late + Duration::hours(1));
let (report, _) = refresh(&mut v, late + Duration::hours(2));
assert_eq!(
report.sync_peers[0].lag_secs, 7_200,
"listing again keeps the start"
);
}
#[test]
fn a_result_for_a_peer_no_longer_listed_is_ignored() {
let mut v = ValueFacts::default();
v.peers_listed(&["u@pi".to_string()], mins(0));
v.peers_listed(&[], mins(0));
v.sync_attempted("u@pi", PeerOutcome::Failed, mins(1));
assert!(refresh(&mut v, mins(2)).0.sync_peers.is_empty());
}
fn store(total: u64, free: Option<u64>) -> StoreSize {
StoreSize {
checked_at: Some(mins(0)),
db_bytes: total,
total_bytes: total,
free_bytes: free,
..StoreSize::default()
}
}
#[test]
fn a_store_larger_than_its_free_space_degrades_until_space_clears_the_band() {
let mut v = ValueFacts::default();
let (report, _) = refresh(&mut v, mins(0));
assert_eq!(report.store, None);
v.store_measured(Some(store(1_000, Some(1_000))));
assert!(refresh(&mut v, mins(0)).1.other.is_empty());
v.store_measured(Some(store(1_000, Some(999))));
let (report, problems) = refresh(&mut v, mins(1));
assert_eq!(report.store, Some(store(1_000, Some(999))));
assert_eq!(problems.other.len(), 1);
assert!(
problems.other[0].contains("compacting or rebuilding the store needs 0 MB"),
"{problems:?}"
);
v.store_measured(Some(store(1_000, Some(1_050))));
assert_eq!(refresh(&mut v, mins(2)).1.other.len(), 1, "inside band");
v.store_measured(Some(store(1_000, Some(1_100))));
assert!(refresh(&mut v, mins(3)).1.other.is_empty());
v.store_measured(Some(store(1_000, Some(999))));
assert_eq!(refresh(&mut v, mins(4)).1.other.len(), 1);
v.store_measured(None);
let (report, problems) = refresh(&mut v, mins(5));
assert_eq!(report.store, None);
assert!(problems.other.is_empty());
v.store_measured(Some(store(1_000, None)));
assert!(refresh(&mut v, mins(6)).1.other.is_empty());
}
#[test]
fn the_store_check_leaves_out_the_archive_that_compaction_never_rewrites() {
let mut v = ValueFacts::default();
v.store_measured(Some(StoreSize {
checked_at: Some(mins(0)),
db_bytes: 100,
wal_bytes: 0,
index_bytes: 100,
archive_bytes: 10_000,
raw_index_bytes: 100,
total_bytes: 10_300,
free_bytes: Some(301),
}));
assert!(refresh(&mut v, mins(1)).1.other.is_empty());
v.store_measured(Some(StoreSize {
free_bytes: Some(299),
..v.store.clone().unwrap()
}));
assert_eq!(refresh(&mut v, mins(2)).1.other.len(), 1);
}
#[test]
fn a_large_wal_is_not_counted_as_space_a_rewrite_needs() {
let mut v = ValueFacts::default();
v.store_measured(Some(StoreSize {
checked_at: Some(mins(0)),
db_bytes: 1_000,
wal_bytes: 600,
index_bytes: 50,
total_bytes: 1_050,
free_bytes: Some(500),
..StoreSize::default()
}));
assert!(refresh(&mut v, mins(1)).1.other.is_empty(), "needs 450");
}
fn resident(rss: u64, limit: Option<u64>) -> ResidentSize {
ResidentSize {
checked_at: Some(mins(0)),
rss_anon_bytes: rss,
limit_bytes: limit,
..ResidentSize::default()
}
}
#[test]
fn resident_size_near_the_limit_degrades_until_it_falls_past_the_band() {
let mut v = ValueFacts::default();
v.resident_measured(Some(resident(840, Some(1_000))));
assert!(refresh(&mut v, mins(0)).1.other.is_empty());
v.resident_measured(Some(resident(850, Some(1_000))));
let (report, problems) = refresh(&mut v, mins(1));
assert_eq!(report.resident, Some(resident(850, Some(1_000))));
assert_eq!(problems.other.len(), 1);
assert!(problems.other[0].contains("85% of its"), "{problems:?}");
v.resident_measured(Some(resident(760, Some(1_000))));
assert_eq!(refresh(&mut v, mins(2)).1.other.len(), 1, "inside band");
v.resident_measured(Some(resident(740, Some(1_000))));
assert!(refresh(&mut v, mins(3)).1.other.is_empty());
v.resident_measured(Some(resident(5_000, None)));
assert!(refresh(&mut v, mins(4)).1.other.is_empty());
v.resident_measured(Some(resident(900, Some(1_000))));
assert_eq!(refresh(&mut v, mins(5)).1.other.len(), 1);
v.resident_measured(None);
let (report, problems) = refresh(&mut v, mins(6));
assert_eq!(report.resident, None);
assert!(problems.other.is_empty());
}
}