1use std::collections::{BTreeMap, HashMap, HashSet, VecDeque};
8use std::hash::Hash;
9
10use crate::health::{counted, human};
11use chrono::{DateTime, Duration, Utc};
12use second_brain_core::residency::ComponentResidency;
13use serde::Serialize;
14use uuid::Uuid;
15
16pub const WINDOW: Duration = Duration::hours(24);
17const MAX_EVENTS: usize = 50_000;
19
20const SESSION_RATE: Band = Band {
21 min_samples: 10,
22 degrade: 0.25,
23 recover: 0.35,
24};
25const EMPTY_OR_LOW: Band = Band {
28 min_samples: 50,
29 degrade: 0.75,
30 recover: 0.65,
31};
32const CORRECTED: Band = Band {
33 min_samples: 20,
34 degrade: 0.20,
35 recover: 0.15,
36};
37const RESIDENT_DEGRADE: f64 = 0.85;
38const RESIDENT_RECOVER: f64 = 0.75;
39const STORE_RECOVER: f64 = 1.1;
43
44#[derive(Debug, Clone, Copy, PartialEq, Eq)]
45pub enum RecallVerdict {
46 Hit,
48 LowScore,
50 Unjudged,
53 Empty,
54}
55
56#[derive(Debug, Clone, Copy, PartialEq, Eq)]
57pub enum PeerOutcome {
58 Synced,
60 Pushed,
63 Failed,
64}
65
66#[derive(Debug, Clone, PartialEq)]
67pub struct RecallEvent {
68 pub session: Option<String>,
70 pub verdict: RecallVerdict,
71 pub memory_ids: Vec<Uuid>,
73}
74
75#[derive(Debug, Clone, Default, PartialEq, Serialize)]
76pub struct RecallReport {
77 pub window_secs: i64,
78 pub recalls: u64,
79 pub hits: u64,
80 pub low_score: u64,
81 pub unjudged: u64,
82 pub empty: u64,
83 pub empty_share: Option<f64>,
85 pub low_score_share: Option<f64>,
87 pub unjudged_share: Option<f64>,
88 pub sessions_with_recall: u64,
89 pub sessions_ingested: Option<u64>,
92 pub session_recall_rate: Option<f64>,
94}
95
96#[derive(Debug, Clone, Default, PartialEq, Serialize)]
97pub struct RecalledReport {
98 pub window_secs: i64,
99 pub memories: u64,
100 pub corrected: u64,
101 pub used: u64,
103 pub corrected_share: Option<f64>,
104 pub used_share: Option<f64>,
105}
106
107#[derive(Debug, Clone, PartialEq, Serialize)]
108pub struct PeerSync {
109 pub peer: String,
110 pub last_success_at: Option<DateTime<Utc>>,
111 pub last_attempt_at: Option<DateTime<Utc>>,
112 pub consecutive_failures: u32,
113 pub lag_secs: i64,
115}
116
117#[derive(Debug, Clone, Default, PartialEq, Eq, Serialize)]
118pub struct StoreSize {
119 pub checked_at: Option<DateTime<Utc>>,
120 pub db_bytes: u64,
122 pub wal_bytes: u64,
125 pub index_bytes: u64,
126 pub archive_bytes: u64,
127 pub raw_index_bytes: u64,
128 pub total_bytes: u64,
129 pub free_bytes: Option<u64>,
131}
132
133#[derive(Debug, Clone, Default, PartialEq, Eq, Serialize)]
134pub struct ResidentSize {
135 pub checked_at: Option<DateTime<Utc>>,
136 pub rss_anon_bytes: u64,
137 pub components: ComponentResidency,
138 pub raw_index_bytes: u64,
139 pub unattributed_bytes: i64,
140 pub limit_bytes: Option<u64>,
143}
144
145struct Band {
150 min_samples: u64,
151 degrade: f64,
152 recover: f64,
153}
154
155#[derive(Debug, Default)]
157struct Latches {
158 session_rate: bool,
159 empty_or_low: bool,
160 corrected: bool,
161 store: bool,
162 resident: bool,
163}
164
165#[derive(Debug, Clone, Copy, Default)]
166struct Flags {
167 corrected: bool,
168 used: bool,
169}
170
171#[derive(Debug)]
174struct Recent<K, V> {
175 entries: HashMap<K, (DateTime<Utc>, V)>,
176 order: VecDeque<(DateTime<Utc>, K)>,
177}
178
179impl<K, V> Default for Recent<K, V> {
180 fn default() -> Self {
181 Self {
182 entries: HashMap::new(),
183 order: VecDeque::new(),
184 }
185 }
186}
187
188impl Band {
189 fn above(&self, latched: bool, count: u64, of: u64) -> bool {
191 if of == 0 || (!latched && of < self.min_samples) {
192 return false;
193 }
194 let share = count as f64 / of as f64;
195 share > if latched { self.recover } else { self.degrade }
196 }
197
198 fn below(&self, latched: bool, count: u64, of: u64) -> bool {
200 if of == 0 || (!latched && of < self.min_samples) {
201 return false;
202 }
203 let share = count as f64 / of as f64;
204 share < if latched { self.recover } else { self.degrade }
205 }
206}
207
208impl<K: Hash + Eq + Clone, V: Default> Recent<K, V> {
209 fn touch(&mut self, key: K, at: DateTime<Utc>) {
210 match self.entries.get_mut(&key) {
211 Some(entry) => entry.0 = at,
212 None => {
213 self.entries.insert(key.clone(), (at, V::default()));
214 }
215 }
216 self.order.push_back((at, key));
217 if self.order.len() > 4 * MAX_EVENTS {
218 let entries = &self.entries;
219 self.order
220 .retain(|(at, key)| entries.get(key).is_some_and(|e| e.0 == *at));
221 }
222 }
223
224 fn get_mut(&mut self, key: &K) -> Option<&mut V> {
225 self.entries.get_mut(key).map(|e| &mut e.1)
226 }
227
228 fn prune(&mut self, cutoff: DateTime<Utc>, mut evicted: impl FnMut(&V)) {
230 while let Some((at, key)) = self.order.front() {
231 let live = self.entries.get(key).is_some_and(|e| e.0 == *at);
232 if live && *at >= cutoff && self.entries.len() <= MAX_EVENTS {
233 break;
234 }
235 if let Some((_, key)) = self.order.pop_front()
236 && live
237 && let Some((_, value)) = self.entries.remove(&key)
238 {
239 evicted(&value);
240 }
241 }
242 }
243
244 fn len(&self) -> usize {
245 self.entries.len()
246 }
247
248 fn count_since(&self, cutoff: DateTime<Utc>) -> usize {
249 self.entries
250 .values()
251 .filter(|(at, _)| *at >= cutoff)
252 .count()
253 }
254
255 fn contains(&self, key: &K) -> bool {
256 self.entries.contains_key(key)
257 }
258}
259
260#[derive(Debug, Default)]
261pub struct ValueFacts {
262 recalls: VecDeque<(DateTime<Utc>, RecallVerdict)>,
263 hits: u64,
264 low_score: u64,
265 unjudged: u64,
266 empty: u64,
267 recall_sessions: Recent<String, ()>,
268 recalled: Recent<Uuid, Flags>,
269 corrected: u64,
270 used: u64,
271 ingested_sessions: Option<HashSet<String>>,
272 session_problem: Option<String>,
275 peers: BTreeMap<String, PeerState>,
276 store: Option<StoreSize>,
277 resident: Option<ResidentSize>,
278 latches: Latches,
279}
280
281#[derive(Debug, Clone, Default)]
282struct PeerState {
283 listed_at: Option<DateTime<Utc>>,
284 last_success_at: Option<DateTime<Utc>>,
285 last_attempt_at: Option<DateTime<Utc>>,
286 consecutive_failures: u32,
287}
288
289#[derive(Debug, Default, PartialEq)]
293pub struct ValueProblems {
294 pub peers: Vec<String>,
295 pub other: Vec<String>,
296}
297
298#[derive(Debug, Clone, Default, PartialEq, Serialize)]
299pub struct ValueReport {
300 pub recall: RecallReport,
301 pub recalled: RecalledReport,
302 pub sync_peers: Vec<PeerSync>,
303 pub store: Option<StoreSize>,
304 pub resident: Option<ResidentSize>,
305}
306
307pub fn session_of(key: &str) -> &str {
310 key.split_once('/').map_or(key, |(session, _)| session)
311}
312
313impl ValueFacts {
314 pub fn recalled(&mut self, event: RecallEvent, now: DateTime<Utc>) {
315 self.recalls.push_back((now, event.verdict));
316 *self.count_of(event.verdict) += 1;
317 if let Some(session) = &event.session {
318 self.recall_sessions
319 .touch(session_of(session).to_string(), now);
320 }
321 for id in event.memory_ids {
322 self.recalled.touch(id, now);
323 }
324 }
325
326 fn count_of(&mut self, verdict: RecallVerdict) -> &mut u64 {
327 match verdict {
328 RecallVerdict::Hit => &mut self.hits,
329 RecallVerdict::LowScore => &mut self.low_score,
330 RecallVerdict::Unjudged => &mut self.unjudged,
331 RecallVerdict::Empty => &mut self.empty,
332 }
333 }
334
335 pub fn corrected(&mut self, id: Uuid) {
337 if let Some(flags) = self.recalled.get_mut(&id)
338 && !flags.corrected
339 {
340 flags.corrected = true;
341 self.corrected += 1;
342 }
343 }
344
345 pub fn used(&mut self, ids: &[Uuid]) {
346 for id in ids {
347 if let Some(flags) = self.recalled.get_mut(id)
348 && !flags.used
349 {
350 flags.used = true;
351 self.used += 1;
352 }
353 }
354 }
355
356 pub fn sessions_ingested(&mut self, keys: Option<Vec<String>>) {
359 self.ingested_sessions =
360 keys.map(|keys| keys.iter().map(|key| session_of(key).to_string()).collect());
361 }
362
363 pub fn peers_listed(&mut self, peers: &[String], now: DateTime<Utc>) {
367 self.peers.retain(|peer, _| peers.contains(peer));
368 for peer in peers {
369 self.peers.entry(peer.clone()).or_insert_with(|| PeerState {
370 listed_at: Some(now),
371 ..PeerState::default()
372 });
373 }
374 }
375
376 pub fn sync_attempted(&mut self, peer: &str, outcome: PeerOutcome, now: DateTime<Utc>) {
377 let Some(state) = self.peers.get_mut(peer) else {
378 return;
379 };
380 match outcome {
381 PeerOutcome::Synced => {
382 state.last_attempt_at = Some(now);
383 state.last_success_at = Some(now);
384 state.consecutive_failures = 0;
385 }
386 PeerOutcome::Pushed => {
387 state.last_attempt_at = Some(now);
388 state.consecutive_failures = 0;
389 }
390 PeerOutcome::Failed => {
391 state.last_attempt_at = Some(now);
392 state.consecutive_failures += 1;
393 }
394 }
395 }
396
397 pub fn store_measured(&mut self, size: Option<StoreSize>) {
399 self.store = size;
400 }
401
402 pub fn resident_measured(&mut self, size: Option<ResidentSize>) {
404 self.resident = size;
405 }
406
407 pub fn refresh(
411 &mut self,
412 now: DateTime<Utc>,
413 started: DateTime<Utc>,
414 sync_overdue: Option<Duration>,
415 ) -> (ValueReport, ValueProblems) {
416 self.prune(now - WINDOW);
417 let mut problems = ValueProblems::default();
418 let recall = self.recall_report(now - WINDOW, &mut problems);
419 let recalled = self.recalled_report(&mut problems);
420 let sync_peers = self.peer_report(now, started, sync_overdue, &mut problems);
421 self.check_store(&mut problems);
422 self.check_resident(&mut problems);
423 let report = ValueReport {
424 recall,
425 recalled,
426 sync_peers,
427 store: self.store.clone(),
428 resident: self.resident.clone(),
429 };
430 (report, problems)
431 }
432
433 fn prune(&mut self, cutoff: DateTime<Utc>) {
434 while let Some(&(at, verdict)) = self.recalls.front() {
435 if at >= cutoff && self.recalls.len() <= MAX_EVENTS {
436 break;
437 }
438 self.recalls.pop_front();
439 *self.count_of(verdict) -= 1;
440 }
441 self.recall_sessions.prune(cutoff - session_grace(), |_| {});
442 let (corrected, used) = (&mut self.corrected, &mut self.used);
443 self.recalled.prune(cutoff, |flags| {
444 *corrected -= u64::from(flags.corrected);
445 *used -= u64::from(flags.used);
446 });
447 }
448
449 fn recall_report(
450 &mut self,
451 cutoff: DateTime<Utc>,
452 problems: &mut ValueProblems,
453 ) -> RecallReport {
454 let recalls = self.recalls.len() as u64;
455 let ingested = self.ingested_sessions.as_ref();
456 let joined = ingested.map(|keys| {
457 keys.iter()
458 .filter(|key| self.recall_sessions.contains(*key))
459 .count() as u64
460 });
461 let sessions_ingested = ingested.map(|keys| keys.len() as u64);
462
463 let judged = recalls - self.unjudged;
464 let bad = self.empty + self.low_score;
465 self.latches.empty_or_low = EMPTY_OR_LOW.above(self.latches.empty_or_low, bad, judged);
466 if self.latches.empty_or_low {
467 problems.other.push(format!(
468 "{} of {judged} judged recalls in the last 24h found nothing that cleared the \
469 relevance bar ({} empty, {} below it)",
470 percent(bad, judged),
471 self.empty,
472 self.low_score
473 ));
474 }
475 if let (Some(joined), Some(of)) = (joined, sessions_ingested) {
476 self.latches.session_rate = SESSION_RATE.below(self.latches.session_rate, joined, of);
477 self.session_problem = self.latches.session_rate.then(|| {
478 format!(
479 "only {joined} of {of} sessions in the last 24h recalled from second-brain; \
480 check its hooks and MCP server with `sb doctor`"
481 )
482 });
483 }
484 if let Some(problem) = &self.session_problem {
485 problems.other.push(problem.clone());
486 }
487
488 RecallReport {
489 window_secs: WINDOW.num_seconds(),
490 recalls,
491 hits: self.hits,
492 low_score: self.low_score,
493 unjudged: self.unjudged,
494 empty: self.empty,
495 empty_share: share(self.empty, judged),
496 low_score_share: share(self.low_score, judged),
497 unjudged_share: share(self.unjudged, recalls),
498 sessions_with_recall: self.recall_sessions.count_since(cutoff) as u64,
499 sessions_ingested,
500 session_recall_rate: joined
501 .zip(sessions_ingested)
502 .and_then(|(j, of)| share(j, of)),
503 }
504 }
505
506 fn recalled_report(&mut self, problems: &mut ValueProblems) -> RecalledReport {
507 let memories = self.recalled.len() as u64;
508 self.latches.corrected = CORRECTED.above(self.latches.corrected, self.corrected, memories);
509 if self.latches.corrected {
510 problems.other.push(format!(
511 "{} of {memories} memories recalled in the last 24h were corrected",
512 self.corrected
513 ));
514 }
515 RecalledReport {
516 window_secs: WINDOW.num_seconds(),
517 memories,
518 corrected: self.corrected,
519 used: self.used,
520 corrected_share: share(self.corrected, memories),
521 used_share: share(self.used, memories),
522 }
523 }
524
525 fn peer_report(
526 &self,
527 now: DateTime<Utc>,
528 started: DateTime<Utc>,
529 sync_overdue: Option<Duration>,
530 problems: &mut ValueProblems,
531 ) -> Vec<PeerSync> {
532 self.peers
533 .iter()
534 .map(|(peer, state)| {
535 let since = state.last_success_at.or(state.listed_at).unwrap_or(started);
536 let lag = (now - since).max(Duration::zero());
537 if sync_overdue.is_some_and(|window| lag >= window) {
538 let failures = match state.consecutive_failures {
539 0 => String::new(),
540 n => format!(
541 " ({} in a row)",
542 counted(u64::from(n), "failure", "failures")
543 ),
544 };
545 problems.peers.push(format!(
546 "no sync with {peer} has succeeded in {}{failures}",
547 human(lag)
548 ));
549 }
550 PeerSync {
551 peer: peer.clone(),
552 last_success_at: state.last_success_at,
553 last_attempt_at: state.last_attempt_at,
554 consecutive_failures: state.consecutive_failures,
555 lag_secs: lag.num_seconds(),
556 }
557 })
558 .collect()
559 }
560
561 fn check_store(&mut self, problems: &mut ValueProblems) {
562 let Some((rewritten, free)) = self.store.as_ref().and_then(|s| {
563 let rewritten =
564 s.db_bytes.saturating_sub(s.wal_bytes) + s.index_bytes + s.raw_index_bytes;
565 s.free_bytes.map(|free| (rewritten, free))
566 }) else {
567 self.latches.store = false;
568 return;
569 };
570 let needed = if self.latches.store {
571 rewritten as f64 * STORE_RECOVER
572 } else {
573 rewritten as f64
574 };
575 self.latches.store = (free as f64) < needed;
576 if self.latches.store {
577 problems.other.push(format!(
578 "compacting or rebuilding the store needs {} MB and its disk has {} MB free",
579 rewritten / 1_000_000,
580 free / 1_000_000
581 ));
582 }
583 }
584
585 fn check_resident(&mut self, problems: &mut ValueProblems) {
586 let Some((rss, limit)) = self.resident.as_ref().and_then(|r| {
587 r.limit_bytes
588 .filter(|&l| l > 0)
589 .map(|l| (r.rss_anon_bytes, l))
590 }) else {
591 self.latches.resident = false;
592 return;
593 };
594 let ratio = rss as f64 / limit as f64;
595 let line = if self.latches.resident {
596 RESIDENT_RECOVER
597 } else {
598 RESIDENT_DEGRADE
599 };
600 self.latches.resident = ratio >= line;
601 if self.latches.resident {
602 problems.other.push(format!(
603 "the daemon holds {} MB resident, {} of its {} MB limit",
604 rss / 1_000_000,
605 percent(rss, limit),
606 limit / 1_000_000
607 ));
608 }
609 }
610}
611
612fn session_grace() -> Duration {
616 Duration::from_std(crate::progress::INTERVAL).unwrap_or(Duration::zero())
617}
618
619fn share(count: u64, of: u64) -> Option<f64> {
620 (of > 0).then(|| count as f64 / of as f64)
621}
622
623fn percent(count: u64, of: u64) -> String {
624 format!("{:.0}%", count as f64 * 100.0 / of as f64)
625}
626
627#[cfg(test)]
628mod tests {
629 use super::*;
630
631 fn t0() -> DateTime<Utc> {
632 DateTime::parse_from_rfc3339("2026-10-31T12:00:00Z")
633 .unwrap()
634 .with_timezone(&Utc)
635 }
636
637 fn mins(m: i64) -> DateTime<Utc> {
638 t0() + Duration::minutes(m)
639 }
640
641 fn id(n: u128) -> Uuid {
642 Uuid::from_u128(n)
643 }
644
645 fn recall(session: &str, verdict: RecallVerdict, ids: &[u128]) -> RecallEvent {
646 RecallEvent {
647 session: Some(session.to_string()),
648 verdict,
649 memory_ids: ids.iter().map(|&n| id(n)).collect(),
650 }
651 }
652
653 fn refresh(v: &mut ValueFacts, now: DateTime<Utc>) -> (ValueReport, ValueProblems) {
654 v.refresh(now, t0(), Some(Duration::hours(24)))
655 }
656
657 #[test]
658 fn subagent_keys_count_as_their_parent_session() {
659 assert_eq!(session_of("abc"), "abc");
660 assert_eq!(session_of("abc/agent-1"), "abc");
661 }
662
663 #[test]
664 fn recall_shares_count_empty_and_low_score_recalls() {
665 let mut v = ValueFacts::default();
666 v.recalled(recall("s1", RecallVerdict::Hit, &[1]), mins(0));
667 v.recalled(recall("s1", RecallVerdict::Empty, &[]), mins(1));
668 v.recalled(recall("s2", RecallVerdict::LowScore, &[2]), mins(2));
669 v.recalled(recall("s2", RecallVerdict::Empty, &[]), mins(3));
670 v.recalled(recall("s2", RecallVerdict::Unjudged, &[]), mins(3));
671 let (report, problems) = refresh(&mut v, mins(4));
672 let r = report.recall;
673 assert_eq!(r.window_secs, WINDOW.num_seconds());
674 assert_eq!((r.recalls, r.hits, r.low_score, r.empty), (5, 1, 1, 2));
675 assert_eq!(r.empty_share, Some(0.5), "over judged recalls");
676 assert_eq!(r.low_score_share, Some(0.25), "over judged recalls");
677 assert_eq!(r.unjudged_share, Some(0.2), "over all recalls");
678 assert_eq!(r.sessions_with_recall, 2);
679 assert_eq!(problems, ValueProblems::default());
680 }
681
682 #[test]
683 fn mostly_empty_recalls_degrade_once_and_recover_past_the_band() {
684 let mut v = ValueFacts::default();
685 for m in 0..49 {
686 v.recalled(recall("s", RecallVerdict::Empty, &[]), mins(m));
687 }
688 assert!(refresh(&mut v, mins(49)).1.other.is_empty(), "too few");
689 v.recalled(recall("s", RecallVerdict::LowScore, &[1]), mins(49));
690 let (_, problems) = refresh(&mut v, mins(50));
691 assert_eq!(problems.other.len(), 1, "{problems:?}");
692 assert!(
693 problems.other[0].contains("of 50 judged recalls in the last 24h"),
694 "{problems:?}"
695 );
696 for m in 50..71 {
697 v.recalled(recall("s", RecallVerdict::Hit, &[1]), mins(m));
698 }
699 let (report, problems) = refresh(&mut v, mins(71));
700 assert_eq!(
701 report.recall.recalls, 71,
702 "50 of 71 is 70%, inside the band"
703 );
704 assert_eq!(problems.other.len(), 1);
705 for m in 71..77 {
706 v.recalled(recall("s", RecallVerdict::Hit, &[1]), mins(m));
707 }
708 assert!(
709 refresh(&mut v, mins(77)).1.other.is_empty(),
710 "50 of 77 is 65%"
711 );
712 }
713
714 #[test]
715 fn a_latched_share_holds_below_the_minimum_until_the_window_empties() {
716 let mut v = ValueFacts::default();
717 for m in 0..50 {
718 v.recalled(recall("s", RecallVerdict::Empty, &[]), mins(m));
719 }
720 assert_eq!(refresh(&mut v, mins(50)).1.other.len(), 1);
721 let first_gone = mins(0) + WINDOW + Duration::seconds(30);
722 let (report, problems) = refresh(&mut v, first_gone);
723 assert_eq!(report.recall.recalls, 49);
724 assert_eq!(problems.other.len(), 1, "below the minimum, still latched");
725 let all_gone = mins(50) + WINDOW;
726 assert!(refresh(&mut v, all_gone).1.other.is_empty());
727 }
728
729 #[test]
730 fn unjudged_recalls_are_left_out_of_the_empty_or_low_share() {
731 let mut v = ValueFacts::default();
732 for m in 0..50 {
733 v.recalled(recall("s", RecallVerdict::Unjudged, &[]), mins(m));
734 }
735 let (report, problems) = refresh(&mut v, mins(50));
736 assert_eq!(report.recall.unjudged, 50);
737 assert_eq!(report.recall.unjudged_share, Some(1.0));
738 assert!(problems.other.is_empty(), "{problems:?}");
739 for m in 50..99 {
740 v.recalled(recall("s", RecallVerdict::Empty, &[]), mins(m));
741 }
742 assert!(refresh(&mut v, mins(99)).1.other.is_empty(), "49 judged");
743 v.recalled(recall("s", RecallVerdict::LowScore, &[]), mins(99));
744 let (_, problems) = refresh(&mut v, mins(100));
745 assert_eq!(problems.other.len(), 1);
746 assert!(
747 problems.other[0].contains("100% of 50 judged recalls in the last 24h"),
748 "{problems:?}"
749 );
750 assert!(
751 problems.other[0].contains("(49 empty, 1 below it)"),
752 "{problems:?}"
753 );
754 }
755
756 #[test]
757 fn a_session_recalled_just_before_the_window_edge_counts_until_the_next_ingest_read() {
758 let mut v = ValueFacts::default();
759 v.recalled(recall("a", RecallVerdict::Hit, &[]), mins(0));
760 v.sessions_ingested(Some(vec!["a".to_string()]));
761 let edge = mins(0) + WINDOW + Duration::minutes(4);
762 let recall = refresh(&mut v, edge).0.recall;
763 assert_eq!(recall.session_recall_rate, Some(1.0));
764 assert_eq!(
765 recall.sessions_with_recall, 0,
766 "outside the reported window"
767 );
768 let later = mins(0) + WINDOW + Duration::minutes(6);
769 assert_eq!(
770 refresh(&mut v, later).0.recall.session_recall_rate,
771 Some(0.0)
772 );
773 }
774
775 #[test]
776 fn recalls_older_than_the_window_drop_out() {
777 let mut v = ValueFacts::default();
778 v.recalled(recall("s1", RecallVerdict::Empty, &[1]), mins(0));
779 v.recalled(recall("s2", RecallVerdict::Hit, &[2]), mins(60));
780 let (report, _) = refresh(&mut v, mins(0) + WINDOW + Duration::minutes(10));
781 assert_eq!(report.recall.recalls, 1);
782 assert_eq!(report.recall.empty, 0);
783 assert_eq!(report.recall.sessions_with_recall, 1);
784 assert_eq!(report.recalled.memories, 1);
785 }
786
787 #[test]
788 fn the_event_window_is_bounded() {
789 let mut v = ValueFacts::default();
790 for n in 0..(MAX_EVENTS as u128 + 10) {
791 v.recalled(recall(&format!("s{n}"), RecallVerdict::Hit, &[n]), mins(0));
792 }
793 let (report, _) = refresh(&mut v, mins(1));
794 assert_eq!(report.recall.recalls, MAX_EVENTS as u64);
795 assert_eq!(report.recall.sessions_with_recall, MAX_EVENTS as u64);
796 assert_eq!(report.recalled.memories, MAX_EVENTS as u64);
797 }
798
799 #[test]
800 fn session_recall_rate_joins_recalling_sessions_with_ingested_ones() {
801 let mut v = ValueFacts::default();
802 let (report, _) = refresh(&mut v, mins(0));
803 assert_eq!(report.recall.sessions_ingested, None);
804 assert_eq!(report.recall.session_recall_rate, None);
805
806 v.recalled(recall("a", RecallVerdict::Hit, &[]), mins(0));
807 v.recalled(recall("b/agent-1", RecallVerdict::Hit, &[]), mins(0));
808 v.recalled(recall("not-ingested", RecallVerdict::Hit, &[]), mins(0));
809 let keys = ["a", "a/agent-2", "b", "c", "d"].map(String::from).to_vec();
810 v.sessions_ingested(Some(keys));
811 let (report, problems) = refresh(&mut v, mins(1));
812 assert_eq!(report.recall.sessions_ingested, Some(4));
813 assert_eq!(report.recall.session_recall_rate, Some(0.5));
814 assert!(problems.other.is_empty(), "under the minimum sample");
815 }
816
817 #[test]
818 fn sessions_that_never_recall_degrade_once_until_they_recover_past_the_band() {
819 let mut v = ValueFacts::default();
820 let keys: Vec<String> = (0..10).map(|n| format!("s{n}")).collect();
821 v.recalled(recall("s0", RecallVerdict::Hit, &[]), mins(0));
822 v.recalled(recall("s1", RecallVerdict::Hit, &[]), mins(0));
823 v.sessions_ingested(Some(keys.clone()));
824 let (_, problems) = refresh(&mut v, mins(1));
825 assert_eq!(problems.other.len(), 1);
826 assert!(
827 problems.other[0].contains("2 of 10 sessions in the last 24h recalled"),
828 "{problems:?}"
829 );
830 v.recalled(recall("s2", RecallVerdict::Hit, &[]), mins(2));
831 assert_eq!(refresh(&mut v, mins(3)).1.other.len(), 1, "inside band");
832 v.recalled(recall("s3", RecallVerdict::Hit, &[]), mins(4));
833 assert!(refresh(&mut v, mins(5)).1.other.is_empty());
834
835 v.sessions_ingested(None);
836 let (report, _) = refresh(&mut v, mins(8));
837 assert_eq!(report.recall.session_recall_rate, None);
838 }
839
840 #[test]
841 fn an_unknown_ingest_count_keeps_the_session_alert_until_a_count_clears_it() {
842 let mut v = ValueFacts::default();
843 let keys: Vec<String> = (0..10).map(|n| format!("s{n}")).collect();
844 v.recalled(recall("s0", RecallVerdict::Hit, &[]), mins(0));
845 v.sessions_ingested(Some(keys.clone()));
846 assert_eq!(refresh(&mut v, mins(1)).1.other.len(), 1);
847 v.sessions_ingested(None);
848 let (report, problems) = refresh(&mut v, mins(2));
849 assert_eq!(report.recall.session_recall_rate, None);
850 assert_eq!(problems.other.len(), 1, "{problems:?}");
851 for n in 1..4 {
852 v.recalled(recall(&format!("s{n}"), RecallVerdict::Hit, &[]), mins(3));
853 }
854 v.sessions_ingested(Some(keys));
855 assert!(refresh(&mut v, mins(4)).1.other.is_empty());
856 }
857
858 #[test]
859 fn corrections_and_uses_count_only_recalled_memories_once_each() {
860 let mut v = ValueFacts::default();
861 v.recalled(recall("s", RecallVerdict::Hit, &[1, 2, 3, 4]), mins(0));
862 v.corrected(id(1));
863 v.corrected(id(1));
864 v.corrected(id(99));
865 v.used(&[id(2), id(3), id(2), id(98)]);
866 let (report, _) = refresh(&mut v, mins(1));
867 let r = report.recalled;
868 assert_eq!((r.memories, r.corrected, r.used), (4, 1, 2));
869 assert_eq!(r.corrected_share, Some(0.25));
870 assert_eq!(r.used_share, Some(0.5));
871
872 let later = mins(0) + WINDOW + Duration::minutes(1);
873 let (report, _) = refresh(&mut v, later);
874 assert_eq!(
875 report.recalled,
876 RecalledReport {
877 window_secs: WINDOW.num_seconds(),
878 ..RecalledReport::default()
879 }
880 );
881 }
882
883 #[test]
884 fn a_memory_corrected_before_it_was_recalled_is_not_counted() {
885 let mut v = ValueFacts::default();
886 v.corrected(id(1));
887 v.recalled(recall("s", RecallVerdict::Hit, &[1]), mins(0));
888 let (report, _) = refresh(&mut v, mins(1));
889 assert_eq!(report.recalled.corrected, 0);
890 }
891
892 #[test]
893 fn many_corrected_recalls_degrade_once_and_recover_past_the_band() {
894 let mut v = ValueFacts::default();
895 let ids: Vec<u128> = (0..20).collect();
896 v.recalled(recall("s", RecallVerdict::Hit, &ids), mins(0));
897 for n in 0..4 {
898 v.corrected(id(n));
899 }
900 assert!(
901 refresh(&mut v, mins(1)).1.other.is_empty(),
902 "20% is the line"
903 );
904 v.corrected(id(4));
905 let (_, problems) = refresh(&mut v, mins(2));
906 assert_eq!(problems.other.len(), 1);
907 assert!(
908 problems.other[0].contains("5 of 20 memories recalled in the last 24h were corrected"),
909 "{problems:?}"
910 );
911 let more: Vec<u128> = (100..110).collect();
912 v.recalled(recall("s", RecallVerdict::Hit, &more), mins(3));
913 assert_eq!(refresh(&mut v, mins(4)).1.other.len(), 1, "inside band");
914 let more: Vec<u128> = (110..120).collect();
915 v.recalled(recall("s", RecallVerdict::Hit, &more), mins(5));
916 assert!(refresh(&mut v, mins(6)).1.other.is_empty());
917 }
918
919 #[test]
920 fn a_peer_lagging_past_the_window_degrades_until_it_syncs() {
921 let mut v = ValueFacts::default();
922 v.peers_listed(&["u@pi".to_string(), "u@mac".to_string()], mins(0));
923 v.sync_attempted("u@pi", PeerOutcome::Synced, mins(0));
924 v.sync_attempted("u@mac", PeerOutcome::Failed, mins(0));
925 v.sync_attempted("u@mac", PeerOutcome::Failed, mins(5));
926 let (report, problems) = refresh(&mut v, mins(10));
927 assert_eq!(report.sync_peers.len(), 2);
928 let mac = &report.sync_peers[0];
929 assert_eq!(mac.peer, "u@mac");
930 assert_eq!(mac.last_success_at, None);
931 assert_eq!(mac.last_attempt_at, Some(mins(5)));
932 assert_eq!(mac.consecutive_failures, 2);
933 assert_eq!(mac.lag_secs, 600, "from startup");
934 assert_eq!(report.sync_peers[1].lag_secs, 600);
935 assert!(problems.peers.is_empty());
936
937 let late = t0() + Duration::hours(24);
938 v.sync_attempted("u@pi", PeerOutcome::Synced, late);
939 let (_, problems) = refresh(&mut v, late);
940 assert_eq!(problems.peers.len(), 1);
941 assert!(
942 problems.peers[0].contains("no sync with u@mac has succeeded in 24h 0m"),
943 "{problems:?}"
944 );
945 assert!(
946 problems.peers[0].contains("2 failures in a row"),
947 "{problems:?}"
948 );
949
950 v.sync_attempted("u@mac", PeerOutcome::Synced, late);
951 assert!(refresh(&mut v, late).1.peers.is_empty());
952
953 v.peers_listed(&["u@pi".to_string()], late);
954 assert_eq!(refresh(&mut v, late).0.sync_peers.len(), 1);
955 assert!(
956 v.refresh(late + Duration::hours(30), t0(), None)
957 .1
958 .peers
959 .is_empty(),
960 "sync checks off"
961 );
962 }
963
964 #[test]
965 fn a_push_ends_a_failure_streak_but_not_the_lag() {
966 let mut v = ValueFacts::default();
967 v.peers_listed(&["u@pi".to_string()], mins(0));
968 v.sync_attempted("u@pi", PeerOutcome::Failed, mins(1));
969 v.sync_attempted("u@pi", PeerOutcome::Pushed, mins(2));
970 let (report, _) = refresh(&mut v, mins(3));
971 let pi = &report.sync_peers[0];
972 assert_eq!(pi.consecutive_failures, 0);
973 assert_eq!(pi.last_attempt_at, Some(mins(2)));
974 assert_eq!(pi.last_success_at, None);
975 assert_eq!(pi.lag_secs, 180);
976 }
977
978 #[test]
979 fn a_peer_listed_late_lags_from_when_it_was_listed() {
980 let mut v = ValueFacts::default();
981 let late = t0() + Duration::hours(30);
982 v.peers_listed(&["u@mac".to_string()], late);
983 let (report, problems) = refresh(&mut v, late + Duration::minutes(5));
984 assert_eq!(report.sync_peers[0].lag_secs, 300);
985 assert!(problems.peers.is_empty());
986 v.peers_listed(&["u@mac".to_string()], late + Duration::hours(1));
987 let (report, _) = refresh(&mut v, late + Duration::hours(2));
988 assert_eq!(
989 report.sync_peers[0].lag_secs, 7_200,
990 "listing again keeps the start"
991 );
992 }
993
994 #[test]
995 fn a_result_for_a_peer_no_longer_listed_is_ignored() {
996 let mut v = ValueFacts::default();
997 v.peers_listed(&["u@pi".to_string()], mins(0));
998 v.peers_listed(&[], mins(0));
999 v.sync_attempted("u@pi", PeerOutcome::Failed, mins(1));
1000 assert!(refresh(&mut v, mins(2)).0.sync_peers.is_empty());
1001 }
1002
1003 fn store(total: u64, free: Option<u64>) -> StoreSize {
1004 StoreSize {
1005 checked_at: Some(mins(0)),
1006 db_bytes: total,
1007 total_bytes: total,
1008 free_bytes: free,
1009 ..StoreSize::default()
1010 }
1011 }
1012
1013 #[test]
1014 fn a_store_larger_than_its_free_space_degrades_until_space_clears_the_band() {
1015 let mut v = ValueFacts::default();
1016 let (report, _) = refresh(&mut v, mins(0));
1017 assert_eq!(report.store, None);
1018 v.store_measured(Some(store(1_000, Some(1_000))));
1019 assert!(refresh(&mut v, mins(0)).1.other.is_empty());
1020 v.store_measured(Some(store(1_000, Some(999))));
1021 let (report, problems) = refresh(&mut v, mins(1));
1022 assert_eq!(report.store, Some(store(1_000, Some(999))));
1023 assert_eq!(problems.other.len(), 1);
1024 assert!(
1025 problems.other[0].contains("compacting or rebuilding the store needs 0 MB"),
1026 "{problems:?}"
1027 );
1028 v.store_measured(Some(store(1_000, Some(1_050))));
1029 assert_eq!(refresh(&mut v, mins(2)).1.other.len(), 1, "inside band");
1030 v.store_measured(Some(store(1_000, Some(1_100))));
1031 assert!(refresh(&mut v, mins(3)).1.other.is_empty());
1032 v.store_measured(Some(store(1_000, Some(999))));
1033 assert_eq!(refresh(&mut v, mins(4)).1.other.len(), 1);
1034 v.store_measured(None);
1035 let (report, problems) = refresh(&mut v, mins(5));
1036 assert_eq!(report.store, None);
1037 assert!(problems.other.is_empty());
1038 v.store_measured(Some(store(1_000, None)));
1039 assert!(refresh(&mut v, mins(6)).1.other.is_empty());
1040 }
1041
1042 #[test]
1043 fn the_store_check_leaves_out_the_archive_that_compaction_never_rewrites() {
1044 let mut v = ValueFacts::default();
1045 v.store_measured(Some(StoreSize {
1046 checked_at: Some(mins(0)),
1047 db_bytes: 100,
1048 wal_bytes: 0,
1049 index_bytes: 100,
1050 archive_bytes: 10_000,
1051 raw_index_bytes: 100,
1052 total_bytes: 10_300,
1053 free_bytes: Some(301),
1054 }));
1055 assert!(refresh(&mut v, mins(1)).1.other.is_empty());
1056 v.store_measured(Some(StoreSize {
1057 free_bytes: Some(299),
1058 ..v.store.clone().unwrap()
1059 }));
1060 assert_eq!(refresh(&mut v, mins(2)).1.other.len(), 1);
1061 }
1062
1063 #[test]
1064 fn a_large_wal_is_not_counted_as_space_a_rewrite_needs() {
1065 let mut v = ValueFacts::default();
1066 v.store_measured(Some(StoreSize {
1067 checked_at: Some(mins(0)),
1068 db_bytes: 1_000,
1069 wal_bytes: 600,
1070 index_bytes: 50,
1071 total_bytes: 1_050,
1072 free_bytes: Some(500),
1073 ..StoreSize::default()
1074 }));
1075 assert!(refresh(&mut v, mins(1)).1.other.is_empty(), "needs 450");
1076 }
1077
1078 fn resident(rss: u64, limit: Option<u64>) -> ResidentSize {
1079 ResidentSize {
1080 checked_at: Some(mins(0)),
1081 rss_anon_bytes: rss,
1082 limit_bytes: limit,
1083 ..ResidentSize::default()
1084 }
1085 }
1086
1087 #[test]
1088 fn resident_size_near_the_limit_degrades_until_it_falls_past_the_band() {
1089 let mut v = ValueFacts::default();
1090 v.resident_measured(Some(resident(840, Some(1_000))));
1091 assert!(refresh(&mut v, mins(0)).1.other.is_empty());
1092 v.resident_measured(Some(resident(850, Some(1_000))));
1093 let (report, problems) = refresh(&mut v, mins(1));
1094 assert_eq!(report.resident, Some(resident(850, Some(1_000))));
1095 assert_eq!(problems.other.len(), 1);
1096 assert!(problems.other[0].contains("85% of its"), "{problems:?}");
1097 v.resident_measured(Some(resident(760, Some(1_000))));
1098 assert_eq!(refresh(&mut v, mins(2)).1.other.len(), 1, "inside band");
1099 v.resident_measured(Some(resident(740, Some(1_000))));
1100 assert!(refresh(&mut v, mins(3)).1.other.is_empty());
1101 v.resident_measured(Some(resident(5_000, None)));
1102 assert!(refresh(&mut v, mins(4)).1.other.is_empty());
1103 v.resident_measured(Some(resident(900, Some(1_000))));
1104 assert_eq!(refresh(&mut v, mins(5)).1.other.len(), 1);
1105 v.resident_measured(None);
1106 let (report, problems) = refresh(&mut v, mins(6));
1107 assert_eq!(report.resident, None);
1108 assert!(problems.other.is_empty());
1109 }
1110}