1use std::collections::{BTreeMap, HashMap, HashSet, VecDeque};
8use std::hash::Hash;
9
10use crate::health::{counted, human};
11use chrono::{DateTime, Duration, Utc};
12use second_brain_core::residency::ComponentResidency;
13use serde::Serialize;
14use uuid::Uuid;
15
16pub const WINDOW: Duration = Duration::hours(24);
17const MAX_EVENTS: usize = 50_000;
19
20const SESSION_RATE: Band = Band {
21 min_samples: 10,
22 degrade: 0.25,
23 recover: 0.35,
24};
25const EMPTY_OR_LOW: Band = Band {
28 min_samples: 50,
29 degrade: 0.75,
30 recover: 0.65,
31};
32const CORRECTED: Band = Band {
33 min_samples: 20,
34 degrade: 0.20,
35 recover: 0.15,
36};
37const RESIDENT_DEGRADE: f64 = 0.85;
38const RESIDENT_RECOVER: f64 = 0.75;
39const STORE_RECOVER: f64 = 1.1;
43
44#[derive(Debug, Clone, Copy, PartialEq, Eq)]
45pub enum RecallVerdict {
46 Hit,
48 LowScore,
50 Unjudged,
53 Empty,
54}
55
56#[derive(Debug, Clone, PartialEq, Eq)]
57pub enum PeerOutcome {
58 Synced,
60 Pushed,
63 Failed(String),
65}
66
67#[derive(Debug, Clone, PartialEq)]
68pub struct RecallEvent {
69 pub session: Option<String>,
71 pub verdict: RecallVerdict,
72 pub memory_ids: Vec<Uuid>,
74}
75
76#[derive(Debug, Clone, Default, PartialEq, Serialize)]
77pub struct RecallReport {
78 pub window_secs: i64,
79 pub recalls: u64,
80 pub hits: u64,
81 pub low_score: u64,
82 pub unjudged: u64,
83 pub empty: u64,
84 pub empty_share: Option<f64>,
86 pub low_score_share: Option<f64>,
88 pub unjudged_share: Option<f64>,
89 pub sessions_with_recall: u64,
90 pub sessions_ingested: Option<u64>,
93 pub session_recall_rate: Option<f64>,
95}
96
97#[derive(Debug, Clone, Default, PartialEq, Serialize)]
98pub struct RecalledReport {
99 pub window_secs: i64,
100 pub memories: u64,
101 pub corrected: u64,
102 pub used: u64,
104 pub corrected_share: Option<f64>,
105 pub used_share: Option<f64>,
106}
107
108#[derive(Debug, Clone, PartialEq, Serialize)]
109pub struct PeerSync {
110 pub peer: String,
111 pub last_success_at: Option<DateTime<Utc>>,
112 pub last_attempt_at: Option<DateTime<Utc>>,
113 pub consecutive_failures: u32,
114 pub last_error: Option<String>,
116 pub lag_secs: i64,
118}
119
120#[derive(Debug, Clone, Default, PartialEq, Eq, Serialize)]
121pub struct StoreSize {
122 pub checked_at: Option<DateTime<Utc>>,
123 pub db_bytes: u64,
125 pub wal_bytes: u64,
128 pub index_bytes: u64,
129 pub archive_bytes: u64,
130 pub raw_index_bytes: u64,
131 pub cold_bytes: u64,
132 pub total_bytes: u64,
133 pub free_bytes: Option<u64>,
135}
136
137#[derive(Debug, Clone, Default, PartialEq, Eq, Serialize)]
138pub struct ResidentSize {
139 pub checked_at: Option<DateTime<Utc>>,
140 pub rss_anon_bytes: u64,
141 pub components: ComponentResidency,
142 pub raw_index_bytes: u64,
143 pub unattributed_bytes: i64,
144 pub limit_bytes: Option<u64>,
147}
148
149struct Band {
154 min_samples: u64,
155 degrade: f64,
156 recover: f64,
157}
158
159#[derive(Debug, Default)]
161struct Latches {
162 session_rate: bool,
163 empty_or_low: bool,
164 corrected: bool,
165 store: bool,
166 resident: bool,
167}
168
169#[derive(Debug, Clone, Copy, Default)]
170struct Flags {
171 corrected: bool,
172 used: bool,
173}
174
175#[derive(Debug)]
178struct Recent<K, V> {
179 entries: HashMap<K, (DateTime<Utc>, V)>,
180 order: VecDeque<(DateTime<Utc>, K)>,
181}
182
183impl<K, V> Default for Recent<K, V> {
184 fn default() -> Self {
185 Self {
186 entries: HashMap::new(),
187 order: VecDeque::new(),
188 }
189 }
190}
191
192impl Band {
193 fn above(&self, latched: bool, count: u64, of: u64) -> bool {
195 if of == 0 || (!latched && of < self.min_samples) {
196 return false;
197 }
198 let share = count as f64 / of as f64;
199 share > if latched { self.recover } else { self.degrade }
200 }
201
202 fn below(&self, latched: bool, count: u64, of: u64) -> bool {
204 if of == 0 || (!latched && of < self.min_samples) {
205 return false;
206 }
207 let share = count as f64 / of as f64;
208 share < if latched { self.recover } else { self.degrade }
209 }
210}
211
212impl<K: Hash + Eq + Clone, V: Default> Recent<K, V> {
213 fn touch(&mut self, key: K, at: DateTime<Utc>) {
214 match self.entries.get_mut(&key) {
215 Some(entry) => entry.0 = at,
216 None => {
217 self.entries.insert(key.clone(), (at, V::default()));
218 }
219 }
220 self.order.push_back((at, key));
221 if self.order.len() > 4 * MAX_EVENTS {
222 let entries = &self.entries;
223 self.order
224 .retain(|(at, key)| entries.get(key).is_some_and(|e| e.0 == *at));
225 }
226 }
227
228 fn get_mut(&mut self, key: &K) -> Option<&mut V> {
229 self.entries.get_mut(key).map(|e| &mut e.1)
230 }
231
232 fn prune(&mut self, cutoff: DateTime<Utc>, mut evicted: impl FnMut(&V)) {
234 while let Some((at, key)) = self.order.front() {
235 let live = self.entries.get(key).is_some_and(|e| e.0 == *at);
236 if live && *at >= cutoff && self.entries.len() <= MAX_EVENTS {
237 break;
238 }
239 if let Some((_, key)) = self.order.pop_front()
240 && live
241 && let Some((_, value)) = self.entries.remove(&key)
242 {
243 evicted(&value);
244 }
245 }
246 }
247
248 fn len(&self) -> usize {
249 self.entries.len()
250 }
251
252 fn count_since(&self, cutoff: DateTime<Utc>) -> usize {
253 self.entries
254 .values()
255 .filter(|(at, _)| *at >= cutoff)
256 .count()
257 }
258
259 fn contains(&self, key: &K) -> bool {
260 self.entries.contains_key(key)
261 }
262}
263
264#[derive(Debug, Default)]
265pub struct ValueFacts {
266 recalls: VecDeque<(DateTime<Utc>, RecallVerdict)>,
267 hits: u64,
268 low_score: u64,
269 unjudged: u64,
270 empty: u64,
271 recall_sessions: Recent<String, ()>,
272 recalled: Recent<Uuid, Flags>,
273 corrected: u64,
274 used: u64,
275 ingested_sessions: Option<HashSet<String>>,
276 session_problem: Option<String>,
279 peers: BTreeMap<String, PeerState>,
280 store: Option<StoreSize>,
281 resident: Option<ResidentSize>,
282 latches: Latches,
283}
284
285#[derive(Debug, Clone, Default)]
286struct PeerState {
287 listed_at: Option<DateTime<Utc>>,
288 last_success_at: Option<DateTime<Utc>>,
289 last_attempt_at: Option<DateTime<Utc>>,
290 consecutive_failures: u32,
291 last_error: Option<String>,
292}
293
294#[derive(Debug, Default, PartialEq)]
298pub struct ValueProblems {
299 pub peers: Vec<String>,
300 pub other: Vec<String>,
301}
302
303#[derive(Debug, Clone, Default, PartialEq, Serialize)]
304pub struct ValueReport {
305 pub recall: RecallReport,
306 pub recalled: RecalledReport,
307 pub sync_peers: Vec<PeerSync>,
308 pub store: Option<StoreSize>,
309 pub resident: Option<ResidentSize>,
310}
311
312pub fn session_of(key: &str) -> &str {
315 key.split_once('/').map_or(key, |(session, _)| session)
316}
317
318impl ValueFacts {
319 pub fn recalled(&mut self, event: RecallEvent, now: DateTime<Utc>) {
320 self.recalls.push_back((now, event.verdict));
321 *self.count_of(event.verdict) += 1;
322 if let Some(session) = &event.session {
323 self.recall_sessions
324 .touch(session_of(session).to_string(), now);
325 }
326 for id in event.memory_ids {
327 self.recalled.touch(id, now);
328 }
329 }
330
331 fn count_of(&mut self, verdict: RecallVerdict) -> &mut u64 {
332 match verdict {
333 RecallVerdict::Hit => &mut self.hits,
334 RecallVerdict::LowScore => &mut self.low_score,
335 RecallVerdict::Unjudged => &mut self.unjudged,
336 RecallVerdict::Empty => &mut self.empty,
337 }
338 }
339
340 pub fn corrected(&mut self, id: Uuid) {
342 if let Some(flags) = self.recalled.get_mut(&id)
343 && !flags.corrected
344 {
345 flags.corrected = true;
346 self.corrected += 1;
347 }
348 }
349
350 pub fn used(&mut self, ids: &[Uuid]) {
351 for id in ids {
352 if let Some(flags) = self.recalled.get_mut(id)
353 && !flags.used
354 {
355 flags.used = true;
356 self.used += 1;
357 }
358 }
359 }
360
361 pub fn sessions_ingested(&mut self, keys: Option<Vec<String>>) {
364 self.ingested_sessions =
365 keys.map(|keys| keys.iter().map(|key| session_of(key).to_string()).collect());
366 }
367
368 pub fn peers_listed(&mut self, peers: &[String], now: DateTime<Utc>) {
372 self.peers.retain(|peer, _| peers.contains(peer));
373 for peer in peers {
374 self.peers.entry(peer.clone()).or_insert_with(|| PeerState {
375 listed_at: Some(now),
376 ..PeerState::default()
377 });
378 }
379 }
380
381 pub fn sync_attempted(&mut self, peer: &str, outcome: PeerOutcome, now: DateTime<Utc>) {
382 let Some(state) = self.peers.get_mut(peer) else {
383 return;
384 };
385 match outcome {
386 PeerOutcome::Synced => {
387 state.last_attempt_at = Some(now);
388 state.last_success_at = Some(now);
389 state.consecutive_failures = 0;
390 state.last_error = None;
391 }
392 PeerOutcome::Pushed => {
393 state.last_attempt_at = Some(now);
394 state.consecutive_failures = 0;
395 state.last_error = None;
396 }
397 PeerOutcome::Failed(error) => {
398 state.last_attempt_at = Some(now);
399 state.consecutive_failures += 1;
400 state.last_error = Some(error);
401 }
402 }
403 }
404
405 pub fn store_measured(&mut self, size: Option<StoreSize>) {
407 self.store = size;
408 }
409
410 pub fn resident_measured(&mut self, size: Option<ResidentSize>) {
412 self.resident = size;
413 }
414
415 pub fn refresh(
419 &mut self,
420 now: DateTime<Utc>,
421 started: DateTime<Utc>,
422 sync_overdue: Option<Duration>,
423 ) -> (ValueReport, ValueProblems) {
424 self.prune(now - WINDOW);
425 let mut problems = ValueProblems::default();
426 let recall = self.recall_report(now - WINDOW, &mut problems);
427 let recalled = self.recalled_report(&mut problems);
428 let sync_peers = self.peer_report(now, started, sync_overdue, &mut problems);
429 self.check_store(&mut problems);
430 self.check_resident(&mut problems);
431 let report = ValueReport {
432 recall,
433 recalled,
434 sync_peers,
435 store: self.store.clone(),
436 resident: self.resident.clone(),
437 };
438 (report, problems)
439 }
440
441 fn prune(&mut self, cutoff: DateTime<Utc>) {
442 while let Some(&(at, verdict)) = self.recalls.front() {
443 if at >= cutoff && self.recalls.len() <= MAX_EVENTS {
444 break;
445 }
446 self.recalls.pop_front();
447 *self.count_of(verdict) -= 1;
448 }
449 self.recall_sessions.prune(cutoff - session_grace(), |_| {});
450 let (corrected, used) = (&mut self.corrected, &mut self.used);
451 self.recalled.prune(cutoff, |flags| {
452 *corrected -= u64::from(flags.corrected);
453 *used -= u64::from(flags.used);
454 });
455 }
456
457 fn recall_report(
458 &mut self,
459 cutoff: DateTime<Utc>,
460 problems: &mut ValueProblems,
461 ) -> RecallReport {
462 let recalls = self.recalls.len() as u64;
463 let ingested = self.ingested_sessions.as_ref();
464 let joined = ingested.map(|keys| {
465 keys.iter()
466 .filter(|key| self.recall_sessions.contains(*key))
467 .count() as u64
468 });
469 let sessions_ingested = ingested.map(|keys| keys.len() as u64);
470
471 let judged = recalls - self.unjudged;
472 let bad = self.empty + self.low_score;
473 self.latches.empty_or_low = EMPTY_OR_LOW.above(self.latches.empty_or_low, bad, judged);
474 if self.latches.empty_or_low {
475 problems.other.push(format!(
476 "{} of {judged} judged recalls in the last 24h found nothing that cleared the \
477 relevance bar ({} empty, {} below it)",
478 percent(bad, judged),
479 self.empty,
480 self.low_score
481 ));
482 }
483 if let (Some(joined), Some(of)) = (joined, sessions_ingested) {
484 self.latches.session_rate = SESSION_RATE.below(self.latches.session_rate, joined, of);
485 self.session_problem = self.latches.session_rate.then(|| {
486 format!(
487 "only {joined} of {of} sessions in the last 24h recalled from second-brain; \
488 check its hooks and MCP server with `sb doctor`"
489 )
490 });
491 }
492 if let Some(problem) = &self.session_problem {
493 problems.other.push(problem.clone());
494 }
495
496 RecallReport {
497 window_secs: WINDOW.num_seconds(),
498 recalls,
499 hits: self.hits,
500 low_score: self.low_score,
501 unjudged: self.unjudged,
502 empty: self.empty,
503 empty_share: share(self.empty, judged),
504 low_score_share: share(self.low_score, judged),
505 unjudged_share: share(self.unjudged, recalls),
506 sessions_with_recall: self.recall_sessions.count_since(cutoff) as u64,
507 sessions_ingested,
508 session_recall_rate: joined
509 .zip(sessions_ingested)
510 .and_then(|(j, of)| share(j, of)),
511 }
512 }
513
514 fn recalled_report(&mut self, problems: &mut ValueProblems) -> RecalledReport {
515 let memories = self.recalled.len() as u64;
516 self.latches.corrected = CORRECTED.above(self.latches.corrected, self.corrected, memories);
517 if self.latches.corrected {
518 problems.other.push(format!(
519 "{} of {memories} memories recalled in the last 24h were corrected",
520 self.corrected
521 ));
522 }
523 RecalledReport {
524 window_secs: WINDOW.num_seconds(),
525 memories,
526 corrected: self.corrected,
527 used: self.used,
528 corrected_share: share(self.corrected, memories),
529 used_share: share(self.used, memories),
530 }
531 }
532
533 fn peer_report(
534 &self,
535 now: DateTime<Utc>,
536 started: DateTime<Utc>,
537 sync_overdue: Option<Duration>,
538 problems: &mut ValueProblems,
539 ) -> Vec<PeerSync> {
540 self.peers
541 .iter()
542 .map(|(peer, state)| {
543 let since = state.last_success_at.or(state.listed_at).unwrap_or(started);
544 let lag = (now - since).max(Duration::zero());
545 if sync_overdue.is_some_and(|window| lag >= window) {
546 let failures = match (state.consecutive_failures, &state.last_error) {
547 (0, _) => String::new(),
548 (n, error) => format!(
549 " ({} in a row{})",
550 counted(u64::from(n), "failure", "failures"),
551 error
552 .as_deref()
553 .and_then(|e| e.lines().next())
554 .map(|line| format!("; the last: {line}"))
555 .unwrap_or_default()
556 ),
557 };
558 problems.peers.push(format!(
559 "no sync with {peer} has succeeded in {}{failures}",
560 human(lag)
561 ));
562 }
563 PeerSync {
564 peer: peer.clone(),
565 last_success_at: state.last_success_at,
566 last_attempt_at: state.last_attempt_at,
567 consecutive_failures: state.consecutive_failures,
568 last_error: state.last_error.clone(),
569 lag_secs: lag.num_seconds(),
570 }
571 })
572 .collect()
573 }
574
575 fn check_store(&mut self, problems: &mut ValueProblems) {
576 let Some((rewritten, free)) = self.store.as_ref().and_then(|s| {
577 let rewritten =
578 s.db_bytes.saturating_sub(s.wal_bytes) + s.index_bytes + s.raw_index_bytes;
579 s.free_bytes.map(|free| (rewritten, free))
580 }) else {
581 self.latches.store = false;
582 return;
583 };
584 let needed = if self.latches.store {
585 rewritten as f64 * STORE_RECOVER
586 } else {
587 rewritten as f64
588 };
589 self.latches.store = (free as f64) < needed;
590 if self.latches.store {
591 problems.other.push(format!(
592 "compacting or rebuilding the store needs {} MB and its disk has {} MB free",
593 rewritten / 1_000_000,
594 free / 1_000_000
595 ));
596 }
597 }
598
599 fn check_resident(&mut self, problems: &mut ValueProblems) {
600 let Some((rss, limit)) = self.resident.as_ref().and_then(|r| {
601 r.limit_bytes
602 .filter(|&l| l > 0)
603 .map(|l| (r.rss_anon_bytes, l))
604 }) else {
605 self.latches.resident = false;
606 return;
607 };
608 let ratio = rss as f64 / limit as f64;
609 let line = if self.latches.resident {
610 RESIDENT_RECOVER
611 } else {
612 RESIDENT_DEGRADE
613 };
614 self.latches.resident = ratio >= line;
615 if self.latches.resident {
616 problems.other.push(format!(
617 "the daemon holds {} MB resident, {} of its {} MB limit",
618 rss / 1_000_000,
619 percent(rss, limit),
620 limit / 1_000_000
621 ));
622 }
623 }
624}
625
626fn session_grace() -> Duration {
630 Duration::from_std(crate::progress::INTERVAL).unwrap_or(Duration::zero())
631}
632
633fn share(count: u64, of: u64) -> Option<f64> {
634 (of > 0).then(|| count as f64 / of as f64)
635}
636
637fn percent(count: u64, of: u64) -> String {
638 format!("{:.0}%", count as f64 * 100.0 / of as f64)
639}
640
641#[cfg(test)]
642mod tests {
643 use super::*;
644
645 fn t0() -> DateTime<Utc> {
646 DateTime::parse_from_rfc3339("2026-10-31T12:00:00Z")
647 .unwrap()
648 .with_timezone(&Utc)
649 }
650
651 fn mins(m: i64) -> DateTime<Utc> {
652 t0() + Duration::minutes(m)
653 }
654
655 fn id(n: u128) -> Uuid {
656 Uuid::from_u128(n)
657 }
658
659 fn recall(session: &str, verdict: RecallVerdict, ids: &[u128]) -> RecallEvent {
660 RecallEvent {
661 session: Some(session.to_string()),
662 verdict,
663 memory_ids: ids.iter().map(|&n| id(n)).collect(),
664 }
665 }
666
667 fn refresh(v: &mut ValueFacts, now: DateTime<Utc>) -> (ValueReport, ValueProblems) {
668 v.refresh(now, t0(), Some(Duration::hours(24)))
669 }
670
671 #[test]
672 fn subagent_keys_count_as_their_parent_session() {
673 assert_eq!(session_of("abc"), "abc");
674 assert_eq!(session_of("abc/agent-1"), "abc");
675 }
676
677 #[test]
678 fn recall_shares_count_empty_and_low_score_recalls() {
679 let mut v = ValueFacts::default();
680 v.recalled(recall("s1", RecallVerdict::Hit, &[1]), mins(0));
681 v.recalled(recall("s1", RecallVerdict::Empty, &[]), mins(1));
682 v.recalled(recall("s2", RecallVerdict::LowScore, &[2]), mins(2));
683 v.recalled(recall("s2", RecallVerdict::Empty, &[]), mins(3));
684 v.recalled(recall("s2", RecallVerdict::Unjudged, &[]), mins(3));
685 let (report, problems) = refresh(&mut v, mins(4));
686 let r = report.recall;
687 assert_eq!(r.window_secs, WINDOW.num_seconds());
688 assert_eq!((r.recalls, r.hits, r.low_score, r.empty), (5, 1, 1, 2));
689 assert_eq!(r.empty_share, Some(0.5), "over judged recalls");
690 assert_eq!(r.low_score_share, Some(0.25), "over judged recalls");
691 assert_eq!(r.unjudged_share, Some(0.2), "over all recalls");
692 assert_eq!(r.sessions_with_recall, 2);
693 assert_eq!(problems, ValueProblems::default());
694 }
695
696 #[test]
697 fn mostly_empty_recalls_degrade_once_and_recover_past_the_band() {
698 let mut v = ValueFacts::default();
699 for m in 0..49 {
700 v.recalled(recall("s", RecallVerdict::Empty, &[]), mins(m));
701 }
702 assert!(refresh(&mut v, mins(49)).1.other.is_empty(), "too few");
703 v.recalled(recall("s", RecallVerdict::LowScore, &[1]), mins(49));
704 let (_, problems) = refresh(&mut v, mins(50));
705 assert_eq!(problems.other.len(), 1, "{problems:?}");
706 assert!(
707 problems.other[0].contains("of 50 judged recalls in the last 24h"),
708 "{problems:?}"
709 );
710 for m in 50..71 {
711 v.recalled(recall("s", RecallVerdict::Hit, &[1]), mins(m));
712 }
713 let (report, problems) = refresh(&mut v, mins(71));
714 assert_eq!(
715 report.recall.recalls, 71,
716 "50 of 71 is 70%, inside the band"
717 );
718 assert_eq!(problems.other.len(), 1);
719 for m in 71..77 {
720 v.recalled(recall("s", RecallVerdict::Hit, &[1]), mins(m));
721 }
722 assert!(
723 refresh(&mut v, mins(77)).1.other.is_empty(),
724 "50 of 77 is 65%"
725 );
726 }
727
728 #[test]
729 fn a_latched_share_holds_below_the_minimum_until_the_window_empties() {
730 let mut v = ValueFacts::default();
731 for m in 0..50 {
732 v.recalled(recall("s", RecallVerdict::Empty, &[]), mins(m));
733 }
734 assert_eq!(refresh(&mut v, mins(50)).1.other.len(), 1);
735 let first_gone = mins(0) + WINDOW + Duration::seconds(30);
736 let (report, problems) = refresh(&mut v, first_gone);
737 assert_eq!(report.recall.recalls, 49);
738 assert_eq!(problems.other.len(), 1, "below the minimum, still latched");
739 let all_gone = mins(50) + WINDOW;
740 assert!(refresh(&mut v, all_gone).1.other.is_empty());
741 }
742
743 #[test]
744 fn unjudged_recalls_are_left_out_of_the_empty_or_low_share() {
745 let mut v = ValueFacts::default();
746 for m in 0..50 {
747 v.recalled(recall("s", RecallVerdict::Unjudged, &[]), mins(m));
748 }
749 let (report, problems) = refresh(&mut v, mins(50));
750 assert_eq!(report.recall.unjudged, 50);
751 assert_eq!(report.recall.unjudged_share, Some(1.0));
752 assert!(problems.other.is_empty(), "{problems:?}");
753 for m in 50..99 {
754 v.recalled(recall("s", RecallVerdict::Empty, &[]), mins(m));
755 }
756 assert!(refresh(&mut v, mins(99)).1.other.is_empty(), "49 judged");
757 v.recalled(recall("s", RecallVerdict::LowScore, &[]), mins(99));
758 let (_, problems) = refresh(&mut v, mins(100));
759 assert_eq!(problems.other.len(), 1);
760 assert!(
761 problems.other[0].contains("100% of 50 judged recalls in the last 24h"),
762 "{problems:?}"
763 );
764 assert!(
765 problems.other[0].contains("(49 empty, 1 below it)"),
766 "{problems:?}"
767 );
768 }
769
770 #[test]
771 fn a_session_recalled_just_before_the_window_edge_counts_until_the_next_ingest_read() {
772 let mut v = ValueFacts::default();
773 v.recalled(recall("a", RecallVerdict::Hit, &[]), mins(0));
774 v.sessions_ingested(Some(vec!["a".to_string()]));
775 let edge = mins(0) + WINDOW + Duration::minutes(4);
776 let recall = refresh(&mut v, edge).0.recall;
777 assert_eq!(recall.session_recall_rate, Some(1.0));
778 assert_eq!(
779 recall.sessions_with_recall, 0,
780 "outside the reported window"
781 );
782 let later = mins(0) + WINDOW + Duration::minutes(6);
783 assert_eq!(
784 refresh(&mut v, later).0.recall.session_recall_rate,
785 Some(0.0)
786 );
787 }
788
789 #[test]
790 fn recalls_older_than_the_window_drop_out() {
791 let mut v = ValueFacts::default();
792 v.recalled(recall("s1", RecallVerdict::Empty, &[1]), mins(0));
793 v.recalled(recall("s2", RecallVerdict::Hit, &[2]), mins(60));
794 let (report, _) = refresh(&mut v, mins(0) + WINDOW + Duration::minutes(10));
795 assert_eq!(report.recall.recalls, 1);
796 assert_eq!(report.recall.empty, 0);
797 assert_eq!(report.recall.sessions_with_recall, 1);
798 assert_eq!(report.recalled.memories, 1);
799 }
800
801 #[test]
802 fn the_event_window_is_bounded() {
803 let mut v = ValueFacts::default();
804 for n in 0..(MAX_EVENTS as u128 + 10) {
805 v.recalled(recall(&format!("s{n}"), RecallVerdict::Hit, &[n]), mins(0));
806 }
807 let (report, _) = refresh(&mut v, mins(1));
808 assert_eq!(report.recall.recalls, MAX_EVENTS as u64);
809 assert_eq!(report.recall.sessions_with_recall, MAX_EVENTS as u64);
810 assert_eq!(report.recalled.memories, MAX_EVENTS as u64);
811 }
812
813 #[test]
814 fn session_recall_rate_joins_recalling_sessions_with_ingested_ones() {
815 let mut v = ValueFacts::default();
816 let (report, _) = refresh(&mut v, mins(0));
817 assert_eq!(report.recall.sessions_ingested, None);
818 assert_eq!(report.recall.session_recall_rate, None);
819
820 v.recalled(recall("a", RecallVerdict::Hit, &[]), mins(0));
821 v.recalled(recall("b/agent-1", RecallVerdict::Hit, &[]), mins(0));
822 v.recalled(recall("not-ingested", RecallVerdict::Hit, &[]), mins(0));
823 let keys = ["a", "a/agent-2", "b", "c", "d"].map(String::from).to_vec();
824 v.sessions_ingested(Some(keys));
825 let (report, problems) = refresh(&mut v, mins(1));
826 assert_eq!(report.recall.sessions_ingested, Some(4));
827 assert_eq!(report.recall.session_recall_rate, Some(0.5));
828 assert!(problems.other.is_empty(), "under the minimum sample");
829 }
830
831 #[test]
832 fn sessions_that_never_recall_degrade_once_until_they_recover_past_the_band() {
833 let mut v = ValueFacts::default();
834 let keys: Vec<String> = (0..10).map(|n| format!("s{n}")).collect();
835 v.recalled(recall("s0", RecallVerdict::Hit, &[]), mins(0));
836 v.recalled(recall("s1", RecallVerdict::Hit, &[]), mins(0));
837 v.sessions_ingested(Some(keys.clone()));
838 let (_, problems) = refresh(&mut v, mins(1));
839 assert_eq!(problems.other.len(), 1);
840 assert!(
841 problems.other[0].contains("2 of 10 sessions in the last 24h recalled"),
842 "{problems:?}"
843 );
844 v.recalled(recall("s2", RecallVerdict::Hit, &[]), mins(2));
845 assert_eq!(refresh(&mut v, mins(3)).1.other.len(), 1, "inside band");
846 v.recalled(recall("s3", RecallVerdict::Hit, &[]), mins(4));
847 assert!(refresh(&mut v, mins(5)).1.other.is_empty());
848
849 v.sessions_ingested(None);
850 let (report, _) = refresh(&mut v, mins(8));
851 assert_eq!(report.recall.session_recall_rate, None);
852 }
853
854 #[test]
855 fn an_unknown_ingest_count_keeps_the_session_alert_until_a_count_clears_it() {
856 let mut v = ValueFacts::default();
857 let keys: Vec<String> = (0..10).map(|n| format!("s{n}")).collect();
858 v.recalled(recall("s0", RecallVerdict::Hit, &[]), mins(0));
859 v.sessions_ingested(Some(keys.clone()));
860 assert_eq!(refresh(&mut v, mins(1)).1.other.len(), 1);
861 v.sessions_ingested(None);
862 let (report, problems) = refresh(&mut v, mins(2));
863 assert_eq!(report.recall.session_recall_rate, None);
864 assert_eq!(problems.other.len(), 1, "{problems:?}");
865 for n in 1..4 {
866 v.recalled(recall(&format!("s{n}"), RecallVerdict::Hit, &[]), mins(3));
867 }
868 v.sessions_ingested(Some(keys));
869 assert!(refresh(&mut v, mins(4)).1.other.is_empty());
870 }
871
872 #[test]
873 fn corrections_and_uses_count_only_recalled_memories_once_each() {
874 let mut v = ValueFacts::default();
875 v.recalled(recall("s", RecallVerdict::Hit, &[1, 2, 3, 4]), mins(0));
876 v.corrected(id(1));
877 v.corrected(id(1));
878 v.corrected(id(99));
879 v.used(&[id(2), id(3), id(2), id(98)]);
880 let (report, _) = refresh(&mut v, mins(1));
881 let r = report.recalled;
882 assert_eq!((r.memories, r.corrected, r.used), (4, 1, 2));
883 assert_eq!(r.corrected_share, Some(0.25));
884 assert_eq!(r.used_share, Some(0.5));
885
886 let later = mins(0) + WINDOW + Duration::minutes(1);
887 let (report, _) = refresh(&mut v, later);
888 assert_eq!(
889 report.recalled,
890 RecalledReport {
891 window_secs: WINDOW.num_seconds(),
892 ..RecalledReport::default()
893 }
894 );
895 }
896
897 #[test]
898 fn a_memory_corrected_before_it_was_recalled_is_not_counted() {
899 let mut v = ValueFacts::default();
900 v.corrected(id(1));
901 v.recalled(recall("s", RecallVerdict::Hit, &[1]), mins(0));
902 let (report, _) = refresh(&mut v, mins(1));
903 assert_eq!(report.recalled.corrected, 0);
904 }
905
906 #[test]
907 fn many_corrected_recalls_degrade_once_and_recover_past_the_band() {
908 let mut v = ValueFacts::default();
909 let ids: Vec<u128> = (0..20).collect();
910 v.recalled(recall("s", RecallVerdict::Hit, &ids), mins(0));
911 for n in 0..4 {
912 v.corrected(id(n));
913 }
914 assert!(
915 refresh(&mut v, mins(1)).1.other.is_empty(),
916 "20% is the line"
917 );
918 v.corrected(id(4));
919 let (_, problems) = refresh(&mut v, mins(2));
920 assert_eq!(problems.other.len(), 1);
921 assert!(
922 problems.other[0].contains("5 of 20 memories recalled in the last 24h were corrected"),
923 "{problems:?}"
924 );
925 let more: Vec<u128> = (100..110).collect();
926 v.recalled(recall("s", RecallVerdict::Hit, &more), mins(3));
927 assert_eq!(refresh(&mut v, mins(4)).1.other.len(), 1, "inside band");
928 let more: Vec<u128> = (110..120).collect();
929 v.recalled(recall("s", RecallVerdict::Hit, &more), mins(5));
930 assert!(refresh(&mut v, mins(6)).1.other.is_empty());
931 }
932
933 #[test]
934 fn a_peer_lagging_past_the_window_degrades_until_it_syncs() {
935 let mut v = ValueFacts::default();
936 v.peers_listed(&["u@pi".to_string(), "u@mac".to_string()], mins(0));
937 v.sync_attempted("u@pi", PeerOutcome::Synced, mins(0));
938 v.sync_attempted("u@mac", PeerOutcome::Failed("refused".into()), mins(0));
939 v.sync_attempted(
940 "u@mac",
941 PeerOutcome::Failed("u@mac refused the sync: pair it with `sb pair`".into()),
942 mins(5),
943 );
944 let (report, problems) = refresh(&mut v, mins(10));
945 assert_eq!(report.sync_peers.len(), 2);
946 let mac = &report.sync_peers[0];
947 assert_eq!(mac.peer, "u@mac");
948 assert_eq!(mac.last_success_at, None);
949 assert_eq!(mac.last_attempt_at, Some(mins(5)));
950 assert_eq!(mac.consecutive_failures, 2);
951 assert_eq!(mac.lag_secs, 600, "from startup");
952 assert_eq!(
953 mac.last_error.as_deref(),
954 Some("u@mac refused the sync: pair it with `sb pair`")
955 );
956 assert_eq!(report.sync_peers[1].lag_secs, 600);
957 assert_eq!(report.sync_peers[1].last_error, None);
958 assert!(problems.peers.is_empty());
959
960 let late = t0() + Duration::hours(24);
961 v.sync_attempted("u@pi", PeerOutcome::Synced, late);
962 let (_, problems) = refresh(&mut v, late);
963 assert_eq!(problems.peers.len(), 1);
964 assert!(
965 problems.peers[0].contains("no sync with u@mac has succeeded in 24h 0m"),
966 "{problems:?}"
967 );
968 assert!(
969 problems.peers[0].contains("2 failures in a row"),
970 "{problems:?}"
971 );
972 assert!(
973 problems.peers[0].contains("the last: u@mac refused the sync: pair it with `sb pair`"),
974 "{problems:?}"
975 );
976
977 v.sync_attempted("u@mac", PeerOutcome::Synced, late);
978 let (report, problems) = refresh(&mut v, late);
979 assert!(problems.peers.is_empty());
980 assert_eq!(report.sync_peers[0].last_error, None);
981
982 v.peers_listed(&["u@pi".to_string()], late);
983 assert_eq!(refresh(&mut v, late).0.sync_peers.len(), 1);
984 assert!(
985 v.refresh(late + Duration::hours(30), t0(), None)
986 .1
987 .peers
988 .is_empty(),
989 "sync checks off"
990 );
991 }
992
993 #[test]
994 fn a_push_ends_a_failure_streak_but_not_the_lag() {
995 let mut v = ValueFacts::default();
996 v.peers_listed(&["u@pi".to_string()], mins(0));
997 v.sync_attempted("u@pi", PeerOutcome::Failed("timed out".into()), mins(1));
998 v.sync_attempted("u@pi", PeerOutcome::Pushed, mins(2));
999 let (report, _) = refresh(&mut v, mins(3));
1000 let pi = &report.sync_peers[0];
1001 assert_eq!(pi.consecutive_failures, 0);
1002 assert_eq!(pi.last_attempt_at, Some(mins(2)));
1003 assert_eq!(pi.last_success_at, None);
1004 assert_eq!(pi.last_error, None);
1005 assert_eq!(pi.lag_secs, 180);
1006 }
1007
1008 #[test]
1009 fn a_peer_listed_late_lags_from_when_it_was_listed() {
1010 let mut v = ValueFacts::default();
1011 let late = t0() + Duration::hours(30);
1012 v.peers_listed(&["u@mac".to_string()], late);
1013 let (report, problems) = refresh(&mut v, late + Duration::minutes(5));
1014 assert_eq!(report.sync_peers[0].lag_secs, 300);
1015 assert!(problems.peers.is_empty());
1016 v.peers_listed(&["u@mac".to_string()], late + Duration::hours(1));
1017 let (report, _) = refresh(&mut v, late + Duration::hours(2));
1018 assert_eq!(
1019 report.sync_peers[0].lag_secs, 7_200,
1020 "listing again keeps the start"
1021 );
1022 }
1023
1024 #[test]
1025 fn a_result_for_a_peer_no_longer_listed_is_ignored() {
1026 let mut v = ValueFacts::default();
1027 v.peers_listed(&["u@pi".to_string()], mins(0));
1028 v.peers_listed(&[], mins(0));
1029 v.sync_attempted("u@pi", PeerOutcome::Failed("timed out".into()), mins(1));
1030 assert!(refresh(&mut v, mins(2)).0.sync_peers.is_empty());
1031 }
1032
1033 fn store(total: u64, free: Option<u64>) -> StoreSize {
1034 StoreSize {
1035 checked_at: Some(mins(0)),
1036 db_bytes: total,
1037 total_bytes: total,
1038 free_bytes: free,
1039 ..StoreSize::default()
1040 }
1041 }
1042
1043 #[test]
1044 fn a_store_larger_than_its_free_space_degrades_until_space_clears_the_band() {
1045 let mut v = ValueFacts::default();
1046 let (report, _) = refresh(&mut v, mins(0));
1047 assert_eq!(report.store, None);
1048 v.store_measured(Some(store(1_000, Some(1_000))));
1049 assert!(refresh(&mut v, mins(0)).1.other.is_empty());
1050 v.store_measured(Some(store(1_000, Some(999))));
1051 let (report, problems) = refresh(&mut v, mins(1));
1052 assert_eq!(report.store, Some(store(1_000, Some(999))));
1053 assert_eq!(problems.other.len(), 1);
1054 assert!(
1055 problems.other[0].contains("compacting or rebuilding the store needs 0 MB"),
1056 "{problems:?}"
1057 );
1058 v.store_measured(Some(store(1_000, Some(1_050))));
1059 assert_eq!(refresh(&mut v, mins(2)).1.other.len(), 1, "inside band");
1060 v.store_measured(Some(store(1_000, Some(1_100))));
1061 assert!(refresh(&mut v, mins(3)).1.other.is_empty());
1062 v.store_measured(Some(store(1_000, Some(999))));
1063 assert_eq!(refresh(&mut v, mins(4)).1.other.len(), 1);
1064 v.store_measured(None);
1065 let (report, problems) = refresh(&mut v, mins(5));
1066 assert_eq!(report.store, None);
1067 assert!(problems.other.is_empty());
1068 v.store_measured(Some(store(1_000, None)));
1069 assert!(refresh(&mut v, mins(6)).1.other.is_empty());
1070 }
1071
1072 #[test]
1073 fn the_store_check_leaves_out_the_archive_that_compaction_never_rewrites() {
1074 let mut v = ValueFacts::default();
1075 v.store_measured(Some(StoreSize {
1076 checked_at: Some(mins(0)),
1077 db_bytes: 100,
1078 wal_bytes: 0,
1079 index_bytes: 100,
1080 archive_bytes: 10_000,
1081 raw_index_bytes: 100,
1082 cold_bytes: 0,
1083 total_bytes: 10_300,
1084 free_bytes: Some(301),
1085 }));
1086 assert!(refresh(&mut v, mins(1)).1.other.is_empty());
1087 v.store_measured(Some(StoreSize {
1088 free_bytes: Some(299),
1089 ..v.store.clone().unwrap()
1090 }));
1091 assert_eq!(refresh(&mut v, mins(2)).1.other.len(), 1);
1092 }
1093
1094 #[test]
1095 fn a_large_wal_is_not_counted_as_space_a_rewrite_needs() {
1096 let mut v = ValueFacts::default();
1097 v.store_measured(Some(StoreSize {
1098 checked_at: Some(mins(0)),
1099 db_bytes: 1_000,
1100 wal_bytes: 600,
1101 index_bytes: 50,
1102 total_bytes: 1_050,
1103 free_bytes: Some(500),
1104 ..StoreSize::default()
1105 }));
1106 assert!(refresh(&mut v, mins(1)).1.other.is_empty(), "needs 450");
1107 }
1108
1109 fn resident(rss: u64, limit: Option<u64>) -> ResidentSize {
1110 ResidentSize {
1111 checked_at: Some(mins(0)),
1112 rss_anon_bytes: rss,
1113 limit_bytes: limit,
1114 ..ResidentSize::default()
1115 }
1116 }
1117
1118 #[test]
1119 fn resident_size_near_the_limit_degrades_until_it_falls_past_the_band() {
1120 let mut v = ValueFacts::default();
1121 v.resident_measured(Some(resident(840, Some(1_000))));
1122 assert!(refresh(&mut v, mins(0)).1.other.is_empty());
1123 v.resident_measured(Some(resident(850, Some(1_000))));
1124 let (report, problems) = refresh(&mut v, mins(1));
1125 assert_eq!(report.resident, Some(resident(850, Some(1_000))));
1126 assert_eq!(problems.other.len(), 1);
1127 assert!(problems.other[0].contains("85% of its"), "{problems:?}");
1128 v.resident_measured(Some(resident(760, Some(1_000))));
1129 assert_eq!(refresh(&mut v, mins(2)).1.other.len(), 1, "inside band");
1130 v.resident_measured(Some(resident(740, Some(1_000))));
1131 assert!(refresh(&mut v, mins(3)).1.other.is_empty());
1132 v.resident_measured(Some(resident(5_000, None)));
1133 assert!(refresh(&mut v, mins(4)).1.other.is_empty());
1134 v.resident_measured(Some(resident(900, Some(1_000))));
1135 assert_eq!(refresh(&mut v, mins(5)).1.other.len(), 1);
1136 v.resident_measured(None);
1137 let (report, problems) = refresh(&mut v, mins(6));
1138 assert_eq!(report.resident, None);
1139 assert!(problems.other.is_empty());
1140 }
1141}