Skip to main content

second_brain_api/
value_metrics.rs

1//! Whether second-brain is worth running: how often sessions recall, how
2//! often recall finds something, what agents do with what it found, and
3//! what the store costs in disk, memory and sync lag. Kept in memory over a
4//! rolling window, so a restart starts every window empty and each check
5//! waits for enough events before it judges.
6
7use std::collections::{BTreeMap, HashMap, HashSet, VecDeque};
8use std::hash::Hash;
9
10use crate::health::{counted, human};
11use chrono::{DateTime, Duration, Utc};
12use second_brain_core::residency::ComponentResidency;
13use serde::Serialize;
14use uuid::Uuid;
15
16pub const WINDOW: Duration = Duration::hours(24);
17/// Bounds memory when a caller recalls far more often than any agent does.
18const MAX_EVENTS: usize = 50_000;
19
20const SESSION_RATE: Band = Band {
21    min_samples: 10,
22    degrade: 0.25,
23    recover: 0.35,
24};
25/// A real store's typed prompts found nothing to inject 17% of the time
26/// (`evals/results/hook_replay.json`).
27const EMPTY_OR_LOW: Band = Band {
28    min_samples: 50,
29    degrade: 0.75,
30    recover: 0.65,
31};
32const CORRECTED: Band = Band {
33    min_samples: 20,
34    degrade: 0.20,
35    recover: 0.15,
36};
37const RESIDENT_DEGRADE: f64 = 0.85;
38const RESIDENT_RECOVER: f64 = 0.75;
39/// Free space must reach this multiple of what compaction and an index
40/// rebuild rewrite (the database and indexes, never the transcript archive)
41/// before a store that outgrew its disk counts as fixed.
42const STORE_RECOVER: f64 = 1.1;
43
44#[derive(Debug, Clone, Copy, PartialEq, Eq)]
45pub enum RecallVerdict {
46    /// Something cleared the relevance bar.
47    Hit,
48    /// Results came back and none cleared the bar.
49    LowScore,
50    /// Results came back without a bar applied although a reranker exists,
51    /// or for a caller that shows only what clears it.
52    Unjudged,
53    Empty,
54}
55
56#[derive(Debug, Clone, PartialEq, Eq)]
57pub enum PeerOutcome {
58    /// Both sides now hold each other's changes.
59    Synced,
60    /// The peer took this device's changes but its own were not pulled, so
61    /// it ends a failure streak without ending the lag.
62    Pushed,
63    /// What went wrong, as the sync loop logged it.
64    Failed(String),
65}
66
67#[derive(Debug, Clone, PartialEq)]
68pub struct RecallEvent {
69    /// The caller's agent session, when it sent one.
70    pub session: Option<String>,
71    pub verdict: RecallVerdict,
72    /// The memories the caller was shown.
73    pub memory_ids: Vec<Uuid>,
74}
75
76#[derive(Debug, Clone, Default, PartialEq, Serialize)]
77pub struct RecallReport {
78    pub window_secs: i64,
79    pub recalls: u64,
80    pub hits: u64,
81    pub low_score: u64,
82    pub unjudged: u64,
83    pub empty: u64,
84    /// Over judged recalls, the pool the threshold judges.
85    pub empty_share: Option<f64>,
86    /// Over judged recalls, the pool the threshold judges.
87    pub low_score_share: Option<f64>,
88    pub unjudged_share: Option<f64>,
89    pub sessions_with_recall: u64,
90    /// None until the first read of the ingest log, or while a backfill
91    /// makes the ingest log's recent sessions mostly old ones.
92    pub sessions_ingested: Option<u64>,
93    /// Ingested sessions that recalled, over ingested sessions.
94    pub session_recall_rate: Option<f64>,
95}
96
97#[derive(Debug, Clone, Default, PartialEq, Serialize)]
98pub struct RecalledReport {
99    pub window_secs: i64,
100    pub memories: u64,
101    pub corrected: u64,
102    /// Fetched by id or cited by id in a note after it was recalled.
103    pub used: u64,
104    pub corrected_share: Option<f64>,
105    pub used_share: Option<f64>,
106}
107
108#[derive(Debug, Clone, PartialEq, Serialize)]
109pub struct PeerSync {
110    pub peer: String,
111    pub last_success_at: Option<DateTime<Utc>>,
112    pub last_attempt_at: Option<DateTime<Utc>>,
113    pub consecutive_failures: u32,
114    /// The error of the newest attempt, cleared by a sync or a push.
115    pub last_error: Option<String>,
116    /// Since the last success, or since startup when none was seen.
117    pub lag_secs: i64,
118}
119
120#[derive(Debug, Clone, Default, PartialEq, Eq, Serialize)]
121pub struct StoreSize {
122    pub checked_at: Option<DateTime<Utc>>,
123    /// The database with its WAL and shared-memory files.
124    pub db_bytes: u64,
125    /// The WAL and shared-memory part of `db_bytes`, which a compaction or
126    /// rebuild reuses rather than needs anew.
127    pub wal_bytes: u64,
128    pub index_bytes: u64,
129    pub archive_bytes: u64,
130    pub raw_index_bytes: u64,
131    pub cold_bytes: u64,
132    pub total_bytes: u64,
133    /// Space left on the store's filesystem; None when it can't be read.
134    pub free_bytes: Option<u64>,
135}
136
137#[derive(Debug, Clone, Default, PartialEq, Eq, Serialize)]
138pub struct ResidentSize {
139    pub checked_at: Option<DateTime<Utc>>,
140    pub rss_anon_bytes: u64,
141    pub components: ComponentResidency,
142    pub raw_index_bytes: u64,
143    pub unattributed_bytes: i64,
144    /// The cgroup's `memory.max`, else the machine's memory; None when
145    /// neither can be read.
146    pub limit_bytes: Option<u64>,
147}
148
149/// Degrades once a share crosses `degrade` and recovers only once it is
150/// back past `recover`, so a share hovering at the line alerts once. Below
151/// `min_samples` it never degrades, but a degraded share holds until the
152/// window empties, so samples aging out do not flip it.
153struct Band {
154    min_samples: u64,
155    degrade: f64,
156    recover: f64,
157}
158
159/// Which checks are degraded right now; each flips only at its band edge.
160#[derive(Debug, Default)]
161struct Latches {
162    session_rate: bool,
163    empty_or_low: bool,
164    corrected: bool,
165    store: bool,
166    resident: bool,
167}
168
169#[derive(Debug, Clone, Copy, Default)]
170struct Flags {
171    corrected: bool,
172    used: bool,
173}
174
175/// Keys seen within the window with the time each was last seen, oldest
176/// first, so pruning is a pop from the front.
177#[derive(Debug)]
178struct Recent<K, V> {
179    entries: HashMap<K, (DateTime<Utc>, V)>,
180    order: VecDeque<(DateTime<Utc>, K)>,
181}
182
183impl<K, V> Default for Recent<K, V> {
184    fn default() -> Self {
185        Self {
186            entries: HashMap::new(),
187            order: VecDeque::new(),
188        }
189    }
190}
191
192impl Band {
193    /// For a share where more is worse.
194    fn above(&self, latched: bool, count: u64, of: u64) -> bool {
195        if of == 0 || (!latched && of < self.min_samples) {
196            return false;
197        }
198        let share = count as f64 / of as f64;
199        share > if latched { self.recover } else { self.degrade }
200    }
201
202    /// For a share where less is worse.
203    fn below(&self, latched: bool, count: u64, of: u64) -> bool {
204        if of == 0 || (!latched && of < self.min_samples) {
205            return false;
206        }
207        let share = count as f64 / of as f64;
208        share < if latched { self.recover } else { self.degrade }
209    }
210}
211
212impl<K: Hash + Eq + Clone, V: Default> Recent<K, V> {
213    fn touch(&mut self, key: K, at: DateTime<Utc>) {
214        match self.entries.get_mut(&key) {
215            Some(entry) => entry.0 = at,
216            None => {
217                self.entries.insert(key.clone(), (at, V::default()));
218            }
219        }
220        self.order.push_back((at, key));
221        if self.order.len() > 4 * MAX_EVENTS {
222            let entries = &self.entries;
223            self.order
224                .retain(|(at, key)| entries.get(key).is_some_and(|e| e.0 == *at));
225        }
226    }
227
228    fn get_mut(&mut self, key: &K) -> Option<&mut V> {
229        self.entries.get_mut(key).map(|e| &mut e.1)
230    }
231
232    /// Calls `evicted` for each entry dropped.
233    fn prune(&mut self, cutoff: DateTime<Utc>, mut evicted: impl FnMut(&V)) {
234        while let Some((at, key)) = self.order.front() {
235            let live = self.entries.get(key).is_some_and(|e| e.0 == *at);
236            if live && *at >= cutoff && self.entries.len() <= MAX_EVENTS {
237                break;
238            }
239            if let Some((_, key)) = self.order.pop_front()
240                && live
241                && let Some((_, value)) = self.entries.remove(&key)
242            {
243                evicted(&value);
244            }
245        }
246    }
247
248    fn len(&self) -> usize {
249        self.entries.len()
250    }
251
252    fn count_since(&self, cutoff: DateTime<Utc>) -> usize {
253        self.entries
254            .values()
255            .filter(|(at, _)| *at >= cutoff)
256            .count()
257    }
258
259    fn contains(&self, key: &K) -> bool {
260        self.entries.contains_key(key)
261    }
262}
263
264#[derive(Debug, Default)]
265pub struct ValueFacts {
266    recalls: VecDeque<(DateTime<Utc>, RecallVerdict)>,
267    hits: u64,
268    low_score: u64,
269    unjudged: u64,
270    empty: u64,
271    recall_sessions: Recent<String, ()>,
272    recalled: Recent<Uuid, Flags>,
273    corrected: u64,
274    used: u64,
275    ingested_sessions: Option<HashSet<String>>,
276    /// The session check's problem while it is degraded, kept through
277    /// passes where the ingested sessions are unknown.
278    session_problem: Option<String>,
279    peers: BTreeMap<String, PeerState>,
280    store: Option<StoreSize>,
281    resident: Option<ResidentSize>,
282    latches: Latches,
283}
284
285#[derive(Debug, Clone, Default)]
286struct PeerState {
287    listed_at: Option<DateTime<Utc>>,
288    last_success_at: Option<DateTime<Utc>>,
289    last_attempt_at: Option<DateTime<Utc>>,
290    consecutive_failures: u32,
291    last_error: Option<String>,
292}
293
294/// What `refresh` concluded: problems that degrade the daemon, with the
295/// per-peer ones apart so a check that already says no sync succeeded at
296/// all can leave them out.
297#[derive(Debug, Default, PartialEq)]
298pub struct ValueProblems {
299    pub peers: Vec<String>,
300    pub other: Vec<String>,
301}
302
303#[derive(Debug, Clone, Default, PartialEq, Serialize)]
304pub struct ValueReport {
305    pub recall: RecallReport,
306    pub recalled: RecalledReport,
307    pub sync_peers: Vec<PeerSync>,
308    pub store: Option<StoreSize>,
309    pub resident: Option<ResidentSize>,
310}
311
312/// The session a transcript or caller belongs to: a subagent's
313/// `<session>/<agent>` key counts as its parent session.
314pub fn session_of(key: &str) -> &str {
315    key.split_once('/').map_or(key, |(session, _)| session)
316}
317
318impl ValueFacts {
319    pub fn recalled(&mut self, event: RecallEvent, now: DateTime<Utc>) {
320        self.recalls.push_back((now, event.verdict));
321        *self.count_of(event.verdict) += 1;
322        if let Some(session) = &event.session {
323            self.recall_sessions
324                .touch(session_of(session).to_string(), now);
325        }
326        for id in event.memory_ids {
327            self.recalled.touch(id, now);
328        }
329    }
330
331    fn count_of(&mut self, verdict: RecallVerdict) -> &mut u64 {
332        match verdict {
333            RecallVerdict::Hit => &mut self.hits,
334            RecallVerdict::LowScore => &mut self.low_score,
335            RecallVerdict::Unjudged => &mut self.unjudged,
336            RecallVerdict::Empty => &mut self.empty,
337        }
338    }
339
340    /// Counts only memories recalled within the window, and each once.
341    pub fn corrected(&mut self, id: Uuid) {
342        if let Some(flags) = self.recalled.get_mut(&id)
343            && !flags.corrected
344        {
345            flags.corrected = true;
346            self.corrected += 1;
347        }
348    }
349
350    pub fn used(&mut self, ids: &[Uuid]) {
351        for id in ids {
352            if let Some(flags) = self.recalled.get_mut(id)
353                && !flags.used
354            {
355                flags.used = true;
356                self.used += 1;
357            }
358        }
359    }
360
361    /// The sessions the ingest log shows captured within the window; None
362    /// while that count would mislead.
363    pub fn sessions_ingested(&mut self, keys: Option<Vec<String>>) {
364        self.ingested_sessions =
365            keys.map(|keys| keys.iter().map(|key| session_of(key).to_string()).collect());
366    }
367
368    /// Forgets peers no longer listed and starts tracking new ones; results
369    /// count only for listed peers, so a sync that ends after its peer was
370    /// removed does not bring it back.
371    pub fn peers_listed(&mut self, peers: &[String], now: DateTime<Utc>) {
372        self.peers.retain(|peer, _| peers.contains(peer));
373        for peer in peers {
374            self.peers.entry(peer.clone()).or_insert_with(|| PeerState {
375                listed_at: Some(now),
376                ..PeerState::default()
377            });
378        }
379    }
380
381    pub fn sync_attempted(&mut self, peer: &str, outcome: PeerOutcome, now: DateTime<Utc>) {
382        let Some(state) = self.peers.get_mut(peer) else {
383            return;
384        };
385        match outcome {
386            PeerOutcome::Synced => {
387                state.last_attempt_at = Some(now);
388                state.last_success_at = Some(now);
389                state.consecutive_failures = 0;
390                state.last_error = None;
391            }
392            PeerOutcome::Pushed => {
393                state.last_attempt_at = Some(now);
394                state.consecutive_failures = 0;
395                state.last_error = None;
396            }
397            PeerOutcome::Failed(error) => {
398                state.last_attempt_at = Some(now);
399                state.consecutive_failures += 1;
400                state.last_error = Some(error);
401            }
402        }
403    }
404
405    /// None when the measurement failed, which drops the old reading.
406    pub fn store_measured(&mut self, size: Option<StoreSize>) {
407        self.store = size;
408    }
409
410    /// None when the measurement failed, which drops the old reading.
411    pub fn resident_measured(&mut self, size: Option<ResidentSize>) {
412        self.resident = size;
413    }
414
415    /// Prunes the windows, moves each check across its band and reports.
416    /// `started` bounds a peer's lag before its first success; `sync_overdue`
417    /// is the alert config's window, None when sync checks are off.
418    pub fn refresh(
419        &mut self,
420        now: DateTime<Utc>,
421        started: DateTime<Utc>,
422        sync_overdue: Option<Duration>,
423    ) -> (ValueReport, ValueProblems) {
424        self.prune(now - WINDOW);
425        let mut problems = ValueProblems::default();
426        let recall = self.recall_report(now - WINDOW, &mut problems);
427        let recalled = self.recalled_report(&mut problems);
428        let sync_peers = self.peer_report(now, started, sync_overdue, &mut problems);
429        self.check_store(&mut problems);
430        self.check_resident(&mut problems);
431        let report = ValueReport {
432            recall,
433            recalled,
434            sync_peers,
435            store: self.store.clone(),
436            resident: self.resident.clone(),
437        };
438        (report, problems)
439    }
440
441    fn prune(&mut self, cutoff: DateTime<Utc>) {
442        while let Some(&(at, verdict)) = self.recalls.front() {
443            if at >= cutoff && self.recalls.len() <= MAX_EVENTS {
444                break;
445            }
446            self.recalls.pop_front();
447            *self.count_of(verdict) -= 1;
448        }
449        self.recall_sessions.prune(cutoff - session_grace(), |_| {});
450        let (corrected, used) = (&mut self.corrected, &mut self.used);
451        self.recalled.prune(cutoff, |flags| {
452            *corrected -= u64::from(flags.corrected);
453            *used -= u64::from(flags.used);
454        });
455    }
456
457    fn recall_report(
458        &mut self,
459        cutoff: DateTime<Utc>,
460        problems: &mut ValueProblems,
461    ) -> RecallReport {
462        let recalls = self.recalls.len() as u64;
463        let ingested = self.ingested_sessions.as_ref();
464        let joined = ingested.map(|keys| {
465            keys.iter()
466                .filter(|key| self.recall_sessions.contains(*key))
467                .count() as u64
468        });
469        let sessions_ingested = ingested.map(|keys| keys.len() as u64);
470
471        let judged = recalls - self.unjudged;
472        let bad = self.empty + self.low_score;
473        self.latches.empty_or_low = EMPTY_OR_LOW.above(self.latches.empty_or_low, bad, judged);
474        if self.latches.empty_or_low {
475            problems.other.push(format!(
476                "{} of {judged} judged recalls in the last 24h found nothing that cleared the \
477                 relevance bar ({} empty, {} below it)",
478                percent(bad, judged),
479                self.empty,
480                self.low_score
481            ));
482        }
483        if let (Some(joined), Some(of)) = (joined, sessions_ingested) {
484            self.latches.session_rate = SESSION_RATE.below(self.latches.session_rate, joined, of);
485            self.session_problem = self.latches.session_rate.then(|| {
486                format!(
487                    "only {joined} of {of} sessions in the last 24h recalled from second-brain; \
488                     check its hooks and MCP server with `sb doctor`"
489                )
490            });
491        }
492        if let Some(problem) = &self.session_problem {
493            problems.other.push(problem.clone());
494        }
495
496        RecallReport {
497            window_secs: WINDOW.num_seconds(),
498            recalls,
499            hits: self.hits,
500            low_score: self.low_score,
501            unjudged: self.unjudged,
502            empty: self.empty,
503            empty_share: share(self.empty, judged),
504            low_score_share: share(self.low_score, judged),
505            unjudged_share: share(self.unjudged, recalls),
506            sessions_with_recall: self.recall_sessions.count_since(cutoff) as u64,
507            sessions_ingested,
508            session_recall_rate: joined
509                .zip(sessions_ingested)
510                .and_then(|(j, of)| share(j, of)),
511        }
512    }
513
514    fn recalled_report(&mut self, problems: &mut ValueProblems) -> RecalledReport {
515        let memories = self.recalled.len() as u64;
516        self.latches.corrected = CORRECTED.above(self.latches.corrected, self.corrected, memories);
517        if self.latches.corrected {
518            problems.other.push(format!(
519                "{} of {memories} memories recalled in the last 24h were corrected",
520                self.corrected
521            ));
522        }
523        RecalledReport {
524            window_secs: WINDOW.num_seconds(),
525            memories,
526            corrected: self.corrected,
527            used: self.used,
528            corrected_share: share(self.corrected, memories),
529            used_share: share(self.used, memories),
530        }
531    }
532
533    fn peer_report(
534        &self,
535        now: DateTime<Utc>,
536        started: DateTime<Utc>,
537        sync_overdue: Option<Duration>,
538        problems: &mut ValueProblems,
539    ) -> Vec<PeerSync> {
540        self.peers
541            .iter()
542            .map(|(peer, state)| {
543                let since = state.last_success_at.or(state.listed_at).unwrap_or(started);
544                let lag = (now - since).max(Duration::zero());
545                if sync_overdue.is_some_and(|window| lag >= window) {
546                    let failures = match (state.consecutive_failures, &state.last_error) {
547                        (0, _) => String::new(),
548                        (n, error) => format!(
549                            " ({} in a row{})",
550                            counted(u64::from(n), "failure", "failures"),
551                            error
552                                .as_deref()
553                                .and_then(|e| e.lines().next())
554                                .map(|line| format!("; the last: {line}"))
555                                .unwrap_or_default()
556                        ),
557                    };
558                    problems.peers.push(format!(
559                        "no sync with {peer} has succeeded in {}{failures}",
560                        human(lag)
561                    ));
562                }
563                PeerSync {
564                    peer: peer.clone(),
565                    last_success_at: state.last_success_at,
566                    last_attempt_at: state.last_attempt_at,
567                    consecutive_failures: state.consecutive_failures,
568                    last_error: state.last_error.clone(),
569                    lag_secs: lag.num_seconds(),
570                }
571            })
572            .collect()
573    }
574
575    fn check_store(&mut self, problems: &mut ValueProblems) {
576        let Some((rewritten, free)) = self.store.as_ref().and_then(|s| {
577            let rewritten =
578                s.db_bytes.saturating_sub(s.wal_bytes) + s.index_bytes + s.raw_index_bytes;
579            s.free_bytes.map(|free| (rewritten, free))
580        }) else {
581            self.latches.store = false;
582            return;
583        };
584        let needed = if self.latches.store {
585            rewritten as f64 * STORE_RECOVER
586        } else {
587            rewritten as f64
588        };
589        self.latches.store = (free as f64) < needed;
590        if self.latches.store {
591            problems.other.push(format!(
592                "compacting or rebuilding the store needs {} MB and its disk has {} MB free",
593                rewritten / 1_000_000,
594                free / 1_000_000
595            ));
596        }
597    }
598
599    fn check_resident(&mut self, problems: &mut ValueProblems) {
600        let Some((rss, limit)) = self.resident.as_ref().and_then(|r| {
601            r.limit_bytes
602                .filter(|&l| l > 0)
603                .map(|l| (r.rss_anon_bytes, l))
604        }) else {
605            self.latches.resident = false;
606            return;
607        };
608        let ratio = rss as f64 / limit as f64;
609        let line = if self.latches.resident {
610            RESIDENT_RECOVER
611        } else {
612            RESIDENT_DEGRADE
613        };
614        self.latches.resident = ratio >= line;
615        if self.latches.resident {
616            problems.other.push(format!(
617                "the daemon holds {} MB resident, {} of its {} MB limit",
618                rss / 1_000_000,
619                percent(rss, limit),
620                limit / 1_000_000
621            ));
622        }
623    }
624}
625
626/// The ingested sessions are read every `progress::INTERVAL`, so a recall
627/// stays joined that long past the window to meet a session the last read
628/// still lists.
629fn session_grace() -> Duration {
630    Duration::from_std(crate::progress::INTERVAL).unwrap_or(Duration::zero())
631}
632
633fn share(count: u64, of: u64) -> Option<f64> {
634    (of > 0).then(|| count as f64 / of as f64)
635}
636
637fn percent(count: u64, of: u64) -> String {
638    format!("{:.0}%", count as f64 * 100.0 / of as f64)
639}
640
641#[cfg(test)]
642mod tests {
643    use super::*;
644
645    fn t0() -> DateTime<Utc> {
646        DateTime::parse_from_rfc3339("2026-10-31T12:00:00Z")
647            .unwrap()
648            .with_timezone(&Utc)
649    }
650
651    fn mins(m: i64) -> DateTime<Utc> {
652        t0() + Duration::minutes(m)
653    }
654
655    fn id(n: u128) -> Uuid {
656        Uuid::from_u128(n)
657    }
658
659    fn recall(session: &str, verdict: RecallVerdict, ids: &[u128]) -> RecallEvent {
660        RecallEvent {
661            session: Some(session.to_string()),
662            verdict,
663            memory_ids: ids.iter().map(|&n| id(n)).collect(),
664        }
665    }
666
667    fn refresh(v: &mut ValueFacts, now: DateTime<Utc>) -> (ValueReport, ValueProblems) {
668        v.refresh(now, t0(), Some(Duration::hours(24)))
669    }
670
671    #[test]
672    fn subagent_keys_count_as_their_parent_session() {
673        assert_eq!(session_of("abc"), "abc");
674        assert_eq!(session_of("abc/agent-1"), "abc");
675    }
676
677    #[test]
678    fn recall_shares_count_empty_and_low_score_recalls() {
679        let mut v = ValueFacts::default();
680        v.recalled(recall("s1", RecallVerdict::Hit, &[1]), mins(0));
681        v.recalled(recall("s1", RecallVerdict::Empty, &[]), mins(1));
682        v.recalled(recall("s2", RecallVerdict::LowScore, &[2]), mins(2));
683        v.recalled(recall("s2", RecallVerdict::Empty, &[]), mins(3));
684        v.recalled(recall("s2", RecallVerdict::Unjudged, &[]), mins(3));
685        let (report, problems) = refresh(&mut v, mins(4));
686        let r = report.recall;
687        assert_eq!(r.window_secs, WINDOW.num_seconds());
688        assert_eq!((r.recalls, r.hits, r.low_score, r.empty), (5, 1, 1, 2));
689        assert_eq!(r.empty_share, Some(0.5), "over judged recalls");
690        assert_eq!(r.low_score_share, Some(0.25), "over judged recalls");
691        assert_eq!(r.unjudged_share, Some(0.2), "over all recalls");
692        assert_eq!(r.sessions_with_recall, 2);
693        assert_eq!(problems, ValueProblems::default());
694    }
695
696    #[test]
697    fn mostly_empty_recalls_degrade_once_and_recover_past_the_band() {
698        let mut v = ValueFacts::default();
699        for m in 0..49 {
700            v.recalled(recall("s", RecallVerdict::Empty, &[]), mins(m));
701        }
702        assert!(refresh(&mut v, mins(49)).1.other.is_empty(), "too few");
703        v.recalled(recall("s", RecallVerdict::LowScore, &[1]), mins(49));
704        let (_, problems) = refresh(&mut v, mins(50));
705        assert_eq!(problems.other.len(), 1, "{problems:?}");
706        assert!(
707            problems.other[0].contains("of 50 judged recalls in the last 24h"),
708            "{problems:?}"
709        );
710        for m in 50..71 {
711            v.recalled(recall("s", RecallVerdict::Hit, &[1]), mins(m));
712        }
713        let (report, problems) = refresh(&mut v, mins(71));
714        assert_eq!(
715            report.recall.recalls, 71,
716            "50 of 71 is 70%, inside the band"
717        );
718        assert_eq!(problems.other.len(), 1);
719        for m in 71..77 {
720            v.recalled(recall("s", RecallVerdict::Hit, &[1]), mins(m));
721        }
722        assert!(
723            refresh(&mut v, mins(77)).1.other.is_empty(),
724            "50 of 77 is 65%"
725        );
726    }
727
728    #[test]
729    fn a_latched_share_holds_below_the_minimum_until_the_window_empties() {
730        let mut v = ValueFacts::default();
731        for m in 0..50 {
732            v.recalled(recall("s", RecallVerdict::Empty, &[]), mins(m));
733        }
734        assert_eq!(refresh(&mut v, mins(50)).1.other.len(), 1);
735        let first_gone = mins(0) + WINDOW + Duration::seconds(30);
736        let (report, problems) = refresh(&mut v, first_gone);
737        assert_eq!(report.recall.recalls, 49);
738        assert_eq!(problems.other.len(), 1, "below the minimum, still latched");
739        let all_gone = mins(50) + WINDOW;
740        assert!(refresh(&mut v, all_gone).1.other.is_empty());
741    }
742
743    #[test]
744    fn unjudged_recalls_are_left_out_of_the_empty_or_low_share() {
745        let mut v = ValueFacts::default();
746        for m in 0..50 {
747            v.recalled(recall("s", RecallVerdict::Unjudged, &[]), mins(m));
748        }
749        let (report, problems) = refresh(&mut v, mins(50));
750        assert_eq!(report.recall.unjudged, 50);
751        assert_eq!(report.recall.unjudged_share, Some(1.0));
752        assert!(problems.other.is_empty(), "{problems:?}");
753        for m in 50..99 {
754            v.recalled(recall("s", RecallVerdict::Empty, &[]), mins(m));
755        }
756        assert!(refresh(&mut v, mins(99)).1.other.is_empty(), "49 judged");
757        v.recalled(recall("s", RecallVerdict::LowScore, &[]), mins(99));
758        let (_, problems) = refresh(&mut v, mins(100));
759        assert_eq!(problems.other.len(), 1);
760        assert!(
761            problems.other[0].contains("100% of 50 judged recalls in the last 24h"),
762            "{problems:?}"
763        );
764        assert!(
765            problems.other[0].contains("(49 empty, 1 below it)"),
766            "{problems:?}"
767        );
768    }
769
770    #[test]
771    fn a_session_recalled_just_before_the_window_edge_counts_until_the_next_ingest_read() {
772        let mut v = ValueFacts::default();
773        v.recalled(recall("a", RecallVerdict::Hit, &[]), mins(0));
774        v.sessions_ingested(Some(vec!["a".to_string()]));
775        let edge = mins(0) + WINDOW + Duration::minutes(4);
776        let recall = refresh(&mut v, edge).0.recall;
777        assert_eq!(recall.session_recall_rate, Some(1.0));
778        assert_eq!(
779            recall.sessions_with_recall, 0,
780            "outside the reported window"
781        );
782        let later = mins(0) + WINDOW + Duration::minutes(6);
783        assert_eq!(
784            refresh(&mut v, later).0.recall.session_recall_rate,
785            Some(0.0)
786        );
787    }
788
789    #[test]
790    fn recalls_older_than_the_window_drop_out() {
791        let mut v = ValueFacts::default();
792        v.recalled(recall("s1", RecallVerdict::Empty, &[1]), mins(0));
793        v.recalled(recall("s2", RecallVerdict::Hit, &[2]), mins(60));
794        let (report, _) = refresh(&mut v, mins(0) + WINDOW + Duration::minutes(10));
795        assert_eq!(report.recall.recalls, 1);
796        assert_eq!(report.recall.empty, 0);
797        assert_eq!(report.recall.sessions_with_recall, 1);
798        assert_eq!(report.recalled.memories, 1);
799    }
800
801    #[test]
802    fn the_event_window_is_bounded() {
803        let mut v = ValueFacts::default();
804        for n in 0..(MAX_EVENTS as u128 + 10) {
805            v.recalled(recall(&format!("s{n}"), RecallVerdict::Hit, &[n]), mins(0));
806        }
807        let (report, _) = refresh(&mut v, mins(1));
808        assert_eq!(report.recall.recalls, MAX_EVENTS as u64);
809        assert_eq!(report.recall.sessions_with_recall, MAX_EVENTS as u64);
810        assert_eq!(report.recalled.memories, MAX_EVENTS as u64);
811    }
812
813    #[test]
814    fn session_recall_rate_joins_recalling_sessions_with_ingested_ones() {
815        let mut v = ValueFacts::default();
816        let (report, _) = refresh(&mut v, mins(0));
817        assert_eq!(report.recall.sessions_ingested, None);
818        assert_eq!(report.recall.session_recall_rate, None);
819
820        v.recalled(recall("a", RecallVerdict::Hit, &[]), mins(0));
821        v.recalled(recall("b/agent-1", RecallVerdict::Hit, &[]), mins(0));
822        v.recalled(recall("not-ingested", RecallVerdict::Hit, &[]), mins(0));
823        let keys = ["a", "a/agent-2", "b", "c", "d"].map(String::from).to_vec();
824        v.sessions_ingested(Some(keys));
825        let (report, problems) = refresh(&mut v, mins(1));
826        assert_eq!(report.recall.sessions_ingested, Some(4));
827        assert_eq!(report.recall.session_recall_rate, Some(0.5));
828        assert!(problems.other.is_empty(), "under the minimum sample");
829    }
830
831    #[test]
832    fn sessions_that_never_recall_degrade_once_until_they_recover_past_the_band() {
833        let mut v = ValueFacts::default();
834        let keys: Vec<String> = (0..10).map(|n| format!("s{n}")).collect();
835        v.recalled(recall("s0", RecallVerdict::Hit, &[]), mins(0));
836        v.recalled(recall("s1", RecallVerdict::Hit, &[]), mins(0));
837        v.sessions_ingested(Some(keys.clone()));
838        let (_, problems) = refresh(&mut v, mins(1));
839        assert_eq!(problems.other.len(), 1);
840        assert!(
841            problems.other[0].contains("2 of 10 sessions in the last 24h recalled"),
842            "{problems:?}"
843        );
844        v.recalled(recall("s2", RecallVerdict::Hit, &[]), mins(2));
845        assert_eq!(refresh(&mut v, mins(3)).1.other.len(), 1, "inside band");
846        v.recalled(recall("s3", RecallVerdict::Hit, &[]), mins(4));
847        assert!(refresh(&mut v, mins(5)).1.other.is_empty());
848
849        v.sessions_ingested(None);
850        let (report, _) = refresh(&mut v, mins(8));
851        assert_eq!(report.recall.session_recall_rate, None);
852    }
853
854    #[test]
855    fn an_unknown_ingest_count_keeps_the_session_alert_until_a_count_clears_it() {
856        let mut v = ValueFacts::default();
857        let keys: Vec<String> = (0..10).map(|n| format!("s{n}")).collect();
858        v.recalled(recall("s0", RecallVerdict::Hit, &[]), mins(0));
859        v.sessions_ingested(Some(keys.clone()));
860        assert_eq!(refresh(&mut v, mins(1)).1.other.len(), 1);
861        v.sessions_ingested(None);
862        let (report, problems) = refresh(&mut v, mins(2));
863        assert_eq!(report.recall.session_recall_rate, None);
864        assert_eq!(problems.other.len(), 1, "{problems:?}");
865        for n in 1..4 {
866            v.recalled(recall(&format!("s{n}"), RecallVerdict::Hit, &[]), mins(3));
867        }
868        v.sessions_ingested(Some(keys));
869        assert!(refresh(&mut v, mins(4)).1.other.is_empty());
870    }
871
872    #[test]
873    fn corrections_and_uses_count_only_recalled_memories_once_each() {
874        let mut v = ValueFacts::default();
875        v.recalled(recall("s", RecallVerdict::Hit, &[1, 2, 3, 4]), mins(0));
876        v.corrected(id(1));
877        v.corrected(id(1));
878        v.corrected(id(99));
879        v.used(&[id(2), id(3), id(2), id(98)]);
880        let (report, _) = refresh(&mut v, mins(1));
881        let r = report.recalled;
882        assert_eq!((r.memories, r.corrected, r.used), (4, 1, 2));
883        assert_eq!(r.corrected_share, Some(0.25));
884        assert_eq!(r.used_share, Some(0.5));
885
886        let later = mins(0) + WINDOW + Duration::minutes(1);
887        let (report, _) = refresh(&mut v, later);
888        assert_eq!(
889            report.recalled,
890            RecalledReport {
891                window_secs: WINDOW.num_seconds(),
892                ..RecalledReport::default()
893            }
894        );
895    }
896
897    #[test]
898    fn a_memory_corrected_before_it_was_recalled_is_not_counted() {
899        let mut v = ValueFacts::default();
900        v.corrected(id(1));
901        v.recalled(recall("s", RecallVerdict::Hit, &[1]), mins(0));
902        let (report, _) = refresh(&mut v, mins(1));
903        assert_eq!(report.recalled.corrected, 0);
904    }
905
906    #[test]
907    fn many_corrected_recalls_degrade_once_and_recover_past_the_band() {
908        let mut v = ValueFacts::default();
909        let ids: Vec<u128> = (0..20).collect();
910        v.recalled(recall("s", RecallVerdict::Hit, &ids), mins(0));
911        for n in 0..4 {
912            v.corrected(id(n));
913        }
914        assert!(
915            refresh(&mut v, mins(1)).1.other.is_empty(),
916            "20% is the line"
917        );
918        v.corrected(id(4));
919        let (_, problems) = refresh(&mut v, mins(2));
920        assert_eq!(problems.other.len(), 1);
921        assert!(
922            problems.other[0].contains("5 of 20 memories recalled in the last 24h were corrected"),
923            "{problems:?}"
924        );
925        let more: Vec<u128> = (100..110).collect();
926        v.recalled(recall("s", RecallVerdict::Hit, &more), mins(3));
927        assert_eq!(refresh(&mut v, mins(4)).1.other.len(), 1, "inside band");
928        let more: Vec<u128> = (110..120).collect();
929        v.recalled(recall("s", RecallVerdict::Hit, &more), mins(5));
930        assert!(refresh(&mut v, mins(6)).1.other.is_empty());
931    }
932
933    #[test]
934    fn a_peer_lagging_past_the_window_degrades_until_it_syncs() {
935        let mut v = ValueFacts::default();
936        v.peers_listed(&["u@pi".to_string(), "u@mac".to_string()], mins(0));
937        v.sync_attempted("u@pi", PeerOutcome::Synced, mins(0));
938        v.sync_attempted("u@mac", PeerOutcome::Failed("refused".into()), mins(0));
939        v.sync_attempted(
940            "u@mac",
941            PeerOutcome::Failed("u@mac refused the sync: pair it with `sb pair`".into()),
942            mins(5),
943        );
944        let (report, problems) = refresh(&mut v, mins(10));
945        assert_eq!(report.sync_peers.len(), 2);
946        let mac = &report.sync_peers[0];
947        assert_eq!(mac.peer, "u@mac");
948        assert_eq!(mac.last_success_at, None);
949        assert_eq!(mac.last_attempt_at, Some(mins(5)));
950        assert_eq!(mac.consecutive_failures, 2);
951        assert_eq!(mac.lag_secs, 600, "from startup");
952        assert_eq!(
953            mac.last_error.as_deref(),
954            Some("u@mac refused the sync: pair it with `sb pair`")
955        );
956        assert_eq!(report.sync_peers[1].lag_secs, 600);
957        assert_eq!(report.sync_peers[1].last_error, None);
958        assert!(problems.peers.is_empty());
959
960        let late = t0() + Duration::hours(24);
961        v.sync_attempted("u@pi", PeerOutcome::Synced, late);
962        let (_, problems) = refresh(&mut v, late);
963        assert_eq!(problems.peers.len(), 1);
964        assert!(
965            problems.peers[0].contains("no sync with u@mac has succeeded in 24h 0m"),
966            "{problems:?}"
967        );
968        assert!(
969            problems.peers[0].contains("2 failures in a row"),
970            "{problems:?}"
971        );
972        assert!(
973            problems.peers[0].contains("the last: u@mac refused the sync: pair it with `sb pair`"),
974            "{problems:?}"
975        );
976
977        v.sync_attempted("u@mac", PeerOutcome::Synced, late);
978        let (report, problems) = refresh(&mut v, late);
979        assert!(problems.peers.is_empty());
980        assert_eq!(report.sync_peers[0].last_error, None);
981
982        v.peers_listed(&["u@pi".to_string()], late);
983        assert_eq!(refresh(&mut v, late).0.sync_peers.len(), 1);
984        assert!(
985            v.refresh(late + Duration::hours(30), t0(), None)
986                .1
987                .peers
988                .is_empty(),
989            "sync checks off"
990        );
991    }
992
993    #[test]
994    fn a_push_ends_a_failure_streak_but_not_the_lag() {
995        let mut v = ValueFacts::default();
996        v.peers_listed(&["u@pi".to_string()], mins(0));
997        v.sync_attempted("u@pi", PeerOutcome::Failed("timed out".into()), mins(1));
998        v.sync_attempted("u@pi", PeerOutcome::Pushed, mins(2));
999        let (report, _) = refresh(&mut v, mins(3));
1000        let pi = &report.sync_peers[0];
1001        assert_eq!(pi.consecutive_failures, 0);
1002        assert_eq!(pi.last_attempt_at, Some(mins(2)));
1003        assert_eq!(pi.last_success_at, None);
1004        assert_eq!(pi.last_error, None);
1005        assert_eq!(pi.lag_secs, 180);
1006    }
1007
1008    #[test]
1009    fn a_peer_listed_late_lags_from_when_it_was_listed() {
1010        let mut v = ValueFacts::default();
1011        let late = t0() + Duration::hours(30);
1012        v.peers_listed(&["u@mac".to_string()], late);
1013        let (report, problems) = refresh(&mut v, late + Duration::minutes(5));
1014        assert_eq!(report.sync_peers[0].lag_secs, 300);
1015        assert!(problems.peers.is_empty());
1016        v.peers_listed(&["u@mac".to_string()], late + Duration::hours(1));
1017        let (report, _) = refresh(&mut v, late + Duration::hours(2));
1018        assert_eq!(
1019            report.sync_peers[0].lag_secs, 7_200,
1020            "listing again keeps the start"
1021        );
1022    }
1023
1024    #[test]
1025    fn a_result_for_a_peer_no_longer_listed_is_ignored() {
1026        let mut v = ValueFacts::default();
1027        v.peers_listed(&["u@pi".to_string()], mins(0));
1028        v.peers_listed(&[], mins(0));
1029        v.sync_attempted("u@pi", PeerOutcome::Failed("timed out".into()), mins(1));
1030        assert!(refresh(&mut v, mins(2)).0.sync_peers.is_empty());
1031    }
1032
1033    fn store(total: u64, free: Option<u64>) -> StoreSize {
1034        StoreSize {
1035            checked_at: Some(mins(0)),
1036            db_bytes: total,
1037            total_bytes: total,
1038            free_bytes: free,
1039            ..StoreSize::default()
1040        }
1041    }
1042
1043    #[test]
1044    fn a_store_larger_than_its_free_space_degrades_until_space_clears_the_band() {
1045        let mut v = ValueFacts::default();
1046        let (report, _) = refresh(&mut v, mins(0));
1047        assert_eq!(report.store, None);
1048        v.store_measured(Some(store(1_000, Some(1_000))));
1049        assert!(refresh(&mut v, mins(0)).1.other.is_empty());
1050        v.store_measured(Some(store(1_000, Some(999))));
1051        let (report, problems) = refresh(&mut v, mins(1));
1052        assert_eq!(report.store, Some(store(1_000, Some(999))));
1053        assert_eq!(problems.other.len(), 1);
1054        assert!(
1055            problems.other[0].contains("compacting or rebuilding the store needs 0 MB"),
1056            "{problems:?}"
1057        );
1058        v.store_measured(Some(store(1_000, Some(1_050))));
1059        assert_eq!(refresh(&mut v, mins(2)).1.other.len(), 1, "inside band");
1060        v.store_measured(Some(store(1_000, Some(1_100))));
1061        assert!(refresh(&mut v, mins(3)).1.other.is_empty());
1062        v.store_measured(Some(store(1_000, Some(999))));
1063        assert_eq!(refresh(&mut v, mins(4)).1.other.len(), 1);
1064        v.store_measured(None);
1065        let (report, problems) = refresh(&mut v, mins(5));
1066        assert_eq!(report.store, None);
1067        assert!(problems.other.is_empty());
1068        v.store_measured(Some(store(1_000, None)));
1069        assert!(refresh(&mut v, mins(6)).1.other.is_empty());
1070    }
1071
1072    #[test]
1073    fn the_store_check_leaves_out_the_archive_that_compaction_never_rewrites() {
1074        let mut v = ValueFacts::default();
1075        v.store_measured(Some(StoreSize {
1076            checked_at: Some(mins(0)),
1077            db_bytes: 100,
1078            wal_bytes: 0,
1079            index_bytes: 100,
1080            archive_bytes: 10_000,
1081            raw_index_bytes: 100,
1082            cold_bytes: 0,
1083            total_bytes: 10_300,
1084            free_bytes: Some(301),
1085        }));
1086        assert!(refresh(&mut v, mins(1)).1.other.is_empty());
1087        v.store_measured(Some(StoreSize {
1088            free_bytes: Some(299),
1089            ..v.store.clone().unwrap()
1090        }));
1091        assert_eq!(refresh(&mut v, mins(2)).1.other.len(), 1);
1092    }
1093
1094    #[test]
1095    fn a_large_wal_is_not_counted_as_space_a_rewrite_needs() {
1096        let mut v = ValueFacts::default();
1097        v.store_measured(Some(StoreSize {
1098            checked_at: Some(mins(0)),
1099            db_bytes: 1_000,
1100            wal_bytes: 600,
1101            index_bytes: 50,
1102            total_bytes: 1_050,
1103            free_bytes: Some(500),
1104            ..StoreSize::default()
1105        }));
1106        assert!(refresh(&mut v, mins(1)).1.other.is_empty(), "needs 450");
1107    }
1108
1109    fn resident(rss: u64, limit: Option<u64>) -> ResidentSize {
1110        ResidentSize {
1111            checked_at: Some(mins(0)),
1112            rss_anon_bytes: rss,
1113            limit_bytes: limit,
1114            ..ResidentSize::default()
1115        }
1116    }
1117
1118    #[test]
1119    fn resident_size_near_the_limit_degrades_until_it_falls_past_the_band() {
1120        let mut v = ValueFacts::default();
1121        v.resident_measured(Some(resident(840, Some(1_000))));
1122        assert!(refresh(&mut v, mins(0)).1.other.is_empty());
1123        v.resident_measured(Some(resident(850, Some(1_000))));
1124        let (report, problems) = refresh(&mut v, mins(1));
1125        assert_eq!(report.resident, Some(resident(850, Some(1_000))));
1126        assert_eq!(problems.other.len(), 1);
1127        assert!(problems.other[0].contains("85% of its"), "{problems:?}");
1128        v.resident_measured(Some(resident(760, Some(1_000))));
1129        assert_eq!(refresh(&mut v, mins(2)).1.other.len(), 1, "inside band");
1130        v.resident_measured(Some(resident(740, Some(1_000))));
1131        assert!(refresh(&mut v, mins(3)).1.other.is_empty());
1132        v.resident_measured(Some(resident(5_000, None)));
1133        assert!(refresh(&mut v, mins(4)).1.other.is_empty());
1134        v.resident_measured(Some(resident(900, Some(1_000))));
1135        assert_eq!(refresh(&mut v, mins(5)).1.other.len(), 1);
1136        v.resident_measured(None);
1137        let (report, problems) = refresh(&mut v, mins(6));
1138        assert_eq!(report.resident, None);
1139        assert!(problems.other.is_empty());
1140    }
1141}