Skip to main content

second_brain_api/
value_metrics.rs

1//! Whether second-brain is worth running: how often sessions recall, how
2//! often recall finds something, what agents do with what it found, and
3//! what the store costs in disk, memory and sync lag. Kept in memory over a
4//! rolling window, so a restart starts every window empty and each check
5//! waits for enough events before it judges.
6
7use std::collections::{BTreeMap, HashMap, HashSet, VecDeque};
8use std::hash::Hash;
9
10use crate::health::{counted, human};
11use chrono::{DateTime, Duration, Utc};
12use second_brain_core::residency::ComponentResidency;
13use serde::Serialize;
14use uuid::Uuid;
15
16pub const WINDOW: Duration = Duration::hours(24);
17/// Bounds memory when a caller recalls far more often than any agent does.
18const MAX_EVENTS: usize = 50_000;
19
20const SESSION_RATE: Band = Band {
21    min_samples: 10,
22    degrade: 0.25,
23    recover: 0.35,
24};
25/// A real store's typed prompts found nothing to inject 17% of the time
26/// (`evals/results/hook_replay.json`).
27const EMPTY_OR_LOW: Band = Band {
28    min_samples: 50,
29    degrade: 0.75,
30    recover: 0.65,
31};
32const CORRECTED: Band = Band {
33    min_samples: 20,
34    degrade: 0.20,
35    recover: 0.15,
36};
37const RESIDENT_DEGRADE: f64 = 0.85;
38const RESIDENT_RECOVER: f64 = 0.75;
39/// Free space must reach this multiple of what compaction and an index
40/// rebuild rewrite (the database and indexes, never the transcript archive)
41/// before a store that outgrew its disk counts as fixed.
42const STORE_RECOVER: f64 = 1.1;
43
44#[derive(Debug, Clone, Copy, PartialEq, Eq)]
45pub enum RecallVerdict {
46    /// Something cleared the relevance bar.
47    Hit,
48    /// Results came back and none cleared the bar.
49    LowScore,
50    /// Results came back without a bar applied although a reranker exists,
51    /// or for a caller that shows only what clears it.
52    Unjudged,
53    Empty,
54}
55
56#[derive(Debug, Clone, Copy, PartialEq, Eq)]
57pub enum PeerOutcome {
58    /// Both sides now hold each other's changes.
59    Synced,
60    /// The peer took this device's changes but its own were not pulled, so
61    /// it ends a failure streak without ending the lag.
62    Pushed,
63    Failed,
64}
65
66#[derive(Debug, Clone, PartialEq)]
67pub struct RecallEvent {
68    /// The caller's agent session, when it sent one.
69    pub session: Option<String>,
70    pub verdict: RecallVerdict,
71    /// The memories the caller was shown.
72    pub memory_ids: Vec<Uuid>,
73}
74
75#[derive(Debug, Clone, Default, PartialEq, Serialize)]
76pub struct RecallReport {
77    pub window_secs: i64,
78    pub recalls: u64,
79    pub hits: u64,
80    pub low_score: u64,
81    pub unjudged: u64,
82    pub empty: u64,
83    /// Over judged recalls, the pool the threshold judges.
84    pub empty_share: Option<f64>,
85    /// Over judged recalls, the pool the threshold judges.
86    pub low_score_share: Option<f64>,
87    pub unjudged_share: Option<f64>,
88    pub sessions_with_recall: u64,
89    /// None until the first read of the ingest log, or while a backfill
90    /// makes the ingest log's recent sessions mostly old ones.
91    pub sessions_ingested: Option<u64>,
92    /// Ingested sessions that recalled, over ingested sessions.
93    pub session_recall_rate: Option<f64>,
94}
95
96#[derive(Debug, Clone, Default, PartialEq, Serialize)]
97pub struct RecalledReport {
98    pub window_secs: i64,
99    pub memories: u64,
100    pub corrected: u64,
101    /// Fetched by id or cited by id in a note after it was recalled.
102    pub used: u64,
103    pub corrected_share: Option<f64>,
104    pub used_share: Option<f64>,
105}
106
107#[derive(Debug, Clone, PartialEq, Serialize)]
108pub struct PeerSync {
109    pub peer: String,
110    pub last_success_at: Option<DateTime<Utc>>,
111    pub last_attempt_at: Option<DateTime<Utc>>,
112    pub consecutive_failures: u32,
113    /// Since the last success, or since startup when none was seen.
114    pub lag_secs: i64,
115}
116
117#[derive(Debug, Clone, Default, PartialEq, Eq, Serialize)]
118pub struct StoreSize {
119    pub checked_at: Option<DateTime<Utc>>,
120    /// The database with its WAL and shared-memory files.
121    pub db_bytes: u64,
122    /// The WAL and shared-memory part of `db_bytes`, which a compaction or
123    /// rebuild reuses rather than needs anew.
124    pub wal_bytes: u64,
125    pub index_bytes: u64,
126    pub archive_bytes: u64,
127    pub raw_index_bytes: u64,
128    pub total_bytes: u64,
129    /// Space left on the store's filesystem; None when it can't be read.
130    pub free_bytes: Option<u64>,
131}
132
133#[derive(Debug, Clone, Default, PartialEq, Eq, Serialize)]
134pub struct ResidentSize {
135    pub checked_at: Option<DateTime<Utc>>,
136    pub rss_anon_bytes: u64,
137    pub components: ComponentResidency,
138    pub raw_index_bytes: u64,
139    pub unattributed_bytes: i64,
140    /// The cgroup's `memory.max`, else the machine's memory; None when
141    /// neither can be read.
142    pub limit_bytes: Option<u64>,
143}
144
145/// Degrades once a share crosses `degrade` and recovers only once it is
146/// back past `recover`, so a share hovering at the line alerts once. Below
147/// `min_samples` it never degrades, but a degraded share holds until the
148/// window empties, so samples aging out do not flip it.
149struct Band {
150    min_samples: u64,
151    degrade: f64,
152    recover: f64,
153}
154
155/// Which checks are degraded right now; each flips only at its band edge.
156#[derive(Debug, Default)]
157struct Latches {
158    session_rate: bool,
159    empty_or_low: bool,
160    corrected: bool,
161    store: bool,
162    resident: bool,
163}
164
165#[derive(Debug, Clone, Copy, Default)]
166struct Flags {
167    corrected: bool,
168    used: bool,
169}
170
171/// Keys seen within the window with the time each was last seen, oldest
172/// first, so pruning is a pop from the front.
173#[derive(Debug)]
174struct Recent<K, V> {
175    entries: HashMap<K, (DateTime<Utc>, V)>,
176    order: VecDeque<(DateTime<Utc>, K)>,
177}
178
179impl<K, V> Default for Recent<K, V> {
180    fn default() -> Self {
181        Self {
182            entries: HashMap::new(),
183            order: VecDeque::new(),
184        }
185    }
186}
187
188impl Band {
189    /// For a share where more is worse.
190    fn above(&self, latched: bool, count: u64, of: u64) -> bool {
191        if of == 0 || (!latched && of < self.min_samples) {
192            return false;
193        }
194        let share = count as f64 / of as f64;
195        share > if latched { self.recover } else { self.degrade }
196    }
197
198    /// For a share where less is worse.
199    fn below(&self, latched: bool, count: u64, of: u64) -> bool {
200        if of == 0 || (!latched && of < self.min_samples) {
201            return false;
202        }
203        let share = count as f64 / of as f64;
204        share < if latched { self.recover } else { self.degrade }
205    }
206}
207
208impl<K: Hash + Eq + Clone, V: Default> Recent<K, V> {
209    fn touch(&mut self, key: K, at: DateTime<Utc>) {
210        match self.entries.get_mut(&key) {
211            Some(entry) => entry.0 = at,
212            None => {
213                self.entries.insert(key.clone(), (at, V::default()));
214            }
215        }
216        self.order.push_back((at, key));
217        if self.order.len() > 4 * MAX_EVENTS {
218            let entries = &self.entries;
219            self.order
220                .retain(|(at, key)| entries.get(key).is_some_and(|e| e.0 == *at));
221        }
222    }
223
224    fn get_mut(&mut self, key: &K) -> Option<&mut V> {
225        self.entries.get_mut(key).map(|e| &mut e.1)
226    }
227
228    /// Calls `evicted` for each entry dropped.
229    fn prune(&mut self, cutoff: DateTime<Utc>, mut evicted: impl FnMut(&V)) {
230        while let Some((at, key)) = self.order.front() {
231            let live = self.entries.get(key).is_some_and(|e| e.0 == *at);
232            if live && *at >= cutoff && self.entries.len() <= MAX_EVENTS {
233                break;
234            }
235            if let Some((_, key)) = self.order.pop_front()
236                && live
237                && let Some((_, value)) = self.entries.remove(&key)
238            {
239                evicted(&value);
240            }
241        }
242    }
243
244    fn len(&self) -> usize {
245        self.entries.len()
246    }
247
248    fn count_since(&self, cutoff: DateTime<Utc>) -> usize {
249        self.entries
250            .values()
251            .filter(|(at, _)| *at >= cutoff)
252            .count()
253    }
254
255    fn contains(&self, key: &K) -> bool {
256        self.entries.contains_key(key)
257    }
258}
259
260#[derive(Debug, Default)]
261pub struct ValueFacts {
262    recalls: VecDeque<(DateTime<Utc>, RecallVerdict)>,
263    hits: u64,
264    low_score: u64,
265    unjudged: u64,
266    empty: u64,
267    recall_sessions: Recent<String, ()>,
268    recalled: Recent<Uuid, Flags>,
269    corrected: u64,
270    used: u64,
271    ingested_sessions: Option<HashSet<String>>,
272    /// The session check's problem while it is degraded, kept through
273    /// passes where the ingested sessions are unknown.
274    session_problem: Option<String>,
275    peers: BTreeMap<String, PeerState>,
276    store: Option<StoreSize>,
277    resident: Option<ResidentSize>,
278    latches: Latches,
279}
280
281#[derive(Debug, Clone, Default)]
282struct PeerState {
283    listed_at: Option<DateTime<Utc>>,
284    last_success_at: Option<DateTime<Utc>>,
285    last_attempt_at: Option<DateTime<Utc>>,
286    consecutive_failures: u32,
287}
288
289/// What `refresh` concluded: problems that degrade the daemon, with the
290/// per-peer ones apart so a check that already says no sync succeeded at
291/// all can leave them out.
292#[derive(Debug, Default, PartialEq)]
293pub struct ValueProblems {
294    pub peers: Vec<String>,
295    pub other: Vec<String>,
296}
297
298#[derive(Debug, Clone, Default, PartialEq, Serialize)]
299pub struct ValueReport {
300    pub recall: RecallReport,
301    pub recalled: RecalledReport,
302    pub sync_peers: Vec<PeerSync>,
303    pub store: Option<StoreSize>,
304    pub resident: Option<ResidentSize>,
305}
306
307/// The session a transcript or caller belongs to: a subagent's
308/// `<session>/<agent>` key counts as its parent session.
309pub fn session_of(key: &str) -> &str {
310    key.split_once('/').map_or(key, |(session, _)| session)
311}
312
313impl ValueFacts {
314    pub fn recalled(&mut self, event: RecallEvent, now: DateTime<Utc>) {
315        self.recalls.push_back((now, event.verdict));
316        *self.count_of(event.verdict) += 1;
317        if let Some(session) = &event.session {
318            self.recall_sessions
319                .touch(session_of(session).to_string(), now);
320        }
321        for id in event.memory_ids {
322            self.recalled.touch(id, now);
323        }
324    }
325
326    fn count_of(&mut self, verdict: RecallVerdict) -> &mut u64 {
327        match verdict {
328            RecallVerdict::Hit => &mut self.hits,
329            RecallVerdict::LowScore => &mut self.low_score,
330            RecallVerdict::Unjudged => &mut self.unjudged,
331            RecallVerdict::Empty => &mut self.empty,
332        }
333    }
334
335    /// Counts only memories recalled within the window, and each once.
336    pub fn corrected(&mut self, id: Uuid) {
337        if let Some(flags) = self.recalled.get_mut(&id)
338            && !flags.corrected
339        {
340            flags.corrected = true;
341            self.corrected += 1;
342        }
343    }
344
345    pub fn used(&mut self, ids: &[Uuid]) {
346        for id in ids {
347            if let Some(flags) = self.recalled.get_mut(id)
348                && !flags.used
349            {
350                flags.used = true;
351                self.used += 1;
352            }
353        }
354    }
355
356    /// The sessions the ingest log shows captured within the window; None
357    /// while that count would mislead.
358    pub fn sessions_ingested(&mut self, keys: Option<Vec<String>>) {
359        self.ingested_sessions =
360            keys.map(|keys| keys.iter().map(|key| session_of(key).to_string()).collect());
361    }
362
363    /// Forgets peers no longer listed and starts tracking new ones; results
364    /// count only for listed peers, so a sync that ends after its peer was
365    /// removed does not bring it back.
366    pub fn peers_listed(&mut self, peers: &[String], now: DateTime<Utc>) {
367        self.peers.retain(|peer, _| peers.contains(peer));
368        for peer in peers {
369            self.peers.entry(peer.clone()).or_insert_with(|| PeerState {
370                listed_at: Some(now),
371                ..PeerState::default()
372            });
373        }
374    }
375
376    pub fn sync_attempted(&mut self, peer: &str, outcome: PeerOutcome, now: DateTime<Utc>) {
377        let Some(state) = self.peers.get_mut(peer) else {
378            return;
379        };
380        match outcome {
381            PeerOutcome::Synced => {
382                state.last_attempt_at = Some(now);
383                state.last_success_at = Some(now);
384                state.consecutive_failures = 0;
385            }
386            PeerOutcome::Pushed => {
387                state.last_attempt_at = Some(now);
388                state.consecutive_failures = 0;
389            }
390            PeerOutcome::Failed => {
391                state.last_attempt_at = Some(now);
392                state.consecutive_failures += 1;
393            }
394        }
395    }
396
397    /// None when the measurement failed, which drops the old reading.
398    pub fn store_measured(&mut self, size: Option<StoreSize>) {
399        self.store = size;
400    }
401
402    /// None when the measurement failed, which drops the old reading.
403    pub fn resident_measured(&mut self, size: Option<ResidentSize>) {
404        self.resident = size;
405    }
406
407    /// Prunes the windows, moves each check across its band and reports.
408    /// `started` bounds a peer's lag before its first success; `sync_overdue`
409    /// is the alert config's window, None when sync checks are off.
410    pub fn refresh(
411        &mut self,
412        now: DateTime<Utc>,
413        started: DateTime<Utc>,
414        sync_overdue: Option<Duration>,
415    ) -> (ValueReport, ValueProblems) {
416        self.prune(now - WINDOW);
417        let mut problems = ValueProblems::default();
418        let recall = self.recall_report(now - WINDOW, &mut problems);
419        let recalled = self.recalled_report(&mut problems);
420        let sync_peers = self.peer_report(now, started, sync_overdue, &mut problems);
421        self.check_store(&mut problems);
422        self.check_resident(&mut problems);
423        let report = ValueReport {
424            recall,
425            recalled,
426            sync_peers,
427            store: self.store.clone(),
428            resident: self.resident.clone(),
429        };
430        (report, problems)
431    }
432
433    fn prune(&mut self, cutoff: DateTime<Utc>) {
434        while let Some(&(at, verdict)) = self.recalls.front() {
435            if at >= cutoff && self.recalls.len() <= MAX_EVENTS {
436                break;
437            }
438            self.recalls.pop_front();
439            *self.count_of(verdict) -= 1;
440        }
441        self.recall_sessions.prune(cutoff - session_grace(), |_| {});
442        let (corrected, used) = (&mut self.corrected, &mut self.used);
443        self.recalled.prune(cutoff, |flags| {
444            *corrected -= u64::from(flags.corrected);
445            *used -= u64::from(flags.used);
446        });
447    }
448
449    fn recall_report(
450        &mut self,
451        cutoff: DateTime<Utc>,
452        problems: &mut ValueProblems,
453    ) -> RecallReport {
454        let recalls = self.recalls.len() as u64;
455        let ingested = self.ingested_sessions.as_ref();
456        let joined = ingested.map(|keys| {
457            keys.iter()
458                .filter(|key| self.recall_sessions.contains(*key))
459                .count() as u64
460        });
461        let sessions_ingested = ingested.map(|keys| keys.len() as u64);
462
463        let judged = recalls - self.unjudged;
464        let bad = self.empty + self.low_score;
465        self.latches.empty_or_low = EMPTY_OR_LOW.above(self.latches.empty_or_low, bad, judged);
466        if self.latches.empty_or_low {
467            problems.other.push(format!(
468                "{} of {judged} judged recalls in the last 24h found nothing that cleared the \
469                 relevance bar ({} empty, {} below it)",
470                percent(bad, judged),
471                self.empty,
472                self.low_score
473            ));
474        }
475        if let (Some(joined), Some(of)) = (joined, sessions_ingested) {
476            self.latches.session_rate = SESSION_RATE.below(self.latches.session_rate, joined, of);
477            self.session_problem = self.latches.session_rate.then(|| {
478                format!(
479                    "only {joined} of {of} sessions in the last 24h recalled from second-brain; \
480                     check its hooks and MCP server with `sb doctor`"
481                )
482            });
483        }
484        if let Some(problem) = &self.session_problem {
485            problems.other.push(problem.clone());
486        }
487
488        RecallReport {
489            window_secs: WINDOW.num_seconds(),
490            recalls,
491            hits: self.hits,
492            low_score: self.low_score,
493            unjudged: self.unjudged,
494            empty: self.empty,
495            empty_share: share(self.empty, judged),
496            low_score_share: share(self.low_score, judged),
497            unjudged_share: share(self.unjudged, recalls),
498            sessions_with_recall: self.recall_sessions.count_since(cutoff) as u64,
499            sessions_ingested,
500            session_recall_rate: joined
501                .zip(sessions_ingested)
502                .and_then(|(j, of)| share(j, of)),
503        }
504    }
505
506    fn recalled_report(&mut self, problems: &mut ValueProblems) -> RecalledReport {
507        let memories = self.recalled.len() as u64;
508        self.latches.corrected = CORRECTED.above(self.latches.corrected, self.corrected, memories);
509        if self.latches.corrected {
510            problems.other.push(format!(
511                "{} of {memories} memories recalled in the last 24h were corrected",
512                self.corrected
513            ));
514        }
515        RecalledReport {
516            window_secs: WINDOW.num_seconds(),
517            memories,
518            corrected: self.corrected,
519            used: self.used,
520            corrected_share: share(self.corrected, memories),
521            used_share: share(self.used, memories),
522        }
523    }
524
525    fn peer_report(
526        &self,
527        now: DateTime<Utc>,
528        started: DateTime<Utc>,
529        sync_overdue: Option<Duration>,
530        problems: &mut ValueProblems,
531    ) -> Vec<PeerSync> {
532        self.peers
533            .iter()
534            .map(|(peer, state)| {
535                let since = state.last_success_at.or(state.listed_at).unwrap_or(started);
536                let lag = (now - since).max(Duration::zero());
537                if sync_overdue.is_some_and(|window| lag >= window) {
538                    let failures = match state.consecutive_failures {
539                        0 => String::new(),
540                        n => format!(
541                            " ({} in a row)",
542                            counted(u64::from(n), "failure", "failures")
543                        ),
544                    };
545                    problems.peers.push(format!(
546                        "no sync with {peer} has succeeded in {}{failures}",
547                        human(lag)
548                    ));
549                }
550                PeerSync {
551                    peer: peer.clone(),
552                    last_success_at: state.last_success_at,
553                    last_attempt_at: state.last_attempt_at,
554                    consecutive_failures: state.consecutive_failures,
555                    lag_secs: lag.num_seconds(),
556                }
557            })
558            .collect()
559    }
560
561    fn check_store(&mut self, problems: &mut ValueProblems) {
562        let Some((rewritten, free)) = self.store.as_ref().and_then(|s| {
563            let rewritten =
564                s.db_bytes.saturating_sub(s.wal_bytes) + s.index_bytes + s.raw_index_bytes;
565            s.free_bytes.map(|free| (rewritten, free))
566        }) else {
567            self.latches.store = false;
568            return;
569        };
570        let needed = if self.latches.store {
571            rewritten as f64 * STORE_RECOVER
572        } else {
573            rewritten as f64
574        };
575        self.latches.store = (free as f64) < needed;
576        if self.latches.store {
577            problems.other.push(format!(
578                "compacting or rebuilding the store needs {} MB and its disk has {} MB free",
579                rewritten / 1_000_000,
580                free / 1_000_000
581            ));
582        }
583    }
584
585    fn check_resident(&mut self, problems: &mut ValueProblems) {
586        let Some((rss, limit)) = self.resident.as_ref().and_then(|r| {
587            r.limit_bytes
588                .filter(|&l| l > 0)
589                .map(|l| (r.rss_anon_bytes, l))
590        }) else {
591            self.latches.resident = false;
592            return;
593        };
594        let ratio = rss as f64 / limit as f64;
595        let line = if self.latches.resident {
596            RESIDENT_RECOVER
597        } else {
598            RESIDENT_DEGRADE
599        };
600        self.latches.resident = ratio >= line;
601        if self.latches.resident {
602            problems.other.push(format!(
603                "the daemon holds {} MB resident, {} of its {} MB limit",
604                rss / 1_000_000,
605                percent(rss, limit),
606                limit / 1_000_000
607            ));
608        }
609    }
610}
611
612/// The ingested sessions are read every `progress::INTERVAL`, so a recall
613/// stays joined that long past the window to meet a session the last read
614/// still lists.
615fn session_grace() -> Duration {
616    Duration::from_std(crate::progress::INTERVAL).unwrap_or(Duration::zero())
617}
618
619fn share(count: u64, of: u64) -> Option<f64> {
620    (of > 0).then(|| count as f64 / of as f64)
621}
622
623fn percent(count: u64, of: u64) -> String {
624    format!("{:.0}%", count as f64 * 100.0 / of as f64)
625}
626
627#[cfg(test)]
628mod tests {
629    use super::*;
630
631    fn t0() -> DateTime<Utc> {
632        DateTime::parse_from_rfc3339("2026-10-31T12:00:00Z")
633            .unwrap()
634            .with_timezone(&Utc)
635    }
636
637    fn mins(m: i64) -> DateTime<Utc> {
638        t0() + Duration::minutes(m)
639    }
640
641    fn id(n: u128) -> Uuid {
642        Uuid::from_u128(n)
643    }
644
645    fn recall(session: &str, verdict: RecallVerdict, ids: &[u128]) -> RecallEvent {
646        RecallEvent {
647            session: Some(session.to_string()),
648            verdict,
649            memory_ids: ids.iter().map(|&n| id(n)).collect(),
650        }
651    }
652
653    fn refresh(v: &mut ValueFacts, now: DateTime<Utc>) -> (ValueReport, ValueProblems) {
654        v.refresh(now, t0(), Some(Duration::hours(24)))
655    }
656
657    #[test]
658    fn subagent_keys_count_as_their_parent_session() {
659        assert_eq!(session_of("abc"), "abc");
660        assert_eq!(session_of("abc/agent-1"), "abc");
661    }
662
663    #[test]
664    fn recall_shares_count_empty_and_low_score_recalls() {
665        let mut v = ValueFacts::default();
666        v.recalled(recall("s1", RecallVerdict::Hit, &[1]), mins(0));
667        v.recalled(recall("s1", RecallVerdict::Empty, &[]), mins(1));
668        v.recalled(recall("s2", RecallVerdict::LowScore, &[2]), mins(2));
669        v.recalled(recall("s2", RecallVerdict::Empty, &[]), mins(3));
670        v.recalled(recall("s2", RecallVerdict::Unjudged, &[]), mins(3));
671        let (report, problems) = refresh(&mut v, mins(4));
672        let r = report.recall;
673        assert_eq!(r.window_secs, WINDOW.num_seconds());
674        assert_eq!((r.recalls, r.hits, r.low_score, r.empty), (5, 1, 1, 2));
675        assert_eq!(r.empty_share, Some(0.5), "over judged recalls");
676        assert_eq!(r.low_score_share, Some(0.25), "over judged recalls");
677        assert_eq!(r.unjudged_share, Some(0.2), "over all recalls");
678        assert_eq!(r.sessions_with_recall, 2);
679        assert_eq!(problems, ValueProblems::default());
680    }
681
682    #[test]
683    fn mostly_empty_recalls_degrade_once_and_recover_past_the_band() {
684        let mut v = ValueFacts::default();
685        for m in 0..49 {
686            v.recalled(recall("s", RecallVerdict::Empty, &[]), mins(m));
687        }
688        assert!(refresh(&mut v, mins(49)).1.other.is_empty(), "too few");
689        v.recalled(recall("s", RecallVerdict::LowScore, &[1]), mins(49));
690        let (_, problems) = refresh(&mut v, mins(50));
691        assert_eq!(problems.other.len(), 1, "{problems:?}");
692        assert!(
693            problems.other[0].contains("of 50 judged recalls in the last 24h"),
694            "{problems:?}"
695        );
696        for m in 50..71 {
697            v.recalled(recall("s", RecallVerdict::Hit, &[1]), mins(m));
698        }
699        let (report, problems) = refresh(&mut v, mins(71));
700        assert_eq!(
701            report.recall.recalls, 71,
702            "50 of 71 is 70%, inside the band"
703        );
704        assert_eq!(problems.other.len(), 1);
705        for m in 71..77 {
706            v.recalled(recall("s", RecallVerdict::Hit, &[1]), mins(m));
707        }
708        assert!(
709            refresh(&mut v, mins(77)).1.other.is_empty(),
710            "50 of 77 is 65%"
711        );
712    }
713
714    #[test]
715    fn a_latched_share_holds_below_the_minimum_until_the_window_empties() {
716        let mut v = ValueFacts::default();
717        for m in 0..50 {
718            v.recalled(recall("s", RecallVerdict::Empty, &[]), mins(m));
719        }
720        assert_eq!(refresh(&mut v, mins(50)).1.other.len(), 1);
721        let first_gone = mins(0) + WINDOW + Duration::seconds(30);
722        let (report, problems) = refresh(&mut v, first_gone);
723        assert_eq!(report.recall.recalls, 49);
724        assert_eq!(problems.other.len(), 1, "below the minimum, still latched");
725        let all_gone = mins(50) + WINDOW;
726        assert!(refresh(&mut v, all_gone).1.other.is_empty());
727    }
728
729    #[test]
730    fn unjudged_recalls_are_left_out_of_the_empty_or_low_share() {
731        let mut v = ValueFacts::default();
732        for m in 0..50 {
733            v.recalled(recall("s", RecallVerdict::Unjudged, &[]), mins(m));
734        }
735        let (report, problems) = refresh(&mut v, mins(50));
736        assert_eq!(report.recall.unjudged, 50);
737        assert_eq!(report.recall.unjudged_share, Some(1.0));
738        assert!(problems.other.is_empty(), "{problems:?}");
739        for m in 50..99 {
740            v.recalled(recall("s", RecallVerdict::Empty, &[]), mins(m));
741        }
742        assert!(refresh(&mut v, mins(99)).1.other.is_empty(), "49 judged");
743        v.recalled(recall("s", RecallVerdict::LowScore, &[]), mins(99));
744        let (_, problems) = refresh(&mut v, mins(100));
745        assert_eq!(problems.other.len(), 1);
746        assert!(
747            problems.other[0].contains("100% of 50 judged recalls in the last 24h"),
748            "{problems:?}"
749        );
750        assert!(
751            problems.other[0].contains("(49 empty, 1 below it)"),
752            "{problems:?}"
753        );
754    }
755
756    #[test]
757    fn a_session_recalled_just_before_the_window_edge_counts_until_the_next_ingest_read() {
758        let mut v = ValueFacts::default();
759        v.recalled(recall("a", RecallVerdict::Hit, &[]), mins(0));
760        v.sessions_ingested(Some(vec!["a".to_string()]));
761        let edge = mins(0) + WINDOW + Duration::minutes(4);
762        let recall = refresh(&mut v, edge).0.recall;
763        assert_eq!(recall.session_recall_rate, Some(1.0));
764        assert_eq!(
765            recall.sessions_with_recall, 0,
766            "outside the reported window"
767        );
768        let later = mins(0) + WINDOW + Duration::minutes(6);
769        assert_eq!(
770            refresh(&mut v, later).0.recall.session_recall_rate,
771            Some(0.0)
772        );
773    }
774
775    #[test]
776    fn recalls_older_than_the_window_drop_out() {
777        let mut v = ValueFacts::default();
778        v.recalled(recall("s1", RecallVerdict::Empty, &[1]), mins(0));
779        v.recalled(recall("s2", RecallVerdict::Hit, &[2]), mins(60));
780        let (report, _) = refresh(&mut v, mins(0) + WINDOW + Duration::minutes(10));
781        assert_eq!(report.recall.recalls, 1);
782        assert_eq!(report.recall.empty, 0);
783        assert_eq!(report.recall.sessions_with_recall, 1);
784        assert_eq!(report.recalled.memories, 1);
785    }
786
787    #[test]
788    fn the_event_window_is_bounded() {
789        let mut v = ValueFacts::default();
790        for n in 0..(MAX_EVENTS as u128 + 10) {
791            v.recalled(recall(&format!("s{n}"), RecallVerdict::Hit, &[n]), mins(0));
792        }
793        let (report, _) = refresh(&mut v, mins(1));
794        assert_eq!(report.recall.recalls, MAX_EVENTS as u64);
795        assert_eq!(report.recall.sessions_with_recall, MAX_EVENTS as u64);
796        assert_eq!(report.recalled.memories, MAX_EVENTS as u64);
797    }
798
799    #[test]
800    fn session_recall_rate_joins_recalling_sessions_with_ingested_ones() {
801        let mut v = ValueFacts::default();
802        let (report, _) = refresh(&mut v, mins(0));
803        assert_eq!(report.recall.sessions_ingested, None);
804        assert_eq!(report.recall.session_recall_rate, None);
805
806        v.recalled(recall("a", RecallVerdict::Hit, &[]), mins(0));
807        v.recalled(recall("b/agent-1", RecallVerdict::Hit, &[]), mins(0));
808        v.recalled(recall("not-ingested", RecallVerdict::Hit, &[]), mins(0));
809        let keys = ["a", "a/agent-2", "b", "c", "d"].map(String::from).to_vec();
810        v.sessions_ingested(Some(keys));
811        let (report, problems) = refresh(&mut v, mins(1));
812        assert_eq!(report.recall.sessions_ingested, Some(4));
813        assert_eq!(report.recall.session_recall_rate, Some(0.5));
814        assert!(problems.other.is_empty(), "under the minimum sample");
815    }
816
817    #[test]
818    fn sessions_that_never_recall_degrade_once_until_they_recover_past_the_band() {
819        let mut v = ValueFacts::default();
820        let keys: Vec<String> = (0..10).map(|n| format!("s{n}")).collect();
821        v.recalled(recall("s0", RecallVerdict::Hit, &[]), mins(0));
822        v.recalled(recall("s1", RecallVerdict::Hit, &[]), mins(0));
823        v.sessions_ingested(Some(keys.clone()));
824        let (_, problems) = refresh(&mut v, mins(1));
825        assert_eq!(problems.other.len(), 1);
826        assert!(
827            problems.other[0].contains("2 of 10 sessions in the last 24h recalled"),
828            "{problems:?}"
829        );
830        v.recalled(recall("s2", RecallVerdict::Hit, &[]), mins(2));
831        assert_eq!(refresh(&mut v, mins(3)).1.other.len(), 1, "inside band");
832        v.recalled(recall("s3", RecallVerdict::Hit, &[]), mins(4));
833        assert!(refresh(&mut v, mins(5)).1.other.is_empty());
834
835        v.sessions_ingested(None);
836        let (report, _) = refresh(&mut v, mins(8));
837        assert_eq!(report.recall.session_recall_rate, None);
838    }
839
840    #[test]
841    fn an_unknown_ingest_count_keeps_the_session_alert_until_a_count_clears_it() {
842        let mut v = ValueFacts::default();
843        let keys: Vec<String> = (0..10).map(|n| format!("s{n}")).collect();
844        v.recalled(recall("s0", RecallVerdict::Hit, &[]), mins(0));
845        v.sessions_ingested(Some(keys.clone()));
846        assert_eq!(refresh(&mut v, mins(1)).1.other.len(), 1);
847        v.sessions_ingested(None);
848        let (report, problems) = refresh(&mut v, mins(2));
849        assert_eq!(report.recall.session_recall_rate, None);
850        assert_eq!(problems.other.len(), 1, "{problems:?}");
851        for n in 1..4 {
852            v.recalled(recall(&format!("s{n}"), RecallVerdict::Hit, &[]), mins(3));
853        }
854        v.sessions_ingested(Some(keys));
855        assert!(refresh(&mut v, mins(4)).1.other.is_empty());
856    }
857
858    #[test]
859    fn corrections_and_uses_count_only_recalled_memories_once_each() {
860        let mut v = ValueFacts::default();
861        v.recalled(recall("s", RecallVerdict::Hit, &[1, 2, 3, 4]), mins(0));
862        v.corrected(id(1));
863        v.corrected(id(1));
864        v.corrected(id(99));
865        v.used(&[id(2), id(3), id(2), id(98)]);
866        let (report, _) = refresh(&mut v, mins(1));
867        let r = report.recalled;
868        assert_eq!((r.memories, r.corrected, r.used), (4, 1, 2));
869        assert_eq!(r.corrected_share, Some(0.25));
870        assert_eq!(r.used_share, Some(0.5));
871
872        let later = mins(0) + WINDOW + Duration::minutes(1);
873        let (report, _) = refresh(&mut v, later);
874        assert_eq!(
875            report.recalled,
876            RecalledReport {
877                window_secs: WINDOW.num_seconds(),
878                ..RecalledReport::default()
879            }
880        );
881    }
882
883    #[test]
884    fn a_memory_corrected_before_it_was_recalled_is_not_counted() {
885        let mut v = ValueFacts::default();
886        v.corrected(id(1));
887        v.recalled(recall("s", RecallVerdict::Hit, &[1]), mins(0));
888        let (report, _) = refresh(&mut v, mins(1));
889        assert_eq!(report.recalled.corrected, 0);
890    }
891
892    #[test]
893    fn many_corrected_recalls_degrade_once_and_recover_past_the_band() {
894        let mut v = ValueFacts::default();
895        let ids: Vec<u128> = (0..20).collect();
896        v.recalled(recall("s", RecallVerdict::Hit, &ids), mins(0));
897        for n in 0..4 {
898            v.corrected(id(n));
899        }
900        assert!(
901            refresh(&mut v, mins(1)).1.other.is_empty(),
902            "20% is the line"
903        );
904        v.corrected(id(4));
905        let (_, problems) = refresh(&mut v, mins(2));
906        assert_eq!(problems.other.len(), 1);
907        assert!(
908            problems.other[0].contains("5 of 20 memories recalled in the last 24h were corrected"),
909            "{problems:?}"
910        );
911        let more: Vec<u128> = (100..110).collect();
912        v.recalled(recall("s", RecallVerdict::Hit, &more), mins(3));
913        assert_eq!(refresh(&mut v, mins(4)).1.other.len(), 1, "inside band");
914        let more: Vec<u128> = (110..120).collect();
915        v.recalled(recall("s", RecallVerdict::Hit, &more), mins(5));
916        assert!(refresh(&mut v, mins(6)).1.other.is_empty());
917    }
918
919    #[test]
920    fn a_peer_lagging_past_the_window_degrades_until_it_syncs() {
921        let mut v = ValueFacts::default();
922        v.peers_listed(&["u@pi".to_string(), "u@mac".to_string()], mins(0));
923        v.sync_attempted("u@pi", PeerOutcome::Synced, mins(0));
924        v.sync_attempted("u@mac", PeerOutcome::Failed, mins(0));
925        v.sync_attempted("u@mac", PeerOutcome::Failed, mins(5));
926        let (report, problems) = refresh(&mut v, mins(10));
927        assert_eq!(report.sync_peers.len(), 2);
928        let mac = &report.sync_peers[0];
929        assert_eq!(mac.peer, "u@mac");
930        assert_eq!(mac.last_success_at, None);
931        assert_eq!(mac.last_attempt_at, Some(mins(5)));
932        assert_eq!(mac.consecutive_failures, 2);
933        assert_eq!(mac.lag_secs, 600, "from startup");
934        assert_eq!(report.sync_peers[1].lag_secs, 600);
935        assert!(problems.peers.is_empty());
936
937        let late = t0() + Duration::hours(24);
938        v.sync_attempted("u@pi", PeerOutcome::Synced, late);
939        let (_, problems) = refresh(&mut v, late);
940        assert_eq!(problems.peers.len(), 1);
941        assert!(
942            problems.peers[0].contains("no sync with u@mac has succeeded in 24h 0m"),
943            "{problems:?}"
944        );
945        assert!(
946            problems.peers[0].contains("2 failures in a row"),
947            "{problems:?}"
948        );
949
950        v.sync_attempted("u@mac", PeerOutcome::Synced, late);
951        assert!(refresh(&mut v, late).1.peers.is_empty());
952
953        v.peers_listed(&["u@pi".to_string()], late);
954        assert_eq!(refresh(&mut v, late).0.sync_peers.len(), 1);
955        assert!(
956            v.refresh(late + Duration::hours(30), t0(), None)
957                .1
958                .peers
959                .is_empty(),
960            "sync checks off"
961        );
962    }
963
964    #[test]
965    fn a_push_ends_a_failure_streak_but_not_the_lag() {
966        let mut v = ValueFacts::default();
967        v.peers_listed(&["u@pi".to_string()], mins(0));
968        v.sync_attempted("u@pi", PeerOutcome::Failed, mins(1));
969        v.sync_attempted("u@pi", PeerOutcome::Pushed, mins(2));
970        let (report, _) = refresh(&mut v, mins(3));
971        let pi = &report.sync_peers[0];
972        assert_eq!(pi.consecutive_failures, 0);
973        assert_eq!(pi.last_attempt_at, Some(mins(2)));
974        assert_eq!(pi.last_success_at, None);
975        assert_eq!(pi.lag_secs, 180);
976    }
977
978    #[test]
979    fn a_peer_listed_late_lags_from_when_it_was_listed() {
980        let mut v = ValueFacts::default();
981        let late = t0() + Duration::hours(30);
982        v.peers_listed(&["u@mac".to_string()], late);
983        let (report, problems) = refresh(&mut v, late + Duration::minutes(5));
984        assert_eq!(report.sync_peers[0].lag_secs, 300);
985        assert!(problems.peers.is_empty());
986        v.peers_listed(&["u@mac".to_string()], late + Duration::hours(1));
987        let (report, _) = refresh(&mut v, late + Duration::hours(2));
988        assert_eq!(
989            report.sync_peers[0].lag_secs, 7_200,
990            "listing again keeps the start"
991        );
992    }
993
994    #[test]
995    fn a_result_for_a_peer_no_longer_listed_is_ignored() {
996        let mut v = ValueFacts::default();
997        v.peers_listed(&["u@pi".to_string()], mins(0));
998        v.peers_listed(&[], mins(0));
999        v.sync_attempted("u@pi", PeerOutcome::Failed, mins(1));
1000        assert!(refresh(&mut v, mins(2)).0.sync_peers.is_empty());
1001    }
1002
1003    fn store(total: u64, free: Option<u64>) -> StoreSize {
1004        StoreSize {
1005            checked_at: Some(mins(0)),
1006            db_bytes: total,
1007            total_bytes: total,
1008            free_bytes: free,
1009            ..StoreSize::default()
1010        }
1011    }
1012
1013    #[test]
1014    fn a_store_larger_than_its_free_space_degrades_until_space_clears_the_band() {
1015        let mut v = ValueFacts::default();
1016        let (report, _) = refresh(&mut v, mins(0));
1017        assert_eq!(report.store, None);
1018        v.store_measured(Some(store(1_000, Some(1_000))));
1019        assert!(refresh(&mut v, mins(0)).1.other.is_empty());
1020        v.store_measured(Some(store(1_000, Some(999))));
1021        let (report, problems) = refresh(&mut v, mins(1));
1022        assert_eq!(report.store, Some(store(1_000, Some(999))));
1023        assert_eq!(problems.other.len(), 1);
1024        assert!(
1025            problems.other[0].contains("compacting or rebuilding the store needs 0 MB"),
1026            "{problems:?}"
1027        );
1028        v.store_measured(Some(store(1_000, Some(1_050))));
1029        assert_eq!(refresh(&mut v, mins(2)).1.other.len(), 1, "inside band");
1030        v.store_measured(Some(store(1_000, Some(1_100))));
1031        assert!(refresh(&mut v, mins(3)).1.other.is_empty());
1032        v.store_measured(Some(store(1_000, Some(999))));
1033        assert_eq!(refresh(&mut v, mins(4)).1.other.len(), 1);
1034        v.store_measured(None);
1035        let (report, problems) = refresh(&mut v, mins(5));
1036        assert_eq!(report.store, None);
1037        assert!(problems.other.is_empty());
1038        v.store_measured(Some(store(1_000, None)));
1039        assert!(refresh(&mut v, mins(6)).1.other.is_empty());
1040    }
1041
1042    #[test]
1043    fn the_store_check_leaves_out_the_archive_that_compaction_never_rewrites() {
1044        let mut v = ValueFacts::default();
1045        v.store_measured(Some(StoreSize {
1046            checked_at: Some(mins(0)),
1047            db_bytes: 100,
1048            wal_bytes: 0,
1049            index_bytes: 100,
1050            archive_bytes: 10_000,
1051            raw_index_bytes: 100,
1052            total_bytes: 10_300,
1053            free_bytes: Some(301),
1054        }));
1055        assert!(refresh(&mut v, mins(1)).1.other.is_empty());
1056        v.store_measured(Some(StoreSize {
1057            free_bytes: Some(299),
1058            ..v.store.clone().unwrap()
1059        }));
1060        assert_eq!(refresh(&mut v, mins(2)).1.other.len(), 1);
1061    }
1062
1063    #[test]
1064    fn a_large_wal_is_not_counted_as_space_a_rewrite_needs() {
1065        let mut v = ValueFacts::default();
1066        v.store_measured(Some(StoreSize {
1067            checked_at: Some(mins(0)),
1068            db_bytes: 1_000,
1069            wal_bytes: 600,
1070            index_bytes: 50,
1071            total_bytes: 1_050,
1072            free_bytes: Some(500),
1073            ..StoreSize::default()
1074        }));
1075        assert!(refresh(&mut v, mins(1)).1.other.is_empty(), "needs 450");
1076    }
1077
1078    fn resident(rss: u64, limit: Option<u64>) -> ResidentSize {
1079        ResidentSize {
1080            checked_at: Some(mins(0)),
1081            rss_anon_bytes: rss,
1082            limit_bytes: limit,
1083            ..ResidentSize::default()
1084        }
1085    }
1086
1087    #[test]
1088    fn resident_size_near_the_limit_degrades_until_it_falls_past_the_band() {
1089        let mut v = ValueFacts::default();
1090        v.resident_measured(Some(resident(840, Some(1_000))));
1091        assert!(refresh(&mut v, mins(0)).1.other.is_empty());
1092        v.resident_measured(Some(resident(850, Some(1_000))));
1093        let (report, problems) = refresh(&mut v, mins(1));
1094        assert_eq!(report.resident, Some(resident(850, Some(1_000))));
1095        assert_eq!(problems.other.len(), 1);
1096        assert!(problems.other[0].contains("85% of its"), "{problems:?}");
1097        v.resident_measured(Some(resident(760, Some(1_000))));
1098        assert_eq!(refresh(&mut v, mins(2)).1.other.len(), 1, "inside band");
1099        v.resident_measured(Some(resident(740, Some(1_000))));
1100        assert!(refresh(&mut v, mins(3)).1.other.is_empty());
1101        v.resident_measured(Some(resident(5_000, None)));
1102        assert!(refresh(&mut v, mins(4)).1.other.is_empty());
1103        v.resident_measured(Some(resident(900, Some(1_000))));
1104        assert_eq!(refresh(&mut v, mins(5)).1.other.len(), 1);
1105        v.resident_measured(None);
1106        let (report, problems) = refresh(&mut v, mins(6));
1107        assert_eq!(report.resident, None);
1108        assert!(problems.other.is_empty());
1109    }
1110}