context_forge/analysis/
recurrence.rs

1use std::collections::{HashMap, HashSet};
2
3/// Configuration for cross-session recurrence scoring.
4#[derive(Debug, Clone)]
5#[non_exhaustive]
6pub struct RecurrenceConfig {
7    /// Minimum number of sessions that must contain a term.
8    pub min_session_frequency: usize,
9    /// Maximum allowed recurrence ratio (`SF / total_sessions`).
10    pub max_session_ratio: f64,
11    /// Terms always excluded from recurrence output.
12    pub term_blocklist: HashSet<String>,
13}
14
15impl Default for RecurrenceConfig {
16    fn default() -> Self {
17        Self {
18            min_session_frequency: 2,
19            max_session_ratio: 0.8,
20            term_blocklist: HashSet::new(),
21        }
22    }
23}
24
25/// Recurrence metrics for a term that passed filtering.
26#[derive(Debug, Clone, PartialEq)]
27pub struct RecurrenceResult {
28    /// The term (unigram, bigram, or trigram).
29    pub term: String,
30    /// Number of sessions containing this term.
31    pub session_frequency: usize,
32    /// Session frequency normalized by total session count.
33    pub recurrence_score: f64,
34}
35
36#[derive(Debug, Clone, Copy, PartialEq)]
37struct FilteredRecurrenceMetrics {
38    session_frequency: usize,
39    recurrence_score: f64,
40}
41
42/// Compute cross-session recurrence for terms.
43///
44/// Input is one term-count map per session. Presence is binary per session;
45/// term count values inside each map are ignored.
46#[must_use]
47#[allow(
48    clippy::implicit_hasher,
49    clippy::cast_precision_loss,
50    reason = "Public API and recurrence score type are intentionally fixed by design"
51)]
52pub fn compute_recurrence(
53    session_term_maps: &[HashMap<String, usize>],
54    config: &RecurrenceConfig,
55) -> Vec<RecurrenceResult> {
56    if session_term_maps.is_empty() {
57        return Vec::new();
58    }
59
60    let total_sessions = session_term_maps.len();
61    let mut session_frequency: HashMap<String, usize> = HashMap::new();
62
63    for term_map in session_term_maps {
64        for term in term_map.keys() {
65            *session_frequency.entry(term.clone()).or_insert(0) += 1;
66        }
67    }
68
69    let mut filtered: HashMap<String, FilteredRecurrenceMetrics> = HashMap::new();
70    for (term, sf) in session_frequency {
71        if config.term_blocklist.contains(&term) {
72            continue;
73        }
74
75        if sf < config.min_session_frequency {
76            continue;
77        }
78
79        let recurrence_score = sf as f64 / total_sessions as f64;
80        if recurrence_score > config.max_session_ratio {
81            continue;
82        }
83
84        filtered.insert(
85            term,
86            FilteredRecurrenceMetrics {
87                session_frequency: sf,
88                recurrence_score,
89            },
90        );
91    }
92
93    let mut suppressed_unigrams: HashSet<String> = HashSet::new();
94    for term in filtered.keys() {
95        if term.contains(' ') {
96            for unigram in term.split_whitespace() {
97                suppressed_unigrams.insert(unigram.to_string());
98            }
99        }
100    }
101
102    for unigram in &suppressed_unigrams {
103        filtered.remove(unigram);
104    }
105
106    let mut results: Vec<RecurrenceResult> = filtered
107        .into_iter()
108        .map(|(term, metrics)| RecurrenceResult {
109            term,
110            session_frequency: metrics.session_frequency,
111            recurrence_score: metrics.recurrence_score,
112        })
113        .collect();
114    results.sort_by(|left, right| {
115        right
116            .recurrence_score
117            .total_cmp(&left.recurrence_score)
118            .then_with(|| left.term.cmp(&right.term))
119    });
120
121    results
122}
123
124#[cfg(test)]
125mod tests {
126    use std::collections::{HashMap, HashSet};
127
128    use super::{compute_recurrence, RecurrenceConfig};
129
130    fn term_map(terms: &[&str]) -> HashMap<String, usize> {
131        let mut map = HashMap::new();
132        for term in terms {
133            map.insert((*term).to_string(), 1);
134        }
135        map
136    }
137
138    fn default_config() -> RecurrenceConfig {
139        RecurrenceConfig::default()
140    }
141
142    #[test]
143    fn compute_recurrence_returns_empty_for_empty_input() {
144        let config = default_config();
145        let results = compute_recurrence(&[], &config);
146        assert!(results.is_empty());
147    }
148
149    #[test]
150    fn compute_recurrence_returns_empty_for_single_session_with_defaults() {
151        let sessions = vec![term_map(&["context", "forge", "context forge"])];
152        let config = default_config();
153
154        let results = compute_recurrence(&sessions, &config);
155        assert!(results.is_empty());
156    }
157
158    #[test]
159    fn compute_recurrence_counts_session_presence_not_raw_frequency() {
160        let mut session_a = HashMap::new();
161        session_a.insert("context".to_string(), 100);
162        session_a.insert("signal".to_string(), 1);
163
164        let mut session_b = HashMap::new();
165        session_b.insert("context".to_string(), 1);
166
167        let session_c = term_map(&["signal"]);
168
169        let sessions = vec![session_a, session_b, session_c];
170        let config = RecurrenceConfig {
171            min_session_frequency: 1,
172            max_session_ratio: 1.0,
173            term_blocklist: HashSet::new(),
174        };
175
176        let results = compute_recurrence(&sessions, &config);
177
178        let context = results
179            .iter()
180            .find(|result| result.term == "context")
181            .expect("context should be present");
182        let signal = results
183            .iter()
184            .find(|result| result.term == "signal")
185            .expect("signal should be present");
186
187        assert_eq!(context.session_frequency, 2);
188        assert_eq!(signal.session_frequency, 2);
189        assert!((context.recurrence_score - (2.0 / 3.0)).abs() < 1e-12);
190    }
191
192    #[test]
193    fn compute_recurrence_excludes_terms_below_low_cutoff() {
194        let sessions = vec![
195            term_map(&["alpha"]),
196            term_map(&["beta"]),
197            term_map(&["alpha"]),
198        ];
199        let config = default_config();
200
201        let results = compute_recurrence(&sessions, &config);
202        assert!(results.iter().all(|result| result.term != "beta"));
203        assert!(results.iter().any(|result| result.term == "alpha"));
204    }
205
206    #[test]
207    fn compute_recurrence_excludes_terms_above_high_cutoff() {
208        let sessions = vec![
209            term_map(&["context", "alpha"]),
210            term_map(&["context", "beta"]),
211            term_map(&["context", "gamma"]),
212            term_map(&["context", "delta"]),
213            term_map(&["context", "epsilon"]),
214        ];
215        let config = default_config();
216
217        let results = compute_recurrence(&sessions, &config);
218        assert!(results.iter().all(|result| result.term != "context"));
219    }
220
221    #[test]
222    fn compute_recurrence_includes_term_at_exact_high_cutoff() {
223        let sessions = vec![
224            term_map(&["threshold"]),
225            term_map(&["threshold"]),
226            term_map(&["threshold"]),
227            term_map(&["threshold", "other"]),
228            term_map(&["other"]),
229        ];
230        let config = default_config();
231
232        let results = compute_recurrence(&sessions, &config);
233        let threshold = results
234            .iter()
235            .find(|result| result.term == "threshold")
236            .expect("threshold should pass at exact 0.8");
237        assert_eq!(threshold.session_frequency, 4);
238        assert!((threshold.recurrence_score - 0.8).abs() < 1e-12);
239    }
240
241    #[test]
242    fn compute_recurrence_can_return_empty_when_band_pass_filters_everything() {
243        let sessions = vec![term_map(&["always"]), term_map(&["always"])];
244        let config = default_config();
245
246        let results = compute_recurrence(&sessions, &config);
247        assert!(results.is_empty());
248    }
249
250    #[test]
251    fn compute_recurrence_excludes_blocklisted_terms() {
252        let sessions = vec![
253            term_map(&["context", "signal"]),
254            term_map(&["context"]),
255            term_map(&["context"]),
256        ];
257        let config = RecurrenceConfig {
258            min_session_frequency: 1,
259            max_session_ratio: 1.0,
260            term_blocklist: HashSet::from(["context".to_string()]),
261        };
262
263        let results = compute_recurrence(&sessions, &config);
264        assert!(results.iter().all(|result| result.term != "context"));
265    }
266
267    #[test]
268    fn compute_recurrence_blocklisted_ngram_does_not_suppress_unigrams() {
269        let sessions = vec![
270            term_map(&["home", "devuser", "home devuser"]),
271            term_map(&["home", "devuser", "home devuser"]),
272            term_map(&["home", "devuser"]),
273            term_map(&["home", "devuser"]),
274            term_map(&["home"]),
275        ];
276
277        let config = RecurrenceConfig {
278            min_session_frequency: 1,
279            max_session_ratio: 1.0,
280            term_blocklist: HashSet::from(["home devuser".to_string()]),
281        };
282
283        let results = compute_recurrence(&sessions, &config);
284        assert!(results.iter().any(|result| result.term == "home"));
285        assert!(results.iter().any(|result| result.term == "devuser"));
286        assert!(results.iter().all(|result| result.term != "home devuser"));
287    }
288
289    #[test]
290    fn compute_recurrence_suppresses_unigrams_when_trigram_survives() {
291        let sessions = vec![
292            term_map(&["context", "forge", "hub", "context forge hub"]),
293            term_map(&["context", "forge", "hub", "context forge hub"]),
294            term_map(&["context", "forge", "hub", "context forge hub"]),
295            term_map(&["context", "forge", "hub"]),
296            term_map(&["other"]),
297        ];
298        let config = default_config();
299
300        let results = compute_recurrence(&sessions, &config);
301        assert!(results
302            .iter()
303            .any(|result| result.term == "context forge hub"));
304        assert!(results.iter().all(|result| result.term != "context"));
305        assert!(results.iter().all(|result| result.term != "forge"));
306        assert!(results.iter().all(|result| result.term != "hub"));
307    }
308
309    #[test]
310    fn compute_recurrence_keeps_bigrams_when_trigram_survives() {
311        let sessions = vec![
312            term_map(&["a", "b", "c", "a b", "b c", "a b c"]),
313            term_map(&["a", "b", "c", "a b", "b c", "a b c"]),
314            term_map(&["a", "b", "c", "a b", "b c", "a b c"]),
315            term_map(&["a", "b", "c", "a b", "b c"]),
316            term_map(&["other"]),
317        ];
318        let config = default_config();
319
320        let results = compute_recurrence(&sessions, &config);
321        assert!(results.iter().any(|result| result.term == "a b c"));
322        assert!(results.iter().any(|result| result.term == "a b"));
323        assert!(results.iter().any(|result| result.term == "b c"));
324    }
325
326    #[test]
327    fn compute_recurrence_bigram_suppresses_constituent_unigrams() {
328        let sessions = vec![
329            term_map(&["token", "model", "token model"]),
330            term_map(&["token", "model", "token model"]),
331            term_map(&["token", "model"]),
332            term_map(&["other"]),
333        ];
334        let config = default_config();
335        let results = compute_recurrence(&sessions, &config);
336        assert!(results.iter().any(|r| r.term == "token model"));
337        assert!(results.iter().all(|r| r.term != "token"));
338        assert!(results.iter().all(|r| r.term != "model"));
339    }
340
341    #[test]
342    fn compute_recurrence_honors_configurable_thresholds() {
343        let sessions = vec![
344            term_map(&["rare", "common"]),
345            term_map(&["common"]),
346            term_map(&["common"]),
347            term_map(&["common"]),
348            term_map(&["common"]),
349        ];
350
351        let config = RecurrenceConfig {
352            min_session_frequency: 1,
353            max_session_ratio: 1.0,
354            term_blocklist: HashSet::new(),
355        };
356
357        let results = compute_recurrence(&sessions, &config);
358        let rare = results
359            .iter()
360            .find(|result| result.term == "rare")
361            .expect("rare should be included with min_session_frequency=1");
362        assert_eq!(rare.session_frequency, 1);
363        assert!((rare.recurrence_score - 0.2).abs() < 1e-12);
364    }
365
366    #[test]
367    fn compute_recurrence_sorts_by_score_desc_then_term_asc() {
368        let sessions = vec![
369            term_map(&["banana", "apple", "carrot"]),
370            term_map(&["banana", "apple"]),
371            term_map(&["banana"]),
372            term_map(&["apple"]),
373            term_map(&["carrot"]),
374        ];
375
376        let config = RecurrenceConfig {
377            min_session_frequency: 1,
378            max_session_ratio: 1.0,
379            term_blocklist: HashSet::new(),
380        };
381
382        let results = compute_recurrence(&sessions, &config);
383        let terms: Vec<&str> = results.iter().map(|result| result.term.as_str()).collect();
384        assert_eq!(terms, vec!["apple", "banana", "carrot"]);
385    }
386}
context_forge/analysis/recurrence.rs

context_forge/analysis/
recurrence.rs
