context_forge/analysis/
mod.rs

1//! Text analysis primitives for importance detection.
2//!
3//! This module is pure computation - no I/O, no storage, no network.
4//! It provides tokenization, stopword filtering, n-gram extraction,
5//! and term count computation.
6
7use std::collections::HashMap;
8
9#[cfg(feature = "parallel")]
10use rayon::prelude::*;
11
12/// Importance classification of passages into categories.
13pub mod classification;
14/// Extraction of importance segments from scored passages.
15pub mod extraction;
16/// Term frequency and session term-map construction.
17pub mod frequency;
18/// Detection of prompt-injection-shaped content.
19pub mod injection;
20/// Lexicon configuration for importance scoring.
21pub mod lexicon;
22/// N-gram extraction over token sequences.
23pub mod ngrams;
24/// Pre-filtering of passages before classification.
25pub mod prefilter;
26/// Cross-session recurrence scoring.
27pub mod recurrence;
28/// Importance scoring of classified passages.
29pub mod scoring;
30/// Text tokenization and stopword filtering.
31pub mod tokenizer;
32
33// Re-export public API
34pub use classification::{
35    classify_passages, ClassificationConfig, ClassifiedPassage, ImportanceCategory, PassageContext,
36};
37pub use extraction::{extract_passages, ExtractedPassage, ExtractionConfig, ExtractionEntry};
38pub use frequency::{term_counts, term_counts_with_ngrams};
39pub use injection::{adjust_weights, scale_budget, InjectionConfig};
40pub use lexicon::Lexicons;
41pub use ngrams::{bigrams, extract, trigrams};
42pub use prefilter::{strip_execution_artifacts, FilterToggle, PrefilterConfig};
43pub use recurrence::{compute_recurrence, RecurrenceConfig, RecurrenceResult};
44pub use scoring::{pack_segments, score_passages, ImportanceSegment, ScoringConfig};
45pub use tokenizer::{Tokenizer, TokenizerConfig};
46
47/// Build per-session term-count maps by pre-filtering, tokenizing, and
48/// computing n-gram term counts for each session's entries.
49///
50/// Each inner `Vec<&str>` represents the raw content strings for one session.
51/// The resulting maps are suitable as input to [`compute_recurrence`].
52#[allow(
53    clippy::implicit_hasher,
54    reason = "HashMap with the default hasher is the natural return type here; generalizing \
55              over S: BuildHasher would add generic noise with no caller benefit"
56)]
57#[must_use]
58pub fn build_session_term_maps(
59    session_contents: &[Vec<&str>],
60    tokenizer: &Tokenizer,
61    prefilter_config: &PrefilterConfig,
62) -> Vec<HashMap<String, usize>> {
63    let session_term_map = |contents: &Vec<&str>| {
64        let mut combined_tokens: Vec<String> = Vec::new();
65        for content in contents {
66            let clean = strip_execution_artifacts(content, prefilter_config);
67            combined_tokens.extend(tokenizer.tokenize(&clean));
68        }
69        term_counts_with_ngrams(&combined_tokens)
70    };
71
72    #[cfg(feature = "parallel")]
73    {
74        session_contents.par_iter().map(session_term_map).collect()
75    }
76    #[cfg(not(feature = "parallel"))]
77    {
78        session_contents.iter().map(session_term_map).collect()
79    }
80}
81
82/// Run `f` inside a scoped rayon thread pool capped at `thread_cap` threads,
83/// or on the global pool when `thread_cap` is `None`.
84///
85/// This crate never configures rayon's *global* thread pool — the global
86/// pool is process-wide and the host application may already own it (for
87/// example, a local LLM server sharing the workstation). Callers that want
88/// to bound CPU usage for a batch analysis pass should wrap that section in
89/// `with_thread_cap(Some(n), || { .. })`. Passing `None` simply calls `f`
90/// directly, running on whatever pool (global or otherwise) is already in
91/// scope.
92///
93/// `Some(0)` is passed through to rayon, which treats zero as "choose the
94/// thread count automatically" — it is not an error.
95///
96/// # Errors
97///
98/// Returns [`crate::Error::InvalidEntry`] if building the scoped thread pool
99/// fails (e.g. the OS refuses to spawn threads).
100#[cfg(feature = "parallel")]
101pub fn with_thread_cap<R: Send>(
102    thread_cap: Option<usize>,
103    f: impl FnOnce() -> R + Send,
104) -> crate::Result<R> {
105    match thread_cap {
106        Some(threads) => {
107            let pool = rayon::ThreadPoolBuilder::new()
108                .num_threads(threads)
109                .build()
110                .map_err(|err| crate::Error::InvalidEntry(format!("rayon pool: {err}")))?;
111            Ok(pool.install(f))
112        }
113        None => Ok(f()),
114    }
115}
116
117#[cfg(all(test, feature = "parallel"))]
118mod parallel_tests {
119    use super::with_thread_cap;
120
121    #[test]
122    fn with_thread_cap_some_installs_scoped_pool() {
123        let threads = with_thread_cap(Some(2), rayon::current_num_threads).unwrap();
124        assert_eq!(
125            threads, 2,
126            "closure should run inside a 2-thread scoped pool"
127        );
128    }
129
130    #[test]
131    fn with_thread_cap_none_calls_directly() {
132        let value = with_thread_cap(None, || 21 * 2).unwrap();
133        assert_eq!(value, 42);
134    }
135
136    #[test]
137    fn with_thread_cap_zero_means_automatic() {
138        // rayon treats num_threads(0) as "choose automatically", not an error.
139        let threads = with_thread_cap(Some(0), rayon::current_num_threads).unwrap();
140        assert!(threads >= 1);
141    }
142
143    #[test]
144    fn with_thread_cap_returns_closure_result_through_pool() {
145        let sum: usize = with_thread_cap(Some(2), || (1..=100).sum()).unwrap();
146        assert_eq!(sum, 5050);
147    }
148}
context_forge/analysis/mod.rs

context_forge/analysis/
mod.rs
