khive_text/
analyzer.rs

1//! Composable text analysis pipeline.
2
3use std::sync::Arc;
4
5use crate::{Analyzer, BoxedTokenizer, TokenFilter, Tokenizer};
6
7/// Composes one Tokenizer and N TokenFilters into a full analysis pipeline.
8pub struct StandardAnalyzer {
9    tokenizer: BoxedTokenizer,
10    filters: Vec<Box<dyn TokenFilter>>,
11}
12
13impl StandardAnalyzer {
14    /// Create an analyzer with the given tokenizer and no filters.
15    pub fn with_tokenizer(tokenizer: impl Tokenizer + 'static) -> Self {
16        Self {
17            tokenizer: Arc::new(tokenizer),
18            filters: Vec::new(),
19        }
20    }
21
22    /// Append a filter to the pipeline. Filters run in insertion order.
23    #[must_use]
24    pub fn filter(mut self, f: impl TokenFilter + 'static) -> Self {
25        self.filters.push(Box::new(f));
26        self
27    }
28}
29
30impl Analyzer for StandardAnalyzer {
31    fn analyze(&self, text: &str) -> Vec<String> {
32        let mut tokens = self.tokenizer.tokenize(text);
33        for f in &self.filters {
34            tokens = tokens.into_iter().filter_map(|t| f.apply(t)).collect();
35        }
36        tokens
37    }
38}
39
40#[cfg(test)]
41mod tests {
42    use super::*;
43    use crate::filter::{LowercaseFilter, MinLengthFilter};
44    use crate::tokenizer::WhitespaceTokenizer;
45
46    #[test]
47    fn pipeline_lowercases() {
48        let a = StandardAnalyzer::with_tokenizer(WhitespaceTokenizer).filter(LowercaseFilter);
49        assert_eq!(a.analyze("Hello WORLD"), vec!["hello", "world"]);
50    }
51
52    #[test]
53    fn pipeline_chains_filters() {
54        let a = StandardAnalyzer::with_tokenizer(WhitespaceTokenizer)
55            .filter(LowercaseFilter)
56            .filter(MinLengthFilter(4));
57        assert_eq!(a.analyze("hi hello world"), vec!["hello", "world"]);
58    }
59
60    #[test]
61    fn empty_input() {
62        let a = StandardAnalyzer::with_tokenizer(WhitespaceTokenizer);
63        assert!(a.analyze("").is_empty());
64    }
65}
khive_text/analyzer.rs

khive_text/
analyzer.rs
