khive_text/
analyzer.rs

1//! Composable text analysis pipeline.
2
3use std::sync::Arc;
4
5use crate::{Analyzer, BoxedTokenizer, TokenFilter, Tokenizer};
6
7/// Runs one tokenizer followed by filters in insertion order.
8///
9/// See `crates/khive-text/docs/api/analyzer-pipeline.md`.
10pub struct StandardAnalyzer {
11    tokenizer: BoxedTokenizer,
12    filters: Vec<Box<dyn TokenFilter>>,
13}
14
15impl StandardAnalyzer {
16    /// Create an analyzer with the given tokenizer and no filters.
17    pub fn with_tokenizer(tokenizer: impl Tokenizer + 'static) -> Self {
18        Self {
19            tokenizer: Arc::new(tokenizer),
20            filters: Vec::new(),
21        }
22    }
23
24    /// Append a filter to the pipeline. Filters run in insertion order.
25    #[must_use]
26    pub fn filter(mut self, f: impl TokenFilter + 'static) -> Self {
27        self.filters.push(Box::new(f));
28        self
29    }
30}
31
32impl Analyzer for StandardAnalyzer {
33    fn analyze(&self, text: &str) -> Vec<String> {
34        let mut tokens = self.tokenizer.tokenize(text);
35        for f in &self.filters {
36            tokens = tokens.into_iter().filter_map(|t| f.apply(t)).collect();
37        }
38        tokens
39    }
40}
41
42#[cfg(test)]
43mod tests {
44    use super::*;
45    use crate::filter::{LowercaseFilter, MinLengthFilter};
46    use crate::tokenizer::WhitespaceTokenizer;
47
48    #[test]
49    fn pipeline_lowercases() {
50        let a = StandardAnalyzer::with_tokenizer(WhitespaceTokenizer).filter(LowercaseFilter);
51        assert_eq!(a.analyze("Hello WORLD"), vec!["hello", "world"]);
52    }
53
54    #[test]
55    fn pipeline_chains_filters() {
56        let a = StandardAnalyzer::with_tokenizer(WhitespaceTokenizer)
57            .filter(LowercaseFilter)
58            .filter(MinLengthFilter(4));
59        assert_eq!(a.analyze("hi hello world"), vec!["hello", "world"]);
60    }
61
62    #[test]
63    fn empty_input() {
64        let a = StandardAnalyzer::with_tokenizer(WhitespaceTokenizer);
65        assert!(a.analyze("").is_empty());
66    }
67}
khive_text/analyzer.rs

khive_text/
analyzer.rs
