lb_tantivy/tokenizer/
simple_tokenizer.rs

1use std::str::CharIndices;
2
3use super::{Token, TokenStream, Tokenizer};
4
5/// Tokenize the text by splitting on whitespaces and punctuation.
6#[derive(Clone, Default)]
7pub struct SimpleTokenizer {
8    token: Token,
9}
10
11/// TokenStream produced by the `SimpleTokenizer`.
12pub struct SimpleTokenStream<'a> {
13    text: &'a str,
14    chars: CharIndices<'a>,
15    token: &'a mut Token,
16}
17
18impl Tokenizer for SimpleTokenizer {
19    type TokenStream<'a> = SimpleTokenStream<'a>;
20    fn token_stream<'a>(&'a mut self, text: &'a str) -> SimpleTokenStream<'a> {
21        self.token.reset();
22        SimpleTokenStream {
23            text,
24            chars: text.char_indices(),
25            token: &mut self.token,
26        }
27    }
28}
29
30impl SimpleTokenStream<'_> {
31    // search for the end of the current token.
32    fn search_token_end(&mut self) -> usize {
33        (&mut self.chars)
34            .filter(|(_, c)| !c.is_alphanumeric())
35            .map(|(offset, _)| offset)
36            .next()
37            .unwrap_or(self.text.len())
38    }
39}
40
41impl TokenStream for SimpleTokenStream<'_> {
42    fn advance(&mut self) -> bool {
43        self.token.text.clear();
44        self.token.position = self.token.position.wrapping_add(1);
45        while let Some((offset_from, c)) = self.chars.next() {
46            if c.is_alphanumeric() {
47                let offset_to = self.search_token_end();
48                self.token.offset_from = offset_from;
49                self.token.offset_to = offset_to;
50                self.token.text.push_str(&self.text[offset_from..offset_to]);
51                return true;
52            }
53        }
54        false
55    }
56
57    fn token(&self) -> &Token {
58        self.token
59    }
60
61    fn token_mut(&mut self) -> &mut Token {
62        self.token
63    }
64}
65
66#[cfg(test)]
67mod tests {
68    use crate::tokenizer::tests::assert_token;
69    use crate::tokenizer::{SimpleTokenizer, TextAnalyzer, Token};
70
71    #[test]
72    fn test_simple_tokenizer() {
73        let tokens = token_stream_helper("Hello, happy tax payer!");
74        assert_eq!(tokens.len(), 4);
75        assert_token(&tokens[0], 0, "Hello", 0, 5);
76        assert_token(&tokens[1], 1, "happy", 7, 12);
77        assert_token(&tokens[2], 2, "tax", 13, 16);
78        assert_token(&tokens[3], 3, "payer", 17, 22);
79    }
80
81    fn token_stream_helper(text: &str) -> Vec<Token> {
82        let mut a = TextAnalyzer::from(SimpleTokenizer::default());
83        let mut token_stream = a.token_stream(text);
84        let mut tokens: Vec<Token> = vec![];
85        let mut add_token = |token: &Token| {
86            tokens.push(token.clone());
87        };
88        token_stream.process(&mut add_token);
89        tokens
90    }
91}
lb_tantivy/tokenizer/simple_tokenizer.rs

lb_tantivy/tokenizer/
simple_tokenizer.rs
