lindera_analysis/token_filter/
lowercase.rs

1use std::borrow::Cow;
2
3use serde_json::Value;
4
5use crate::token_filter::TokenFilter;
6use lindera::LinderaResult;
7use lindera::token::Token;
8
9pub const LOWERCASE_TOKEN_FILTER_NAME: &str = "lowercase";
10
11pub type LowercaseTokenFilterConfig = Value;
12
13/// Normalizes token text to lowercase.
14///
15#[derive(Clone, Debug)]
16pub struct LowercaseTokenFilter {}
17
18impl LowercaseTokenFilter {
19    pub fn new() -> Self {
20        Self {}
21    }
22
23    pub fn from_config(_config: &LowercaseTokenFilterConfig) -> LinderaResult<Self> {
24        Ok(Self::new())
25    }
26}
27
28impl Default for LowercaseTokenFilter {
29    fn default() -> Self {
30        Self::new()
31    }
32}
33
34impl TokenFilter for LowercaseTokenFilter {
35    fn name(&self) -> &'static str {
36        LOWERCASE_TOKEN_FILTER_NAME
37    }
38
39    fn apply(&self, tokens: &mut Vec<Token<'_>>) -> LinderaResult<()> {
40        for token in tokens.iter_mut() {
41            // Skip the allocating case-fold for tokens with no uppercase
42            // characters (e.g. any CJK-only token), which is a no-op.
43            if token.surface.chars().any(|c| c.is_uppercase()) {
44                token.surface = Cow::Owned(token.surface.to_lowercase());
45            }
46        }
47
48        Ok(())
49    }
50}
51
52#[cfg(test)]
53mod tests {
54    #[test]
55    #[cfg(feature = "embed-ipadic")]
56    fn test_lowercase_token_filter_apply_ipadic() {
57        use std::borrow::Cow;
58
59        use crate::token_filter::TokenFilter;
60        use crate::token_filter::lowercase::LowercaseTokenFilter;
61        use lindera::dictionary::{DictionaryKind, WordId, load_embedded_dictionary};
62        use lindera::token::Token;
63        use lindera_dictionary::viterbi::LexType;
64
65        let filter = LowercaseTokenFilter::new();
66
67        let dictionary = load_embedded_dictionary(DictionaryKind::IPADIC).unwrap();
68
69        let mut tokens: Vec<Token> = vec![Token {
70            surface: Cow::Borrowed("Rust"),
71            byte_start: 0,
72            byte_end: 4,
73            position: 0,
74            position_length: 1,
75            word_id: WordId::new(LexType::System, 4294967295),
76            dictionary: &dictionary,
77            user_dictionary: None,
78            details: Some(vec![Cow::Borrowed("UNK")]),
79        }];
80
81        filter.apply(&mut tokens).unwrap();
82
83        assert_eq!(tokens.len(), 1);
84        assert_eq!(&tokens[0].surface, "rust");
85    }
86
87    #[test]
88    #[cfg(feature = "embed-ipadic")]
89    fn test_lowercase_token_filter_apply_cjk_no_alloc() {
90        use std::borrow::Cow;
91
92        use crate::token_filter::TokenFilter;
93        use crate::token_filter::lowercase::LowercaseTokenFilter;
94        use lindera::dictionary::{DictionaryKind, WordId, load_embedded_dictionary};
95        use lindera::token::Token;
96        use lindera_dictionary::viterbi::LexType;
97
98        let filter = LowercaseTokenFilter::new();
99
100        let dictionary = load_embedded_dictionary(DictionaryKind::IPADIC).unwrap();
101
102        let mut tokens: Vec<Token> = vec![Token {
103            surface: Cow::Borrowed("東京"),
104            byte_start: 0,
105            byte_end: 6,
106            position: 0,
107            position_length: 1,
108            word_id: WordId::new(LexType::System, 4294967295),
109            dictionary: &dictionary,
110            user_dictionary: None,
111            details: Some(vec![Cow::Borrowed("UNK")]),
112        }];
113
114        filter.apply(&mut tokens).unwrap();
115
116        assert_eq!(&tokens[0].surface, "東京");
117        // The fast path must leave CJK-only tokens untouched (no reallocation).
118        assert!(matches!(tokens[0].surface, Cow::Borrowed(_)));
119    }
120}
lindera_analysis/token_filter/lowercase.rs

lindera_analysis/token_filter/
lowercase.rs
