lindera_analysis/token_filter/
length.rs

1use serde_json::Value;
2
3use crate::token_filter::TokenFilter;
4use lindera::LinderaResult;
5use lindera::token::Token;
6
7pub const LENGTH_TOKEN_FILTER_NAME: &str = "length";
8
9pub type LengthTokenFilterConfig = Value;
10
11/// Keep only tokens with the specified number of characters of text.
12///
13#[derive(Clone, Debug)]
14pub struct LengthTokenFilter {
15    min: Option<usize>,
16    max: Option<usize>,
17}
18
19impl LengthTokenFilter {
20    pub fn new(min: Option<usize>, max: Option<usize>) -> Self {
21        Self { min, max }
22    }
23
24    pub fn from_config(config: &LengthTokenFilterConfig) -> LinderaResult<Self> {
25        let min: Option<usize> = config
26            .get("min")
27            .and_then(|v| v.as_u64())
28            .map(|v| v as usize);
29        let max: Option<usize> = config
30            .get("max")
31            .and_then(|v| v.as_u64())
32            .map(|v| v as usize);
33
34        Ok(Self::new(min, max))
35    }
36}
37
38impl TokenFilter for LengthTokenFilter {
39    fn name(&self) -> &'static str {
40        LENGTH_TOKEN_FILTER_NAME
41    }
42
43    fn apply(&self, tokens: &mut Vec<Token<'_>>) -> LinderaResult<()> {
44        tokens.retain(|token| {
45            let len = token.surface.chars().count();
46            if let Some(min) = self.min
47                && len < min
48            {
49                return false;
50            }
51            if let Some(max) = self.max
52                && len > max
53            {
54                return false;
55            }
56            true
57        });
58
59        Ok(())
60    }
61}
62
63#[cfg(test)]
64mod tests {
65    use crate::token_filter::length::{LengthTokenFilter, LengthTokenFilterConfig};
66
67    #[test]
68    fn test_length_token_filter_confige() {
69        let config_str = r#"
70            {
71                "min": 1,
72                "max": 3
73            }
74            "#;
75        let result: Result<LengthTokenFilterConfig, _> = serde_json::from_str(config_str);
76        assert!(result.is_ok());
77
78        let config_str = r#"
79            {
80                "min": 1
81            }
82            "#;
83        let result: Result<LengthTokenFilterConfig, _> = serde_json::from_str(config_str);
84        assert!(result.is_ok());
85
86        let config_str = r#"
87            {
88                "max": 2
89            }
90            "#;
91        let result: Result<LengthTokenFilterConfig, _> = serde_json::from_str(config_str);
92        assert!(result.is_ok());
93    }
94
95    #[test]
96    fn test_length_token_filter() {
97        let config_str = r#"
98            {
99                "min": 1,
100                "max": 3
101            }
102            "#;
103        let config: LengthTokenFilterConfig = serde_json::from_str(config_str).unwrap();
104        let result = LengthTokenFilter::from_config(&config);
105        assert!(result.is_ok());
106
107        let config_str = r#"
108            {
109                "min": 1
110            }
111            "#;
112        let config: LengthTokenFilterConfig = serde_json::from_str(config_str).unwrap();
113        let result = LengthTokenFilter::from_config(&config);
114        assert!(result.is_ok());
115
116        let config_str = r#"
117            {
118                "max": 2
119            }
120            "#;
121        let config: LengthTokenFilterConfig = serde_json::from_str(config_str).unwrap();
122        let result = LengthTokenFilter::from_config(&config);
123        assert!(result.is_ok());
124    }
125
126    #[test]
127    #[cfg(feature = "embed-ipadic")]
128    fn test_length_token_filter_apply_ipadic() {
129        use std::borrow::Cow;
130
131        use crate::token_filter::TokenFilter;
132        use lindera::dictionary::{DictionaryKind, WordId, load_embedded_dictionary};
133        use lindera::token::Token;
134        use lindera_dictionary::viterbi::LexType;
135
136        let config_str = r#"
137            {
138                "min": 2,
139                "max": 3
140            }
141            "#;
142        let config: LengthTokenFilterConfig = serde_json::from_str(config_str).unwrap();
143        let filter = LengthTokenFilter::from_config(&config).unwrap();
144
145        let dictionary = load_embedded_dictionary(DictionaryKind::IPADIC).unwrap();
146
147        let mut tokens: Vec<Token> = vec![
148            Token {
149                surface: Cow::Borrowed("すもも"),
150                byte_start: 0,
151                byte_end: 9,
152                position: 0,
153                position_length: 1,
154                word_id: WordId::new(LexType::System, 36165),
155                dictionary: &dictionary,
156                user_dictionary: None,
157                details: Some(vec![
158                    Cow::Borrowed("名詞"),
159                    Cow::Borrowed("一般"),
160                    Cow::Borrowed("*"),
161                    Cow::Borrowed("*"),
162                    Cow::Borrowed("*"),
163                    Cow::Borrowed("*"),
164                    Cow::Borrowed("すもも"),
165                    Cow::Borrowed("スモモ"),
166                    Cow::Borrowed("スモモ"),
167                ]),
168            },
169            Token {
170                surface: Cow::Borrowed("も"),
171                byte_start: 9,
172                byte_end: 12,
173                position: 1,
174                position_length: 1,
175                word_id: WordId::new(LexType::System, 73246),
176                dictionary: &dictionary,
177                user_dictionary: None,
178                details: Some(vec![
179                    Cow::Borrowed("助詞"),
180                    Cow::Borrowed("係助詞"),
181                    Cow::Borrowed("*"),
182                    Cow::Borrowed("*"),
183                    Cow::Borrowed("*"),
184                    Cow::Borrowed("*"),
185                    Cow::Borrowed("も"),
186                    Cow::Borrowed("モ"),
187                    Cow::Borrowed("モ"),
188                ]),
189            },
190            Token {
191                surface: Cow::Borrowed("もも"),
192                byte_start: 12,
193                byte_end: 18,
194                position: 2,
195                position_length: 1,
196                word_id: WordId::new(LexType::System, 74990),
197                dictionary: &dictionary,
198                user_dictionary: None,
199                details: Some(vec![
200                    Cow::Borrowed("名詞"),
201                    Cow::Borrowed("一般"),
202                    Cow::Borrowed("*"),
203                    Cow::Borrowed("*"),
204                    Cow::Borrowed("*"),
205                    Cow::Borrowed("*"),
206                    Cow::Borrowed("もも"),
207                    Cow::Borrowed("モモ"),
208                    Cow::Borrowed("モモ"),
209                ]),
210            },
211            Token {
212                surface: Cow::Borrowed("も"),
213                byte_start: 18,
214                byte_end: 21,
215                position: 3,
216                position_length: 1,
217                word_id: WordId::new(LexType::System, 73246),
218                dictionary: &dictionary,
219                user_dictionary: None,
220                details: Some(vec![
221                    Cow::Borrowed("助詞"),
222                    Cow::Borrowed("係助詞"),
223                    Cow::Borrowed("*"),
224                    Cow::Borrowed("*"),
225                    Cow::Borrowed("*"),
226                    Cow::Borrowed("*"),
227                    Cow::Borrowed("も"),
228                    Cow::Borrowed("モ"),
229                    Cow::Borrowed("モ"),
230                ]),
231            },
232            Token {
233                surface: Cow::Borrowed("もも"),
234                byte_start: 21,
235                byte_end: 27,
236                position: 4,
237                position_length: 1,
238                word_id: WordId::new(LexType::System, 74990),
239                dictionary: &dictionary,
240                user_dictionary: None,
241                details: Some(vec![
242                    Cow::Borrowed("名詞"),
243                    Cow::Borrowed("一般"),
244                    Cow::Borrowed("*"),
245                    Cow::Borrowed("*"),
246                    Cow::Borrowed("*"),
247                    Cow::Borrowed("*"),
248                    Cow::Borrowed("もも"),
249                    Cow::Borrowed("モモ"),
250                    Cow::Borrowed("モモ"),
251                ]),
252            },
253            Token {
254                surface: Cow::Borrowed("の"),
255                byte_start: 27,
256                byte_end: 30,
257                position: 5,
258                position_length: 1,
259                word_id: WordId::new(LexType::System, 55831),
260                dictionary: &dictionary,
261                user_dictionary: None,
262                details: Some(vec![
263                    Cow::Borrowed("助詞"),
264                    Cow::Borrowed("連体化"),
265                    Cow::Borrowed("*"),
266                    Cow::Borrowed("*"),
267                    Cow::Borrowed("*"),
268                    Cow::Borrowed("*"),
269                    Cow::Borrowed("の"),
270                    Cow::Borrowed("ノ"),
271                    Cow::Borrowed("ノ"),
272                ]),
273            },
274            Token {
275                surface: Cow::Borrowed("うち"),
276                byte_start: 30,
277                byte_end: 36,
278                position: 6,
279                position_length: 1,
280                word_id: WordId::new(LexType::System, 8029),
281                dictionary: &dictionary,
282                user_dictionary: None,
283                details: Some(vec![
284                    Cow::Borrowed("名詞"),
285                    Cow::Borrowed("非自立"),
286                    Cow::Borrowed("副詞可能"),
287                    Cow::Borrowed("*"),
288                    Cow::Borrowed("*"),
289                    Cow::Borrowed("*"),
290                    Cow::Borrowed("うち"),
291                    Cow::Borrowed("ウチ"),
292                    Cow::Borrowed("ウチ"),
293                ]),
294            },
295        ];
296
297        filter.apply(&mut tokens).unwrap();
298
299        assert_eq!(tokens.len(), 4);
300        assert_eq!(&tokens[0].surface, "すもも");
301        assert_eq!(&tokens[1].surface, "もも");
302        assert_eq!(&tokens[2].surface, "もも");
303        assert_eq!(&tokens[3].surface, "うち");
304    }
305}
lindera_analysis/token_filter/length.rs

lindera_analysis/token_filter/
length.rs
