lindera_analysis/
token_filter.rs

1mod compound;
2/// This module defines various token filters and provides functionality to load them.
3///
4/// # Modules
5/// - `japanese_base_form`: Contains the Japanese base form token filter.
6/// - `japanese_compound_word`: Contains the Japanese compound word token filter.
7/// - `japanese_kana`: Contains the Japanese kana token filter.
8/// - `japanese_katakana_stem`: Contains the Japanese katakana stem token filter.
9/// - `japanese_keep_tags`: Contains the Japanese keep tags token filter.
10/// - `japanese_number`: Contains the Japanese number token filter.
11/// - `japanese_reading_form`: Contains the Japanese reading form token filter.
12/// - `japanese_stop_tags`: Contains the Japanese stop tags token filter.
13/// - `keep_words`: Contains the keep words token filter.
14/// - `korean_compound_word`: Contains the Korean compound word token filter.
15/// - `korean_keep_tags`: Contains the Korean keep tags token filter.
16/// - `korean_number`: Contains the Korean number token filter.
17/// - `korean_reading_form`: Contains the Korean reading form token filter.
18/// - `korean_stop_tags`: Contains the Korean stop tags token filter.
19/// - `length`: Contains the length token filter.
20/// - `lowercase`: Contains the lowercase token filter.
21/// - `mapping`: Contains the mapping token filter.
22/// - `remove_diacritical_mark`: Contains the remove diacritical mark token filter.
23/// - `stop_words`: Contains the stop words token filter.
24/// - `uppercase`: Contains the uppercase token filter.
25///
26/// # Traits
27/// - `TokenFilter`: A trait for token filters that can be applied to a vector of tokens.
28/// - `TokenFilterClone`: A trait for cloning boxed token filters.
29///
30/// # Structs
31/// - `BoxTokenFilter`: A boxed token filter that implements `TokenFilter`.
32/// - `TokenFilterLoader`: A loader for creating token filters from configuration values.
33///
34/// # Usage
35/// The `TokenFilterLoader` struct provides methods to load token filters from configuration values
36/// or command-line flags. The `TokenFilter` trait defines the interface for token filters, and
37/// `BoxTokenFilter` is a boxed implementation of a token filter.
38pub mod japanese_base_form;
39pub mod japanese_compound_word;
40pub mod japanese_kana;
41pub mod japanese_katakana_stem;
42pub mod japanese_keep_tags;
43pub mod japanese_number;
44pub mod japanese_reading_form;
45pub mod japanese_stop_tags;
46pub mod keep_words;
47pub mod korean_compound_word;
48pub mod korean_keep_tags;
49pub mod korean_number;
50pub mod korean_reading_form;
51pub mod korean_stop_tags;
52pub mod length;
53pub mod lowercase;
54pub mod mapping;
55mod numeral;
56pub mod remove_diacritical_mark;
57pub mod stop_words;
58mod tags;
59pub mod uppercase;
60
61use serde_json::Value;
62use std::ops::Deref;
63
64use crate::parse_cli_flag;
65use crate::token_filter::japanese_base_form::{
66    JAPANESE_BASE_FORM_TOKEN_FILTER_NAME, JapaneseBaseFormTokenFilter,
67};
68use crate::token_filter::japanese_compound_word::{
69    JAPANESE_COMPOUND_WORD_TOKEN_FILTER_NAME, JapaneseCompoundWordTokenFilter,
70};
71use crate::token_filter::japanese_kana::{
72    JAPANESE_KANA_TOKEN_FILTER_NAME, JapaneseKanaTokenFilter,
73};
74use crate::token_filter::japanese_katakana_stem::{
75    JAPANESE_KATAKANA_STEM_TOKEN_FILTER_NAME, JapaneseKatakanaStemTokenFilter,
76};
77use crate::token_filter::japanese_keep_tags::{
78    JAPANESE_KEEP_TAGS_TOKEN_FILTER_NAME, JapaneseKeepTagsTokenFilter,
79};
80use crate::token_filter::japanese_number::{
81    JAPANESE_NUMBER_TOKEN_FILTER_NAME, JapaneseNumberTokenFilter,
82};
83use crate::token_filter::japanese_reading_form::{
84    JAPANESE_READING_FORM_TOKEN_FILTER_NAME, JapaneseReadingFormTokenFilter,
85};
86use crate::token_filter::japanese_stop_tags::{
87    JAPANESE_STOP_TAGS_TOKEN_FILTER_NAME, JapaneseStopTagsTokenFilter,
88};
89use crate::token_filter::keep_words::{KEEP_WORDS_TOKEN_FILTER_NAME, KeepWordsTokenFilter};
90use crate::token_filter::korean_compound_word::{
91    KOREAN_COMPOUND_WORD_TOKEN_FILTER_NAME, KoreanCompoundWordTokenFilter,
92};
93use crate::token_filter::korean_keep_tags::{
94    KOREAN_KEEP_TAGS_TOKEN_FILTER_NAME, KoreanKeepTagsTokenFilter,
95};
96use crate::token_filter::korean_number::{
97    KOREAN_NUMBER_TOKEN_FILTER_NAME, KoreanNumberTokenFilter,
98};
99use crate::token_filter::korean_reading_form::{
100    KOREAN_READING_FORM_TOKEN_FILTER_NAME, KoreanReadingFormTokenFilter,
101};
102use crate::token_filter::korean_stop_tags::{
103    KOREAN_STOP_TAGS_TOKEN_FILTER_NAME, KoreanStopTagsTokenFilter,
104};
105use crate::token_filter::length::{LENGTH_TOKEN_FILTER_NAME, LengthTokenFilter};
106use crate::token_filter::lowercase::{LOWERCASE_TOKEN_FILTER_NAME, LowercaseTokenFilter};
107use crate::token_filter::mapping::{MAPPING_TOKEN_FILTER_NAME, MappingTokenFilter};
108use crate::token_filter::remove_diacritical_mark::{
109    REMOVE_DIACRITICAL_TOKEN_FILTER_NAME, RemoveDiacriticalMarkTokenFilter,
110};
111use crate::token_filter::stop_words::{STOP_WORDS_TOKEN_FILTER_NAME, StopWordsTokenFilter};
112use crate::token_filter::uppercase::{UPPERCASE_TOKEN_FILTER_NAME, UppercaseTokenFilter};
113use crate::{LinderaErrorKind, LinderaResult};
114use lindera::token::Token;
115
116/// A trait for token filters that can be applied to a vector of tokens.
117///
118/// This trait requires the implementor to be `'static`, `Send`, `Sync`, and
119/// implement the `TokenFilterClone` trait. It provides methods to get the
120/// name of the filter and to apply the filter to a mutable vector of tokens.
121///
122/// # Required Methods
123///
124/// - `name`: Returns the name of the token filter as a static string slice.
125/// - `apply`: Applies the token filter to a mutable vector of tokens, returning
126///   a `LinderaResult<()>`.
127pub trait TokenFilter: 'static + Send + Sync + TokenFilterClone {
128    fn name(&self) -> &'static str;
129    fn apply(&self, tokens: &mut Vec<Token<'_>>) -> LinderaResult<()>;
130}
131
132/// A `BoxTokenFilter` is a wrapper around a boxed trait object that implements
133/// the `TokenFilter` trait. This allows for dynamic dispatch of different
134/// `TokenFilter` implementations at runtime. The `BoxTokenFilter` ensures that
135/// the contained `TokenFilter` is thread-safe (`Send` and `Sync`) and has a
136/// static lifetime.
137pub struct BoxTokenFilter(Box<dyn TokenFilter + 'static + Send + Sync>);
138
139impl Deref for BoxTokenFilter {
140    type Target = dyn TokenFilter;
141
142    fn deref(&self) -> &dyn TokenFilter {
143        &*self.0
144    }
145}
146
147impl<T: TokenFilter> From<T> for BoxTokenFilter {
148    fn from(token_filter: T) -> BoxTokenFilter {
149        BoxTokenFilter(Box::new(token_filter))
150    }
151}
152
153/// A trait for cloning token filters.
154///
155/// This trait provides a method `box_clone` which allows for cloning
156/// a token filter and returning it as a boxed trait object.
157pub trait TokenFilterClone {
158    fn box_clone(&self) -> BoxTokenFilter;
159}
160
161impl<T: TokenFilter + Clone + 'static> TokenFilterClone for T {
162    fn box_clone(&self) -> BoxTokenFilter {
163        BoxTokenFilter::from(self.clone())
164    }
165}
166
167pub struct TokenFilterLoader {}
168
169impl TokenFilterLoader {
170    pub fn load_from_value(kind: &str, value: &Value) -> LinderaResult<BoxTokenFilter> {
171        // Creates a `BoxTokenFilter` based on the provided `kind` and `value`.
172        //
173        // The function matches the `kind` against various predefined token filter names
174        // and constructs the corresponding token filter using the configuration derived
175        // from `value`. If the `kind` does not match any of the predefined names, an error
176        // is returned.
177        //
178        // # Parameters
179        // - `kind`: A string slice that specifies the type of token filter to create.
180        // - `value`: A `serde_json::Value` that contains the configuration for the token filter.
181        //
182        // # Returns
183        // - `Result<BoxTokenFilter, LinderaError>`: A boxed token filter if the `kind` is recognized,
184        //   otherwise an error indicating that the token filter is unsupported.
185        //
186        // # Errors
187        // - Returns `LinderaErrorKind::Deserialize` if the `kind` is not supported or if there is an
188        //   error in creating the token filter from the provided `value`.
189        let token_filter = match kind {
190            JAPANESE_BASE_FORM_TOKEN_FILTER_NAME => {
191                BoxTokenFilter::from(JapaneseBaseFormTokenFilter::from_config(value)?)
192            }
193            JAPANESE_COMPOUND_WORD_TOKEN_FILTER_NAME => {
194                BoxTokenFilter::from(JapaneseCompoundWordTokenFilter::from_config(value)?)
195            }
196            JAPANESE_KANA_TOKEN_FILTER_NAME => {
197                BoxTokenFilter::from(JapaneseKanaTokenFilter::from_config(value)?)
198            }
199            JAPANESE_KATAKANA_STEM_TOKEN_FILTER_NAME => {
200                BoxTokenFilter::from(JapaneseKatakanaStemTokenFilter::from_config(value)?)
201            }
202            JAPANESE_KEEP_TAGS_TOKEN_FILTER_NAME => {
203                BoxTokenFilter::from(JapaneseKeepTagsTokenFilter::from_config(value)?)
204            }
205            JAPANESE_NUMBER_TOKEN_FILTER_NAME => {
206                BoxTokenFilter::from(JapaneseNumberTokenFilter::from_config(value)?)
207            }
208            JAPANESE_READING_FORM_TOKEN_FILTER_NAME => {
209                BoxTokenFilter::from(JapaneseReadingFormTokenFilter::from_config(value)?)
210            }
211            JAPANESE_STOP_TAGS_TOKEN_FILTER_NAME => {
212                BoxTokenFilter::from(JapaneseStopTagsTokenFilter::from_config(value)?)
213            }
214            KEEP_WORDS_TOKEN_FILTER_NAME => {
215                BoxTokenFilter::from(KeepWordsTokenFilter::from_config(value)?)
216            }
217            KOREAN_COMPOUND_WORD_TOKEN_FILTER_NAME => {
218                BoxTokenFilter::from(KoreanCompoundWordTokenFilter::from_config(value)?)
219            }
220            KOREAN_KEEP_TAGS_TOKEN_FILTER_NAME => {
221                BoxTokenFilter::from(KoreanKeepTagsTokenFilter::from_config(value)?)
222            }
223            KOREAN_NUMBER_TOKEN_FILTER_NAME => {
224                BoxTokenFilter::from(KoreanNumberTokenFilter::from_config(value)?)
225            }
226            KOREAN_READING_FORM_TOKEN_FILTER_NAME => {
227                BoxTokenFilter::from(KoreanReadingFormTokenFilter::from_config(value)?)
228            }
229            KOREAN_STOP_TAGS_TOKEN_FILTER_NAME => {
230                BoxTokenFilter::from(KoreanStopTagsTokenFilter::from_config(value)?)
231            }
232            LENGTH_TOKEN_FILTER_NAME => {
233                BoxTokenFilter::from(LengthTokenFilter::from_config(value)?)
234            }
235            LOWERCASE_TOKEN_FILTER_NAME => {
236                BoxTokenFilter::from(LowercaseTokenFilter::from_config(value)?)
237            }
238            MAPPING_TOKEN_FILTER_NAME => {
239                BoxTokenFilter::from(MappingTokenFilter::from_config(value)?)
240            }
241            REMOVE_DIACRITICAL_TOKEN_FILTER_NAME => {
242                BoxTokenFilter::from(RemoveDiacriticalMarkTokenFilter::from_config(value)?)
243            }
244            STOP_WORDS_TOKEN_FILTER_NAME => {
245                BoxTokenFilter::from(StopWordsTokenFilter::from_config(value)?)
246            }
247            UPPERCASE_TOKEN_FILTER_NAME => {
248                BoxTokenFilter::from(UppercaseTokenFilter::from_config(value)?)
249            }
250            _ => {
251                return Err(LinderaErrorKind::Deserialize
252                    .with_error(anyhow::anyhow!("unsupported token filter: {kind}")));
253            }
254        };
255
256        Ok(token_filter)
257    }
258
259    /// Loads a token filter based on a CLI flag string.
260    ///
261    /// # Arguments
262    ///
263    /// * `cli_flag` - A string slice representing the command-line interface (CLI) flag used to specify the token filter. The flag typically contains both the filter kind and its arguments.
264    ///
265    /// # Returns
266    ///
267    /// Returns a `LinderaResult<BoxTokenFilter>`, which is a boxed token filter, or an error if the CLI flag is invalid or the filter configuration cannot be loaded.
268    ///
269    /// # Process
270    ///
271    /// 1. **Parse CLI flag**:
272    ///    - The `parse_cli_flag` function is called to extract the filter kind and its arguments from the `cli_flag` string.
273    /// 2. **Load filter from parsed values**:
274    ///    - The filter kind and arguments are passed to `load_from_value`, which constructs the appropriate token filter based on the parsed values.
275    ///
276    /// # Errors
277    ///
278    /// - If the CLI flag cannot be parsed, an error is returned.
279    /// - If the filter kind or its configuration is invalid, an error is returned during the filter loading process.
280    ///
281    /// # Details
282    ///
283    /// - The CLI flag is parsed into a filter kind and arguments. These are then used to load the appropriate token filter using the `load_from_value` function.
284    pub fn load_from_cli_flag(cli_flag: &str) -> LinderaResult<BoxTokenFilter> {
285        let (kind, args) = parse_cli_flag(cli_flag)?;
286
287        let character_filter = Self::load_from_value(kind, &args)?;
288
289        Ok(character_filter)
290    }
291}
lindera_analysis/token_filter.rs

lindera_analysis/
token_filter.rs
