lindera_analysis/
token_filter.rs

1/// This module defines various token filters and provides functionality to load them.
2///
3/// # Modules
4/// - `japanese_base_form`: Contains the Japanese base form token filter.
5/// - `japanese_compound_word`: Contains the Japanese compound word token filter.
6/// - `japanese_kana`: Contains the Japanese kana token filter.
7/// - `japanese_katakana_stem`: Contains the Japanese katakana stem token filter.
8/// - `japanese_keep_tags`: Contains the Japanese keep tags token filter.
9/// - `japanese_number`: Contains the Japanese number token filter.
10/// - `japanese_reading_form`: Contains the Japanese reading form token filter.
11/// - `japanese_stop_tags`: Contains the Japanese stop tags token filter.
12/// - `keep_words`: Contains the keep words token filter.
13/// - `korean_keep_tags`: Contains the Korean keep tags token filter.
14/// - `korean_reading_form`: Contains the Korean reading form token filter.
15/// - `korean_stop_tags`: Contains the Korean stop tags token filter.
16/// - `length`: Contains the length token filter.
17/// - `lowercase`: Contains the lowercase token filter.
18/// - `mapping`: Contains the mapping token filter.
19/// - `remove_diacritical_mark`: Contains the remove diacritical mark token filter.
20/// - `stop_words`: Contains the stop words token filter.
21/// - `uppercase`: Contains the uppercase token filter.
22///
23/// # Traits
24/// - `TokenFilter`: A trait for token filters that can be applied to a vector of tokens.
25/// - `TokenFilterClone`: A trait for cloning boxed token filters.
26///
27/// # Structs
28/// - `BoxTokenFilter`: A boxed token filter that implements `TokenFilter`.
29/// - `TokenFilterLoader`: A loader for creating token filters from configuration values.
30///
31/// # Usage
32/// The `TokenFilterLoader` struct provides methods to load token filters from configuration values
33/// or command-line flags. The `TokenFilter` trait defines the interface for token filters, and
34/// `BoxTokenFilter` is a boxed implementation of a token filter.
35pub mod japanese_base_form;
36pub mod japanese_compound_word;
37pub mod japanese_kana;
38pub mod japanese_katakana_stem;
39pub mod japanese_keep_tags;
40pub mod japanese_number;
41pub mod japanese_reading_form;
42pub mod japanese_stop_tags;
43pub mod keep_words;
44pub mod korean_keep_tags;
45pub mod korean_reading_form;
46pub mod korean_stop_tags;
47pub mod length;
48pub mod lowercase;
49pub mod mapping;
50pub mod remove_diacritical_mark;
51pub mod stop_words;
52mod tags;
53pub mod uppercase;
54
55use serde_json::Value;
56use std::ops::Deref;
57
58use crate::parse_cli_flag;
59use crate::token_filter::japanese_base_form::{
60    JAPANESE_BASE_FORM_TOKEN_FILTER_NAME, JapaneseBaseFormTokenFilter,
61};
62use crate::token_filter::japanese_compound_word::{
63    JAPANESE_COMPOUND_WORD_TOKEN_FILTER_NAME, JapaneseCompoundWordTokenFilter,
64};
65use crate::token_filter::japanese_kana::{
66    JAPANESE_KANA_TOKEN_FILTER_NAME, JapaneseKanaTokenFilter,
67};
68use crate::token_filter::japanese_katakana_stem::{
69    JAPANESE_KATAKANA_STEM_TOKEN_FILTER_NAME, JapaneseKatakanaStemTokenFilter,
70};
71use crate::token_filter::japanese_keep_tags::{
72    JAPANESE_KEEP_TAGS_TOKEN_FILTER_NAME, JapaneseKeepTagsTokenFilter,
73};
74use crate::token_filter::japanese_number::{
75    JAPANESE_NUMBER_TOKEN_FILTER_NAME, JapaneseNumberTokenFilter,
76};
77use crate::token_filter::japanese_reading_form::{
78    JAPANESE_READING_FORM_TOKEN_FILTER_NAME, JapaneseReadingFormTokenFilter,
79};
80use crate::token_filter::japanese_stop_tags::{
81    JAPANESE_STOP_TAGS_TOKEN_FILTER_NAME, JapaneseStopTagsTokenFilter,
82};
83use crate::token_filter::keep_words::{KEEP_WORDS_TOKEN_FILTER_NAME, KeepWordsTokenFilter};
84use crate::token_filter::korean_keep_tags::{
85    KOREAN_KEEP_TAGS_TOKEN_FILTER_NAME, KoreanKeepTagsTokenFilter,
86};
87use crate::token_filter::korean_reading_form::{
88    KOREAN_READING_FORM_TOKEN_FILTER_NAME, KoreanReadingFormTokenFilter,
89};
90use crate::token_filter::korean_stop_tags::{
91    KOREAN_STOP_TAGS_TOKEN_FILTER_NAME, KoreanStopTagsTokenFilter,
92};
93use crate::token_filter::length::{LENGTH_TOKEN_FILTER_NAME, LengthTokenFilter};
94use crate::token_filter::lowercase::{LOWERCASE_TOKEN_FILTER_NAME, LowercaseTokenFilter};
95use crate::token_filter::mapping::{MAPPING_TOKEN_FILTER_NAME, MappingTokenFilter};
96use crate::token_filter::remove_diacritical_mark::{
97    REMOVE_DIACRITICAL_TOKEN_FILTER_NAME, RemoveDiacriticalMarkTokenFilter,
98};
99use crate::token_filter::stop_words::{STOP_WORDS_TOKEN_FILTER_NAME, StopWordsTokenFilter};
100use crate::token_filter::uppercase::{UPPERCASE_TOKEN_FILTER_NAME, UppercaseTokenFilter};
101use crate::{LinderaErrorKind, LinderaResult};
102use lindera::token::Token;
103
104/// A trait for token filters that can be applied to a vector of tokens.
105///
106/// This trait requires the implementor to be `'static`, `Send`, `Sync`, and
107/// implement the `TokenFilterClone` trait. It provides methods to get the
108/// name of the filter and to apply the filter to a mutable vector of tokens.
109///
110/// # Required Methods
111///
112/// - `name`: Returns the name of the token filter as a static string slice.
113/// - `apply`: Applies the token filter to a mutable vector of tokens, returning
114///   a `LinderaResult<()>`.
115pub trait TokenFilter: 'static + Send + Sync + TokenFilterClone {
116    fn name(&self) -> &'static str;
117    fn apply(&self, tokens: &mut Vec<Token<'_>>) -> LinderaResult<()>;
118}
119
120/// A `BoxTokenFilter` is a wrapper around a boxed trait object that implements
121/// the `TokenFilter` trait. This allows for dynamic dispatch of different
122/// `TokenFilter` implementations at runtime. The `BoxTokenFilter` ensures that
123/// the contained `TokenFilter` is thread-safe (`Send` and `Sync`) and has a
124/// static lifetime.
125pub struct BoxTokenFilter(Box<dyn TokenFilter + 'static + Send + Sync>);
126
127impl Deref for BoxTokenFilter {
128    type Target = dyn TokenFilter;
129
130    fn deref(&self) -> &dyn TokenFilter {
131        &*self.0
132    }
133}
134
135impl<T: TokenFilter> From<T> for BoxTokenFilter {
136    fn from(token_filter: T) -> BoxTokenFilter {
137        BoxTokenFilter(Box::new(token_filter))
138    }
139}
140
141/// A trait for cloning token filters.
142///
143/// This trait provides a method `box_clone` which allows for cloning
144/// a token filter and returning it as a boxed trait object.
145pub trait TokenFilterClone {
146    fn box_clone(&self) -> BoxTokenFilter;
147}
148
149impl<T: TokenFilter + Clone + 'static> TokenFilterClone for T {
150    fn box_clone(&self) -> BoxTokenFilter {
151        BoxTokenFilter::from(self.clone())
152    }
153}
154
155pub struct TokenFilterLoader {}
156
157impl TokenFilterLoader {
158    pub fn load_from_value(kind: &str, value: &Value) -> LinderaResult<BoxTokenFilter> {
159        // Creates a `BoxTokenFilter` based on the provided `kind` and `value`.
160        //
161        // The function matches the `kind` against various predefined token filter names
162        // and constructs the corresponding token filter using the configuration derived
163        // from `value`. If the `kind` does not match any of the predefined names, an error
164        // is returned.
165        //
166        // # Parameters
167        // - `kind`: A string slice that specifies the type of token filter to create.
168        // - `value`: A `serde_json::Value` that contains the configuration for the token filter.
169        //
170        // # Returns
171        // - `Result<BoxTokenFilter, LinderaError>`: A boxed token filter if the `kind` is recognized,
172        //   otherwise an error indicating that the token filter is unsupported.
173        //
174        // # Errors
175        // - Returns `LinderaErrorKind::Deserialize` if the `kind` is not supported or if there is an
176        //   error in creating the token filter from the provided `value`.
177        let token_filter = match kind {
178            JAPANESE_BASE_FORM_TOKEN_FILTER_NAME => {
179                BoxTokenFilter::from(JapaneseBaseFormTokenFilter::from_config(value)?)
180            }
181            JAPANESE_COMPOUND_WORD_TOKEN_FILTER_NAME => {
182                BoxTokenFilter::from(JapaneseCompoundWordTokenFilter::from_config(value)?)
183            }
184            JAPANESE_KANA_TOKEN_FILTER_NAME => {
185                BoxTokenFilter::from(JapaneseKanaTokenFilter::from_config(value)?)
186            }
187            JAPANESE_KATAKANA_STEM_TOKEN_FILTER_NAME => {
188                BoxTokenFilter::from(JapaneseKatakanaStemTokenFilter::from_config(value)?)
189            }
190            JAPANESE_KEEP_TAGS_TOKEN_FILTER_NAME => {
191                BoxTokenFilter::from(JapaneseKeepTagsTokenFilter::from_config(value)?)
192            }
193            JAPANESE_NUMBER_TOKEN_FILTER_NAME => {
194                BoxTokenFilter::from(JapaneseNumberTokenFilter::from_config(value)?)
195            }
196            JAPANESE_READING_FORM_TOKEN_FILTER_NAME => {
197                BoxTokenFilter::from(JapaneseReadingFormTokenFilter::from_config(value)?)
198            }
199            JAPANESE_STOP_TAGS_TOKEN_FILTER_NAME => {
200                BoxTokenFilter::from(JapaneseStopTagsTokenFilter::from_config(value)?)
201            }
202            KEEP_WORDS_TOKEN_FILTER_NAME => {
203                BoxTokenFilter::from(KeepWordsTokenFilter::from_config(value)?)
204            }
205            KOREAN_KEEP_TAGS_TOKEN_FILTER_NAME => {
206                BoxTokenFilter::from(KoreanKeepTagsTokenFilter::from_config(value)?)
207            }
208            KOREAN_READING_FORM_TOKEN_FILTER_NAME => {
209                BoxTokenFilter::from(KoreanReadingFormTokenFilter::from_config(value)?)
210            }
211            KOREAN_STOP_TAGS_TOKEN_FILTER_NAME => {
212                BoxTokenFilter::from(KoreanStopTagsTokenFilter::from_config(value)?)
213            }
214            LENGTH_TOKEN_FILTER_NAME => {
215                BoxTokenFilter::from(LengthTokenFilter::from_config(value)?)
216            }
217            LOWERCASE_TOKEN_FILTER_NAME => {
218                BoxTokenFilter::from(LowercaseTokenFilter::from_config(value)?)
219            }
220            MAPPING_TOKEN_FILTER_NAME => {
221                BoxTokenFilter::from(MappingTokenFilter::from_config(value)?)
222            }
223            REMOVE_DIACRITICAL_TOKEN_FILTER_NAME => {
224                BoxTokenFilter::from(RemoveDiacriticalMarkTokenFilter::from_config(value)?)
225            }
226            STOP_WORDS_TOKEN_FILTER_NAME => {
227                BoxTokenFilter::from(StopWordsTokenFilter::from_config(value)?)
228            }
229            UPPERCASE_TOKEN_FILTER_NAME => {
230                BoxTokenFilter::from(UppercaseTokenFilter::from_config(value)?)
231            }
232            _ => {
233                return Err(LinderaErrorKind::Deserialize
234                    .with_error(anyhow::anyhow!("unsupported token filter: {kind}")));
235            }
236        };
237
238        Ok(token_filter)
239    }
240
241    /// Loads a token filter based on a CLI flag string.
242    ///
243    /// # Arguments
244    ///
245    /// * `cli_flag` - A string slice representing the command-line interface (CLI) flag used to specify the token filter. The flag typically contains both the filter kind and its arguments.
246    ///
247    /// # Returns
248    ///
249    /// Returns a `LinderaResult<BoxTokenFilter>`, which is a boxed token filter, or an error if the CLI flag is invalid or the filter configuration cannot be loaded.
250    ///
251    /// # Process
252    ///
253    /// 1. **Parse CLI flag**:
254    ///    - The `parse_cli_flag` function is called to extract the filter kind and its arguments from the `cli_flag` string.
255    /// 2. **Load filter from parsed values**:
256    ///    - The filter kind and arguments are passed to `load_from_value`, which constructs the appropriate token filter based on the parsed values.
257    ///
258    /// # Errors
259    ///
260    /// - If the CLI flag cannot be parsed, an error is returned.
261    /// - If the filter kind or its configuration is invalid, an error is returned during the filter loading process.
262    ///
263    /// # Details
264    ///
265    /// - The CLI flag is parsed into a filter kind and arguments. These are then used to load the appropriate token filter using the `load_from_value` function.
266    pub fn load_from_cli_flag(cli_flag: &str) -> LinderaResult<BoxTokenFilter> {
267        let (kind, args) = parse_cli_flag(cli_flag)?;
268
269        let character_filter = Self::load_from_value(kind, &args)?;
270
271        Ok(character_filter)
272    }
273}
lindera_analysis/token_filter.rs

lindera_analysis/
token_filter.rs
