lindera_analysis/character_filter/
regex.rs

1use regex::Regex;
2use serde_json::Value;
3
4use crate::character_filter::{CharacterFilter, OffsetMapping, Transformation};
5use lindera::LinderaResult;
6use lindera::error::LinderaErrorKind;
7
8pub const REGEX_CHARACTER_FILTER_NAME: &str = "regex";
9
10pub type RegexCharacterFilterConfig = Value;
11
12/// Character filter that uses a regular expression for the target of replace string.
13///
14#[derive(Clone, Debug)]
15pub struct RegexCharacterFilter {
16    replacement: String,
17    regex: Regex,
18}
19
20impl RegexCharacterFilter {
21    pub fn new(pattern: &str, replacement: &str) -> LinderaResult<Self> {
22        let regex = Regex::new(pattern).map_err(|err| LinderaErrorKind::Args.with_error(err))?;
23
24        Ok(Self {
25            replacement: replacement.to_string(),
26            regex,
27        })
28    }
29
30    pub fn from_config(config: &RegexCharacterFilterConfig) -> LinderaResult<Self> {
31        let pattern = config
32            .get("pattern")
33            .ok_or_else(|| {
34                LinderaErrorKind::Deserialize.with_error(anyhow::anyhow!("missing pattern config."))
35            })?
36            .as_str()
37            .ok_or_else(|| {
38                LinderaErrorKind::Deserialize.with_error(anyhow::anyhow!("invalid pattern config."))
39            })?;
40
41        let replacement = config
42            .get("replacement")
43            .ok_or_else(|| {
44                LinderaErrorKind::Deserialize
45                    .with_error(anyhow::anyhow!("missing replacement config."))
46            })?
47            .as_str()
48            .ok_or_else(|| {
49                LinderaErrorKind::Deserialize
50                    .with_error(anyhow::anyhow!("invalid replacement config."))
51            })?;
52
53        Self::new(pattern, replacement)
54    }
55}
56
57impl CharacterFilter for RegexCharacterFilter {
58    fn name(&self) -> &'static str {
59        REGEX_CHARACTER_FILTER_NAME
60    }
61
62    /// Apply the filter using the OffsetMapping API
63    fn apply(&self, text: &mut String) -> LinderaResult<OffsetMapping> {
64        let mut filtered_text = String::with_capacity(text.len());
65        let mut mapping = OffsetMapping::new();
66        let mut last_match_end = 0;
67
68        for mat in self.regex.find_iter(text) {
69            let input_start = mat.start();
70            let input_len = mat.len();
71            let replacement_text = self.replacement.as_str();
72            let replacement_len = replacement_text.len();
73
74            // Append the text before the match
75            filtered_text.push_str(&text[last_match_end..input_start]);
76
77            // Record transformation if text changed
78            if input_len != replacement_len {
79                let transformation = Transformation::new(
80                    input_start,
81                    input_start + input_len,
82                    filtered_text.len(),
83                    filtered_text.len() + replacement_len,
84                );
85                mapping.add_transformation(transformation);
86            }
87
88            // Apply the replacement
89            filtered_text.push_str(replacement_text);
90
91            last_match_end = input_start + input_len;
92        }
93
94        // Append the remaining text after the last match
95        filtered_text.push_str(&text[last_match_end..]);
96
97        *text = filtered_text;
98        Ok(mapping)
99    }
100}
101
102#[cfg(test)]
103mod tests {
104    use crate::character_filter::CharacterFilter;
105    use crate::character_filter::regex::{RegexCharacterFilter, RegexCharacterFilterConfig};
106
107    #[test]
108    fn test_regex_character_filter_config() {
109        let config_str = r#"
110        {
111            "pattern": "リンデラ",
112            "replacement": "Lindera"
113        }
114        "#;
115        let result: Result<RegexCharacterFilterConfig, _> = serde_json::from_str(config_str);
116        assert!(result.is_ok());
117    }
118
119    #[test]
120    fn test_regex_character_filter_from_config() {
121        let config_str = r#"
122        {
123            "pattern": "リンデラ",
124            "replacement": "Lindera"
125        }
126        "#;
127        let config: RegexCharacterFilterConfig = serde_json::from_str(config_str).unwrap();
128
129        let result = RegexCharacterFilter::from_config(&config);
130        assert!(result.is_ok());
131    }
132
133    #[test]
134    fn test_regex_character_filter_apply() {
135        {
136            let config_str = r#"
137            {
138                "pattern": "リンデラ",
139                "replacement": "Lindera"
140            }
141            "#;
142            let config: RegexCharacterFilterConfig = serde_json::from_str(config_str).unwrap();
143
144            let filter = RegexCharacterFilter::from_config(&config).unwrap();
145            let original_text = "リンデラは形態素解析器です。";
146            let mut text = original_text.to_string();
147            let mapping = filter.apply(&mut text).unwrap();
148            assert_eq!("Linderaは形態素解析器です。", text.as_str());
149
150            // Verify transformation: "リンデラ"(0-12) → "Lindera"(0-7)
151            assert_eq!(1, mapping.transformations.len());
152            let transform = &mapping.transformations[0];
153            assert_eq!(0, transform.original_start);
154            assert_eq!(12, transform.original_end);
155            assert_eq!(0, transform.filtered_start);
156            assert_eq!(7, transform.filtered_end);
157
158            // Test text fragments
159            let start = 0;
160            let end = 7;
161            assert_eq!("Lindera", &text[start..end]);
162            let correct_start = mapping.correct_offset(start, text.len());
163            let correct_end = mapping.correct_offset(end, text.len());
164            assert_eq!(0, correct_start);
165            assert_eq!(12, correct_end);
166            assert_eq!("リンデラ", &original_text[correct_start..correct_end]);
167        }
168
169        {
170            let config_str = r#"
171            {
172                "pattern": "\\s{2,}",
173                "replacement": " "
174            }
175            "#;
176            let config: RegexCharacterFilterConfig = serde_json::from_str(config_str).unwrap();
177
178            let filter = RegexCharacterFilter::from_config(&config).unwrap();
179            let original_text = "a     b     c";
180            let mut text = original_text.to_string();
181            let mapping = filter.apply(&mut text).unwrap();
182            assert_eq!("a b c", text.as_str());
183
184            // Verify transformations: two groups of spaces compressed
185            assert_eq!(2, mapping.transformations.len());
186            let transform1 = &mapping.transformations[0];
187            assert_eq!(1, transform1.original_start);
188            assert_eq!(6, transform1.original_end);
189            assert_eq!(1, transform1.filtered_start);
190            assert_eq!(2, transform1.filtered_end);
191
192            let transform2 = &mapping.transformations[1];
193            assert_eq!(7, transform2.original_start);
194            assert_eq!(12, transform2.original_end);
195            assert_eq!(3, transform2.filtered_start);
196            assert_eq!(4, transform2.filtered_end);
197
198            // Test text fragments
199            let start = 2;
200            let end = 3;
201            assert_eq!("b", &text[start..end]);
202            let correct_start = mapping.correct_offset(start, text.len());
203            let correct_end = mapping.correct_offset(end, text.len());
204            assert_eq!(6, correct_start);
205            assert_eq!(7, correct_end);
206            assert_eq!("b", &original_text[correct_start..correct_end]);
207        }
208    }
209}
lindera_analysis/character_filter/regex.rs

lindera_analysis/character_filter/
regex.rs
