lindera_analysis/
tokenizer.rs

1use std::borrow::Cow;
2use std::env;
3use std::fs::File;
4use std::io::Read;
5use std::path::Path;
6
7use serde_json::{Value, json};
8
9use crate::character_filter::{BoxCharacterFilter, CharacterFilterLoader, OffsetMapping};
10use crate::token_filter::{BoxTokenFilter, TokenFilterLoader};
11use lindera::LinderaResult;
12use lindera::dictionary::Lattice;
13use lindera::error::LinderaErrorKind;
14use lindera::mode::Mode;
15use lindera::segmenter::Segmenter;
16use lindera::token::Token;
17
18pub type TokenizerConfig = Value;
19
20fn yaml_to_config(file_path: &Path) -> LinderaResult<TokenizerConfig> {
21    let mut input_read = File::open(file_path).map_err(|err| {
22        LinderaErrorKind::Io.with_error(err).add_context(format!(
23            "Failed to open tokenizer config file: {}",
24            file_path.display()
25        ))
26    })?;
27
28    let mut buffer = Vec::new();
29    input_read.read_to_end(&mut buffer).map_err(|err| {
30        LinderaErrorKind::Io.with_error(err).add_context(format!(
31            "Failed to read tokenizer config file: {}",
32            file_path.display()
33        ))
34    })?;
35
36    match serde_yaml_ng::from_slice::<serde_yaml_ng::Value>(&buffer) {
37        Ok(value) => {
38            // Check if the value is a mapping.
39            match value {
40                serde_yaml_ng::Value::Mapping(_) => {
41                    Ok(serde_json::to_value(value).map_err(|err| {
42                        LinderaErrorKind::Deserialize
43                            .with_error(err)
44                            .add_context(format!(
45                                "Failed to convert YAML to JSON for config file: {}",
46                                file_path.display()
47                            ))
48                    })?)
49                }
50                _ => Err(LinderaErrorKind::Deserialize
51                    .with_error(anyhow::anyhow!("Invalid YAML"))
52                    .add_context(format!(
53                        "Config file must contain a YAML mapping: {}",
54                        file_path.display()
55                    ))),
56            }
57        }
58        Err(err) => Err(LinderaErrorKind::Deserialize
59            .with_error(err)
60            .add_context(format!(
61                "Failed to parse YAML config file: {}",
62                file_path.display()
63            ))),
64    }
65}
66
67/// Returns the default configuration as a `serde_json::Value`.
68fn empty_config() -> Value {
69    json!({
70        "segmenter": {},
71        "character_filters": [],
72        "token_filters": []
73    })
74}
75
76/// Ensures that the configuration contains the required keys with default values if absent.
77fn ensure_keys(mut config: Value) -> Value {
78    if config.get("segmenter").is_none() {
79        config["segmenter"] = json!({});
80    }
81
82    if config.get("character_filters").is_none() {
83        config["character_filters"] = json!([]);
84    }
85
86    if config.get("token_filters").is_none() {
87        config["token_filters"] = json!([]);
88    }
89
90    config
91}
92
93#[derive(Debug)]
94pub struct TokenizerBuilder {
95    config: TokenizerConfig,
96}
97
98impl TokenizerBuilder {
99    pub fn new() -> LinderaResult<Self> {
100        if let Ok(config_path) = env::var("LINDERA_CONFIG_PATH") {
101            Self::from_file(Path::new(&config_path))
102        } else {
103            Ok(Self {
104                config: empty_config(),
105            })
106        }
107    }
108
109    pub fn from_file(file_path: &Path) -> LinderaResult<Self> {
110        let config = yaml_to_config(file_path)?;
111
112        Ok(TokenizerBuilder {
113            config: ensure_keys(config),
114        })
115    }
116
117    pub fn from_config(config: TokenizerConfig) -> LinderaResult<Self> {
118        Ok(TokenizerBuilder {
119            config: ensure_keys(config),
120        })
121    }
122
123    pub fn set_segmenter_mode(&mut self, mode: &Mode) -> &mut Self {
124        self.config["segmenter"]["mode"] = json!(mode.as_str());
125        self
126    }
127
128    pub fn set_segmenter_dictionary(&mut self, uri: &str) -> &mut Self {
129        self.config["segmenter"]["dictionary"] = json!(uri);
130        self
131    }
132
133    pub fn set_segmenter_user_dictionary(&mut self, uri: &str) -> &mut Self {
134        self.config["segmenter"]["user_dictionary"] = json!(uri);
135        self
136    }
137
138    pub fn set_segmenter_keep_whitespace(&mut self, keep_whitespace: bool) -> &mut Self {
139        self.config["segmenter"]["keep_whitespace"] = json!(keep_whitespace);
140        self
141    }
142
143    /// Set whether to route filesystem-loaded dictionaries through
144    /// memory-mapped reads. Ignored for `embedded://` dictionaries.
145    ///
146    /// # Arguments
147    ///
148    /// * `use_mmap` - Whether to request memory-mapped dictionary loading.
149    ///
150    /// # Returns
151    ///
152    /// A mutable reference to `self`, for chaining.
153    pub fn set_segmenter_use_mmap(&mut self, use_mmap: bool) -> &mut Self {
154        self.config["segmenter"]["use_mmap"] = json!(use_mmap);
155        self
156    }
157
158    pub fn append_character_filter(&mut self, kind: &str, args: &Value) -> &mut Self {
159        if let Some(array) = self.config["character_filters"].as_array_mut() {
160            array.push(json!({ "kind": kind, "args": args }));
161        }
162        self
163    }
164
165    pub fn append_token_filter(&mut self, kind: &str, args: &Value) -> &mut Self {
166        if let Some(array) = self.config["token_filters"].as_array_mut() {
167            array.push(json!({ "kind": kind, "args": args }));
168        }
169        self
170    }
171
172    pub fn build(&self) -> LinderaResult<Tokenizer> {
173        Tokenizer::from_config(&self.config).map_err(|err| {
174            LinderaErrorKind::Parse.with_error(anyhow::anyhow!("failed to build tokenizer: {err}"))
175        })
176    }
177}
178
179pub struct Tokenizer {
180    /// Segmenter
181    /// The `segmenter` field is an instance of the `Segmenter` struct, which is responsible for
182    /// segmenting text into tokens. This is a core component of the tokenizer, enabling it to
183    /// break down input text into manageable and meaningful units for further processing.
184    pub segmenter: Segmenter,
185
186    /// Character filters
187    /// A vector of boxed character filters that will be applied to the input text
188    /// before tokenization. Each character filter is responsible for transforming
189    /// the input text in a specific way, such as normalizing characters or removing
190    /// unwanted characters.
191    pub character_filters: Vec<BoxCharacterFilter>,
192
193    /// Token filters
194    /// A vector of boxed token filters that will be applied to the tokens during tokenization.
195    /// Each token filter is a boxed trait object implementing the `TokenFilter` trait, allowing
196    /// for various transformations and processing steps to be applied to the tokens.
197    pub token_filters: Vec<BoxTokenFilter>,
198}
199
200impl Tokenizer {
201    /// Creates a new `Tokenizer` instance from a provided `Segmenter`.
202    ///
203    /// # Arguments
204    ///
205    /// * `segmenter` - An instance of the `Segmenter` struct, which is responsible for the core tokenization process.
206    ///
207    /// # Returns
208    ///
209    /// Returns a new `Tokenizer` instance that uses the provided `segmenter` for tokenization, with empty character and token filters.
210    ///
211    /// # Details
212    ///
213    /// - `segmenter`: The segmenter is responsible for handling the actual segmentation and tokenization of text. It is passed into the `Tokenizer` during initialization.
214    /// - `character_filters`: This is initialized as an empty vector and can be modified later to include character filters.
215    /// - `token_filters`: This is also initialized as an empty vector and can be modified later to include token filters.
216    pub fn new(segmenter: Segmenter) -> Self {
217        Self {
218            segmenter,
219            character_filters: Vec::new(),
220            token_filters: Vec::new(),
221        }
222    }
223
224    pub fn from_config(config: &TokenizerConfig) -> LinderaResult<Self> {
225        let segmenter_config = config.get("segmenter").ok_or_else(|| {
226            LinderaErrorKind::Deserialize.with_error(anyhow::anyhow!("missing segmenter config."))
227        })?;
228        let segmenter = Segmenter::from_config(segmenter_config)?;
229
230        // Create a tokenizer from the segmenter.
231        let mut tokenizer = Tokenizer::new(segmenter);
232
233        // Load character filter settings from the tokenizer config if it is not empty.
234        if let Some(character_filter_settings) = config["character_filters"].as_array() {
235            for character_filter_setting in character_filter_settings {
236                let character_filter_name = character_filter_setting["kind"].as_str();
237                if let Some(character_filter_name) = character_filter_name {
238                    // Append a character filter to the tokenizer.
239                    tokenizer.append_character_filter(CharacterFilterLoader::load_from_value(
240                        character_filter_name,
241                        &character_filter_setting["args"],
242                    )?);
243                }
244            }
245        }
246
247        // Load token filter settings from the tokenizer config if it is not empty.
248        if let Some(token_filter_settings) = config["token_filters"].as_array() {
249            for token_filter_setting in token_filter_settings {
250                let token_filter_name = token_filter_setting["kind"].as_str();
251                if let Some(token_filter_name) = token_filter_name {
252                    // Append a token filter to the tokenizer.
253                    tokenizer.append_token_filter(TokenFilterLoader::load_from_value(
254                        token_filter_name,
255                        &token_filter_setting["args"],
256                    )?);
257                }
258            }
259        }
260
261        Ok(tokenizer)
262    }
263
264    /// Appends a character filter to the tokenizer.
265    ///
266    /// # Arguments
267    ///
268    /// * `character_filter` - A `BoxCharacterFilter` that will be added to the tokenizer. This filter will be applied to the text during the tokenization process.
269    ///
270    /// # Returns
271    ///
272    /// Returns a mutable reference to `Self`, allowing for method chaining.
273    ///
274    /// # Details
275    ///
276    /// - This method adds a new character filter to the `Tokenizer`'s `character_filters` vector.
277    /// - It returns a mutable reference to `self`, allowing multiple character filters to be appended in a chain of method calls.
278    pub fn append_character_filter(&mut self, character_filter: BoxCharacterFilter) -> &mut Self {
279        self.character_filters.push(character_filter);
280
281        self
282    }
283
284    /// Appends a token filter to the tokenizer.
285    ///
286    /// # Arguments
287    ///
288    /// * `token_filter` - A `BoxTokenFilter` that will be added to the tokenizer. This filter will be applied to the tokens after they are segmented.
289    ///
290    /// # Returns
291    ///
292    /// Returns a mutable reference to `Self`, allowing for method chaining.
293    ///
294    /// # Details
295    ///
296    /// - This method adds a new token filter to the `Tokenizer`'s `token_filters` vector.
297    /// - It returns a mutable reference to `self`, allowing multiple token filters to be appended in a chain of method calls.
298    pub fn append_token_filter(&mut self, token_filter: BoxTokenFilter) -> &mut Self {
299        self.token_filters.push(token_filter);
300
301        self
302    }
303
304    /// Tokenizes the input text using the tokenizer's segmenter, character filters, and token filters.
305    ///
306    /// # Arguments
307    ///
308    /// * `text` - A reference to the input text (`&str`) that will be tokenized.
309    ///
310    /// # Returns
311    ///
312    /// Returns a `LinderaResult` containing a vector of `Token`s, where each `Token` represents a segment of the tokenized text.
313    ///
314    /// # Process
315    ///
316    /// 1. **Apply character filters**:
317    ///    - If any character filters are defined, they are applied to the input text before tokenization.
318    ///    - The `offsets`, `diffs`, and `text_len` are recorded for each character filter.
319    /// 2. **Segment the text**:
320    ///    - The `segmenter` divides the (potentially filtered) text into tokens.
321    /// 3. **Apply token filters**:
322    ///    - If any token filters are defined, they are applied to the segmented tokens.
323    /// 4. **Correct token offsets**:
324    ///    - If character filters were applied, the byte offsets of each token are corrected to account for changes introduced by those filters.
325    ///
326    /// # Errors
327    ///
328    /// - Returns an error if any of the character or token filters fail during processing.
329    /// - Returns an error if the segmentation process fails.
330    ///
331    /// # Details
332    ///
333    /// - `Cow<'a, str>` is used for the `normalized_text`, allowing the function to either borrow the original text or create an owned version if the text needs modification.
334    /// - If no character filters are applied, the original `text` is used as-is for segmentation.
335    /// - Token offsets are adjusted after the tokenization process if character filters were applied to ensure the byte positions of each token are accurate relative to the original text.
336    pub fn tokenize<'a>(&'a self, text: &'a str) -> LinderaResult<Vec<Token<'a>>> {
337        let mut lattice = Lattice::default();
338        self.tokenize_with_lattice(text, &mut lattice)
339    }
340
341    /// Tokenizes the input text using the tokenizer's segmenter, character filters, and token filters.
342    ///
343    /// # Arguments
344    ///
345    /// * `text` - A reference to the input text (`&str`) that will be tokenized.
346    /// * `lattice` - A mutable reference to a `Lattice` structure. This allows reusing the lattice across multiple calls to avoid memory allocation.
347    ///
348    /// # Returns
349    ///
350    /// Returns a `LinderaResult` containing a vector of `Token`s, where each `Token` represents a segment of the tokenized text.
351    ///
352    /// # Process
353    ///
354    /// 1. **Apply character filters**:
355    ///    - If any character filters are defined, they are applied to the input text before tokenization.
356    ///    - The `offsets`, `diffs`, and `text_len` are recorded for each character filter.
357    /// 2. **Segment the text**:
358    ///    - The `segmenter` divides the (potentially filtered) text into tokens.
359    /// 3. **Apply token filters**:
360    ///    - If any token filters are defined, they are applied to the segmented tokens.
361    /// 4. **Correct token offsets**:
362    ///    - If character filters were applied, the byte offsets of each token are corrected to account for changes introduced by those filters.
363    ///
364    /// # Errors
365    ///
366    /// - Returns an error if any of the character or token filters fail during processing.
367    /// - Returns an error if the segmentation process fails.
368    ///
369    /// # Details
370    ///
371    /// - `Cow<'a, str>` is used for the `normalized_text`, allowing the function to either borrow the original text or create an owned version if the text needs modification.
372    /// - If no character filters are applied, the original `text` is used as-is for segmentation.
373    /// - Token offsets are adjusted after the tokenization process if character filters were applied to ensure the byte positions of each token are accurate relative to the original text.
374    pub fn tokenize_with_lattice<'a>(
375        &'a self,
376        text: &'a str,
377        lattice: &mut Lattice,
378    ) -> LinderaResult<Vec<Token<'a>>> {
379        let mut normalized_text: Cow<'a, str> = Cow::Borrowed(text);
380
381        let mut offset_mappings: Vec<OffsetMapping> =
382            Vec::with_capacity(self.character_filters.len());
383
384        // Apply character filters to the text if it is not empty.
385        // Optimize: Only convert to mutable when we have filters to apply
386        if !self.character_filters.is_empty() {
387            // Convert to owned string once for all filters
388            let text_mut = normalized_text.to_mut();
389
390            for character_filter in &self.character_filters {
391                let mapping = character_filter.apply(text_mut)?;
392
393                if !mapping.is_empty() {
394                    // Record the offset mapping of each character filter in reverse order
395                    // since we need to apply corrections in reverse order
396                    offset_mappings.push(mapping);
397                }
398            }
399        }
400
401        // Store the final text length for offset correction
402        let final_text_len = normalized_text.len();
403
404        // Segment a text.
405        // Segment a text.
406        let mut tokens = self
407            .segmenter
408            .segment_with_lattice(normalized_text, lattice)?;
409
410        // Apply token filters to the tokens if they are not empty.
411        for token_filter in &self.token_filters {
412            token_filter.apply(&mut tokens)?;
413        }
414
415        // Correct token offsets if character filters are applied.
416        // Apply corrections in reverse order (last filter first)
417        if !offset_mappings.is_empty() {
418            for token in tokens.iter_mut() {
419                // Apply corrections in reverse order to undo the transformations
420                for mapping in offset_mappings.iter().rev() {
421                    // Override start.
422                    token.byte_start = mapping.correct_offset(token.byte_start, final_text_len);
423                    // Override end.
424                    token.byte_end = mapping.correct_offset(token.byte_end, final_text_len);
425                }
426            }
427        }
428
429        Ok(tokens)
430    }
431
432    /// Tokenizes the input text and returns the top-N results.
433    ///
434    /// Each result is a `Vec<Token>` with character/token filters applied.
435    /// Results are ordered by cost (best first).
436    pub fn tokenize_nbest<'a>(
437        &'a self,
438        text: &'a str,
439        n: usize,
440        unique: bool,
441        cost_threshold: Option<i64>,
442    ) -> LinderaResult<Vec<(Vec<Token<'a>>, i64)>> {
443        let mut lattice = Lattice::default();
444        self.tokenize_nbest_with_lattice(text, &mut lattice, n, unique, cost_threshold)
445    }
446
447    /// Tokenizes the input text and returns the top-N results with costs.
448    /// Each result is a (tokens, cost) pair.
449    /// If `unique` is true, results with the same word boundaries are deduplicated.
450    /// If `cost_threshold` is Some(t), paths whose cost exceeds best_cost + t
451    /// are discarded.
452    pub fn tokenize_nbest_with_lattice<'a>(
453        &'a self,
454        text: &'a str,
455        lattice: &mut Lattice,
456        n: usize,
457        unique: bool,
458        cost_threshold: Option<i64>,
459    ) -> LinderaResult<Vec<(Vec<Token<'a>>, i64)>> {
460        let mut normalized_text: Cow<'a, str> = Cow::Borrowed(text);
461
462        let mut offset_mappings: Vec<OffsetMapping> =
463            Vec::with_capacity(self.character_filters.len());
464
465        if !self.character_filters.is_empty() {
466            let text_mut = normalized_text.to_mut();
467            for character_filter in &self.character_filters {
468                let mapping = character_filter.apply(text_mut)?;
469                if !mapping.is_empty() {
470                    offset_mappings.push(mapping);
471                }
472            }
473        }
474
475        let final_text_len = normalized_text.len();
476
477        let mut all_results = self.segmenter.segment_nbest_with_lattice(
478            normalized_text,
479            lattice,
480            n,
481            unique,
482            cost_threshold,
483        )?;
484
485        // Apply token filters and offset corrections to each result
486        for (tokens, _cost) in &mut all_results {
487            for token_filter in &self.token_filters {
488                token_filter.apply(tokens)?;
489            }
490
491            if !offset_mappings.is_empty() {
492                for token in tokens.iter_mut() {
493                    for mapping in offset_mappings.iter().rev() {
494                        token.byte_start = mapping.correct_offset(token.byte_start, final_text_len);
495                        token.byte_end = mapping.correct_offset(token.byte_end, final_text_len);
496                    }
497                }
498            }
499        }
500
501        Ok(all_results)
502    }
503}
504
505impl Clone for Tokenizer {
506    /// Creates a deep clone of the `Tokenizer` instance, including all character filters, token filters, and the segmenter.
507    ///
508    /// # Returns
509    ///
510    /// Returns a new `Tokenizer` instance that is a deep clone of the current instance. All internal filters and the segmenter are cloned.
511    ///
512    /// # Details
513    ///
514    /// - **Character Filters**: Each character filter is cloned by calling its `box_clone` method, which ensures that any dynamically dispatched filters are properly cloned.
515    /// - **Token Filters**: Similarly, each token filter is cloned using the `box_clone` method to handle dynamic dispatch.
516    /// - **Segmenter**: The segmenter is cloned using its `clone` method.
517    ///
518    /// # Notes
519    ///
520    /// - This method performs deep cloning, meaning that all internal filters and segmenter instances are fully duplicated.
521    /// - The `box_clone` method is used to clone the dynamically dispatched filter objects (`BoxCharacterFilter` and `BoxTokenFilter`).
522    fn clone(&self) -> Self {
523        let character_filters: Vec<BoxCharacterFilter> = self
524            .character_filters
525            .iter()
526            .map(|filter| filter.box_clone())
527            .collect();
528
529        let token_filters: Vec<BoxTokenFilter> = self
530            .token_filters
531            .iter()
532            .map(|filter| filter.box_clone())
533            .collect();
534
535        Tokenizer {
536            character_filters,
537            segmenter: self.segmenter.clone(),
538            token_filters,
539        }
540    }
541}
542
543#[cfg(test)]
544mod tests {
545    use super::TokenizerBuilder;
546
547    #[test]
548    fn test_set_segmenter_use_mmap_writes_flat_key_under_segmenter() {
549        let mut builder = TokenizerBuilder::new().unwrap();
550        builder.set_segmenter_use_mmap(true);
551
552        assert_eq!(builder.config["segmenter"]["use_mmap"], true);
553    }
554
555    #[cfg(feature = "embed-ipadic")]
556    #[test]
557    fn test_tokenizer_config_from_slice() {
558        use std::path::PathBuf;
559
560        use crate::tokenizer::yaml_to_config;
561
562        let config_file = PathBuf::from(env!("CARGO_MANIFEST_DIR"))
563            .join("../resources")
564            .join("config")
565            .join("lindera.yml");
566
567        let result = yaml_to_config(&config_file);
568
569        assert!(result.is_ok());
570    }
571
572    #[test]
573    #[cfg(feature = "embed-ipadic")]
574    fn test_tokenizer_config_clone() {
575        use std::path::PathBuf;
576
577        use crate::tokenizer::yaml_to_config;
578
579        let config_file = PathBuf::from(env!("CARGO_MANIFEST_DIR"))
580            .join("../resources")
581            .join("config")
582            .join("lindera.yml");
583
584        let tokenizer_config = yaml_to_config(&config_file).unwrap();
585
586        let cloned_tokenizer_config = tokenizer_config.clone();
587
588        assert_eq!(tokenizer_config, cloned_tokenizer_config);
589    }
590
591    #[test]
592    #[cfg(feature = "embed-ipadic")]
593    fn test_tokenize_ipadic() {
594        use std::borrow::Cow;
595        use std::path::PathBuf;
596
597        use crate::tokenizer::TokenizerBuilder;
598
599        let config_file = PathBuf::from(env!("CARGO_MANIFEST_DIR"))
600            .join("../resources")
601            .join("config")
602            .join("lindera.yml");
603
604        let builder = TokenizerBuilder::from_file(&config_file).unwrap();
605
606        let tokenizer = builder.build().unwrap();
607
608        {
609            let text = "ﾘﾝﾃﾞﾗは形態素解析ｴﾝｼﾞﾝです。";
610            let mut tokens = tokenizer.tokenize(text).unwrap();
611            let mut tokens_iter = tokens.iter_mut();
612            {
613                let token = tokens_iter.next().unwrap();
614                assert_eq!(token.surface, Cow::Borrowed("Lindera"));
615                assert_eq!(token.byte_start, 0);
616                assert_eq!(token.byte_end, 15);
617                assert_eq!(token.position, 0);
618                assert_eq!(token.position_length, 1);
619                assert!(token.word_id.is_unknown());
620                assert_eq!(
621                    token.details,
622                    Some(vec![
623                        Cow::Borrowed("名詞"),
624                        Cow::Borrowed("固有名詞"),
625                        Cow::Borrowed("組織"),
626                        Cow::Borrowed("*"),
627                        Cow::Borrowed("*"),
628                        Cow::Borrowed("*"),
629                        Cow::Borrowed("*"),
630                        Cow::Borrowed("*"),
631                        Cow::Borrowed("*"),
632                    ])
633                );
634            }
635            {
636                let token = tokens_iter.next().unwrap();
637                assert_eq!(token.surface, Cow::Borrowed("形態素"));
638                assert_eq!(token.byte_start, 18);
639                assert_eq!(token.byte_end, 27);
640                assert_eq!(token.position, 2);
641                assert_eq!(token.position_length, 1);
642                assert_eq!(
643                    token.details,
644                    Some(vec![
645                        Cow::Borrowed("名詞"),
646                        Cow::Borrowed("一般"),
647                        Cow::Borrowed("*"),
648                        Cow::Borrowed("*"),
649                        Cow::Borrowed("*"),
650                        Cow::Borrowed("*"),
651                        Cow::Borrowed("形態素"),
652                        Cow::Borrowed("ケイタイソ"),
653                        Cow::Borrowed("ケイタイソ"),
654                    ])
655                );
656            }
657            {
658                let token = tokens_iter.next().unwrap();
659                assert_eq!(token.surface, Cow::Borrowed("解析"));
660                assert_eq!(token.byte_start, 27);
661                assert_eq!(token.byte_end, 33);
662                assert_eq!(token.position, 3);
663                assert_eq!(token.position_length, 1);
664                assert_eq!(
665                    token.details,
666                    Some(vec![
667                        Cow::Borrowed("名詞"),
668                        Cow::Borrowed("サ変接続"),
669                        Cow::Borrowed("*"),
670                        Cow::Borrowed("*"),
671                        Cow::Borrowed("*"),
672                        Cow::Borrowed("*"),
673                        Cow::Borrowed("解析"),
674                        Cow::Borrowed("カイセキ"),
675                        Cow::Borrowed("カイセキ"),
676                    ])
677                );
678            }
679            {
680                let token = tokens_iter.next().unwrap();
681                assert_eq!(token.surface, Cow::Borrowed("エンジン"));
682                assert_eq!(token.byte_start, 33);
683                assert_eq!(token.byte_end, 48);
684                assert_eq!(token.position, 4);
685                assert_eq!(token.position_length, 1);
686                assert_eq!(
687                    token.details,
688                    Some(vec![
689                        Cow::Borrowed("名詞"),
690                        Cow::Borrowed("一般"),
691                        Cow::Borrowed("*"),
692                        Cow::Borrowed("*"),
693                        Cow::Borrowed("*"),
694                        Cow::Borrowed("*"),
695                        Cow::Borrowed("エンジン"),
696                        Cow::Borrowed("エンジン"),
697                        Cow::Borrowed("エンジン"),
698                    ])
699                );
700            }
701
702            let mut tokens_iter = tokens.iter();
703            {
704                let token = tokens_iter.next().unwrap();
705                let start = token.byte_start;
706                let end = token.byte_end;
707                assert_eq!(token.surface, Cow::Borrowed("Lindera"));
708                assert_eq!(&text[start..end], "ﾘﾝﾃﾞﾗ");
709            }
710        }
711
712        {
713            let text = "１０㌎のｶﾞｿﾘﾝ";
714            let mut tokens = tokenizer.tokenize(text).unwrap();
715            let mut tokens_iter = tokens.iter_mut();
716            {
717                // "10" (unknown NUMERIC) and "ガロン" (名詞,接尾,助数詞) are merged
718                // by the japanese_compound_word filter into "10ガロン" with tag "名詞,数".
719                let token = tokens_iter.next().unwrap();
720                assert_eq!(token.surface, Cow::Owned::<str>("10ガロン".into()));
721                assert_eq!(token.byte_start, 0);
722                assert_eq!(token.byte_end, 9);
723                assert_eq!(token.position, 0);
724                assert_eq!(token.position_length, 2);
725            }
726            {
727                let token = tokens_iter.next().unwrap();
728                assert_eq!(token.surface, Cow::Borrowed("ガソリン"));
729                assert_eq!(token.byte_start, 12);
730                assert_eq!(token.byte_end, 27);
731                assert_eq!(token.position, 3);
732                assert_eq!(token.position_length, 1);
733                assert_eq!(
734                    token.details,
735                    Some(vec![
736                        Cow::Borrowed("名詞"),
737                        Cow::Borrowed("一般"),
738                        Cow::Borrowed("*"),
739                        Cow::Borrowed("*"),
740                        Cow::Borrowed("*"),
741                        Cow::Borrowed("*"),
742                        Cow::Borrowed("ガソリン"),
743                        Cow::Borrowed("ガソリン"),
744                        Cow::Borrowed("ガソリン"),
745                    ])
746                );
747            }
748
749            let mut tokens_iter = tokens.iter();
750            {
751                // "10" and "ガロン" are merged by the japanese_compound_word filter
752                let token = tokens_iter.next().unwrap();
753                let start = token.byte_start;
754                let end = token.byte_end;
755                assert_eq!(token.surface, Cow::Owned::<str>("10ガロン".into()));
756                assert_eq!(&text[start..end], "１０㌎");
757            }
758            {
759                let token = tokens_iter.next().unwrap();
760                let start = token.byte_start;
761                let end = token.byte_end;
762                assert_eq!(token.surface, Cow::Borrowed("ガソリン"));
763                assert_eq!(&text[start..end], "ｶﾞｿﾘﾝ");
764            }
765        }
766
767        {
768            let text = "お釣りは百三十四円です。";
769            let mut tokens = tokenizer.tokenize(text).unwrap();
770            let mut tokens_iter = tokens.iter_mut();
771            {
772                let token = tokens_iter.next().unwrap();
773                assert_eq!(token.surface, Cow::Borrowed("お釣り"));
774                assert_eq!(token.byte_start, 0);
775                assert_eq!(token.byte_end, 9);
776                assert_eq!(token.position, 0);
777                assert_eq!(token.position_length, 1);
778                assert_eq!(
779                    token.details,
780                    Some(vec![
781                        Cow::Borrowed("名詞"),
782                        Cow::Borrowed("一般"),
783                        Cow::Borrowed("*"),
784                        Cow::Borrowed("*"),
785                        Cow::Borrowed("*"),
786                        Cow::Borrowed("*"),
787                        Cow::Borrowed("お釣り"),
788                        Cow::Borrowed("オツリ"),
789                        Cow::Borrowed("オツリ"),
790                    ])
791                );
792            }
793            {
794                let token = tokens_iter.next().unwrap();
795                assert_eq!(token.surface, Cow::Borrowed("134円"));
796                assert_eq!(token.byte_start, 12);
797                assert_eq!(token.byte_end, 27);
798                assert_eq!(token.position, 2);
799                assert_eq!(token.position_length, 5);
800                assert_eq!(
801                    token.details,
802                    Some(vec![
803                        Cow::Borrowed("名詞"),
804                        Cow::Borrowed("数"),
805                        Cow::Borrowed("*"),
806                        Cow::Borrowed("*"),
807                        Cow::Borrowed("*"),
808                        Cow::Borrowed("*"),
809                        Cow::Borrowed("*"),
810                        Cow::Borrowed("*"),
811                        Cow::Borrowed("*"),
812                    ])
813                );
814            }
815        }
816
817        {
818            let text = "ここは騒々しい";
819            let mut tokens = tokenizer.tokenize(text).unwrap();
820            let mut tokens_iter = tokens.iter_mut();
821            {
822                let token = tokens_iter.next().unwrap();
823                assert_eq!(token.surface, Cow::Borrowed("ここ"));
824                assert_eq!(token.byte_start, 0);
825                assert_eq!(token.byte_end, 6);
826                assert_eq!(token.position, 0);
827                assert_eq!(token.position_length, 1);
828                assert_eq!(
829                    token.details,
830                    Some(vec![
831                        Cow::Borrowed("名詞"),
832                        Cow::Borrowed("代名詞"),
833                        Cow::Borrowed("一般"),
834                        Cow::Borrowed("*"),
835                        Cow::Borrowed("*"),
836                        Cow::Borrowed("*"),
837                        Cow::Borrowed("ここ"),
838                        Cow::Borrowed("ココ"),
839                        Cow::Borrowed("ココ"),
840                    ])
841                );
842            }
843            {
844                let token = tokens_iter.next().unwrap();
845                assert_eq!(token.surface, Cow::Borrowed("騒騒しい"));
846                assert_eq!(token.byte_start, 9);
847                assert_eq!(token.byte_end, 21);
848                assert_eq!(token.position, 2);
849                assert_eq!(token.position_length, 1);
850                assert_eq!(
851                    token.details,
852                    Some(vec![
853                        Cow::Borrowed("形容詞"),
854                        Cow::Borrowed("自立"),
855                        Cow::Borrowed("*"),
856                        Cow::Borrowed("*"),
857                        Cow::Borrowed("形容詞・イ段"),
858                        Cow::Borrowed("基本形"),
859                        Cow::Borrowed("騒騒しい"),
860                        Cow::Borrowed("ソウゾウシイ"),
861                        Cow::Borrowed("ソーゾーシイ"),
862                    ])
863                );
864            }
865        }
866    }
867
868    #[test]
869    #[cfg(not(windows))]
870    #[should_panic(expected = "No such file or directory")]
871    fn test_create_tokenizer_builder_from_non_existent_file() {
872        use std::path::PathBuf;
873
874        use crate::tokenizer::TokenizerBuilder;
875
876        let config_file = PathBuf::from(env!("CARGO_MANIFEST_DIR"))
877            .join("../resources")
878            .join("config")
879            .join("non_existent_file.yml");
880
881        TokenizerBuilder::from_file(&config_file).unwrap();
882    }
883
884    #[test]
885    #[cfg(windows)]
886    #[should_panic(expected = "The system cannot find the file specified.")]
887    fn test_create_tokenizer_builder_from_non_existent_file() {
888        use std::path::PathBuf;
889
890        use crate::tokenizer::TokenizerBuilder;
891
892        let config_file = PathBuf::from(env!("CARGO_MANIFEST_DIR"))
893            .join("../resources")
894            .join("config")
895            .join("non_existent_file.yml");
896
897        TokenizerBuilder::from_file(&config_file).unwrap();
898    }
899
900    #[test]
901    #[should_panic(expected = "Invalid YAML")]
902    fn test_create_tokenizer_builder_from_invalid_file() {
903        use std::path::PathBuf;
904
905        use crate::tokenizer::TokenizerBuilder;
906
907        let config_file = PathBuf::from(env!("CARGO_MANIFEST_DIR"))
908            .join("../resources")
909            .join("config")
910            .join("invalid.yml");
911
912        TokenizerBuilder::from_file(&config_file).unwrap();
913    }
914
915    #[test]
916    #[cfg(feature = "embed-ipadic")]
917    fn test_tokenize_nbest_1best_matches_tokenize() {
918        use crate::tokenizer::TokenizerBuilder;
919
920        let mut builder = TokenizerBuilder::new().unwrap();
921        builder.set_segmenter_dictionary("embedded://ipadic");
922
923        let tokenizer = builder.build().unwrap();
924
925        let text = "すもももももももものうち";
926        let normal_tokens = tokenizer.tokenize(text).unwrap();
927        let nbest_results = tokenizer.tokenize_nbest(text, 1, false, None).unwrap();
928
929        assert_eq!(nbest_results.len(), 1);
930        let (nbest_tokens, _cost) = &nbest_results[0];
931        assert_eq!(normal_tokens.len(), nbest_tokens.len());
932        for (normal, nbest) in normal_tokens.iter().zip(nbest_tokens.iter()) {
933            assert_eq!(normal.surface.as_ref(), nbest.surface.as_ref());
934        }
935    }
936
937    #[test]
938    #[cfg(feature = "embed-ipadic")]
939    fn test_tokenize_nbest_multiple_results() {
940        use crate::tokenizer::TokenizerBuilder;
941
942        let mut builder = TokenizerBuilder::new().unwrap();
943        builder.set_segmenter_dictionary("embedded://ipadic");
944
945        let tokenizer = builder.build().unwrap();
946
947        let text = "すもももももももものうち";
948        let results = tokenizer.tokenize_nbest(text, 5, false, None).unwrap();
949
950        // Should return multiple results for ambiguous text
951        assert!(results.len() >= 2);
952
953        // All results should cover the full text
954        for (tokens, _cost) in &results {
955            assert!(!tokens.is_empty());
956        }
957    }
958}
lindera_analysis/tokenizer.rs

lindera_analysis/
tokenizer.rs
