lindera_analysis/
tokenizer.rs

1use std::borrow::Cow;
2use std::env;
3use std::fs::File;
4use std::io::Read;
5use std::path::Path;
6
7use serde_json::{Value, json};
8
9use crate::character_filter::{BoxCharacterFilter, CharacterFilterLoader, OffsetMapping};
10use crate::token_filter::{BoxTokenFilter, TokenFilterLoader};
11use lindera::LinderaResult;
12use lindera::dictionary::Lattice;
13use lindera::error::LinderaErrorKind;
14use lindera::mode::Mode;
15use lindera::segmenter::Segmenter;
16use lindera::token::Token;
17
18pub type TokenizerConfig = Value;
19
20fn yaml_to_config(file_path: &Path) -> LinderaResult<TokenizerConfig> {
21    let mut input_read = File::open(file_path).map_err(|err| {
22        LinderaErrorKind::Io.with_error(err).add_context(format!(
23            "Failed to open tokenizer config file: {}",
24            file_path.display()
25        ))
26    })?;
27
28    let mut buffer = Vec::new();
29    input_read.read_to_end(&mut buffer).map_err(|err| {
30        LinderaErrorKind::Io.with_error(err).add_context(format!(
31            "Failed to read tokenizer config file: {}",
32            file_path.display()
33        ))
34    })?;
35
36    match serde_yaml_ng::from_slice::<serde_yaml_ng::Value>(&buffer) {
37        Ok(value) => {
38            // Check if the value is a mapping.
39            match value {
40                serde_yaml_ng::Value::Mapping(_) => {
41                    Ok(serde_json::to_value(value).map_err(|err| {
42                        LinderaErrorKind::Deserialize
43                            .with_error(err)
44                            .add_context(format!(
45                                "Failed to convert YAML to JSON for config file: {}",
46                                file_path.display()
47                            ))
48                    })?)
49                }
50                _ => Err(LinderaErrorKind::Deserialize
51                    .with_error(anyhow::anyhow!("Invalid YAML"))
52                    .add_context(format!(
53                        "Config file must contain a YAML mapping: {}",
54                        file_path.display()
55                    ))),
56            }
57        }
58        Err(err) => Err(LinderaErrorKind::Deserialize
59            .with_error(err)
60            .add_context(format!(
61                "Failed to parse YAML config file: {}",
62                file_path.display()
63            ))),
64    }
65}
66
67/// Returns the default configuration as a `serde_json::Value`.
68fn empty_config() -> Value {
69    json!({
70        "segmenter": {},
71        "character_filters": [],
72        "token_filters": []
73    })
74}
75
76/// Ensures that the configuration contains the required keys with default values if absent.
77fn ensure_keys(mut config: Value) -> Value {
78    if config.get("segmenter").is_none() {
79        config["segmenter"] = json!({});
80    }
81
82    if config.get("character_filters").is_none() {
83        config["character_filters"] = json!([]);
84    }
85
86    if config.get("token_filters").is_none() {
87        config["token_filters"] = json!([]);
88    }
89
90    config
91}
92
93#[derive(Debug)]
94pub struct TokenizerBuilder {
95    config: TokenizerConfig,
96}
97
98impl TokenizerBuilder {
99    pub fn new() -> LinderaResult<Self> {
100        if let Ok(config_path) = env::var("LINDERA_CONFIG_PATH") {
101            Self::from_file(Path::new(&config_path))
102        } else {
103            Ok(Self {
104                config: empty_config(),
105            })
106        }
107    }
108
109    pub fn from_file(file_path: &Path) -> LinderaResult<Self> {
110        let config = yaml_to_config(file_path)?;
111
112        Ok(TokenizerBuilder {
113            config: ensure_keys(config),
114        })
115    }
116
117    pub fn from_config(config: TokenizerConfig) -> LinderaResult<Self> {
118        Ok(TokenizerBuilder {
119            config: ensure_keys(config),
120        })
121    }
122
123    pub fn set_segmenter_mode(&mut self, mode: &Mode) -> &mut Self {
124        self.config["segmenter"]["mode"] = json!(mode.as_str());
125        self
126    }
127
128    pub fn set_segmenter_dictionary(&mut self, uri: &str) -> &mut Self {
129        self.config["segmenter"]["dictionary"] = json!(uri);
130        self
131    }
132
133    pub fn set_segmenter_user_dictionary(&mut self, uri: &str) -> &mut Self {
134        self.config["segmenter"]["user_dictionary"] = json!(uri);
135        self
136    }
137
138    pub fn set_segmenter_keep_whitespace(&mut self, keep_whitespace: bool) -> &mut Self {
139        self.config["segmenter"]["keep_whitespace"] = json!(keep_whitespace);
140        self
141    }
142
143    /// Set whether to route filesystem-loaded dictionaries through
144    /// memory-mapped reads. Ignored for `embedded://` dictionaries.
145    ///
146    /// # Arguments
147    ///
148    /// * `use_mmap` - Whether to request memory-mapped dictionary loading.
149    ///
150    /// # Returns
151    ///
152    /// A mutable reference to `self`, for chaining.
153    pub fn set_segmenter_use_mmap(&mut self, use_mmap: bool) -> &mut Self {
154        self.config["segmenter"]["use_mmap"] = json!(use_mmap);
155        self
156    }
157
158    pub fn append_character_filter(&mut self, kind: &str, args: &Value) -> &mut Self {
159        if let Some(array) = self.config["character_filters"].as_array_mut() {
160            array.push(json!({ "kind": kind, "args": args }));
161        }
162        self
163    }
164
165    pub fn append_token_filter(&mut self, kind: &str, args: &Value) -> &mut Self {
166        if let Some(array) = self.config["token_filters"].as_array_mut() {
167            array.push(json!({ "kind": kind, "args": args }));
168        }
169        self
170    }
171
172    pub fn build(&self) -> LinderaResult<Tokenizer> {
173        Tokenizer::from_config(&self.config).map_err(|err| {
174            LinderaErrorKind::Parse.with_error(anyhow::anyhow!("failed to build tokenizer: {err}"))
175        })
176    }
177}
178
179pub struct Tokenizer {
180    /// Segmenter
181    /// The `segmenter` field is an instance of the `Segmenter` struct, which is responsible for
182    /// segmenting text into tokens. This is a core component of the tokenizer, enabling it to
183    /// break down input text into manageable and meaningful units for further processing.
184    pub segmenter: Segmenter,
185
186    /// Character filters
187    /// A vector of boxed character filters that will be applied to the input text
188    /// before tokenization. Each character filter is responsible for transforming
189    /// the input text in a specific way, such as normalizing characters or removing
190    /// unwanted characters.
191    pub character_filters: Vec<BoxCharacterFilter>,
192
193    /// Token filters
194    /// A vector of boxed token filters that will be applied to the tokens during tokenization.
195    /// Each token filter is a boxed trait object implementing the `TokenFilter` trait, allowing
196    /// for various transformations and processing steps to be applied to the tokens.
197    pub token_filters: Vec<BoxTokenFilter>,
198}
199
200impl Tokenizer {
201    /// Creates a new `Tokenizer` instance from a provided `Segmenter`.
202    ///
203    /// # Arguments
204    ///
205    /// * `segmenter` - An instance of the `Segmenter` struct, which is responsible for the core tokenization process.
206    ///
207    /// # Returns
208    ///
209    /// Returns a new `Tokenizer` instance that uses the provided `segmenter` for tokenization, with empty character and token filters.
210    ///
211    /// # Details
212    ///
213    /// - `segmenter`: The segmenter is responsible for handling the actual segmentation and tokenization of text. It is passed into the `Tokenizer` during initialization.
214    /// - `character_filters`: This is initialized as an empty vector and can be modified later to include character filters.
215    /// - `token_filters`: This is also initialized as an empty vector and can be modified later to include token filters.
216    pub fn new(segmenter: Segmenter) -> Self {
217        Self {
218            segmenter,
219            character_filters: Vec::new(),
220            token_filters: Vec::new(),
221        }
222    }
223
224    pub fn from_config(config: &TokenizerConfig) -> LinderaResult<Self> {
225        let segmenter_config = config.get("segmenter").ok_or_else(|| {
226            LinderaErrorKind::Deserialize.with_error(anyhow::anyhow!("missing segmenter config."))
227        })?;
228        let segmenter = Segmenter::from_config(segmenter_config)?;
229
230        // Create a tokenizer from the segmenter.
231        let mut tokenizer = Tokenizer::new(segmenter);
232
233        // Load character filter settings from the tokenizer config if it is not empty.
234        if let Some(character_filter_settings) = config["character_filters"].as_array() {
235            for character_filter_setting in character_filter_settings {
236                let character_filter_name = character_filter_setting["kind"].as_str();
237                if let Some(character_filter_name) = character_filter_name {
238                    // Append a character filter to the tokenizer.
239                    tokenizer.append_character_filter(CharacterFilterLoader::load_from_value(
240                        character_filter_name,
241                        &character_filter_setting["args"],
242                    )?);
243                }
244            }
245        }
246
247        // Load token filter settings from the tokenizer config if it is not empty.
248        if let Some(token_filter_settings) = config["token_filters"].as_array() {
249            for token_filter_setting in token_filter_settings {
250                let token_filter_name = token_filter_setting["kind"].as_str();
251                if let Some(token_filter_name) = token_filter_name {
252                    // Append a token filter to the tokenizer.
253                    tokenizer.append_token_filter(TokenFilterLoader::load_from_value(
254                        token_filter_name,
255                        &token_filter_setting["args"],
256                    )?);
257                }
258            }
259        }
260
261        Ok(tokenizer)
262    }
263
264    /// Appends a character filter to the tokenizer.
265    ///
266    /// # Arguments
267    ///
268    /// * `character_filter` - A `BoxCharacterFilter` that will be added to the tokenizer. This filter will be applied to the text during the tokenization process.
269    ///
270    /// # Returns
271    ///
272    /// Returns a mutable reference to `Self`, allowing for method chaining.
273    ///
274    /// # Details
275    ///
276    /// - This method adds a new character filter to the `Tokenizer`'s `character_filters` vector.
277    /// - It returns a mutable reference to `self`, allowing multiple character filters to be appended in a chain of method calls.
278    pub fn append_character_filter(&mut self, character_filter: BoxCharacterFilter) -> &mut Self {
279        self.character_filters.push(character_filter);
280
281        self
282    }
283
284    /// Appends a token filter to the tokenizer.
285    ///
286    /// # Arguments
287    ///
288    /// * `token_filter` - A `BoxTokenFilter` that will be added to the tokenizer. This filter will be applied to the tokens after they are segmented.
289    ///
290    /// # Returns
291    ///
292    /// Returns a mutable reference to `Self`, allowing for method chaining.
293    ///
294    /// # Details
295    ///
296    /// - This method adds a new token filter to the `Tokenizer`'s `token_filters` vector.
297    /// - It returns a mutable reference to `self`, allowing multiple token filters to be appended in a chain of method calls.
298    pub fn append_token_filter(&mut self, token_filter: BoxTokenFilter) -> &mut Self {
299        self.token_filters.push(token_filter);
300
301        self
302    }
303
304    /// Tokenizes the input text using the tokenizer's segmenter, character filters, and token filters.
305    ///
306    /// # Arguments
307    ///
308    /// * `text` - A reference to the input text (`&str`) that will be tokenized.
309    ///
310    /// # Returns
311    ///
312    /// Returns a `LinderaResult` containing a vector of `Token`s, where each `Token` represents a segment of the tokenized text.
313    ///
314    /// # Process
315    ///
316    /// 1. **Apply character filters**:
317    ///    - If any character filters are defined, they are applied to the input text before tokenization.
318    ///    - The `offsets`, `diffs`, and `text_len` are recorded for each character filter.
319    /// 2. **Segment the text**:
320    ///    - The `segmenter` divides the (potentially filtered) text into tokens.
321    /// 3. **Apply token filters**:
322    ///    - If any token filters are defined, they are applied to the segmented tokens.
323    /// 4. **Correct token offsets**:
324    ///    - If character filters were applied, the byte offsets of each token are corrected to account for changes introduced by those filters.
325    ///
326    /// # Errors
327    ///
328    /// - Returns an error if any of the character or token filters fail during processing.
329    /// - Returns an error if the segmentation process fails.
330    ///
331    /// # Details
332    ///
333    /// - `Cow<'a, str>` is used for the `normalized_text`, allowing the function to either borrow the original text or create an owned version if the text needs modification.
334    /// - If no character filters are applied, the original `text` is used as-is for segmentation.
335    /// - Token offsets are adjusted after the tokenization process if character filters were applied to ensure the byte positions of each token are accurate relative to the original text.
336    pub fn tokenize<'a>(&'a self, text: &'a str) -> LinderaResult<Vec<Token<'a>>> {
337        let mut lattice = Lattice::default();
338        self.tokenize_with_lattice(text, &mut lattice)
339    }
340
341    /// Tokenizes the input text using the tokenizer's segmenter, character filters, and token filters.
342    ///
343    /// # Arguments
344    ///
345    /// * `text` - A reference to the input text (`&str`) that will be tokenized.
346    /// * `lattice` - A mutable reference to a `Lattice` structure. This allows reusing the lattice across multiple calls to avoid memory allocation.
347    ///
348    /// # Returns
349    ///
350    /// Returns a `LinderaResult` containing a vector of `Token`s, where each `Token` represents a segment of the tokenized text.
351    ///
352    /// # Process
353    ///
354    /// 1. **Apply character filters**:
355    ///    - If any character filters are defined, they are applied to the input text before tokenization.
356    ///    - The `offsets`, `diffs`, and `text_len` are recorded for each character filter.
357    /// 2. **Segment the text**:
358    ///    - The `segmenter` divides the (potentially filtered) text into tokens.
359    /// 3. **Apply token filters**:
360    ///    - If any token filters are defined, they are applied to the segmented tokens.
361    /// 4. **Correct token offsets**:
362    ///    - If character filters were applied, the byte offsets of each token are corrected to account for changes introduced by those filters.
363    ///
364    /// # Errors
365    ///
366    /// - Returns an error if any of the character or token filters fail during processing.
367    /// - Returns an error if the segmentation process fails.
368    ///
369    /// # Details
370    ///
371    /// - `Cow<'a, str>` is used for the `normalized_text`, allowing the function to either borrow the original text or create an owned version if the text needs modification.
372    /// - If no character filters are applied, the original `text` is used as-is for segmentation.
373    /// - Token offsets are adjusted after the tokenization process if character filters were applied to ensure the byte positions of each token are accurate relative to the original text.
374    pub fn tokenize_with_lattice<'a>(
375        &'a self,
376        text: &'a str,
377        lattice: &mut Lattice,
378    ) -> LinderaResult<Vec<Token<'a>>> {
379        let mut normalized_text: Cow<'a, str> = Cow::Borrowed(text);
380
381        let mut offset_mappings: Vec<OffsetMapping> =
382            Vec::with_capacity(self.character_filters.len());
383
384        // Apply character filters to the text if it is not empty.
385        // Optimize: Only convert to mutable when we have filters to apply
386        if !self.character_filters.is_empty() {
387            // Convert to owned string once for all filters
388            let text_mut = normalized_text.to_mut();
389
390            for character_filter in &self.character_filters {
391                let mapping = character_filter.apply(text_mut)?;
392
393                if !mapping.is_empty() {
394                    // Record the offset mapping of each character filter in reverse order
395                    // since we need to apply corrections in reverse order
396                    offset_mappings.push(mapping);
397                }
398            }
399        }
400
401        // Store the final text length for offset correction
402        let final_text_len = normalized_text.len();
403
404        // Segment a text.
405        // Segment a text.
406        let mut tokens = self
407            .segmenter
408            .segment_with_lattice(normalized_text, lattice)?;
409
410        // Apply token filters to the tokens if they are not empty.
411        for token_filter in &self.token_filters {
412            token_filter.apply(&mut tokens)?;
413        }
414
415        // Correct token offsets if character filters are applied.
416        correct_offsets(&mut tokens, &offset_mappings, final_text_len);
417
418        Ok(tokens)
419    }
420
421    /// Tokenizes the input text and returns the top-N results.
422    ///
423    /// Each result is a `Vec<Token>` with character/token filters applied.
424    /// Results are ordered by cost (best first).
425    pub fn tokenize_nbest<'a>(
426        &'a self,
427        text: &'a str,
428        n: usize,
429        unique: bool,
430        cost_threshold: Option<i64>,
431    ) -> LinderaResult<Vec<(Vec<Token<'a>>, i64)>> {
432        let mut lattice = Lattice::default();
433        self.tokenize_nbest_with_lattice(text, &mut lattice, n, unique, cost_threshold)
434    }
435
436    /// Tokenizes the input text and returns the top-N results with costs.
437    /// Each result is a (tokens, cost) pair.
438    /// If `unique` is true, results with the same word boundaries are deduplicated.
439    /// If `cost_threshold` is Some(t), paths whose cost exceeds best_cost + t
440    /// are discarded.
441    pub fn tokenize_nbest_with_lattice<'a>(
442        &'a self,
443        text: &'a str,
444        lattice: &mut Lattice,
445        n: usize,
446        unique: bool,
447        cost_threshold: Option<i64>,
448    ) -> LinderaResult<Vec<(Vec<Token<'a>>, i64)>> {
449        let mut normalized_text: Cow<'a, str> = Cow::Borrowed(text);
450
451        let mut offset_mappings: Vec<OffsetMapping> =
452            Vec::with_capacity(self.character_filters.len());
453
454        if !self.character_filters.is_empty() {
455            let text_mut = normalized_text.to_mut();
456            for character_filter in &self.character_filters {
457                let mapping = character_filter.apply(text_mut)?;
458                if !mapping.is_empty() {
459                    offset_mappings.push(mapping);
460                }
461            }
462        }
463
464        let final_text_len = normalized_text.len();
465
466        let mut all_results = self.segmenter.segment_nbest_with_lattice(
467            normalized_text,
468            lattice,
469            n,
470            unique,
471            cost_threshold,
472        )?;
473
474        // Apply token filters and offset corrections to each result
475        for (tokens, _cost) in &mut all_results {
476            for token_filter in &self.token_filters {
477                token_filter.apply(tokens)?;
478            }
479
480            correct_offsets(tokens, &offset_mappings, final_text_len);
481        }
482
483        Ok(all_results)
484    }
485}
486
487/// Corrects token byte offsets back to the original (pre-filter) text by
488/// applying the character filters' offset mappings in reverse order (last
489/// filter first). A no-op when `offset_mappings` is empty.
490///
491/// Shared by `tokenize_with_lattice`, `tokenize_nbest_with_lattice`, and
492/// `AnalysisWorker`.
493///
494/// # 引数
495///
496/// * `tokens` - The tokens whose `byte_start`/`byte_end` are corrected in
497///   place.
498/// * `offset_mappings` - Non-empty mappings recorded by the character
499///   filters, in application order.
500/// * `final_text_len` - Length in bytes of the fully filtered text.
501pub(crate) fn correct_offsets(
502    tokens: &mut [Token<'_>],
503    offset_mappings: &[OffsetMapping],
504    final_text_len: usize,
505) {
506    if offset_mappings.is_empty() {
507        return;
508    }
509    for token in tokens.iter_mut() {
510        // Apply corrections in reverse order to undo the transformations.
511        for mapping in offset_mappings.iter().rev() {
512            token.byte_start = mapping.correct_offset(token.byte_start, final_text_len);
513            token.byte_end = mapping.correct_offset(token.byte_end, final_text_len);
514        }
515    }
516}
517
518impl Clone for Tokenizer {
519    /// Creates a deep clone of the `Tokenizer` instance, including all character filters, token filters, and the segmenter.
520    ///
521    /// # Returns
522    ///
523    /// Returns a new `Tokenizer` instance that is a deep clone of the current instance. All internal filters and the segmenter are cloned.
524    ///
525    /// # Details
526    ///
527    /// - **Character Filters**: Each character filter is cloned by calling its `box_clone` method, which ensures that any dynamically dispatched filters are properly cloned.
528    /// - **Token Filters**: Similarly, each token filter is cloned using the `box_clone` method to handle dynamic dispatch.
529    /// - **Segmenter**: The segmenter is cloned using its `clone` method.
530    ///
531    /// # Notes
532    ///
533    /// - This method performs deep cloning, meaning that all internal filters and segmenter instances are fully duplicated.
534    /// - The `box_clone` method is used to clone the dynamically dispatched filter objects (`BoxCharacterFilter` and `BoxTokenFilter`).
535    fn clone(&self) -> Self {
536        let character_filters: Vec<BoxCharacterFilter> = self
537            .character_filters
538            .iter()
539            .map(|filter| filter.box_clone())
540            .collect();
541
542        let token_filters: Vec<BoxTokenFilter> = self
543            .token_filters
544            .iter()
545            .map(|filter| filter.box_clone())
546            .collect();
547
548        Tokenizer {
549            character_filters,
550            segmenter: self.segmenter.clone(),
551            token_filters,
552        }
553    }
554}
555
556#[cfg(test)]
557mod tests {
558    use super::TokenizerBuilder;
559
560    #[test]
561    fn test_set_segmenter_use_mmap_writes_flat_key_under_segmenter() {
562        let mut builder = TokenizerBuilder::new().unwrap();
563        builder.set_segmenter_use_mmap(true);
564
565        assert_eq!(builder.config["segmenter"]["use_mmap"], true);
566    }
567
568    #[cfg(feature = "embed-ipadic")]
569    #[test]
570    fn test_tokenizer_config_from_slice() {
571        use std::path::PathBuf;
572
573        use crate::tokenizer::yaml_to_config;
574
575        let config_file = PathBuf::from(env!("CARGO_MANIFEST_DIR"))
576            .join("../resources")
577            .join("config")
578            .join("lindera.yml");
579
580        let result = yaml_to_config(&config_file);
581
582        assert!(result.is_ok());
583    }
584
585    #[test]
586    #[cfg(feature = "embed-ipadic")]
587    fn test_tokenizer_config_clone() {
588        use std::path::PathBuf;
589
590        use crate::tokenizer::yaml_to_config;
591
592        let config_file = PathBuf::from(env!("CARGO_MANIFEST_DIR"))
593            .join("../resources")
594            .join("config")
595            .join("lindera.yml");
596
597        let tokenizer_config = yaml_to_config(&config_file).unwrap();
598
599        let cloned_tokenizer_config = tokenizer_config.clone();
600
601        assert_eq!(tokenizer_config, cloned_tokenizer_config);
602    }
603
604    #[test]
605    #[cfg(feature = "embed-ipadic")]
606    fn test_tokenize_ipadic() {
607        use std::borrow::Cow;
608        use std::path::PathBuf;
609
610        use crate::tokenizer::TokenizerBuilder;
611
612        let config_file = PathBuf::from(env!("CARGO_MANIFEST_DIR"))
613            .join("../resources")
614            .join("config")
615            .join("lindera.yml");
616
617        let builder = TokenizerBuilder::from_file(&config_file).unwrap();
618
619        let tokenizer = builder.build().unwrap();
620
621        {
622            let text = "ﾘﾝﾃﾞﾗは形態素解析ｴﾝｼﾞﾝです。";
623            let mut tokens = tokenizer.tokenize(text).unwrap();
624            let mut tokens_iter = tokens.iter_mut();
625            {
626                let token = tokens_iter.next().unwrap();
627                assert_eq!(token.surface, Cow::Borrowed("Lindera"));
628                assert_eq!(token.byte_start, 0);
629                assert_eq!(token.byte_end, 15);
630                assert_eq!(token.position, 0);
631                assert_eq!(token.position_length, 1);
632                assert!(token.word_id.is_unknown());
633                assert_eq!(
634                    token.details,
635                    Some(vec![
636                        Cow::Borrowed("名詞"),
637                        Cow::Borrowed("固有名詞"),
638                        Cow::Borrowed("組織"),
639                        Cow::Borrowed("*"),
640                        Cow::Borrowed("*"),
641                        Cow::Borrowed("*"),
642                        Cow::Borrowed("*"),
643                        Cow::Borrowed("*"),
644                        Cow::Borrowed("*"),
645                    ])
646                );
647            }
648            {
649                let token = tokens_iter.next().unwrap();
650                assert_eq!(token.surface, Cow::Borrowed("形態素"));
651                assert_eq!(token.byte_start, 18);
652                assert_eq!(token.byte_end, 27);
653                assert_eq!(token.position, 2);
654                assert_eq!(token.position_length, 1);
655                assert_eq!(
656                    token.details,
657                    Some(vec![
658                        Cow::Borrowed("名詞"),
659                        Cow::Borrowed("一般"),
660                        Cow::Borrowed("*"),
661                        Cow::Borrowed("*"),
662                        Cow::Borrowed("*"),
663                        Cow::Borrowed("*"),
664                        Cow::Borrowed("形態素"),
665                        Cow::Borrowed("ケイタイソ"),
666                        Cow::Borrowed("ケイタイソ"),
667                    ])
668                );
669            }
670            {
671                let token = tokens_iter.next().unwrap();
672                assert_eq!(token.surface, Cow::Borrowed("解析"));
673                assert_eq!(token.byte_start, 27);
674                assert_eq!(token.byte_end, 33);
675                assert_eq!(token.position, 3);
676                assert_eq!(token.position_length, 1);
677                assert_eq!(
678                    token.details,
679                    Some(vec![
680                        Cow::Borrowed("名詞"),
681                        Cow::Borrowed("サ変接続"),
682                        Cow::Borrowed("*"),
683                        Cow::Borrowed("*"),
684                        Cow::Borrowed("*"),
685                        Cow::Borrowed("*"),
686                        Cow::Borrowed("解析"),
687                        Cow::Borrowed("カイセキ"),
688                        Cow::Borrowed("カイセキ"),
689                    ])
690                );
691            }
692            {
693                let token = tokens_iter.next().unwrap();
694                assert_eq!(token.surface, Cow::Borrowed("エンジン"));
695                assert_eq!(token.byte_start, 33);
696                assert_eq!(token.byte_end, 48);
697                assert_eq!(token.position, 4);
698                assert_eq!(token.position_length, 1);
699                assert_eq!(
700                    token.details,
701                    Some(vec![
702                        Cow::Borrowed("名詞"),
703                        Cow::Borrowed("一般"),
704                        Cow::Borrowed("*"),
705                        Cow::Borrowed("*"),
706                        Cow::Borrowed("*"),
707                        Cow::Borrowed("*"),
708                        Cow::Borrowed("エンジン"),
709                        Cow::Borrowed("エンジン"),
710                        Cow::Borrowed("エンジン"),
711                    ])
712                );
713            }
714
715            let mut tokens_iter = tokens.iter();
716            {
717                let token = tokens_iter.next().unwrap();
718                let start = token.byte_start;
719                let end = token.byte_end;
720                assert_eq!(token.surface, Cow::Borrowed("Lindera"));
721                assert_eq!(&text[start..end], "ﾘﾝﾃﾞﾗ");
722            }
723        }
724
725        {
726            let text = "１０㌎のｶﾞｿﾘﾝ";
727            let mut tokens = tokenizer.tokenize(text).unwrap();
728            let mut tokens_iter = tokens.iter_mut();
729            {
730                // "10" (unknown NUMERIC) and "ガロン" (名詞,接尾,助数詞) are merged
731                // by the japanese_compound_word filter into "10ガロン" with tag "名詞,数".
732                let token = tokens_iter.next().unwrap();
733                assert_eq!(token.surface, Cow::Owned::<str>("10ガロン".into()));
734                assert_eq!(token.byte_start, 0);
735                assert_eq!(token.byte_end, 9);
736                assert_eq!(token.position, 0);
737                assert_eq!(token.position_length, 2);
738            }
739            {
740                let token = tokens_iter.next().unwrap();
741                assert_eq!(token.surface, Cow::Borrowed("ガソリン"));
742                assert_eq!(token.byte_start, 12);
743                assert_eq!(token.byte_end, 27);
744                assert_eq!(token.position, 3);
745                assert_eq!(token.position_length, 1);
746                assert_eq!(
747                    token.details,
748                    Some(vec![
749                        Cow::Borrowed("名詞"),
750                        Cow::Borrowed("一般"),
751                        Cow::Borrowed("*"),
752                        Cow::Borrowed("*"),
753                        Cow::Borrowed("*"),
754                        Cow::Borrowed("*"),
755                        Cow::Borrowed("ガソリン"),
756                        Cow::Borrowed("ガソリン"),
757                        Cow::Borrowed("ガソリン"),
758                    ])
759                );
760            }
761
762            let mut tokens_iter = tokens.iter();
763            {
764                // "10" and "ガロン" are merged by the japanese_compound_word filter
765                let token = tokens_iter.next().unwrap();
766                let start = token.byte_start;
767                let end = token.byte_end;
768                assert_eq!(token.surface, Cow::Owned::<str>("10ガロン".into()));
769                assert_eq!(&text[start..end], "１０㌎");
770            }
771            {
772                let token = tokens_iter.next().unwrap();
773                let start = token.byte_start;
774                let end = token.byte_end;
775                assert_eq!(token.surface, Cow::Borrowed("ガソリン"));
776                assert_eq!(&text[start..end], "ｶﾞｿﾘﾝ");
777            }
778        }
779
780        {
781            let text = "お釣りは百三十四円です。";
782            let mut tokens = tokenizer.tokenize(text).unwrap();
783            let mut tokens_iter = tokens.iter_mut();
784            {
785                let token = tokens_iter.next().unwrap();
786                assert_eq!(token.surface, Cow::Borrowed("お釣り"));
787                assert_eq!(token.byte_start, 0);
788                assert_eq!(token.byte_end, 9);
789                assert_eq!(token.position, 0);
790                assert_eq!(token.position_length, 1);
791                assert_eq!(
792                    token.details,
793                    Some(vec![
794                        Cow::Borrowed("名詞"),
795                        Cow::Borrowed("一般"),
796                        Cow::Borrowed("*"),
797                        Cow::Borrowed("*"),
798                        Cow::Borrowed("*"),
799                        Cow::Borrowed("*"),
800                        Cow::Borrowed("お釣り"),
801                        Cow::Borrowed("オツリ"),
802                        Cow::Borrowed("オツリ"),
803                    ])
804                );
805            }
806            {
807                let token = tokens_iter.next().unwrap();
808                assert_eq!(token.surface, Cow::Borrowed("134円"));
809                assert_eq!(token.byte_start, 12);
810                assert_eq!(token.byte_end, 27);
811                assert_eq!(token.position, 2);
812                assert_eq!(token.position_length, 5);
813                assert_eq!(
814                    token.details,
815                    Some(vec![
816                        Cow::Borrowed("名詞"),
817                        Cow::Borrowed("数"),
818                        Cow::Borrowed("*"),
819                        Cow::Borrowed("*"),
820                        Cow::Borrowed("*"),
821                        Cow::Borrowed("*"),
822                        Cow::Borrowed("*"),
823                        Cow::Borrowed("*"),
824                        Cow::Borrowed("*"),
825                    ])
826                );
827            }
828        }
829
830        {
831            let text = "ここは騒々しい";
832            let mut tokens = tokenizer.tokenize(text).unwrap();
833            let mut tokens_iter = tokens.iter_mut();
834            {
835                let token = tokens_iter.next().unwrap();
836                assert_eq!(token.surface, Cow::Borrowed("ここ"));
837                assert_eq!(token.byte_start, 0);
838                assert_eq!(token.byte_end, 6);
839                assert_eq!(token.position, 0);
840                assert_eq!(token.position_length, 1);
841                assert_eq!(
842                    token.details,
843                    Some(vec![
844                        Cow::Borrowed("名詞"),
845                        Cow::Borrowed("代名詞"),
846                        Cow::Borrowed("一般"),
847                        Cow::Borrowed("*"),
848                        Cow::Borrowed("*"),
849                        Cow::Borrowed("*"),
850                        Cow::Borrowed("ここ"),
851                        Cow::Borrowed("ココ"),
852                        Cow::Borrowed("ココ"),
853                    ])
854                );
855            }
856            {
857                let token = tokens_iter.next().unwrap();
858                assert_eq!(token.surface, Cow::Borrowed("騒騒しい"));
859                assert_eq!(token.byte_start, 9);
860                assert_eq!(token.byte_end, 21);
861                assert_eq!(token.position, 2);
862                assert_eq!(token.position_length, 1);
863                assert_eq!(
864                    token.details,
865                    Some(vec![
866                        Cow::Borrowed("形容詞"),
867                        Cow::Borrowed("自立"),
868                        Cow::Borrowed("*"),
869                        Cow::Borrowed("*"),
870                        Cow::Borrowed("形容詞・イ段"),
871                        Cow::Borrowed("基本形"),
872                        Cow::Borrowed("騒騒しい"),
873                        Cow::Borrowed("ソウゾウシイ"),
874                        Cow::Borrowed("ソーゾーシイ"),
875                    ])
876                );
877            }
878        }
879    }
880
881    #[test]
882    #[cfg(not(windows))]
883    #[should_panic(expected = "No such file or directory")]
884    fn test_create_tokenizer_builder_from_non_existent_file() {
885        use std::path::PathBuf;
886
887        use crate::tokenizer::TokenizerBuilder;
888
889        let config_file = PathBuf::from(env!("CARGO_MANIFEST_DIR"))
890            .join("../resources")
891            .join("config")
892            .join("non_existent_file.yml");
893
894        TokenizerBuilder::from_file(&config_file).unwrap();
895    }
896
897    #[test]
898    #[cfg(windows)]
899    #[should_panic(expected = "The system cannot find the file specified.")]
900    fn test_create_tokenizer_builder_from_non_existent_file() {
901        use std::path::PathBuf;
902
903        use crate::tokenizer::TokenizerBuilder;
904
905        let config_file = PathBuf::from(env!("CARGO_MANIFEST_DIR"))
906            .join("../resources")
907            .join("config")
908            .join("non_existent_file.yml");
909
910        TokenizerBuilder::from_file(&config_file).unwrap();
911    }
912
913    #[test]
914    #[should_panic(expected = "Invalid YAML")]
915    fn test_create_tokenizer_builder_from_invalid_file() {
916        use std::path::PathBuf;
917
918        use crate::tokenizer::TokenizerBuilder;
919
920        let config_file = PathBuf::from(env!("CARGO_MANIFEST_DIR"))
921            .join("../resources")
922            .join("config")
923            .join("invalid.yml");
924
925        TokenizerBuilder::from_file(&config_file).unwrap();
926    }
927
928    #[test]
929    #[cfg(feature = "embed-ipadic")]
930    fn test_tokenize_nbest_1best_matches_tokenize() {
931        use crate::tokenizer::TokenizerBuilder;
932
933        let mut builder = TokenizerBuilder::new().unwrap();
934        builder.set_segmenter_dictionary("embedded://ipadic");
935
936        let tokenizer = builder.build().unwrap();
937
938        let text = "すもももももももものうち";
939        let normal_tokens = tokenizer.tokenize(text).unwrap();
940        let nbest_results = tokenizer.tokenize_nbest(text, 1, false, None).unwrap();
941
942        assert_eq!(nbest_results.len(), 1);
943        let (nbest_tokens, _cost) = &nbest_results[0];
944        assert_eq!(normal_tokens.len(), nbest_tokens.len());
945        for (normal, nbest) in normal_tokens.iter().zip(nbest_tokens.iter()) {
946            assert_eq!(normal.surface.as_ref(), nbest.surface.as_ref());
947        }
948    }
949
950    #[test]
951    #[cfg(feature = "embed-ipadic")]
952    fn test_tokenize_nbest_multiple_results() {
953        use crate::tokenizer::TokenizerBuilder;
954
955        let mut builder = TokenizerBuilder::new().unwrap();
956        builder.set_segmenter_dictionary("embedded://ipadic");
957
958        let tokenizer = builder.build().unwrap();
959
960        let text = "すもももももももものうち";
961        let results = tokenizer.tokenize_nbest(text, 5, false, None).unwrap();
962
963        // Should return multiple results for ambiguous text
964        assert!(results.len() >= 2);
965
966        // All results should cover the full text
967        for (tokens, _cost) in &results {
968            assert!(!tokens.is_empty());
969        }
970    }
971}
lindera_analysis/tokenizer.rs

lindera_analysis/
tokenizer.rs
