lindera_analysis/
tokenizer.rs

1use std::borrow::Cow;
2use std::env;
3use std::fs::File;
4use std::io::Read;
5use std::path::Path;
6
7use serde_json::{Value, json};
8
9use crate::character_filter::{BoxCharacterFilter, CharacterFilterLoader, OffsetMapping};
10use crate::token_filter::{BoxTokenFilter, TokenFilterLoader};
11use lindera::LinderaResult;
12use lindera::dictionary::Lattice;
13use lindera::error::LinderaErrorKind;
14use lindera::mode::Mode;
15use lindera::segmenter::Segmenter;
16use lindera::token::Token;
17
18pub type TokenizerConfig = Value;
19
20fn yaml_to_config(file_path: &Path) -> LinderaResult<TokenizerConfig> {
21    let mut input_read = File::open(file_path).map_err(|err| {
22        LinderaErrorKind::Io.with_error(err).add_context(format!(
23            "Failed to open tokenizer config file: {}",
24            file_path.display()
25        ))
26    })?;
27
28    let mut buffer = Vec::new();
29    input_read.read_to_end(&mut buffer).map_err(|err| {
30        LinderaErrorKind::Io.with_error(err).add_context(format!(
31            "Failed to read tokenizer config file: {}",
32            file_path.display()
33        ))
34    })?;
35
36    match serde_yaml_ng::from_slice::<serde_yaml_ng::Value>(&buffer) {
37        Ok(value) => {
38            // Check if the value is a mapping.
39            match value {
40                serde_yaml_ng::Value::Mapping(_) => {
41                    Ok(serde_json::to_value(value).map_err(|err| {
42                        LinderaErrorKind::Deserialize
43                            .with_error(err)
44                            .add_context(format!(
45                                "Failed to convert YAML to JSON for config file: {}",
46                                file_path.display()
47                            ))
48                    })?)
49                }
50                _ => Err(LinderaErrorKind::Deserialize
51                    .with_error(anyhow::anyhow!("Invalid YAML"))
52                    .add_context(format!(
53                        "Config file must contain a YAML mapping: {}",
54                        file_path.display()
55                    ))),
56            }
57        }
58        Err(err) => Err(LinderaErrorKind::Deserialize
59            .with_error(err)
60            .add_context(format!(
61                "Failed to parse YAML config file: {}",
62                file_path.display()
63            ))),
64    }
65}
66
67/// Returns the default configuration as a `serde_json::Value`.
68fn empty_config() -> Value {
69    json!({
70        "segmenter": {},
71        "character_filters": [],
72        "token_filters": []
73    })
74}
75
76/// Ensures that the configuration contains the required keys with default values if absent.
77fn ensure_keys(mut config: Value) -> Value {
78    if config.get("segmenter").is_none() {
79        config["segmenter"] = json!({});
80    }
81
82    if config.get("character_filters").is_none() {
83        config["character_filters"] = json!([]);
84    }
85
86    if config.get("token_filters").is_none() {
87        config["token_filters"] = json!([]);
88    }
89
90    config
91}
92
93#[derive(Debug)]
94pub struct TokenizerBuilder {
95    config: TokenizerConfig,
96}
97
98impl TokenizerBuilder {
99    pub fn new() -> LinderaResult<Self> {
100        if let Ok(config_path) = env::var("LINDERA_CONFIG_PATH") {
101            Self::from_file(Path::new(&config_path))
102        } else {
103            Ok(Self {
104                config: empty_config(),
105            })
106        }
107    }
108
109    pub fn from_file(file_path: &Path) -> LinderaResult<Self> {
110        let config = yaml_to_config(file_path)?;
111
112        Ok(TokenizerBuilder {
113            config: ensure_keys(config),
114        })
115    }
116
117    pub fn from_config(config: TokenizerConfig) -> LinderaResult<Self> {
118        Ok(TokenizerBuilder {
119            config: ensure_keys(config),
120        })
121    }
122
123    pub fn set_segmenter_mode(&mut self, mode: &Mode) -> &mut Self {
124        self.config["segmenter"]["mode"] = json!(mode.as_str());
125        self
126    }
127
128    pub fn set_segmenter_dictionary(&mut self, uri: &str) -> &mut Self {
129        self.config["segmenter"]["dictionary"] = json!(uri);
130        self
131    }
132
133    pub fn set_segmenter_user_dictionary(&mut self, uri: &str) -> &mut Self {
134        self.config["segmenter"]["user_dictionary"] = json!(uri);
135        self
136    }
137
138    pub fn set_segmenter_keep_whitespace(&mut self, keep_whitespace: bool) -> &mut Self {
139        self.config["segmenter"]["keep_whitespace"] = json!(keep_whitespace);
140        self
141    }
142
143    /// Caps unknown-word grouping (MeCab's `max-grouping-size` semantics);
144    /// `0` or an absent key means unbounded grouping, the default.
145    ///
146    /// # Arguments
147    ///
148    /// * `max_grouping_len` - Maximum grouped characters beyond the first.
149    ///
150    /// # Returns
151    ///
152    /// A mutable reference to `self`, for chaining.
153    pub fn set_segmenter_max_grouping_len(&mut self, max_grouping_len: usize) -> &mut Self {
154        self.config["segmenter"]["max_grouping_len"] = json!(max_grouping_len);
155        self
156    }
157
158    /// Enables/disables the unknown-word length ladder (see
159    /// `Segmenter::unknown_word_ladder`; defaults to `true`).
160    ///
161    /// # Arguments
162    ///
163    /// * `unknown_word_ladder` - Whether to emit the length ladder.
164    ///
165    /// # Returns
166    ///
167    /// A mutable reference to `self`, for chaining.
168    pub fn set_segmenter_unknown_word_ladder(&mut self, unknown_word_ladder: bool) -> &mut Self {
169        self.config["segmenter"]["unknown_word_ladder"] = json!(unknown_word_ladder);
170        self
171    }
172
173    /// Set whether to route filesystem-loaded dictionaries through
174    /// memory-mapped reads. Ignored for `embedded://` dictionaries.
175    ///
176    /// # Arguments
177    ///
178    /// * `use_mmap` - Whether to request memory-mapped dictionary loading.
179    ///
180    /// # Returns
181    ///
182    /// A mutable reference to `self`, for chaining.
183    pub fn set_segmenter_use_mmap(&mut self, use_mmap: bool) -> &mut Self {
184        self.config["segmenter"]["use_mmap"] = json!(use_mmap);
185        self
186    }
187
188    pub fn append_character_filter(&mut self, kind: &str, args: &Value) -> &mut Self {
189        if let Some(array) = self.config["character_filters"].as_array_mut() {
190            array.push(json!({ "kind": kind, "args": args }));
191        }
192        self
193    }
194
195    pub fn append_token_filter(&mut self, kind: &str, args: &Value) -> &mut Self {
196        if let Some(array) = self.config["token_filters"].as_array_mut() {
197            array.push(json!({ "kind": kind, "args": args }));
198        }
199        self
200    }
201
202    pub fn build(&self) -> LinderaResult<Tokenizer> {
203        Tokenizer::from_config(&self.config).map_err(|err| {
204            LinderaErrorKind::Parse.with_error(anyhow::anyhow!("failed to build tokenizer: {err}"))
205        })
206    }
207}
208
209pub struct Tokenizer {
210    /// Segmenter
211    /// The `segmenter` field is an instance of the `Segmenter` struct, which is responsible for
212    /// segmenting text into tokens. This is a core component of the tokenizer, enabling it to
213    /// break down input text into manageable and meaningful units for further processing.
214    pub segmenter: Segmenter,
215
216    /// Character filters
217    /// A vector of boxed character filters that will be applied to the input text
218    /// before tokenization. Each character filter is responsible for transforming
219    /// the input text in a specific way, such as normalizing characters or removing
220    /// unwanted characters.
221    pub character_filters: Vec<BoxCharacterFilter>,
222
223    /// Token filters
224    /// A vector of boxed token filters that will be applied to the tokens during tokenization.
225    /// Each token filter is a boxed trait object implementing the `TokenFilter` trait, allowing
226    /// for various transformations and processing steps to be applied to the tokens.
227    pub token_filters: Vec<BoxTokenFilter>,
228}
229
230impl Tokenizer {
231    /// Creates a new `Tokenizer` instance from a provided `Segmenter`.
232    ///
233    /// # Arguments
234    ///
235    /// * `segmenter` - An instance of the `Segmenter` struct, which is responsible for the core tokenization process.
236    ///
237    /// # Returns
238    ///
239    /// Returns a new `Tokenizer` instance that uses the provided `segmenter` for tokenization, with empty character and token filters.
240    ///
241    /// # Details
242    ///
243    /// - `segmenter`: The segmenter is responsible for handling the actual segmentation and tokenization of text. It is passed into the `Tokenizer` during initialization.
244    /// - `character_filters`: This is initialized as an empty vector and can be modified later to include character filters.
245    /// - `token_filters`: This is also initialized as an empty vector and can be modified later to include token filters.
246    pub fn new(segmenter: Segmenter) -> Self {
247        Self {
248            segmenter,
249            character_filters: Vec::new(),
250            token_filters: Vec::new(),
251        }
252    }
253
254    pub fn from_config(config: &TokenizerConfig) -> LinderaResult<Self> {
255        let segmenter_config = config.get("segmenter").ok_or_else(|| {
256            LinderaErrorKind::Deserialize.with_error(anyhow::anyhow!("missing segmenter config."))
257        })?;
258        let segmenter = Segmenter::from_config(segmenter_config)?;
259
260        // Create a tokenizer from the segmenter.
261        let mut tokenizer = Tokenizer::new(segmenter);
262
263        // Load character filter settings from the tokenizer config if it is not empty.
264        if let Some(character_filter_settings) = config["character_filters"].as_array() {
265            for character_filter_setting in character_filter_settings {
266                let character_filter_name = character_filter_setting["kind"].as_str();
267                if let Some(character_filter_name) = character_filter_name {
268                    // Append a character filter to the tokenizer.
269                    tokenizer.append_character_filter(CharacterFilterLoader::load_from_value(
270                        character_filter_name,
271                        &character_filter_setting["args"],
272                    )?);
273                }
274            }
275        }
276
277        // Load token filter settings from the tokenizer config if it is not empty.
278        if let Some(token_filter_settings) = config["token_filters"].as_array() {
279            for token_filter_setting in token_filter_settings {
280                let token_filter_name = token_filter_setting["kind"].as_str();
281                if let Some(token_filter_name) = token_filter_name {
282                    // Append a token filter to the tokenizer.
283                    tokenizer.append_token_filter(TokenFilterLoader::load_from_value(
284                        token_filter_name,
285                        &token_filter_setting["args"],
286                    )?);
287                }
288            }
289        }
290
291        Ok(tokenizer)
292    }
293
294    /// Appends a character filter to the tokenizer.
295    ///
296    /// # Arguments
297    ///
298    /// * `character_filter` - A `BoxCharacterFilter` that will be added to the tokenizer. This filter will be applied to the text during the tokenization process.
299    ///
300    /// # Returns
301    ///
302    /// Returns a mutable reference to `Self`, allowing for method chaining.
303    ///
304    /// # Details
305    ///
306    /// - This method adds a new character filter to the `Tokenizer`'s `character_filters` vector.
307    /// - It returns a mutable reference to `self`, allowing multiple character filters to be appended in a chain of method calls.
308    pub fn append_character_filter(&mut self, character_filter: BoxCharacterFilter) -> &mut Self {
309        self.character_filters.push(character_filter);
310
311        self
312    }
313
314    /// Appends a token filter to the tokenizer.
315    ///
316    /// # Arguments
317    ///
318    /// * `token_filter` - A `BoxTokenFilter` that will be added to the tokenizer. This filter will be applied to the tokens after they are segmented.
319    ///
320    /// # Returns
321    ///
322    /// Returns a mutable reference to `Self`, allowing for method chaining.
323    ///
324    /// # Details
325    ///
326    /// - This method adds a new token filter to the `Tokenizer`'s `token_filters` vector.
327    /// - It returns a mutable reference to `self`, allowing multiple token filters to be appended in a chain of method calls.
328    pub fn append_token_filter(&mut self, token_filter: BoxTokenFilter) -> &mut Self {
329        self.token_filters.push(token_filter);
330
331        self
332    }
333
334    /// Tokenizes the input text using the tokenizer's segmenter, character filters, and token filters.
335    ///
336    /// # Arguments
337    ///
338    /// * `text` - A reference to the input text (`&str`) that will be tokenized.
339    ///
340    /// # Returns
341    ///
342    /// Returns a `LinderaResult` containing a vector of `Token`s, where each `Token` represents a segment of the tokenized text.
343    ///
344    /// # Process
345    ///
346    /// 1. **Apply character filters**:
347    ///    - If any character filters are defined, they are applied to the input text before tokenization.
348    ///    - The `offsets`, `diffs`, and `text_len` are recorded for each character filter.
349    /// 2. **Segment the text**:
350    ///    - The `segmenter` divides the (potentially filtered) text into tokens.
351    /// 3. **Apply token filters**:
352    ///    - If any token filters are defined, they are applied to the segmented tokens.
353    /// 4. **Correct token offsets**:
354    ///    - If character filters were applied, the byte offsets of each token are corrected to account for changes introduced by those filters.
355    ///
356    /// # Errors
357    ///
358    /// - Returns an error if any of the character or token filters fail during processing.
359    /// - Returns an error if the segmentation process fails.
360    ///
361    /// # Details
362    ///
363    /// - `Cow<'a, str>` is used for the `normalized_text`, allowing the function to either borrow the original text or create an owned version if the text needs modification.
364    /// - If no character filters are applied, the original `text` is used as-is for segmentation.
365    /// - Token offsets are adjusted after the tokenization process if character filters were applied to ensure the byte positions of each token are accurate relative to the original text.
366    pub fn tokenize<'a>(&'a self, text: &'a str) -> LinderaResult<Vec<Token<'a>>> {
367        let mut lattice = Lattice::default();
368        self.tokenize_with_lattice(text, &mut lattice)
369    }
370
371    /// Tokenizes the input text using the tokenizer's segmenter, character filters, and token filters.
372    ///
373    /// # Arguments
374    ///
375    /// * `text` - A reference to the input text (`&str`) that will be tokenized.
376    /// * `lattice` - A mutable reference to a `Lattice` structure. This allows reusing the lattice across multiple calls to avoid memory allocation.
377    ///
378    /// # Returns
379    ///
380    /// Returns a `LinderaResult` containing a vector of `Token`s, where each `Token` represents a segment of the tokenized text.
381    ///
382    /// # Process
383    ///
384    /// 1. **Apply character filters**:
385    ///    - If any character filters are defined, they are applied to the input text before tokenization.
386    ///    - The `offsets`, `diffs`, and `text_len` are recorded for each character filter.
387    /// 2. **Segment the text**:
388    ///    - The `segmenter` divides the (potentially filtered) text into tokens.
389    /// 3. **Apply token filters**:
390    ///    - If any token filters are defined, they are applied to the segmented tokens.
391    /// 4. **Correct token offsets**:
392    ///    - If character filters were applied, the byte offsets of each token are corrected to account for changes introduced by those filters.
393    ///
394    /// # Errors
395    ///
396    /// - Returns an error if any of the character or token filters fail during processing.
397    /// - Returns an error if the segmentation process fails.
398    ///
399    /// # Details
400    ///
401    /// - `Cow<'a, str>` is used for the `normalized_text`, allowing the function to either borrow the original text or create an owned version if the text needs modification.
402    /// - If no character filters are applied, the original `text` is used as-is for segmentation.
403    /// - Token offsets are adjusted after the tokenization process if character filters were applied to ensure the byte positions of each token are accurate relative to the original text.
404    pub fn tokenize_with_lattice<'a>(
405        &'a self,
406        text: &'a str,
407        lattice: &mut Lattice,
408    ) -> LinderaResult<Vec<Token<'a>>> {
409        let mut normalized_text: Cow<'a, str> = Cow::Borrowed(text);
410
411        let mut offset_mappings: Vec<OffsetMapping> =
412            Vec::with_capacity(self.character_filters.len());
413
414        // Apply character filters to the text if it is not empty.
415        // Optimize: Only convert to mutable when we have filters to apply
416        if !self.character_filters.is_empty() {
417            // Convert to owned string once for all filters
418            let text_mut = normalized_text.to_mut();
419
420            for character_filter in &self.character_filters {
421                let mapping = character_filter.apply(text_mut)?;
422
423                if !mapping.is_empty() {
424                    // Record the offset mapping of each character filter in reverse order
425                    // since we need to apply corrections in reverse order
426                    offset_mappings.push(mapping);
427                }
428            }
429        }
430
431        // Store the final text length for offset correction
432        let final_text_len = normalized_text.len();
433
434        // Segment a text.
435        // Segment a text.
436        let mut tokens = self
437            .segmenter
438            .segment_with_lattice(normalized_text, lattice)?;
439
440        // Apply token filters to the tokens if they are not empty.
441        for token_filter in &self.token_filters {
442            token_filter.apply(&mut tokens)?;
443        }
444
445        // Correct token offsets if character filters are applied.
446        correct_offsets(&mut tokens, &offset_mappings, final_text_len);
447
448        Ok(tokens)
449    }
450
451    /// Tokenizes the input text and returns the top-N results.
452    ///
453    /// Each result is a `Vec<Token>` with character/token filters applied.
454    /// Results are ordered by cost (best first).
455    pub fn tokenize_nbest<'a>(
456        &'a self,
457        text: &'a str,
458        n: usize,
459        unique: bool,
460        cost_threshold: Option<i64>,
461    ) -> LinderaResult<Vec<(Vec<Token<'a>>, i64)>> {
462        let mut lattice = Lattice::default();
463        self.tokenize_nbest_with_lattice(text, &mut lattice, n, unique, cost_threshold)
464    }
465
466    /// Tokenizes the input text and returns the top-N results with costs.
467    /// Each result is a (tokens, cost) pair.
468    /// If `unique` is true, results with the same word boundaries are deduplicated.
469    /// If `cost_threshold` is Some(t), paths whose cost exceeds best_cost + t
470    /// are discarded.
471    pub fn tokenize_nbest_with_lattice<'a>(
472        &'a self,
473        text: &'a str,
474        lattice: &mut Lattice,
475        n: usize,
476        unique: bool,
477        cost_threshold: Option<i64>,
478    ) -> LinderaResult<Vec<(Vec<Token<'a>>, i64)>> {
479        let mut normalized_text: Cow<'a, str> = Cow::Borrowed(text);
480
481        let mut offset_mappings: Vec<OffsetMapping> =
482            Vec::with_capacity(self.character_filters.len());
483
484        if !self.character_filters.is_empty() {
485            let text_mut = normalized_text.to_mut();
486            for character_filter in &self.character_filters {
487                let mapping = character_filter.apply(text_mut)?;
488                if !mapping.is_empty() {
489                    offset_mappings.push(mapping);
490                }
491            }
492        }
493
494        let final_text_len = normalized_text.len();
495
496        let mut all_results = self.segmenter.segment_nbest_with_lattice(
497            normalized_text,
498            lattice,
499            n,
500            unique,
501            cost_threshold,
502        )?;
503
504        // Apply token filters and offset corrections to each result
505        for (tokens, _cost) in &mut all_results {
506            for token_filter in &self.token_filters {
507                token_filter.apply(tokens)?;
508            }
509
510            correct_offsets(tokens, &offset_mappings, final_text_len);
511        }
512
513        Ok(all_results)
514    }
515}
516
517/// Corrects token byte offsets back to the original (pre-filter) text by
518/// applying the character filters' offset mappings in reverse order (last
519/// filter first). A no-op when `offset_mappings` is empty.
520///
521/// Shared by `tokenize_with_lattice`, `tokenize_nbest_with_lattice`, and
522/// `AnalysisWorker`.
523///
524/// # 引数
525///
526/// * `tokens` - The tokens whose `byte_start`/`byte_end` are corrected in
527///   place.
528/// * `offset_mappings` - Non-empty mappings recorded by the character
529///   filters, in application order.
530/// * `final_text_len` - Length in bytes of the fully filtered text.
531pub(crate) fn correct_offsets(
532    tokens: &mut [Token<'_>],
533    offset_mappings: &[OffsetMapping],
534    final_text_len: usize,
535) {
536    if offset_mappings.is_empty() {
537        return;
538    }
539    for token in tokens.iter_mut() {
540        // Apply corrections in reverse order to undo the transformations.
541        for mapping in offset_mappings.iter().rev() {
542            token.byte_start = mapping.correct_offset(token.byte_start, final_text_len);
543            token.byte_end = mapping.correct_offset(token.byte_end, final_text_len);
544        }
545    }
546}
547
548impl Clone for Tokenizer {
549    /// Creates a deep clone of the `Tokenizer` instance, including all character filters, token filters, and the segmenter.
550    ///
551    /// # Returns
552    ///
553    /// Returns a new `Tokenizer` instance that is a deep clone of the current instance. All internal filters and the segmenter are cloned.
554    ///
555    /// # Details
556    ///
557    /// - **Character Filters**: Each character filter is cloned by calling its `box_clone` method, which ensures that any dynamically dispatched filters are properly cloned.
558    /// - **Token Filters**: Similarly, each token filter is cloned using the `box_clone` method to handle dynamic dispatch.
559    /// - **Segmenter**: The segmenter is cloned using its `clone` method.
560    ///
561    /// # Notes
562    ///
563    /// - This method performs deep cloning, meaning that all internal filters and segmenter instances are fully duplicated.
564    /// - The `box_clone` method is used to clone the dynamically dispatched filter objects (`BoxCharacterFilter` and `BoxTokenFilter`).
565    fn clone(&self) -> Self {
566        let character_filters: Vec<BoxCharacterFilter> = self
567            .character_filters
568            .iter()
569            .map(|filter| filter.box_clone())
570            .collect();
571
572        let token_filters: Vec<BoxTokenFilter> = self
573            .token_filters
574            .iter()
575            .map(|filter| filter.box_clone())
576            .collect();
577
578        Tokenizer {
579            character_filters,
580            segmenter: self.segmenter.clone(),
581            token_filters,
582        }
583    }
584}
585
586#[cfg(test)]
587mod tests {
588    use super::TokenizerBuilder;
589
590    #[test]
591    fn test_set_segmenter_use_mmap_writes_flat_key_under_segmenter() {
592        let mut builder = TokenizerBuilder::new().unwrap();
593        builder.set_segmenter_use_mmap(true);
594
595        assert_eq!(builder.config["segmenter"]["use_mmap"], true);
596    }
597
598    #[cfg(feature = "embed-ipadic")]
599    #[test]
600    fn test_tokenizer_config_from_slice() {
601        use std::path::PathBuf;
602
603        use crate::tokenizer::yaml_to_config;
604
605        let config_file = PathBuf::from(env!("CARGO_MANIFEST_DIR"))
606            .join("../resources")
607            .join("config")
608            .join("lindera.yml");
609
610        let result = yaml_to_config(&config_file);
611
612        assert!(result.is_ok());
613    }
614
615    #[test]
616    #[cfg(feature = "embed-ipadic")]
617    fn test_tokenizer_config_clone() {
618        use std::path::PathBuf;
619
620        use crate::tokenizer::yaml_to_config;
621
622        let config_file = PathBuf::from(env!("CARGO_MANIFEST_DIR"))
623            .join("../resources")
624            .join("config")
625            .join("lindera.yml");
626
627        let tokenizer_config = yaml_to_config(&config_file).unwrap();
628
629        let cloned_tokenizer_config = tokenizer_config.clone();
630
631        assert_eq!(tokenizer_config, cloned_tokenizer_config);
632    }
633
634    #[test]
635    #[cfg(feature = "embed-ipadic")]
636    fn test_tokenize_ipadic() {
637        use std::borrow::Cow;
638        use std::path::PathBuf;
639
640        use crate::tokenizer::TokenizerBuilder;
641
642        let config_file = PathBuf::from(env!("CARGO_MANIFEST_DIR"))
643            .join("../resources")
644            .join("config")
645            .join("lindera.yml");
646
647        let builder = TokenizerBuilder::from_file(&config_file).unwrap();
648
649        let tokenizer = builder.build().unwrap();
650
651        {
652            let text = "ﾘﾝﾃﾞﾗは形態素解析ｴﾝｼﾞﾝです。";
653            let mut tokens = tokenizer.tokenize(text).unwrap();
654            let mut tokens_iter = tokens.iter_mut();
655            {
656                let token = tokens_iter.next().unwrap();
657                assert_eq!(token.surface, Cow::Borrowed("Lindera"));
658                assert_eq!(token.byte_start, 0);
659                assert_eq!(token.byte_end, 15);
660                assert_eq!(token.position, 0);
661                assert_eq!(token.position_length, 1);
662                assert!(token.word_id.is_unknown());
663                assert_eq!(
664                    token.details,
665                    Some(vec![
666                        Cow::Borrowed("名詞"),
667                        Cow::Borrowed("固有名詞"),
668                        Cow::Borrowed("組織"),
669                        Cow::Borrowed("*"),
670                        Cow::Borrowed("*"),
671                        Cow::Borrowed("*"),
672                        Cow::Borrowed("*"),
673                        Cow::Borrowed("*"),
674                        Cow::Borrowed("*"),
675                    ])
676                );
677            }
678            {
679                let token = tokens_iter.next().unwrap();
680                assert_eq!(token.surface, Cow::Borrowed("形態素"));
681                assert_eq!(token.byte_start, 18);
682                assert_eq!(token.byte_end, 27);
683                assert_eq!(token.position, 2);
684                assert_eq!(token.position_length, 1);
685                assert_eq!(
686                    token.details,
687                    Some(vec![
688                        Cow::Borrowed("名詞"),
689                        Cow::Borrowed("一般"),
690                        Cow::Borrowed("*"),
691                        Cow::Borrowed("*"),
692                        Cow::Borrowed("*"),
693                        Cow::Borrowed("*"),
694                        Cow::Borrowed("形態素"),
695                        Cow::Borrowed("ケイタイソ"),
696                        Cow::Borrowed("ケイタイソ"),
697                    ])
698                );
699            }
700            {
701                let token = tokens_iter.next().unwrap();
702                assert_eq!(token.surface, Cow::Borrowed("解析"));
703                assert_eq!(token.byte_start, 27);
704                assert_eq!(token.byte_end, 33);
705                assert_eq!(token.position, 3);
706                assert_eq!(token.position_length, 1);
707                assert_eq!(
708                    token.details,
709                    Some(vec![
710                        Cow::Borrowed("名詞"),
711                        Cow::Borrowed("サ変接続"),
712                        Cow::Borrowed("*"),
713                        Cow::Borrowed("*"),
714                        Cow::Borrowed("*"),
715                        Cow::Borrowed("*"),
716                        Cow::Borrowed("解析"),
717                        Cow::Borrowed("カイセキ"),
718                        Cow::Borrowed("カイセキ"),
719                    ])
720                );
721            }
722            {
723                let token = tokens_iter.next().unwrap();
724                assert_eq!(token.surface, Cow::Borrowed("エンジン"));
725                assert_eq!(token.byte_start, 33);
726                assert_eq!(token.byte_end, 48);
727                assert_eq!(token.position, 4);
728                assert_eq!(token.position_length, 1);
729                assert_eq!(
730                    token.details,
731                    Some(vec![
732                        Cow::Borrowed("名詞"),
733                        Cow::Borrowed("一般"),
734                        Cow::Borrowed("*"),
735                        Cow::Borrowed("*"),
736                        Cow::Borrowed("*"),
737                        Cow::Borrowed("*"),
738                        Cow::Borrowed("エンジン"),
739                        Cow::Borrowed("エンジン"),
740                        Cow::Borrowed("エンジン"),
741                    ])
742                );
743            }
744
745            let mut tokens_iter = tokens.iter();
746            {
747                let token = tokens_iter.next().unwrap();
748                let start = token.byte_start;
749                let end = token.byte_end;
750                assert_eq!(token.surface, Cow::Borrowed("Lindera"));
751                assert_eq!(&text[start..end], "ﾘﾝﾃﾞﾗ");
752            }
753        }
754
755        {
756            let text = "１０㌎のｶﾞｿﾘﾝ";
757            let mut tokens = tokenizer.tokenize(text).unwrap();
758            let mut tokens_iter = tokens.iter_mut();
759            {
760                // "10" (unknown NUMERIC) and "ガロン" (名詞,接尾,助数詞) are merged
761                // by the japanese_compound_word filter into "10ガロン" with tag "名詞,数".
762                let token = tokens_iter.next().unwrap();
763                assert_eq!(token.surface, Cow::Owned::<str>("10ガロン".into()));
764                assert_eq!(token.byte_start, 0);
765                assert_eq!(token.byte_end, 9);
766                assert_eq!(token.position, 0);
767                assert_eq!(token.position_length, 2);
768            }
769            {
770                let token = tokens_iter.next().unwrap();
771                assert_eq!(token.surface, Cow::Borrowed("ガソリン"));
772                assert_eq!(token.byte_start, 12);
773                assert_eq!(token.byte_end, 27);
774                assert_eq!(token.position, 3);
775                assert_eq!(token.position_length, 1);
776                assert_eq!(
777                    token.details,
778                    Some(vec![
779                        Cow::Borrowed("名詞"),
780                        Cow::Borrowed("一般"),
781                        Cow::Borrowed("*"),
782                        Cow::Borrowed("*"),
783                        Cow::Borrowed("*"),
784                        Cow::Borrowed("*"),
785                        Cow::Borrowed("ガソリン"),
786                        Cow::Borrowed("ガソリン"),
787                        Cow::Borrowed("ガソリン"),
788                    ])
789                );
790            }
791
792            let mut tokens_iter = tokens.iter();
793            {
794                // "10" and "ガロン" are merged by the japanese_compound_word filter
795                let token = tokens_iter.next().unwrap();
796                let start = token.byte_start;
797                let end = token.byte_end;
798                assert_eq!(token.surface, Cow::Owned::<str>("10ガロン".into()));
799                assert_eq!(&text[start..end], "１０㌎");
800            }
801            {
802                let token = tokens_iter.next().unwrap();
803                let start = token.byte_start;
804                let end = token.byte_end;
805                assert_eq!(token.surface, Cow::Borrowed("ガソリン"));
806                assert_eq!(&text[start..end], "ｶﾞｿﾘﾝ");
807            }
808        }
809
810        {
811            let text = "お釣りは百三十四円です。";
812            let mut tokens = tokenizer.tokenize(text).unwrap();
813            let mut tokens_iter = tokens.iter_mut();
814            {
815                let token = tokens_iter.next().unwrap();
816                assert_eq!(token.surface, Cow::Borrowed("お釣り"));
817                assert_eq!(token.byte_start, 0);
818                assert_eq!(token.byte_end, 9);
819                assert_eq!(token.position, 0);
820                assert_eq!(token.position_length, 1);
821                assert_eq!(
822                    token.details,
823                    Some(vec![
824                        Cow::Borrowed("名詞"),
825                        Cow::Borrowed("一般"),
826                        Cow::Borrowed("*"),
827                        Cow::Borrowed("*"),
828                        Cow::Borrowed("*"),
829                        Cow::Borrowed("*"),
830                        Cow::Borrowed("お釣り"),
831                        Cow::Borrowed("オツリ"),
832                        Cow::Borrowed("オツリ"),
833                    ])
834                );
835            }
836            {
837                let token = tokens_iter.next().unwrap();
838                assert_eq!(token.surface, Cow::Borrowed("134円"));
839                assert_eq!(token.byte_start, 12);
840                assert_eq!(token.byte_end, 27);
841                assert_eq!(token.position, 2);
842                assert_eq!(token.position_length, 5);
843                assert_eq!(
844                    token.details,
845                    Some(vec![
846                        Cow::Borrowed("名詞"),
847                        Cow::Borrowed("数"),
848                        Cow::Borrowed("*"),
849                        Cow::Borrowed("*"),
850                        Cow::Borrowed("*"),
851                        Cow::Borrowed("*"),
852                        Cow::Borrowed("*"),
853                        Cow::Borrowed("*"),
854                        Cow::Borrowed("*"),
855                    ])
856                );
857            }
858        }
859
860        {
861            let text = "ここは騒々しい";
862            let mut tokens = tokenizer.tokenize(text).unwrap();
863            let mut tokens_iter = tokens.iter_mut();
864            {
865                let token = tokens_iter.next().unwrap();
866                assert_eq!(token.surface, Cow::Borrowed("ここ"));
867                assert_eq!(token.byte_start, 0);
868                assert_eq!(token.byte_end, 6);
869                assert_eq!(token.position, 0);
870                assert_eq!(token.position_length, 1);
871                assert_eq!(
872                    token.details,
873                    Some(vec![
874                        Cow::Borrowed("名詞"),
875                        Cow::Borrowed("代名詞"),
876                        Cow::Borrowed("一般"),
877                        Cow::Borrowed("*"),
878                        Cow::Borrowed("*"),
879                        Cow::Borrowed("*"),
880                        Cow::Borrowed("ここ"),
881                        Cow::Borrowed("ココ"),
882                        Cow::Borrowed("ココ"),
883                    ])
884                );
885            }
886            {
887                let token = tokens_iter.next().unwrap();
888                assert_eq!(token.surface, Cow::Borrowed("騒騒しい"));
889                assert_eq!(token.byte_start, 9);
890                assert_eq!(token.byte_end, 21);
891                assert_eq!(token.position, 2);
892                assert_eq!(token.position_length, 1);
893                assert_eq!(
894                    token.details,
895                    Some(vec![
896                        Cow::Borrowed("形容詞"),
897                        Cow::Borrowed("自立"),
898                        Cow::Borrowed("*"),
899                        Cow::Borrowed("*"),
900                        Cow::Borrowed("形容詞・イ段"),
901                        Cow::Borrowed("基本形"),
902                        Cow::Borrowed("騒騒しい"),
903                        Cow::Borrowed("ソウゾウシイ"),
904                        Cow::Borrowed("ソーゾーシイ"),
905                    ])
906                );
907            }
908        }
909    }
910
911    #[test]
912    #[cfg(not(windows))]
913    #[should_panic(expected = "No such file or directory")]
914    fn test_create_tokenizer_builder_from_non_existent_file() {
915        use std::path::PathBuf;
916
917        use crate::tokenizer::TokenizerBuilder;
918
919        let config_file = PathBuf::from(env!("CARGO_MANIFEST_DIR"))
920            .join("../resources")
921            .join("config")
922            .join("non_existent_file.yml");
923
924        TokenizerBuilder::from_file(&config_file).unwrap();
925    }
926
927    #[test]
928    #[cfg(windows)]
929    #[should_panic(expected = "The system cannot find the file specified.")]
930    fn test_create_tokenizer_builder_from_non_existent_file() {
931        use std::path::PathBuf;
932
933        use crate::tokenizer::TokenizerBuilder;
934
935        let config_file = PathBuf::from(env!("CARGO_MANIFEST_DIR"))
936            .join("../resources")
937            .join("config")
938            .join("non_existent_file.yml");
939
940        TokenizerBuilder::from_file(&config_file).unwrap();
941    }
942
943    #[test]
944    #[should_panic(expected = "Invalid YAML")]
945    fn test_create_tokenizer_builder_from_invalid_file() {
946        use std::path::PathBuf;
947
948        use crate::tokenizer::TokenizerBuilder;
949
950        let config_file = PathBuf::from(env!("CARGO_MANIFEST_DIR"))
951            .join("../resources")
952            .join("config")
953            .join("invalid.yml");
954
955        TokenizerBuilder::from_file(&config_file).unwrap();
956    }
957
958    #[test]
959    #[cfg(feature = "embed-ipadic")]
960    fn test_tokenize_nbest_1best_matches_tokenize() {
961        use crate::tokenizer::TokenizerBuilder;
962
963        let mut builder = TokenizerBuilder::new().unwrap();
964        builder.set_segmenter_dictionary("embedded://ipadic");
965
966        let tokenizer = builder.build().unwrap();
967
968        let text = "すもももももももものうち";
969        let normal_tokens = tokenizer.tokenize(text).unwrap();
970        let nbest_results = tokenizer.tokenize_nbest(text, 1, false, None).unwrap();
971
972        assert_eq!(nbest_results.len(), 1);
973        let (nbest_tokens, _cost) = &nbest_results[0];
974        assert_eq!(normal_tokens.len(), nbest_tokens.len());
975        for (normal, nbest) in normal_tokens.iter().zip(nbest_tokens.iter()) {
976            assert_eq!(normal.surface.as_ref(), nbest.surface.as_ref());
977        }
978    }
979
980    #[test]
981    #[cfg(feature = "embed-ipadic")]
982    fn test_tokenize_nbest_multiple_results() {
983        use crate::tokenizer::TokenizerBuilder;
984
985        let mut builder = TokenizerBuilder::new().unwrap();
986        builder.set_segmenter_dictionary("embedded://ipadic");
987
988        let tokenizer = builder.build().unwrap();
989
990        let text = "すもももももももものうち";
991        let results = tokenizer.tokenize_nbest(text, 5, false, None).unwrap();
992
993        // Should return multiple results for ambiguous text
994        assert!(results.len() >= 2);
995
996        // All results should cover the full text
997        for (tokens, _cost) in &results {
998            assert!(!tokens.is_empty());
999        }
1000    }
1001}
lindera_analysis/tokenizer.rs

lindera_analysis/
tokenizer.rs
