lindera_analysis/
tokenizer.rs

1use std::borrow::Cow;
2use std::env;
3use std::fs::File;
4use std::io::Read;
5use std::path::Path;
6
7use serde_json::{Value, json};
8
9use crate::character_filter::{BoxCharacterFilter, CharacterFilterLoader, OffsetMapping};
10use crate::token_filter::{BoxTokenFilter, TokenFilterLoader};
11use lindera::LinderaResult;
12use lindera::dictionary::Lattice;
13use lindera::error::LinderaErrorKind;
14use lindera::mode::Mode;
15use lindera::segmenter::Segmenter;
16use lindera::space_penalty::SpacePenaltyConfig;
17use lindera::token::Token;
18
19pub type TokenizerConfig = Value;
20
21fn yaml_to_config(file_path: &Path) -> LinderaResult<TokenizerConfig> {
22    let mut input_read = File::open(file_path).map_err(|err| {
23        LinderaErrorKind::Io.with_error(err).add_context(format!(
24            "Failed to open tokenizer config file: {}",
25            file_path.display()
26        ))
27    })?;
28
29    let mut buffer = Vec::new();
30    input_read.read_to_end(&mut buffer).map_err(|err| {
31        LinderaErrorKind::Io.with_error(err).add_context(format!(
32            "Failed to read tokenizer config file: {}",
33            file_path.display()
34        ))
35    })?;
36
37    match serde_yaml_ng::from_slice::<serde_yaml_ng::Value>(&buffer) {
38        Ok(value) => {
39            // Check if the value is a mapping.
40            match value {
41                serde_yaml_ng::Value::Mapping(_) => {
42                    Ok(serde_json::to_value(value).map_err(|err| {
43                        LinderaErrorKind::Deserialize
44                            .with_error(err)
45                            .add_context(format!(
46                                "Failed to convert YAML to JSON for config file: {}",
47                                file_path.display()
48                            ))
49                    })?)
50                }
51                _ => Err(LinderaErrorKind::Deserialize
52                    .with_error(anyhow::anyhow!("Invalid YAML"))
53                    .add_context(format!(
54                        "Config file must contain a YAML mapping: {}",
55                        file_path.display()
56                    ))),
57            }
58        }
59        Err(err) => Err(LinderaErrorKind::Deserialize
60            .with_error(err)
61            .add_context(format!(
62                "Failed to parse YAML config file: {}",
63                file_path.display()
64            ))),
65    }
66}
67
68/// Returns the default configuration as a `serde_json::Value`.
69fn empty_config() -> Value {
70    json!({
71        "segmenter": {},
72        "character_filters": [],
73        "token_filters": []
74    })
75}
76
77/// Ensures that the configuration contains the required keys with default values if absent.
78fn ensure_keys(mut config: Value) -> Value {
79    if config.get("segmenter").is_none() {
80        config["segmenter"] = json!({});
81    }
82
83    if config.get("character_filters").is_none() {
84        config["character_filters"] = json!([]);
85    }
86
87    if config.get("token_filters").is_none() {
88        config["token_filters"] = json!([]);
89    }
90
91    config
92}
93
94#[derive(Debug)]
95pub struct TokenizerBuilder {
96    config: TokenizerConfig,
97}
98
99impl TokenizerBuilder {
100    pub fn new() -> LinderaResult<Self> {
101        if let Ok(config_path) = env::var("LINDERA_CONFIG_PATH") {
102            Self::from_file(Path::new(&config_path))
103        } else {
104            Ok(Self {
105                config: empty_config(),
106            })
107        }
108    }
109
110    pub fn from_file(file_path: &Path) -> LinderaResult<Self> {
111        let config = yaml_to_config(file_path)?;
112
113        Ok(TokenizerBuilder {
114            config: ensure_keys(config),
115        })
116    }
117
118    pub fn from_config(config: TokenizerConfig) -> LinderaResult<Self> {
119        Ok(TokenizerBuilder {
120            config: ensure_keys(config),
121        })
122    }
123
124    pub fn set_segmenter_mode(&mut self, mode: &Mode) -> &mut Self {
125        self.config["segmenter"]["mode"] = json!(mode.as_str());
126        self
127    }
128
129    pub fn set_segmenter_dictionary(&mut self, uri: &str) -> &mut Self {
130        self.config["segmenter"]["dictionary"] = json!(uri);
131        self
132    }
133
134    pub fn set_segmenter_user_dictionary(&mut self, uri: &str) -> &mut Self {
135        self.config["segmenter"]["user_dictionary"] = json!(uri);
136        self
137    }
138
139    pub fn set_segmenter_keep_whitespace(&mut self, keep_whitespace: bool) -> &mut Self {
140        self.config["segmenter"]["keep_whitespace"] = json!(keep_whitespace);
141        self
142    }
143
144    /// Caps unknown-word grouping (MeCab's `max-grouping-size` semantics);
145    /// `0` or an absent key means unbounded grouping, the default.
146    ///
147    /// # Arguments
148    ///
149    /// * `max_grouping_len` - Maximum grouped characters beyond the first.
150    ///
151    /// # Returns
152    ///
153    /// A mutable reference to `self`, for chaining.
154    pub fn set_segmenter_max_grouping_len(&mut self, max_grouping_len: usize) -> &mut Self {
155        self.config["segmenter"]["max_grouping_len"] = json!(max_grouping_len);
156        self
157    }
158
159    /// Enables/disables the unknown-word length ladder (see
160    /// `Segmenter::unknown_word_ladder`; defaults to `true`).
161    ///
162    /// # Arguments
163    ///
164    /// * `unknown_word_ladder` - Whether to emit the length ladder.
165    ///
166    /// # Returns
167    ///
168    /// A mutable reference to `self`, for chaining.
169    pub fn set_segmenter_unknown_word_ladder(&mut self, unknown_word_ladder: bool) -> &mut Self {
170        self.config["segmenter"]["unknown_word_ladder"] = json!(unknown_word_ladder);
171        self
172    }
173
174    /// Sets the left-space penalty rules (see `Segmenter::space_penalty`).
175    /// Pass `None` to turn the penalty off (`space_penalty: false`), which
176    /// also overrides the rules the dictionary ships and applies by default;
177    /// leave the key unset to keep those.
178    ///
179    /// # Arguments
180    ///
181    /// * `space_penalty` - The rules, or `None` to disable the penalty.
182    ///
183    /// # Returns
184    ///
185    /// A mutable reference to `self`, for chaining.
186    pub fn set_segmenter_space_penalty(
187        &mut self,
188        space_penalty: Option<&SpacePenaltyConfig>,
189    ) -> &mut Self {
190        self.config["segmenter"]["space_penalty"] = match space_penalty {
191            Some(config) => json!(config),
192            None => json!(false),
193        };
194        self
195    }
196
197    /// Enables the left-space penalty with the rules the dictionary ships in
198    /// its `metadata.json` (see `Segmenter::space_penalty_from_dictionary`),
199    /// i.e. `space_penalty: true` in the config; `false` turns it off. The
200    /// shipped rules are already the default when the key is absent, so
201    /// `true` mainly makes a dictionary that ships none fail loudly.
202    ///
203    /// # Arguments
204    ///
205    /// * `enabled` - Whether to use the dictionary's rules.
206    ///
207    /// # Returns
208    ///
209    /// A mutable reference to `self`, for chaining.
210    pub fn set_segmenter_space_penalty_from_dictionary(&mut self, enabled: bool) -> &mut Self {
211        self.config["segmenter"]["space_penalty"] = json!(enabled);
212        self
213    }
214
215    /// Removes any left-space penalty setting, so the Segmenter falls back to
216    /// its default: the rules the dictionary ships, if any. This undoes
217    /// [`Self::set_segmenter_space_penalty`] and
218    /// [`Self::set_segmenter_space_penalty_from_dictionary`].
219    ///
220    /// # Returns
221    ///
222    /// A mutable reference to `self`, for chaining.
223    pub fn reset_segmenter_space_penalty(&mut self) -> &mut Self {
224        if let Some(segmenter) = self.config["segmenter"].as_object_mut() {
225            segmenter.remove("space_penalty");
226        }
227        self
228    }
229
230    /// Set whether to route filesystem-loaded dictionaries through
231    /// memory-mapped reads. Ignored for `embedded://` dictionaries.
232    ///
233    /// # Arguments
234    ///
235    /// * `use_mmap` - Whether to request memory-mapped dictionary loading.
236    ///
237    /// # Returns
238    ///
239    /// A mutable reference to `self`, for chaining.
240    pub fn set_segmenter_use_mmap(&mut self, use_mmap: bool) -> &mut Self {
241        self.config["segmenter"]["use_mmap"] = json!(use_mmap);
242        self
243    }
244
245    pub fn append_character_filter(&mut self, kind: &str, args: &Value) -> &mut Self {
246        if let Some(array) = self.config["character_filters"].as_array_mut() {
247            array.push(json!({ "kind": kind, "args": args }));
248        }
249        self
250    }
251
252    pub fn append_token_filter(&mut self, kind: &str, args: &Value) -> &mut Self {
253        if let Some(array) = self.config["token_filters"].as_array_mut() {
254            array.push(json!({ "kind": kind, "args": args }));
255        }
256        self
257    }
258
259    pub fn build(&self) -> LinderaResult<Tokenizer> {
260        Tokenizer::from_config(&self.config).map_err(|err| {
261            LinderaErrorKind::Parse.with_error(anyhow::anyhow!("failed to build tokenizer: {err}"))
262        })
263    }
264}
265
266pub struct Tokenizer {
267    /// Segmenter
268    /// The `segmenter` field is an instance of the `Segmenter` struct, which is responsible for
269    /// segmenting text into tokens. This is a core component of the tokenizer, enabling it to
270    /// break down input text into manageable and meaningful units for further processing.
271    pub segmenter: Segmenter,
272
273    /// Character filters
274    /// A vector of boxed character filters that will be applied to the input text
275    /// before tokenization. Each character filter is responsible for transforming
276    /// the input text in a specific way, such as normalizing characters or removing
277    /// unwanted characters.
278    pub character_filters: Vec<BoxCharacterFilter>,
279
280    /// Token filters
281    /// A vector of boxed token filters that will be applied to the tokens during tokenization.
282    /// Each token filter is a boxed trait object implementing the `TokenFilter` trait, allowing
283    /// for various transformations and processing steps to be applied to the tokens.
284    pub token_filters: Vec<BoxTokenFilter>,
285}
286
287impl Tokenizer {
288    /// Creates a new `Tokenizer` instance from a provided `Segmenter`.
289    ///
290    /// # Arguments
291    ///
292    /// * `segmenter` - An instance of the `Segmenter` struct, which is responsible for the core tokenization process.
293    ///
294    /// # Returns
295    ///
296    /// Returns a new `Tokenizer` instance that uses the provided `segmenter` for tokenization, with empty character and token filters.
297    ///
298    /// # Details
299    ///
300    /// - `segmenter`: The segmenter is responsible for handling the actual segmentation and tokenization of text. It is passed into the `Tokenizer` during initialization.
301    /// - `character_filters`: This is initialized as an empty vector and can be modified later to include character filters.
302    /// - `token_filters`: This is also initialized as an empty vector and can be modified later to include token filters.
303    pub fn new(segmenter: Segmenter) -> Self {
304        Self {
305            segmenter,
306            character_filters: Vec::new(),
307            token_filters: Vec::new(),
308        }
309    }
310
311    pub fn from_config(config: &TokenizerConfig) -> LinderaResult<Self> {
312        let segmenter_config = config.get("segmenter").ok_or_else(|| {
313            LinderaErrorKind::Deserialize.with_error(anyhow::anyhow!("missing segmenter config."))
314        })?;
315        let segmenter = Segmenter::from_config(segmenter_config)?;
316
317        // Create a tokenizer from the segmenter.
318        let mut tokenizer = Tokenizer::new(segmenter);
319
320        // Load character filter settings from the tokenizer config if it is not empty.
321        if let Some(character_filter_settings) = config["character_filters"].as_array() {
322            for character_filter_setting in character_filter_settings {
323                let character_filter_name = character_filter_setting["kind"].as_str();
324                if let Some(character_filter_name) = character_filter_name {
325                    // Append a character filter to the tokenizer.
326                    tokenizer.append_character_filter(CharacterFilterLoader::load_from_value(
327                        character_filter_name,
328                        &character_filter_setting["args"],
329                    )?);
330                }
331            }
332        }
333
334        // Load token filter settings from the tokenizer config if it is not empty.
335        if let Some(token_filter_settings) = config["token_filters"].as_array() {
336            for token_filter_setting in token_filter_settings {
337                let token_filter_name = token_filter_setting["kind"].as_str();
338                if let Some(token_filter_name) = token_filter_name {
339                    // Append a token filter to the tokenizer.
340                    tokenizer.append_token_filter(TokenFilterLoader::load_from_value(
341                        token_filter_name,
342                        &token_filter_setting["args"],
343                    )?);
344                }
345            }
346        }
347
348        Ok(tokenizer)
349    }
350
351    /// Appends a character filter to the tokenizer.
352    ///
353    /// # Arguments
354    ///
355    /// * `character_filter` - A `BoxCharacterFilter` that will be added to the tokenizer. This filter will be applied to the text during the tokenization process.
356    ///
357    /// # Returns
358    ///
359    /// Returns a mutable reference to `Self`, allowing for method chaining.
360    ///
361    /// # Details
362    ///
363    /// - This method adds a new character filter to the `Tokenizer`'s `character_filters` vector.
364    /// - It returns a mutable reference to `self`, allowing multiple character filters to be appended in a chain of method calls.
365    pub fn append_character_filter(&mut self, character_filter: BoxCharacterFilter) -> &mut Self {
366        self.character_filters.push(character_filter);
367
368        self
369    }
370
371    /// Appends a token filter to the tokenizer.
372    ///
373    /// # Arguments
374    ///
375    /// * `token_filter` - A `BoxTokenFilter` that will be added to the tokenizer. This filter will be applied to the tokens after they are segmented.
376    ///
377    /// # Returns
378    ///
379    /// Returns a mutable reference to `Self`, allowing for method chaining.
380    ///
381    /// # Details
382    ///
383    /// - This method adds a new token filter to the `Tokenizer`'s `token_filters` vector.
384    /// - It returns a mutable reference to `self`, allowing multiple token filters to be appended in a chain of method calls.
385    pub fn append_token_filter(&mut self, token_filter: BoxTokenFilter) -> &mut Self {
386        self.token_filters.push(token_filter);
387
388        self
389    }
390
391    /// Tokenizes the input text using the tokenizer's segmenter, character filters, and token filters.
392    ///
393    /// # Arguments
394    ///
395    /// * `text` - A reference to the input text (`&str`) that will be tokenized.
396    ///
397    /// # Returns
398    ///
399    /// Returns a `LinderaResult` containing a vector of `Token`s, where each `Token` represents a segment of the tokenized text.
400    ///
401    /// # Process
402    ///
403    /// 1. **Apply character filters**:
404    ///    - If any character filters are defined, they are applied to the input text before tokenization.
405    ///    - The `offsets`, `diffs`, and `text_len` are recorded for each character filter.
406    /// 2. **Segment the text**:
407    ///    - The `segmenter` divides the (potentially filtered) text into tokens.
408    /// 3. **Apply token filters**:
409    ///    - If any token filters are defined, they are applied to the segmented tokens.
410    /// 4. **Correct token offsets**:
411    ///    - If character filters were applied, the byte offsets of each token are corrected to account for changes introduced by those filters.
412    ///
413    /// # Errors
414    ///
415    /// - Returns an error if any of the character or token filters fail during processing.
416    /// - Returns an error if the segmentation process fails.
417    ///
418    /// # Details
419    ///
420    /// - `Cow<'a, str>` is used for the `normalized_text`, allowing the function to either borrow the original text or create an owned version if the text needs modification.
421    /// - If no character filters are applied, the original `text` is used as-is for segmentation.
422    /// - Token offsets are adjusted after the tokenization process if character filters were applied to ensure the byte positions of each token are accurate relative to the original text.
423    pub fn tokenize<'a>(&'a self, text: &'a str) -> LinderaResult<Vec<Token<'a>>> {
424        let mut lattice = Lattice::default();
425        self.tokenize_with_lattice(text, &mut lattice)
426    }
427
428    /// Tokenizes the input text using the tokenizer's segmenter, character filters, and token filters.
429    ///
430    /// # Arguments
431    ///
432    /// * `text` - A reference to the input text (`&str`) that will be tokenized.
433    /// * `lattice` - A mutable reference to a `Lattice` structure. This allows reusing the lattice across multiple calls to avoid memory allocation.
434    ///
435    /// # Returns
436    ///
437    /// Returns a `LinderaResult` containing a vector of `Token`s, where each `Token` represents a segment of the tokenized text.
438    ///
439    /// # Process
440    ///
441    /// 1. **Apply character filters**:
442    ///    - If any character filters are defined, they are applied to the input text before tokenization.
443    ///    - The `offsets`, `diffs`, and `text_len` are recorded for each character filter.
444    /// 2. **Segment the text**:
445    ///    - The `segmenter` divides the (potentially filtered) text into tokens.
446    /// 3. **Apply token filters**:
447    ///    - If any token filters are defined, they are applied to the segmented tokens.
448    /// 4. **Correct token offsets**:
449    ///    - If character filters were applied, the byte offsets of each token are corrected to account for changes introduced by those filters.
450    ///
451    /// # Errors
452    ///
453    /// - Returns an error if any of the character or token filters fail during processing.
454    /// - Returns an error if the segmentation process fails.
455    ///
456    /// # Details
457    ///
458    /// - `Cow<'a, str>` is used for the `normalized_text`, allowing the function to either borrow the original text or create an owned version if the text needs modification.
459    /// - If no character filters are applied, the original `text` is used as-is for segmentation.
460    /// - Token offsets are adjusted after the tokenization process if character filters were applied to ensure the byte positions of each token are accurate relative to the original text.
461    pub fn tokenize_with_lattice<'a>(
462        &'a self,
463        text: &'a str,
464        lattice: &mut Lattice,
465    ) -> LinderaResult<Vec<Token<'a>>> {
466        let mut normalized_text: Cow<'a, str> = Cow::Borrowed(text);
467
468        let mut offset_mappings: Vec<OffsetMapping> =
469            Vec::with_capacity(self.character_filters.len());
470
471        // Apply character filters to the text if it is not empty.
472        // Optimize: Only convert to mutable when we have filters to apply
473        if !self.character_filters.is_empty() {
474            // Convert to owned string once for all filters
475            let text_mut = normalized_text.to_mut();
476
477            for character_filter in &self.character_filters {
478                let mapping = character_filter.apply(text_mut)?;
479
480                if !mapping.is_empty() {
481                    // Record the offset mapping of each character filter in reverse order
482                    // since we need to apply corrections in reverse order
483                    offset_mappings.push(mapping);
484                }
485            }
486        }
487
488        // Store the final text length for offset correction
489        let final_text_len = normalized_text.len();
490
491        // Segment a text.
492        // Segment a text.
493        let mut tokens = self
494            .segmenter
495            .segment_with_lattice(normalized_text, lattice)?;
496
497        // Apply token filters to the tokens if they are not empty.
498        for token_filter in &self.token_filters {
499            token_filter.apply(&mut tokens)?;
500        }
501
502        // Correct token offsets if character filters are applied.
503        correct_offsets(&mut tokens, &offset_mappings, final_text_len);
504
505        Ok(tokens)
506    }
507
508    /// Tokenizes the input text and returns the top-N results.
509    ///
510    /// Each result is a `Vec<Token>` with character/token filters applied.
511    /// Results are ordered by cost (best first).
512    pub fn tokenize_nbest<'a>(
513        &'a self,
514        text: &'a str,
515        n: usize,
516        unique: bool,
517        cost_threshold: Option<i64>,
518    ) -> LinderaResult<Vec<(Vec<Token<'a>>, i64)>> {
519        let mut lattice = Lattice::default();
520        self.tokenize_nbest_with_lattice(text, &mut lattice, n, unique, cost_threshold)
521    }
522
523    /// Tokenizes the input text and returns the top-N results with costs.
524    /// Each result is a (tokens, cost) pair.
525    /// If `unique` is true, results with the same word boundaries are deduplicated.
526    /// If `cost_threshold` is Some(t), paths whose cost exceeds best_cost + t
527    /// are discarded.
528    pub fn tokenize_nbest_with_lattice<'a>(
529        &'a self,
530        text: &'a str,
531        lattice: &mut Lattice,
532        n: usize,
533        unique: bool,
534        cost_threshold: Option<i64>,
535    ) -> LinderaResult<Vec<(Vec<Token<'a>>, i64)>> {
536        let mut normalized_text: Cow<'a, str> = Cow::Borrowed(text);
537
538        let mut offset_mappings: Vec<OffsetMapping> =
539            Vec::with_capacity(self.character_filters.len());
540
541        if !self.character_filters.is_empty() {
542            let text_mut = normalized_text.to_mut();
543            for character_filter in &self.character_filters {
544                let mapping = character_filter.apply(text_mut)?;
545                if !mapping.is_empty() {
546                    offset_mappings.push(mapping);
547                }
548            }
549        }
550
551        let final_text_len = normalized_text.len();
552
553        let mut all_results = self.segmenter.segment_nbest_with_lattice(
554            normalized_text,
555            lattice,
556            n,
557            unique,
558            cost_threshold,
559        )?;
560
561        // Apply token filters and offset corrections to each result
562        for (tokens, _cost) in &mut all_results {
563            for token_filter in &self.token_filters {
564                token_filter.apply(tokens)?;
565            }
566
567            correct_offsets(tokens, &offset_mappings, final_text_len);
568        }
569
570        Ok(all_results)
571    }
572}
573
574/// Corrects token byte offsets back to the original (pre-filter) text by
575/// applying the character filters' offset mappings in reverse order (last
576/// filter first). A no-op when `offset_mappings` is empty.
577///
578/// Shared by `tokenize_with_lattice`, `tokenize_nbest_with_lattice`, and
579/// `AnalysisWorker`.
580///
581/// # 引数
582///
583/// * `tokens` - The tokens whose `byte_start`/`byte_end` are corrected in
584///   place.
585/// * `offset_mappings` - Non-empty mappings recorded by the character
586///   filters, in application order.
587/// * `final_text_len` - Length in bytes of the fully filtered text.
588pub(crate) fn correct_offsets(
589    tokens: &mut [Token<'_>],
590    offset_mappings: &[OffsetMapping],
591    final_text_len: usize,
592) {
593    if offset_mappings.is_empty() {
594        return;
595    }
596    for token in tokens.iter_mut() {
597        // Apply corrections in reverse order to undo the transformations.
598        for mapping in offset_mappings.iter().rev() {
599            token.byte_start = mapping.correct_offset(token.byte_start, final_text_len);
600            token.byte_end = mapping.correct_offset(token.byte_end, final_text_len);
601        }
602    }
603}
604
605impl Clone for Tokenizer {
606    /// Creates a deep clone of the `Tokenizer` instance, including all character filters, token filters, and the segmenter.
607    ///
608    /// # Returns
609    ///
610    /// Returns a new `Tokenizer` instance that is a deep clone of the current instance. All internal filters and the segmenter are cloned.
611    ///
612    /// # Details
613    ///
614    /// - **Character Filters**: Each character filter is cloned by calling its `box_clone` method, which ensures that any dynamically dispatched filters are properly cloned.
615    /// - **Token Filters**: Similarly, each token filter is cloned using the `box_clone` method to handle dynamic dispatch.
616    /// - **Segmenter**: The segmenter is cloned using its `clone` method.
617    ///
618    /// # Notes
619    ///
620    /// - This method performs deep cloning, meaning that all internal filters and segmenter instances are fully duplicated.
621    /// - The `box_clone` method is used to clone the dynamically dispatched filter objects (`BoxCharacterFilter` and `BoxTokenFilter`).
622    fn clone(&self) -> Self {
623        let character_filters: Vec<BoxCharacterFilter> = self
624            .character_filters
625            .iter()
626            .map(|filter| filter.box_clone())
627            .collect();
628
629        let token_filters: Vec<BoxTokenFilter> = self
630            .token_filters
631            .iter()
632            .map(|filter| filter.box_clone())
633            .collect();
634
635        Tokenizer {
636            character_filters,
637            segmenter: self.segmenter.clone(),
638            token_filters,
639        }
640    }
641}
642
643#[cfg(test)]
644mod tests {
645    use lindera::space_penalty::{SpacePenaltyConfig, SpacePenaltyRule};
646
647    use super::TokenizerBuilder;
648
649    #[test]
650    fn test_set_segmenter_use_mmap_writes_flat_key_under_segmenter() {
651        let mut builder = TokenizerBuilder::new().unwrap();
652        builder.set_segmenter_use_mmap(true);
653
654        assert_eq!(builder.config["segmenter"]["use_mmap"], true);
655    }
656
657    /// Opting out must write `false`: an absent or `null` key means "keep the
658    /// rules the dictionary ships", which is the default since 6.1.
659    #[test]
660    fn test_set_segmenter_space_penalty_writes_false_true_or_rules() {
661        let mut builder = TokenizerBuilder::new().unwrap();
662        assert!(builder.config["segmenter"].get("space_penalty").is_none());
663
664        builder.set_segmenter_space_penalty(None);
665        assert_eq!(builder.config["segmenter"]["space_penalty"], false);
666
667        builder.set_segmenter_space_penalty_from_dictionary(true);
668        assert_eq!(builder.config["segmenter"]["space_penalty"], true);
669
670        let rules = SpacePenaltyConfig::new(vec![SpacePenaltyRule::new(["JKS"], 6000)]);
671        builder.set_segmenter_space_penalty(Some(&rules));
672        assert_eq!(
673            builder.config["segmenter"]["space_penalty"]["rules"][0]["cost"],
674            6000
675        );
676    }
677
678    /// Resetting removes the key again, so the dictionary default applies.
679    #[test]
680    fn test_reset_segmenter_space_penalty_removes_the_key() {
681        let mut builder = TokenizerBuilder::new().unwrap();
682        // Resetting an unset key is a no-op.
683        builder.reset_segmenter_space_penalty();
684        assert!(builder.config["segmenter"].get("space_penalty").is_none());
685
686        builder.set_segmenter_space_penalty(None);
687        builder.reset_segmenter_space_penalty();
688        assert!(builder.config["segmenter"].get("space_penalty").is_none());
689        // Other segmenter keys are untouched.
690        builder.set_segmenter_keep_whitespace(true);
691        builder.set_segmenter_space_penalty_from_dictionary(true);
692        builder.reset_segmenter_space_penalty();
693        assert_eq!(builder.config["segmenter"]["keep_whitespace"], true);
694    }
695
696    #[cfg(feature = "embed-ipadic")]
697    #[test]
698    fn test_tokenizer_config_from_slice() {
699        use std::path::PathBuf;
700
701        use crate::tokenizer::yaml_to_config;
702
703        let config_file = PathBuf::from(env!("CARGO_MANIFEST_DIR"))
704            .join("../resources")
705            .join("config")
706            .join("lindera.yml");
707
708        let result = yaml_to_config(&config_file);
709
710        assert!(result.is_ok());
711    }
712
713    #[test]
714    #[cfg(feature = "embed-ipadic")]
715    fn test_tokenizer_config_clone() {
716        use std::path::PathBuf;
717
718        use crate::tokenizer::yaml_to_config;
719
720        let config_file = PathBuf::from(env!("CARGO_MANIFEST_DIR"))
721            .join("../resources")
722            .join("config")
723            .join("lindera.yml");
724
725        let tokenizer_config = yaml_to_config(&config_file).unwrap();
726
727        let cloned_tokenizer_config = tokenizer_config.clone();
728
729        assert_eq!(tokenizer_config, cloned_tokenizer_config);
730    }
731
732    /// The Python and Node.js docs link their `resources/lindera.yml` as a complete example, so
733    /// both have to stay equal to the shipped configuration, which `test_tokenize_ipadic` loads
734    /// and checks. Comments are not part of the parsed configuration and may differ.
735    #[test]
736    fn test_binding_sample_configs_match_shipped_config() {
737        use std::path::PathBuf;
738
739        use crate::tokenizer::yaml_to_config;
740
741        let root = PathBuf::from(env!("CARGO_MANIFEST_DIR")).join("..");
742        let shipped = yaml_to_config(&root.join("resources/config/lindera.yml")).unwrap();
743
744        for sample in [
745            "lindera-python/resources/lindera.yml",
746            "lindera-nodejs/resources/lindera.yml",
747        ] {
748            let config = yaml_to_config(&root.join(sample)).unwrap();
749            assert_eq!(
750                config, shipped,
751                "{sample} differs from the shipped configuration"
752            );
753        }
754    }
755
756    #[test]
757    #[cfg(feature = "embed-ipadic")]
758    fn test_tokenize_ipadic() {
759        use std::borrow::Cow;
760        use std::path::PathBuf;
761
762        use crate::tokenizer::TokenizerBuilder;
763
764        let config_file = PathBuf::from(env!("CARGO_MANIFEST_DIR"))
765            .join("../resources")
766            .join("config")
767            .join("lindera.yml");
768
769        let builder = TokenizerBuilder::from_file(&config_file).unwrap();
770
771        let tokenizer = builder.build().unwrap();
772
773        {
774            let text = "ﾘﾝﾃﾞﾗは形態素解析ｴﾝｼﾞﾝです。";
775            let mut tokens = tokenizer.tokenize(text).unwrap();
776            let mut tokens_iter = tokens.iter_mut();
777            {
778                let token = tokens_iter.next().unwrap();
779                assert_eq!(token.surface, Cow::Borrowed("Lindera"));
780                assert_eq!(token.byte_start, 0);
781                assert_eq!(token.byte_end, 15);
782                assert_eq!(token.position, 0);
783                assert_eq!(token.position_length, 1);
784                assert!(token.word_id.is_unknown());
785                assert_eq!(
786                    token.details,
787                    Some(vec![
788                        Cow::Borrowed("名詞"),
789                        Cow::Borrowed("固有名詞"),
790                        Cow::Borrowed("組織"),
791                        Cow::Borrowed("*"),
792                        Cow::Borrowed("*"),
793                        Cow::Borrowed("*"),
794                        Cow::Borrowed("*"),
795                        Cow::Borrowed("*"),
796                        Cow::Borrowed("*"),
797                    ])
798                );
799            }
800            {
801                let token = tokens_iter.next().unwrap();
802                assert_eq!(token.surface, Cow::Borrowed("形態素"));
803                assert_eq!(token.byte_start, 18);
804                assert_eq!(token.byte_end, 27);
805                assert_eq!(token.position, 2);
806                assert_eq!(token.position_length, 1);
807                assert_eq!(
808                    token.details,
809                    Some(vec![
810                        Cow::Borrowed("名詞"),
811                        Cow::Borrowed("一般"),
812                        Cow::Borrowed("*"),
813                        Cow::Borrowed("*"),
814                        Cow::Borrowed("*"),
815                        Cow::Borrowed("*"),
816                        Cow::Borrowed("形態素"),
817                        Cow::Borrowed("ケイタイソ"),
818                        Cow::Borrowed("ケイタイソ"),
819                    ])
820                );
821            }
822            {
823                let token = tokens_iter.next().unwrap();
824                assert_eq!(token.surface, Cow::Borrowed("解析"));
825                assert_eq!(token.byte_start, 27);
826                assert_eq!(token.byte_end, 33);
827                assert_eq!(token.position, 3);
828                assert_eq!(token.position_length, 1);
829                assert_eq!(
830                    token.details,
831                    Some(vec![
832                        Cow::Borrowed("名詞"),
833                        Cow::Borrowed("サ変接続"),
834                        Cow::Borrowed("*"),
835                        Cow::Borrowed("*"),
836                        Cow::Borrowed("*"),
837                        Cow::Borrowed("*"),
838                        Cow::Borrowed("解析"),
839                        Cow::Borrowed("カイセキ"),
840                        Cow::Borrowed("カイセキ"),
841                    ])
842                );
843            }
844            {
845                let token = tokens_iter.next().unwrap();
846                assert_eq!(token.surface, Cow::Borrowed("エンジン"));
847                assert_eq!(token.byte_start, 33);
848                assert_eq!(token.byte_end, 48);
849                assert_eq!(token.position, 4);
850                assert_eq!(token.position_length, 1);
851                assert_eq!(
852                    token.details,
853                    Some(vec![
854                        Cow::Borrowed("名詞"),
855                        Cow::Borrowed("一般"),
856                        Cow::Borrowed("*"),
857                        Cow::Borrowed("*"),
858                        Cow::Borrowed("*"),
859                        Cow::Borrowed("*"),
860                        Cow::Borrowed("エンジン"),
861                        Cow::Borrowed("エンジン"),
862                        Cow::Borrowed("エンジン"),
863                    ])
864                );
865            }
866
867            let mut tokens_iter = tokens.iter();
868            {
869                let token = tokens_iter.next().unwrap();
870                let start = token.byte_start;
871                let end = token.byte_end;
872                assert_eq!(token.surface, Cow::Borrowed("Lindera"));
873                assert_eq!(&text[start..end], "ﾘﾝﾃﾞﾗ");
874            }
875        }
876
877        {
878            let text = "１０㌎のｶﾞｿﾘﾝ";
879            let mut tokens = tokenizer.tokenize(text).unwrap();
880            let mut tokens_iter = tokens.iter_mut();
881            {
882                // "10" (unknown NUMERIC) is converted by japanese_number; the counter
883                // "ガロン" (名詞,接尾,助数詞) is not merged into it by the shipped configuration.
884                let token = tokens_iter.next().unwrap();
885                assert_eq!(token.surface, Cow::Owned::<str>("10".into()));
886                assert_eq!(token.byte_start, 0);
887                assert_eq!(token.byte_end, 6);
888                assert_eq!(token.position, 0);
889                assert_eq!(token.position_length, 1);
890            }
891            {
892                let token = tokens_iter.next().unwrap();
893                assert_eq!(token.surface, Cow::Borrowed("ガロン"));
894                assert_eq!(token.byte_start, 6);
895                assert_eq!(token.byte_end, 9);
896                assert_eq!(token.position, 1);
897                assert_eq!(token.position_length, 1);
898                assert_eq!(&token.details()[0..3], ["名詞", "接尾", "助数詞"]);
899            }
900            {
901                let token = tokens_iter.next().unwrap();
902                assert_eq!(token.surface, Cow::Borrowed("ガソリン"));
903                assert_eq!(token.byte_start, 12);
904                assert_eq!(token.byte_end, 27);
905                assert_eq!(token.position, 3);
906                assert_eq!(token.position_length, 1);
907                assert_eq!(
908                    token.details,
909                    Some(vec![
910                        Cow::Borrowed("名詞"),
911                        Cow::Borrowed("一般"),
912                        Cow::Borrowed("*"),
913                        Cow::Borrowed("*"),
914                        Cow::Borrowed("*"),
915                        Cow::Borrowed("*"),
916                        Cow::Borrowed("ガソリン"),
917                        Cow::Borrowed("ガソリン"),
918                        Cow::Borrowed("ガソリン"),
919                    ])
920                );
921            }
922
923            let mut tokens_iter = tokens.iter();
924            {
925                let token = tokens_iter.next().unwrap();
926                let start = token.byte_start;
927                let end = token.byte_end;
928                assert_eq!(token.surface, Cow::Owned::<str>("10".into()));
929                assert_eq!(&text[start..end], "１０");
930            }
931            {
932                let token = tokens_iter.next().unwrap();
933                let start = token.byte_start;
934                let end = token.byte_end;
935                assert_eq!(token.surface, Cow::Borrowed("ガロン"));
936                assert_eq!(&text[start..end], "㌎");
937            }
938            {
939                let token = tokens_iter.next().unwrap();
940                let start = token.byte_start;
941                let end = token.byte_end;
942                assert_eq!(token.surface, Cow::Borrowed("ガソリン"));
943                assert_eq!(&text[start..end], "ｶﾞｿﾘﾝ");
944            }
945        }
946
947        {
948            let text = "お釣りは百三十四円です。";
949            let mut tokens = tokenizer.tokenize(text).unwrap();
950            let mut tokens_iter = tokens.iter_mut();
951            {
952                let token = tokens_iter.next().unwrap();
953                assert_eq!(token.surface, Cow::Borrowed("お釣り"));
954                assert_eq!(token.byte_start, 0);
955                assert_eq!(token.byte_end, 9);
956                assert_eq!(token.position, 0);
957                assert_eq!(token.position_length, 1);
958                assert_eq!(
959                    token.details,
960                    Some(vec![
961                        Cow::Borrowed("名詞"),
962                        Cow::Borrowed("一般"),
963                        Cow::Borrowed("*"),
964                        Cow::Borrowed("*"),
965                        Cow::Borrowed("*"),
966                        Cow::Borrowed("*"),
967                        Cow::Borrowed("お釣り"),
968                        Cow::Borrowed("オツリ"),
969                        Cow::Borrowed("オツリ"),
970                    ])
971                );
972            }
973            {
974                // The shipped configuration merges the numeral tokens only, so the counter
975                // stays a separate token and the number is converted correctly.
976                let token = tokens_iter.next().unwrap();
977                assert_eq!(token.surface, Cow::Borrowed("134"));
978                assert_eq!(token.byte_start, 12);
979                assert_eq!(token.byte_end, 24);
980                assert_eq!(token.position, 2);
981                assert_eq!(token.position_length, 4);
982                assert_eq!(
983                    token.details,
984                    Some(vec![
985                        Cow::Borrowed("名詞"),
986                        Cow::Borrowed("数"),
987                        Cow::Borrowed("*"),
988                        Cow::Borrowed("*"),
989                        Cow::Borrowed("*"),
990                        Cow::Borrowed("*"),
991                        Cow::Borrowed("*"),
992                        Cow::Borrowed("*"),
993                        Cow::Borrowed("*"),
994                    ])
995                );
996            }
997            {
998                let token = tokens_iter.next().unwrap();
999                assert_eq!(token.surface, Cow::Borrowed("円"));
1000                assert_eq!(token.byte_start, 24);
1001                assert_eq!(token.byte_end, 27);
1002                assert_eq!(token.position, 6);
1003                assert_eq!(token.position_length, 1);
1004                assert_eq!(
1005                    token.details,
1006                    Some(vec![
1007                        Cow::Borrowed("名詞"),
1008                        Cow::Borrowed("接尾"),
1009                        Cow::Borrowed("助数詞"),
1010                        Cow::Borrowed("*"),
1011                        Cow::Borrowed("*"),
1012                        Cow::Borrowed("*"),
1013                        Cow::Borrowed("円"),
1014                        Cow::Borrowed("エン"),
1015                        Cow::Borrowed("エン"),
1016                    ])
1017                );
1018            }
1019        }
1020
1021        {
1022            // The counter stays a separate token. Merged into the number token it would make the
1023            // token something other than a numeral, and `一万円` would be left unconverted.
1024            let text = "一万円";
1025            let tokens = tokenizer.tokenize(text).unwrap();
1026            let surfaces: Vec<&str> = tokens.iter().map(|token| token.surface.as_ref()).collect();
1027            assert_eq!(surfaces, ["10000", "円"]);
1028        }
1029
1030        {
1031            let text = "ここは騒々しい";
1032            let mut tokens = tokenizer.tokenize(text).unwrap();
1033            let mut tokens_iter = tokens.iter_mut();
1034            {
1035                let token = tokens_iter.next().unwrap();
1036                assert_eq!(token.surface, Cow::Borrowed("ここ"));
1037                assert_eq!(token.byte_start, 0);
1038                assert_eq!(token.byte_end, 6);
1039                assert_eq!(token.position, 0);
1040                assert_eq!(token.position_length, 1);
1041                assert_eq!(
1042                    token.details,
1043                    Some(vec![
1044                        Cow::Borrowed("名詞"),
1045                        Cow::Borrowed("代名詞"),
1046                        Cow::Borrowed("一般"),
1047                        Cow::Borrowed("*"),
1048                        Cow::Borrowed("*"),
1049                        Cow::Borrowed("*"),
1050                        Cow::Borrowed("ここ"),
1051                        Cow::Borrowed("ココ"),
1052                        Cow::Borrowed("ココ"),
1053                    ])
1054                );
1055            }
1056            {
1057                let token = tokens_iter.next().unwrap();
1058                assert_eq!(token.surface, Cow::Borrowed("騒騒しい"));
1059                assert_eq!(token.byte_start, 9);
1060                assert_eq!(token.byte_end, 21);
1061                assert_eq!(token.position, 2);
1062                assert_eq!(token.position_length, 1);
1063                assert_eq!(
1064                    token.details,
1065                    Some(vec![
1066                        Cow::Borrowed("形容詞"),
1067                        Cow::Borrowed("自立"),
1068                        Cow::Borrowed("*"),
1069                        Cow::Borrowed("*"),
1070                        Cow::Borrowed("形容詞・イ段"),
1071                        Cow::Borrowed("基本形"),
1072                        Cow::Borrowed("騒騒しい"),
1073                        Cow::Borrowed("ソウゾウシイ"),
1074                        Cow::Borrowed("ソーゾーシイ"),
1075                    ])
1076                );
1077            }
1078        }
1079    }
1080
1081    #[test]
1082    #[cfg(not(windows))]
1083    #[should_panic(expected = "No such file or directory")]
1084    fn test_create_tokenizer_builder_from_non_existent_file() {
1085        use std::path::PathBuf;
1086
1087        use crate::tokenizer::TokenizerBuilder;
1088
1089        let config_file = PathBuf::from(env!("CARGO_MANIFEST_DIR"))
1090            .join("../resources")
1091            .join("config")
1092            .join("non_existent_file.yml");
1093
1094        TokenizerBuilder::from_file(&config_file).unwrap();
1095    }
1096
1097    #[test]
1098    #[cfg(windows)]
1099    #[should_panic(expected = "The system cannot find the file specified.")]
1100    fn test_create_tokenizer_builder_from_non_existent_file() {
1101        use std::path::PathBuf;
1102
1103        use crate::tokenizer::TokenizerBuilder;
1104
1105        let config_file = PathBuf::from(env!("CARGO_MANIFEST_DIR"))
1106            .join("../resources")
1107            .join("config")
1108            .join("non_existent_file.yml");
1109
1110        TokenizerBuilder::from_file(&config_file).unwrap();
1111    }
1112
1113    #[test]
1114    #[should_panic(expected = "Invalid YAML")]
1115    fn test_create_tokenizer_builder_from_invalid_file() {
1116        use std::path::PathBuf;
1117
1118        use crate::tokenizer::TokenizerBuilder;
1119
1120        let config_file = PathBuf::from(env!("CARGO_MANIFEST_DIR"))
1121            .join("../resources")
1122            .join("config")
1123            .join("invalid.yml");
1124
1125        TokenizerBuilder::from_file(&config_file).unwrap();
1126    }
1127
1128    #[test]
1129    #[cfg(feature = "embed-ipadic")]
1130    fn test_tokenize_nbest_1best_matches_tokenize() {
1131        use crate::tokenizer::TokenizerBuilder;
1132
1133        let mut builder = TokenizerBuilder::new().unwrap();
1134        builder.set_segmenter_dictionary("embedded://ipadic");
1135
1136        let tokenizer = builder.build().unwrap();
1137
1138        let text = "すもももももももものうち";
1139        let normal_tokens = tokenizer.tokenize(text).unwrap();
1140        let nbest_results = tokenizer.tokenize_nbest(text, 1, false, None).unwrap();
1141
1142        assert_eq!(nbest_results.len(), 1);
1143        let (nbest_tokens, _cost) = &nbest_results[0];
1144        assert_eq!(normal_tokens.len(), nbest_tokens.len());
1145        for (normal, nbest) in normal_tokens.iter().zip(nbest_tokens.iter()) {
1146            assert_eq!(normal.surface.as_ref(), nbest.surface.as_ref());
1147        }
1148    }
1149
1150    #[test]
1151    #[cfg(feature = "embed-ipadic")]
1152    fn test_tokenize_nbest_multiple_results() {
1153        use crate::tokenizer::TokenizerBuilder;
1154
1155        let mut builder = TokenizerBuilder::new().unwrap();
1156        builder.set_segmenter_dictionary("embedded://ipadic");
1157
1158        let tokenizer = builder.build().unwrap();
1159
1160        let text = "すもももももももものうち";
1161        let results = tokenizer.tokenize_nbest(text, 5, false, None).unwrap();
1162
1163        // Should return multiple results for ambiguous text
1164        assert!(results.len() >= 2);
1165
1166        // All results should cover the full text
1167        for (tokens, _cost) in &results {
1168            assert!(!tokens.is_empty());
1169        }
1170    }
1171}
lindera_analysis/tokenizer.rs

lindera_analysis/
tokenizer.rs
