charabia 0.9.9

A simple library to detect the language, tokenize the text and normalize the tokens
Documentation 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157

use std::borrow::Cow;

use wana_kana::{ConvertJapanese, IsJapaneseStr, Options};

use super::{Normalizer, NormalizerOption};
use crate::detection::{Language, Script};
use crate::Token;

/// Japanese specialized [`Normalizer`].
///
/// This Normalizer uses [`wana_kana`] internally to convert Katakana and Kanji to Hiragana
///
/// The input and output of `is_hiragana` will have identical char len according to manual testing [1],
/// therefore the `options.create_char_map` should be ignored for now [2].
///
/// [wana_kana]: https://docs.rs/wana_kana/latest/wana_kana/
/// [1]: https://github.com/meilisearch/charabia/pull/149#issuecomment-1273540805
/// [2]: https://github.com/meilisearch/charabia/pull/149#discussion_r991337772
pub struct JapaneseNormalizer;

impl Normalizer for JapaneseNormalizer {
    // converting katakana to hiragana doesn't change the characters length,
    // so the `normalize` method is overloaded to skip the useless char_map computing.
    fn normalize<'o>(&self, mut token: Token<'o>, _options: &NormalizerOption) -> Token<'o> {
        // Convert Katakana to Hiragana
        let dst = token.lemma().to_hiragana_with_opt(Options {
            pass_romaji: true, // Otherwise 'ダメ駄目だめHi' would become 'だめ駄目だめひ'
            ..Default::default()
        });

        token.lemma = Cow::Owned(dst);
        token
    }

    fn should_normalize(&self, token: &Token) -> bool {
        token.script == Script::Cj
            && matches!(token.language, None | Some(Language::Jpn))
            && !token.lemma().is_hiragana()
    }
}

#[cfg(test)]
mod test {
    use std::borrow::Cow::Owned;

    use crate::normalizer::test::test_normalizer;
    use crate::token::TokenKind;

    // base tokens to normalize.
    fn tokens() -> Vec<Token<'static>> {
        vec![
            Token {
                lemma: Owned("だめ".to_string()),
                char_end: 2,
                byte_end: 6,
                script: Script::Cj,
                language: Some(Language::Jpn),
                ..Default::default()
            },
            Token {
                lemma: Owned("だめ".to_string()),
                char_end: 2,
                byte_end: 6,
                char_map: Some(vec![(3, 3), (3, 3)]),
                script: Script::Cj,
                language: Some(Language::Jpn),
                ..Default::default()
            },
            Token {
                lemma: Owned("ダメ駄目だめHi".to_string()),
                char_end: 8,
                byte_end: 20,
                script: Script::Cj,
                language: Some(Language::Jpn),
                ..Default::default()
            },
        ]
    }

    // expected result of the current Normalizer.
    fn normalizer_result() -> Vec<Token<'static>> {
        vec![
            Token {
                lemma: Owned("だめ".to_string()),
                char_end: 2,
                byte_end: 6,
                script: Script::Cj,
                language: Some(Language::Jpn),
                ..Default::default()
            },
            Token {
                lemma: Owned("だめ".to_string()),
                char_end: 2,
                byte_end: 6,
                char_map: Some(vec![(3, 3), (3, 3)]),
                script: Script::Cj,
                language: Some(Language::Jpn),
                ..Default::default()
            },
            Token {
                lemma: Owned("だめ駄目だめHi".to_string()),
                char_end: 8,
                byte_end: 20,
                script: Script::Cj,
                language: Some(Language::Jpn),
                ..Default::default()
            },
        ]
    }

    // expected result of the complete Normalizer pieline.
    fn normalized_tokens() -> Vec<Token<'static>> {
        vec![
            Token {
                lemma: Owned("た\u{3099}め".to_string()),
                char_end: 2,
                byte_end: 6,
                char_map: Some(vec![(3, 6), (3, 3)]),
                script: Script::Cj,
                language: Some(Language::Jpn),
                kind: TokenKind::Word,
                ..Default::default()
            },
            Token {
                lemma: Owned("た\u{3099}め".to_string()),
                char_end: 2,
                byte_end: 6,
                char_map: Some(vec![(3, 6), (3, 3)]),
                script: Script::Cj,
                language: Some(Language::Jpn),
                kind: TokenKind::Word,
                ..Default::default()
            },
            Token {
                lemma: Owned("た\u{3099}め駄目た\u{3099}めHi".to_string()),
                char_end: 8,
                byte_end: 20,
                char_map: Some(vec![
                    (3, 6),
                    (3, 3),
                    (3, 3),
                    (3, 3),
                    (3, 6),
                    (3, 3),
                    (1, 1),
                    (1, 1),
                ]),
                script: Script::Cj,
                language: Some(Language::Jpn),
                kind: TokenKind::Word,
                ..Default::default()
            },
        ]
    }

    test_normalizer!(JapaneseNormalizer, tokens(), normalizer_result(), normalized_tokens());
}