yuru_core/
normalize.rs

1use unicode_normalization::UnicodeNormalization;
2
3pub fn normalize(text: &str) -> String {
4    text.nfkc()
5        .flat_map(char::to_lowercase)
6        .map(fold_width_compatible_char)
7        .collect()
8}
9
10pub fn fold_width_compatible_char(ch: char) -> char {
11    match ch {
12        '-' | '\u{2010}' | '\u{2011}' | '\u{2012}' | '\u{2013}' | '\u{2014}' | '\u{2015}'
13        | '\u{2212}' | '\u{30a0}' | '\u{30fc}' | '\u{fe58}' | '\u{fe63}' | '\u{ff0d}'
14        | '\u{ff70}' => '-',
15        _ => ch,
16    }
17}
18
19pub fn katakana_to_hiragana(text: &str) -> String {
20    text.chars()
21        .map(|ch| {
22            if ('ァ'..='ヶ').contains(&ch) {
23                char::from_u32(ch as u32 - 0x60).unwrap_or(ch)
24            } else {
25                ch
26            }
27        })
28        .collect()
29}
30
31pub fn hiragana_to_katakana(text: &str) -> String {
32    text.chars()
33        .map(|ch| {
34            if ('ぁ'..='ゖ').contains(&ch) {
35                char::from_u32(ch as u32 + 0x60).unwrap_or(ch)
36            } else {
37                ch
38            }
39        })
40        .collect()
41}
42
43pub fn contains_kana(text: &str) -> bool {
44    text.chars()
45        .any(|ch| ('ぁ'..='ゖ').contains(&ch) || ('ァ'..='ヶ').contains(&ch))
46}
47
48#[cfg(test)]
49mod tests {
50    use super::*;
51
52    #[test]
53    fn normalize_ascii_lowercase() {
54        assert_eq!(normalize("README.MD"), "readme.md");
55    }
56
57    #[test]
58    fn normalize_fullwidth_ascii_nfkc() {
59        assert_eq!(normalize("ＡＢＣ１２３"), "abc123");
60    }
61
62    #[test]
63    fn normalize_halfwidth_katakana() {
64        assert_eq!(normalize("ｶﾒﾗ"), "カメラ");
65    }
66
67    #[test]
68    fn normalize_folds_dash_and_prolonged_sound_width_variants() {
69        assert_eq!(normalize("ハッピー-ｰ－―−゠"), "ハッピ-------");
70    }
71
72    #[test]
73    fn normalize_folds_fullwidth_space() {
74        assert_eq!(normalize("2025年8月　PDF"), "2025年8月 pdf");
75    }
76
77    #[test]
78    fn katakana_to_hiragana_basic() {
79        assert_eq!(katakana_to_hiragana("カメラ"), "かめら");
80    }
81
82    #[test]
83    fn hiragana_to_katakana_basic() {
84        assert_eq!(hiragana_to_katakana("しんじゅく"), "シンジュク");
85    }
86}
yuru_core/normalize.rs

yuru_core/
normalize.rs
