ik-rs 0.3.6

chinese segment, ik-analyzer for rust
Documentation 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102

use unicode_blocks;

#[derive(Debug)]
pub enum CharType {
    USELESS,
    ARABIC,
    ENGLISH,
    CHINESE,
    OtherCjk,
}

impl From<char> for CharType {
    fn from(input: char) -> Self {
        if input >= '0' && input <= '9' {
            return CharType::ARABIC;
        } else if (input >= 'a' && input <= 'z') || (input >= 'A' && input <= 'Z') {
            return CharType::ENGLISH;
        } else {
            if let Some(ub) = unicode_blocks::find_unicode_block(input) {
                if ub == unicode_blocks::CJK_UNIFIED_IDEOGRAPHS
                    || ub == unicode_blocks::CJK_COMPATIBILITY_IDEOGRAPHS
                    || ub == unicode_blocks::CJK_UNIFIED_IDEOGRAPHS_EXTENSION_A
                {
                    return CharType::CHINESE;
                } else if ub == unicode_blocks::HALFWIDTH_AND_FULLWIDTH_FORMS
                    || ub == unicode_blocks::HANGUL_SYLLABLES
                    || ub == unicode_blocks::HANGUL_JAMO
                    || ub == unicode_blocks::HANGUL_COMPATIBILITY_JAMO
                    || ub == unicode_blocks::HIRAGANA
                    || ub == unicode_blocks::KATAKANA
                    || ub == unicode_blocks::KATAKANA_PHONETIC_EXTENSIONS
                {
                    return CharType::OtherCjk;
                }
            }
        }
        return CharType::USELESS;
    }
}

// full char -> half char && lowercase
pub fn regularize(input: char) -> char {
    let mut input_code = u32::from(input);
    if input_code == 12288 {
        input_code -= 12256; // whitespace
    } else if input_code >= 65281 && input_code <= 65374 {
        input_code -= 65248; // full char
    } else if input_code >= 'A' as u32 && input_code <= 'Z' as u32 {
        input_code += 32; // lowercase
    }
    char::from_u32(input_code).unwrap()
}

pub fn regularize_str(input: &str) -> String {
    let mut regular_str = String::from("");
    for c in input.chars() {
        regular_str.push(regularize(c));
    }
    regular_str
}

#[allow(dead_code)]
pub fn regular_char_enum(input: &str) -> Vec<(char, CharType)> {
    let mut pair = Vec::with_capacity(input.chars().count());
    for c in input.chars() {
        let r_char = regularize(c);
        pair.push((r_char, CharType::from(r_char)));
    }
    pair
}

pub fn utf8_slice(s: &str, begin: usize, end: usize) -> &str {
    if end < begin {
        return "";
    }
    s.char_indices()
        .nth(begin)
        .and_then(|(start_pos, _)| {
            if end >= utf8_len(s) {
                return Some(&s[start_pos..]);
            }
            s[start_pos..]
                .char_indices()
                .nth(end - begin)
                .map(|(end_pos, _)| &s[start_pos..start_pos + end_pos])
        })
        .unwrap_or("")
}

#[allow(dead_code)]
pub fn utf8_from(s: &str, begin: usize) -> &str {
    utf8_slice(s, begin, utf8_len(s))
}

#[allow(dead_code)]
pub fn utf8_till(s: &str, end: usize) -> &str {
    utf8_slice(s, 0, end)
}

pub fn utf8_len(s: &str) -> usize {
    s.chars().count()
}