dictx-index 0.1.0

Index builder and binary entry storage for DictX.
Documentation 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50

pub fn expand_for_search(input: &str) -> String {
    let mut out = String::with_capacity(input.len() * 2);
    out.push_str(input);

    let cjk: Vec<char> = input.chars().filter(|ch| is_cjk(*ch)).collect();
    if cjk.is_empty() {
        return out;
    }

    out.push(' ');
    for ch in &cjk {
        out.push(*ch);
        out.push(' ');
    }
    for window in cjk.windows(2) {
        out.extend(window);
        out.push(' ');
    }
    for window in cjk.windows(3) {
        out.extend(window);
        out.push(' ');
    }
    out
}

pub fn is_cjk(ch: char) -> bool {
    matches!(
        ch as u32,
        0x3400..=0x4DBF
            | 0x4E00..=0x9FFF
            | 0xF900..=0xFAFF
            | 0x20000..=0x2A6DF
            | 0x2A700..=0x2B73F
            | 0x2B740..=0x2B81F
            | 0x2B820..=0x2CEAF
    )
}

#[cfg(test)]
mod tests {
    use super::*;

    #[test]
    fn expands_cjk_unigrams_and_bigrams() {
        let text = expand_for_search("苹果");
        assert!(text.contains("苹"));
        assert!(text.contains("果"));
        assert!(text.contains("苹果"));
    }
}