rishe 0.1.0

Rishe (ریشه): High-performance Persian NLP engine in Rust — tokenization, SBD, lemmatization, ezafe detection, informal→formal normalization
Documentation 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41

pub struct Detokenizer;

impl Detokenizer {
    #[inline]
    pub fn detokenize<S: AsRef<str>>(tokens: &[S]) -> String {
        if tokens.is_empty() {
            return String::new();
        }
        let cap: usize = tokens.iter().map(|t| t.as_ref().len() + 1).sum();
        let mut out = String::with_capacity(cap);
        let mut no_space_before_next = true;

        for tok_ref in tokens {
            let tok = tok_ref.as_ref();
            if tok.is_empty() {
                continue;
            }
            if matches!(tok, "«" | "(" | "[" | "{") {
                if !no_space_before_next {
                    out.push(' ');
                }
                out.push_str(tok);
                no_space_before_next = true;
            } else if matches!(
                tok,
                "." | "،" | "؛" | ":" | "؟" | "?" | "!" | ")" | "]" | "}" | "»" | "%" | "٪"
            ) || tok.chars().all(|c| matches!(c, '.' | '؟' | '?' | '!'))
            {
                out.push_str(tok);
                no_space_before_next = false;
            } else {
                if !no_space_before_next {
                    out.push(' ');
                }
                out.push_str(tok);
                no_space_before_next = false;
            }
        }
        out
    }
}