rishe 0.3.0

Rishe (ریشه): High-performance Persian NLP engine in Rust — tokenization, SBD, lemmatization, ezafe detection, informal→formal normalization
Documentation 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34

use crate::morphology::{MorphologyEngine, ATTACHED_NOUN_SUFFIXES, ZWNJ_NOUN_SUFFIXES};
use std::borrow::Cow;

pub struct Stemmer;

impl Stemmer {
    pub fn stem<'a>(word: &'a str, morph: &MorphologyEngine) -> Cow<'a, str> {
        for &suffix in ZWNJ_NOUN_SUFFIXES {
            if let Some(stem) = word.strip_suffix(suffix) {
                if !stem.is_empty() && morph.is_known_noun(stem) {
                    return Cow::Borrowed(stem);
                }
            }
        }

        if morph.is_known_noun(word) {
            return Cow::Borrowed(word);
        }

        for &suffix in ATTACHED_NOUN_SUFFIXES {
            if let Some(stem) = word.strip_suffix(suffix) {
                if stem.chars().count() >= 2 && morph.is_known_noun(stem) {
                    return Cow::Borrowed(stem);
                }
            }
        }

        let lemma = morph.lemmatize(word);
        if lemma.as_ref() != word {
            return lemma;
        }
        Cow::Borrowed(word)
    }
}