rishe 0.4.0

Rishe (ریشه): High-performance Persian NLP engine in Rust — tokenization, SBD, lemmatization, ezafe detection, informal→formal normalization, stemming, spell checking, embeddings
Documentation 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30

use std::collections::HashSet;
use std::sync::OnceLock;

const STOPWORDS_TEXT: &str = include_str!("../data/stopwords.txt");

fn set() -> &'static HashSet<&'static str> {
    static SET: OnceLock<HashSet<&'static str>> = OnceLock::new();
    SET.get_or_init(|| {
        STOPWORDS_TEXT
            .lines()
            .map(str::trim)
            .filter(|line| !line.is_empty() && !line.starts_with('#'))
            .collect()
    })
}

pub fn is_stopword(word: &str) -> bool {
    set().contains(word)
}

pub fn remove_stopwords(tokens: Vec<String>) -> Vec<String> {
    tokens
        .into_iter()
        .filter(|token| !is_stopword(token))
        .collect()
}

pub fn stopwords() -> Vec<String> {
    set().iter().map(|word| (*word).to_string()).collect()
}