rustam 0.1.0

Full Persian NLP pipeline for Rust — normalization, tokenization, stemming, lemmatization, conjugation, spell correction, and more
Documentation 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112

use std::collections::HashMap;

use crate::types::{VerbRoots, WordEntry};

// ---------------------------------------------------------------------------
// Embedded data files compiled into the binary at build time
// ---------------------------------------------------------------------------

/// Embedded word lexicon (`words.dat`).
pub const WORDS_DAT: &str = include_str!("../data/words.dat");
/// Embedded verb root list (`verbs.dat`).
pub const VERBS_DAT: &str = include_str!("../data/verbs.dat");
/// Embedded stopword list (`stopwords.dat`).
pub const STOPWORDS_DAT: &str = include_str!("../data/stopwords.dat");
/// Embedded abbreviation list (`abbreviations.dat`).
pub const ABBREVIATIONS_DAT: &str = include_str!("../data/abbreviations.dat");
/// Embedded informal word map (`iwords.dat`).
pub const INFORMAL_WORDS_DAT: &str = include_str!("../data/iwords.dat");
/// Embedded informal verb list (`iverbs.dat`).
pub const INFORMAL_VERBS_DAT: &str = include_str!("../data/iverbs.dat");

// ---------------------------------------------------------------------------
// Parsing helpers
// ---------------------------------------------------------------------------

/// Parses the lexicon file into a HashMap of word → WordEntry.
///
/// Each line has the format: `word\tfrequency\tPOS1,POS2,...`
pub fn parse_words(source: &str) -> HashMap<String, WordEntry> {
    let mut map = HashMap::new();
    for line in source.lines() {
        let parts: Vec<&str> = line.trim().splitn(3, '\t').collect();
        if parts.len() != 3 {
            continue;
        }
        let word = parts[0].to_string();
        let Ok(frequency) = parts[1].parse::<u64>() else {
            continue;
        };
        let pos_tags = parts[2].split(',').map(str::to_string).collect();
        map.insert(word, WordEntry { frequency, pos_tags });
    }
    map
}

/// Parses the verb file into a Vec of VerbRoots.
///
/// Each non-empty line has the format `past#present`.
pub fn parse_verbs(source: &str) -> Vec<VerbRoots> {
    source
        .lines()
        .filter(|l| !l.trim().is_empty())
        .filter_map(|line| {
            let line = line.trim();
            let mut parts = line.splitn(2, '#');
            let past = parts.next()?.to_string();
            let present = parts.next()?.to_string();
            Some(VerbRoots { past, present })
        })
        .collect()
}

/// Returns the list of stopwords as owned strings.
pub fn parse_stopwords(source: &str) -> Vec<String> {
    let mut words: Vec<String> = source
        .lines()
        .map(|l| l.trim().to_string())
        .filter(|l| !l.is_empty())
        .collect::<std::collections::HashSet<_>>()
        .into_iter()
        .collect();
    words.sort();
    words
}

/// Returns the list of abbreviations as owned strings.
pub fn parse_abbreviations(source: &str) -> Vec<String> {
    source
        .lines()
        .map(|l| l.trim().to_string())
        .filter(|l| !l.is_empty())
        .collect()
}

// ---------------------------------------------------------------------------
// Convenience accessors over embedded data
// ---------------------------------------------------------------------------

/// Returns the full word lexicon parsed from the embedded `words.dat`.
pub fn words_list() -> std::collections::HashMap<String, crate::types::WordEntry> {
    parse_words(WORDS_DAT)
}

/// Returns all verb root pairs from the embedded `verbs.dat`.
pub fn verbs_list() -> Vec<crate::types::VerbRoots> {
    parse_verbs(VERBS_DAT)
}

/// Returns the stopword list from the embedded `stopwords.dat`.
pub fn stopwords_list() -> Vec<String> {
    parse_stopwords(STOPWORDS_DAT)
}

/// Returns all past roots from the embedded `verbs.dat`.
pub fn past_roots() -> Vec<String> {
    parse_verbs(VERBS_DAT).into_iter().map(|v| v.past).collect()
}

/// Returns all present roots from the embedded `verbs.dat`.
pub fn present_roots() -> Vec<String> {
    parse_verbs(VERBS_DAT).into_iter().map(|v| v.present).collect()
}