rustam 0.1.0

Full Persian NLP pipeline for Rust — normalization, tokenization, stemming, lemmatization, conjugation, spell correction, and more
Documentation 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79

//! Demonstrates every normalization step available in [`Normalizer`].
//!
//! Run with:
//! ```
//! cargo run --example normalize
//! ```

use rustam::{Normalizer, NormalizerConfig};

fn main() {
    let n = Normalizer::new();

    // -----------------------------------------------------------------------
    // Full pipeline (all steps enabled)
    // -----------------------------------------------------------------------
    let raw = "اِعلاممممم کَرد : « زمین لرزه ای به بُزرگیِ 6 دهم ریشتر ...»";
    println!("Input:      {raw}");
    println!("Normalized: {}\n", n.normalize(raw));

    // -----------------------------------------------------------------------
    // Individual steps
    // -----------------------------------------------------------------------

    // 1. Remove Arabic diacritics (tashkeel)
    let diacritics = "حَذفِ اِعراب";
    println!("Diacritics in:  {diacritics}");
    println!("Diacritics out: {}\n", n.remove_diacritics(diacritics));

    // 2. Convert digits to Persian (ASCII → ۰–۹, Arabic-Indic → ۰–۹)
    let digits = "score: 98.6 و ٤٢ امتیاز";
    println!("Digits in:  {digits}");
    println!("Digits out: {}\n", n.persian_number(digits));

    // 3. Persian typographic style (guillemets, ellipsis, decimal point)
    let style_in = "\"نرمال‌سازی\" و 6.5 درصد و ...";
    println!("Style in:  {style_in}");
    println!("Style out: {}\n", n.persian_style(style_in));

    // 4. Remove decorative Unicode special characters
    let special = "پیامبر اکرم \u{FDFA} بزرگ است";
    println!("Specials in:  {special}");
    println!("Specials out: {}\n", n.remove_specials_chars(&special));

    // 5. Reduce excessive character repetition (3+ → ≤2)
    let repeated = "خیلیییییی زیباست";
    println!("Repeated in:  {repeated}");
    println!("Repeated out: {}\n", n.decrease_repeated_chars(repeated));

    // 6. Separate می/نمی prefix with ZWNJ
    let mi_in = "میروم و نمیدانم";
    println!("می in:  {mi_in}");
    println!("می out: {}\n", n.seperate_mi(mi_in));

    // 7. Replace Unicode ligatures
    let ligature = "\u{FDF2} الله";  // U+FDF2 ARABIC LIGATURE ALLAH
    println!("Ligature in:  {ligature}");
    println!("Ligature out: {}\n", n.unicodes_replacement(ligature));

    // 8. Fix spacing around punctuation
    let spacing = "کتاب ،که خوب بود،  را خواندم  .";
    println!("Spacing in:  {spacing}");
    println!("Spacing out: {}\n", n.correct_spacing(spacing));

    // -----------------------------------------------------------------------
    // Custom config: only diacritics removal and Persian numbers
    // -----------------------------------------------------------------------
    let minimal = Normalizer::with_config(NormalizerConfig {
        correct_spacing: false,
        remove_diacritics: true,
        remove_specials_chars: false,
        decrease_repeated_chars: false,
        persian_style: false,
        persian_numbers: true,
        unicodes_replacement: false,
        seperate_mi: false,
    });
    println!("Minimal config input:  {raw}");
    println!("Minimal config output: {}", minimal.normalize(raw));
}