rustam 0.1.0

Full Persian NLP pipeline for Rust — normalization, tokenization, stemming, lemmatization, conjugation, spell correction, and more
Documentation 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73

//! Demonstrates stemming and lemmatization.
//!
//! Run with:
//! ```
//! cargo run --example stem_lemmatize
//! ```

use rustam::{Lemmatizer, Stemmer};

fn main() {
    // -----------------------------------------------------------------------
    // Stemmer
    // -----------------------------------------------------------------------
    let stemmer = Stemmer::new();

    let words = [
        "کتاب‌ها",    // books → کتاب (removes ‌ها plural)
        "کتابی",      // a book → کتاب (removes ی indef.)
        "کتاب‌هایی",  // some books → کتاب
        "اندیشه‌اش",  // his thought → اندیشه
        "خانۀ",       // house (ۀ form) → خانه
        "رفته‌ام",    // I have gone → رفته
        "می‌خواهند",  // they want → می‌خواه
    ];

    println!("=== Stemmer ===");
    for w in &words {
        println!("  {:20} → {}", w, stemmer.stem(w));
    }

    // -----------------------------------------------------------------------
    // Lemmatizer — no POS hint
    // -----------------------------------------------------------------------
    let lem = Lemmatizer::new();

    let nouns_and_adj = [
        ("کتاب‌ها",     ""),     // books
        ("اجتماعی",    "ADJ"),  // social (passthrough for adj)
        ("مردم",       "PRON"), // people (passthrough for pronoun)
        ("آتشفشان",    ""),     // volcano (known word)
    ];

    println!("\n=== Lemmatizer (nouns / adjectives) ===");
    for (w, pos) in &nouns_and_adj {
        println!("  {:20} [{}]  → {}", w, pos, lem.lemmatize(w, pos));
    }

    // Verbs — returned as past#present
    let verbs = [
        ("می‌روم",           ""),  // I go   → رفت#رو
        ("می‌رفتم",          ""),  // I went → رفت#رو
        ("رفته‌ام",          ""),  // I have gone
        ("گفته_شده_است",    ""),  // (compound) has been said → گفت#گو
        ("خواهند_رفت",      ""),  // they will go
        ("می‌بینند",        ""),  // they see → دید#بین
    ];

    println!("\n=== Lemmatizer (verbs) ===");
    for (w, pos) in &verbs {
        println!("  {:25} → {}", w, lem.lemmatize(w, pos));
    }

    // -----------------------------------------------------------------------
    // Lemmatizer with joined verb parts disabled
    //   (use when the input is already pre-split, not joined with _)
    // -----------------------------------------------------------------------
    let lem_no_join = Lemmatizer::with_joined_parts(false);
    println!("\n=== Lemmatizer (joined_parts=false) ===");
    println!(
        "  رفته است  → {}",
        lem_no_join.lemmatize("رفته", "")
    );
}