rustam 0.1.0

Full Persian NLP pipeline for Rust — normalization, tokenization, stemming, lemmatization, conjugation, spell correction, and more
Documentation 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88

#![allow(missing_docs)]
//! Reader for the Persica Persian news corpus (CSV-like, 7 lines per record).

use std::{
    fs::File,
    io::{BufRead, BufReader},
    iter,
    path::PathBuf,
};

/// A document from the Persica corpus.
pub struct PersicaDoc {
    pub id: i64,
    pub title: String,
    pub text: String,
    pub date: String,
    pub time: String,
    pub category: String,
    pub category2: String,
}

/// Reads the Persica Persian news corpus.
pub struct PersicaReader {
    path: PathBuf,
}

impl PersicaReader {
    /// Opens the Persica corpus file.
    pub fn new(path: impl Into<PathBuf>) -> Self {
        Self { path: path.into() }
    }

    /// Returns an iterator over all documents.
    pub fn docs(&self) -> Box<dyn Iterator<Item = PersicaDoc> + '_> {
        let file = match File::open(&self.path) {
            Ok(f) => f,
            Err(e) => {
                eprintln!("PersicaReader: cannot open {:?}: {}", self.path, e);
                return Box::new(iter::empty());
            }
        };

        let mut lines: Vec<String> = BufReader::new(file)
            .lines()
            .filter_map(|l| l.ok())
            .collect();

        // Strip BOM from first line if present
        if let Some(first) = lines.first_mut() {
            if first.starts_with('\u{FEFF}') {
                *first = first.trim_start_matches('\u{FEFF}').to_string();
            }
        }

        let mut docs: Vec<PersicaDoc> = Vec::new();
        let mut i = 0;
        while i + 6 < lines.len() {
            let chunk = &lines[i..i + 7];

            fn strip_trailing_comma(s: &str) -> String {
                s.trim_end_matches(',').to_string()
            }

            let id_str = strip_trailing_comma(chunk[0].trim());
            let id: i64 = id_str.parse().unwrap_or(0);
            let title = strip_trailing_comma(chunk[1].trim());
            let text = strip_trailing_comma(chunk[2].trim());
            let date = strip_trailing_comma(chunk[3].trim());
            let time = strip_trailing_comma(chunk[4].trim());
            let category = strip_trailing_comma(chunk[5].trim());
            let category2 = strip_trailing_comma(chunk[6].trim());

            docs.push(PersicaDoc {
                id,
                title,
                text,
                date,
                time,
                category,
                category2,
            });

            i += 7;
        }

        Box::new(docs.into_iter())
    }
}