rustam 0.1.0

Full Persian NLP pipeline for Rust — normalization, tokenization, stemming, lemmatization, conjugation, spell correction, and more
Documentation 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53

#![allow(missing_docs)]
//! Reader for the MirasText Persian corpus.

use std::{
    fs::File,
    io::{BufRead, BufReader},
    iter,
    path::PathBuf,
};

/// A document from the MirasText corpus.
pub struct MirasDoc {
    pub text: String,
}

/// Reads the MirasText Persian corpus (plain text, `***`-delimited fields).
pub struct MirasTextReader {
    path: PathBuf,
}

impl MirasTextReader {
    /// Opens the MirasText corpus file.
    pub fn new(path: impl Into<PathBuf>) -> Self {
        Self { path: path.into() }
    }

    /// Returns an iterator over all documents.
    pub fn docs(&self) -> Box<dyn Iterator<Item = MirasDoc> + '_> {
        let file = match File::open(&self.path) {
            Ok(f) => f,
            Err(e) => {
                eprintln!("MirasTextReader: cannot open {:?}: {}", self.path, e);
                return Box::new(iter::empty());
            }
        };

        let docs: Vec<MirasDoc> = BufReader::new(file)
            .lines()
            .filter_map(|l| l.ok())
            .filter_map(|line| {
                let text = line.splitn(2, "***").next()?.to_string();
                if text.is_empty() { None } else { Some(MirasDoc { text }) }
            })
            .collect();

        Box::new(docs.into_iter())
    }

    /// Returns an iterator over all document texts.
    pub fn texts(&self) -> Box<dyn Iterator<Item = String> + '_> {
        Box::new(self.docs().map(|d| d.text))
    }
}