rustam 0.1.0

Full Persian NLP pipeline for Rust — normalization, tokenization, stemming, lemmatization, conjugation, spell correction, and more
Documentation 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96

#![allow(missing_docs)]
//! Reader for the PN Summary Persian news summarization corpus.

use std::{
    fs::File,
    io::{BufRead, BufReader},
    path::PathBuf,
};

/// A document from the PN Summary corpus.
pub struct PnSummaryDoc {
    pub id: String,
    pub title: String,
    pub article: String,
    pub summary: String,
    pub category: String,
    pub categories: Vec<String>,
    pub network: String,
    pub link: String,
}

/// Reads the PN Summary Persian news summarization corpus.
pub struct PnSummaryReader {
    root: PathBuf,
    subset: String,
}

impl PnSummaryReader {
    /// Opens the PN Summary corpus directory with optional subset filter.
    pub fn new(root: impl Into<PathBuf>, subset: impl Into<String>) -> Self {
        Self {
            root: root.into(),
            subset: subset.into(),
        }
    }

    /// Returns an iterator over all documents.
    pub fn docs(&self) -> Box<dyn Iterator<Item = PnSummaryDoc> + '_> {
        let files: Vec<_> = match std::fs::read_dir(&self.root) {
            Ok(entries) => entries
                .flatten()
                .map(|e| e.path())
                .filter(|p| {
                    p.extension().map_or(false, |e| e == "csv") && {
                        let fname = p.file_name().and_then(|n| n.to_str()).unwrap_or("");
                        self.subset == "all" || fname.starts_with(self.subset.as_str())
                    }
                })
                .collect(),
            Err(e) => {
                eprintln!("PnSummaryReader: cannot read dir {:?}: {}", self.root, e);
                return Box::new(std::iter::empty());
            }
        };

        let mut docs: Vec<PnSummaryDoc> = Vec::new();

        for file_path in &files {
            let file = match File::open(file_path) {
                Ok(f) => f,
                Err(e) => {
                    eprintln!("PnSummaryReader: cannot open {:?}: {}", file_path, e);
                    continue;
                }
            };
            let mut first = true;
            for line in BufReader::new(file).lines().filter_map(|l| l.ok()) {
                // Skip header row
                if first {
                    first = false;
                    continue;
                }
                let cols: Vec<&str> = line.splitn(8, '\t').collect();
                if cols.len() < 8 {
                    continue;
                }
                let categories: Vec<String> = cols[4]
                    .split('+')
                    .map(|s| s.to_string())
                    .collect();
                docs.push(PnSummaryDoc {
                    id: cols[0].to_string(),
                    title: cols[1].to_string(),
                    article: cols[2].to_string(),
                    summary: cols[3].to_string(),
                    category: cols[4].to_string(),
                    categories,
                    network: cols[5].to_string(),
                    link: cols[6].to_string(),
                });
            }
        }

        Box::new(docs.into_iter())
    }
}