sentencex 0.1.26

Sentence segmentation library with wide language support optimized for speed and utility.
Documentation 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130

mod am;
mod ar;
mod bg;
mod bn;
mod ca;
mod da;
mod de;
mod el;
mod en;
mod es;
mod fallbacks;
mod fi;
mod fr;
mod gu;
mod hi;
mod hy;
mod it;
mod ja;
mod kk;
mod kn;
mod language;
mod list_markers;
mod ml;
mod mr;
mod my;
mod nl;
mod pa;
mod pl;
mod pt;
mod ru;
mod sk;
mod ta;
mod te;
mod uk;

pub use am::Amharic;
pub use ar::Arabic;
pub use bg::Bulgarian;
pub use bn::Bengali;
pub use ca::Catalan;
pub use da::Danish;
pub use de::German;
pub use el::Greek;
pub use en::English;
pub use es::Spanish;
pub use fallbacks::get_fallbacks;
pub use fi::Finnish;
pub use fr::French;
pub use gu::Gujarati;
pub use hi::Hindi;
pub use hy::Armenian;
pub use it::Italian;
pub use ja::Japanese;
pub use kk::Kazakh;
pub use kn::Kannada;
pub use language::Language;
pub use ml::Malayalam;
pub use mr::Marathi;
pub use my::Burmese;
pub use nl::Dutch;
pub use pa::Punjabi;
pub use pl::Polish;
pub use pt::Portuguese;
pub use ru::Russian;
pub use sk::Slovak;
pub use ta::Tamil;
pub use te::Telugu;
pub use uk::Ukrainian;

#[cfg(test)]
mod tests {
    use std::fs;

    use super::*;

    pub fn run_language_tests<T: Language>(language: T, test_file: &str) {
        let raw = fs::read_to_string(test_file).expect("Failed to read test file");
        let content = raw.replace("\r\n", "\n");
        let test_cases: Vec<&str> = content.split("===").collect();

        for case in test_cases {
            // Explicit '#' comment handling. This lets fixture files include
            // section headers and structural comments anywhere — not just at
            // the very top of the file — without silently skipping cases that
            // happen to share a chunk with leading comments.
            let cleaned: String = case
                .lines()
                .filter(|line| !line.trim_start().starts_with('#'))
                .collect::<Vec<_>>()
                .join("\n");

            let cleaned = cleaned.trim();
            if cleaned.is_empty() {
                continue;
            }

            let parts: Vec<&str> = cleaned
                .split("---")
                .map(|part| part.trim())
                .filter(|part| !part.is_empty())
                .collect();

            if parts.len() != 2 {
                continue; // Skip malformed test cases
            }

            // Comparison normalises whitespace on both sides — any run of
            // whitespace (spaces, tabs, newlines) collapses to a single
            // space.
            let normalise = |s: &str| s.split_whitespace().collect::<Vec<_>>().join(" ");
            let input = parts[0];

            let expected: Vec<String> = parts[1]
                .lines()
                .map(|s| normalise(&s))
                .filter(|s| !s.is_empty())
                .collect();

            let result = language.segment(&input);

            let actual: Vec<String> = result
                .iter()
                .map(|item| normalise(item))
                .filter(|s| !s.is_empty())
                .collect();

            assert_eq!(actual, expected, "Failed for input: \n{}", input);
        }
    }
}