sentencex 0.1.31

Sentence segmentation library with wide language support optimized for speed and utility.
Documentation 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70

use rustc_hash::FxHashSet;
use std::sync::LazyLock;

use crate::constants::ROMAN_NUMERALS;

use super::Language;
use super::language::continues_after_boundary;
use super::parse_lowercase_word_list;

#[derive(Debug, Clone)]
pub struct Slovak {}

static SLOVAK_ABBREVIATIONS: LazyLock<FxHashSet<String>> =
    LazyLock::new(|| parse_lowercase_word_list([include_str!("./abbrev/sk.txt")]));

static SLOVAK_ALL_ABBREVIATIONS: LazyLock<FxHashSet<String>> = LazyLock::new(|| {
    let mut abbreviations = SLOVAK_ABBREVIATIONS.clone();
    abbreviations.extend(ROMAN_NUMERALS.iter().map(|&s| s.to_string()));
    abbreviations.extend(ROMAN_NUMERALS.iter().map(|&s| s.to_uppercase()));
    abbreviations
});

const MONTHS: [&str; 24] = [
    "Január",
    "Február",
    "Marec",
    "Apríl",
    "Máj",
    "Jún",
    "Júl",
    "August",
    "September",
    "Október",
    "November",
    "December",
    "Januára",
    "Februára",
    "Marca",
    "Apríla",
    "Mája",
    "Júna",
    "Júla",
    "Augusta",
    "Septembra",
    "Októbra",
    "Novembra",
    "Decembra",
];

impl Language for Slovak {
    fn get_abbreviations(&self) -> &FxHashSet<String> {
        &SLOVAK_ALL_ABBREVIATIONS
    }

    fn continue_in_next_word(&self, text_after_boundary: &str) -> bool {
        continues_after_boundary(text_after_boundary, &MONTHS)
    }
}

#[cfg(test)]
mod tests {
    use crate::languages::tests::run_language_tests;

    use super::*;

    #[test]
    fn test_segment() {
        run_language_tests(Slovak {}, "tests/sk.txt");
    }
}