readsight 1.0.0

Multilingual readability library — 86 languages, 17 formulas, TeX-based syllable counting via the Frank M. Liang algorithm.
Documentation 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121

//! Unit tests for the text splitter (ported from the PHP suite).

use readsight::language::Language;
use readsight::text::TextSplitter;

fn en_language() -> Language {
    Language::from_json(
        r#"{
        "code": "en-us",
        "name": "English (US)",
        "nativeName": "English (US)",
        "script": "Latin",
        "hyphenMins": {"left": 2, "right": 2},
        "letterPattern": "[A-Za-z]",
        "wordSplitPattern": "[^A-Za-z'-]+",
        "sentenceBoundaryPattern": "[.!?]+"
    }"#,
    )
    .unwrap()
}

fn splitter() -> TextSplitter {
    TextSplitter::new(&en_language()).unwrap()
}

#[test]
fn split_words_simple() {
    assert_eq!(
        splitter().split_words("The quick brown fox"),
        vec!["The", "quick", "brown", "fox"]
    );
}

#[test]
fn split_words_with_punctuation() {
    assert_eq!(
        splitter().split_words("Hello, world! How are you?"),
        vec!["Hello", "world", "How", "are", "you"]
    );
}

#[test]
fn split_words_empty() {
    assert_eq!(splitter().split_words(""), Vec::<String>::new());
}

#[test]
fn count_words_and_empty() {
    assert_eq!(splitter().count_words("The quick brown fox"), 4);
    assert_eq!(splitter().count_words(""), 0);
}

#[test]
fn count_letters_cases() {
    assert_eq!(splitter().count_letters("The quick brown"), 13);
    assert_eq!(splitter().count_letters(""), 0);
    assert_eq!(splitter().count_letters("Hello 123 world!"), 10);
    assert_eq!(splitter().count_letters("Hello @world #2024!"), 10);
}

#[test]
fn count_sentences_cases() {
    assert_eq!(splitter().count_sentences("One. Two! Three?"), 3);
    assert_eq!(splitter().count_sentences(""), 0);
    assert_eq!(
        splitter().count_sentences("Hello world without punctuation"),
        1
    );
}

#[test]
fn count_long_words() {
    assert_eq!(
        splitter().count_long_words("short but longer words here", 5),
        1
    );
}

#[test]
fn words_with_hyphens_and_apostrophes() {
    let s = splitter();
    let hy = s.split_words("self-contained well-known re-evaluate");
    assert_eq!(hy.len(), 3);
    assert_eq!(hy[0], "self-contained");
    assert_eq!(
        s.split_words("don't can't it's"),
        vec!["don't", "can't", "it's"]
    );
}

#[test]
fn numbers_split_words() {
    let words = splitter().split_words("Hello 123 world 456test");
    assert!(words.contains(&"Hello".to_string()));
    assert!(words.contains(&"world".to_string()));
    assert!(words.contains(&"test".to_string()));
}

#[test]
fn whitespace_variants_collapse() {
    assert_eq!(
        splitter().split_words("Hello    world"),
        vec!["Hello", "world"]
    );
    assert_eq!(
        splitter().split_words("Hello\tworld\nagain"),
        vec!["Hello", "world", "again"]
    );
}

#[test]
fn only_punctuation_no_words() {
    assert_eq!(splitter().split_words("!!! ??? ..."), Vec::<String>::new());
}

#[test]
fn split_sentences_trims() {
    let parts = splitter().split_sentences("One. Two! Three?");
    assert_eq!(parts, vec!["One", "Two", "Three"]);
    assert_eq!(splitter().split_sentences(""), Vec::<String>::new());
}