text_statistics/
lib.rs

1//! # text-statistics
2//!
3//! Comprehensive text statistics: word, sentence, and character counts,
4//! average word length, average sentence length, estimated reading time,
5//! and a lexile-like difficulty estimate. Pure Rust, **zero dependencies**,
6//! deterministic.
7//!
8//! These are the same measurements the [BeLikeNative](https://belikenative.com/)
9//! AI writing assistant surfaces while you write — turned into a small,
10//! embeddable library.
11//!
12//! ## Quick example
13//!
14//! ```
15//! use text_statistics::stats;
16//!
17//! let s = stats("The cat sat on the mat. It was a good day for a nap.");
18//! assert_eq!(s.words, 14);
19//! assert_eq!(s.sentences, 2);
20//! assert!(s.avg_word_length > 2.5);
21//! assert!(s.estimated_reading_seconds < 60.0);
22//! assert!(s.difficulty_band() == text_statistics::Difficulty::Easy);
23//! ```
24
25// ============================================================================
26// Tunable constants. Kept as `pub` so callers can calibrate against their own
27// corpora; the defaults match common publishing conventions.
28// ============================================================================
29
30/// Average adult reading speed, in words per minute. 200 wpm is the
31/// widely-cited average for silent reading of English prose.
32pub const WORDS_PER_MINUTE: f64 = 200.0;
33
34/// Lexile-like difficulty reference: characters per word at grade-5 level.
35/// Below this average word length the text is considered "easy".
36pub const EASY_AVG_WORD_LENGTH: f64 = 4.5;
37
38/// Above this average word length the text is considered "difficult"
39/// (roughly grade-11+ vocabulary density).
40pub const HARD_AVG_WORD_LENGTH: f64 = 6.5;
41
42/// Above this average sentence length (in words) the text is considered
43/// "difficult" to follow.
44pub const HARD_AVG_SENTENCE_LENGTH: f64 = 20.0;
45
46/// A coarse difficulty band derived from the lexile-like estimate.
47#[derive(Debug, Clone, Copy, PartialEq, Eq)]
48pub enum Difficulty {
49    /// Short words, short sentences — early-grade reading.
50    Easy,
51    /// Typical adult prose.
52    Medium,
53    /// Long words or long sentences — advanced reading.
54    Hard,
55}
56
57/// The full bundle of computed statistics for a piece of text.
58///
59/// All fields are computed from a single pass over the input. Counts are
60/// `u64` so they compose cleanly with very large documents; averages and
61/// durations are `f64`.
62#[derive(Debug, Clone, PartialEq)]
63pub struct TextStats {
64    /// Number of whitespace-or-punctuation-delimited word tokens.
65    pub words: u64,
66    /// Number of sentence-ending units (`.` `!` `?`), with a trailing
67    /// sentence counted even if the final terminator is omitted.
68    pub sentences: u64,
69    /// Total number of Unicode scalar values (Rust `char`s), including
70    /// whitespace and punctuation.
71    pub characters: u64,
72    /// Characters excluding ASCII whitespace.
73    pub characters_no_spaces: u64,
74    /// Average number of characters per word (letters only, per word).
75    pub avg_word_length: f64,
76    /// Average number of words per sentence.
77    pub avg_sentence_length: f64,
78    /// Estimated reading time in seconds (at [`WORDS_PER_MINUTE`]).
79    pub estimated_reading_seconds: f64,
80    /// Estimated reading time in whole minutes (rounded up, minimum 1 if
81    /// the text is non-empty).
82    pub estimated_reading_minutes: u64,
83    /// A 0–100 lexile-like difficulty estimate; higher means harder.
84    pub difficulty_score: f64,
85}
86
87impl TextStats {
88    /// Map the continuous [`difficulty_score`] to a coarse band.
89    ///
90    /// [`difficulty_score`]: TextStats::difficulty_score
91    #[must_use]
92    pub fn difficulty_band(&self) -> Difficulty {
93        if self.difficulty_score < 33.0 {
94            Difficulty::Easy
95        } else if self.difficulty_score < 66.0 {
96            Difficulty::Medium
97        } else {
98            Difficulty::Hard
99        }
100    }
101}
102
103/// Split prose into word tokens.
104///
105/// A word is a maximal run of ASCII letters or digits or apostrophes. This
106/// keeps contractions (`don't`) as a single token while dropping pure
107/// punctuation/whitespace.
108fn word_tokens(text: &str) -> Vec<String> {
109    text.split(|c: char| !(c.is_alphanumeric() || c == '\''))
110        .filter(|s| !s.is_empty())
111        .map(|s| s.to_ascii_lowercase())
112        .collect()
113}
114
115/// Count sentence-ending units. Abbreviations are not specially handled —
116/// this is a deterministic token count, not a parse.
117fn count_sentences(text: &str) -> u64 {
118    let n = text.chars().filter(|&c| c == '.' || c == '!' || c == '?').count() as u64;
119    // If the text has words but no terminators, treat it as one sentence.
120    if n == 0 && text.chars().any(char::is_alphanumeric) {
121        1
122    } else {
123        n
124    }
125}
126
127/// Compute the full [`TextStats`] bundle for `text`.
128///
129/// Empty input yields all-zero counts (and zero difficulty). The function is
130/// total: it never panics and allocates only for the temporary word list.
131#[must_use]
132pub fn stats(text: &str) -> TextStats {
133    let words = word_tokens(text);
134    let word_count = words.len() as u64;
135    let sentences = count_sentences(text);
136    let characters = text.chars().count() as u64;
137    let characters_no_spaces = text.chars().filter(|c| !c.is_whitespace()).count() as u64;
138
139    // Average word length: count letters (alphabetic) to keep punctuation
140    // out of the figure. Digits are excluded so "12345" does not inflate it.
141    let total_letters: u64 = words.iter().map(|w| w.chars().filter(|c| c.is_alphabetic()).count() as u64).sum();
142    let avg_word_length = if word_count == 0 {
143        0.0
144    } else {
145        total_letters as f64 / word_count as f64
146    };
147
148    let avg_sentence_length = if sentences == 0 {
149        0.0
150    } else {
151        word_count as f64 / sentences as f64
152    };
153
154    // Reading time. At least one second for any non-empty text so a single
155    // word is not reported as "0 seconds".
156    let estimated_reading_seconds = if word_count == 0 {
157        0.0
158    } else {
159        let secs = (word_count as f64 / WORDS_PER_MINUTE) * 60.0;
160        secs.max(1.0)
161    };
162    let estimated_reading_minutes = if word_count == 0 {
163        0
164    } else {
165        ((estimated_reading_seconds / 60.0).ceil() as u64).max(1)
166    };
167
168    // Lexile-like difficulty: blend word length and sentence length signals.
169    // Each signal contributes 0..50; together they span 0..100.
170    let word_signal = clamp01((avg_word_length - EASY_AVG_WORD_LENGTH) / (HARD_AVG_WORD_LENGTH - EASY_AVG_WORD_LENGTH)) * 50.0;
171    let sentence_signal = clamp01(avg_sentence_length / HARD_AVG_SENTENCE_LENGTH) * 50.0;
172    let difficulty_score = word_signal + sentence_signal;
173
174    TextStats {
175        words: word_count,
176        sentences,
177        characters,
178        characters_no_spaces,
179        avg_word_length,
180        avg_sentence_length,
181        estimated_reading_seconds,
182        estimated_reading_minutes,
183        difficulty_score,
184    }
185}
186
187#[inline]
188fn clamp01(x: f64) -> f64 {
189    if x < 0.0 {
190        0.0
191    } else if x > 1.0 {
192        1.0
193    } else {
194        x
195    }
196}
197
198#[cfg(test)]
199mod tests {
200    use super::*;
201
202    #[test]
203    fn empty_input_is_all_zero() {
204        let s = stats("");
205        assert_eq!(s.words, 0);
206        assert_eq!(s.sentences, 0);
207        assert_eq!(s.characters, 0);
208        assert_eq!(s.avg_word_length, 0.0);
209        assert_eq!(s.difficulty_score, 0.0);
210        assert_eq!(s.difficulty_band(), Difficulty::Easy);
211    }
212
213    #[test]
214    fn counts_simple_sentence() {
215        let s = stats("The cat sat on the mat.");
216        assert_eq!(s.words, 6);
217        assert_eq!(s.sentences, 1);
218        // Letters: the(3) cat(3) sat(3) on(2) the(3) mat(3) = 17 / 6 words.
219        assert!((s.avg_word_length - (17.0 / 6.0)).abs() < 1e-9);
220        assert_eq!(s.avg_sentence_length, 6.0);
221    }
222
223    #[test]
224    fn counts_multiple_sentences_with_mixed_terminators() {
225        let s = stats("Hello world! Are you ready? Let's go.");
226        assert_eq!(s.sentences, 3);
227        assert_eq!(s.words, 7); // hello world are you ready let's go
228    }
229
230    #[test]
231    fn contraction_is_one_word() {
232        let s = stats("don't can't won't");
233        assert_eq!(s.words, 3);
234    }
235
236    #[test]
237    fn words_without_terminator_still_count_as_sentence() {
238        let s = stats("just a fragment with no ending");
239        assert_eq!(s.sentences, 1);
240        assert_eq!(s.words, 6);
241    }
242
243    #[test]
244    fn character_counts() {
245        let s = stats("ab c");
246        assert_eq!(s.characters, 4); // a b space c
247        assert_eq!(s.characters_no_spaces, 3);
248    }
249
250    #[test]
251    fn reading_time_at_200_wpm() {
252        // 400 words at 200 wpm = 2 minutes = 120 seconds.
253        let text: String = std::iter::repeat("word ").take(400).collect();
254        let s = stats(&text);
255        assert!((s.estimated_reading_seconds - 120.0).abs() < 1e-6);
256        assert_eq!(s.estimated_reading_minutes, 2);
257    }
258
259    #[test]
260    fn single_word_minimum_one_second() {
261        let s = stats("hi");
262        assert!(s.estimated_reading_seconds >= 1.0);
263        assert_eq!(s.estimated_reading_minutes, 1);
264    }
265
266    #[test]
267    fn easy_prose_is_easy_band() {
268        let s = stats("The cat sat on the mat. The dog ran. It was fun.");
269        assert_eq!(s.difficulty_band(), Difficulty::Easy);
270        assert!(s.difficulty_score < 33.0);
271    }
272
273    #[test]
274    fn long_words_and_sentences_are_hard() {
275        // Long, multi-syllable words and a long sentence.
276        let text = "Notwithstanding the fundamental institutional accommodations, the multidisciplinary organizational representatives nevertheless demonstrated extraordinary quantitative methodological considerations throughout the extensively protracted deliberations.";
277        let s = stats(text);
278        assert_eq!(s.difficulty_band(), Difficulty::Hard);
279        assert!(s.difficulty_score > 66.0);
280    }
281
282    #[test]
283    fn difficulty_score_bounded_0_to_100() {
284        // Pathological inputs should not overflow the band.
285        let very_long: String = std::iter::repeat("internationalization ").take(500).collect();
286        let s = stats(&very_long);
287        assert!(s.difficulty_score <= 100.0);
288        assert!(s.difficulty_score >= 0.0);
289    }
290
291    #[test]
292    fn digits_excluded_from_avg_word_length() {
293        // "12345" has no letters -> 0 letters / 1 word = 0 avg length.
294        let s = stats("12345");
295        assert_eq!(s.words, 1);
296        assert_eq!(s.avg_word_length, 0.0);
297    }
298
299    #[test]
300    fn deterministic_on_repeated_calls() {
301        let t = "The quick brown fox jumps over the lazy dog.";
302        let a = stats(t);
303        let b = stats(t);
304        assert_eq!(a, b);
305    }
306}
text_statistics/lib.rs

text_statistics/
lib.rs
