Skip to main content

text_stats/
lib.rs

1#![warn(clippy::all, clippy::pedantic, clippy::nursery)]
2#![doc = include_str!("../README.md")]
3
4//! Word and character counts for text in any language.
5//!
6//! Counting is done the way translation tools do it, but with real Unicode segmentation instead of
7//! splitting on spaces, so it also works for languages that do not use spaces:
8//!
9//! - **words** are the *word-like* segments of ICU4X's word segmenter: letters, numbers and CJK
10//!   ideographs/kana, but not whitespace, punctuation or emoji. `don't` and `3.14` are one word,
11//!   `well-known` is two. With the `dictionary` feature (on by default) Chinese, Japanese, Thai, Lao, Khmer
12//!   and Burmese are split into real words; without it a run of such text is not split and counts as one word.
13//! - **characters** are extended grapheme clusters, i.e. what a reader sees as one character (`e` plus a
14//!   combining accent, or a family emoji, is one), not bytes or `char`s.
15//! - **characters without whitespace** are the same clusters, minus the ones that start with whitespace.
16//!
17//! No language needs to be given: the segmenter works on mixed-script text directly.
18//!
19//! ```
20//! use text_stats::count;
21//!
22//! let counts = count("Don't panic, it's 3.14 o'clock!");
23//! assert_eq!(counts.words, 5);
24//! assert_eq!(counts.characters, 31);
25//! assert_eq!(counts.characters_no_whitespace, 27);
26//! ```
27
28use icu_segmenter::{
29    GraphemeClusterSegmenter, GraphemeClusterSegmenterBorrowed, WordSegmenter, WordSegmenterBorrowed,
30    options::WordBreakInvariantOptions,
31};
32use std::{
33    iter::Sum,
34    ops::{Add, AddAssign},
35    sync::OnceLock,
36};
37
38/// Word and character counts of a text. Counts of several texts can be added.
39#[derive(Clone, Copy, Debug, Default, Eq, PartialEq)]
40pub struct Counts {
41    /// Word-like segments: letters, numbers, CJK ideographs and kana. Punctuation and whitespace are not words.
42    pub words: u32,
43    /// Extended grapheme clusters, whitespace included.
44    pub characters: u32,
45    /// Extended grapheme clusters that do not start with whitespace.
46    pub characters_no_whitespace: u32,
47}
48
49impl Add for Counts {
50    type Output = Self;
51
52    fn add(self, other: Self) -> Self {
53        Self {
54            words: self.words.saturating_add(other.words),
55            characters: self.characters.saturating_add(other.characters),
56            characters_no_whitespace: self
57                .characters_no_whitespace
58                .saturating_add(other.characters_no_whitespace),
59        }
60    }
61}
62
63impl AddAssign for Counts {
64    fn add_assign(&mut self, other: Self) {
65        *self = *self + other;
66    }
67}
68
69impl Sum for Counts {
70    fn sum<I: Iterator<Item = Self>>(iter: I) -> Self {
71        iter.fold(Self::default(), Add::add)
72    }
73}
74
75/// Counts words and characters. Building a `Counter` loads the segmentation data, so build one and reuse it;
76/// [`count`] does that for you with a shared instance. A `Counter` can be used from many threads at once.
77pub struct Counter {
78    words: WordSegmenterBorrowed<'static>,
79    graphemes: GraphemeClusterSegmenterBorrowed<'static>,
80}
81
82impl Default for Counter {
83    fn default() -> Self {
84        Self::new()
85    }
86}
87
88impl Counter {
89    #[must_use]
90    pub fn new() -> Self {
91        let options = WordBreakInvariantOptions::default();
92
93        #[cfg(feature = "lstm")]
94        let words = WordSegmenter::new_auto(options);
95        #[cfg(all(feature = "dictionary", not(feature = "lstm")))]
96        let words = WordSegmenter::new_dictionary(options);
97        #[cfg(not(any(feature = "dictionary", feature = "lstm")))]
98        let words = WordSegmenter::new_for_non_complex_scripts(options);
99
100        Self {
101            words,
102            graphemes: GraphemeClusterSegmenter::new(),
103        }
104    }
105
106    /// Counts the words and characters of `text`. Line breaks (`\n`, `\r\n`) count as whitespace characters.
107    #[must_use]
108    pub fn count(&self, text: &str) -> Counts {
109        let mut words = 0usize;
110        let mut segments = self.words.segment_str(text);
111
112        // The segment preceding each boundary is the one that is classified.
113        while segments.next().is_some() {
114            if segments.is_word_like() {
115                words += 1;
116            }
117        }
118
119        let mut characters = 0usize;
120        let mut characters_no_whitespace = 0usize;
121        let mut boundaries = self.graphemes.segment_str(text);
122        let mut start = boundaries.next();
123
124        for end in boundaries {
125            if let Some(start) = start {
126                characters += 1;
127
128                // `start` is a boundary from the segmenter, so it is on a `char` boundary and the slice is not empty.
129                if text[start..end]
130                    .chars()
131                    .next()
132                    .is_some_and(|first| !first.is_whitespace())
133                {
134                    characters_no_whitespace += 1;
135                }
136            }
137
138            start = Some(end);
139        }
140
141        Counts {
142            words: u32::try_from(words).unwrap_or(u32::MAX),
143            characters: u32::try_from(characters).unwrap_or(u32::MAX),
144            characters_no_whitespace: u32::try_from(characters_no_whitespace).unwrap_or(u32::MAX),
145        }
146    }
147}
148
149/// Counts the words and characters of `text` with a shared [`Counter`].
150#[must_use]
151pub fn count(text: &str) -> Counts {
152    static COUNTER: OnceLock<Counter> = OnceLock::new();
153
154    COUNTER.get_or_init(Counter::new).count(text)
155}
156
157#[cfg(test)]
158mod tests {
159    use super::*;
160
161    fn counts(words: u32, characters: u32, characters_no_whitespace: u32) -> Counts {
162        Counts {
163            words,
164            characters,
165            characters_no_whitespace,
166        }
167    }
168
169    #[test]
170    fn empty_and_whitespace() {
171        assert_eq!(count(""), Counts::default());
172        assert_eq!(count(" \t\r\n "), counts(0, 4, 0));
173    }
174
175    #[test]
176    fn english() {
177        assert_eq!(count("Hello, world!"), counts(2, 13, 12));
178        // Apostrophes and decimal points stay inside a word, hyphens split.
179        assert_eq!(count("don't 3.14 well-known"), counts(4, 21, 19));
180    }
181
182    #[test]
183    fn other_space_delimited_scripts() {
184        assert_eq!(count("Привет, мир!"), counts(2, 12, 11));
185        assert_eq!(count("γειά σου κόσμε"), counts(3, 14, 12));
186        assert_eq!(count("مرحبا بالعالم"), counts(2, 13, 12));
187        assert_eq!(count("안녕하세요 세계"), counts(2, 8, 7));
188    }
189
190    #[test]
191    fn grapheme_clusters_are_single_characters() {
192        // `e` + combining acute, a flag (two regional indicators) and a ZWJ family emoji.
193        assert_eq!(count("e\u{301}"), counts(1, 1, 1));
194        assert_eq!(count("\u{1F1E9}\u{1F1EA}"), counts(0, 1, 1));
195        assert_eq!(count("\u{1F468}\u{200D}\u{1F469}\u{200D}\u{1F467}"), counts(0, 1, 1));
196    }
197
198    #[test]
199    fn crlf_is_one_whitespace_character() {
200        assert_eq!(count("a\r\nb"), counts(2, 3, 2));
201    }
202
203    #[test]
204    fn counts_add_up() {
205        let total: Counts = ["one two", "three"].into_iter().map(count).sum();
206        assert_eq!(total, counts(3, 12, 11));
207    }
208
209    #[cfg(not(any(feature = "dictionary", feature = "lstm")))]
210    #[test]
211    fn cjk_without_dictionary_is_one_word_per_run() {
212        // Documented limitation: no segmentation model is available, so the whole run is one word.
213        let result = count("今日は晴れ");
214        assert_eq!(result.characters, 5);
215        assert_eq!(result.words, 1);
216    }
217
218    #[cfg(any(feature = "dictionary", feature = "lstm"))]
219    #[test]
220    fn mixed_scripts_and_cjk_punctuation() {
221        // Punctuation of any script is never a word.
222        let result = count("\u{300C}Hello\u{3001}\u{4E16}\u{754C}\u{3002}\u{300D}");
223
224        assert_eq!(result.characters, 11);
225        assert!(result.words >= 2 && result.words <= 3, "{result:?}");
226    }
227
228    #[cfg(any(feature = "dictionary", feature = "lstm"))]
229    #[test]
230    fn japanese_and_chinese_are_segmented_into_words() {
231        // "It is very nice weather today" - more than one word, fewer than the number of characters.
232        let japanese = count(
233            "\u{4ECA}\u{65E5}\u{306F}\u{3068}\u{3066}\u{3082}\u{3044}\u{3044}\u{5929}\u{6C17}\u{3067}\u{3059}\u{306D}",
234        );
235        assert!(
236            japanese.words > 1 && japanese.words < japanese.characters,
237            "{japanese:?}"
238        );
239
240        let chinese = count(
241            "\u{4ECA}\u{5929}\u{5929}\u{6C14}\u{5F88}\u{597D}\u{FF0C}\u{6211}\u{4EEC}\u{53BB}\u{516C}\u{56ED}\u{6563}\\
242             \
243             u{6B65}",
244        );
245        assert!(chinese.words > 1 && chinese.words < chinese.characters, "{chinese:?}");
246    }
247
248    #[cfg(any(feature = "dictionary", feature = "lstm"))]
249    #[test]
250    fn thai_is_segmented_into_words() {
251        // "Hello, world" in Thai, written without spaces.
252        let thai =
253            count("\u{0E2A}\u{0E27}\u{0E31}\u{0E2A}\u{0E14}\u{0E35}\u{0E0A}\u{0E32}\u{0E27}\u{0E42}\u{0E25}\u{0E01}");
254        assert!(thai.words >= 2, "{thai:?}");
255    }
256}