1#![warn(clippy::all, clippy::pedantic, clippy::nursery)]
2#![doc = include_str!("../README.md")]
3
4use icu_segmenter::{
29 GraphemeClusterSegmenter, GraphemeClusterSegmenterBorrowed, WordSegmenter, WordSegmenterBorrowed,
30 options::WordBreakInvariantOptions,
31};
32use std::{
33 iter::Sum,
34 ops::{Add, AddAssign},
35 sync::OnceLock,
36};
37
38#[derive(Clone, Copy, Debug, Default, Eq, PartialEq)]
40pub struct Counts {
41 pub words: u32,
43 pub characters: u32,
45 pub characters_no_whitespace: u32,
47}
48
49impl Add for Counts {
50 type Output = Self;
51
52 fn add(self, other: Self) -> Self {
53 Self {
54 words: self.words.saturating_add(other.words),
55 characters: self.characters.saturating_add(other.characters),
56 characters_no_whitespace: self
57 .characters_no_whitespace
58 .saturating_add(other.characters_no_whitespace),
59 }
60 }
61}
62
63impl AddAssign for Counts {
64 fn add_assign(&mut self, other: Self) {
65 *self = *self + other;
66 }
67}
68
69impl Sum for Counts {
70 fn sum<I: Iterator<Item = Self>>(iter: I) -> Self {
71 iter.fold(Self::default(), Add::add)
72 }
73}
74
75pub struct Counter {
78 words: WordSegmenterBorrowed<'static>,
79 graphemes: GraphemeClusterSegmenterBorrowed<'static>,
80}
81
82impl Default for Counter {
83 fn default() -> Self {
84 Self::new()
85 }
86}
87
88impl Counter {
89 #[must_use]
90 pub fn new() -> Self {
91 let options = WordBreakInvariantOptions::default();
92
93 #[cfg(feature = "lstm")]
94 let words = WordSegmenter::new_auto(options);
95 #[cfg(all(feature = "dictionary", not(feature = "lstm")))]
96 let words = WordSegmenter::new_dictionary(options);
97 #[cfg(not(any(feature = "dictionary", feature = "lstm")))]
98 let words = WordSegmenter::new_for_non_complex_scripts(options);
99
100 Self {
101 words,
102 graphemes: GraphemeClusterSegmenter::new(),
103 }
104 }
105
106 #[must_use]
108 pub fn count(&self, text: &str) -> Counts {
109 let mut words = 0usize;
110 let mut segments = self.words.segment_str(text);
111
112 while segments.next().is_some() {
114 if segments.is_word_like() {
115 words += 1;
116 }
117 }
118
119 let mut characters = 0usize;
120 let mut characters_no_whitespace = 0usize;
121 let mut boundaries = self.graphemes.segment_str(text);
122 let mut start = boundaries.next();
123
124 for end in boundaries {
125 if let Some(start) = start {
126 characters += 1;
127
128 if text[start..end]
130 .chars()
131 .next()
132 .is_some_and(|first| !first.is_whitespace())
133 {
134 characters_no_whitespace += 1;
135 }
136 }
137
138 start = Some(end);
139 }
140
141 Counts {
142 words: u32::try_from(words).unwrap_or(u32::MAX),
143 characters: u32::try_from(characters).unwrap_or(u32::MAX),
144 characters_no_whitespace: u32::try_from(characters_no_whitespace).unwrap_or(u32::MAX),
145 }
146 }
147}
148
149#[must_use]
151pub fn count(text: &str) -> Counts {
152 static COUNTER: OnceLock<Counter> = OnceLock::new();
153
154 COUNTER.get_or_init(Counter::new).count(text)
155}
156
157#[cfg(test)]
158mod tests {
159 use super::*;
160
161 fn counts(words: u32, characters: u32, characters_no_whitespace: u32) -> Counts {
162 Counts {
163 words,
164 characters,
165 characters_no_whitespace,
166 }
167 }
168
169 #[test]
170 fn empty_and_whitespace() {
171 assert_eq!(count(""), Counts::default());
172 assert_eq!(count(" \t\r\n "), counts(0, 4, 0));
173 }
174
175 #[test]
176 fn english() {
177 assert_eq!(count("Hello, world!"), counts(2, 13, 12));
178 assert_eq!(count("don't 3.14 well-known"), counts(4, 21, 19));
180 }
181
182 #[test]
183 fn other_space_delimited_scripts() {
184 assert_eq!(count("Привет, мир!"), counts(2, 12, 11));
185 assert_eq!(count("γειά σου κόσμε"), counts(3, 14, 12));
186 assert_eq!(count("مرحبا بالعالم"), counts(2, 13, 12));
187 assert_eq!(count("안녕하세요 세계"), counts(2, 8, 7));
188 }
189
190 #[test]
191 fn grapheme_clusters_are_single_characters() {
192 assert_eq!(count("e\u{301}"), counts(1, 1, 1));
194 assert_eq!(count("\u{1F1E9}\u{1F1EA}"), counts(0, 1, 1));
195 assert_eq!(count("\u{1F468}\u{200D}\u{1F469}\u{200D}\u{1F467}"), counts(0, 1, 1));
196 }
197
198 #[test]
199 fn crlf_is_one_whitespace_character() {
200 assert_eq!(count("a\r\nb"), counts(2, 3, 2));
201 }
202
203 #[test]
204 fn counts_add_up() {
205 let total: Counts = ["one two", "three"].into_iter().map(count).sum();
206 assert_eq!(total, counts(3, 12, 11));
207 }
208
209 #[cfg(not(any(feature = "dictionary", feature = "lstm")))]
210 #[test]
211 fn cjk_without_dictionary_is_one_word_per_run() {
212 let result = count("今日は晴れ");
214 assert_eq!(result.characters, 5);
215 assert_eq!(result.words, 1);
216 }
217
218 #[cfg(any(feature = "dictionary", feature = "lstm"))]
219 #[test]
220 fn mixed_scripts_and_cjk_punctuation() {
221 let result = count("\u{300C}Hello\u{3001}\u{4E16}\u{754C}\u{3002}\u{300D}");
223
224 assert_eq!(result.characters, 11);
225 assert!(result.words >= 2 && result.words <= 3, "{result:?}");
226 }
227
228 #[cfg(any(feature = "dictionary", feature = "lstm"))]
229 #[test]
230 fn japanese_and_chinese_are_segmented_into_words() {
231 let japanese = count(
233 "\u{4ECA}\u{65E5}\u{306F}\u{3068}\u{3066}\u{3082}\u{3044}\u{3044}\u{5929}\u{6C17}\u{3067}\u{3059}\u{306D}",
234 );
235 assert!(
236 japanese.words > 1 && japanese.words < japanese.characters,
237 "{japanese:?}"
238 );
239
240 let chinese = count(
241 "\u{4ECA}\u{5929}\u{5929}\u{6C14}\u{5F88}\u{597D}\u{FF0C}\u{6211}\u{4EEC}\u{53BB}\u{516C}\u{56ED}\u{6563}\\
242 \
243 u{6B65}",
244 );
245 assert!(chinese.words > 1 && chinese.words < chinese.characters, "{chinese:?}");
246 }
247
248 #[cfg(any(feature = "dictionary", feature = "lstm"))]
249 #[test]
250 fn thai_is_segmented_into_words() {
251 let thai =
253 count("\u{0E2A}\u{0E27}\u{0E31}\u{0E2A}\u{0E14}\u{0E35}\u{0E0A}\u{0E32}\u{0E27}\u{0E42}\u{0E25}\u{0E01}");
254 assert!(thai.words >= 2, "{thai:?}");
255 }
256}