acorn/analyzer/readability/
mod.rs

1//! # Readability utilities
2//!
3//! Analyze readabilty of prose using modern readability metrics.
4use crate::prelude::HashMap;
5use crate::util::constants::app::{
6    MAX_ALLOWED_ARI, MAX_ALLOWED_CLI, MAX_ALLOWED_FKGL, MAX_ALLOWED_FRES, MAX_ALLOWED_GFI, MAX_ALLOWED_LIX, MAX_ALLOWED_SMOG,
7};
8use crate::util::find_first;
9use crate::util::Label;
10use aho_corasick::{AhoCorasickBuilder, MatchKind};
11use derive_more::Display;
12use dotenvy::dotenv;
13use fancy_regex::Regex;
14use itertools::Itertools;
15use tracing::warn;
16use tracing::{debug, trace};
17
18pub mod constants;
19use constants::{
20    ACRONYM_TOKEN, DOUBLE, DOUBLE_SYLLABIC_FOUR, DOUBLE_SYLLABIC_ONE, DOUBLE_SYLLABIC_THREE, DOUBLE_SYLLABIC_TWO, IRREGULAR_NOUNS,
21    IRREGULAR_NOUNS_INVERTED, NEED_TO_BE_FIXED, NON_ALPHABETIC, PLURAL_TO_SINGULAR, PROBLEMATIC_WORDS, SAME_SINGULAR_PLURAL, SINGLE,
22    SINGLE_SYLLABIC_ONE, SINGLE_SYLLABIC_TWO, TRIPLE, VOWEL, WORD_SYLLABLES,
23};
24
25/// Readability Type
26#[derive(Clone, Copy, Debug, Default, Display, PartialEq)]
27pub enum ReadabilityType {
28    /// Automated Readability Index (ARI)
29    ///
30    /// See [`automated_readability_index`]
31    #[display("ari")]
32    ARI,
33    /// Coleman-Liau Index (CLI)
34    ///
35    /// See [`coleman_liau_index`]
36    #[display("cli")]
37    CLI,
38    /// Flesch-Kincaid Grade Level (FKGL)
39    ///
40    /// See [`flesch_kincaid_grade_level`]
41    #[default]
42    #[display("fkgl")]
43    FKGL,
44    /// Flesch Reading Ease (FRES)
45    ///
46    /// See [`flesch_reading_ease_score`]
47    #[display("fres")]
48    FRES,
49    /// Gunning Fog Index (GFI)
50    ///
51    /// See [`gunning_fog_index`]
52    #[display("gfi")]
53    GFI,
54    /// Lix (abbreviation of Swedish läsbarhetsindex)
55    ///
56    /// See [`lix`]
57    #[display("lix")]
58    Lix,
59    /// SMOG Index (SMOG)
60    ///
61    /// See [`smog`]
62    #[display("smog")]
63    SMOG,
64}
65impl From<ReadabilityType> for String {
66    fn from(value: ReadabilityType) -> Self {
67        value.to_string()
68    }
69}
70impl From<String> for ReadabilityType {
71    fn from(value: String) -> Self {
72        ReadabilityType::from_string(&value)
73    }
74}
75impl From<&str> for ReadabilityType {
76    fn from(value: &str) -> Self {
77        ReadabilityType::from_string(value)
78    }
79}
80impl ReadabilityType {
81    /// Calculate Readability for a given text and readability type
82    pub fn calculate(self, text: &str) -> f64 {
83        match self {
84            | ReadabilityType::ARI => automated_readability_index(text),
85            | ReadabilityType::CLI => coleman_liau_index(text),
86            | ReadabilityType::FKGL => flesch_kincaid_grade_level(text),
87            | ReadabilityType::FRES => flesch_reading_ease_score(text),
88            | ReadabilityType::GFI => gunning_fog_index(text),
89            | ReadabilityType::Lix => lix(text),
90            | ReadabilityType::SMOG => smog(text),
91        }
92    }
93    /// Get Readability Type from string
94    pub fn from_string(value: &str) -> ReadabilityType {
95        match value.to_lowercase().replace("-", " ").as_str() {
96            | "ari" | "automated readability index" => ReadabilityType::ARI,
97            | "cli" | "coleman liau index" => ReadabilityType::CLI,
98            | "fkgl" | "flesch kincaid grade level" => ReadabilityType::FKGL,
99            | "fres" | "flesch reading ease score" => ReadabilityType::FRES,
100            | "gfi" | "gunning fog index" => ReadabilityType::GFI,
101            | "lix" => ReadabilityType::Lix,
102            | "smog" | "simple measure of gobbledygook" => ReadabilityType::SMOG,
103            | _ => {
104                warn!(value, "=> {} Unknown Readability Type", Label::using());
105                ReadabilityType::default()
106            }
107        }
108    }
109    /// Get maximum allowed value for a given readability type
110    pub fn maximum_allowed(self) -> f64 {
111        match self {
112            | ReadabilityType::ARI => MAX_ALLOWED_ARI,
113            | ReadabilityType::CLI => MAX_ALLOWED_CLI,
114            | ReadabilityType::FKGL => MAX_ALLOWED_FKGL,
115            | ReadabilityType::FRES => MAX_ALLOWED_FRES,
116            | ReadabilityType::GFI => MAX_ALLOWED_GFI,
117            | ReadabilityType::Lix => MAX_ALLOWED_LIX,
118            | ReadabilityType::SMOG => MAX_ALLOWED_SMOG,
119        }
120    }
121    /// Get maximum allowed value for a given readability type, from environment file
122    pub fn maximum_allowed_from_env(self) -> Option<f64> {
123        match dotenv() {
124            | Ok(_) => {
125                let variables = dotenvy::vars().collect::<Vec<(String, String)>>();
126                let pair = match self {
127                    | ReadabilityType::ARI => find_first(variables, "MAX_ALLOWED_ARI"),
128                    | ReadabilityType::CLI => find_first(variables, "MAX_ALLOWED_CLI"),
129                    | ReadabilityType::FKGL => find_first(variables, "MAX_ALLOWED_FKGL"),
130                    | ReadabilityType::FRES => find_first(variables, "MAX_ALLOWED_FRES"),
131                    | ReadabilityType::GFI => find_first(variables, "MAX_ALLOWED_GFI"),
132                    | ReadabilityType::Lix => find_first(variables, "MAX_ALLOWED_LIX"),
133                    | ReadabilityType::SMOG => find_first(variables, "MAX_ALLOWED_SMOG"),
134                };
135                match pair {
136                    | Some((_, value)) => value.parse::<f64>().ok(),
137                    | None => None,
138                }
139            }
140            | Err(_) => None,
141        }
142    }
143}
144/// Expand acronyms in a given text using the provided acronym map
145/// ### Example
146/// ```rust
147/// use acorn::analyzer::readability::expand_acronyms;
148/// use acorn::prelude::HashMap;
149///
150/// let acronyms = HashMap::from([("CLI".to_string(), "Command Line Interface".to_string())]);
151/// let text = "Use the CLI for automation.";
152/// let expanded = expand_acronyms(text, &acronyms);
153/// assert_eq!(expanded, "Use the Command Line Interface for automation.");
154/// ```
155pub fn expand_acronyms(text: &str, acronyms: &HashMap<String, String>) -> String {
156    const AHO_MAP_THRESHOLD: usize = 32;
157    const AHO_TEXT_THRESHOLD: usize = 256;
158    fn expand_with_regex(text: &str, normalized: &HashMap<String, String>) -> String {
159        ACRONYM_TOKEN
160            .replace_all(text, |captures: &fancy_regex::Captures<'_>| {
161                captures.get(0).map_or(String::new(), |value| {
162                    let token = value.as_str();
163                    normalized.get(&token.to_lowercase()).cloned().unwrap_or_else(|| token.to_string())
164                })
165            })
166            .to_string()
167    }
168    fn is_word_character(character: char) -> bool {
169        character == '_' || character.is_alphanumeric()
170    }
171    fn is_token_boundary(text: &str, start: usize, end: usize) -> bool {
172        let previous = text.get(..start).and_then(|segment| segment.chars().next_back());
173        let next = text.get(end..).and_then(|segment| segment.chars().next());
174        !matches!(previous, Some(value) if is_word_character(value)) && !matches!(next, Some(value) if is_word_character(value))
175    }
176    let normalized = acronyms
177        .iter()
178        .map(|(key, value)| (key.to_lowercase(), value.clone()))
179        .collect::<HashMap<String, String>>();
180    match normalized.is_empty() {
181        | true => text.to_string(),
182        | false if normalized.len() <= AHO_MAP_THRESHOLD || text.len() < AHO_TEXT_THRESHOLD => expand_with_regex(text, &normalized),
183        | false => {
184            let patterns = normalized
185                .keys()
186                .map(String::as_str)
187                .sorted_by(|left, right| right.len().cmp(&left.len()).then_with(|| left.cmp(right)))
188                .collect::<Vec<_>>();
189            let matcher = AhoCorasickBuilder::new()
190                .ascii_case_insensitive(true)
191                .match_kind(MatchKind::LeftmostLongest)
192                .build(patterns);
193            match matcher {
194                | Ok(value) => {
195                    let (result, last) = value
196                        .find_iter(text)
197                        .fold((String::with_capacity(text.len()), 0usize), |(result, last), found| {
198                            let start = found.start();
199                            let end = found.end();
200                            let replacement = is_token_boundary(text, start, end)
201                                .then(|| normalized.get(&text[start..end].to_lowercase()).cloned())
202                                .flatten();
203                            match replacement {
204                                | Some(value) => {
205                                    let chunk = [result, text[last..start].to_string(), value];
206                                    (chunk.concat(), end)
207                                }
208                                | None => (result, last),
209                            }
210                        });
211                    [result, text[last..].to_string()].concat()
212                }
213                | Err(_) => expand_with_regex(text, &normalized),
214            }
215        }
216    }
217}
218/// Count the number of "complex words"[^complex] in a given text
219///
220/// [^complex]: Words with 3 or more syllables
221pub fn complex_word_count(text: &str) -> u32 {
222    words(text).iter().filter(|word| syllable_count(word) > 2).count() as u32
223}
224/// Count the number of letters in a given text
225///
226/// Does NOT count white space or punctuation
227pub fn letter_count(text: &str) -> u32 {
228    text.chars()
229        .filter(|c| !(c.is_whitespace() || NON_ALPHABETIC.is_match(&c.to_string()).unwrap_or_default()))
230        .count() as u32
231}
232/// Count the number of "long words"[^long] in a given text
233///
234/// [^long]: Words with more than 6 letters
235pub fn long_word_count(text: &str) -> u32 {
236    words(text).iter().filter(|word| word.len() > 6).count() as u32
237}
238/// Count the number of sentences in a given text
239pub fn sentence_count(text: &str) -> u32 {
240    fn is_wrapper(character: char) -> bool {
241        matches!(character, ')' | ']' | '}' | '"' | '\'')
242    }
243    fn next_meaningful(chars: &[char], index: usize) -> Option<char> {
244        chars
245            .iter()
246            .skip(index.saturating_add(1))
247            .copied()
248            .find(|character| !(character.is_whitespace() || is_wrapper(*character)))
249    }
250    fn previous_meaningful(chars: &[char], index: usize) -> Option<char> {
251        chars
252            .get(..index)
253            .into_iter()
254            .flatten()
255            .rev()
256            .copied()
257            .find(|character| !(character.is_whitespace() || is_wrapper(*character)))
258    }
259    fn token_before(chars: &[char], index: usize) -> String {
260        chars.get(..index).map_or(String::new(), |slice| {
261            slice
262                .iter()
263                .rev()
264                .take_while(|character| !character.is_whitespace())
265                .copied()
266                .collect::<Vec<_>>()
267                .into_iter()
268                .rev()
269                .collect::<String>()
270                .trim_matches(|character: char| !character.is_alphanumeric() && character != '.')
271                .to_lowercase()
272        })
273    }
274    fn suppresses_boundary(token: &str, next: Option<char>) -> bool {
275        matches!(token, "e.g" | "i.e")
276            || matches!(token, "mr" | "mrs" | "ms" | "dr" | "prof" | "sr" | "jr" | "vs" | "st")
277                && matches!(next, Some(value) if value.is_alphabetic())
278    }
279    let chars = text.chars().collect::<Vec<_>>();
280    chars
281        .iter()
282        .enumerate()
283        .filter(|(index, character)| match character {
284            | '!' | '?' => previous_meaningful(&chars, *index).is_some(),
285            | '.' => {
286                let previous = previous_meaningful(&chars, *index);
287                let next = next_meaningful(&chars, *index);
288                let token = token_before(&chars, *index);
289                !matches!(chars.get(index.saturating_add(1)), Some('.'))
290                    && !matches!(chars.get(index.wrapping_sub(1)), Some('.'))
291                    && !matches!((previous, next), (Some(left), Some(right)) if left.is_ascii_digit() && right.is_ascii_digit())
292                    && !matches!(next, Some(value) if value.is_ascii_lowercase())
293                    && !suppresses_boundary(&token, next)
294                    && previous.is_some()
295            }
296            | _ => false,
297        })
298        .count() as u32
299}
300/// Get list of words in a given text
301pub fn words(text: &str) -> Vec<String> {
302    text.split_whitespace().map(String::from).collect()
303}
304/// Count the number of words in a given text
305///
306/// See [`words`]
307pub fn word_count(text: &str) -> u32 {
308    words(text).len() as u32
309}
310/// Automated Readability Index (ARI)
311///
312/// The formula was derived from a large dataset of texts used in US schools.
313/// The result is a number that corresponds with a US grade level.
314///
315/// Requires counting letters, words, and sentences
316///
317/// See <https://en.wikipedia.org/wiki/Automated_readability_index> for more information
318pub fn automated_readability_index(text: &str) -> f64 {
319    let letters = letter_count(text);
320    let words = word_count(text);
321    let sentences = sentence_count(text);
322    debug!(letters, words, sentences, "=> {}", Label::using());
323    let score = 4.71 * (letters as f64 / words as f64) + 0.5 * (words as f64 / sentences as f64) - 21.43;
324    (score * 100.0).round() / 100.0
325}
326/// Coleman-Liau Index (CLI)
327///
328/// Requires counting letters, words, and sentences
329pub fn coleman_liau_index(text: &str) -> f64 {
330    let letters = letter_count(text);
331    let words = word_count(text);
332    let sentences = sentence_count(text);
333    debug!(letters, words, sentences, "=> {}", Label::using());
334    let score = (0.0588 * 100.0 * (letters as f64 / words as f64)) - (0.296 * 100.0 * (sentences as f64 / words as f64)) - 15.8;
335    (score * 100.0).round() / 100.0
336}
337/// Flesch-Kincaid Grade Level (FKGL)[^cite]
338///
339/// Arguably the most popular readability test.
340///
341/// The result is a number that corresponds with a US grade level.
342///
343/// Requires counting words, sentences, and syllables
344///
345/// See <https://en.wikipedia.org/wiki/Flesch%E2%80%93Kincaid_readability_tests> for more information
346///
347/// [^cite]: Flesch, R. (1948). A new readability yardstick. Journal of Applied Psychology, 32(3), 221-233. <https://doi.org/10.1037/h0057532>
348pub fn flesch_kincaid_grade_level(text: &str) -> f64 {
349    let words = word_count(text);
350    let sentences = sentence_count(text);
351    let syllables = syllable_count(text);
352    debug!(words, sentences, syllables, "=> {}", Label::using());
353    let score = 0.39 * (words as f64 / sentences as f64) + 11.8 * (syllables as f64 / words as f64) - 15.59;
354    (score * 100.0).round() / 100.0
355}
356/// Flesch Reading Ease Score (FRES)
357///
358/// FRES range is 100 (very easy) - 0 (extremely difficult)
359///
360/// Requires counting words, sentences, and syllables
361///
362/// See <https://en.wikipedia.org/wiki/Flesch%E2%80%93Kincaid_readability_tests> for more information
363pub fn flesch_reading_ease_score(text: &str) -> f64 {
364    let words = word_count(text);
365    let sentences = sentence_count(text);
366    let syllables = syllable_count(text);
367    debug!(words, sentences, syllables, "=> {}", Label::using());
368    let score = 206.835 - (1.015 * words as f64 / sentences as f64) - (84.6 * syllables as f64 / words as f64);
369    (score * 100.0).round() / 100.0
370}
371/// Gunning Fog Index (GFI)
372///
373/// Estimates the years of formal education a person needs to understand the text on the first reading
374///
375/// Requires counting words, sentences, and "complex words" (see [complex_word_count])
376///
377/// See <https://en.wikipedia.org/wiki/Gunning_fog_index> for more information
378pub fn gunning_fog_index(text: &str) -> f64 {
379    let words = word_count(text);
380    let complex_words = complex_word_count(text);
381    let sentences = sentence_count(text);
382    let score = 0.4 * ((words as f64 / sentences as f64) + (100.0 * (complex_words as f64 / words as f64)));
383    (score * 100.0).round() / 100.0
384}
385/// Lix (abbreviation of Swedish läsbarhetsindex)
386///
387/// Indicates the difficulty of reading a text
388///
389/// Requires counting words, sentences, and long words (see [long_word_count])
390///
391/// "Lix" is an abbreviation of *läsbarhetsindex*, which means "readability index" in Swedish
392///
393/// See <https://en.wikipedia.org/wiki/Lix_(readability_test)> for more information
394pub fn lix(text: &str) -> f64 {
395    let words = word_count(text);
396    let sentences = sentence_count(text);
397    let long_words = long_word_count(text);
398    let score = (words as f64 / sentences as f64) + 100.0 * (long_words as f64 / words as f64);
399    (score * 100.0).round() / 100.0
400}
401/// Simple Measure of Gobbledygook (SMOG)
402///
403/// Estimates the years of education needed to understand a piece of writing
404///
405/// **Caution**: SMOG formula was normalized on 30-sentence samples
406///
407/// Requires counting sentences, and "complex words" (see [complex_word_count])
408///
409/// See <https://en.wikipedia.org/wiki/SMOG> for more information
410pub fn smog(text: &str) -> f64 {
411    let sentences = sentence_count(text);
412    let complex_words = complex_word_count(text);
413    let score = 1.0430 * (30.0 * (complex_words as f64 / sentences as f64)).sqrt() + 3.1291;
414    (score * 100.0).round() / 100.0
415}
416/// Get the singular form of a word (e.g. "people" -> "person")
417///
418/// Adapted from the PHP library, [Text-Statistics](https://github.com/DaveChild/Text-Statistics)
419pub fn singular_form(word: &str) -> String {
420    match word.to_lowercase().as_str() {
421        | value if SAME_SINGULAR_PLURAL.contains(&value) => value.to_string(),
422        | value if IRREGULAR_NOUNS.contains_key(&value) => value.to_string(),
423        | value if IRREGULAR_NOUNS_INVERTED.contains_key(&value) => match IRREGULAR_NOUNS_INVERTED.get(value) {
424            | Some(value) => value.to_string(),
425            | None => value.to_string(),
426        },
427        | value => {
428            let pair = PLURAL_TO_SINGULAR.iter().find(|(pattern, _)| {
429                #[allow(clippy::unwrap_used)]
430                Regex::new(pattern).unwrap().is_match(value).unwrap_or(false)
431            });
432            match pair {
433                | Some((pattern, replacement)) => {
434                    trace!(pattern, replacement, value, "=> {} Singular form conversion", Label::using());
435                    #[allow(clippy::unwrap_used)]
436                    let re = Regex::new(pattern).unwrap();
437                    re.replace_all(value, *replacement).to_string()
438                }
439                | None => value.to_string(),
440            }
441        }
442    }
443}
444/// Count the number of syllables in a given text
445/// ### Example
446/// ```rust
447/// use acorn::analyzer::readability::syllable_count;
448///
449/// let sentence = "The quick brown fox jumps over the lazy dog.";
450/// assert_eq!(syllable_count(sentence), 11);
451/// ```
452pub fn syllable_count(text: &str) -> usize {
453    fn syllables(word: String) -> usize {
454        let singular = singular_form(&word);
455        let normalized = word.to_lowercase();
456        if let Some(value) = WORD_SYLLABLES.get(&normalized).or_else(|| WORD_SYLLABLES.get(&singular)) {
457            return *value;
458        }
459        match word.as_str() {
460            | "" => 0,
461            | value if value.len() < 3 => 1,
462            | value if PROBLEMATIC_WORDS.contains_key(value) => match PROBLEMATIC_WORDS.get(value) {
463                | Some(x) => *x,
464                | None => 0,
465            },
466            | _ if PROBLEMATIC_WORDS.contains_key(&singular.as_str()) => match PROBLEMATIC_WORDS.get(singular.as_str()) {
467                | Some(x) => *x,
468                | None => 0,
469            },
470            | value if NEED_TO_BE_FIXED.contains_key(value) => match NEED_TO_BE_FIXED.get(value) {
471                | Some(x) => *x,
472                | None => 0,
473            },
474            | _ if NEED_TO_BE_FIXED.contains_key(&singular.as_str()) => match NEED_TO_BE_FIXED.get(singular.as_str()) {
475                | Some(x) => *x,
476                | None => 0,
477            },
478            | _ => {
479                #[allow(clippy::arithmetic_side_effects)]
480                {
481                    let mut count: isize = 0;
482                    let mut input = word.to_lowercase();
483                    count += 3 * TRIPLE.find_iter(&input).count() as isize;
484                    input = TRIPLE.replace_all(&input, "").to_string();
485                    count += 2 * DOUBLE.find_iter(&input).count() as isize;
486                    input = DOUBLE.replace_all(&input, "").to_string();
487                    count += SINGLE.find_iter(&input).count() as isize;
488                    input = SINGLE.replace_all(&input, "").to_string();
489                    count -= SINGLE_SYLLABIC_ONE.find_iter(&input).count() as isize;
490                    count -= SINGLE_SYLLABIC_TWO.find_iter(&input).count() as isize;
491                    count += DOUBLE_SYLLABIC_ONE.find_iter(&input).count() as isize;
492                    count += DOUBLE_SYLLABIC_TWO.find_iter(&input).count() as isize;
493                    count += DOUBLE_SYLLABIC_THREE.find_iter(&input).count() as isize;
494                    count += DOUBLE_SYLLABIC_FOUR.find_iter(&input).count() as isize;
495                    count += VOWEL.split(&input).filter_map(Result::ok).filter(|x| !x.is_empty()).count() as isize;
496                    count.max(1).cast_unsigned()
497                }
498            }
499        }
500    }
501    let tokens = text.split_whitespace().flat_map(tokenize).collect::<Vec<String>>();
502    tokens.into_iter().map(syllables).sum()
503}
504/// Break text into tokens
505///
506/// Currently replaces `é` and `ë` with `-e`, splits on hyphens, and removes non-alphabetic characters.
507///
508/// This function is a good entry point for adding support for the nuacnces of 'scientific" texts
509pub(crate) fn tokenize(value: &str) -> Vec<String> {
510    value
511        .replace("é", "-e")
512        .replace("ë", "-e")
513        .split('-')
514        .map(|x| NON_ALPHABETIC.replace_all(x, "").to_lowercase())
515        .collect::<Vec<_>>()
516}
517
518#[cfg(test)]
519mod tests;
acorn/analyzer/readability/mod.rs

acorn/analyzer/readability/
mod.rs
