text-statistics 0.1.0

//! # text-statistics
//!
//! Comprehensive text statistics: word, sentence, and character counts,
//! average word length, average sentence length, estimated reading time,
//! and a lexile-like difficulty estimate. Pure Rust, **zero dependencies**,
//! deterministic.
//!
//! These are the same measurements the [BeLikeNative](https://belikenative.com/)
//! AI writing assistant surfaces while you write — turned into a small,
//! embeddable library.
//!
//! ## Quick example
//!
//! ```
//! use text_statistics::stats;
//!
//! let s = stats("The cat sat on the mat. It was a good day for a nap.");
//! assert_eq!(s.words, 14);
//! assert_eq!(s.sentences, 2);
//! assert!(s.avg_word_length > 2.5);
//! assert!(s.estimated_reading_seconds < 60.0);
//! assert!(s.difficulty_band() == text_statistics::Difficulty::Easy);
//! ```

// ============================================================================
// Tunable constants. Kept as `pub` so callers can calibrate against their own
// corpora; the defaults match common publishing conventions.
// ============================================================================

/// Average adult reading speed, in words per minute. 200 wpm is the
/// widely-cited average for silent reading of English prose.
pub const WORDS_PER_MINUTE: f64 = 200.0;

/// Lexile-like difficulty reference: characters per word at grade-5 level.
/// Below this average word length the text is considered "easy".
pub const EASY_AVG_WORD_LENGTH: f64 = 4.5;

/// Above this average word length the text is considered "difficult"
/// (roughly grade-11+ vocabulary density).
pub const HARD_AVG_WORD_LENGTH: f64 = 6.5;

/// Above this average sentence length (in words) the text is considered
/// "difficult" to follow.
pub const HARD_AVG_SENTENCE_LENGTH: f64 = 20.0;

/// A coarse difficulty band derived from the lexile-like estimate.
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub enum Difficulty {
    /// Short words, short sentences — early-grade reading.
    Easy,
    /// Typical adult prose.
    Medium,
    /// Long words or long sentences — advanced reading.
    Hard,
}

/// The full bundle of computed statistics for a piece of text.
///
/// All fields are computed from a single pass over the input. Counts are
/// `u64` so they compose cleanly with very large documents; averages and
/// durations are `f64`.
#[derive(Debug, Clone, PartialEq)]
pub struct TextStats {
    /// Number of whitespace-or-punctuation-delimited word tokens.
    pub words: u64,
    /// Number of sentence-ending units (`.` `!` `?`), with a trailing
    /// sentence counted even if the final terminator is omitted.
    pub sentences: u64,
    /// Total number of Unicode scalar values (Rust `char`s), including
    /// whitespace and punctuation.
    pub characters: u64,
    /// Characters excluding ASCII whitespace.
    pub characters_no_spaces: u64,
    /// Average number of characters per word (letters only, per word).
    pub avg_word_length: f64,
    /// Average number of words per sentence.
    pub avg_sentence_length: f64,
    /// Estimated reading time in seconds (at [`WORDS_PER_MINUTE`]).
    pub estimated_reading_seconds: f64,
    /// Estimated reading time in whole minutes (rounded up, minimum 1 if
    /// the text is non-empty).
    pub estimated_reading_minutes: u64,
    /// A 0–100 lexile-like difficulty estimate; higher means harder.
    pub difficulty_score: f64,
}

impl TextStats {
    /// Map the continuous [`difficulty_score`] to a coarse band.
    ///
    /// [`difficulty_score`]: TextStats::difficulty_score
    #[must_use]
    pub fn difficulty_band(&self) -> Difficulty {
        if self.difficulty_score < 33.0 {
            Difficulty::Easy
        } else if self.difficulty_score < 66.0 {
            Difficulty::Medium
        } else {
            Difficulty::Hard
        }
    }
}

/// Split prose into word tokens.
///
/// A word is a maximal run of ASCII letters or digits or apostrophes. This
/// keeps contractions (`don't`) as a single token while dropping pure
/// punctuation/whitespace.
fn word_tokens(text: &str) -> Vec<String> {
    text.split(|c: char| !(c.is_alphanumeric() || c == '\''))
        .filter(|s| !s.is_empty())
        .map(|s| s.to_ascii_lowercase())
        .collect()
}

/// Count sentence-ending units. Abbreviations are not specially handled —
/// this is a deterministic token count, not a parse.
fn count_sentences(text: &str) -> u64 {
    let n = text.chars().filter(|&c| c == '.' || c == '!' || c == '?').count() as u64;
    // If the text has words but no terminators, treat it as one sentence.
    if n == 0 && text.chars().any(char::is_alphanumeric) {
        1
    } else {
        n
    }
}

/// Compute the full [`TextStats`] bundle for `text`.
///
/// Empty input yields all-zero counts (and zero difficulty). The function is
/// total: it never panics and allocates only for the temporary word list.
#[must_use]
pub fn stats(text: &str) -> TextStats {
    let words = word_tokens(text);
    let word_count = words.len() as u64;
    let sentences = count_sentences(text);
    let characters = text.chars().count() as u64;
    let characters_no_spaces = text.chars().filter(|c| !c.is_whitespace()).count() as u64;

    // Average word length: count letters (alphabetic) to keep punctuation
    // out of the figure. Digits are excluded so "12345" does not inflate it.
    let total_letters: u64 = words.iter().map(|w| w.chars().filter(|c| c.is_alphabetic()).count() as u64).sum();
    let avg_word_length = if word_count == 0 {
        0.0
    } else {
        total_letters as f64 / word_count as f64
    };

    let avg_sentence_length = if sentences == 0 {
        0.0
    } else {
        word_count as f64 / sentences as f64
    };

    // Reading time. At least one second for any non-empty text so a single
    // word is not reported as "0 seconds".
    let estimated_reading_seconds = if word_count == 0 {
        0.0
    } else {
        let secs = (word_count as f64 / WORDS_PER_MINUTE) * 60.0;
        secs.max(1.0)
    };
    let estimated_reading_minutes = if word_count == 0 {
        0
    } else {
        ((estimated_reading_seconds / 60.0).ceil() as u64).max(1)
    };

    // Lexile-like difficulty: blend word length and sentence length signals.
    // Each signal contributes 0..50; together they span 0..100.
    let word_signal = clamp01((avg_word_length - EASY_AVG_WORD_LENGTH) / (HARD_AVG_WORD_LENGTH - EASY_AVG_WORD_LENGTH)) * 50.0;
    let sentence_signal = clamp01(avg_sentence_length / HARD_AVG_SENTENCE_LENGTH) * 50.0;
    let difficulty_score = word_signal + sentence_signal;

    TextStats {
        words: word_count,
        sentences,
        characters,
        characters_no_spaces,
        avg_word_length,
        avg_sentence_length,
        estimated_reading_seconds,
        estimated_reading_minutes,
        difficulty_score,
    }
}

#[inline]
fn clamp01(x: f64) -> f64 {
    if x < 0.0 {
        0.0
    } else if x > 1.0 {
        1.0
    } else {
        x
    }
}

#[cfg(test)]
mod tests {
    use super::*;

    #[test]
    fn empty_input_is_all_zero() {
        let s = stats("");
        assert_eq!(s.words, 0);
        assert_eq!(s.sentences, 0);
        assert_eq!(s.characters, 0);
        assert_eq!(s.avg_word_length, 0.0);
        assert_eq!(s.difficulty_score, 0.0);
        assert_eq!(s.difficulty_band(), Difficulty::Easy);
    }

    #[test]
    fn counts_simple_sentence() {
        let s = stats("The cat sat on the mat.");
        assert_eq!(s.words, 6);
        assert_eq!(s.sentences, 1);
        // Letters: the(3) cat(3) sat(3) on(2) the(3) mat(3) = 17 / 6 words.
        assert!((s.avg_word_length - (17.0 / 6.0)).abs() < 1e-9);
        assert_eq!(s.avg_sentence_length, 6.0);
    }

    #[test]
    fn counts_multiple_sentences_with_mixed_terminators() {
        let s = stats("Hello world! Are you ready? Let's go.");
        assert_eq!(s.sentences, 3);
        assert_eq!(s.words, 7); // hello world are you ready let's go
    }

    #[test]
    fn contraction_is_one_word() {
        let s = stats("don't can't won't");
        assert_eq!(s.words, 3);
    }

    #[test]
    fn words_without_terminator_still_count_as_sentence() {
        let s = stats("just a fragment with no ending");
        assert_eq!(s.sentences, 1);
        assert_eq!(s.words, 6);
    }

    #[test]
    fn character_counts() {
        let s = stats("ab c");
        assert_eq!(s.characters, 4); // a b space c
        assert_eq!(s.characters_no_spaces, 3);
    }

    #[test]
    fn reading_time_at_200_wpm() {
        // 400 words at 200 wpm = 2 minutes = 120 seconds.
        let text: String = std::iter::repeat("word ").take(400).collect();
        let s = stats(&text);
        assert!((s.estimated_reading_seconds - 120.0).abs() < 1e-6);
        assert_eq!(s.estimated_reading_minutes, 2);
    }

    #[test]
    fn single_word_minimum_one_second() {
        let s = stats("hi");
        assert!(s.estimated_reading_seconds >= 1.0);
        assert_eq!(s.estimated_reading_minutes, 1);
    }

    #[test]
    fn easy_prose_is_easy_band() {
        let s = stats("The cat sat on the mat. The dog ran. It was fun.");
        assert_eq!(s.difficulty_band(), Difficulty::Easy);
        assert!(s.difficulty_score < 33.0);
    }

    #[test]
    fn long_words_and_sentences_are_hard() {
        // Long, multi-syllable words and a long sentence.
        let text = "Notwithstanding the fundamental institutional accommodations, the multidisciplinary organizational representatives nevertheless demonstrated extraordinary quantitative methodological considerations throughout the extensively protracted deliberations.";
        let s = stats(text);
        assert_eq!(s.difficulty_band(), Difficulty::Hard);
        assert!(s.difficulty_score > 66.0);
    }

    #[test]
    fn difficulty_score_bounded_0_to_100() {
        // Pathological inputs should not overflow the band.
        let very_long: String = std::iter::repeat("internationalization ").take(500).collect();
        let s = stats(&very_long);
        assert!(s.difficulty_score <= 100.0);
        assert!(s.difficulty_score >= 0.0);
    }

    #[test]
    fn digits_excluded_from_avg_word_length() {
        // "12345" has no letters -> 0 letters / 1 word = 0 avg length.
        let s = stats("12345");
        assert_eq!(s.words, 1);
        assert_eq!(s.avg_word_length, 0.0);
    }

    #[test]
    fn deterministic_on_repeated_calls() {
        let t = "The quick brown fox jumps over the lazy dog.";
        let a = stats(t);
        let b = stats(t);
        assert_eq!(a, b);
    }
}