readsight 1.0.0

Multilingual readability library — 86 languages, 17 formulas, TeX-based syllable counting via the Frank M. Liang algorithm.
Documentation
//! Coverage-focused tests: exercise every interpretation branch, error variant,
//! and edge path not already hit by the golden/parity suites.

use std::collections::BTreeMap;

use readsight::error::Error;
use readsight::formula::impls::{
    AutomatedReadabilityIndex, ColemanLiau, Crawford, DaleChall, FernandezHuerta,
    FleschKincaidGradeLevel, FleschReadingEase, FogPL, Gulpease, GunningFog, GutierrezPolini, Lix,
    Osman, SmogIndex, Spache, SzigrisztPazos, WienerSachtextformel,
};
use readsight::formula::{registry, Formula};
use readsight::language::Language;
use readsight::text::TextSplitter;
use readsight::{Config, ReadSight, Script, TextStatistics};

fn generic_language() -> Language {
    Language::from_json(
        r#"{
        "code": "en-us", "name": "x", "nativeName": "x", "script": "Latin",
        "hyphenMins": {"left": 2, "right": 2},
        "letterPattern": "[a-z]", "wordSplitPattern": "[^a-z]+", "sentenceBoundaryPattern": "[.]+",
        "formulas": {"lix": {"longWordThreshold": 6},
                     "flesch_reading_ease": {"base": 206.835, "aslMult": 1.015, "aswMult": 84.6}}
    }"#,
    )
    .unwrap()
}

fn stats_for(t: f64) -> TextStatistics {
    let word_count = 100i64;
    let sentence_count = (60.0 - t / 8.0).max(1.0) as i64;
    let hist1 = (100.0 - t / 4.0).max(0.0) as i64;
    let mut histogram: BTreeMap<i64, i64> = BTreeMap::new();
    if hist1 > 0 {
        histogram.insert(1, hist1);
    }
    histogram.insert(2, ((t / 8.0) as i64).max(0));
    histogram.insert(3, ((t / 8.0) as i64).max(0));
    TextStatistics {
        letter_count: (t * 6.0) as i64,
        word_count,
        sentence_count,
        syllable_count: (t * 4.0) as i64,
        polysyllable_count: (t / 4.0).min(100.0) as i64,
        average_syllables_per_word: t * 0.04,
        average_words_per_sentence: t,
        long_word_count: (t / 4.0).min(100.0) as i64,
        syllable_histogram: histogram,
    }
}

#[test]
fn formula_interpretation_branches_all_reachable() {
    let lang = generic_language();
    let wiener = WienerSachtextformel;

    let mut seen: BTreeMap<&'static str, std::collections::BTreeSet<String>> = BTreeMap::new();

    let universal: Vec<Box<dyn Formula>> = vec![
        Box::new(GunningFog),
        Box::new(SmogIndex),
        Box::new(ColemanLiau),
        Box::new(AutomatedReadabilityIndex),
        Box::new(Lix),
        Box::new(FleschReadingEase),
        Box::new(FleschKincaidGradeLevel),
        Box::new(Gulpease),
        Box::new(FernandezHuerta),
        Box::new(SzigrisztPazos),
        Box::new(GutierrezPolini),
        Box::new(Crawford),
        Box::new(FogPL),
        Box::new(DaleChall),
        Box::new(Spache),
        Box::new(Osman),
    ];

    let run =
        |s: &TextStatistics,
         seen: &mut BTreeMap<&'static str, std::collections::BTreeSet<String>>| {
            for f in &universal {
                let r = f.calculate(s, &lang);
                assert!(r.score.is_finite());
                seen.entry(f.name())
                    .or_default()
                    .insert(r.interpretation.clone());
            }
            for v in 1..=4 {
                let r = wiener.calculate_variant(s, &lang, v).unwrap();
                seen.entry("wiener")
                    .or_default()
                    .insert(r.interpretation.clone());
            }
        };

    // Sweep 1: large word_count, sweep averages/counts.
    for step in 0..=400 {
        let s = stats_for(step as f64);
        run(&s, &mut seen);
    }

    // Sweep 2: small word_count so per-100/ratio driven formulas
    // (Crawford, Osman, Gutiérrez, Gulpease, ...) reach their high bands.
    for step in 0..=300 {
        let t = step as f64;
        let mut histogram: BTreeMap<i64, i64> = BTreeMap::new();
        histogram.insert(1, (10.0 - t / 30.0).max(0.0) as i64);
        let s = TextStatistics {
            letter_count: (t * 0.3) as i64,
            word_count: 10,
            sentence_count: (step as i64).max(1),
            syllable_count: (t * 0.2) as i64,
            polysyllable_count: (t / 30.0) as i64,
            average_syllables_per_word: t * 0.01,
            average_words_per_sentence: t * 0.05,
            long_word_count: (t / 30.0) as i64,
            syllable_histogram: histogram,
        };
        run(&s, &mut seen);
    }

    // Sweep 3: high-magnitude asl (word/sentence) so Osman and the fog/grade
    // formulas reach their low/high bands.
    for step in 1..=160 {
        let wc = step as i64;
        let mut histogram: BTreeMap<i64, i64> = BTreeMap::new();
        histogram.insert(1, wc);
        let s = TextStatistics {
            letter_count: wc * 2,
            word_count: wc,
            sentence_count: 1,
            syllable_count: wc,
            polysyllable_count: wc / 2,
            average_syllables_per_word: step as f64 * 0.05,
            average_words_per_sentence: step as f64,
            long_word_count: wc / 2,
            syllable_histogram: histogram,
        };
        run(&s, &mut seen);
    }

    assert!(seen[&"gunning_fog"].len() >= 6);
    assert!(seen[&"flesch_reading_ease"].len() >= 7);
    assert!(seen[&"lix"].len() >= 6);
    assert!(seen[&"dale_chall"].len() >= 7);
    assert!(seen[&"spache"].len() >= 5);
    assert!(seen[&"wiener"].len() >= 6);
    assert!(seen[&"szigriszt_pazos"].len() >= 7);
    assert!(seen[&"crawford"].len() >= 5);
    assert!(seen[&"osman"].len() >= 5);
    assert!(seen[&"gulpease"].len() >= 4);
    assert!(seen[&"gutierrez_polini"].len() >= 5);
    assert!(seen[&"fernandez_huerta"].len() >= 7);
    assert!(seen[&"fog_pl"].len() >= 5);
    assert!(seen[&"flesch_kincaid_grade_level"].len() >= 6);
    assert!(seen[&"coleman_liau"].len() >= 6);
    assert!(seen[&"ari"].len() >= 6);
    assert!(seen[&"smog"].len() >= 4);
}

#[test]
fn formula_metadata_accessors() {
    let f = GunningFog;
    assert_eq!(f.name(), "gunning_fog");
    assert!(!f.description().is_empty());
    assert_eq!(f.supported_languages(), &["*"]);
    assert!(!Osman.description().is_empty());
    assert_eq!(Osman.supported_languages(), &["ar"]);
    assert!(!WienerSachtextformel.description().is_empty());
}

#[test]
fn every_formula_metadata_is_populated() {
    let formulas: Vec<Box<dyn Formula>> = vec![
        Box::new(GunningFog),
        Box::new(SmogIndex),
        Box::new(ColemanLiau),
        Box::new(AutomatedReadabilityIndex),
        Box::new(Lix),
        Box::new(FleschReadingEase),
        Box::new(FleschKincaidGradeLevel),
        Box::new(WienerSachtextformel),
        Box::new(Gulpease),
        Box::new(FernandezHuerta),
        Box::new(SzigrisztPazos),
        Box::new(GutierrezPolini),
        Box::new(Crawford),
        Box::new(FogPL),
        Box::new(DaleChall),
        Box::new(Spache),
        Box::new(Osman),
    ];
    for f in &formulas {
        assert!(!f.name().is_empty());
        assert!(!f.description().is_empty());
        assert!(!f.supported_languages().is_empty());
    }
}

#[test]
fn formulas_handle_zero_word_count() {
    // Exercises the `word_count > 0` else-branches (Gunning Fog, LIX, Wiener).
    let lang = generic_language();
    let zero = TextStatistics {
        letter_count: 0,
        word_count: 0,
        sentence_count: 0,
        syllable_count: 0,
        polysyllable_count: 0,
        average_syllables_per_word: 0.0,
        average_words_per_sentence: 0.0,
        long_word_count: 0,
        syllable_histogram: BTreeMap::new(),
    };
    assert!(GunningFog.calculate(&zero, &lang).score.is_finite());
    assert!(Lix.calculate(&zero, &lang).score.is_finite());
    let w = WienerSachtextformel
        .calculate_variant(&zero, &lang, 1)
        .unwrap();
    assert!(w.score.is_finite());
    assert!(WienerSachtextformel
        .calculate_variant(&zero, &lang, 2)
        .unwrap()
        .score
        .is_finite());
}

#[test]
fn registry_default_constructor() {
    use readsight::formula::FormulaRegistry;
    let empty = FormulaRegistry::default();
    assert!(!empty.has("smog"));
    assert!(empty.list_names().is_empty());
}

#[test]
fn registry_operations() {
    let mut reg = registry::create();
    assert_eq!(reg.list_names().len(), 17);
    assert!(reg.has("smog"));
    assert!(!reg.has("nope"));
    assert!(reg.get("smog").is_some());
    assert!(reg.get("nope").is_none());

    let lang = generic_language();
    let stats = stats_for(20.0);
    assert!(reg.calculate("smog", &lang, &stats).is_ok());
    match reg.calculate("nope", &lang, &stats) {
        Err(Error::UnsupportedFormula { formula, .. }) => assert_eq!(formula, "nope"),
        _ => panic!("expected UnsupportedFormula"),
    }

    // re-register (overwrite path)
    reg.register(Box::new(SmogIndex));
    assert_eq!(reg.list_names().len(), 17);

    let en = ReadSight::new("en-us").unwrap();
    let for_en = reg.list_for_language(en.language());
    assert!(for_en.contains(&"smog".to_string()));
    assert!(!for_en.contains(&"gulpease".to_string()));
}

#[test]
fn all_script_variants_as_str() {
    let cases = [
        (Script::Latin, "Latin"),
        (Script::Cyrillic, "Cyrillic"),
        (Script::Arabic, "Arabic"),
        (Script::Hebrew, "Hebrew"),
        (Script::Devanagari, "Devanagari"),
        (Script::Bengali, "Bengali"),
        (Script::Greek, "Greek"),
        (Script::Armenian, "Armenian"),
        (Script::Georgian, "Georgian"),
        (Script::Thai, "Thai"),
        (Script::Tamil, "Tamil"),
        (Script::Telugu, "Telugu"),
        (Script::Kannada, "Kannada"),
        (Script::Malayalam, "Malayalam"),
        (Script::Gujarati, "Gujarati"),
        (Script::Gurmukhi, "Gurmukhi"),
        (Script::Odia, "Odia"),
        (Script::Ethiopic, "Ethiopic"),
        (Script::Coptic, "Coptic"),
        (Script::CJK, "CJK"),
        (Script::Other, "Other"),
    ];
    for (script, name) in cases {
        assert_eq!(script.as_str(), name);
    }
}

#[test]
fn error_variants_display_and_source() {
    use std::error::Error as _;

    let io = Error::from(std::io::Error::other("boom"));
    assert!(io.to_string().contains("I/O error"));
    assert!(io.source().is_some());

    let json = Error::from(serde_json::from_str::<serde_json::Value>("not json").unwrap_err());
    assert!(json.to_string().contains("JSON error"));
    assert!(json.source().is_some());

    let parse = Error::PatternParse {
        token: "a2".into(),
        line: 3,
        file: "f.tex".into(),
    };
    assert!(parse.to_string().contains("a2"));
    assert!(parse.source().is_none());
}

#[test]
fn language_value_parsing_edges() {
    let lang = Language::from_json(
        r#"{
        "code": "xx", "name": "x", "nativeName": "x", "script": "Other",
        "hyphenMins": {"left": 1, "right": 1},
        "letterPattern": "[a-z]", "wordSplitPattern": "[^a-z]+", "sentenceBoundaryPattern": "[.]+",
        "formulas": {"flesch_reading_ease": {"base": "150.5", "aslMult": true}}
    }"#,
    )
    .unwrap();
    // numeric string is parsed
    assert_eq!(
        lang.formula_number("flesch_reading_ease", "base"),
        Some(150.5)
    );
    // non-numeric (bool) yields None
    assert_eq!(lang.formula_number("flesch_reading_ease", "aslMult"), None);
    // missing key / missing formula yield None
    assert_eq!(lang.formula_number("flesch_reading_ease", "zzz"), None);
    assert_eq!(lang.formula_number("nope", "base"), None);
    assert_eq!(lang.syllable_mode, "tex"); // default when absent
    assert!(lang.syllable_heuristics.is_none());
}

#[test]
fn analyzer_empty_text_average_paths() {
    let rs = ReadSight::new("en-us").unwrap();
    assert_eq!(rs.average_syllables_per_word(""), 0.0);
    assert_eq!(rs.average_words_per_sentence(""), 0.0);
    // split_syllables delegate
    let parts = rs.split_syllables("communication");
    assert!(!parts.is_empty());
}

#[test]
fn invalid_regex_pattern_errors() {
    let lang = Language::from_json(
        r#"{
        "code": "xx", "name": "x", "nativeName": "x", "script": "Other",
        "hyphenMins": {"left": 1, "right": 1},
        "letterPattern": "[unterminated", "wordSplitPattern": "[^a-z]+", "sentenceBoundaryPattern": "[.]+"
    }"#,
    )
    .unwrap();
    assert!(matches!(
        TextSplitter::new(&lang),
        Err(Error::InvalidPattern(_))
    ));
}

#[test]
fn filesystem_missing_language_is_none() {
    let base = std::path::PathBuf::from(env!("CARGO_MANIFEST_DIR")).join("data");
    let config = Config::from_dirs(base.join("patterns"), base.join("languages"));
    assert!(matches!(
        ReadSight::with_config("zz-nonexistent", config),
        Err(Error::UnsupportedLanguage(_))
    ));
}