readsight 1.0.2

Multilingual readability library — 86 languages, 17 formulas, TeX-based syllable counting via the Frank M. Liang algorithm.
Documentation
//! Parity tests against golden vectors generated from the canonical PHP library.
//!
//! Golden data lives in `tests/golden/` and is produced by `temp/gen_golden.php`.

use std::collections::BTreeMap;
use std::fs;
use std::path::PathBuf;

use readsight::{FormulaResult, ReadSight};
use serde_json::Value;

const TOL: f64 = 1e-9;

fn golden_dir() -> PathBuf {
    PathBuf::from(env!("CARGO_MANIFEST_DIR")).join("tests/golden")
}

fn load(name: &str) -> Value {
    let path = golden_dir().join(name);
    let text = fs::read_to_string(&path)
        .unwrap_or_else(|e| panic!("failed to read {}: {e}", path.display()));
    serde_json::from_str(&text).expect("valid golden JSON")
}

fn approx(a: f64, b: f64, ctx: &str) {
    assert!(
        (a - b).abs() <= TOL || (a.is_nan() && b.is_nan()),
        "{ctx}: got {a}, expected {b} (|diff|={})",
        (a - b).abs()
    );
}

fn opt_f64(v: &Value) -> Option<f64> {
    match v {
        Value::Null => None,
        Value::Number(n) => n.as_f64(),
        _ => panic!("expected number or null, got {v:?}"),
    }
}

fn assert_result(got: &FormulaResult, exp: &Value, ctx: &str) {
    assert_eq!(
        got.formula_name,
        exp["formula_name"].as_str().unwrap(),
        "{ctx}: formula_name"
    );
    assert_eq!(
        got.language_code,
        exp["language_code"].as_str().unwrap(),
        "{ctx}: language_code"
    );
    approx(
        got.score,
        exp["score"].as_f64().unwrap(),
        &format!("{ctx}: score"),
    );

    match (got.grade_level, opt_f64(&exp["grade_level"])) {
        (None, None) => {}
        (Some(a), Some(b)) => approx(a, b, &format!("{ctx}: grade_level")),
        (a, b) => panic!("{ctx}: grade_level mismatch got {a:?} expected {b:?}"),
    }

    assert_eq!(
        got.interpretation,
        exp["interpretation"].as_str().unwrap(),
        "{ctx}: interpretation"
    );

    let exp_inputs = exp["inputs"].as_object().unwrap();
    assert_eq!(
        got.inputs.len(),
        exp_inputs.len(),
        "{ctx}: inputs key count (got {:?}, expected {:?})",
        got.inputs.keys().collect::<Vec<_>>(),
        exp_inputs.keys().collect::<Vec<_>>()
    );
    for (key, exp_val) in exp_inputs {
        let got_val = got
            .inputs
            .get(key)
            .unwrap_or_else(|| panic!("{ctx}: missing input key {key}"));
        approx(
            *got_val,
            exp_val.as_f64().unwrap(),
            &format!("{ctx}: input {key}"),
        );
    }
}

#[test]
fn supported_languages_match() {
    let expected: Vec<String> =
        serde_json::from_value(load("languages.json")).expect("languages array");
    let got = ReadSight::supported_languages(None);
    assert_eq!(got.len(), 86, "expected 86 languages");
    assert_eq!(got, expected, "supported languages list (order + values)");
}

#[test]
fn syllable_vectors_match() {
    let golden = load("syllable.json");
    let obj = golden.as_object().unwrap();
    for (lang, entry) in obj {
        let rs = ReadSight::new(lang).unwrap_or_else(|e| panic!("engine {lang}: {e}"));

        for (word, exp) in entry["syllable_count"].as_object().unwrap() {
            let got = rs.syllable_count(word);
            assert_eq!(got, exp.as_i64().unwrap(), "{lang} syllable_count({word})");
        }
        for (word, exp) in entry["split_word"].as_object().unwrap() {
            let got = rs.split_word(word);
            let exp: Vec<String> = serde_json::from_value(exp.clone()).unwrap();
            assert_eq!(got, exp, "{lang} split_word({word})");
        }
        for (word, exp) in entry["split_syllables"].as_object().unwrap() {
            let got = rs.split_syllables(word);
            let exp: Vec<String> = serde_json::from_value(exp.clone()).unwrap();
            assert_eq!(got, exp, "{lang} split_syllables({word})");
        }
    }
}

#[test]
fn analyze_and_formula_vectors_match() {
    let golden = load("analyze.json");
    let obj = golden.as_object().unwrap();

    for (code, entry) in obj {
        let rs = ReadSight::new(code).unwrap_or_else(|e| panic!("engine {code}: {e}"));

        // supported formulas list (registration order)
        let exp_formulas: Vec<String> =
            serde_json::from_value(entry["supported_formulas"].clone()).unwrap();
        assert_eq!(
            rs.supported_formulas(),
            exp_formulas,
            "{code}: supported_formulas"
        );

        for text_key in ["latin", "cyrillic"] {
            let block = &entry[text_key];
            let text = sample_text(text_key);
            let ctx = format!("{code}/{text_key}");

            // stats
            let stats = rs
                .analyze(text)
                .unwrap_or_else(|e| panic!("{ctx}: analyze {e}"));
            let exp_stats = &block["stats"];
            assert_eq!(
                stats.letter_count,
                exp_stats["letter_count"].as_i64().unwrap(),
                "{ctx}: letter_count"
            );
            assert_eq!(
                stats.word_count,
                exp_stats["word_count"].as_i64().unwrap(),
                "{ctx}: word_count"
            );
            assert_eq!(
                stats.sentence_count,
                exp_stats["sentence_count"].as_i64().unwrap(),
                "{ctx}: sentence_count"
            );
            assert_eq!(
                stats.syllable_count,
                exp_stats["syllable_count"].as_i64().unwrap(),
                "{ctx}: syllable_count"
            );
            assert_eq!(
                stats.polysyllable_count,
                exp_stats["polysyllable_count"].as_i64().unwrap(),
                "{ctx}: polysyllable_count"
            );
            approx(
                stats.average_syllables_per_word,
                exp_stats["average_syllables_per_word"].as_f64().unwrap(),
                &format!("{ctx}: average_syllables_per_word"),
            );
            approx(
                stats.average_words_per_sentence,
                exp_stats["average_words_per_sentence"].as_f64().unwrap(),
                &format!("{ctx}: average_words_per_sentence"),
            );
            assert_eq!(
                stats.long_word_count,
                exp_stats["long_word_count"].as_i64().unwrap(),
                "{ctx}: long_word_count"
            );
            let exp_hist: BTreeMap<i64, i64> = exp_stats["syllable_histogram"]
                .as_object()
                .unwrap()
                .iter()
                .map(|(k, v)| (k.parse().unwrap(), v.as_i64().unwrap()))
                .collect();
            assert_eq!(stats.syllable_histogram, exp_hist, "{ctx}: histogram");

            // formulas
            for (fname, exp) in block["formulas"].as_object().unwrap() {
                let got = rs
                    .score(fname, text)
                    .unwrap_or_else(|e| panic!("{ctx}: score {fname} {e}"));
                assert_result(&got, exp, &format!("{ctx}: {fname}"));
            }

            // wiener variants
            if let Some(wiener) = block["wiener"].as_object() {
                for (vk, exp) in wiener {
                    let variant: i32 = vk.trim_start_matches('v').parse().unwrap();
                    let got = rs
                        .wiener_sachtextformel(text, variant)
                        .unwrap_or_else(|e| panic!("{ctx}: wiener {variant} {e}"));
                    assert_result(&got, exp, &format!("{ctx}: wiener v{variant}"));
                }
            }
        }
    }
}

fn sample_text(key: &str) -> &'static str {
    match key {
        "latin" => {
            "The quick brown fox jumps over the lazy dog. This sentence provides a simple readability sample for testing purposes and evaluation."
        }
        "cyrillic" => {
            "Быстрая коричневая лиса прыгает через ленивую собаку. Это предложение служит хорошим примером для тестирования читабельности текста."
        }
        _ => unreachable!(),
    }
}