use std::collections::BTreeMap;
use readsight::error::Error;
use readsight::formula::impls::{
AutomatedReadabilityIndex, ColemanLiau, Crawford, DaleChall, FernandezHuerta,
FleschKincaidGradeLevel, FleschReadingEase, FogPL, Gulpease, GunningFog, GutierrezPolini, Lix,
Osman, SmogIndex, Spache, SzigrisztPazos, WienerSachtextformel,
};
use readsight::formula::{registry, Formula};
use readsight::language::Language;
use readsight::text::TextSplitter;
use readsight::{Config, ReadSight, Script, TextStatistics};
fn generic_language() -> Language {
Language::from_json(
r#"{
"code": "en-us", "name": "x", "nativeName": "x", "script": "Latin",
"hyphenMins": {"left": 2, "right": 2},
"letterPattern": "[a-z]", "wordSplitPattern": "[^a-z]+", "sentenceBoundaryPattern": "[.]+",
"formulas": {"lix": {"longWordThreshold": 6},
"flesch_reading_ease": {"base": 206.835, "aslMult": 1.015, "aswMult": 84.6}}
}"#,
)
.unwrap()
}
fn stats_for(t: f64) -> TextStatistics {
let word_count = 100i64;
let sentence_count = (60.0 - t / 8.0).max(1.0) as i64;
let hist1 = (100.0 - t / 4.0).max(0.0) as i64;
let mut histogram: BTreeMap<i64, i64> = BTreeMap::new();
if hist1 > 0 {
histogram.insert(1, hist1);
}
histogram.insert(2, ((t / 8.0) as i64).max(0));
histogram.insert(3, ((t / 8.0) as i64).max(0));
TextStatistics {
letter_count: (t * 6.0) as i64,
word_count,
sentence_count,
syllable_count: (t * 4.0) as i64,
polysyllable_count: (t / 4.0).min(100.0) as i64,
average_syllables_per_word: t * 0.04,
average_words_per_sentence: t,
long_word_count: (t / 4.0).min(100.0) as i64,
syllable_histogram: histogram,
}
}
#[test]
fn formula_interpretation_branches_all_reachable() {
let lang = generic_language();
let wiener = WienerSachtextformel;
let mut seen: BTreeMap<&'static str, std::collections::BTreeSet<String>> = BTreeMap::new();
let universal: Vec<Box<dyn Formula>> = vec![
Box::new(GunningFog),
Box::new(SmogIndex),
Box::new(ColemanLiau),
Box::new(AutomatedReadabilityIndex),
Box::new(Lix),
Box::new(FleschReadingEase),
Box::new(FleschKincaidGradeLevel),
Box::new(Gulpease),
Box::new(FernandezHuerta),
Box::new(SzigrisztPazos),
Box::new(GutierrezPolini),
Box::new(Crawford),
Box::new(FogPL),
Box::new(DaleChall),
Box::new(Spache),
Box::new(Osman),
];
let run =
|s: &TextStatistics,
seen: &mut BTreeMap<&'static str, std::collections::BTreeSet<String>>| {
for f in &universal {
let r = f.calculate(s, &lang);
assert!(r.score.is_finite());
seen.entry(f.name())
.or_default()
.insert(r.interpretation.clone());
}
for v in 1..=4 {
let r = wiener.calculate_variant(s, &lang, v).unwrap();
seen.entry("wiener")
.or_default()
.insert(r.interpretation.clone());
}
};
for step in 0..=400 {
let s = stats_for(step as f64);
run(&s, &mut seen);
}
for step in 0..=300 {
let t = step as f64;
let mut histogram: BTreeMap<i64, i64> = BTreeMap::new();
histogram.insert(1, (10.0 - t / 30.0).max(0.0) as i64);
let s = TextStatistics {
letter_count: (t * 0.3) as i64,
word_count: 10,
sentence_count: (step as i64).max(1),
syllable_count: (t * 0.2) as i64,
polysyllable_count: (t / 30.0) as i64,
average_syllables_per_word: t * 0.01,
average_words_per_sentence: t * 0.05,
long_word_count: (t / 30.0) as i64,
syllable_histogram: histogram,
};
run(&s, &mut seen);
}
for step in 1..=160 {
let wc = step as i64;
let mut histogram: BTreeMap<i64, i64> = BTreeMap::new();
histogram.insert(1, wc);
let s = TextStatistics {
letter_count: wc * 2,
word_count: wc,
sentence_count: 1,
syllable_count: wc,
polysyllable_count: wc / 2,
average_syllables_per_word: step as f64 * 0.05,
average_words_per_sentence: step as f64,
long_word_count: wc / 2,
syllable_histogram: histogram,
};
run(&s, &mut seen);
}
assert!(seen[&"gunning_fog"].len() >= 6);
assert!(seen[&"flesch_reading_ease"].len() >= 7);
assert!(seen[&"lix"].len() >= 6);
assert!(seen[&"dale_chall"].len() >= 7);
assert!(seen[&"spache"].len() >= 5);
assert!(seen[&"wiener"].len() >= 6);
assert!(seen[&"szigriszt_pazos"].len() >= 7);
assert!(seen[&"crawford"].len() >= 5);
assert!(seen[&"osman"].len() >= 5);
assert!(seen[&"gulpease"].len() >= 4);
assert!(seen[&"gutierrez_polini"].len() >= 5);
assert!(seen[&"fernandez_huerta"].len() >= 7);
assert!(seen[&"fog_pl"].len() >= 5);
assert!(seen[&"flesch_kincaid_grade_level"].len() >= 6);
assert!(seen[&"coleman_liau"].len() >= 6);
assert!(seen[&"ari"].len() >= 6);
assert!(seen[&"smog"].len() >= 4);
}
#[test]
fn formula_metadata_accessors() {
let f = GunningFog;
assert_eq!(f.name(), "gunning_fog");
assert!(!f.description().is_empty());
assert_eq!(f.supported_languages(), &["*"]);
assert!(!Osman.description().is_empty());
assert_eq!(Osman.supported_languages(), &["ar"]);
assert!(!WienerSachtextformel.description().is_empty());
}
#[test]
fn every_formula_metadata_is_populated() {
let formulas: Vec<Box<dyn Formula>> = vec![
Box::new(GunningFog),
Box::new(SmogIndex),
Box::new(ColemanLiau),
Box::new(AutomatedReadabilityIndex),
Box::new(Lix),
Box::new(FleschReadingEase),
Box::new(FleschKincaidGradeLevel),
Box::new(WienerSachtextformel),
Box::new(Gulpease),
Box::new(FernandezHuerta),
Box::new(SzigrisztPazos),
Box::new(GutierrezPolini),
Box::new(Crawford),
Box::new(FogPL),
Box::new(DaleChall),
Box::new(Spache),
Box::new(Osman),
];
for f in &formulas {
assert!(!f.name().is_empty());
assert!(!f.description().is_empty());
assert!(!f.supported_languages().is_empty());
}
}
#[test]
fn formulas_handle_zero_word_count() {
let lang = generic_language();
let zero = TextStatistics {
letter_count: 0,
word_count: 0,
sentence_count: 0,
syllable_count: 0,
polysyllable_count: 0,
average_syllables_per_word: 0.0,
average_words_per_sentence: 0.0,
long_word_count: 0,
syllable_histogram: BTreeMap::new(),
};
assert!(GunningFog.calculate(&zero, &lang).score.is_finite());
assert!(Lix.calculate(&zero, &lang).score.is_finite());
let w = WienerSachtextformel
.calculate_variant(&zero, &lang, 1)
.unwrap();
assert!(w.score.is_finite());
assert!(WienerSachtextformel
.calculate_variant(&zero, &lang, 2)
.unwrap()
.score
.is_finite());
}
#[test]
fn registry_default_constructor() {
use readsight::formula::FormulaRegistry;
let empty = FormulaRegistry::default();
assert!(!empty.has("smog"));
assert!(empty.list_names().is_empty());
}
#[test]
fn registry_operations() {
let mut reg = registry::create();
assert_eq!(reg.list_names().len(), 17);
assert!(reg.has("smog"));
assert!(!reg.has("nope"));
assert!(reg.get("smog").is_some());
assert!(reg.get("nope").is_none());
let lang = generic_language();
let stats = stats_for(20.0);
assert!(reg.calculate("smog", &lang, &stats).is_ok());
match reg.calculate("nope", &lang, &stats) {
Err(Error::UnsupportedFormula { formula, .. }) => assert_eq!(formula, "nope"),
_ => panic!("expected UnsupportedFormula"),
}
reg.register(Box::new(SmogIndex));
assert_eq!(reg.list_names().len(), 17);
let en = ReadSight::new("en-us").unwrap();
let for_en = reg.list_for_language(en.language());
assert!(for_en.contains(&"smog".to_string()));
assert!(!for_en.contains(&"gulpease".to_string()));
}
#[test]
fn all_script_variants_as_str() {
let cases = [
(Script::Latin, "Latin"),
(Script::Cyrillic, "Cyrillic"),
(Script::Arabic, "Arabic"),
(Script::Hebrew, "Hebrew"),
(Script::Devanagari, "Devanagari"),
(Script::Bengali, "Bengali"),
(Script::Greek, "Greek"),
(Script::Armenian, "Armenian"),
(Script::Georgian, "Georgian"),
(Script::Thai, "Thai"),
(Script::Tamil, "Tamil"),
(Script::Telugu, "Telugu"),
(Script::Kannada, "Kannada"),
(Script::Malayalam, "Malayalam"),
(Script::Gujarati, "Gujarati"),
(Script::Gurmukhi, "Gurmukhi"),
(Script::Odia, "Odia"),
(Script::Ethiopic, "Ethiopic"),
(Script::Coptic, "Coptic"),
(Script::CJK, "CJK"),
(Script::Other, "Other"),
];
for (script, name) in cases {
assert_eq!(script.as_str(), name);
}
}
#[test]
fn error_variants_display_and_source() {
use std::error::Error as _;
let io = Error::from(std::io::Error::other("boom"));
assert!(io.to_string().contains("I/O error"));
assert!(io.source().is_some());
let json = Error::from(serde_json::from_str::<serde_json::Value>("not json").unwrap_err());
assert!(json.to_string().contains("JSON error"));
assert!(json.source().is_some());
let parse = Error::PatternParse {
token: "a2".into(),
line: 3,
file: "f.tex".into(),
};
assert!(parse.to_string().contains("a2"));
assert!(parse.source().is_none());
}
#[test]
fn language_value_parsing_edges() {
let lang = Language::from_json(
r#"{
"code": "xx", "name": "x", "nativeName": "x", "script": "Other",
"hyphenMins": {"left": 1, "right": 1},
"letterPattern": "[a-z]", "wordSplitPattern": "[^a-z]+", "sentenceBoundaryPattern": "[.]+",
"formulas": {"flesch_reading_ease": {"base": "150.5", "aslMult": true}}
}"#,
)
.unwrap();
assert_eq!(
lang.formula_number("flesch_reading_ease", "base"),
Some(150.5)
);
assert_eq!(lang.formula_number("flesch_reading_ease", "aslMult"), None);
assert_eq!(lang.formula_number("flesch_reading_ease", "zzz"), None);
assert_eq!(lang.formula_number("nope", "base"), None);
assert_eq!(lang.syllable_mode, "tex"); assert!(lang.syllable_heuristics.is_none());
}
#[test]
fn analyzer_empty_text_average_paths() {
let rs = ReadSight::new("en-us").unwrap();
assert_eq!(rs.average_syllables_per_word(""), 0.0);
assert_eq!(rs.average_words_per_sentence(""), 0.0);
let parts = rs.split_syllables("communication");
assert!(!parts.is_empty());
}
#[test]
fn invalid_regex_pattern_errors() {
let lang = Language::from_json(
r#"{
"code": "xx", "name": "x", "nativeName": "x", "script": "Other",
"hyphenMins": {"left": 1, "right": 1},
"letterPattern": "[unterminated", "wordSplitPattern": "[^a-z]+", "sentenceBoundaryPattern": "[.]+"
}"#,
)
.unwrap();
assert!(matches!(
TextSplitter::new(&lang),
Err(Error::InvalidPattern(_))
));
}
#[test]
fn filesystem_missing_language_is_none() {
let base = std::path::PathBuf::from(env!("CARGO_MANIFEST_DIR")).join("data");
let config = Config::from_dirs(base.join("patterns"), base.join("languages"));
assert!(matches!(
ReadSight::with_config("zz-nonexistent", config),
Err(Error::UnsupportedLanguage(_))
));
}