packset_core/
prose.rs

1//! Hemingway-style complexity for the seat pack.
2//!
3//! A memory a model has to parse twice wastes the 1375 / 2200 caps. The grade
4//! is the Automated Readability Index, which is what Hemingway uses. Adverbs,
5//! passive be-verbs and long sentences are counted alongside it. No editor
6//! binary and no network: the whole check is arithmetic over the text.
7
8/// A sentence at or past this many words reads hard.
9pub const HARD_WORDS: usize = 20;
10/// A sentence at or past this many words reads very hard.
11pub const VERY_HARD_WORDS: usize = 30;
12/// Hemingway aims at grade 9. Only "very hard" is refused.
13pub const MAX_GRADE: f64 = 14.0;
14/// Adverbs as a fraction of words.
15pub const MAX_ADVERB_RATIO: f64 = 0.12;
16/// One claim per atom, and a claim is at most this many sentences.
17pub const MAX_ATOM_SENTENCES: usize = 2;
18
19/// What the text is being written into.
20#[derive(Debug, Clone, Copy, PartialEq, Eq)]
21pub enum Role {
22    /// One atom: one claim, so the sentence count is capped too.
23    Atom,
24    /// A card on disk: only the readability rules apply.
25    File,
26}
27
28/// Why text is too complex for the working core.
29#[derive(Debug, Clone, PartialEq, Eq)]
30pub struct ProseError(pub String);
31
32impl std::fmt::Display for ProseError {
33    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
34        f.write_str(&self.0)
35    }
36}
37
38impl std::error::Error for ProseError {}
39
40/// Readability counts for one piece of text.
41#[derive(Debug, Clone, PartialEq)]
42pub struct Report {
43    /// Word tokens.
44    pub words: usize,
45    /// Sentences, or zero when there are no words at all.
46    pub sentences: usize,
47    /// Automated Readability Index, absent under eight words.
48    pub grade: Option<f64>,
49    /// Flesch reading ease, absent under eight words.
50    pub ease: Option<f64>,
51    /// Adverb tokens.
52    pub adverbs: usize,
53    /// Adverbs over words.
54    pub adverb_ratio: f64,
55    /// Sentences carrying a be-verb near a past participle.
56    pub passives: usize,
57    /// Sentences at or past [`HARD_WORDS`].
58    pub hard_sentences: usize,
59    /// Sentences at or past [`VERY_HARD_WORDS`].
60    pub very_hard_sentences: usize,
61}
62
63/// Words ending in `ly` that are not adverbs.
64const FALSE_LY: &[&str] = &[
65    "only", "family", "apply", "early", "daily", "weekly", "monthly", "yearly", "supply", "reply",
66    "imply", "comply", "ally", "belly", "fly", "sly", "july",
67];
68
69const PLAIN_ADVERBS: &[&str] = &[
70    "very",
71    "really",
72    "quite",
73    "just",
74    "actually",
75    "basically",
76    "literally",
77    "seriously",
78    "extremely",
79    "incredibly",
80    "totally",
81    "definitely",
82    "probably",
83    "certainly",
84];
85
86const BE_VERBS: &[&str] = &["am", "is", "are", "was", "were", "be", "been", "being"];
87
88/// Word tokens: a letter, then letters, apostrophes or hyphens.
89fn words_of(text: &str) -> Vec<&str> {
90    let bytes = text.as_bytes();
91    let mut out = Vec::new();
92    let mut i = 0;
93    while i < bytes.len() {
94        if bytes[i].is_ascii_alphabetic() {
95            let start = i;
96            i += 1;
97            while i < bytes.len()
98                && (bytes[i].is_ascii_alphabetic() || bytes[i] == b'\'' || bytes[i] == b'-')
99            {
100                i += 1;
101            }
102            out.push(&text[start..i]);
103        } else {
104            i += 1;
105        }
106    }
107    out
108}
109
110/// Sentences, split at a `.`, `!` or `?` that ends a run of terminators and
111/// is followed by whitespace or the end of the text. A full stop inside a
112/// token (`0.9.3`, `127.0.0.1`, `Cargo.lock`) is not a boundary.
113fn sentences_of(text: &str) -> Vec<&str> {
114    let bytes = text.as_bytes();
115    let terminator = |b: u8| matches!(b, b'.' | b'!' | b'?');
116    let mut out = Vec::new();
117    let mut start = 0usize;
118    for (at, &b) in bytes.iter().enumerate() {
119        if !terminator(b) {
120            continue;
121        }
122        let ends = match bytes.get(at + 1) {
123            None => true,
124            Some(&next) => next.is_ascii_whitespace(),
125        };
126        if !ends {
127            continue;
128        }
129        out.push(&text[start..at]);
130        start = at + 1;
131    }
132    if start < text.len() {
133        out.push(&text[start..]);
134    }
135    out.into_iter()
136        .filter(|piece| !words_of(piece).is_empty())
137        .collect()
138}
139
140fn syllables(word: &str) -> usize {
141    let token: String = word
142        .chars()
143        .filter(|c| c.is_ascii_alphabetic())
144        .map(|c| c.to_ascii_lowercase())
145        .collect();
146    if token.is_empty() {
147        return 1;
148    }
149    let mut count = 0usize;
150    let mut prev_vowel = false;
151    for ch in token.chars() {
152        let is_vowel = matches!(ch, 'a' | 'e' | 'i' | 'o' | 'u' | 'y');
153        if is_vowel && !prev_vowel {
154            count += 1;
155        }
156        prev_vowel = is_vowel;
157    }
158    if token.ends_with('e') && count > 1 {
159        count -= 1;
160    }
161    count.max(1)
162}
163
164fn is_adverb(word: &str) -> bool {
165    let lower = word.to_ascii_lowercase();
166    if FALSE_LY.contains(&lower.as_str()) {
167        return false;
168    }
169    PLAIN_ADVERBS.contains(&lower.as_str()) || (lower.len() > 2 && lower.ends_with("ly"))
170}
171
172fn has_be_verb(sentence: &str) -> bool {
173    words_of(sentence)
174        .iter()
175        .any(|w| BE_VERBS.contains(&w.to_ascii_lowercase().as_str()))
176}
177
178fn has_participle(sentence: &str) -> bool {
179    words_of(sentence).iter().any(|w| {
180        let lower = w.to_ascii_lowercase();
181        lower.len() > 2 && (lower.ends_with("ed") || lower.ends_with("en"))
182    })
183}
184
185/// Grade, ease, and the Hemingway-style counts.
186#[must_use]
187pub fn assess(text: &str) -> Report {
188    let words = words_of(text);
189    let mut sentences: Vec<&str> = sentences_of(text);
190    if sentences.is_empty() && !words.is_empty() {
191        sentences = vec![text];
192    }
193    let n_words = words.len();
194    let n_sent = sentences.len().max(1);
195    let n_chars: usize = words.iter().map(|w| w.len()).sum();
196    let n_syl: usize = words.iter().map(|w| syllables(w)).sum();
197
198    let adverbs = words.iter().filter(|w| is_adverb(w)).count();
199    let mut passives = 0usize;
200    let mut hard = 0usize;
201    let mut very_hard = 0usize;
202    for sentence in &sentences {
203        let n = words_of(sentence).len();
204        if n >= VERY_HARD_WORDS {
205            very_hard += 1;
206        } else if n >= HARD_WORDS {
207            hard += 1;
208        }
209        if has_be_verb(sentence) && has_participle(sentence) {
210            passives += 1;
211        }
212    }
213
214    let (grade, ease) = if n_words >= 8 {
215        let w = n_words as f64;
216        let s = n_sent as f64;
217        let g = round2(4.71 * (n_chars as f64 / w) + 0.5 * (w / s) - 21.43);
218        let e = if n_syl > 0 {
219            Some(round2(
220                206.835 - 1.015 * (w / s) - 84.6 * (n_syl as f64 / w),
221            ))
222        } else {
223            None
224        };
225        (Some(g), e)
226    } else {
227        (None, None)
228    };
229
230    let adverb_ratio = if n_words > 0 {
231        adverbs as f64 / n_words as f64
232    } else {
233        0.0
234    };
235
236    Report {
237        words: n_words,
238        sentences: if words.is_empty() { 0 } else { sentences.len() },
239        grade,
240        ease,
241        adverbs,
242        adverb_ratio: round3(adverb_ratio),
243        passives,
244        hard_sentences: hard,
245        very_hard_sentences: very_hard,
246    }
247}
248
249fn round2(v: f64) -> f64 {
250    (v * 100.0).round() / 100.0
251}
252
253fn round3(v: f64) -> f64 {
254    (v * 1000.0).round() / 1000.0
255}
256
257/// Refuse text too hard for the working core.
258///
259/// # Errors
260///
261/// Returns [`ProseError`] when an atom carries more than one claim, when a
262/// sentence reads very hard, or when the grade or the adverb ratio is past its
263/// ceiling.
264pub fn refuse(text: &str, role: Role) -> Result<Report, ProseError> {
265    let report = assess(text);
266    if role == Role::Atom {
267        if report.sentences > MAX_ATOM_SENTENCES {
268            return Err(ProseError(format!(
269                "atom has {} sentences; one claim is at most {MAX_ATOM_SENTENCES}",
270                report.sentences
271            )));
272        }
273        if report.very_hard_sentences > 0 {
274            return Err(ProseError("atom sentence is very hard to read".into()));
275        }
276    }
277    if report.words >= 12 {
278        if let Some(grade) = report.grade {
279            if grade > MAX_GRADE {
280                return Err(ProseError(format!(
281                    "readability grade {grade} exceeds {MAX_GRADE}"
282                )));
283            }
284            if report.adverb_ratio > MAX_ADVERB_RATIO {
285                return Err(ProseError(format!(
286                    "adverb ratio {} exceeds {MAX_ADVERB_RATIO}",
287                    report.adverb_ratio
288                )));
289            }
290        }
291    }
292    Ok(report)
293}
294
295#[cfg(test)]
296mod tests {
297    use super::*;
298
299    /// A number is not the end of a sentence, and neither is a version.
300    #[test]
301    fn a_decimal_point_does_not_end_a_sentence() {
302        assert_eq!(
303            sentences_of("BM25+ beats BM25: 0.635 vs 0.615 hit@1 on turns. It is the default.")
304                .len(),
305            2
306        );
307        assert_eq!(
308            sentences_of("Cargo.lock pins the client. Bump it with cargo update.").len(),
309            2
310        );
311        assert_eq!(
312            sentences_of("packsetd listens on 127.0.0.1 only and never on localhost.").len(),
313            1
314        );
315        assert_eq!(sentences_of("Really?! Yes. No").len(), 3);
316        assert_eq!(
317            sentences_of("the tracker has 0.9.3 now, but keep at it.").len(),
318            1
319        );
320        assert_eq!(sentences_of("One. Two! Three?").len(), 3);
321        assert_eq!(sentences_of("no terminator at all").len(), 1);
322        assert_eq!(sentences_of("...").len(), 0);
323    }
324
325    #[test]
326    fn a_plain_claim_passes_as_an_atom() {
327        let text = "Reviews open with a reproducibility check.";
328        assert!(refuse(text, Role::Atom).is_ok());
329    }
330
331    #[test]
332    fn one_claim_means_at_most_two_sentences() {
333        let three = "One thing. Another thing. A third thing.";
334        let err = refuse(three, Role::Atom).unwrap_err();
335        assert!(err.0.contains("3 sentences"), "{err}");
336        // The same text is fine in a card, which holds more than one claim.
337        assert!(refuse(three, Role::File).is_ok());
338    }
339
340    #[test]
341    fn a_short_text_is_not_graded() {
342        let report = assess("Too short to grade.");
343        assert_eq!(report.grade, None);
344        assert_eq!(report.ease, None);
345    }
346
347    #[test]
348    fn ly_words_that_are_not_adverbs_do_not_count() {
349        assert_eq!(assess("only family apply early daily").adverbs, 0);
350        assert_eq!(assess("quickly").adverbs, 1);
351        assert_eq!(assess("very").adverbs, 1);
352    }
353
354    #[test]
355    fn a_passive_is_a_be_verb_near_a_participle() {
356        assert_eq!(assess("The header was parsed by the reader.").passives, 1);
357        assert_eq!(assess("The reader parses the header.").passives, 0);
358    }
359
360    #[test]
361    fn sentence_length_bands_are_counted() {
362        let hard = "word ".repeat(HARD_WORDS) + ".";
363        assert_eq!(assess(&hard).hard_sentences, 1);
364        assert_eq!(assess(&hard).very_hard_sentences, 0);
365        let very = "word ".repeat(VERY_HARD_WORDS) + ".";
366        assert_eq!(assess(&very).very_hard_sentences, 1);
367        assert_eq!(assess(&very).hard_sentences, 0);
368    }
369
370    #[test]
371    fn a_very_hard_sentence_is_refused_in_an_atom_only() {
372        let very = "word ".repeat(VERY_HARD_WORDS) + ".";
373        assert!(refuse(&very, Role::Atom).is_err());
374        // In a card the length alone does not refuse; the grade decides.
375        let report = assess(&very);
376        assert_eq!(report.very_hard_sentences, 1);
377    }
378
379    #[test]
380    fn empty_text_has_no_sentences() {
381        let report = assess("");
382        assert_eq!(report.words, 0);
383        assert_eq!(report.sentences, 0);
384        assert_eq!(report.adverb_ratio, 0.0);
385    }
386
387    #[test]
388    fn syllable_counting_drops_a_silent_e() {
389        assert_eq!(syllables("make"), 1);
390        assert_eq!(syllables("the"), 1);
391        assert_eq!(syllables("reproducibility"), 7);
392        assert_eq!(syllables(""), 1);
393    }
394}
packset_core/prose.rs

packset_core/
prose.rs
