Skip to main content

cellix_doc/
prose.rs

1//! Spelling and grammar for writing. Harper checks plain English, so what isn't prose is
2//! blanked out first, keeping every other char where it was, and what it finds still lands on
3//! the right words in the source.
4
5use std::collections::HashSet;
6use std::sync::Arc;
7
8use harper_core::language_detection::is_doc_likely_english;
9use harper_core::linting::{LintGroup, LintKind, Linter, Suggestion};
10use harper_core::spell::FstDictionary;
11use harper_core::{Dialect, Document};
12
13/// One thing to look at, as a range of chars in the source.
14#[derive(Clone, Debug, PartialEq, Eq)]
15pub struct Issue {
16    pub start: usize,
17    pub end: usize,
18    pub message: String,
19    /// What it could be instead.
20    pub fixes: Vec<String>,
21    pub spelling: bool,
22}
23
24/// What a document is written in, which says what isn't prose.
25#[derive(Clone, Copy, Debug, PartialEq, Eq)]
26pub enum Source {
27    Markdown,
28    Latex,
29}
30
31fn blank(chars: &mut [char], from: usize, to: usize) {
32    let to = to.min(chars.len());
33    for c in &mut chars[from..to] {
34        if *c != '\n' {
35            *c = ' ';
36        }
37    }
38}
39
40fn find_from(chars: &[char], from: usize, wanted: &[char]) -> Option<usize> {
41    (from..chars.len().saturating_sub(wanted.len() - 1)).find(|at| chars[*at..].starts_with(wanted))
42}
43
44fn markdown(text: &str) -> Vec<char> {
45    let mut chars: Vec<char> = text.chars().collect();
46    let mut at = 0;
47    let mut line_start = true;
48    let mut fenced = false;
49    while at < chars.len() {
50        let c = chars[at];
51        if line_start {
52            let end = chars[at..]
53                .iter()
54                .position(|c| *c == '\n')
55                .map_or(chars.len(), |n| at + n);
56            let line: String = chars[at..end].iter().collect();
57            let trimmed = line.trim_start();
58            let fence = trimmed.starts_with("```");
59            fenced ^= fence;
60            let table_rule = line.contains('-') && line.chars().all(|c| "|-: ".contains(c));
61            if fence || fenced || table_rule {
62                blank(&mut chars, at, end);
63                at = end;
64                line_start = false;
65                continue;
66            }
67            // A line's marker isn't a word.
68            let marker = trimmed
69                .chars()
70                .take_while(|c| "#>-*+ ".contains(*c) || c.is_ascii_digit() || *c == '.')
71                .count();
72            let indent = line.chars().count() - trimmed.chars().count();
73            blank(&mut chars, at, at + indent + marker);
74            at += indent + marker;
75            line_start = false;
76            continue;
77        }
78        match c {
79            '\n' => {
80                line_start = true;
81                at += 1;
82            }
83            '`' | '$' => {
84                let double = chars.get(at + 1) == Some(&c);
85                let wanted = if double { vec![c, c] } else { vec![c] };
86                match find_from(&chars, at + wanted.len(), &wanted) {
87                    Some(close) => {
88                        blank(&mut chars, at, close + wanted.len());
89                        at = close + wanted.len();
90                    }
91                    None => at += 1,
92                }
93            }
94            // A link keeps its words, and loses its address. An image loses both.
95            ']' if chars.get(at + 1) == Some(&'(') => {
96                let close = find_from(&chars, at, &[')']).unwrap_or(at);
97                blank(&mut chars, at, close + 1);
98                at = close + 1;
99            }
100            '*' | '_' | '|' | '[' | ']' | '{' | '}' | '~' | '=' | '<' | '>' | '^' => {
101                chars[at] = ' ';
102                at += 1;
103            }
104            'h' if chars[at..].starts_with(&['h', 't', 't', 'p']) => {
105                let end = chars[at..]
106                    .iter()
107                    .position(|c| c.is_whitespace())
108                    .map_or(chars.len(), |n| at + n);
109                blank(&mut chars, at, end);
110                at = end;
111            }
112            _ => at += 1,
113        }
114    }
115    chars
116}
117
118/// Commands whose argument is words, not a name.
119const WORDS_INSIDE: [&str; 12] = [
120    "section",
121    "subsection",
122    "subsubsection",
123    "chapter",
124    "paragraph",
125    "caption",
126    "title",
127    "textbf",
128    "textit",
129    "emph",
130    "underline",
131    "footnote",
132];
133/// Environments with no prose in them.
134const NOT_PROSE: [&str; 13] = [
135    "equation",
136    "align",
137    "gather",
138    "multline",
139    "displaymath",
140    "verbatim",
141    "lstlisting",
142    "minted",
143    "tikzpicture",
144    "tikzcd",
145    "tabular",
146    "thebibliography",
147    "algorithmic",
148];
149
150fn latex(text: &str) -> Vec<char> {
151    let mut chars: Vec<char> = text.chars().collect();
152    let begin: Vec<char> = "\\begin{document}".chars().collect();
153    let mut at = find_from(&chars, 0, &begin).map_or(0, |found| {
154        blank(&mut chars, 0, found + begin.len());
155        found + begin.len()
156    });
157    while at < chars.len() {
158        match chars[at] {
159            '%' if at == 0 || chars[at - 1] != '\\' => {
160                let end = chars[at..]
161                    .iter()
162                    .position(|c| *c == '\n')
163                    .map_or(chars.len(), |n| at + n);
164                blank(&mut chars, at, end);
165                at = end;
166            }
167            '$' => match find_from(&chars, at + 1, &['$']) {
168                Some(close) => {
169                    blank(&mut chars, at, close + 1);
170                    at = close + 1;
171                }
172                None => at += 1,
173            },
174            '\\' => {
175                let name: String = chars[at + 1..]
176                    .iter()
177                    .take_while(|c| c.is_ascii_alphabetic() || **c == '*')
178                    .collect();
179                let mut end = at + 1 + name.chars().count().max(1);
180                if name == "begin" || name == "end" {
181                    let close = find_from(&chars, end, &['}']).unwrap_or(end);
182                    let env: String = chars[end..close]
183                        .iter()
184                        .filter(|c| !"{*".contains(**c))
185                        .collect();
186                    if name == "begin" && NOT_PROSE.contains(&env.as_str()) {
187                        let wanted: Vec<char> = format!("\\end{{{env}").chars().collect();
188                        let stop = find_from(&chars, close, &wanted).unwrap_or(close);
189                        blank(&mut chars, at, stop);
190                        at = stop;
191                        continue;
192                    }
193                    blank(&mut chars, at, close + 1);
194                    at = close + 1;
195                    continue;
196                }
197                if name.is_empty() && chars.get(at + 1) == Some(&'[') {
198                    let close = find_from(&chars, at, &['\\', ']']).unwrap_or(at);
199                    blank(&mut chars, at, close + 2);
200                    at = close + 2;
201                    continue;
202                }
203                if chars.get(end) == Some(&'[')
204                    && let Some(close) = find_from(&chars, end, &[']'])
205                {
206                    end = close + 1;
207                }
208                let keeps = WORDS_INSIDE.contains(&name.trim_end_matches('*'));
209                let has_argument = chars.get(end) == Some(&'{');
210                if keeps && has_argument {
211                    blank(&mut chars, at, end + 1);
212                    at = end + 1;
213                } else if has_argument {
214                    let close = find_from(&chars, end, &['}']).unwrap_or(end);
215                    blank(&mut chars, at, close + 1);
216                    at = close + 1;
217                } else {
218                    blank(&mut chars, at, end);
219                    at = end;
220                }
221            }
222            '{' | '}' | '~' | '&' => {
223                chars[at] = ' ';
224                at += 1;
225            }
226            _ => at += 1,
227        }
228    }
229    chars
230}
231
232/// The prose of a document, the same length as its source.
233pub fn prose(text: &str, source: Source) -> String {
234    let chars = match source {
235        Source::Markdown => markdown(text),
236        Source::Latex => latex(text),
237    };
238    chars.into_iter().collect()
239}
240
241/// Kinds of finding worth a writer's attention. Spacing and capitals are left out, since
242/// the blanks above would set them off.
243fn worth_showing(kind: LintKind) -> bool {
244    matches!(
245        kind,
246        LintKind::Spelling
247            | LintKind::Typo
248            | LintKind::Grammar
249            | LintKind::Agreement
250            | LintKind::Repetition
251            | LintKind::Malapropism
252            | LintKind::Eggcorn
253            | LintKind::WordOrder
254            | LintKind::Usage
255    )
256}
257
258/// Words that aren't English to begin with: names like ImageNet and CIFAR, anything with a
259/// digit in it, a letter or two left by a formula, and what keyboards call their keys.
260fn technical(word: &str) -> bool {
261    const KEYS: [&str; 4] = ["Ctrl", "Cmd", "Esc", "Alt"];
262    let mut letters = word.chars();
263    let first = letters.next();
264    let inner_capital = letters.any(char::is_uppercase);
265    word.chars().count() <= 2
266        || inner_capital
267        || first.is_some_and(|c| c.is_ascii_digit())
268        || word.chars().any(|c| c.is_ascii_digit())
269        || KEYS.contains(&word)
270}
271
272/// Whether a suggestion is only the other side of the Atlantic: minimises for minimizes,
273/// colour for color. Either is right, and the writer knows which they mean.
274fn regional(word: &str, fix: &Suggestion) -> bool {
275    let Suggestion::ReplaceWith(fix) = fix else {
276        return false;
277    };
278    let fix: String = fix.iter().collect::<String>().to_lowercase();
279    let word = word.to_lowercase();
280    let plain = |text: &str| {
281        text.replace("ise", "ize")
282            .replace("isa", "iza")
283            .replace("yse", "yze")
284            .replace("our", "or")
285            .replace("tre", "ter")
286            .replace("ll", "l")
287    };
288    plain(&word) == plain(&fix)
289}
290
291/// Does the checking. Starting it takes a moment, so it lives on its own thread.
292pub struct Checker {
293    dictionary: Arc<FstDictionary>,
294    group: LintGroup,
295    /// Words the writer has said are right, in lower case.
296    pub known: HashSet<String>,
297}
298
299impl Checker {
300    pub fn new(known: HashSet<String>) -> Self {
301        let dictionary = FstDictionary::curated();
302        let group = LintGroup::new_curated(dictionary.clone(), Dialect::American);
303        Self {
304            dictionary,
305            group,
306            known,
307        }
308    }
309
310    /// What's wrong with a document, or nothing if it doesn't read as English.
311    pub fn check(&mut self, text: &str, source: Source) -> Vec<Issue> {
312        let cleaned = prose(text, source);
313        let chars: Vec<char> = cleaned.chars().collect();
314        let document = Document::new_plain_english_curated(&cleaned);
315        if !is_doc_likely_english(&document, self.dictionary.as_ref()) {
316            return Vec::new();
317        }
318        let mut issues = Vec::new();
319        for lint in self.group.lint(&document) {
320            if !worth_showing(lint.lint_kind) {
321                continue;
322            }
323            let (start, end) = (lint.span.start, lint.span.end.min(chars.len()));
324            let word: String = chars[start..end].iter().collect();
325            let spelling = lint.lint_kind == LintKind::Spelling;
326            if spelling && (self.known.contains(&word.to_lowercase()) || technical(&word)) {
327                continue;
328            }
329            if spelling && lint.suggestions.iter().any(|fix| regional(&word, fix)) {
330                continue;
331            }
332            let fixes = lint
333                .suggestions
334                .iter()
335                .filter_map(|fix| match fix {
336                    Suggestion::ReplaceWith(text) => Some(text.iter().collect()),
337                    Suggestion::Remove => Some(String::new()),
338                    Suggestion::InsertAfter(_) => None,
339                })
340                .take(5)
341                .collect();
342            issues.push(Issue {
343                start,
344                end,
345                message: lint.message,
346                fixes,
347                spelling,
348            });
349        }
350        issues
351    }
352}
353
354#[cfg(test)]
355mod tests {
356    use super::*;
357
358    #[test]
359    fn prose_stays_where_it_was_and_the_rest_goes() {
360        let text = "# Title\n\nSome `code` and $x^2$ and [a link](http://x.y) here.\n\n```\nfn broken\n```\n";
361        let cleaned = prose(text, Source::Markdown);
362        assert_eq!(cleaned.chars().count(), text.chars().count());
363        assert!(cleaned.contains("Title"));
364        assert!(cleaned.contains("a link"));
365        for gone in ["code", "x^2", "http", "broken", "#"] {
366            assert!(!cleaned.contains(gone), "{gone}: {cleaned}");
367        }
368    }
369
370    #[test]
371    fn a_papers_commands_go_and_its_words_stay() {
372        let text = "\\documentclass{article}\n\\usepackage{tikz}\n\\begin{document}\n\\section{Intro words}\nText \\cite{knuth} with \\emph{stress} and $x$.\n\\begin{equation}\nE = mc^2\n\\end{equation}\n% a comment\n\\end{document}\n";
373        let cleaned = prose(text, Source::Latex);
374        assert_eq!(cleaned.chars().count(), text.chars().count());
375        assert!(cleaned.contains("Intro words") && cleaned.contains("stress"));
376        for gone in ["documentclass", "knuth", "mc^2", "comment", "tikz"] {
377            assert!(!cleaned.contains(gone), "{gone}: {cleaned}");
378        }
379    }
380
381    #[test]
382    fn a_misspelling_is_found_with_its_fix() {
383        let mut checker = Checker::new(HashSet::new());
384        let text = "This sentence has a mispelled word in the middle of it.\n";
385        let issues = checker.check(text, Source::Markdown);
386        let issue = issues.iter().find(|i| i.spelling).expect("found");
387        let word: String = text
388            .chars()
389            .skip(issue.start)
390            .take(issue.end - issue.start)
391            .collect();
392        assert_eq!(word, "mispelled");
393        assert!(
394            issue.fixes.iter().any(|f| f == "misspelled"),
395            "{:?}",
396            issue.fixes
397        );
398    }
399
400    #[test]
401    fn a_word_the_writer_knows_is_left_alone() {
402        let known = HashSet::from(["mispelled".to_string()]);
403        let mut checker = Checker::new(known);
404        let text = "This sentence has a mispelled word in the middle of it.\n";
405        assert!(
406            checker
407                .check(text, Source::Markdown)
408                .iter()
409                .all(|i| !i.spelling)
410        );
411    }
412
413    #[test]
414    fn names_and_keys_are_not_misspellings() {
415        let mut checker = Checker::new(HashSet::new());
416        let text =
417            "We trained on ImageNet and CIFAR-10, drew it in TikZ, and pressed Ctrl to go.\n";
418        let found = checker.check(text, Source::Markdown);
419        assert!(found.iter().all(|i| !i.spelling), "{found:?}");
420    }
421
422    #[test]
423    fn british_spelling_is_not_a_mistake() {
424        let mut checker = Checker::new(HashSet::new());
425        let text = "This minimises the loss, and the colour of the emphasised centre stays.\n";
426        let found = checker.check(text, Source::Markdown);
427        assert!(found.iter().all(|i| !i.spelling), "{found:?}");
428    }
429
430    #[test]
431    fn what_is_not_prose_is_not_checked() {
432        let mut checker = Checker::new(HashSet::new());
433        let text = "A plain sentence with `qwrtyzz` in code and nothing wrong around it.\n";
434        assert!(checker.check(text, Source::Markdown).is_empty());
435    }
436}