1use std::collections::HashSet;
6use std::sync::Arc;
7
8use harper_core::language_detection::is_doc_likely_english;
9use harper_core::linting::{LintGroup, LintKind, Linter, Suggestion};
10use harper_core::spell::FstDictionary;
11use harper_core::{Dialect, Document};
12
13#[derive(Clone, Debug, PartialEq, Eq)]
15pub struct Issue {
16 pub start: usize,
17 pub end: usize,
18 pub message: String,
19 pub fixes: Vec<String>,
21 pub spelling: bool,
22}
23
24#[derive(Clone, Copy, Debug, PartialEq, Eq)]
26pub enum Source {
27 Markdown,
28 Latex,
29}
30
31fn blank(chars: &mut [char], from: usize, to: usize) {
32 let to = to.min(chars.len());
33 for c in &mut chars[from..to] {
34 if *c != '\n' {
35 *c = ' ';
36 }
37 }
38}
39
40fn find_from(chars: &[char], from: usize, wanted: &[char]) -> Option<usize> {
41 (from..chars.len().saturating_sub(wanted.len() - 1)).find(|at| chars[*at..].starts_with(wanted))
42}
43
44fn markdown(text: &str) -> Vec<char> {
45 let mut chars: Vec<char> = text.chars().collect();
46 let mut at = 0;
47 let mut line_start = true;
48 let mut fenced = false;
49 while at < chars.len() {
50 let c = chars[at];
51 if line_start {
52 let end = chars[at..]
53 .iter()
54 .position(|c| *c == '\n')
55 .map_or(chars.len(), |n| at + n);
56 let line: String = chars[at..end].iter().collect();
57 let trimmed = line.trim_start();
58 let fence = trimmed.starts_with("```");
59 fenced ^= fence;
60 let table_rule = line.contains('-') && line.chars().all(|c| "|-: ".contains(c));
61 if fence || fenced || table_rule {
62 blank(&mut chars, at, end);
63 at = end;
64 line_start = false;
65 continue;
66 }
67 let marker = trimmed
69 .chars()
70 .take_while(|c| "#>-*+ ".contains(*c) || c.is_ascii_digit() || *c == '.')
71 .count();
72 let indent = line.chars().count() - trimmed.chars().count();
73 blank(&mut chars, at, at + indent + marker);
74 at += indent + marker;
75 line_start = false;
76 continue;
77 }
78 match c {
79 '\n' => {
80 line_start = true;
81 at += 1;
82 }
83 '`' | '$' => {
84 let double = chars.get(at + 1) == Some(&c);
85 let wanted = if double { vec![c, c] } else { vec![c] };
86 match find_from(&chars, at + wanted.len(), &wanted) {
87 Some(close) => {
88 blank(&mut chars, at, close + wanted.len());
89 at = close + wanted.len();
90 }
91 None => at += 1,
92 }
93 }
94 ']' if chars.get(at + 1) == Some(&'(') => {
96 let close = find_from(&chars, at, &[')']).unwrap_or(at);
97 blank(&mut chars, at, close + 1);
98 at = close + 1;
99 }
100 '*' | '_' | '|' | '[' | ']' | '{' | '}' | '~' | '=' | '<' | '>' | '^' => {
101 chars[at] = ' ';
102 at += 1;
103 }
104 'h' if chars[at..].starts_with(&['h', 't', 't', 'p']) => {
105 let end = chars[at..]
106 .iter()
107 .position(|c| c.is_whitespace())
108 .map_or(chars.len(), |n| at + n);
109 blank(&mut chars, at, end);
110 at = end;
111 }
112 _ => at += 1,
113 }
114 }
115 chars
116}
117
118const WORDS_INSIDE: [&str; 12] = [
120 "section",
121 "subsection",
122 "subsubsection",
123 "chapter",
124 "paragraph",
125 "caption",
126 "title",
127 "textbf",
128 "textit",
129 "emph",
130 "underline",
131 "footnote",
132];
133const NOT_PROSE: [&str; 13] = [
135 "equation",
136 "align",
137 "gather",
138 "multline",
139 "displaymath",
140 "verbatim",
141 "lstlisting",
142 "minted",
143 "tikzpicture",
144 "tikzcd",
145 "tabular",
146 "thebibliography",
147 "algorithmic",
148];
149
150fn latex(text: &str) -> Vec<char> {
151 let mut chars: Vec<char> = text.chars().collect();
152 let begin: Vec<char> = "\\begin{document}".chars().collect();
153 let mut at = find_from(&chars, 0, &begin).map_or(0, |found| {
154 blank(&mut chars, 0, found + begin.len());
155 found + begin.len()
156 });
157 while at < chars.len() {
158 match chars[at] {
159 '%' if at == 0 || chars[at - 1] != '\\' => {
160 let end = chars[at..]
161 .iter()
162 .position(|c| *c == '\n')
163 .map_or(chars.len(), |n| at + n);
164 blank(&mut chars, at, end);
165 at = end;
166 }
167 '$' => match find_from(&chars, at + 1, &['$']) {
168 Some(close) => {
169 blank(&mut chars, at, close + 1);
170 at = close + 1;
171 }
172 None => at += 1,
173 },
174 '\\' => {
175 let name: String = chars[at + 1..]
176 .iter()
177 .take_while(|c| c.is_ascii_alphabetic() || **c == '*')
178 .collect();
179 let mut end = at + 1 + name.chars().count().max(1);
180 if name == "begin" || name == "end" {
181 let close = find_from(&chars, end, &['}']).unwrap_or(end);
182 let env: String = chars[end..close]
183 .iter()
184 .filter(|c| !"{*".contains(**c))
185 .collect();
186 if name == "begin" && NOT_PROSE.contains(&env.as_str()) {
187 let wanted: Vec<char> = format!("\\end{{{env}").chars().collect();
188 let stop = find_from(&chars, close, &wanted).unwrap_or(close);
189 blank(&mut chars, at, stop);
190 at = stop;
191 continue;
192 }
193 blank(&mut chars, at, close + 1);
194 at = close + 1;
195 continue;
196 }
197 if name.is_empty() && chars.get(at + 1) == Some(&'[') {
198 let close = find_from(&chars, at, &['\\', ']']).unwrap_or(at);
199 blank(&mut chars, at, close + 2);
200 at = close + 2;
201 continue;
202 }
203 if chars.get(end) == Some(&'[')
204 && let Some(close) = find_from(&chars, end, &[']'])
205 {
206 end = close + 1;
207 }
208 let keeps = WORDS_INSIDE.contains(&name.trim_end_matches('*'));
209 let has_argument = chars.get(end) == Some(&'{');
210 if keeps && has_argument {
211 blank(&mut chars, at, end + 1);
212 at = end + 1;
213 } else if has_argument {
214 let close = find_from(&chars, end, &['}']).unwrap_or(end);
215 blank(&mut chars, at, close + 1);
216 at = close + 1;
217 } else {
218 blank(&mut chars, at, end);
219 at = end;
220 }
221 }
222 '{' | '}' | '~' | '&' => {
223 chars[at] = ' ';
224 at += 1;
225 }
226 _ => at += 1,
227 }
228 }
229 chars
230}
231
232pub fn prose(text: &str, source: Source) -> String {
234 let chars = match source {
235 Source::Markdown => markdown(text),
236 Source::Latex => latex(text),
237 };
238 chars.into_iter().collect()
239}
240
241fn worth_showing(kind: LintKind) -> bool {
244 matches!(
245 kind,
246 LintKind::Spelling
247 | LintKind::Typo
248 | LintKind::Grammar
249 | LintKind::Agreement
250 | LintKind::Repetition
251 | LintKind::Malapropism
252 | LintKind::Eggcorn
253 | LintKind::WordOrder
254 | LintKind::Usage
255 )
256}
257
258fn technical(word: &str) -> bool {
261 const KEYS: [&str; 4] = ["Ctrl", "Cmd", "Esc", "Alt"];
262 let mut letters = word.chars();
263 let first = letters.next();
264 let inner_capital = letters.any(char::is_uppercase);
265 word.chars().count() <= 2
266 || inner_capital
267 || first.is_some_and(|c| c.is_ascii_digit())
268 || word.chars().any(|c| c.is_ascii_digit())
269 || KEYS.contains(&word)
270}
271
272fn regional(word: &str, fix: &Suggestion) -> bool {
275 let Suggestion::ReplaceWith(fix) = fix else {
276 return false;
277 };
278 let fix: String = fix.iter().collect::<String>().to_lowercase();
279 let word = word.to_lowercase();
280 let plain = |text: &str| {
281 text.replace("ise", "ize")
282 .replace("isa", "iza")
283 .replace("yse", "yze")
284 .replace("our", "or")
285 .replace("tre", "ter")
286 .replace("ll", "l")
287 };
288 plain(&word) == plain(&fix)
289}
290
291pub struct Checker {
293 dictionary: Arc<FstDictionary>,
294 group: LintGroup,
295 pub known: HashSet<String>,
297}
298
299impl Checker {
300 pub fn new(known: HashSet<String>) -> Self {
301 let dictionary = FstDictionary::curated();
302 let group = LintGroup::new_curated(dictionary.clone(), Dialect::American);
303 Self {
304 dictionary,
305 group,
306 known,
307 }
308 }
309
310 pub fn check(&mut self, text: &str, source: Source) -> Vec<Issue> {
312 let cleaned = prose(text, source);
313 let chars: Vec<char> = cleaned.chars().collect();
314 let document = Document::new_plain_english_curated(&cleaned);
315 if !is_doc_likely_english(&document, self.dictionary.as_ref()) {
316 return Vec::new();
317 }
318 let mut issues = Vec::new();
319 for lint in self.group.lint(&document) {
320 if !worth_showing(lint.lint_kind) {
321 continue;
322 }
323 let (start, end) = (lint.span.start, lint.span.end.min(chars.len()));
324 let word: String = chars[start..end].iter().collect();
325 let spelling = lint.lint_kind == LintKind::Spelling;
326 if spelling && (self.known.contains(&word.to_lowercase()) || technical(&word)) {
327 continue;
328 }
329 if spelling && lint.suggestions.iter().any(|fix| regional(&word, fix)) {
330 continue;
331 }
332 let fixes = lint
333 .suggestions
334 .iter()
335 .filter_map(|fix| match fix {
336 Suggestion::ReplaceWith(text) => Some(text.iter().collect()),
337 Suggestion::Remove => Some(String::new()),
338 Suggestion::InsertAfter(_) => None,
339 })
340 .take(5)
341 .collect();
342 issues.push(Issue {
343 start,
344 end,
345 message: lint.message,
346 fixes,
347 spelling,
348 });
349 }
350 issues
351 }
352}
353
354#[cfg(test)]
355mod tests {
356 use super::*;
357
358 #[test]
359 fn prose_stays_where_it_was_and_the_rest_goes() {
360 let text = "# Title\n\nSome `code` and $x^2$ and [a link](http://x.y) here.\n\n```\nfn broken\n```\n";
361 let cleaned = prose(text, Source::Markdown);
362 assert_eq!(cleaned.chars().count(), text.chars().count());
363 assert!(cleaned.contains("Title"));
364 assert!(cleaned.contains("a link"));
365 for gone in ["code", "x^2", "http", "broken", "#"] {
366 assert!(!cleaned.contains(gone), "{gone}: {cleaned}");
367 }
368 }
369
370 #[test]
371 fn a_papers_commands_go_and_its_words_stay() {
372 let text = "\\documentclass{article}\n\\usepackage{tikz}\n\\begin{document}\n\\section{Intro words}\nText \\cite{knuth} with \\emph{stress} and $x$.\n\\begin{equation}\nE = mc^2\n\\end{equation}\n% a comment\n\\end{document}\n";
373 let cleaned = prose(text, Source::Latex);
374 assert_eq!(cleaned.chars().count(), text.chars().count());
375 assert!(cleaned.contains("Intro words") && cleaned.contains("stress"));
376 for gone in ["documentclass", "knuth", "mc^2", "comment", "tikz"] {
377 assert!(!cleaned.contains(gone), "{gone}: {cleaned}");
378 }
379 }
380
381 #[test]
382 fn a_misspelling_is_found_with_its_fix() {
383 let mut checker = Checker::new(HashSet::new());
384 let text = "This sentence has a mispelled word in the middle of it.\n";
385 let issues = checker.check(text, Source::Markdown);
386 let issue = issues.iter().find(|i| i.spelling).expect("found");
387 let word: String = text
388 .chars()
389 .skip(issue.start)
390 .take(issue.end - issue.start)
391 .collect();
392 assert_eq!(word, "mispelled");
393 assert!(
394 issue.fixes.iter().any(|f| f == "misspelled"),
395 "{:?}",
396 issue.fixes
397 );
398 }
399
400 #[test]
401 fn a_word_the_writer_knows_is_left_alone() {
402 let known = HashSet::from(["mispelled".to_string()]);
403 let mut checker = Checker::new(known);
404 let text = "This sentence has a mispelled word in the middle of it.\n";
405 assert!(
406 checker
407 .check(text, Source::Markdown)
408 .iter()
409 .all(|i| !i.spelling)
410 );
411 }
412
413 #[test]
414 fn names_and_keys_are_not_misspellings() {
415 let mut checker = Checker::new(HashSet::new());
416 let text =
417 "We trained on ImageNet and CIFAR-10, drew it in TikZ, and pressed Ctrl to go.\n";
418 let found = checker.check(text, Source::Markdown);
419 assert!(found.iter().all(|i| !i.spelling), "{found:?}");
420 }
421
422 #[test]
423 fn british_spelling_is_not_a_mistake() {
424 let mut checker = Checker::new(HashSet::new());
425 let text = "This minimises the loss, and the colour of the emphasised centre stays.\n";
426 let found = checker.check(text, Source::Markdown);
427 assert!(found.iter().all(|i| !i.spelling), "{found:?}");
428 }
429
430 #[test]
431 fn what_is_not_prose_is_not_checked() {
432 let mut checker = Checker::new(HashSet::new());
433 let text = "A plain sentence with `qwrtyzz` in code and nothing wrong around it.\n";
434 assert!(checker.check(text, Source::Markdown).is_empty());
435 }
436}