use std::path::{Path, PathBuf};
use anyhow::Result;
use crate::dict::PhraseBigrams;
use crate::engine::Engine;
use crate::format::{self, Options};
use crate::token::{Token, tokenize_with_lowercase};
#[derive(Debug, Clone)]
pub struct Compound {
pub text: String,
pub byte_offset: usize,
}
#[derive(Debug, Clone)]
pub struct Misspelling {
pub path: PathBuf,
pub line: usize,
pub col: usize,
pub byte_offset: usize,
pub word: String,
pub suggestions: Option<Vec<String>>,
pub compound: Option<Compound>,
}
#[derive(Debug, Clone)]
pub struct Warning {
pub path: PathBuf,
pub line: usize,
pub kind: crate::directive::Kind,
pub message: String,
}
impl std::fmt::Display for Warning {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
write!(f, "{}:{}: {}", self.path.display(), self.line, self.message)
}
}
#[derive(Debug, Default)]
pub struct Checked {
pub misspellings: Vec<Misspelling>,
pub warnings: Vec<Warning>,
}
pub fn check_file(path: &Path, opts: Options, engine: &Engine) -> Result<Checked> {
let src = std::fs::read_to_string(path)?;
let line_starts = LineStarts::new(&src);
let analysis = format::analyze(&src, path, opts);
let allow = &analysis.allow;
let accepted = |w: &str| allow.contains(w) || engine.check(w);
let warnings = analysis
.warnings
.iter()
.map(|w| Warning {
path: path.to_path_buf(),
line: line_starts.locate(w.byte_offset).0 + 1,
kind: w.kind.clone(),
message: w.message.clone(),
})
.collect();
let tokenized = tokenize_with_lowercase(&src, &analysis.skips);
let tokens = &tokenized.tokens;
let lowercase = &tokenized.lowercase;
let gap_clean = &tokenized.gap_clean;
let phrases = engine.phrase_bigrams();
let phrases_cs = engine.phrase_bigrams_cs();
let mut out = Vec::new();
for (i, tok) in tokens.iter().enumerate() {
if phrase_covered(i, tokens, lowercase, gap_clean, phrases, phrases_cs) {
continue;
}
if tok.word.contains('-') {
if accepted(&tok.word) {
continue;
}
let compound = Compound {
text: tok.word.clone(),
byte_offset: tok.byte_range.start,
};
for (part, range) in split_hyphen_parts(&tok.word, tok.byte_range.start) {
if part.is_empty() || !part.chars().any(|c| c.is_alphabetic()) {
continue;
}
if accepted(part) {
continue;
}
out.push(build_misspelling(
path,
&line_starts,
part,
&range,
Some(&compound),
));
}
} else {
if accepted(&tok.word) {
continue;
}
out.push(build_misspelling(
path,
&line_starts,
&tok.word,
&tok.byte_range,
None,
));
}
}
Ok(Checked {
misspellings: out,
warnings,
})
}
pub(crate) fn phrase_covered(
i: usize,
tokens: &[Token],
lowercase: &[String],
gap_clean: &[bool],
phrases: &PhraseBigrams,
phrases_cs: &PhraseBigrams,
) -> bool {
let w = &lowercase[i];
if i > 0
&& gap_clean[i]
&& (phrases.contains(&lowercase[i - 1], w)
|| phrases_cs.contains(&tokens[i - 1].word, &tokens[i].word))
{
return true;
}
if i + 1 < tokens.len()
&& gap_clean[i + 1]
&& (phrases.contains(w, &lowercase[i + 1])
|| phrases_cs.contains(&tokens[i].word, &tokens[i + 1].word))
{
return true;
}
false
}
fn build_misspelling(
path: &Path,
line_starts: &LineStarts,
word: &str,
byte_range: &std::ops::Range<usize>,
compound: Option<&Compound>,
) -> Misspelling {
let (line0, col0) = line_starts.locate(byte_range.start);
Misspelling {
path: path.to_path_buf(),
line: line0 + 1,
col: col0 + 1,
byte_offset: byte_range.start,
word: word.to_string(),
suggestions: None,
compound: compound.cloned(),
}
}
fn split_hyphen_parts(word: &str, base: usize) -> Vec<(&str, std::ops::Range<usize>)> {
let mut out = Vec::new();
let bytes = word.as_bytes();
let mut start = 0usize;
let mut i = 0usize;
while i <= bytes.len() {
if i == bytes.len() || bytes[i] == b'-' {
if i > start {
out.push((&word[start..i], base + start..base + i));
}
start = i + 1;
}
i += 1;
}
out
}
pub struct LineStarts {
starts: Vec<usize>,
}
impl LineStarts {
pub fn new(src: &str) -> Self {
let mut starts = vec![0usize];
for (i, b) in src.bytes().enumerate() {
if b == b'\n' {
starts.push(i + 1);
}
}
Self { starts }
}
pub fn locate(&self, byte_offset: usize) -> (usize, usize) {
let line = self
.starts
.partition_point(|&s| s <= byte_offset)
.saturating_sub(1);
let col = byte_offset - self.starts[line];
(line, col)
}
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn split_parts_basic() {
let parts = split_hyphen_parts("Tzeya-Gan", 100);
assert_eq!(parts, vec![("Tzeya", 100..105), ("Gan", 106..109)]);
}
#[test]
fn split_parts_double_hyphen_drops_empty() {
let parts = split_hyphen_parts("year--another", 0);
assert_eq!(parts.len(), 2);
assert_eq!(parts[0], ("year", 0..4));
assert_eq!(parts[1], ("another", 6..13));
}
#[test]
fn split_parts_multi() {
let parts = split_hyphen_parts("Seneca-by-Stone", 10);
assert_eq!(
parts,
vec![("Seneca", 10..16), ("by", 17..19), ("Stone", 20..25),]
);
}
fn check_str_with(src: &str, working: crate::dict::WorkingDict) -> Vec<String> {
let s = crate::testutil::scratch("check");
let path = s.path("x.md");
std::fs::write(&path, src).unwrap();
let sys = crate::sysdict::resolve_embedded();
let dict = crate::engine::load_dictionary(&sys.aff, &sys.dic).unwrap();
let engine =
crate::engine::Engine::new(dict, working, std::path::PathBuf::from("/dev/null"));
let checked = check_file(&path, Options::default(), &engine).unwrap();
checked.misspellings.into_iter().map(|m| m.word).collect()
}
fn check_str(src: &str) -> Vec<String> {
check_str_with(src, crate::dict::WorkingDict::default())
}
fn check_md(src: &str, opts: Options) -> Checked {
let s = crate::testutil::scratch("check-md");
let path = s.path("x.md");
std::fs::write(&path, src).unwrap();
let sys = crate::sysdict::resolve_embedded();
let dict = crate::engine::load_dictionary(&sys.aff, &sys.dic).unwrap();
let engine = crate::engine::Engine::new(
dict,
crate::dict::WorkingDict::default(),
std::path::PathBuf::from("/dev/null"),
);
check_file(&path, opts, &engine).unwrap()
}
fn words(c: &Checked) -> Vec<&str> {
c.misspellings.iter().map(|m| m.word.as_str()).collect()
}
const AUDIT: Options = Options {
format: format::Format::Auto,
directives: false,
};
#[test]
fn off_region_between_blocks_is_skipped() {
let src = "teh one\n\n<!-- redink:off -->\n\nZe vorld ees\nending\n\n<!-- redink:on -->\n\ncdoe two\n";
let c = check_md(src, Options::default());
assert_eq!(words(&c), vec!["teh", "cdoe"]);
assert!(c.warnings.is_empty(), "{:?}", c.warnings);
}
#[test]
fn off_marker_hugging_a_paragraph() {
let src = "<!-- redink:off -->\nvorld ees\n<!-- redink:on -->\ncdoe\n";
assert_eq!(words(&check_md(src, Options::default())), vec!["cdoe"]);
}
#[test]
fn off_region_inline_within_a_sentence() {
let src = "She said <!-- redink:off -->\"Ze vorld\"<!-- redink:on --> and teh rest.\n";
assert_eq!(words(&check_md(src, Options::default())), vec!["teh"]);
}
#[test]
fn directive_in_code_is_inert() {
let src = "```\n<!-- redink:off -->\n```\n\nvorld\n\n`<!-- redink:allow cdoe -->` cdoe\n";
let c = check_md(src, Options::default());
assert_eq!(words(&c), vec!["vorld", "cdoe"]);
assert!(c.warnings.is_empty(), "{:?}", c.warnings);
}
#[test]
fn unclosed_off_reports_its_line() {
let src = "cdoe\n\n<!-- redink:off -->\n\nvorld\n";
let c = check_md(src, Options::default());
assert_eq!(words(&c), vec!["cdoe"]);
assert_eq!(c.warnings.len(), 1);
assert_eq!(c.warnings[0].line, 3);
}
#[test]
fn allow_list_is_file_wide_and_covers_compounds() {
let src = "Vorld first.\n\n<!-- redink:allow vorld =Kesh -->\n\nThe vorld-Kesh pact, Kesh's oath; kesh and cdoe.\n";
let c = check_md(src, Options::default());
assert_eq!(words(&c), vec!["kesh", "cdoe"]);
}
#[test]
fn no_ignore_checks_everything() {
let src = "<!-- redink:allow cdoe -->\n\ncdoe <!-- redink:off -->vorld\n";
let c = check_md(src, AUDIT);
assert_eq!(words(&c), vec!["cdoe", "vorld"]);
assert!(
c.warnings.is_empty(),
"audit mode should not interpret directives"
);
}
#[test]
fn plain_text_ignores_directives() {
let s = crate::testutil::scratch("check-txt");
let path = s.path("x.txt");
std::fs::write(&path, "<!-- redink:off --> vorld").unwrap();
let sys = crate::sysdict::resolve_embedded();
let dict = crate::engine::load_dictionary(&sys.aff, &sys.dic).unwrap();
let engine = crate::engine::Engine::new(
dict,
crate::dict::WorkingDict::default(),
std::path::PathBuf::from("/dev/null"),
);
let c = check_file(&path, Options::default(), &engine).unwrap();
assert!(words(&c).contains(&"vorld"));
}
#[test]
fn phrase_matching() {
let m = check_str("It was, per se, fine. But se alone is a typo.");
assert!(
!m.contains(&"se".to_string()) || m.iter().filter(|w| *w == "se").count() == 1,
"expected exactly one 'se' (the standalone): {m:?}"
);
assert!(m.contains(&"se".to_string()));
}
#[test]
fn roman_numerals_and_acronyms_skipped() {
let m = check_str("Chapter XIV bore MK17 rifles and XVII flags.");
assert!(
!m.iter().any(|w| w.chars().all(|c| c.is_ascii_uppercase())),
"all-caps token flagged: {m:?}"
);
}
#[test]
fn numeric_possessives_and_single_letters_not_flagged() {
let m = check_str("The relocation 2/1's pacing; 2/5's architecture; the rune Θ held.");
assert!(!m.contains(&"1's".to_string()), "1's flagged: {m:?}");
assert!(!m.contains(&"5's".to_string()), "5's flagged: {m:?}");
assert!(!m.contains(&"Θ".to_string()), "Θ flagged: {m:?}");
}
#[test]
fn latin_phrases_accepted() {
let m = check_str("A de facto rule; in vitro; ad hoc; ipso facto; bona fide.");
for bad in ["facto", "vitro", "ipso", "bona", "fide"] {
assert!(!m.contains(&bad.to_string()), "{bad} flagged: {m:?}");
}
}
#[test]
fn case_sensitive_phrase_exact_match_only() {
let mut wd = crate::dict::WorkingDict::default();
wd.add_entry("Tzeya Gan", true);
let m = check_str_with("The Tzeya Gan rose. The tzeya gan fell.", wd);
assert!(
!m.contains(&"Tzeya".to_string()),
"exact phrase flagged: {m:?}"
);
assert!(
!m.contains(&"Gan".to_string()),
"exact phrase flagged: {m:?}"
);
assert_eq!(
m,
vec!["tzeya".to_string(), "gan".to_string()],
"wrong-case phrase must still be flagged: {m:?}"
);
}
#[test]
fn case_insensitive_phrase_still_fuzzy() {
let mut wd = crate::dict::WorkingDict::default();
wd.add_entry("tzeya gan", false);
let m = check_str_with("The tzeya gan rose. The TZeya GAN fell.", wd);
assert!(m.is_empty(), "ci phrase should accept any casing: {m:?}");
}
#[test]
fn phrases_do_not_match_across_sentence_boundary() {
let mut wd = crate::dict::WorkingDict::default();
wd.add_entry("tzeya gan", false);
let m = check_str_with("The tzeya. Gan rose.", wd);
assert_eq!(
m,
vec!["tzeya".to_string(), "Gan".to_string()],
"sentence-straddled bigram matched: {m:?}"
);
}
#[test]
fn phrases_do_not_match_across_paragraph_break() {
let mut wd = crate::dict::WorkingDict::default();
wd.add_entry("tzeya gan", false);
let m = check_str_with("The tzeya\n\nGan rose.", wd);
assert_eq!(
m,
vec!["tzeya".to_string(), "Gan".to_string()],
"paragraph-straddled bigram matched: {m:?}"
);
}
#[test]
fn phrases_match_across_soft_line_wrap() {
let mut wd = crate::dict::WorkingDict::default();
wd.add_entry("tzeya gan", false);
let m = check_str_with("The tzeya\nGan rose.", wd);
assert!(m.is_empty(), "soft-wrapped phrase should match: {m:?}");
}
#[test]
fn latin_phrase_not_matched_across_sentence_boundary() {
let m = check_str("It was de. Facto rules applied.");
assert!(
m.contains(&"de".to_string()) && m.contains(&"Facto".to_string()),
"sentence-straddled 'de facto' not flagged: {m:?}"
);
}
}