redink 0.1.10

A spellchecker for prose — a TUI for humans, a JSON CLI for agents
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
//! Driving a spellcheck pass over files into a list of [`Misspelling`] records.

use std::path::{Path, PathBuf};

use anyhow::Result;

use crate::dict::PhraseBigrams;
use crate::engine::Engine;
use crate::format::{self, Options};
use crate::token::{Token, tokenize_with_lowercase};

/// The hyphenated token a flagged part belongs to. Carries its position as
/// well as its text: an editor that fixes one part has to be able to say
/// exactly where the compound sits in order to keep the text current.
#[derive(Debug, Clone)]
pub struct Compound {
    pub text: String,
    /// Absolute byte offset of the compound's start in the file.
    pub byte_offset: usize,
}

/// A single misspelled-word occurrence in a file.
#[derive(Debug, Clone)]
pub struct Misspelling {
    pub path: PathBuf,
    /// 1-based line number.
    pub line: usize,
    /// 1-based byte column within the line.
    pub col: usize,
    /// Absolute byte offset of the word start in the file.
    pub byte_offset: usize,
    pub word: String,
    /// Filled in later (batch, deduplicated) by the caller; `None` until then.
    pub suggestions: Option<Vec<String>>,
    /// If this occurrence is a part of a hyphenated compound that was not
    /// recognized as a whole, that whole token. `None` for plain
    /// (non-compound) words.
    pub compound: Option<Compound>,
}

/// A problem with an in-file directive, located for display.
#[derive(Debug, Clone)]
pub struct Warning {
    pub path: PathBuf,
    /// 1-based line number of the offending comment.
    pub line: usize,
    pub kind: crate::directive::Kind,
    pub message: String,
}

impl std::fmt::Display for Warning {
    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
        write!(f, "{}:{}: {}", self.path.display(), self.line, self.message)
    }
}

/// The outcome of checking one file.
#[derive(Debug, Default)]
pub struct Checked {
    /// Misspellings in source order.
    pub misspellings: Vec<Misspelling>,
    pub warnings: Vec<Warning>,
}

/// Check a single file, returning its misspellings in source order and any
/// directive warnings. Suggestions are left unset; callers attach them in a
/// deduplicated pass.
pub fn check_file(path: &Path, opts: Options, engine: &Engine) -> Result<Checked> {
    let src = std::fs::read_to_string(path)?;
    let line_starts = LineStarts::new(&src);
    let analysis = format::analyze(&src, path, opts);
    let allow = &analysis.allow;
    let accepted = |w: &str| allow.contains(w) || engine.check(w);
    let warnings = analysis
        .warnings
        .iter()
        .map(|w| Warning {
            path: path.to_path_buf(),
            line: line_starts.locate(w.byte_offset).0 + 1,
            kind: w.kind.clone(),
            message: w.message.clone(),
        })
        .collect();
    let tokenized = tokenize_with_lowercase(&src, &analysis.skips);
    let tokens = &tokenized.tokens;
    let lowercase = &tokenized.lowercase;
    let gap_clean = &tokenized.gap_clean;
    let phrases = engine.phrase_bigrams();
    let phrases_cs = engine.phrase_bigrams_cs();

    let mut out = Vec::new();
    for (i, tok) in tokens.iter().enumerate() {
        // Phrase matching: a token that forms a known bigram with its neighbour
        // (e.g. "se" in "per se") is accepted, so fragment words are only let
        // through in their phrase context and still flagged elsewhere.
        if phrase_covered(i, tokens, lowercase, gap_clean, phrases, phrases_cs) {
            continue;
        }
        if tok.word.contains('-') {
            // Stage 1: the whole compound. If any layer recognizes it, accept.
            if accepted(&tok.word) {
                continue;
            }
            // Stage 2: split on hyphen-runs and check each non-empty part.
            let compound = Compound {
                text: tok.word.clone(),
                byte_offset: tok.byte_range.start,
            };
            for (part, range) in split_hyphen_parts(&tok.word, tok.byte_range.start) {
                if part.is_empty() || !part.chars().any(|c| c.is_alphabetic()) {
                    continue;
                }
                if accepted(part) {
                    continue;
                }
                out.push(build_misspelling(
                    path,
                    &line_starts,
                    part,
                    &range,
                    Some(&compound),
                ));
            }
        } else {
            if accepted(&tok.word) {
                continue;
            }
            out.push(build_misspelling(
                path,
                &line_starts,
                &tok.word,
                &tok.byte_range,
                None,
            ));
        }
    }
    Ok(Checked {
        misspellings: out,
        warnings,
    })
}

/// True if `tokens[i]` forms a known phrase bigram with either neighbour.
/// A bigram only counts when the source gap between the two words is clean
/// (whitespace-only, at most one newline) — so "per. Se" or "per\n\nse"
/// never match "per se", while a soft line wrap still does. Regular phrases
/// match case-insensitively (via the parallel `lowercase` array); exact-case
/// phrases (`=Tzeya Gan`) require the neighbouring words to match as written.
pub(crate) fn phrase_covered(
    i: usize,
    tokens: &[Token],
    lowercase: &[String],
    gap_clean: &[bool],
    phrases: &PhraseBigrams,
    phrases_cs: &PhraseBigrams,
) -> bool {
    let w = &lowercase[i];
    if i > 0
        && gap_clean[i]
        && (phrases.contains(&lowercase[i - 1], w)
            || phrases_cs.contains(&tokens[i - 1].word, &tokens[i].word))
    {
        return true;
    }
    if i + 1 < tokens.len()
        && gap_clean[i + 1]
        && (phrases.contains(w, &lowercase[i + 1])
            || phrases_cs.contains(&tokens[i].word, &tokens[i + 1].word))
    {
        return true;
    }
    false
}

fn build_misspelling(
    path: &Path,
    line_starts: &LineStarts,
    word: &str,
    byte_range: &std::ops::Range<usize>,
    compound: Option<&Compound>,
) -> Misspelling {
    let (line0, col0) = line_starts.locate(byte_range.start);
    Misspelling {
        path: path.to_path_buf(),
        line: line0 + 1,
        col: col0 + 1,
        byte_offset: byte_range.start,
        word: word.to_string(),
        suggestions: None,
        compound: compound.cloned(),
    }
}

/// Split `word` on hyphens into `(part, absolute_byte_range)` pairs, skipping
/// empty parts (so `year--another` yields only `year` and `another`). `base` is
/// the absolute byte offset at which `word` begins in the source.
fn split_hyphen_parts(word: &str, base: usize) -> Vec<(&str, std::ops::Range<usize>)> {
    let mut out = Vec::new();
    let bytes = word.as_bytes();
    let mut start = 0usize;
    let mut i = 0usize;
    while i <= bytes.len() {
        if i == bytes.len() || bytes[i] == b'-' {
            if i > start {
                out.push((&word[start..i], base + start..base + i));
            }
            start = i + 1;
        }
        i += 1;
    }
    out
}

/// Map an absolute byte offset to a 0-based (line, byte-column).
pub struct LineStarts {
    starts: Vec<usize>,
}

impl LineStarts {
    pub fn new(src: &str) -> Self {
        let mut starts = vec![0usize];
        for (i, b) in src.bytes().enumerate() {
            if b == b'\n' {
                starts.push(i + 1);
            }
        }
        Self { starts }
    }

    pub fn locate(&self, byte_offset: usize) -> (usize, usize) {
        let line = self
            .starts
            .partition_point(|&s| s <= byte_offset)
            .saturating_sub(1);
        let col = byte_offset - self.starts[line];
        (line, col)
    }
}

#[cfg(test)]
mod tests {
    use super::*;

    #[test]
    fn split_parts_basic() {
        let parts = split_hyphen_parts("Tzeya-Gan", 100);
        assert_eq!(parts, vec![("Tzeya", 100..105), ("Gan", 106..109)]);
    }

    #[test]
    fn split_parts_double_hyphen_drops_empty() {
        let parts = split_hyphen_parts("year--another", 0);
        assert_eq!(parts.len(), 2);
        assert_eq!(parts[0], ("year", 0..4));
        assert_eq!(parts[1], ("another", 6..13));
    }

    #[test]
    fn split_parts_multi() {
        let parts = split_hyphen_parts("Seneca-by-Stone", 10);
        assert_eq!(
            parts,
            vec![("Seneca", 10..16), ("by", 17..19), ("Stone", 20..25),]
        );
    }

    fn check_str_with(src: &str, working: crate::dict::WorkingDict) -> Vec<String> {
        let s = crate::testutil::scratch("check");
        let path = s.path("x.md");
        std::fs::write(&path, src).unwrap();

        let sys = crate::sysdict::resolve_embedded();
        let dict = crate::engine::load_dictionary(&sys.aff, &sys.dic).unwrap();
        let engine =
            crate::engine::Engine::new(dict, working, std::path::PathBuf::from("/dev/null"));
        let checked = check_file(&path, Options::default(), &engine).unwrap();
        checked.misspellings.into_iter().map(|m| m.word).collect()
    }

    fn check_str(src: &str) -> Vec<String> {
        check_str_with(src, crate::dict::WorkingDict::default())
    }

    /// Check `src` as a Markdown file under `opts`, returning everything.
    fn check_md(src: &str, opts: Options) -> Checked {
        let s = crate::testutil::scratch("check-md");
        let path = s.path("x.md");
        std::fs::write(&path, src).unwrap();
        let sys = crate::sysdict::resolve_embedded();
        let dict = crate::engine::load_dictionary(&sys.aff, &sys.dic).unwrap();
        let engine = crate::engine::Engine::new(
            dict,
            crate::dict::WorkingDict::default(),
            std::path::PathBuf::from("/dev/null"),
        );
        check_file(&path, opts, &engine).unwrap()
    }

    fn words(c: &Checked) -> Vec<&str> {
        c.misspellings.iter().map(|m| m.word.as_str()).collect()
    }

    const AUDIT: Options = Options {
        format: format::Format::Auto,
        directives: false,
    };

    #[test]
    fn off_region_between_blocks_is_skipped() {
        let src = "teh one\n\n<!-- redink:off -->\n\nZe vorld ees\nending\n\n<!-- redink:on -->\n\ncdoe two\n";
        let c = check_md(src, Options::default());
        assert_eq!(words(&c), vec!["teh", "cdoe"]);
        assert!(c.warnings.is_empty(), "{:?}", c.warnings);
    }

    /// A marker directly above a paragraph (no blank line) still works: the
    /// HTML block ends at the comment's line.
    #[test]
    fn off_marker_hugging_a_paragraph() {
        let src = "<!-- redink:off -->\nvorld ees\n<!-- redink:on -->\ncdoe\n";
        assert_eq!(words(&check_md(src, Options::default())), vec!["cdoe"]);
    }

    #[test]
    fn off_region_inline_within_a_sentence() {
        let src = "She said <!-- redink:off -->\"Ze vorld\"<!-- redink:on --> and teh rest.\n";
        assert_eq!(words(&check_md(src, Options::default())), vec!["teh"]);
    }

    /// A directive shown in a code block is an example, not an instruction.
    #[test]
    fn directive_in_code_is_inert() {
        let src = "```\n<!-- redink:off -->\n```\n\nvorld\n\n`<!-- redink:allow cdoe -->` cdoe\n";
        let c = check_md(src, Options::default());
        assert_eq!(words(&c), vec!["vorld", "cdoe"]);
        assert!(c.warnings.is_empty(), "{:?}", c.warnings);
    }

    #[test]
    fn unclosed_off_reports_its_line() {
        let src = "cdoe\n\n<!-- redink:off -->\n\nvorld\n";
        let c = check_md(src, Options::default());
        assert_eq!(words(&c), vec!["cdoe"]);
        assert_eq!(c.warnings.len(), 1);
        assert_eq!(c.warnings[0].line, 3);
    }

    #[test]
    fn allow_list_is_file_wide_and_covers_compounds() {
        let src = "Vorld first.\n\n<!-- redink:allow vorld =Kesh -->\n\nThe vorld-Kesh pact, Kesh's oath; kesh and cdoe.\n";
        let c = check_md(src, Options::default());
        assert_eq!(words(&c), vec!["kesh", "cdoe"]);
    }

    #[test]
    fn no_ignore_checks_everything() {
        let src = "<!-- redink:allow cdoe -->\n\ncdoe <!-- redink:off -->vorld\n";
        let c = check_md(src, AUDIT);
        assert_eq!(words(&c), vec!["cdoe", "vorld"]);
        assert!(
            c.warnings.is_empty(),
            "audit mode should not interpret directives"
        );
    }

    /// Directives are a Markdown feature: in plain text a "comment" is prose.
    #[test]
    fn plain_text_ignores_directives() {
        let s = crate::testutil::scratch("check-txt");
        let path = s.path("x.txt");
        std::fs::write(&path, "<!-- redink:off --> vorld").unwrap();
        let sys = crate::sysdict::resolve_embedded();
        let dict = crate::engine::load_dictionary(&sys.aff, &sys.dic).unwrap();
        let engine = crate::engine::Engine::new(
            dict,
            crate::dict::WorkingDict::default(),
            std::path::PathBuf::from("/dev/null"),
        );
        let c = check_file(&path, Options::default(), &engine).unwrap();
        assert!(words(&c).contains(&"vorld"));
    }

    #[test]
    fn phrase_matching() {
        // "se" is accepted inside "per se" but flagged on its own.
        let m = check_str("It was, per se, fine. But se alone is a typo.");
        assert!(
            !m.contains(&"se".to_string()) || m.iter().filter(|w| *w == "se").count() == 1,
            "expected exactly one 'se' (the standalone): {m:?}"
        );
        assert!(m.contains(&"se".to_string()));
    }

    #[test]
    fn roman_numerals_and_acronyms_skipped() {
        let m = check_str("Chapter XIV bore MK17 rifles and XVII flags.");
        assert!(
            !m.iter().any(|w| w.chars().all(|c| c.is_ascii_uppercase())),
            "all-caps token flagged: {m:?}"
        );
    }

    /// Scene-label fractions like "2/1's" tokenize to numeric possessives
    /// ("1's"), and lone foreign letters ("Θ") are notation — neither is a
    /// misspelling.
    #[test]
    fn numeric_possessives_and_single_letters_not_flagged() {
        let m = check_str("The relocation 2/1's pacing; 2/5's architecture; the rune Θ held.");
        assert!(!m.contains(&"1's".to_string()), "1's flagged: {m:?}");
        assert!(!m.contains(&"5's".to_string()), "5's flagged: {m:?}");
        assert!(!m.contains(&"Θ".to_string()), "Θ flagged: {m:?}");
    }

    #[test]
    fn latin_phrases_accepted() {
        let m = check_str("A de facto rule; in vitro; ad hoc; ipso facto; bona fide.");
        // none of the Latin fragment words should be flagged
        for bad in ["facto", "vitro", "ipso", "bona", "fide"] {
            assert!(!m.contains(&bad.to_string()), "{bad} flagged: {m:?}");
        }
    }

    /// An exact-case phrase accepts its fragment words only when the
    /// neighbouring text matches the phrase as written.
    #[test]
    fn case_sensitive_phrase_exact_match_only() {
        let mut wd = crate::dict::WorkingDict::default();
        wd.add_entry("Tzeya Gan", true);
        let m = check_str_with("The Tzeya Gan rose. The tzeya gan fell.", wd);
        assert!(
            !m.contains(&"Tzeya".to_string()),
            "exact phrase flagged: {m:?}"
        );
        assert!(
            !m.contains(&"Gan".to_string()),
            "exact phrase flagged: {m:?}"
        );
        assert_eq!(
            m,
            vec!["tzeya".to_string(), "gan".to_string()],
            "wrong-case phrase must still be flagged: {m:?}"
        );
    }

    /// A case-insensitive phrase still accepts any casing (the cs layer must
    /// not narrow existing behaviour).
    #[test]
    fn case_insensitive_phrase_still_fuzzy() {
        let mut wd = crate::dict::WorkingDict::default();
        wd.add_entry("tzeya gan", false);
        let m = check_str_with("The tzeya gan rose. The TZeya GAN fell.", wd);
        assert!(m.is_empty(), "ci phrase should accept any casing: {m:?}");
    }

    /// A phrase must not match when its words straddle a sentence boundary:
    /// "tzeya." ending one sentence and "Gan" opening the next is not the
    /// phrase "tzeya gan".
    #[test]
    fn phrases_do_not_match_across_sentence_boundary() {
        let mut wd = crate::dict::WorkingDict::default();
        wd.add_entry("tzeya gan", false);
        let m = check_str_with("The tzeya. Gan rose.", wd);
        assert_eq!(
            m,
            vec!["tzeya".to_string(), "Gan".to_string()],
            "sentence-straddled bigram matched: {m:?}"
        );
    }

    /// Ditto for a paragraph break between the two words.
    #[test]
    fn phrases_do_not_match_across_paragraph_break() {
        let mut wd = crate::dict::WorkingDict::default();
        wd.add_entry("tzeya gan", false);
        let m = check_str_with("The tzeya\n\nGan rose.", wd);
        assert_eq!(
            m,
            vec!["tzeya".to_string(), "Gan".to_string()],
            "paragraph-straddled bigram matched: {m:?}"
        );
    }

    /// A soft line wrap within a paragraph is still one phrase — hard-wrapped
    /// manuscripts must not lose coverage.
    #[test]
    fn phrases_match_across_soft_line_wrap() {
        let mut wd = crate::dict::WorkingDict::default();
        wd.add_entry("tzeya gan", false);
        let m = check_str_with("The tzeya\nGan rose.", wd);
        assert!(m.is_empty(), "soft-wrapped phrase should match: {m:?}");
    }

    /// The bundled Latin list obeys the same rule: "de. Facto…" is not
    /// "de facto" (both fragments are flagged standalone).
    #[test]
    fn latin_phrase_not_matched_across_sentence_boundary() {
        let m = check_str("It was de. Facto rules applied.");
        assert!(
            m.contains(&"de".to_string()) && m.contains(&"Facto".to_string()),
            "sentence-straddled 'de facto' not flagged: {m:?}"
        );
    }
}