use std::path::PathBuf;
use hashbrown::HashSet;
use spellbook::Dictionary;
use crate::dict::{PhraseBigrams, WorkingDict, canonical, strip_possessive};
const MIN_SUGGEST_LEN: usize = 3;
pub const SUGGEST_LIMIT: usize = 9;
const CUSTOM_SUGGEST_LIMIT: usize = 3;
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub enum Suggest {
Fast,
Thorough,
}
pub struct Engine {
dict: Dictionary,
custom_dict: Dictionary,
working_path: PathBuf,
working: WorkingDict,
session_ignore: HashSet<String>,
phrase_bigrams: PhraseBigrams,
phrase_bigrams_cs: PhraseBigrams,
}
impl Engine {
pub fn new(dict: Dictionary, working: WorkingDict, working_path: PathBuf) -> Self {
let custom_dict = build_custom_dict(&working.ci, &working.cs);
let mut this = Self {
dict,
custom_dict,
working_path,
working,
session_ignore: HashSet::new(),
phrase_bigrams: PhraseBigrams::default(),
phrase_bigrams_cs: PhraseBigrams::default(),
};
this.rebuild_phrase_bigrams();
this
}
pub fn working_path(&self) -> &std::path::Path {
&self.working_path
}
pub fn working_word_count(&self) -> usize {
self.working.ci.len() + self.working.cs.len()
}
pub fn check(&self, word: &str) -> bool {
let stem = strip_possessive(word);
if is_all_caps_alnum(word) || stem.is_some_and(is_all_caps_alnum) {
return true;
}
if stem.is_some_and(|s| !s.chars().any(|c| c.is_alphabetic())) {
return true;
}
if is_non_ascii_single_letter(word) || stem.is_some_and(is_non_ascii_single_letter) {
return true;
}
if self.user_layer_has(word) || stem.is_some_and(|s| self.user_layer_has(s)) {
return true;
}
self.dict.check(word)
}
fn user_layer_has(&self, word: &str) -> bool {
if self.working.cs.contains(word) {
return true;
}
if self.session_ignore.is_empty() && self.working.ci.is_empty() {
return false;
}
if word.is_ascii() && !word.bytes().any(|b| b.is_ascii_uppercase()) {
return self.session_ignore.contains(word) || self.working.ci.contains(word);
}
let lower = word.to_lowercase();
self.session_ignore.contains(&lower) || self.working.ci.contains(&lower)
}
pub fn suggest(&self, word: &str, mode: Suggest) -> Vec<String> {
let mut out: Vec<String> = Vec::new();
if let Some(stem) = strip_possessive(word) {
let suffix = &word[stem.len()..];
for s in self.suggest_forms(stem, mode) {
let mut form = s;
form.push_str(suffix);
if !out.contains(&form) {
out.push(form);
}
}
}
for s in self.suggest_forms(word, mode) {
if !out.contains(&s) {
out.push(s);
}
}
out
}
fn suggest_forms(&self, word: &str, mode: Suggest) -> Vec<String> {
let mut custom = Vec::new();
self.custom_dict.suggest(word, &mut custom);
custom.retain(|s| s.chars().count() >= MIN_SUGGEST_LEN);
custom.truncate(CUSTOM_SUGGEST_LIMIT);
let mut system = Vec::new();
match mode {
Suggest::Thorough => self.dict.suggest(word, &mut system),
Suggest::Fast => {
self.dict
.checker()
.into_suggester()
.with_ngram_suggestions(false)
.suggest(word, &mut system);
system.retain(|s| s.chars().count() >= MIN_SUGGEST_LEN);
if system.is_empty() {
self.dict.suggest(word, &mut system);
}
}
}
system.retain(|s| s.chars().count() >= MIN_SUGGEST_LEN);
merge_by_distance(word, custom, system)
}
pub fn ignore_session(&mut self, word: &str) {
self.session_ignore.insert(canonical(word).to_lowercase());
}
pub fn add_ci(&mut self, word: &str) {
let stored = canonical(word).to_lowercase();
if self.working.add_ci(word) {
let _ = self.custom_dict.add(&stored);
}
}
pub fn add_cs(&mut self, word: &str) {
let stored = canonical(word);
if self.working.add_cs(word) {
let _ = self.custom_dict.add(&stored);
}
}
pub fn add_phrase(&mut self, text: &str, sensitive: bool) {
let is_phrase = crate::dict::normalize_phrase(text).is_some();
match self.working.add_entry(text, sensitive) {
crate::dict::AddOutcome::Added if is_phrase => self.rebuild_phrase_bigrams(),
crate::dict::AddOutcome::Added => {
let stored = if sensitive {
canonical(text)
} else {
canonical(text).to_lowercase()
};
let _ = self.custom_dict.add(&stored);
}
_ => {}
}
}
fn rebuild_phrase_bigrams(&mut self) {
self.phrase_bigrams = crate::dict::build_phrase_bigrams(&self.working.phrases);
self.phrase_bigrams_cs = crate::dict::PhraseBigrams::from_phrases(
self.working.phrases_cs.iter().map(String::as_str),
);
}
#[allow(dead_code)]
pub fn remove(&mut self, word: &str) -> bool {
let removed = self.working.remove(word);
if removed {
self.custom_dict = build_custom_dict(&self.working.ci, &self.working.cs);
self.rebuild_phrase_bigrams();
}
removed
}
pub fn phrase_bigrams(&self) -> &PhraseBigrams {
&self.phrase_bigrams
}
pub fn phrase_bigrams_cs(&self) -> &PhraseBigrams {
&self.phrase_bigrams_cs
}
pub fn save_working(&self) -> anyhow::Result<()> {
crate::dict::save(&self.working_path, &self.working)
}
}
fn merge_by_distance(word: &str, custom: Vec<String>, system: Vec<String>) -> Vec<String> {
if custom.is_empty() {
return system;
}
let mut distances: Vec<usize> = system.iter().map(|s| edit_distance(word, s)).collect();
let mut out = system;
for candidate in custom {
if out.contains(&candidate) {
continue;
}
let distance = edit_distance(word, &candidate);
let at = distances
.iter()
.position(|&d| d >= distance)
.unwrap_or(out.len());
out.insert(at, candidate);
distances.insert(at, distance);
}
out
}
fn edit_distance(a: &str, b: &str) -> usize {
let a: Vec<char> = a.chars().flat_map(char::to_lowercase).collect();
let b: Vec<char> = b.chars().flat_map(char::to_lowercase).collect();
if a.is_empty() {
return b.len();
}
if b.is_empty() {
return a.len();
}
let width = b.len() + 1;
let mut two_ago = vec![0usize; width];
let mut one_ago: Vec<usize> = (0..width).collect();
let mut current = vec![0usize; width];
for i in 1..=a.len() {
current[0] = i;
for j in 1..=b.len() {
let cost = usize::from(a[i - 1] != b[j - 1]);
let mut best = (one_ago[j] + 1)
.min(current[j - 1] + 1)
.min(one_ago[j - 1] + cost);
if i > 1 && j > 1 && a[i - 1] == b[j - 2] && a[i - 2] == b[j - 1] {
best = best.min(two_ago[j - 2] + 1);
}
current[j] = best;
}
std::mem::swap(&mut two_ago, &mut one_ago);
std::mem::swap(&mut one_ago, &mut current);
}
one_ago[b.len()]
}
fn build_custom_dict(ci: &HashSet<String>, cs: &HashSet<String>) -> Dictionary {
let mut dict = Dictionary::new("", "0\n").expect("empty dictionary is valid");
for word in ci.iter().chain(cs.iter()) {
let _ = dict.add(word);
}
dict
}
fn is_all_caps_alnum(word: &str) -> bool {
let mut has_letter = false;
word.chars().all(|c| {
if c.is_ascii_uppercase() {
has_letter = true;
true
} else {
c.is_ascii_digit()
}
}) && has_letter
}
fn is_non_ascii_single_letter(word: &str) -> bool {
let mut chars = word.chars();
matches!(chars.next(), Some(c) if c.is_alphabetic() && !c.is_ascii()) && chars.next().is_none()
}
pub fn load_dictionary(aff: &str, dic: &str) -> anyhow::Result<Dictionary> {
Dictionary::new(aff, dic).map_err(|e| anyhow::anyhow!("failed to parse dictionary: {e:?}"))
}
#[cfg(test)]
mod tests {
use super::*;
use crate::dict::WorkingDict;
use std::path::PathBuf;
fn engine_with_ci(word: &str) -> Engine {
let sys = crate::sysdict::resolve_embedded();
let dict = load_dictionary(&sys.aff, &sys.dic).unwrap();
let mut wd = WorkingDict::default();
wd.add_ci(word);
Engine::new(dict, wd, PathBuf::from("/dev/null"))
}
#[test]
fn possessive_accepted_when_base_added() {
let e = engine_with_ci("thorne");
assert!(e.check("Thorne"));
assert!(e.check("thorne"));
assert!(e.check("Thorne's")); assert!(e.check("Thorne\u{2019}s")); assert!(!e.check("Thornex"));
}
#[test]
fn add_via_possessive_form_registers_stem() {
let sys = crate::sysdict::resolve_embedded();
let dict = load_dictionary(&sys.aff, &sys.dic).unwrap();
let wd = WorkingDict::default();
let mut e = Engine::new(dict, wd, PathBuf::from("/dev/null"));
e.add_ci("Atrax's");
assert!(e.check("Atrax"));
assert!(e.check("Atrax's"));
assert!(e.check("atrax"));
}
#[test]
fn add_phrase_layers_and_roundtrip() {
let s = crate::testutil::scratch("addphrase");
let path = s.path("t.dic");
let _ = std::fs::remove_file(&path);
let sys = crate::sysdict::resolve_embedded();
let dict = load_dictionary(&sys.aff, &sys.dic).unwrap();
let mut e = Engine::new(dict, WorkingDict::default(), path.clone());
e.add_phrase("hobbit", false);
assert!(e.check("Hobbit"));
e.add_phrase("tzeya gan", false);
assert!(
e.phrase_bigrams().contains("tzeya", "gan"),
"ci bigram missing after add_phrase"
);
e.add_phrase("Tzeya Gan", true);
assert!(
e.phrase_bigrams_cs().contains("Tzeya", "Gan"),
"cs bigram missing after add_phrase"
);
e.add_phrase("per se", false);
assert!(e.phrase_bigrams().contains("per", "se"));
e.save_working().unwrap();
let loaded = crate::dict::load(&path).unwrap();
assert!(loaded.ci.contains("hobbit"));
assert!(loaded.phrases.contains("tzeya gan"));
assert!(loaded.phrases.contains("per se"));
assert!(!loaded.phrases_cs.contains("Tzeya Gan"));
let _ = std::fs::remove_file(&path);
}
#[test]
fn ignore_via_possessive_suppresses_stem() {
let sys = crate::sysdict::resolve_embedded();
let dict = load_dictionary(&sys.aff, &sys.dic).unwrap();
let wd = WorkingDict::default();
let mut e = Engine::new(dict, wd, PathBuf::from("/dev/null"));
e.ignore_session("Atrax's");
assert!(e.check("Atrax"));
assert!(e.check("Atrax's"));
}
#[test]
fn vendored_dict_accepts_else_possessive() {
let sys = crate::sysdict::resolve_embedded();
let dict = load_dictionary(&sys.aff, &sys.dic).unwrap();
let e = Engine::new(dict, WorkingDict::default(), PathBuf::from("/dev/null"));
assert!(e.check("else"));
assert!(e.check("else's"), "local patch else->else/M missing?");
assert!(e.check("anyone else's".split(' ').nth(1).unwrap()));
}
#[test]
fn vendored_dict_accepts_saddler_possessive() {
let sys = crate::sysdict::resolve_embedded();
let dict = load_dictionary(&sys.aff, &sys.dic).unwrap();
let e = Engine::new(dict, WorkingDict::default(), PathBuf::from("/dev/null"));
assert!(e.check("saddler"));
assert!(e.check("saddler's"), "patch saddler/S->saddler/SM missing?");
assert!(e.check("saddlers"), "plural (S flag) lost in patching?");
}
#[test]
fn all_caps_alnum_accepted() {
let sys = crate::sysdict::resolve_embedded();
let dict = load_dictionary(&sys.aff, &sys.dic).unwrap();
let e = Engine::new(dict, WorkingDict::default(), PathBuf::from("/dev/null"));
for w in [
"XVII",
"NASA",
"M16",
"3M",
"R2D2",
"NASA's",
"NASA\u{2019}S",
] {
assert!(e.check(w), "{w} should be skipped as all-caps alnum");
}
assert!(!e.check("Ml6"));
assert!(!e.check("nasaa"));
}
#[test]
fn numeric_possessives_accepted() {
let sys = crate::sysdict::resolve_embedded();
let dict = load_dictionary(&sys.aff, &sys.dic).unwrap();
let e = Engine::new(dict, WorkingDict::default(), PathBuf::from("/dev/null"));
for w in ["0's", "1's", "5's", "42's", "42\u{2019}s"] {
assert!(e.check(w), "{w} should be skipped as a numeric possessive");
}
assert!(!e.check("1x's"));
}
#[test]
fn non_ascii_single_letters_accepted() {
let sys = crate::sysdict::resolve_embedded();
let dict = load_dictionary(&sys.aff, &sys.dic).unwrap();
let e = Engine::new(dict, WorkingDict::default(), PathBuf::from("/dev/null"));
for w in ["Θ", "é", "Æ", "Ω"] {
assert!(
e.check(w),
"{w} should be skipped as a single non-ASCII letter"
);
}
assert!(!e.check("ΘΘ"));
assert!(e.check("a"), "'a' is a real word");
}
#[test]
fn edit_distance_counts_a_transposition_as_one_edit() {
assert_eq!(edit_distance("ab", "ba"), 1);
assert_eq!(edit_distance("recieve", "receive"), 1);
assert_eq!(edit_distance("athrones", "thrones"), 1);
assert_eq!(edit_distance("athrones", "athrune"), 2);
assert_eq!(edit_distance("", "abc"), 3);
assert_eq!(edit_distance("abc", "abc"), 0);
assert_eq!(edit_distance("gondr", "Gondor"), 1);
}
#[test]
fn merge_places_custom_by_distance_and_wins_ties() {
let system = || vec!["xaxb".to_string(), "xaxbc".to_string(), "yyyy".to_string()];
let out = merge_by_distance("xaxa", vec!["xaxa".to_string()], system());
assert_eq!(out[0], "xaxa");
let out = merge_by_distance("xaxa", vec!["zaxa".to_string()], system());
assert_eq!(out[0], "zaxa", "tie should go to the working dictionary");
let out = merge_by_distance("xaxa", vec!["xaxbcd".to_string()], system());
assert_eq!(out, vec!["xaxb", "xaxbc", "xaxbcd", "yyyy"]);
let out = merge_by_distance("xaxa", vec!["xaxbc".to_string()], system());
assert_eq!(out, system());
}
#[test]
fn custom_suggestions_are_merged_not_pinned() {
let e = engine_with_ci("athrune");
let sugs = e.suggest("athrones", Suggest::Thorough);
let coinage = sugs.iter().position(|s| s == "athrune");
let system = sugs.iter().position(|s| s == "thrones");
assert!(coinage.is_some(), "coinage missing entirely: {sugs:?}");
assert!(system.is_some(), "vendored dict lost 'thrones'? {sugs:?}");
assert!(
system < coinage,
"the closer system word should lead: {sugs:?}"
);
let sugs = e.suggest("atrune", Suggest::Thorough);
assert_eq!(
sugs.first().map(String::as_str),
Some("athrune"),
"a tied coinage should lead: {sugs:?}"
);
}
#[test]
fn fast_and_thorough_agree_on_ordinary_typos() {
let e = engine_with_ci("athrune");
for word in ["recieve", "teh", "atrune", "seperate", "wrold"] {
let fast = e.suggest(word, Suggest::Fast);
let thorough = e.suggest(word, Suggest::Thorough);
assert!(!fast.is_empty(), "fast found nothing for {word}");
assert_eq!(
fast.first(),
thorough.first(),
"fast/thorough disagree on the best suggestion for {word}"
);
}
}
#[test]
fn suggestions_drop_very_short() {
let sys = crate::sysdict::resolve_embedded();
let dict = load_dictionary(&sys.aff, &sys.dic).unwrap();
let e = Engine::new(dict, WorkingDict::default(), PathBuf::from("/dev/null"));
let sugs = e.suggest("se", Suggest::Thorough);
assert!(
sugs.iter().all(|s| s.chars().count() >= 3),
"short leak: {sugs:?}"
);
assert!(sugs.contains(&"see".to_string()));
}
#[test]
fn suggestions_include_working_dictionary() {
let e = engine_with_ci("Thorne");
let sugs = e.suggest("Thorn", Suggest::Thorough);
assert!(
sugs.contains(&"Thorne".to_string()),
"custom suggestion missing: {sugs:?}"
);
}
#[test]
fn suggestions_include_entries_added_after_startup() {
let sys = crate::sysdict::resolve_embedded();
let dict = load_dictionary(&sys.aff, &sys.dic).unwrap();
let mut e = Engine::new(dict, WorkingDict::default(), PathBuf::from("/dev/null"));
e.add_cs("Gondor");
let sugs = e.suggest("gondr", Suggest::Thorough);
assert!(
sugs.contains(&"Gondor".to_string()),
"custom suggestion missing: {sugs:?}"
);
}
#[test]
fn suggest_possessive_finds_ci_stem() {
let e = engine_with_ci("Thyri");
let sugs = e.suggest("Thryi's", Suggest::Thorough);
assert!(
sugs.contains(&"Thyri's".to_string()),
"expected Thyri's (re-attached) in {sugs:?}"
);
}
#[test]
fn suggest_possessive_finds_cs_stem() {
let sys = crate::sysdict::resolve_embedded();
let dict = load_dictionary(&sys.aff, &sys.dic).unwrap();
let mut e = Engine::new(dict, WorkingDict::default(), PathBuf::from("/dev/null"));
e.add_cs("Thyri");
let sugs = e.suggest("Thryi's", Suggest::Thorough);
assert!(
sugs.contains(&"Thyri's".to_string()),
"expected Thyri's (re-attached) in {sugs:?}"
);
}
#[test]
fn suggest_possessive_prioritizes_stem_derived() {
let e = engine_with_ci("Thyri");
let sugs = e.suggest("Thryi's", Suggest::Thorough);
let stem_idx = sugs.iter().position(|s| s == "Thyri's");
let other_idx = sugs.iter().position(|s| s == "Thrift's");
assert!(stem_idx.is_some(), "Thyri's missing: {sugs:?}");
if let (Some(stem), Some(other)) = (stem_idx, other_idx) {
assert!(stem < other, "stem-derived should precede system: {sugs:?}");
}
}
}