lang_check/

names.rs

//! Opt-in detection of human names in spelling diagnostics.
//!
//! Spell checkers flag names constantly, and the only escape hatch is adding each one to
//! a dictionary — which never converges. This module decides whether a token that an
//! engine flagged as a misspelling is in fact a person's name, so the diagnostic can be
//! dropped.
//!
//! # Engine independence
//!
//! Nothing here knows which engine produced the diagnostic. The only inputs are the
//! flagged token, the surrounding document text, and the engine's own `suggestions`
//! list — which both Harper and `LanguageTool` populate. See [`crate::suppression`].
//!
//! # Why no single signal is trusted
//!
//! Measurement against a live `LanguageTool` server showed real surnames sitting at edit
//! distance 1 from real words (`Hoare`→`Hare`, `Ackermann`→`Ackerman`), so "no plausible
//! correction exists" cannot carry the decision alone. In the other direction, the
//! crowdsourced name lists that feed the gazetteer contain hundreds of entries that are
//! actually common misspellings (`thier`, `balck`, `alway`); those are pruned at build
//! time, but the lesson stands — a gazetteer hit alone must not silence a diagnostic
//! either.
//!
//! So the verdict requires **at least two independent signals** and a score above a
//! configurable threshold. Missing a name costs a stray squiggle; suppressing a real
//! typo costs trust in every squiggle, so the bias is deliberately toward staying noisy.

use fst::Set;
use regex::Regex;
use std::sync::LazyLock;

/// The compiled gazetteer, embedded at build time.
///
/// Generated by `scripts/build-name-gazetteer.py` + `examples/build-name-fst.rs`.
static NAME_FST_BYTES: &[u8] = include_bytes!("../dictionaries/names/names.fst");

static GAZETTEER: LazyLock<Option<Set<&'static [u8]>>> =
    LazyLock::new(|| Set::new(NAME_FST_BYTES).ok());

/// How much corroboration is required before a diagnostic is dropped.
#[derive(Debug, Clone, Copy, PartialEq, Eq, Default, serde::Serialize, serde::Deserialize)]
#[serde(rename_all = "lowercase")]
pub enum Aggressiveness {
    /// Only suppress when the evidence is overwhelming.
    Conservative,
    /// Default: a gazetteer hit plus one corroborating signal, or equivalent.
    #[default]
    Balanced,
    /// Suppress on weaker evidence. Expect occasional real typos to be missed.
    Aggressive,
}

impl Aggressiveness {
    const fn threshold(self) -> f32 {
        match self {
            Self::Conservative => 4.0,
            Self::Balanced => 3.0,
            Self::Aggressive => 2.0,
        }
    }
}

/// An individual piece of evidence that a token is a name.
#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
pub enum NameSignal {
    /// The token (or a morphological base of it) is a known given name or surname.
    Gazetteer,
    /// No engine suggestion is close enough for the token to be a plausible typo.
    Orphan,
    /// The engine offered no correction at all.
    NoSuggestions,
    /// The same spelling occurs more than once in the document.
    Repetition,
    /// A title, salutation, possessive or citation pattern surrounds the token.
    Context,
    /// Capitalised somewhere other than the start of a sentence.
    Shape,
}

impl NameSignal {
    const fn weight(self) -> f32 {
        match self {
            Self::Gazetteer | Self::Orphan => 2.0,
            Self::NoSuggestions | Self::Context => 1.5,
            Self::Repetition | Self::Shape => 1.0,
        }
    }

    /// Short stable tag, surfaced in the inspector.
    #[must_use]
    pub const fn tag(self) -> &'static str {
        match self {
            Self::Gazetteer => "gazetteer",
            Self::Orphan => "orphan",
            Self::NoSuggestions => "no-suggestions",
            Self::Repetition => "repetition",
            Self::Context => "context",
            Self::Shape => "shape",
        }
    }
}

/// The outcome of examining one flagged token.
#[derive(Debug, Clone, Default)]
pub struct NameVerdict {
    pub is_name: bool,
    pub score: f32,
    pub signals: Vec<NameSignal>,
}

impl NameVerdict {
    /// Comma-separated signal tags, for the inspector payload.
    #[must_use]
    pub fn signal_tags(&self) -> String {
        self.signals
            .iter()
            .map(|s| s.tag())
            .collect::<Vec<_>>()
            .join(",")
    }
}

/// Everything the detector needs about one flagged token.
pub struct NameQuery<'a> {
    /// The flagged token, exactly as it appears in the document.
    pub token: &'a str,
    /// The full document text.
    pub text: &'a str,
    /// Byte offset of the token within `text`.
    pub start_byte: usize,
    /// Byte offset of the token end within `text`.
    pub end_byte: usize,
    /// The engine's proposed corrections.
    pub suggestions: &'a [String],
}

/// Titles that precede a name. Deliberately multilingual: the extension checks German,
/// French and Spanish documents too.
static HONORIFIC_BEFORE: LazyLock<Regex> = LazyLock::new(|| {
    Regex::new(
        r"(?ix)
        \b(
            mr | mrs | ms | miss | dr | prof(essor)? | sir | dame | lord | lady |
            rev | hon | capt | sgt | st |
            herr | frau | fr | fraeulein | fräulein |
            monsieur | madame | mme | mlle | m |
            senor | senora | señor | señora | sr | sra | srta |
            dott | ing
        )\.?\s+$",
    )
    .expect("valid honorific pattern")
});

/// Salutations and bylines: "Dear X", "Hi X", "by X".
static SALUTATION_BEFORE: LazyLock<Regex> = LazyLock::new(|| {
    Regex::new(
        r"(?ix)\b(dear|hi|hello|hey|attn|regards|sincerely|cc|by|von|van|de|del|della|der)\s+$",
    )
    .expect("valid salutation pattern")
});

/// Possessive or academic-citation shapes immediately following the token.
static CITATION_AFTER: LazyLock<Regex> = LazyLock::new(|| {
    Regex::new(r"(?ix)^( ['’]s\b | \s+et\s+al\b | \s*,\s*\d{4}\b | \s+\(\d{4}\) )")
        .expect("valid citation pattern")
});

/// A capitalised word immediately after the token — "Jon Sterling".
static CAPITALISED_AFTER: LazyLock<Regex> =
    LazyLock::new(|| Regex::new(r"^\s+\p{Lu}\p{Ll}+").expect("valid capitalised pattern"));

/// Detects human names among flagged tokens.
pub struct NameFilter {
    aggressiveness: Aggressiveness,
    /// Language tag the document is being checked as, e.g. `en-US`, `de-DE`.
    language: String,
}

impl NameFilter {
    #[must_use]
    pub fn new(aggressiveness: Aggressiveness, language: &str) -> Self {
        Self {
            aggressiveness,
            language: language.to_ascii_lowercase(),
        }
    }

    /// Whether capitalisation carries information in this language.
    ///
    /// German capitalises every noun, so "Bäcker" tells you nothing about whether the
    /// token is a surname or the word for baker. Giving shape a vote there would make
    /// the filter noticeably more eager on exactly the language where it should not be.
    fn capitalisation_is_informative(&self) -> bool {
        !self.language.starts_with("de")
    }

    /// Examine one flagged token.
    #[must_use]
    pub fn evaluate(&self, query: &NameQuery<'_>) -> NameVerdict {
        let mut signals = Vec::new();

        if gazetteer_contains(query.token) {
            signals.push(NameSignal::Gazetteer);
        }

        match min_suggestion_distance(query.token, query.suggestions) {
            None => signals.push(NameSignal::NoSuggestions),
            Some(distance) if distance >= 3 => signals.push(NameSignal::Orphan),
            Some(_) => {}
        }

        if occurrences(query.text, query.token) > 1 {
            signals.push(NameSignal::Repetition);
        }

        if self.has_name_context(query) {
            signals.push(NameSignal::Context);
        }

        if self.capitalisation_is_informative()
            && is_capitalised(query.token)
            && !starts_sentence(query.text, query.start_byte)
        {
            signals.push(NameSignal::Shape);
        }

        let score: f32 = signals.iter().map(|s| s.weight()).sum();
        // Two independent signals minimum. A gazetteer hit alone is not enough: the
        // name lists are crowdsourced and a lone hit on a lowercase token in running
        // text is far more likely to be a typo than a person.
        let is_name = signals.len() >= 2 && score >= self.aggressiveness.threshold();

        NameVerdict {
            is_name,
            score,
            signals,
        }
    }

    fn has_name_context(&self, query: &NameQuery<'_>) -> bool {
        let before = preceding_window(query.text, query.start_byte);
        let after = following_window(query.text, query.end_byte);

        HONORIFIC_BEFORE.is_match(before)
            || SALUTATION_BEFORE.is_match(before)
            || CITATION_AFTER.is_match(after)
            || (self.capitalisation_is_informative()
                && is_capitalised(query.token)
                && CAPITALISED_AFTER.is_match(after))
    }
}

/// Case-insensitive gazetteer lookup, trying morphological bases in turn.
fn gazetteer_contains(token: &str) -> bool {
    let Some(set) = GAZETTEER.as_ref() else {
        return false;
    };
    let lowered = token.to_lowercase();
    morphological_bases(&lowered)
        .into_iter()
        .any(|candidate| set.contains(candidate.as_bytes()))
}

/// The token plus the inflected forms a name commonly takes.
///
/// English possessives (`Merkel's`) and German genitives (`Merkels`, `Bachs`) would
/// otherwise miss a gazetteer that stores base forms only.
fn morphological_bases(lowered: &str) -> Vec<String> {
    let mut bases = vec![lowered.to_string()];

    for possessive in ["'s", "\u{2019}s"] {
        if let Some(stem) = lowered.strip_suffix(possessive)
            && stem.len() >= 2
        {
            bases.push(stem.to_string());
        }
    }
    // Genitive/plural -s, and the German weak-declension -n / -en.
    for suffix in ["s", "en", "n"] {
        if let Some(stem) = lowered.strip_suffix(suffix)
            && stem.len() >= 3
        {
            bases.push(stem.to_string());
        }
    }

    bases
}

/// Smallest edit distance between the token and any single-word suggestion.
///
/// Returns `None` when the engine offered nothing usable. Suggestions containing
/// whitespace are ignored: `LanguageTool` answers `Abramsky` with `Abram sky`, a word-split
/// proposal that is one edit away by character count but is not evidence that the token
/// is a misspelling of a known word.
fn min_suggestion_distance(token: &str, suggestions: &[String]) -> Option<usize> {
    let lowered = token.to_lowercase();
    suggestions
        .iter()
        .filter(|s| !s.chars().any(char::is_whitespace))
        .map(|s| strsim::damerau_levenshtein(&lowered, &s.to_lowercase()))
        .min()
}

/// How many times `token` appears in `text` as a whole word.
///
/// Names recur; typos are usually one-off. Counting stops at 2 because that is all the
/// signal needs, which keeps this cheap on large documents.
fn occurrences(text: &str, token: &str) -> usize {
    if token.is_empty() {
        return 0;
    }
    let mut count = 0;
    let mut cursor = 0;
    while let Some(found) = text[cursor..].find(token) {
        let start = cursor + found;
        let end = start + token.len();
        let before_ok = start == 0
            || !text[..start]
                .chars()
                .next_back()
                .is_some_and(char::is_alphanumeric);
        let after_ok = end >= text.len()
            || !text[end..]
                .chars()
                .next()
                .is_some_and(char::is_alphanumeric);
        if before_ok && after_ok {
            count += 1;
            if count > 1 {
                return count;
            }
        }
        cursor = end.max(start + 1);
        if cursor >= text.len() {
            break;
        }
    }
    count
}

fn is_capitalised(token: &str) -> bool {
    token.chars().next().is_some_and(char::is_uppercase)
}

/// Whether the token at `start` opens a sentence.
///
/// Sentence-initial capitalisation is not evidence of a name, so the shape signal has to
/// exclude it.
fn starts_sentence(text: &str, start: usize) -> bool {
    let preceding = text[..start.min(text.len())].trim_end();
    preceding.chars().next_back().is_none_or(|c| {
        matches!(
            c,
            '.' | '!' | '?' | ':' | ';' | '\n' | '"' | '\'' | '(' | '['
        )
    })
}

const WINDOW: usize = 48;

fn preceding_window(text: &str, start: usize) -> &str {
    let start = start.min(text.len());
    let lo = text.floor_char_boundary(start.saturating_sub(WINDOW));
    &text[lo..start]
}

fn following_window(text: &str, end: usize) -> &str {
    let end = end.min(text.len());
    let hi = text.ceil_char_boundary((end + WINDOW).min(text.len()));
    &text[end..hi]
}

#[cfg(test)]
mod tests;