lang_check/

suppression.rs

//! Shared post-check diagnostic suppression.
//!
//! Every entry point (protobuf server, LSP, CLI, background indexer) runs the same
//! suppression pass once a diagnostic has been produced. Keeping it here means the four
//! paths cannot drift apart — previously the user-dictionary check existed only in the
//! server and LSP paths, so `language-check check` reported spelling errors for words the
//! user had explicitly whitelisted.
//!
//! Ordering requirements for callers:
//! - diagnostic offsets must already be rebased to **document** coordinates, so `text` is
//!   the whole document and surrounding context is visible;
//! - `unified_id` must already be populated (only [`crate::orchestrator::Orchestrator`]
//!   does that), since the spelling check keys on it.

use tracing::debug;

use crate::checker::Diagnostic;
use crate::dictionary::Dictionary;
use crate::hashing::{DiagnosticFingerprint, IgnoreStore};
use crate::ignore_rules::{IgnoreParser, ResolvedDirectives};
use crate::morphology::AffixAnalyzer;
use crate::names::{NameFilter, NameQuery, NameVerdict};
use crate::prose::is_spelling_category;
use crate::text_util::{min_suggestion_distance, safe_slice};

/// The suppression sources available at a given call site.
///
/// Each is optional because the entry points differ in what they have: the CLI has no
/// ignore store, and the name filter is only present when the user opts in.
#[derive(Default, Clone, Copy)]
pub struct SuppressionContext<'a> {
    pub ignore: Option<&'a IgnoreStore>,
    pub dictionary: Option<&'a Dictionary>,
    pub morphology: Option<&'a AffixAnalyzer>,
    pub names: Option<&'a NameFilter>,
    pub directives: Option<&'a InlineDirectives>,
}

/// Inline `lang-check-*` directives resolved from a whole document.
///
/// Resolved once per document and shared by every diagnostic, because the
/// directives live in comments that the prose extractors strip: a range of
/// extracted prose never contains the `lang-check-begin` that governs it, and
/// its offsets are range-local. Both only line up at the call sites that hold
/// the document, which is where [`retain_visible`] runs.
#[derive(Debug, Clone, Default)]
pub struct InlineDirectives {
    resolved: ResolvedDirectives,
}

impl InlineDirectives {
    #[must_use]
    pub fn parse(text: &str) -> Self {
        let directives = IgnoreParser::parse_directives(text);
        Self {
            resolved: IgnoreParser::resolve_all(text, &directives),
        }
    }

    /// Whether anything was found, so callers can skip the per-diagnostic work.
    #[must_use]
    pub const fn is_empty(&self) -> bool {
        self.resolved.ignore_ranges.is_empty() && self.resolved.regions.is_empty()
    }

    /// Whether `diagnostic` falls inside a suppressing directive.
    ///
    /// `text` is the document the directives were parsed from, and the
    /// diagnostic's offsets must index into it.
    #[must_use]
    pub fn suppresses(&self, diagnostic: &Diagnostic, text: &str) -> bool {
        IgnoreParser::should_ignore(diagnostic, &self.resolved.ignore_ranges)
            || IgnoreParser::should_ignore_by_region(diagnostic, text, &self.resolved.regions)
    }
}

impl<'a> SuppressionContext<'a> {
    #[must_use]
    pub const fn new() -> Self {
        Self {
            ignore: None,
            dictionary: None,
            morphology: None,
            names: None,
            directives: None,
        }
    }

    #[must_use]
    pub const fn with_ignore(mut self, ignore: &'a IgnoreStore) -> Self {
        self.ignore = Some(ignore);
        self
    }

    #[must_use]
    pub const fn with_dictionary(mut self, dictionary: &'a Dictionary) -> Self {
        self.dictionary = Some(dictionary);
        self
    }

    #[must_use]
    pub const fn with_morphology(mut self, morphology: &'a AffixAnalyzer) -> Self {
        self.morphology = Some(morphology);
        self
    }

    #[must_use]
    pub const fn with_names(mut self, names: &'a NameFilter) -> Self {
        self.names = Some(names);
        self
    }

    #[must_use]
    pub const fn with_directives(mut self, directives: &'a InlineDirectives) -> Self {
        self.directives = Some(directives);
        self
    }
}

/// What the suppression pass decided about one diagnostic.
enum Outcome {
    /// Show it.
    Keep,
    /// Drop it (ignore store or user dictionary).
    Drop,
    /// Drop it because the token is a human name.
    DropAsName(NameVerdict),
}

/// The single decision point. Both public entry points route through this so the four
/// call sites cannot diverge again.
fn classify(diagnostic: &Diagnostic, text: &str, ctx: &SuppressionContext<'_>) -> Outcome {
    if let Some(directives) = ctx.directives
        && !directives.is_empty()
        && directives.suppresses(diagnostic, text)
    {
        return Outcome::Drop;
    }

    if let Some(ignore) = ctx.ignore {
        let fingerprint = DiagnosticFingerprint::new(
            &diagnostic.message,
            text,
            diagnostic.start_byte as usize,
            diagnostic.end_byte as usize,
        );
        if ignore.is_ignored(&fingerprint) {
            return Outcome::Drop;
        }
    }

    if !is_spelling_category(&diagnostic.unified_id) {
        return Outcome::Keep;
    }

    let word = safe_slice(
        text,
        diagnostic.start_byte as usize,
        diagnostic.end_byte as usize,
    );

    if let Some(dictionary) = ctx.dictionary
        && dictionary.contains(word)
    {
        return Outcome::Drop;
    }

    if is_known_derivation(word, diagnostic, ctx) {
        return Outcome::Drop;
    }

    let Some(filter) = ctx.names else {
        return Outcome::Keep;
    };
    let verdict = filter.evaluate(&NameQuery {
        token: word,
        text,
        start_byte: diagnostic.start_byte as usize,
        end_byte: diagnostic.end_byte as usize,
        suggestions: &diagnostic.suggestions,
    });
    if verdict.is_name {
        Outcome::DropAsName(verdict)
    } else {
        Outcome::Keep
    }
}

/// Whether the token is an affixed form of known material, and not a typo.
///
/// Decomposition alone is not enough. Measured over single-edit misspellings of common
/// English words it accepts about 1% of them — `untill` as `un` + `till`, `intergrate`
/// as `inter` + `grate` — and every one of those has its correction sitting in the
/// engine's own suggestion list one edit away. A token that close to a real word is a
/// slip, whatever else it also parses as, so the suggestions veto the analysis.
///
/// No suggestions at all is the opposite evidence: the engine could think of nothing
/// this might have been, which is what a genuinely novel coinage looks like.
fn is_known_derivation(word: &str, diagnostic: &Diagnostic, ctx: &SuppressionContext<'_>) -> bool {
    let Some(analyzer) = ctx.morphology else {
        return false;
    };
    if min_suggestion_distance(word, &diagnostic.suggestions).is_some_and(|d| d <= 1) {
        return false;
    }
    let Some(analysis) = analyzer.analyze(word, ctx.dictionary) else {
        return false;
    };
    // Suppression is otherwise silent, and a feature that hides squiggles without
    // saying why cannot be debugged from a bug report.
    debug!(word, decomposition = %analysis.describe(), "Suppressed as an affixed form");
    true
}

/// Whether `diagnostic` should be dropped before reaching the user.
///
/// `text` is the full document; `diagnostic`'s offsets must index into it.
#[must_use]
pub fn should_suppress(diagnostic: &Diagnostic, text: &str, ctx: &SuppressionContext<'_>) -> bool {
    !matches!(classify(diagnostic, text, ctx), Outcome::Keep)
}

/// A span the name filter recognised, for the inspector.
#[derive(Debug, Clone)]
pub struct DetectedName {
    pub start_byte: u32,
    pub end_byte: u32,
    pub confidence: f32,
    pub signals: String,
}

/// Drop suppressed diagnostics, reporting any dropped because they were names.
///
/// The report exists so the inspector can show exactly what the filter silenced and on
/// what evidence — without it the feature is invisible and undebuggable. Call sites that
/// don't surface it simply discard the return value.
pub fn retain_visible(
    diagnostics: &mut Vec<Diagnostic>,
    text: &str,
    ctx: &SuppressionContext<'_>,
) -> Vec<DetectedName> {
    let mut detected = Vec::new();
    diagnostics.retain(|d| match classify(d, text, ctx) {
        Outcome::Keep => true,
        Outcome::Drop => false,
        Outcome::DropAsName(verdict) => {
            detected.push(DetectedName {
                start_byte: d.start_byte,
                end_byte: d.end_byte,
                confidence: verdict.score,
                signals: verdict.signal_tags(),
            });
            false
        }
    });
    detected
}

#[cfg(test)]
mod tests {
    use super::*;

    fn spelling_diagnostic(start: u32, end: u32) -> Diagnostic {
        Diagnostic {
            start_byte: start,
            end_byte: end,
            message: "Possible spelling mistake found.".to_string(),
            suggestions: vec![],
            rule_id: "languagetool.MORFOLOGIK_RULE_EN_US".to_string(),
            severity: 2,
            unified_id: "spelling.typo".to_string(),
            confidence: 1.0,
            language: String::new(),
            pack_installable: false,
        }
    }

    /// A spelling diagnostic carrying the corrections an engine would actually offer.
    fn with_suggestions(start: u32, end: u32, suggestions: &[&str]) -> Diagnostic {
        Diagnostic {
            suggestions: suggestions.iter().map(|s| (*s).to_string()).collect(),
            ..spelling_diagnostic(start, end)
        }
    }

    #[test]
    fn an_affixed_form_of_a_known_word_is_suppressed() {
        let text = "every subalgebra here";
        let analyzer = AffixAnalyzer::new("en-US");
        let ctx = SuppressionContext::new().with_morphology(&analyzer);
        assert!(should_suppress(&spelling_diagnostic(6, 16), text, &ctx));
    }

    #[test]
    fn morphology_is_inert_until_it_is_supplied() {
        let text = "every subalgebra here";
        assert!(!should_suppress(
            &spelling_diagnostic(6, 16),
            text,
            &SuppressionContext::new()
        ));
    }

    #[test]
    fn a_close_suggestion_vetoes_the_decomposition() {
        // `untill` parses as un + till, and `till` really is a word. The engine's own
        // correction, one edit away, is what says otherwise.
        let text = "wait untill then";
        let analyzer = AffixAnalyzer::new("en-US");
        let ctx = SuppressionContext::new().with_morphology(&analyzer);
        assert!(!should_suppress(
            &with_suggestions(5, 11, &["until"]),
            text,
            &ctx
        ));
        // Without that evidence the same token is accepted, which is exactly why the
        // guard is not optional.
        assert!(should_suppress(&spelling_diagnostic(5, 11), text, &ctx));
    }

    #[test]
    fn a_distant_suggestion_does_not_veto() {
        // Engines answer novel coinages with something, but not something close.
        let text = "the subadditivity holds";
        let analyzer = AffixAnalyzer::new("en-US");
        let ctx = SuppressionContext::new().with_morphology(&analyzer);
        assert!(should_suppress(
            &with_suggestions(4, 17, &["subjectivity"]),
            text,
            &ctx
        ));
    }

    #[test]
    fn morphology_does_not_touch_non_spelling_diagnostics() {
        let text = "every subalgebra here";
        let analyzer = AffixAnalyzer::new("en-US");
        let ctx = SuppressionContext::new().with_morphology(&analyzer);
        let mut d = spelling_diagnostic(6, 16);
        d.unified_id = "grammar.agreement".to_string();
        assert!(!should_suppress(&d, text, &ctx));
    }

    #[test]
    fn empty_context_suppresses_nothing() {
        let text = "Ackermann wrote this.";
        let d = spelling_diagnostic(0, 9);
        assert!(!should_suppress(&d, text, &SuppressionContext::new()));
    }

    #[test]
    fn dictionary_word_is_suppressed() {
        let text = "Ackermann wrote this.";
        let mut dict = Dictionary::new();
        dict.add_word("ackermann").unwrap();
        let ctx = SuppressionContext::new().with_dictionary(&dict);
        assert!(should_suppress(&spelling_diagnostic(0, 9), text, &ctx));
    }

    #[test]
    fn dictionary_lookup_is_case_insensitive() {
        let text = "ACKERMANN wrote this.";
        let mut dict = Dictionary::new();
        dict.add_word("Ackermann").unwrap();
        let ctx = SuppressionContext::new().with_dictionary(&dict);
        assert!(should_suppress(&spelling_diagnostic(0, 9), text, &ctx));
    }

    #[test]
    fn dictionary_does_not_suppress_non_spelling_diagnostics() {
        let text = "Ackermann wrote this.";
        let mut dict = Dictionary::new();
        dict.add_word("ackermann").unwrap();
        let mut d = spelling_diagnostic(0, 9);
        d.unified_id = "grammar.agreement".to_string();
        let ctx = SuppressionContext::new().with_dictionary(&dict);
        assert!(!should_suppress(&d, text, &ctx));
    }

    #[test]
    fn multibyte_spans_do_not_panic() {
        let text = "Grüße von Müller.";
        let mut dict = Dictionary::new();
        dict.add_word("müller").unwrap();
        let start = text.find("Müller").unwrap() as u32;
        let ctx = SuppressionContext::new().with_dictionary(&dict);
        // "Müller" is 7 bytes because of the umlaut.
        assert!(should_suppress(
            &spelling_diagnostic(start, start + 7),
            text,
            &ctx
        ));
    }

    #[test]
    fn detected_names_are_reported_for_the_inspector() {
        use crate::names::{Aggressiveness, NameFilter};

        let text = "The logic of Hoare is central.";
        let filter = NameFilter::new(Aggressiveness::Balanced, "en-US");
        let ctx = SuppressionContext::new().with_names(&filter);

        let start = text.find("Hoare").unwrap() as u32;
        let mut d = spelling_diagnostic(start, start + 5);
        d.suggestions = vec!["Hare".to_string(), "Hoar".to_string()];
        let mut diagnostics = vec![d];

        let detected = retain_visible(&mut diagnostics, text, &ctx);

        assert!(diagnostics.is_empty(), "the name should have been dropped");
        assert_eq!(detected.len(), 1);
        assert_eq!(detected[0].start_byte, start);
        assert_eq!(detected[0].end_byte, start + 5);
        assert!(detected[0].confidence > 0.0);
        assert!(
            detected[0].signals.contains("gazetteer"),
            "signals were {}",
            detected[0].signals
        );
    }

    #[test]
    fn nothing_is_reported_without_a_name_filter() {
        let text = "The logic of Hoare is central.";
        let start = text.find("Hoare").unwrap() as u32;
        let mut diagnostics = vec![spelling_diagnostic(start, start + 5)];
        let detected = retain_visible(&mut diagnostics, text, &SuppressionContext::new());
        assert_eq!(diagnostics.len(), 1, "opt-in: must stay flagged");
        assert!(detected.is_empty());
    }

    #[test]
    fn retain_visible_drops_only_suppressed() {
        let text = "Ackermann met Hoare.";
        let mut dict = Dictionary::new();
        dict.add_word("ackermann").unwrap();
        let ctx = SuppressionContext::new().with_dictionary(&dict);

        let hoare_start = text.find("Hoare").unwrap() as u32;
        let mut diagnostics = vec![
            spelling_diagnostic(0, 9),
            spelling_diagnostic(hoare_start, hoare_start + 5),
        ];
        retain_visible(&mut diagnostics, text, &ctx);

        assert_eq!(diagnostics.len(), 1);
        assert_eq!(diagnostics[0].start_byte, hoare_start);
    }
}