uqa-analysis 0.3.5

//
// Unified Query Algebra
//
// Copyright (c) 2023-2026 Cognica, Inc.
//

//! Character-level filters that run before tokenization.

use std::collections::BTreeMap;

use serde::{Deserialize, Serialize};

use crate::error::AnalysisResult;
use crate::FilteredText;
use uqa_core::memory::MemoryBudget;

mod compiled;
#[cfg(feature = "kuromoji")]
mod iteration;
mod replacement;
mod stream;
mod width;
pub(crate) use compiled::PreparedCharFilter;

#[derive(Debug, Clone, Serialize, Deserialize)]
#[serde(tag = "type", rename_all = "snake_case")]
pub enum CharFilter {
    // The alias keeps catalogs persisted before the stable tag existed
    // deserializable: releases up to 0.1.2 wrote the derived spelling.
    #[serde(rename = "html_strip", alias = "h_t_m_l_strip")]
    HTMLStrip,
    /// Fold fullwidth ASCII and halfwidth Katakana, preserving original source spans.
    ///
    /// ```
    /// use uqa_analysis::CharFilter;
    /// let filtered = CharFilter::CJKWidth.filter_with_offsets("ｶﾞＡ①")?;
    /// assert_eq!(filtered.as_str(), "ガA①");
    /// assert_eq!(filtered.source_offsets(0..3)?.utf16, 0..2);
    /// # Ok::<(), uqa_analysis::AnalysisError>(())
    /// ```
    #[serde(rename = "cjk_width")]
    CJKWidth,
    /// Expand Japanese horizontal iteration marks while retaining original source coordinates.
    ///
    /// ```
    /// use uqa_analysis::CharFilter;
    /// let filter = CharFilter::KuromojiIterationMark { normalize_kanji: true, normalize_kana: true };
    /// assert_eq!(filter.filter("時々 なゝ 🙂々")?, "時時 など 🙂々");
    /// # Ok::<(), uqa_analysis::AnalysisError>(())
    /// ```
    #[cfg(feature = "kuromoji")]
    #[serde(rename = "kuromoji_iteration_mark")]
    KuromojiIterationMark {
        #[serde(default = "default_iteration_normalization")]
        normalize_kanji: bool,
        #[serde(default = "default_iteration_normalization")]
        normalize_kana: bool,
    },
    Mapping {
        mapping: BTreeMap<String, String>,
    },
    PatternReplace {
        pattern: String,
        #[serde(default)]
        replacement: String,
    },
}

#[cfg(feature = "kuromoji")]
fn default_iteration_normalization() -> bool {
    true
}

impl CharFilter {
    /// Validate configuration without filtering input.
    pub fn validate(&self) -> AnalysisResult<()> {
        match self {
            CharFilter::PatternReplace { .. } => self.prepare().map(|_| ()),
            _ => Ok(()),
        }
    }

    pub fn filter(&self, text: &str) -> AnalysisResult<String> {
        Ok(self.filter_with_offsets(text)?.into_string())
    }

    /// Transform text while retaining source coordinates for the result.
    pub fn filter_with_offsets<'a>(&self, text: &'a str) -> AnalysisResult<FilteredText<'a>> {
        self.filter_mapped(FilteredText::new(text))
    }

    /// Apply this stage to previously filtered text without losing its original source.
    pub fn filter_mapped<'a>(&self, text: FilteredText<'a>) -> AnalysisResult<FilteredText<'a>> {
        self.prepare()?.filter_mapped(text)
    }

    /// Transform a borrowed input while retaining source buffers and regex search workspace under the caller's byte allowance. Immutable configuration preparation is separate. Literal, built-in HTML, regex range and capture searches, source copying, and coordinate construction poll during execution. Search scratch is released before returning the retained source result.
    ///
    /// ```
    /// use uqa_analysis::CharFilter;
    /// use uqa_core::memory::MemoryBudget;
    /// let budget = MemoryBudget::new(16 * 1024);
    /// let filtered = CharFilter::HTMLStrip.filter_with_offsets_budgeted(
    ///     "<b>한&amp;🙂</b>", &budget, &mut || Ok(()),
    /// )?;
    /// let retained = filtered.clone();
    /// drop(filtered);
    /// assert_eq!(retained.as_str(), " 한&🙂 ");
    /// assert_eq!(retained.source_offsets(1..4)?.utf8, 3..6);
    /// assert!(budget.used() > 0);
    /// drop(retained);
    /// assert_eq!(budget.used(), 0);
    /// # Ok::<(), uqa_analysis::AnalysisError>(())
    /// ```
    pub fn filter_with_offsets_budgeted<'a>(
        &self,
        text: &'a str,
        budget: &MemoryBudget,
        poll: &mut dyn FnMut() -> AnalysisResult<()>,
    ) -> AnalysisResult<FilteredText<'a>> {
        self.filter_mapped_budgeted(FilteredText::new(text), budget, poll)
    }

    /// New source, edit, and coordinate buffers use `budget`; retained input allocations keep their original shared leases.
    pub fn filter_mapped_budgeted<'a>(
        &self,
        text: FilteredText<'a>,
        budget: &MemoryBudget,
        poll: &mut dyn FnMut() -> AnalysisResult<()>,
    ) -> AnalysisResult<FilteredText<'a>> {
        poll()?;
        self.prepare()?.filter_mapped_budgeted(text, budget, poll)
    }
}

const HTML_ENTITIES: &[(&str, &str)] = &[
    ("&amp;", "&"),
    ("&lt;", "<"),
    ("&gt;", ">"),
    ("&quot;", "\""),
    ("&#39;", "'"),
    ("&apos;", "'"),
    ("&nbsp;", " "),
];

/// Order mapping entries longest-key-first so that, e.g., the rule
/// `aa -> X` fires before `a -> Y`.
fn mapping_longest_first(m: &BTreeMap<String, String>) -> Vec<(String, String)> {
    let mut entries: Vec<(String, String)> =
        m.iter().map(|(k, v)| (k.clone(), v.clone())).collect();
    entries.sort_by(|a, b| b.0.len().cmp(&a.0.len()).then_with(|| a.0.cmp(&b.0)));
    entries
}

#[cfg(test)]
mod tests {
    use super::*;

    #[cfg(not(feature = "kuromoji"))]
    #[test]
    fn japanese_iteration_marks_require_the_kuromoji_feature() {
        assert!(
            serde_json::from_str::<CharFilter>(r#"{"type":"kuromoji_iteration_mark"}"#).is_err()
        );
    }

    #[test]
    fn html_strip_removes_tags_and_decodes_entities() {
        let f = CharFilter::HTMLStrip;
        assert_eq!(
            f.filter("<p>hello &amp; world</p>").unwrap(),
            " hello & world ".to_string()
        );
    }

    #[test]
    fn mapping_replaces_longest_first() {
        // Longest-first ordering: `aa` consumes the prefix before the
        // single-`a` rule sees it, leaving nothing for the second rule.
        // Without longest-first ordering the single-char rule would fire
        // twice and produce "YYb".
        let mut m = BTreeMap::new();
        m.insert("aa".to_string(), "X".to_string());
        m.insert("a".to_string(), "Y".to_string());
        let f = CharFilter::Mapping { mapping: m };
        assert_eq!(f.filter("aab").unwrap(), "Xb");

        // A 'a' that wasn't in the longer rule's match still gets replaced
        // by the shorter rule.
        assert_eq!(f.filter("aba").unwrap(), "YbY");
    }

    #[test]
    fn pattern_replace_uses_regex() {
        let f = CharFilter::PatternReplace {
            pattern: r"\d+".to_string(),
            replacement: "#".to_string(),
        };
        assert_eq!(f.filter("a1b22c").unwrap(), "a#b#c");
    }
}