zeppelin-embed 0.4.2

//! Versioned character folding: the normalization half of the epoch.
//!
//! # Task 12 D3 — full NFKC folding, not a narrow table
//!
//! The plan originally shipped a narrow hand-written folding table because
//! the static-library budget was scarce. The owner raised the gate to 5 MB
//! on 2026-08-23 and set the cardinal rule — best and fastest lexical search
//! — which reversed D3 to "vendor the full tables". This module implements
//! the reversed decision.
//!
//! The deciding ground is retrieval quality, not bytes. A partial folding
//! table is precisely the quiet tokenizer defect the research blames for
//! quality gaps: diacritics, width variants, and compatibility forms must
//! normalize the way Lucene's analyzers do, or the loss lands before scoring
//! ever starts (`research/02a:265`, `:269`). The measured cost of being
//! right here is 87 KB of table against ~3.5 MB of headroom.
//!
//! # What the fold is
//!
//! One lookup per codepoint, in Lucene's `ICUFoldingFilter` shape: NFKC,
//! then Unicode case folding, then canonical decomposition with nonspacing
//! marks removed, then recomposition. Collapsing all four into a single
//! precomputed mapping is what makes it fast — no runtime canonical
//! ordering, no composition pass, no decomposition buffer. `Café`, `CAFÉ`,
//! `café`, `ｃａｆｅ`, and `cafe\u{301}` all reach `cafe`.
//!
//! `fold_table.rs` is generated by `tools/generate-fold-table.py` from the
//! Unicode Character Database and committed. It is not a build script;
//! build scripts and host probing are prohibited. Regenerating it against a
//! newer Unicode version changes the meaning of every existing index, so it
//! is a deliberate epoch bump that task 21 migrates — never a quiet fix.
//!
//! # The one deliberate departure from NFKC
//!
//! NFKC does not map curly quotation marks or Unicode hyphens onto their
//! ASCII forms, so under pure NFKC `don’t` and `don't` are different terms,
//! and `i‑485` typed with a non-breaking hyphen misses `i-485`. Search
//! engines fix this with a mapping filter ahead of normalization; Lucene
//! ships `MappingCharFilter` for exactly this. `PUNCTUATION_EQUIVALENTS`
//! is that filter, kept explicit and tiny so it is auditable, and it is an
//! epoch-digest input like everything else here.

use super::fold_table::{FOLD_BLOB, FOLD_KEYS, FOLD_OFFSETS, UNICODE_VERSION};

/// The folding rule revision, an epoch-digest input.
///
/// Bump this whenever the fold changes for any input, including a Unicode
/// version change in the generated table.
pub(crate) const FOLDING_TABLE_VERSION: u16 = 2;

/// The Unicode version the generated table was built from.
pub(crate) const FOLDING_UNICODE_VERSION: &str = UNICODE_VERSION;

/// Punctuation treated as equivalent before segmentation and folding.
///
/// Deliberately beyond NFKC; see the module docs.
const PUNCTUATION_EQUIVALENTS: [(char, char); 8] = [
    ('\u{2018}', '\''), // LEFT SINGLE QUOTATION MARK
    ('\u{2019}', '\''), // RIGHT SINGLE QUOTATION MARK
    ('\u{02BC}', '\''), // MODIFIER LETTER APOSTROPHE
    ('\u{FF07}', '\''), // FULLWIDTH APOSTROPHE
    ('\u{2010}', '-'),  // HYPHEN
    ('\u{2011}', '-'),  // NON-BREAKING HYPHEN
    ('\u{FE63}', '-'),  // SMALL HYPHEN-MINUS
    ('\u{FF0D}', '-'),  // FULLWIDTH HYPHEN-MINUS
];

/// Applies the punctuation-equivalence filter to one character.
pub(crate) fn punctuation_equivalent(value: char) -> char {
    let mut index = 0;
    while index < PUNCTUATION_EQUIVALENTS.len() {
        match PUNCTUATION_EQUIVALENTS.get(index) {
            Some((from, to)) if *from == value => return *to,
            _ => index += 1,
        }
    }
    value
}

/// Looks up the generated fold for one codepoint.
///
/// Returns `None` when the codepoint folds to itself, which is the common
/// case and needs no allocation or copy beyond the character itself.
fn table_fold(value: char) -> Option<&'static str> {
    let code = value as u32;
    let position = FOLD_KEYS.binary_search(&code).ok()?;
    let start = usize::try_from(*FOLD_OFFSETS.get(position)?).ok()?;
    let end = usize::try_from(*FOLD_OFFSETS.get(position + 1)?).ok()?;
    FOLD_BLOB.get(start..end)
}

/// Folds one character into `output`.
///
/// A character whose fold is empty — a combining mark — appends nothing.
pub(crate) fn fold_char(value: char, output: &mut String) {
    let value = punctuation_equivalent(value);
    // ASCII is the overwhelmingly common case and needs no table probe.
    if value.is_ascii() {
        output.push(value.to_ascii_lowercase());
        return;
    }
    match table_fold(value) {
        Some(folded) => output.push_str(folded),
        None => output.push(value),
    }
}

/// Folds a whole string.
#[must_use]
pub(crate) fn fold(input: &str) -> String {
    let mut output = String::with_capacity(input.len());
    for value in input.chars() {
        fold_char(value, &mut output);
    }
    output
}

/// Returns the character a segmenter should classify `value` as.
///
/// Segmentation runs on the original bytes so offsets stay exact, but it
/// must agree with folding about which characters are equivalent: a curly
/// apostrophe has to join `don’t` exactly as a straight one joins `don't`,
/// and a fullwidth digit has to count as alphanumeric.
pub(crate) fn segmentation_equivalent(value: char) -> char {
    let value = punctuation_equivalent(value);
    if value.is_ascii() {
        return value;
    }
    match table_fold(value) {
        Some(folded) => {
            let mut characters = folded.chars();
            match (characters.next(), characters.next()) {
                // A single-character fold stands in for the original.
                (Some(single), None) => single,
                // Multi-character folds (ß -> ss) and empty folds (marks)
                // keep the original for classification purposes; the
                // original is already alphanumeric or already a mark.
                _ => value,
            }
        }
        None => value,
    }
}

#[cfg(test)]
#[allow(clippy::expect_used, clippy::indexing_slicing, clippy::unwrap_used)]
mod tests {
    use super::*;

    #[test]
    fn case_folds_to_lowercase() {
        assert_eq!(fold("HELLO World"), "hello world");
        assert_eq!(fold("\u{0410}\u{0411}"), "\u{0430}\u{0431}");
    }

    #[test]
    fn diacritics_fold_to_their_ascii_base() {
        assert_eq!(fold("Café"), "cafe");
        assert_eq!(fold("naïve"), "naive");
        assert_eq!(fold("ÄÖÜ"), "aou");
        assert_eq!(fold("straße"), "strasse");
        assert_eq!(fold("Ångström"), "angstrom");
    }

    #[test]
    fn decomposed_and_precomposed_diacritics_fold_alike() {
        assert_eq!(fold("cafe\u{0301}"), fold("caf\u{00E9}"));
        assert_eq!(fold("cafe\u{0301}"), "cafe");
    }

    #[test]
    fn compatibility_forms_fold_to_ascii() {
        assert_eq!(fold("\u{FB01}nd"), "find");
        assert_eq!(fold("\u{FF21}\u{FF22}"), "ab");
        assert_eq!(fold("x\u{00B2}"), "x2");
        assert_eq!(fold("\u{2168}"), "ix");
    }

    #[test]
    fn punctuation_equivalents_collapse_beyond_nfkc() {
        assert_eq!(fold("don\u{2019}t"), fold("don't"));
        assert_eq!(fold("i\u{2011}485"), fold("i-485"));
        assert_eq!(segmentation_equivalent('\u{2019}'), '\'');
        assert_eq!(segmentation_equivalent('\u{2011}'), '-');
    }

    #[test]
    fn fullwidth_digits_stay_alphanumeric_for_segmentation() {
        assert!(segmentation_equivalent('\u{FF15}').is_alphanumeric());
        assert_eq!(fold("\u{FF15}"), "5");
    }

    #[test]
    fn combining_marks_fold_away() {
        assert_eq!(fold("\u{0301}"), "");
        assert_eq!(fold("\u{0300}\u{0302}"), "");
    }

    #[test]
    fn ideographs_are_left_alone() {
        assert_eq!(fold("\u{4E2D}\u{6587}"), "\u{4E2D}\u{6587}");
    }

    #[test]
    fn folding_is_idempotent() {
        for sample in [
            "Café",
            "straße",
            "\u{FB01}nd",
            "HELLO",
            "\u{FF21}\u{FF22}",
            "don\u{2019}t",
            "\u{4E2D}\u{6587}",
            "cafe\u{0301}",
            "\u{2168}",
        ] {
            let once = fold(sample);
            assert_eq!(fold(&once), once, "fold is not stable on {sample:?}");
        }
    }

    #[test]
    fn the_generated_table_is_sorted_and_addressable() {
        assert!(
            FOLD_KEYS
                .windows(2)
                .all(|pair| match (pair.first(), pair.get(1)) {
                    (Some(left), Some(right)) => left < right,
                    _ => true,
                }),
            "FOLD_KEYS must be strictly ascending for binary search"
        );
        assert_eq!(FOLD_OFFSETS.len(), FOLD_KEYS.len() + 1);
        for index in 0..FOLD_KEYS.len() {
            let key = *FOLD_KEYS.get(index).expect("in range");
            let value = char::from_u32(key).expect("table keys are scalar values");
            assert!(
                table_fold(value).is_some(),
                "entry {key:#X} is unaddressable"
            );
        }
    }

    #[test]
    fn the_unicode_version_is_pinned() {
        assert_eq!(FOLDING_UNICODE_VERSION, "14.0.0");
    }
}