relon_unicode/

full_case_folding.rs

//! v3++ b-6 full Unicode case folding (UAX #21).
//!
//! Extends the v3+ a-4 simple folding (1:1 codepoint mappings in
//! [`super::case_folding`]) with three additional rules drawn from
//! UCD 14.0.0:
//!
//!   * **Multi-codepoint unconditional mappings** — `ß` -> `SS`, the
//!     Latin / Armenian ligatures (`fi` -> `FI`, …), and the Greek
//!     letters with combining marks pre-applied (`ΐ`, `ΰ`, …). Tables
//!     live in [`FULL_UPPER_FOLDING`] and [`FULL_LOWER_FOLDING`],
//!     generated by `tools/gen_full_case_folding.py` from
//!     `data/SpecialCasing.txt`.
//!
//!   * **Greek final sigma context** — `Σ` (U+03A3) lowercases to
//!     `ς` (U+03C2) when it appears at the end of a word (no following
//!     cased codepoint, ignoring case-ignorable codepoints in between),
//!     else `σ` (U+03C3). The right-scan uses [`is_cased`] and
//!     [`is_case_ignorable`], whose backing ranges come from
//!     `data/DerivedCoreProperties.txt`.
//!
//!   * **Turkish / Azerbaijani locale overrides** — the `I`/`İ` /
//!     `ı`/`i` quartet behaves differently in `tr` and `az` locales
//!     (no-dot dotless-i conventions). [`TURKISH_LOWER_FOLDING`] and
//!     [`TURKISH_UPPER_FOLDING`] hold the four override entries; the
//!     locale-aware stdlib bodies fall back to the default tables on
//!     a miss.
//!
//! ### Data layout (wire format for the wasm-AOT data section)
//!
//! Each FULL entry is `(input_cp: u32, out0: u32, out1: u32, out2: u32,
//! out_len: u8)` — 20 bytes serialised, with the trailing `out_len`
//! padded to a 4-byte stride for natural alignment. Up-to-3 codepoint
//! outputs cover every SpecialCasing.txt unconditional entry (the
//! longest is 3 cps, e.g. `U+0390` -> `0399 0308 0301`).
//!
//! Ranges (`CASED_RANGES`, `CASE_IGNORABLE_RANGES`) use the same
//! `(start, end)` inclusive shape as the existing combining-mark /
//! whitespace tables — runtime binary search reuses the same arithmetic
//! against an 8-byte stride.

// The build script-style generated file lives next to this module so
// the include path stays short. Re-run `tools/gen_full_case_folding.py`
// after a UCD bump.
include!("full_case_folding_data.rs");

// ---------------------------------------------------------------------------
// Public API
// ---------------------------------------------------------------------------

/// Public view of the full multi-codepoint upper-folding table.
/// Sorted ascending by input codepoint.
pub fn full_upper_folding() -> &'static [(u32, u32, u32, u32, u8)] {
    FULL_UPPER_FOLDING
}

/// Public view of the full multi-codepoint lower-folding table.
/// Currently a single entry (U+0130 → i + combining-dot-above).
pub fn full_lower_folding() -> &'static [(u32, u32, u32, u32, u8)] {
    FULL_LOWER_FOLDING
}

/// Public view of the Turkish / Azerbaijani upper-folding overrides.
/// Used only when the caller passes a `tr` / `az` locale.
pub fn turkish_upper_folding() -> &'static [(u32, u32, u32, u32, u8)] {
    TURKISH_UPPER_FOLDING
}

/// Public view of the Turkish / Azerbaijani lower-folding overrides.
pub fn turkish_lower_folding() -> &'static [(u32, u32, u32, u32, u8)] {
    TURKISH_LOWER_FOLDING
}

/// Sorted ranges of Cased codepoints (UCD `Cased` derived property).
pub fn cased_ranges() -> &'static [(u32, u32)] {
    CASED_RANGES
}

/// Sorted ranges of Case_Ignorable codepoints (UCD `Case_Ignorable`
/// derived property).
pub fn case_ignorable_ranges() -> &'static [(u32, u32)] {
    CASE_IGNORABLE_RANGES
}

/// `true` when `cp` carries the UCD `Cased` derived property.
#[inline]
pub fn is_cased(cp: u32) -> bool {
    range_contains(CASED_RANGES, cp)
}

/// `true` when `cp` carries the UCD `Case_Ignorable` derived property.
#[inline]
pub fn is_case_ignorable(cp: u32) -> bool {
    range_contains(CASE_IGNORABLE_RANGES, cp)
}

/// UAX #21 Final_Sigma context check: returns `true` when `Σ` at
/// `cps[anchor]` is at the end of a word — no later codepoint in
/// `cps[anchor + 1 ..]` is cased, after skipping case-ignorable
/// codepoints in between. The argument is a slice over decoded
/// codepoints (post-UTF-8 decode); the helper consults
/// [`is_cased`] and [`is_case_ignorable`] internally.
pub fn is_final_sigma_context(cps: &[u32], anchor: usize) -> bool {
    // Need at least one cased codepoint to the left of `anchor` to
    // count as "end of word" — otherwise the sigma is a stand-alone
    // codepoint and the canonical rule still applies (per UAX #21,
    // `Final_Sigma` requires `Before` = there is a preceding cased
    // codepoint plus only case-ignorables between).
    let mut has_cased_before = false;
    for &cp in cps[..anchor].iter().rev() {
        if is_cased(cp) {
            has_cased_before = true;
            break;
        }
        if !is_case_ignorable(cp) {
            break;
        }
    }
    if !has_cased_before {
        return false;
    }
    // After: any cased codepoint after the sigma (ignoring case
    // ignorables) disqualifies the final-sigma form.
    for &cp in cps[anchor + 1..].iter() {
        if is_cased(cp) {
            return false;
        }
        if !is_case_ignorable(cp) {
            // Non-cased, non-ignorable breaks the scan — sigma is
            // word-final.
            return true;
        }
    }
    true
}

/// Locale resolution: returns `true` when the BCP-47-ish `locale`
/// string selects the Turkish / Azerbaijani branch. Only the leading
/// two ASCII characters are compared; subtags (`-TR`, `_AZ`, ...) are
/// ignored so `tr`, `tr-TR`, `tr_TR`, `az_AZ` all match.
#[inline]
pub fn is_turkish_locale(locale: &str) -> bool {
    let bytes = locale.as_bytes();
    if bytes.len() < 2 {
        return false;
    }
    let a = bytes[0].to_ascii_lowercase();
    let b = bytes[1].to_ascii_lowercase();
    // After the two-letter prefix the only valid follow-on for a BCP-47
    // tag is `-`, `_`, or end-of-string.
    let boundary_ok = bytes.len() == 2 || matches!(bytes[2], b'-' | b'_');
    boundary_ok && ((a == b't' && b == b'r') || (a == b'a' && b == b'z'))
}

// ---------------------------------------------------------------------------
// Helpers (private)
// ---------------------------------------------------------------------------

fn range_contains(ranges: &[(u32, u32)], cp: u32) -> bool {
    super::cp_in_ranges(cp, ranges)
}

/// Look up a FULL fold entry by codepoint, returning `(out_len,
/// [out0, out1, out2])`. Callers index the leading `out_len` slots.
#[inline]
pub fn full_upper_entry(cp: u32) -> Option<(u8, [u32; 3])> {
    entry_for(FULL_UPPER_FOLDING, cp)
}

#[inline]
pub fn full_lower_entry(cp: u32) -> Option<(u8, [u32; 3])> {
    entry_for(FULL_LOWER_FOLDING, cp)
}

#[inline]
pub fn turkish_upper_entry(cp: u32) -> Option<(u8, [u32; 3])> {
    entry_for(TURKISH_UPPER_FOLDING, cp)
}

#[inline]
pub fn turkish_lower_entry(cp: u32) -> Option<(u8, [u32; 3])> {
    entry_for(TURKISH_LOWER_FOLDING, cp)
}

fn entry_for(table: &'static [(u32, u32, u32, u32, u8)], cp: u32) -> Option<(u8, [u32; 3])> {
    let idx = table.binary_search_by_key(&cp, |&(k, _, _, _, _)| k).ok()?;
    let (_, a, b, c, n) = table[idx];
    Some((n, [a, b, c]))
}

// ---------------------------------------------------------------------------
// Wire-format encoders (used by the wasm-AOT data-section pass)
// ---------------------------------------------------------------------------

/// Encode a FULL fold table for the wasm data section.
///
/// Layout: 4-byte LE entry count followed by `count * 20` bytes —
/// each entry is `(input_cp: u32 LE, out0: u32 LE, out1: u32 LE,
/// out2: u32 LE, out_len: u32 LE)`. `out_len` is widened from `u8` on
/// the wire to keep the per-entry stride at a clean 20 bytes, matching
/// the runtime helper's `4 + mid * 20` rebase arithmetic.
pub fn encode_full_table_bytes(table: &[(u32, u32, u32, u32, u8)]) -> Vec<u8> {
    let mut bytes = Vec::with_capacity(4 + table.len() * 20);
    bytes.extend_from_slice(&(table.len() as u32).to_le_bytes());
    for (k, a, b, c, n) in table {
        bytes.extend_from_slice(&k.to_le_bytes());
        bytes.extend_from_slice(&a.to_le_bytes());
        bytes.extend_from_slice(&b.to_le_bytes());
        bytes.extend_from_slice(&c.to_le_bytes());
        bytes.extend_from_slice(&(*n as u32).to_le_bytes());
    }
    bytes
}

/// Byte size of an encoded FULL fold table — header + 20 bytes per
/// entry. Codegen uses this to pre-size the data section.
pub fn encoded_full_table_size(table: &[(u32, u32, u32, u32, u8)]) -> usize {
    4 + table.len() * 20
}

/// Encode a FULL table into the simple 8-byte stride layout the
/// `__casefold_lookup` helper expects (one input cp -> one output cp).
/// Multi-codepoint entries (out_len > 1) emit the FIRST output slot
/// only — caller-side logic is expected to detect this case via a
/// parallel `out_len` check, or to use the full 20-byte encoder when
/// the trailing codepoints matter.
///
/// Used by the wasm-AOT locale-aware bodies to share the existing
/// `__casefold_lookup` helper while still consulting the Turkish
/// override entries (which today happen to all be 1:1 mappings).
pub fn encode_simple_view_bytes(table: &[(u32, u32, u32, u32, u8)]) -> Vec<u8> {
    let mut bytes = Vec::with_capacity(4 + table.len() * 8);
    bytes.extend_from_slice(&(table.len() as u32).to_le_bytes());
    for (k, a, _b, _c, _n) in table {
        bytes.extend_from_slice(&k.to_le_bytes());
        bytes.extend_from_slice(&a.to_le_bytes());
    }
    bytes
}

/// Encode a range table — same shape as the combining-mark /
/// whitespace ranges. Delegates to [`super::encode_u32_pair_table`].
pub fn encode_ranges_bytes(table: &[(u32, u32)]) -> Vec<u8> {
    super::encode_u32_pair_table(table)
}

#[cfg(test)]
mod tests {
    use super::*;

    #[test]
    fn sharp_s_upper_is_ss() {
        let (len, slots) = full_upper_entry(0x00DF).expect("ß entry");
        assert_eq!(len, 2);
        assert_eq!(&slots[..2], &[0x0053, 0x0053]);
    }

    #[test]
    fn fi_ligature_upper_is_fi() {
        let (len, slots) = full_upper_entry(0xFB01).expect("fi entry");
        assert_eq!(len, 2);
        assert_eq!(&slots[..2], &[0x0046, 0x0049]);
    }

    #[test]
    fn capital_i_with_dot_lower_is_i_plus_dot() {
        let (len, slots) = full_lower_entry(0x0130).expect("İ entry");
        assert_eq!(len, 2);
        assert_eq!(&slots[..2], &[0x0069, 0x0307]);
    }

    #[test]
    fn turkish_capital_i_lower_is_dotless() {
        let (len, slots) = turkish_lower_entry(0x0049).expect("tr I entry");
        assert_eq!(len, 1);
        assert_eq!(slots[0], 0x0131);
    }

    #[test]
    fn cased_basic_letters() {
        assert!(is_cased(0x0041)); // 'A'
        assert!(is_cased(0x03A3)); // 'Σ'
        assert!(!is_cased(0x0020)); // space
        assert!(!is_cased(0x0030)); // '0'
    }

    #[test]
    fn case_ignorable_basic_marks() {
        assert!(is_case_ignorable(0x0027)); // apostrophe
        assert!(is_case_ignorable(0x0301)); // combining acute
        assert!(!is_case_ignorable(0x0041)); // 'A' is cased, not ignorable
    }

    #[test]
    fn final_sigma_context_at_word_end() {
        // "OΣ" with Σ at position 1 — has cased before, nothing after.
        // (note: anchor codepoint is the sigma at index 1)
        let cps: Vec<u32> = "OΣ".chars().map(|c| c as u32).collect();
        assert!(is_final_sigma_context(&cps, 1));
    }

    #[test]
    fn non_final_sigma_in_middle_of_word() {
        // "ΣΑ" — sigma at start, cased letter follows.
        let cps: Vec<u32> = "ΣΑ".chars().map(|c| c as u32).collect();
        assert!(!is_final_sigma_context(&cps, 0));
    }

    #[test]
    fn final_sigma_through_case_ignorable() {
        // "OΣ'" — apostrophe is case-ignorable, sigma at index 1 is
        // followed only by ignorables, so the form is final.
        let cps: Vec<u32> = "OΣ'".chars().map(|c| c as u32).collect();
        assert!(is_final_sigma_context(&cps, 1));
    }

    #[test]
    fn locale_match_tr_and_az() {
        assert!(is_turkish_locale("tr"));
        assert!(is_turkish_locale("TR"));
        assert!(is_turkish_locale("tr-TR"));
        assert!(is_turkish_locale("az_AZ"));
        assert!(!is_turkish_locale("en"));
        assert!(!is_turkish_locale("de-DE"));
        assert!(!is_turkish_locale(""));
        assert!(!is_turkish_locale("tron")); // tr-prefix is not a locale match
    }

    #[test]
    fn encode_full_table_layout() {
        let toy: &[(u32, u32, u32, u32, u8)] = &[(0x00DF, 0x0053, 0x0053, 0x0000, 2)];
        let bytes = encode_full_table_bytes(toy);
        assert_eq!(bytes.len(), 4 + 20);
        assert_eq!(&bytes[0..4], &1u32.to_le_bytes());
        assert_eq!(&bytes[4..8], &0x00DFu32.to_le_bytes());
        assert_eq!(&bytes[8..12], &0x0053u32.to_le_bytes());
        assert_eq!(&bytes[20..24], &2u32.to_le_bytes());
    }

    #[test]
    fn upper_table_sorted_and_non_empty() {
        let table = FULL_UPPER_FOLDING;
        assert!(!table.is_empty());
        for win in table.windows(2) {
            assert!(win[0].0 < win[1].0, "FULL upper table must be sorted asc");
        }
    }

    #[test]
    fn lower_table_sorted() {
        let table = FULL_LOWER_FOLDING;
        for win in table.windows(2) {
            assert!(win[0].0 < win[1].0, "FULL lower table must be sorted asc");
        }
    }
}