disarm 0.11.0

//! Compose-only "local NFC" at lookup time (#475 / #477).
//!
//! The confusables and transliteration tables are keyed per code point on the
//! *precomposed* form (`ї` U+0457 → …), so a *decomposed* input (`і` U+0456 +
//! combining diaeresis U+0308) never reaches the entry — the base maps and the mark
//! survives, which lets an attacker evade the fold/romanization by decomposing.
//!
//! Fixing this by normalizing the whole *input* to NFC is wrong: NFC also *decomposes*
//! composition-excluded singletons (Hebrew presentation forms like `שׂ` U+FB2B), which
//! destroys their precomposed table entry, and it changes character counts. Instead,
//! [`composed`] makes only the *lookup* form-complete: it canonically composes each
//! base + combining-mark cluster (compose-only, never decomposes the input) so any
//! normal form reaches the same precomposed entry, while
//!
//! * a starter **not** followed by a combining mark (ASCII, CJK, a lone precomposed
//!   `שׂ`) is yielded verbatim — the hot path pays one combining-class check per char,
//!   in the spirit of the #458 guard, and a bare presentation form is never decomposed
//!   (so the #478 regression class cannot recur);
//! * a composition-**excluded** cluster (`ש` + sin dot, KA + nukta) reaches its
//!   precomposed scalar through the build-time widening map (#481): canonical NFC alone
//!   leaves it decomposed, so the map is consulted on the cluster's NFC string;
//! * the **whole** cluster is composed (canonical order, multi-mark): Vietnamese `ệ`
//!   (base + two marks) and polytonic Greek `ᾷ` reach their precomposed scalar;
//! * conjoining Hangul jamo (`General_Category=Lo`, so the mark gate misses them) are
//!   composed into syllables by pure arithmetic (#483), so a decomposed syllable run
//!   romanizes like the precomposed one.

use std::collections::VecDeque;

use unicode_normalization::char::is_combining_mark;
use unicode_normalization::UnicodeNormalization;

// #481: the compose-at-lookup *widening* map — `phf::Map<&'static str, char>` from a
// fully canonically-decomposed cluster to its composition-EXCLUDED precomposed scalar
// (KA+nukta → QA U+0958, shin+sin-dot → U+FB2B). These do not recompose under canonical
// NFC, so without this table they would stay decomposed and miss the precomposed entry.
// Generated by build.rs from `src/tables/data/excluded_compositions.tsv`.
include!(concat!(env!("OUT_DIR"), "/excluded_compositions_phf.rs"));

/// Iterate `text` yielding `(char, byte_offset)` with each base+mark cluster locally
/// NFC-composed. `byte_offset` is the start of the cluster (or the char) in the
/// original `text` — for diagnostics like `find_untranslatable`. See the module docs.
pub(crate) fn composed(text: &str) -> Composed<'_> {
    Composed {
        text,
        iter: text.char_indices().peekable(),
        pending: VecDeque::new(),
        scratch: String::new(),
    }
}

/// Apply [`composed`] over the whole string: a borrowed `Cow` when `text` has no
/// combining mark (nothing can compose — the ASCII / common case pays one scan), an
/// owned, cluster-composed string otherwise. This is "local NFC on each grapheme
/// cluster": it lets a per-code-point engine see the precomposed form without ever
/// decomposing a composition-excluded singleton.
pub(crate) fn compose_str(text: &str) -> std::borrow::Cow<'_, str> {
    if needs_composition(text) {
        std::borrow::Cow::Owned(composed(text).map(|(c, _)| c).collect())
    } else {
        std::borrow::Cow::Borrowed(text)
    }
}

/// True if [`composed`] could change `text` — the cheap gate that decides whether to run
/// the compose pass at all. Two triggers, found anywhere in `text`: a combining mark
/// (General_Category=Mark, which also catches the spacing marks Mc in Brahmic two-part
/// vowels), or a conjoining Hangul **L** jamo — a leading *consonant*, not a string
/// position — (General_Category=Lo, which the mark test misses — #483). Input with
/// neither (ASCII, CJK, lone precomposed letters) takes a borrow/identity fast path.
pub(crate) fn needs_composition(text: &str) -> bool {
    text.chars().any(could_compose)
}

/// Per-char predicate for [`needs_composition`]: is `c` a combining mark (GC=Mark) or a
/// conjoining Hangul **L** jamo? Behaviour-identical to `is_combining_mark(c) || L-jamo`,
/// but with a cheap range fast-path over U+0000–U+058F — the dense Latin / Greek /
/// Cyrillic / Armenian letter region that dominates real non-ASCII text. In that whole
/// region a character is GC=Mark *only* inside two small sub-blocks (U+0300–036F
/// Combining Diacritical Marks, U+0483–0489 Combining Cyrillic), and no conjoining jamo
/// (those start at U+1100) can appear — so a couple of integer range checks settle it
/// without the `is_combining_mark` Unicode-property **trie lookup**. Only U+0590 and
/// above (Hebrew/Arabic/Indic marks, the Mc spacing vowels, jamo, …) pay the precise
/// lookup. This is the per-character cost that the compose-at-lookup pre-scan (#477)
/// otherwise adds to every non-ASCII transliterate/confusables call.
#[inline]
pub(crate) fn could_compose(c: char) -> bool {
    let u = c as u32;
    if u < 0x0590 {
        return (0x0300..=0x036F).contains(&u) || (0x0483..=0x0489).contains(&u);
    }
    is_combining_mark(c) || (HANGUL_L_BASE..=HANGUL_L_LAST).contains(&u)
}

pub(crate) struct Composed<'a> {
    text: &'a str,
    iter: std::iter::Peekable<std::str::CharIndices<'a>>,
    /// Chars of a composed cluster's NFC result, waiting to be yielded.
    pending: VecDeque<(char, usize)>,
    /// Reused buffer for a cluster's NFC form (L-6): mark-heavy input would otherwise
    /// allocate a fresh `String` per cluster. Cleared and refilled each cluster, so its
    /// capacity is retained across the whole iteration — one alloc, not one-per-cluster.
    scratch: String,
}

// Conjoining Hangul jamo composition (#483). The L/V/T ranges and the syllable formula
// are the standard Unicode algorithm (UAX #15 §16) — total over these ranges, a
// bijection, so no table or validation is needed beyond the range checks. Scope is the
// modern conjoining jamo only; archaic jamo (Extended-A/B) never appear in the NFD of a
// standard precomposed syllable.
const HANGUL_S_BASE: u32 = 0xAC00;
const HANGUL_L_BASE: u32 = 0x1100;
const HANGUL_L_LAST: u32 = 0x1112; // 19 leading consonants
const HANGUL_V_BASE: u32 = 0x1161;
const HANGUL_V_LAST: u32 = 0x1175; // 21 vowels
const HANGUL_T_BASE: u32 = 0x11A7; // index 0 = no trailing consonant
const HANGUL_T_FIRST: u32 = 0x11A8;
const HANGUL_T_LAST: u32 = 0x11C2; // 27 trailing consonants
const HANGUL_T_COUNT: u32 = 28;
const HANGUL_N_COUNT: u32 = 21 * HANGUL_T_COUNT; // V_COUNT * T_COUNT = 588

impl Composed<'_> {
    /// If `l` is a leading jamo (L) followed by a vowel jamo (V), consume the V (and an
    /// optional trailing jamo T) and return the composed syllable. Otherwise consume
    /// nothing and return `None`, so a lone L — or an L not followed by a V — is yielded
    /// verbatim by the caller. Pure arithmetic, gated on a cheap range check.
    fn try_compose_hangul(&mut self, l: char) -> Option<char> {
        let lc = l as u32;
        if !(HANGUL_L_BASE..=HANGUL_L_LAST).contains(&lc) {
            return None;
        }
        let vc = self.iter.peek()?.1 as u32;
        if !(HANGUL_V_BASE..=HANGUL_V_LAST).contains(&vc) {
            return None;
        }
        self.iter.next(); // consume V
        let lv = HANGUL_S_BASE
            + (lc - HANGUL_L_BASE) * HANGUL_N_COUNT
            + (vc - HANGUL_V_BASE) * HANGUL_T_COUNT;
        // Optional trailing consonant.
        if let Some(&(_, t)) = self.iter.peek() {
            let tc = t as u32;
            if (HANGUL_T_FIRST..=HANGUL_T_LAST).contains(&tc) {
                self.iter.next(); // consume T
                return char::from_u32(lv + (tc - HANGUL_T_BASE));
            }
        }
        char::from_u32(lv)
    }
}

impl Iterator for Composed<'_> {
    type Item = (char, usize);

    fn next(&mut self) -> Option<Self::Item> {
        if let Some(item) = self.pending.pop_front() {
            return Some(item);
        }
        let (start, ch) = self.iter.next()?;

        // #483: conjoining Hangul jamo are General_Category=Lo, so the GC=Mark gate below
        // never fires on them. Compose an L+V[+T] run into its precomposed syllable by
        // pure index arithmetic (no table, no reordering, no exclusions) so a decomposed
        // syllable reaches the same per-code-point romanization path as the precomposed
        // form. A partial run (lone L, L+T with no V) does not compose.
        if let Some(syllable) = self.try_compose_hangul(ch) {
            return Some((syllable, start));
        }

        // A char anchors a cluster iff it is followed by a combining mark. The follower
        // test is General_Category=Mark, not ccc != 0: Brahmic two-part vowels compose a
        // base with a *spacing* mark (Mc, ccc == 0) — Bengali `ো` U+09CB = U+09C7 +
        // U+09BE — which a ccc test misses. The anchor may itself be a mark: Tibetan
        // vowel combinations like `◌ཱི` U+0F73 = U+0F71 + U+0F72 are mark-only excluded
        // compositions with no base. A char with no following mark (ASCII / CJK / a lone
        // precomposed letter or presentation form, or a lone trailing mark) is yielded
        // verbatim — the hot path. Only a cluster that hits the widening map or canonical
        // NFC actually changes; everything else round-trips.
        let next_is_mark = self.iter.peek().is_some_and(|&(_, c)| is_combining_mark(c));
        if !next_is_mark {
            return Some((ch, start));
        }

        // Collect the cluster: anchor + the run of following combining marks.
        let mut end = start + ch.len_utf8();
        while let Some(&(j, c)) = self.iter.peek() {
            if !is_combining_mark(c) {
                break;
            }
            end = j + c.len_utf8();
            self.iter.next();
        }

        // Local NFC composes the cluster (canonical order, full multi-mark). A
        // composition-EXCLUDED cluster (KA+nukta, shin+sin-dot) does not recompose under
        // NFC, so consult the widening map (#481): the cluster's NFC string keys the
        // precomposed scalar. Offsets collapse to the cluster start — exact enough for
        // diagnostics, and the common single-mark cluster yields one char anyway.
        //
        // Reuse `self.scratch` for the NFC form (L-6): take it out (so `recompose_excluded`
        // can borrow `&mut self.pending`), refill, consume, then put it back with its grown
        // capacity — no per-cluster `String` allocation on mark-heavy input.
        let mut scratch = std::mem::take(&mut self.scratch);
        scratch.clear();
        scratch.extend(self.text[start..end].nfc());
        self.recompose_excluded(&scratch, start);
        self.scratch = scratch;
        self.pending.pop_front()
    }
}

impl Composed<'_> {
    /// Queue the NFC'd cluster onto `pending`, recomposing composition-excluded base+mark
    /// sequences via the widening map.
    ///
    /// Fast path — the whole cluster is itself an excluded composition (the common
    /// excluded-pair case: KA+nukta, shin+sin-dot): one O(1) lookup, no scan, no
    /// allocation. A plain base+mark that canonically composed (`é`, `ệ`) is one char and
    /// also takes a cheap miss-then-emit.
    ///
    /// General path — greedy longest-prefix matching, needed because a whole-cluster
    /// lookup is *not* enough: `.nfc()` decomposes an excluded singleton (`ড়` U+09DC → ড +
    /// nukta), and when that singleton is followed by an *unrelated* mark (a visarga) the
    /// cluster is `ড nukta visarga`, which misses the 2-char `ড nukta` key — leaving the
    /// singleton decomposed and making the fold oscillate (compose ⇄ decompose,
    /// non-idempotent). So at each position match the longest map key that fits, emit the
    /// precomposed scalar, and advance past it; otherwise emit one char. The scan walks the
    /// `&str` by char boundaries (a fixed stack array of cut points, no `Vec`).
    fn recompose_excluded(&mut self, nfc: &str, start: usize) {
        // Whole-cluster fast path: the common excluded pair resolves in one lookup.
        if let Some(&precomposed) = EXCLUDED_COMPOSITIONS.get(nfc) {
            self.pending.push_back((precomposed, start));
            return;
        }
        let mut rest = nfc;
        while !rest.is_empty() {
            if let Some((precomposed, len)) = excluded_prefix(rest) {
                self.pending.push_back((precomposed, start));
                rest = &rest[len..];
            } else {
                // SAFETY of unwrap: `rest` is non-empty, so it has a first char.
                let ch = rest.chars().next().unwrap_or('\u{FFFD}');
                self.pending.push_back((ch, start));
                rest = &rest[ch.len_utf8()..];
            }
        }
    }
}

/// Longest composition-excluded key (≥ 2 chars) that is a char-prefix of `s`, as
/// `(precomposed, byte_len)`. Keys are at most `EXCLUDED_COMPOSITIONS_MAX_KEY_CHARS`
/// chars; record each candidate cut point in a fixed stack array (no heap), then probe
/// longest-first so a 3-char excluded stack wins over its 2-char head.
fn excluded_prefix(s: &str) -> Option<(char, usize)> {
    let mut cuts = [0usize; EXCLUDED_COMPOSITIONS_MAX_KEY_CHARS];
    let mut count = 0;
    for (i, (off, ch)) in s.char_indices().enumerate() {
        if i >= EXCLUDED_COMPOSITIONS_MAX_KEY_CHARS {
            break;
        }
        cuts[i] = off + ch.len_utf8();
        count = i + 1;
    }
    (2..=count).rev().find_map(|n| {
        let end = cuts[n - 1];
        EXCLUDED_COMPOSITIONS.get(&s[..end]).map(|&pc| (pc, end))
    })
}

#[cfg(test)]
mod tests {
    use super::*;

    fn chars(text: &str) -> Vec<char> {
        composed(text).map(|(c, _)| c).collect()
    }

    #[test]
    fn ascii_and_mark_free_pass_through_verbatim() {
        assert_eq!(chars("abc 123"), "abc 123".chars().collect::<Vec<_>>());
        assert_eq!(chars("日本語"), vec!['日', '本', '語']);
        // a lone precomposed char with no following mark is untouched
        assert_eq!(chars("\u{0457}"), vec!['\u{0457}']); // precomposed ї
    }

    #[test]
    fn decomposed_homoglyph_composes_to_precomposed() {
        // і (U+0456) + combining diaeresis (U+0308) → ї (U+0457)
        assert_eq!(chars("\u{0456}\u{0308}"), vec!['\u{0457}']);
        // embedded in text, offsets preserved at the cluster start
        // bytes: a@0, і@1 (2B), ◌̈@3 (2B), b@5 — composed ї keeps the cluster start.
        let got: Vec<_> = composed("a\u{0456}\u{0308}b").collect();
        assert_eq!(got, vec![('a', 0), ('\u{0457}', 1), ('b', 5)]);
    }

    #[test]
    fn multi_mark_clusters_compose_fully() {
        // Vietnamese ệ = e + ◌̣ (U+0323, ccc 220) + ◌̂ (U+0302, ccc 230) → U+1EC7
        assert_eq!(chars("e\u{0323}\u{0302}"), vec!['\u{1EC7}']);
        // …in the other input order too (canonical reordering happens inside NFC)
        assert_eq!(chars("e\u{0302}\u{0323}"), vec!['\u{1EC7}']);
        // polytonic Greek ᾷ (U+1FB7) = α + ◌̃-ish marks → precomposed
        assert_eq!(chars("\u{03B1}\u{0342}\u{0345}"), vec!['\u{1FB7}']);
    }

    #[test]
    fn brahmic_two_part_vowels_compose_via_spacing_marks() {
        // Bengali O (U+09CB) = E (U+09C7) + AA (U+09BE) — both spacing marks (Mc,
        // ccc 0). A ccc-only gate misses these; the category=Mark gate composes them.
        assert_eq!(chars("\u{09C7}\u{09BE}"), vec!['\u{09CB}']);
        // Tamil AU (U+0B94) = O (U+0B92, a *letter* base) + AU length mark (U+0BD7).
        assert_eq!(chars("\u{0B92}\u{0BD7}"), vec!['\u{0B94}']);
        // Kannada OO (U+0CCB) = E (U+0CC6) + UU (U+0CC2) + length mark (U+0CD5).
        assert_eq!(chars("\u{0CC6}\u{0CC2}\u{0CD5}"), vec!['\u{0CCB}']);
    }

    #[test]
    fn composition_excluded_pairs_compose_via_widening_map() {
        // #481: a composition-EXCLUDED cluster now reaches its precomposed scalar through
        // the widening map (canonical NFC alone leaves it decomposed). ש (U+05E9) + sin
        // dot (U+05C2) → שׂ U+FB2B; KA (U+0915) + nukta (U+093C) → QA U+0958. This is the
        // form-invariant *and* correct result — and the decompose still never happens at
        // lookup: a *bare* presentation form (no following mark) is left untouched, so the
        // #478 regression class (decomposing raw U+FB2B → "sh") cannot recur.
        assert_eq!(chars("\u{05E9}\u{05C2}"), vec!['\u{FB2B}']);
        assert_eq!(chars("\u{0915}\u{093C}"), vec!['\u{0958}']);
        assert_eq!(chars("\u{FB2B}"), vec!['\u{FB2B}']); // bare presentation form, untouched

        // Mark-only excluded composition (no base to anchor): Tibetan VOWEL SIGN II
        // U+0F73 = U+0F71 (AA, ccc 129) + U+0F72 (I) composes via the map even though
        // both pieces are combining marks.
        assert_eq!(chars("\u{0F71}\u{0F72}"), vec!['\u{0F73}']);
    }

    #[test]
    fn compose_str_borrows_mark_free_and_owns_composed() {
        use std::borrow::Cow;
        // mark-free input borrows (no allocation, identity)
        assert!(matches!(compose_str("hello"), Cow::Borrowed("hello")));
        assert!(matches!(compose_str("\u{FB2B}"), Cow::Borrowed(_))); // lone שׂ, no mark
                                                                      // mark-bearing input is composed into an owned string
        assert_eq!(compose_str("\u{0456}\u{0308}"), "\u{0457}"); // і+◌̈ → ї
        assert_eq!(compose_str("e\u{0323}\u{0302}"), "\u{1EC7}"); // ệ
                                                                  // excluded base+mark now composes via the widening map (#481)
        assert_eq!(compose_str("\u{05E9}\u{05C2}"), "\u{FB2B}");
    }

    #[test]
    fn legitimate_accents_recompose_unchanged() {
        // café decomposed (e + ◌́) recomposes to é — a no-op for already-NFC callers.
        assert_eq!(chars("cafe\u{0301}"), vec!['c', 'a', 'f', 'é']);
    }

    #[test]
    fn conjoining_hangul_jamo_compose_to_syllables() {
        // #483: conjoining jamo are General_Category=Lo (so the GC=Mark gate never fires);
        // compose them arithmetically into syllables.
        // L+V → LV syllable: ᄀ U+1100 + ᅡ U+1161 → 가 U+AC00
        assert_eq!(chars("\u{1100}\u{1161}"), vec!['\u{AC00}']);
        // L+V+T → LVT syllable: ᄒ U+1112 + ᅡ U+1161 + ᆫ U+11AB → 한 U+D55C
        assert_eq!(chars("\u{1112}\u{1161}\u{11AB}"), vec!['\u{D55C}']);
        // multi-syllable run (처리 NFD = LV LV) → two precomposed syllables
        assert_eq!(
            chars("\u{110E}\u{1165}\u{1105}\u{1175}"),
            vec!['\u{CC98}', '\u{B9AC}']
        );
    }

    #[test]
    fn partial_hangul_jamo_left_alone() {
        // #483 criterion 4: a lone L, a lone V, or L+T without a V are not valid syllable
        // sequences — leave them untouched.
        assert_eq!(chars("\u{1100}"), vec!['\u{1100}']); // lone L
        assert_eq!(chars("\u{1161}"), vec!['\u{1161}']); // lone V
        assert_eq!(chars("\u{1100}\u{11AB}"), vec!['\u{1100}', '\u{11AB}']); // L + T, no V
                                                                             // an L followed by a non-jamo letter stays separate
        assert_eq!(chars("\u{1100}a"), vec!['\u{1100}', 'a']);
    }

    #[test]
    fn excluded_singleton_with_trailing_unrelated_mark_recomposes() {
        // Regression: an excluded precomposed singleton FOLLOWED BY an unrelated mark.
        // `ড়` U+09DC (excluded singleton = ড U+09A1 + nukta U+09BC) then ◌ঃ visarga
        // U+0983. The cluster collects all three; `.nfc()` *decomposes* the excluded
        // singleton (ড nukta visarga), and a whole-cluster map lookup misses the 2-char
        // key because of the trailing visarga — leaving the singleton decomposed, which
        // makes the fold oscillate (compose → decompose → compose) and non-idempotent.
        // Greedy prefix matching recomposes `ড nukta` → U+09DC and keeps the visarga.
        assert_eq!(chars("\u{09DC}\u{0983}"), vec!['\u{09DC}', '\u{0983}']);
        // the proptest seed that surfaced it: Hebrew FB31 (excluded) + Oriya visarga.
        assert_eq!(chars("\u{FB31}\u{0B03}"), vec!['\u{FB31}', '\u{0B03}']);
        // and the decomposed form of the same input lands on the identical scalar —
        // form-invariance, which is what idempotency of the whole fold reduces to here.
        assert_eq!(
            chars("\u{09A1}\u{09BC}\u{0983}"),
            vec!['\u{09DC}', '\u{0983}']
        );
    }

    #[test]
    fn compose_str_composes_conjoining_jamo() {
        use std::borrow::Cow;
        // the gate must trigger on jamo even though they are not combining marks
        assert_eq!(compose_str("\u{1100}\u{1161}"), "\u{AC00}"); // ᄀ+ᅡ → 가
                                                                 // jamo-free / mark-free input still borrows (hot path unchanged)
        assert!(matches!(compose_str("hello"), Cow::Borrowed("hello")));
    }
}