llmtrace_security/

normalise.rs

//! Unicode normalisation layer for security analysis.
//!
//! This module provides text normalisation as a preprocessing step before all
//! security analysis.  It applies a multi-stage pipeline to defeat Unicode-based
//! evasion techniques:
//!
//! 1. **NFKC normalisation** — compatibility decomposition + canonical composition
//! 2. **Diacritics stripping** — removes combining marks to defeat accent evasion
//!    (IS-031)
//! 3. **Invisible character stripping** — removes zero-width, tag, and control
//!    characters (IS-022)
//! 4. **Homoglyph mapping** — maps Cyrillic, Greek, upside-down, and Braille
//!    characters to ASCII equivalents (IS-021, IS-015)
//! 5. **Emoji stripping** — removes emoji to defeat emoji-smuggling attacks
//!    (IS-020)
//!
//! # Why?
//!
//! Attackers can bypass regex-based detection by using visually identical but
//! distinct Unicode code points — for example, Cyrillic `а` (U+0430) instead
//! of Latin `a` (U+0061), embedding zero-width characters inside keywords,
//! using upside-down letters, encoding text in Braille, adding diacritics, or
//! interspersing emoji characters.  Normalising text before analysis neutralises
//! these evasion techniques.

use unicode_normalization::UnicodeNormalization;

/// Characters that are zero-width or invisible and should be stripped.
const ZERO_WIDTH_CHARS: &[char] = &[
    '\u{200B}', // Zero-width space
    '\u{200C}', // Zero-width non-joiner
    '\u{200D}', // Zero-width joiner
    '\u{FEFF}', // BOM / zero-width no-break space
    '\u{00AD}', // Soft hyphen
    '\u{2060}', // Word joiner
    '\u{2028}', // Line separator
    '\u{2029}', // Paragraph separator
    // Bidirectional control characters (U+202A-U+202E)
    '\u{202A}', // Left-to-right embedding
    '\u{202B}', // Right-to-left embedding
    '\u{202C}', // Pop directional formatting
    '\u{202D}', // Left-to-right override
    '\u{202E}', // Right-to-left override
    // Bidirectional isolate characters (U+2066-U+2069)
    '\u{2066}', // Left-to-right isolate
    '\u{2067}', // Right-to-left isolate
    '\u{2068}', // First strong isolate
    '\u{2069}', // Pop directional isolate
];

/// Normalise text for security analysis.
///
/// This function applies a multi-stage normalisation pipeline:
/// 1. NFKC normalisation (compatibility decomposition + canonical composition)
/// 2. Diacritics stripping via NFD decomposition and combining mark removal
/// 3. Zero-width, invisible, and Unicode tag character stripping
/// 4. Homoglyph mapping (Cyrillic, Greek, upside-down text, Braille → ASCII)
/// 5. Emoji stripping
///
/// # Examples
///
/// ```
/// use llmtrace_security::normalise::normalise_text;
///
/// // NFKC normalisation: fullwidth "Ａ" → "A"
/// assert_eq!(normalise_text("\u{FF21}"), "A");
///
/// // Zero-width stripping
/// assert_eq!(normalise_text("he\u{200B}llo"), "hello");
///
/// // Homoglyph mapping: Cyrillic "а" → Latin "a"
/// assert_eq!(normalise_text("\u{0430}"), "a");
///
/// // Diacritics stripping: "café" → "cafe"
/// assert_eq!(normalise_text("caf\u{00E9}"), "cafe");
///
/// // Emoji stripping: "he😀llo" → "hello"
/// assert_eq!(normalise_text("he\u{1F600}llo"), "hello");
/// ```
pub fn normalise_text(input: &str) -> String {
    // Step 1: NFKC normalisation
    let nfkc: String = input.nfkc().collect();

    // Step 2: Strip diacritics (NFD decomposition + combining mark removal)
    let without_diacritics = strip_diacritics(&nfkc);

    // Step 3: Strip zero-width, invisible, and tag characters
    let stripped: String = without_diacritics
        .chars()
        .filter(|c| !ZERO_WIDTH_CHARS.contains(c) && !is_tag_character(*c))
        .collect();

    // Step 4: Map homoglyphs to ASCII equivalents
    let mapped: String = stripped.chars().map(map_homoglyph).collect();

    // Step 5: Strip emoji characters
    strip_emoji(&mapped)
}

/// Strip emoji characters from text.
///
/// Removes characters in standard Unicode emoji ranges including emoticons,
/// pictographs, transport symbols, dingbats, variation selectors, and skin
/// tone modifiers.  Emoji are removed entirely (not replaced with spaces) to
/// prevent attackers from using them as word separators to bypass detection.
///
/// # Examples
///
/// ```
/// use llmtrace_security::normalise::strip_emoji;
///
/// assert_eq!(strip_emoji("hello 🌍 world"), "hello  world");
/// assert_eq!(strip_emoji("ig🔥no📌re"), "ignore");
/// ```
pub fn strip_emoji(input: &str) -> String {
    input.chars().filter(|c| !is_emoji(*c)).collect()
}

/// Strip diacritics (combining marks) from text.
///
/// Applies NFD (canonical decomposition) to separate base characters from
/// combining marks, then removes all combining marks.  This converts accented
/// characters to their base forms (e.g., "é" → "e", "ñ" → "n").
///
/// # Examples
///
/// ```
/// use llmtrace_security::normalise::strip_diacritics;
///
/// assert_eq!(strip_diacritics("café"), "cafe");
/// assert_eq!(strip_diacritics("résumé"), "resume");
/// assert_eq!(strip_diacritics("naïve"), "naive");
/// ```
pub fn strip_diacritics(input: &str) -> String {
    input.nfd().filter(|c| !is_combining_mark(*c)).collect()
}

/// Returns `true` if the character is an emoji.
///
/// Covers standard Unicode emoji ranges: emoticons, miscellaneous symbols,
/// transport/map symbols, alchemical symbols, geometric shapes extended,
/// supplemental arrows, dingbats, variation selectors, and skin tone modifiers.
fn is_emoji(c: char) -> bool {
    let cp = c as u32;
    matches!(
        cp,
        0x1F600..=0x1F64F   // Emoticons
        | 0x1F300..=0x1F5FF // Misc Symbols and Pictographs
        | 0x1F680..=0x1F6FF // Transport and Map Symbols
        | 0x1F700..=0x1F77F // Alchemical Symbols
        | 0x1F780..=0x1F7FF // Geometric Shapes Extended
        | 0x1F800..=0x1F8FF // Supplemental Arrows-C
        | 0x1F900..=0x1F9FF // Supplemental Symbols and Pictographs
        | 0x1FA00..=0x1FA6F // Chess Symbols
        | 0x1FA70..=0x1FAFF // Symbols and Pictographs Extended-A
        | 0x2600..=0x26FF   // Miscellaneous Symbols
        | 0x2700..=0x27BF   // Dingbats
        | 0xFE00..=0xFE0F // Variation Selectors
                          // Skin tone modifiers (U+1F3FB–U+1F3FF) are covered by
                          // Misc Symbols and Pictographs (U+1F300–U+1F5FF) above.
    )
}

/// Returns `true` if the character is a Unicode combining mark.
///
/// Covers the principal combining diacritical mark blocks used to add accents
/// and other modifications to base characters.
fn is_combining_mark(c: char) -> bool {
    let cp = c as u32;
    matches!(
        cp,
        0x0300..=0x036F   // Combining Diacritical Marks
        | 0x0483..=0x0489 // Combining Cyrillic
        | 0x1AB0..=0x1AFF // Combining Diacritical Marks Extended
        | 0x1DC0..=0x1DFF // Combining Diacritical Marks Supplement
        | 0x20D0..=0x20FF // Combining Diacritical Marks for Symbols
        | 0xFE20..=0xFE2F // Combining Half Marks
    )
}

/// Returns `true` if the character is a Unicode tag character.
///
/// Tag characters (U+E0001–U+E007F) duplicate ASCII but are invisible.  They
/// were designed for language tagging but can be exploited to smuggle hidden
/// text through LLM pipelines.
fn is_tag_character(c: char) -> bool {
    let cp = c as u32;
    (0xE0001..=0xE007F).contains(&cp)
}

/// Map a single character to its ASCII equivalent if it is a known homoglyph.
///
/// Covers the most common Cyrillic-to-Latin confusables, Greek confusables,
/// upside-down (flipped) Latin letters, and Braille Grade 1 letter patterns.
fn map_homoglyph(c: char) -> char {
    match c {
        // =================================================================
        // Cyrillic → Latin (lowercase)
        // =================================================================
        '\u{0430}' => 'a', // Cyrillic а
        '\u{0435}' => 'e', // Cyrillic е
        '\u{043E}' => 'o', // Cyrillic о
        '\u{0440}' => 'p', // Cyrillic р
        '\u{0441}' => 'c', // Cyrillic с
        '\u{0445}' => 'x', // Cyrillic х
        '\u{0443}' => 'y', // Cyrillic у
        '\u{0456}' => 'i', // Cyrillic і (Ukrainian i)
        '\u{0458}' => 'j', // Cyrillic ј
        '\u{04BB}' => 'h', // Cyrillic һ

        // =================================================================
        // Cyrillic → Latin (uppercase)
        // =================================================================
        '\u{0410}' => 'A', // Cyrillic А
        '\u{0412}' => 'B', // Cyrillic В
        '\u{0415}' => 'E', // Cyrillic Е
        '\u{041A}' => 'K', // Cyrillic К
        '\u{041C}' => 'M', // Cyrillic М
        '\u{041D}' => 'H', // Cyrillic Н
        '\u{041E}' => 'O', // Cyrillic О
        '\u{0420}' => 'P', // Cyrillic Р
        '\u{0421}' => 'C', // Cyrillic С
        '\u{0422}' => 'T', // Cyrillic Т
        '\u{0425}' => 'X', // Cyrillic Х

        // =================================================================
        // Greek → Latin
        // =================================================================
        '\u{03BF}' => 'o', // Greek omicron ο
        '\u{03B1}' => 'a', // Greek alpha α (after NFKC, still distinct)
        '\u{0391}' => 'A', // Greek Alpha Α
        '\u{0392}' => 'B', // Greek Beta Β
        '\u{0395}' => 'E', // Greek Epsilon Ε
        '\u{0396}' => 'Z', // Greek Zeta Ζ
        '\u{0397}' => 'H', // Greek Eta Η
        '\u{0399}' => 'I', // Greek Iota Ι
        '\u{039A}' => 'K', // Greek Kappa Κ
        '\u{039B}' => 'V', // Greek Lambda Λ (upside-down V)
        '\u{039C}' => 'M', // Greek Mu Μ
        '\u{039D}' => 'N', // Greek Nu Ν
        '\u{039F}' => 'O', // Greek Omicron Ο
        '\u{03A1}' => 'P', // Greek Rho Ρ
        '\u{03A4}' => 'T', // Greek Tau Τ
        '\u{03A5}' => 'Y', // Greek Upsilon Υ
        '\u{03A7}' => 'X', // Greek Chi Χ

        // =================================================================
        // Upside-down / flipped Latin (lowercase)  — IS-021
        // =================================================================
        '\u{0250}' => 'a', // ɐ  (turned a)
        '\u{0254}' => 'c', // ɔ  (open o / turned c)
        '\u{01DD}' => 'e', // ǝ  (turned e)
        '\u{025F}' => 'f', // ɟ  (dotless j with stroke / turned f)
        '\u{0183}' => 'g', // ƃ  (b with topbar / turned g)
        '\u{0265}' => 'h', // ɥ  (turned h)
        '\u{0131}' => 'i', // ı  (dotless i)
        '\u{027E}' => 'j', // ɾ  (r with fishhook / turned j)
        '\u{029E}' => 'k', // ʞ  (turned k)
        '\u{026F}' => 'm', // ɯ  (turned m)
        '\u{0279}' => 'r', // ɹ  (turned r)
        '\u{0287}' => 't', // ʇ  (turned t)
        '\u{028C}' => 'v', // ʌ  (turned v / caret)
        '\u{028D}' => 'w', // ʍ  (turned w)
        '\u{028E}' => 'y', // ʎ  (turned y)

        // =================================================================
        // Upside-down / flipped Latin (uppercase)  — IS-021
        //
        // NOTE: Characters handled by NFKC are omitted to avoid dead arms:
        //   Ⅎ (U+2132) → F,  ⅁ (U+2141) → G,  ⅄ (U+2144) → Y
        //   ſ (U+017F) → s   (NFKC; task specifies J but NFKC wins)
        // =================================================================
        '\u{2200}' => 'A', // ∀  (for-all / turned A)
        '\u{15FA}' => 'B', // ᗺ  (Canadian Syllabics Carrier SI / turned B)
        '\u{0186}' => 'C', // Ɔ  (open O / turned C)
        '\u{15E1}' => 'D', // ᗡ  (Canadian Syllabics Carrier THE / turned D)
        '\u{018E}' => 'E', // Ǝ  (reversed E)
        '\u{02E5}' => 'L', // ˥  (modifier letter extra-high tone bar / turned L)
        '\u{0500}' => 'P', // Ԁ  (Cyrillic Komi De / turned P)
        '\u{1D1A}' => 'R', // ᴚ  (Latin letter small capital turned R)
        '\u{22A5}' => 'T', // ⊥  (up tack / turned T)
        '\u{2229}' => 'U', // ∩  (intersection / turned U)

        // =================================================================
        // Braille Grade 1 → ASCII  — IS-015
        //
        // Standard Braille encoding where each dot pattern maps to a letter.
        // U+2800 (blank) maps to space.
        // =================================================================
        '\u{2800}' => ' ', // ⠀  (blank)
        '\u{2801}' => 'a', // ⠁  (dot 1)
        '\u{2803}' => 'b', // ⠃  (dots 1-2)
        '\u{2809}' => 'c', // ⠉  (dots 1-4)
        '\u{2819}' => 'd', // ⠙  (dots 1-4-5)
        '\u{2811}' => 'e', // ⠑  (dots 1-5)
        '\u{280B}' => 'f', // ⠋  (dots 1-2-4)
        '\u{281B}' => 'g', // ⠛  (dots 1-2-4-5)
        '\u{2813}' => 'h', // ⠓  (dots 1-2-5)
        '\u{280A}' => 'i', // ⠊  (dots 2-4)
        '\u{281A}' => 'j', // ⠚  (dots 2-4-5)
        '\u{2805}' => 'k', // ⠅  (dots 1-3)
        '\u{2807}' => 'l', // ⠇  (dots 1-2-3)
        '\u{280D}' => 'm', // ⠍  (dots 1-3-4)
        '\u{281D}' => 'n', // ⠝  (dots 1-3-4-5)
        '\u{2815}' => 'o', // ⠕  (dots 1-3-5)
        '\u{280F}' => 'p', // ⠏  (dots 1-2-3-4)
        '\u{281F}' => 'q', // ⠟  (dots 1-2-3-4-5)
        '\u{2817}' => 'r', // ⠗  (dots 1-2-3-5)
        '\u{280E}' => 's', // ⠎  (dots 2-3-4)
        '\u{281E}' => 't', // ⠞  (dots 2-3-4-5)
        '\u{2825}' => 'u', // ⠥  (dots 1-3-6)
        '\u{2827}' => 'v', // ⠧  (dots 1-2-3-6)
        '\u{283A}' => 'w', // ⠺  (dots 2-4-5-6)
        '\u{282D}' => 'x', // ⠭  (dots 1-3-4-6)
        '\u{283D}' => 'y', // ⠽  (dots 1-3-4-5-6)
        '\u{2835}' => 'z', // ⠵  (dots 1-3-5-6)

        _ => c,
    }
}

// ===========================================================================
// Tests
// ===========================================================================

#[cfg(test)]
mod tests {
    use super::*;

    // -- NFKC normalisation ------------------------------------------------

    #[test]
    fn test_nfkc_fullwidth_to_ascii() {
        // Fullwidth "ＨＥＬＬＯ" → "HELLO"
        assert_eq!(
            normalise_text("\u{FF28}\u{FF25}\u{FF2C}\u{FF2C}\u{FF2F}"),
            "HELLO"
        );
    }

    #[test]
    fn test_nfkc_superscript_digits() {
        // Superscript "²" → "2"
        assert_eq!(normalise_text("\u{00B2}"), "2");
    }

    #[test]
    fn test_nfkc_ligature_fi() {
        // Ligature "fi" → "fi"
        assert_eq!(normalise_text("\u{FB01}"), "fi");
    }

    #[test]
    fn test_nfkc_roman_numeral() {
        // Roman numeral Ⅳ (U+2163) → "IV"
        assert_eq!(normalise_text("\u{2163}"), "IV");
    }

    #[test]
    fn test_nfkc_preserves_normal_ascii() {
        let text = "Hello, world! 123";
        assert_eq!(normalise_text(text), text);
    }

    // -- Zero-width character stripping ------------------------------------

    #[test]
    fn test_strip_zero_width_space() {
        assert_eq!(normalise_text("ig\u{200B}nore"), "ignore");
    }

    #[test]
    fn test_strip_zero_width_non_joiner() {
        assert_eq!(normalise_text("in\u{200C}structions"), "instructions");
    }

    #[test]
    fn test_strip_zero_width_joiner() {
        assert_eq!(normalise_text("pr\u{200D}ompt"), "prompt");
    }

    #[test]
    fn test_strip_bom() {
        assert_eq!(normalise_text("\u{FEFF}hello"), "hello");
    }

    #[test]
    fn test_strip_soft_hyphen() {
        assert_eq!(normalise_text("ig\u{00AD}nore"), "ignore");
    }

    #[test]
    fn test_strip_word_joiner() {
        assert_eq!(normalise_text("sys\u{2060}tem"), "system");
    }

    #[test]
    fn test_strip_line_separator() {
        assert_eq!(normalise_text("a\u{2028}b"), "ab");
    }

    #[test]
    fn test_strip_paragraph_separator() {
        assert_eq!(normalise_text("a\u{2029}b"), "ab");
    }

    #[test]
    fn test_strip_bidi_controls() {
        let input = "\u{202A}system\u{202C}: override\u{202E}";
        assert_eq!(normalise_text(input), "system: override");
    }

    #[test]
    fn test_strip_bidi_isolates() {
        let input = "\u{2066}ignore\u{2069} previous";
        assert_eq!(normalise_text(input), "ignore previous");
    }

    #[test]
    fn test_strip_multiple_zero_width_in_keyword() {
        // "i\u{200B}g\u{200C}n\u{200D}o\u{FEFF}re" → "ignore"
        assert_eq!(
            normalise_text("i\u{200B}g\u{200C}n\u{200D}o\u{FEFF}re"),
            "ignore"
        );
    }

    // -- Unicode tag character stripping (IS-022) --------------------------

    #[test]
    fn test_strip_tag_language_tag() {
        // U+E0001 (LANGUAGE TAG) should be stripped
        assert_eq!(normalise_text("hello\u{E0001}world"), "helloworld");
    }

    #[test]
    fn test_strip_tag_characters_range() {
        // Tag characters U+E0020–U+E007E embed invisible ASCII-equivalent text
        let input = "safe\u{E0069}\u{E0067}\u{E006E}\u{E006F}\u{E0072}\u{E0065}text";
        assert_eq!(normalise_text(input), "safetext");
    }

    #[test]
    fn test_strip_tag_cancel_tag() {
        // U+E007F (CANCEL TAG) should also be stripped
        assert_eq!(normalise_text("a\u{E007F}b"), "ab");
    }

    #[test]
    fn test_strip_all_tag_range() {
        // Ensure the full tag range U+E0001–U+E007F is stripped
        let mut input = String::from("start");
        for cp in 0xE0001..=0xE007Fu32 {
            if let Some(c) = char::from_u32(cp) {
                input.push(c);
            }
        }
        input.push_str("end");
        assert_eq!(normalise_text(&input), "startend");
    }

    // -- Homoglyph mapping --------------------------------------------------

    #[test]
    fn test_cyrillic_a_to_latin_a() {
        assert_eq!(normalise_text("\u{0430}"), "a");
    }

    #[test]
    fn test_cyrillic_e_to_latin_e() {
        assert_eq!(normalise_text("\u{0435}"), "e");
    }

    #[test]
    fn test_cyrillic_o_to_latin_o() {
        assert_eq!(normalise_text("\u{043E}"), "o");
    }

    #[test]
    fn test_cyrillic_p_to_latin_p() {
        assert_eq!(normalise_text("\u{0440}"), "p");
    }

    #[test]
    fn test_cyrillic_c_to_latin_c() {
        assert_eq!(normalise_text("\u{0441}"), "c");
    }

    #[test]
    fn test_mixed_script_homoglyph_attack() {
        // "ignоre" with Cyrillic о (U+043E) → "ignore" with Latin o
        let malicious = "ign\u{043E}re";
        assert_eq!(normalise_text(malicious), "ignore");
    }

    #[test]
    fn test_full_cyrillic_word_looks_like_ignore() {
        // Cyrillic: і + g + n + о + r + е
        let malicious = "\u{0456}gnor\u{0435}";
        assert_eq!(normalise_text(malicious), "ignore");
    }

    #[test]
    fn test_cyrillic_uppercase_confusables() {
        // Cyrillic А, С, Е, О, Р → Latin A, C, E, O, P
        let text = "\u{0410}\u{0421}\u{0415}\u{041E}\u{0420}";
        assert_eq!(normalise_text(text), "ACEOP");
    }

    #[test]
    fn test_greek_omicron_to_latin_o() {
        assert_eq!(normalise_text("\u{03BF}"), "o");
    }

    #[test]
    fn test_greek_uppercase_confusables() {
        // Greek Α, Β, Ε → Latin A, B, E
        let text = "\u{0391}\u{0392}\u{0395}";
        assert_eq!(normalise_text(text), "ABE");
    }

    // -- Upside-down text mapping (IS-021) ---------------------------------

    #[test]
    fn test_upside_down_individual_chars() {
        assert_eq!(map_homoglyph('\u{0250}'), 'a'); // ɐ
        assert_eq!(map_homoglyph('\u{0254}'), 'c'); // ɔ
        assert_eq!(map_homoglyph('\u{01DD}'), 'e'); // ǝ
        assert_eq!(map_homoglyph('\u{025F}'), 'f'); // ɟ
        assert_eq!(map_homoglyph('\u{0183}'), 'g'); // ƃ
        assert_eq!(map_homoglyph('\u{0265}'), 'h'); // ɥ
        assert_eq!(map_homoglyph('\u{0131}'), 'i'); // ı
        assert_eq!(map_homoglyph('\u{027E}'), 'j'); // ɾ
        assert_eq!(map_homoglyph('\u{029E}'), 'k'); // ʞ
        assert_eq!(map_homoglyph('\u{026F}'), 'm'); // ɯ
        assert_eq!(map_homoglyph('\u{0279}'), 'r'); // ɹ
        assert_eq!(map_homoglyph('\u{0287}'), 't'); // ʇ
        assert_eq!(map_homoglyph('\u{028C}'), 'v'); // ʌ
        assert_eq!(map_homoglyph('\u{028D}'), 'w'); // ʍ
        assert_eq!(map_homoglyph('\u{028E}'), 'y'); // ʎ
    }

    #[test]
    fn test_upside_down_uppercase_chars() {
        assert_eq!(map_homoglyph('\u{2200}'), 'A'); // ∀
        assert_eq!(map_homoglyph('\u{15FA}'), 'B'); // ᗺ
        assert_eq!(map_homoglyph('\u{0186}'), 'C'); // Ɔ
        assert_eq!(map_homoglyph('\u{15E1}'), 'D'); // ᗡ
        assert_eq!(map_homoglyph('\u{018E}'), 'E'); // Ǝ
        assert_eq!(map_homoglyph('\u{02E5}'), 'L'); // ˥
        assert_eq!(map_homoglyph('\u{0500}'), 'P'); // Ԁ
        assert_eq!(map_homoglyph('\u{1D1A}'), 'R'); // ᴚ
        assert_eq!(map_homoglyph('\u{22A5}'), 'T'); // ⊥
        assert_eq!(map_homoglyph('\u{2229}'), 'U'); // ∩
        assert_eq!(map_homoglyph('\u{039B}'), 'V'); // Λ
    }

    #[test]
    fn test_upside_down_word_hello() {
        // "ɥǝllo" → "hello" (ɥ→h, ǝ→e, l→l, l→l, o→o)
        assert_eq!(normalise_text("\u{0265}\u{01DD}llo"), "hello");
    }

    #[test]
    fn test_upside_down_word_attack() {
        // "ɐʇʇɐɔʞ" → "attack" (ɐ→a, ʇ→t, ʇ→t, ɐ→a, ɔ→c, ʞ→k)
        assert_eq!(
            normalise_text("\u{0250}\u{0287}\u{0287}\u{0250}\u{0254}\u{029E}"),
            "attack"
        );
    }

    #[test]
    fn test_upside_down_word_text() {
        // "ʇǝxʇ" → "text" (ʇ→t, ǝ→e, x→x, ʇ→t)
        assert_eq!(normalise_text("\u{0287}\u{01DD}x\u{0287}"), "text");
    }

    // -- Braille-to-ASCII mapping (IS-015) ---------------------------------

    #[test]
    fn test_braille_individual_letters() {
        assert_eq!(map_homoglyph('\u{2801}'), 'a');
        assert_eq!(map_homoglyph('\u{2803}'), 'b');
        assert_eq!(map_homoglyph('\u{2809}'), 'c');
        assert_eq!(map_homoglyph('\u{2819}'), 'd');
        assert_eq!(map_homoglyph('\u{2811}'), 'e');
        assert_eq!(map_homoglyph('\u{280B}'), 'f');
        assert_eq!(map_homoglyph('\u{281B}'), 'g');
        assert_eq!(map_homoglyph('\u{2813}'), 'h');
        assert_eq!(map_homoglyph('\u{280A}'), 'i');
        assert_eq!(map_homoglyph('\u{281A}'), 'j');
        assert_eq!(map_homoglyph('\u{2805}'), 'k');
        assert_eq!(map_homoglyph('\u{2807}'), 'l');
        assert_eq!(map_homoglyph('\u{280D}'), 'm');
        assert_eq!(map_homoglyph('\u{281D}'), 'n');
        assert_eq!(map_homoglyph('\u{2815}'), 'o');
        assert_eq!(map_homoglyph('\u{280F}'), 'p');
        assert_eq!(map_homoglyph('\u{281F}'), 'q');
        assert_eq!(map_homoglyph('\u{2817}'), 'r');
        assert_eq!(map_homoglyph('\u{280E}'), 's');
        assert_eq!(map_homoglyph('\u{281E}'), 't');
        assert_eq!(map_homoglyph('\u{2825}'), 'u');
        assert_eq!(map_homoglyph('\u{2827}'), 'v');
        assert_eq!(map_homoglyph('\u{283A}'), 'w');
        assert_eq!(map_homoglyph('\u{282D}'), 'x');
        assert_eq!(map_homoglyph('\u{283D}'), 'y');
        assert_eq!(map_homoglyph('\u{2835}'), 'z');
    }

    #[test]
    fn test_braille_blank_to_space() {
        assert_eq!(map_homoglyph('\u{2800}'), ' ');
    }

    #[test]
    fn test_braille_word_hello() {
        // ⠓⠑⠇⠇⠕ → "hello"
        assert_eq!(
            normalise_text("\u{2813}\u{2811}\u{2807}\u{2807}\u{2815}"),
            "hello"
        );
    }

    #[test]
    fn test_braille_word_ignore() {
        // ⠊⠛⠝⠕⠗⠑ → "ignore"
        assert_eq!(
            normalise_text("\u{280A}\u{281B}\u{281D}\u{2815}\u{2817}\u{2811}"),
            "ignore"
        );
    }

    #[test]
    fn test_braille_with_spaces() {
        // ⠊⠛⠝⠕⠗⠑⠀⠞⠓⠊⠎ → "ignore this"
        assert_eq!(
            normalise_text(
                "\u{280A}\u{281B}\u{281D}\u{2815}\u{2817}\u{2811}\u{2800}\u{281E}\u{2813}\u{280A}\u{280E}"
            ),
            "ignore this"
        );
    }

    // -- Diacritics stripping (IS-031) -------------------------------------

    #[test]
    fn test_diacritics_cafe() {
        assert_eq!(normalise_text("café"), "cafe");
    }

    #[test]
    fn test_diacritics_resume() {
        assert_eq!(normalise_text("résumé"), "resume");
    }

    #[test]
    fn test_diacritics_naive() {
        assert_eq!(normalise_text("naïve"), "naive");
    }

    #[test]
    fn test_diacritics_ignore_evasion() {
        // "ïgnörë" → "ignore"
        assert_eq!(normalise_text("ïgnörë"), "ignore");
    }

    #[test]
    fn test_diacritics_multiple_accents() {
        // Various accented Latin characters
        assert_eq!(normalise_text("àáâãäå"), "aaaaaa");
        assert_eq!(normalise_text("èéêë"), "eeee");
        assert_eq!(normalise_text("ñ"), "n");
    }

    #[test]
    fn test_strip_diacritics_standalone() {
        assert_eq!(strip_diacritics("café"), "cafe");
        assert_eq!(strip_diacritics("résumé"), "resume");
        assert_eq!(strip_diacritics("naïve"), "naive");
    }

    // -- Emoji stripping (IS-020) ------------------------------------------

    #[test]
    fn test_strip_emoji_simple() {
        assert_eq!(normalise_text("he😀llo"), "hello");
    }

    #[test]
    fn test_strip_emoji_multiple() {
        assert_eq!(normalise_text("ig🔥no📌re"), "ignore");
    }

    #[test]
    fn test_strip_emoji_skin_tone() {
        // Waving hand + skin tone modifier — both should be stripped
        assert_eq!(normalise_text("a\u{1F44B}\u{1F3FD}b"), "ab");
    }

    #[test]
    fn test_strip_emoji_zwj_sequence() {
        // Family emoji ZWJ sequence: 👨‍👩‍👧‍👦
        // ZWJ (U+200D) is already in ZERO_WIDTH_CHARS, individual emoji are stripped
        assert_eq!(
            normalise_text("a\u{1F468}\u{200D}\u{1F469}\u{200D}\u{1F467}\u{200D}\u{1F466}b"),
            "ab"
        );
    }

    #[test]
    fn test_strip_emoji_variation_selectors() {
        // Variation selector should be stripped
        assert_eq!(normalise_text("a\u{FE0F}b"), "ab");
    }

    #[test]
    fn test_strip_emoji_misc_symbols() {
        // ☀ (U+2600) — misc symbols range
        assert_eq!(normalise_text("a\u{2600}b"), "ab");
    }

    #[test]
    fn test_strip_emoji_dingbats() {
        // ✂ (U+2702) — dingbats range
        assert_eq!(normalise_text("a\u{2702}b"), "ab");
    }

    #[test]
    fn test_strip_emoji_transport() {
        // 🚀 (U+1F680) — transport range
        assert_eq!(normalise_text("a\u{1F680}b"), "ab");
    }

    #[test]
    fn test_strip_emoji_standalone_function() {
        assert_eq!(strip_emoji("hello 🌍 world"), "hello  world");
        assert_eq!(strip_emoji("ig🔥no📌re"), "ignore");
        assert_eq!(strip_emoji("no emoji here"), "no emoji here");
    }

    #[test]
    fn test_strip_emoji_preserves_text_between() {
        assert_eq!(
            normalise_text("Ignore 🎯 previous 🔥 instructions"),
            "Ignore  previous  instructions"
        );
    }

    // -- Combined attacks --------------------------------------------------

    #[test]
    fn test_combined_zero_width_and_homoglyph() {
        // "ign\u{200B}\u{043E}re" — zero-width space + Cyrillic о
        let malicious = "ign\u{200B}\u{043E}re";
        assert_eq!(normalise_text(malicious), "ignore");
    }

    #[test]
    fn test_combined_fullwidth_and_zero_width() {
        // Fullwidth "Ｓ" + zero-width + "ystem"
        let malicious = "\u{FF33}\u{200B}ystem";
        assert_eq!(normalise_text(malicious), "System");
    }

    #[test]
    fn test_realistic_evasion_ignore_previous_instructions() {
        // Attacker uses: Cyrillic і, zero-width space, Cyrillic о
        let evasion = "\u{0456}gn\u{200B}\u{043E}re previ\u{043E}us instructi\u{043E}ns";
        let normalised = normalise_text(evasion);
        assert_eq!(normalised, "ignore previous instructions");
    }

    #[test]
    fn test_combined_emoji_and_diacritics() {
        // Emoji interleaved with accented text
        assert_eq!(normalise_text("ïg🔥nörë"), "ignore");
    }

    #[test]
    fn test_combined_emoji_and_upside_down() {
        // Upside-down text with emoji interleaved
        assert_eq!(normalise_text("\u{0265}😀\u{01DD}llo"), "hello");
    }

    #[test]
    fn test_combined_braille_and_zero_width() {
        // Braille "hello" with zero-width chars inserted
        assert_eq!(
            normalise_text("\u{2813}\u{200B}\u{2811}\u{200C}\u{2807}\u{2807}\u{2815}"),
            "hello"
        );
    }

    #[test]
    fn test_combined_all_evasion_techniques() {
        // A single string mixing: diacritics + zero-width + Cyrillic homoglyph +
        // emoji + upside-down + tag characters
        let evasion = concat!(
            "ï",         // i with diaeresis → i (diacritics)
            "\u{200B}",  // zero-width space (stripped)
            "\u{0441}",  // Cyrillic с → c (homoglyph)
            "🔥",        // emoji (stripped)
            "\u{0250}",  // ɐ → a (upside-down)
            "\u{E0041}", // tag A (stripped)
            "\u{0287}",  // ʇ → t (upside-down)
        );
        assert_eq!(normalise_text(evasion), "icat");
    }

    #[test]
    fn test_empty_string() {
        assert_eq!(normalise_text(""), "");
    }

    #[test]
    fn test_only_zero_width_chars() {
        assert_eq!(normalise_text("\u{200B}\u{200C}\u{200D}\u{FEFF}"), "");
    }

    #[test]
    fn test_preserves_normal_unicode() {
        // CJK text should pass through unchanged (emoji is stripped)
        let text = "你好世界";
        assert_eq!(normalise_text(text), text);
    }

    #[test]
    fn test_emoji_stripped_from_cjk_text() {
        // Emoji next to CJK: emoji stripped, CJK preserved
        assert_eq!(normalise_text("你好世界 🌍"), "你好世界 ");
    }

    #[test]
    fn test_diacritics_stripped_from_accented_latin() {
        // Accented characters have diacritics removed for security analysis
        let result = normalise_text("café résumé naïve");
        assert_eq!(result, "cafe resume naive");
    }
}