urdu-text 0.1.0

Unicode normalization for Urdu: folds Arabic and Persian letterforms to their Urdu codepoints, strips aerab and tatweel, and expands PDF presentation forms.
Documentation
//! Character tables and pure per-character lookups.
//!
//! Every linguistic judgement call this crate makes lives in this file, and
//! every one of them carries a comment saying *why*. If you disagree with a
//! decision, it is one `grep` away — you do not have to read the normalizer to
//! find it.
//!
//! The guiding rule: **fold only what is shape-identical and
//! semantically identical in Urdu.** When a fold would change meaning, it is
//! listed under "never folded" below and left alone, even though several
//! Arabic-oriented normalizers do apply it.

/// The tatweel (kashida) character, `U+0640`.
///
/// Tatweel is a typographic elongation mark. It carries no meaning at all —
/// it exists purely to stretch a word so a line of text can be justified.
/// `کتـــاب` and `کتاب` are the same word.
pub const TATWEEL: char = '\u{0640}';

// ---------------------------------------------------------------------------
// Letter folding
// ---------------------------------------------------------------------------

/// Fold a single character to its canonical Urdu form.
///
/// Returns `ch` unchanged if there is nothing to fold. This is the operation
/// that makes text typed on an Arabic keyboard compare equal to the same text
/// typed on an Urdu keyboard.
///
/// # What folds
///
/// | From | To | Reason |
/// |------|----|--------|
/// | `ك` U+0643, `ڪ` U+06AA | `ک` U+06A9 | Arabic kaf and swash kaf render identically to Urdu keheh |
/// | `ي` U+064A, `ى` U+0649, `ې` U+06D0, U+063D, U+063F | `ی` U+06CC | All render as Urdu choti ye |
/// | `ه` U+0647, `ە` U+06D5 | `ہ` U+06C1 | Arabic heh renders as Urdu gol he |
/// | `ۀ` U+06C0 | `ۂ` U+06C2 | Same izafat form, Urdu-preferred codepoint |
/// | `ة` U+0629 | `ۃ` U+06C3 | Teh marbuta, Urdu-preferred codepoint |
/// | `٠`–`٩` U+0660–U+0669 | `۰`–`۹` U+06F0–U+06F9 | Arabic-Indic digits render as Urdu digits |
///
/// # What never folds
///
/// These four are the reason this crate exists. Arabic normalizers apply them
/// and silently corrupt Urdu:
///
/// - `آ` U+0622 is **never** `ا` — it is a distinct letter, not a decorated
///   alef. `آم` ("mango") and `ام` are different words.
/// - `ے` U+06D2 (bari ye) is **never** `ی` — `کے` ("of") and `کی` ("whose")
///   are different words.
/// - `ھ` U+06BE (do-chashmi he) is **never** `ہ` — it forms aspirated
///   consonants. `تھا` ("was") becomes `تہا`, which is not a word.
/// - Hamza `U+0654` / `U+0655` are not diacritics; see [`is_aerab_char`].
///
/// # Examples
///
/// ```
/// use urdu_text::fold_letter;
///
/// assert_eq!(fold_letter('\u{0643}'), '\u{06A9}'); // ك -> ک
/// assert_eq!(fold_letter('\u{0622}'), '\u{0622}'); // آ stays آ
/// assert_eq!(fold_letter('\u{06D2}'), '\u{06D2}'); // ے stays ے
/// assert_eq!(fold_letter('a'), 'a');
/// ```
#[must_use]
pub fn fold_letter(ch: char) -> char {
    match ch {
        // --- kaf family -> keheh U+06A9 ک ---------------------------------
        // U+0643 is the Arabic kaf, produced by every Arabic keyboard layout
        // and by most Arabic fonts' default. In Urdu typography it is drawn
        // exactly like keheh. U+06AA is the swash kaf, a Sindhi/older-Urdu
        // codepoint that renders the same way.
        '\u{0643}' | '\u{06AA}' => '\u{06A9}',

        // --- ye family -> farsi yeh U+06CC ی ------------------------------
        // U+064A Arabic yeh carries two dots below in isolated/final position
        // in Arabic fonts but is drawn dotless-final in Urdu fonts, i.e. the
        // same as U+06CC. U+0649 alef maksura is visually identical in Urdu.
        // U+06D0 is Pashto/Kurdish e. U+063D and U+063F are decorated farsi
        // yeh variants that appear in transliteration data.
        //
        // Deliberately absent: U+06D2 ے bari ye. See the doc comment above.
        '\u{064A}' | '\u{0649}' | '\u{06D0}' | '\u{063D}' | '\u{063F}' => '\u{06CC}',

        // --- he family -> gol he U+06C1 ہ ---------------------------------
        // U+0647 Arabic heh is what an Arabic keyboard produces for the sound
        // Urdu writes with U+06C1. U+06D5 (ae) is the Kurdish/Pashto shape,
        // identical in Urdu rendering.
        //
        // Deliberately absent: U+06BE ھ do-chashmi he. See above — it is a
        // different letter that changes consonants, not a variant spelling.
        '\u{0647}' | '\u{06D5}' => '\u{06C1}',

        // --- he-with-hamza (izafat) ---------------------------------------
        // U+06C0 (heh with yeh above) and U+06C2 (heh goal with hamza above)
        // are the same izafat construction. U+06C2 is the Urdu-preferred
        // codepoint because its base is heh *goal*, matching U+06C1 above.
        '\u{06C0}' => '\u{06C2}',

        // --- teh marbuta ---------------------------------------------------
        // U+0629 is the Arabic teh marbuta; U+06C3 is the "goal" variant used
        // in Urdu orthography for the same letter.
        //
        // OPEN QUESTION: some Urdu style guides normalize Arabic loanwords
        // further, ة -> ہ, on the grounds that Urdu pronounces the final
        // sound as /h/. That is a spelling reform, not a shape fold, so this
        // crate does not do it. Flagged in the README for a native speaker.
        '\u{0629}' => '\u{06C3}',

        // --- digits: Arabic-Indic -> Extended Arabic-Indic (Urdu) ----------
        // U+0660..=U+0669 and U+06F0..=U+06F9 encode the same ten digits.
        // Urdu uses the U+06Fx block; four of its glyphs (۴ ۶ ۷ and ۸ in some
        // fonts) are drawn differently from the Arabic ones, so this is a
        // convention choice, not a pure shape fold — but the numeric value is
        // identical, which is what a search index cares about.
        '\u{0660}'..='\u{0669}' => {
            // Safe by construction: the arm above bounds ch to a 10-wide range
            // and U+06F0 + 9 is a valid scalar value.
            char::from_u32(ch as u32 - 0x0660 + 0x06F0).unwrap_or(ch)
        }

        _ => ch,
    }
}

// ---------------------------------------------------------------------------
// Aerab (diacritics)
// ---------------------------------------------------------------------------

/// Is this character an aerab (harakat / diacritic) that carries no meaning
/// once removed?
///
/// Urdu is normally written without aerab. When they do appear — in textbooks,
/// poetry, or religious text — they are pronunciation aids layered on top of a
/// spelling that is already complete. `کِتاب` and `کتاب` are the same word.
///
/// # Hamza is excluded, on purpose
///
/// `U+0654` (hamza above) and `U+0655` (hamza below) sit in the middle of the
/// Unicode combining-marks range and are classified as combining marks, so a
/// naive "strip all combining marks" implementation removes them. In Urdu
/// hamza is **part of the spelling**: `کوئی` ("someone") becomes `کوی`, which
/// is not a word. This function returns `false` for both, which is why the
/// aerab ranges below have a hole at U+0654–U+0655.
///
/// # Examples
///
/// ```
/// use urdu_text::is_aerab_char;
///
/// assert!(is_aerab_char('\u{064E}'));  // zabar
/// assert!(is_aerab_char('\u{0670}'));  // khari zabar (superscript alef)
/// assert!(!is_aerab_char('\u{0654}')); // hamza above — NOT an aerab
/// assert!(!is_aerab_char('\u{0655}')); // hamza below — NOT an aerab
/// assert!(!is_aerab_char('ک'));
/// ```
#[must_use]
pub fn is_aerab_char(ch: char) -> bool {
    matches!(ch,
        // Quranic annotation marks that precede the main block: sallallahou
        // alayhe wasallam, alayhe assallam, high seen, small high marks, etc.
        '\u{0610}'..='\u{061A}'

        // The core harakat: fathatan, dammatan, kasratan, fatha, damma,
        // kasra, shadda, sukun, maddah above, hamza above*, hamza below*,
        // subscript alef... — *except* U+0654 and U+0655.
        //
        // U+064B..=U+0653 stops one short of U+0654 hamza above.
        | '\u{064B}'..='\u{0653}'
        // U+0656..=U+065F resumes one past U+0655 hamza below.
        | '\u{0656}'..='\u{065F}'

        // U+0670 superscript alef (khari zabar) — a vowel length mark.
        | '\u{0670}'

        // Quranic recitation and annotation marks in the Arabic Extended
        // block. U+06DD (end of ayah) and U+06DE (start of rub el hizb) are
        // enclosing/standalone symbols rather than marks, so the ranges skip
        // them: ..=06DC, then 06DF.. .
        | '\u{06D6}'..='\u{06DC}'
        | '\u{06DF}'..='\u{06E8}'
        // U+06E9 is PLACE OF SAJDAH, a standalone symbol — skipped.
        | '\u{06EA}'..='\u{06ED}'

        // Arabic Extended-A tail: Quranic and extended-orthography marks.
        | '\u{08D3}'..='\u{08FF}'
    )
}

// ---------------------------------------------------------------------------
// Format / invisible marks
// ---------------------------------------------------------------------------

/// Is this an invisible formatting or bidirectional-control character?
///
/// These are stripped because they are invisible: two strings that differ only
/// by a zero-width joiner look identical to a human and must compare equal.
/// They are extremely common in text copied out of PDFs and web pages.
pub(crate) fn is_format_mark(ch: char) -> bool {
    matches!(ch,
        // ZWSP, ZWNJ, ZWJ, LRM, RLM.
        //
        // NOTE on U+200C ZWNJ: in Urdu it is sometimes used deliberately to
        // block a ligature that would otherwise form. Removing it is a
        // rendering change, though not a spelling change, so it is included
        // here but flagged in the README as an open question.
        '\u{200B}'..='\u{200F}'

        // Arabic letter mark — bidi control specific to Arabic script.
        | '\u{061C}'

        // Explicit bidi embedding/override controls (LRE, RLE, PDF, LRO, RLO).
        | '\u{202A}'..='\u{202E}'

        // Bidi isolate controls (LRI, RLI, FSI, PDI).
        | '\u{2066}'..='\u{2069}'

        // BOM / zero-width no-break space. Turns up at the head of files read
        // as UTF-8 without BOM handling, and mid-string in concatenated text.
        | '\u{FEFF}'
    )
}

// ---------------------------------------------------------------------------
// Digits
// ---------------------------------------------------------------------------

/// Map an Arabic-Indic or Urdu digit to the corresponding ASCII digit.
///
/// Covers both `U+0660`–`U+0669` (Arabic-Indic) and `U+06F0`–`U+06F9`
/// (Extended Arabic-Indic, used by Urdu). Returns `None` for anything else,
/// including characters that are already ASCII digits.
///
/// This is off by default in [`crate::normalize`] because converting `۲۰۲۴` to
/// `2024` changes how the text *reads*, which is a decision only the caller
/// can make. It is the right choice when you are indexing for search; it is
/// the wrong choice when you are round-tripping text for display.
///
/// # Examples
///
/// ```
/// use urdu_text::to_ascii_digit;
///
/// assert_eq!(to_ascii_digit('\u{06F5}'), Some('5')); // ۵ Urdu five
/// assert_eq!(to_ascii_digit('\u{0665}'), Some('5')); // ٥ Arabic-Indic five
/// assert_eq!(to_ascii_digit('5'), None);             // already ASCII
/// assert_eq!(to_ascii_digit('ک'), None);
/// ```
#[must_use]
pub fn to_ascii_digit(ch: char) -> Option<char> {
    let value = match ch {
        '\u{0660}'..='\u{0669}' => ch as u32 - 0x0660,
        '\u{06F0}'..='\u{06F9}' => ch as u32 - 0x06F0,
        _ => return None,
    };
    // value is 0..=9, so b'0' + value is always a valid ASCII digit.
    char::from_u32(u32::from(b'0') + value)
}

// ---------------------------------------------------------------------------
// Arabic Presentation Forms-B
// ---------------------------------------------------------------------------

/// Base letters for `U+FE80`–`U+FEF4`, indexed by `ch as u32 - 0xFE80`.
///
/// Presentation forms are the pre-shaped isolated/initial/medial/final glyphs
/// that PDF generators and older Windows software emit instead of the abstract
/// letter plus a shaping engine. `ﻛ` (U+FEDB, kaf initial form) *is* the letter
/// kaf — it just has the joining already baked in. Text extracted from a PDF is
/// routinely full of these, and it will not match anything typed on a keyboard
/// until they are mapped back.
///
/// This table maps each form to its **Arabic** base codepoint. Folding to Urdu
/// happens afterwards, by running the result through [`fold_letter`], so the
/// two tables stay independent: `U+FEDB -> U+0643 -> U+06A9`.
///
/// Ligatures (`U+FEF5`–`U+FEFC`) expand to two characters and so live in
/// [`expand_ligature`] instead.
#[rustfmt::skip]
const PRESENTATION_FORMS: [char; 117] = [
    '\u{0621}',                                     // FE80  hamza
    '\u{0622}', '\u{0622}',                         // FE81..FE82  alef with madda  (NOTE: stays آ)
    '\u{0623}', '\u{0623}',                         // FE83..FE84  alef with hamza above
    '\u{0624}', '\u{0624}',                         // FE85..FE86  waw with hamza above
    '\u{0625}', '\u{0625}',                         // FE87..FE88  alef with hamza below
    '\u{0626}', '\u{0626}', '\u{0626}', '\u{0626}', // FE89..FE8C  yeh with hamza above
    '\u{0627}', '\u{0627}',                         // FE8D..FE8E  alef
    '\u{0628}', '\u{0628}', '\u{0628}', '\u{0628}', // FE8F..FE92  beh
    '\u{0629}', '\u{0629}',                         // FE93..FE94  teh marbuta
    '\u{062A}', '\u{062A}', '\u{062A}', '\u{062A}', // FE95..FE98  teh
    '\u{062B}', '\u{062B}', '\u{062B}', '\u{062B}', // FE99..FE9C  theh
    '\u{062C}', '\u{062C}', '\u{062C}', '\u{062C}', // FE9D..FEA0  jeem
    '\u{062D}', '\u{062D}', '\u{062D}', '\u{062D}', // FEA1..FEA4  hah
    '\u{062E}', '\u{062E}', '\u{062E}', '\u{062E}', // FEA5..FEA8  khah
    '\u{062F}', '\u{062F}',                         // FEA9..FEAA  dal
    '\u{0630}', '\u{0630}',                         // FEAB..FEAC  thal
    '\u{0631}', '\u{0631}',                         // FEAD..FEAE  reh
    '\u{0632}', '\u{0632}',                         // FEAF..FEB0  zain
    '\u{0633}', '\u{0633}', '\u{0633}', '\u{0633}', // FEB1..FEB4  seen
    '\u{0634}', '\u{0634}', '\u{0634}', '\u{0634}', // FEB5..FEB8  sheen
    '\u{0635}', '\u{0635}', '\u{0635}', '\u{0635}', // FEB9..FEBC  sad
    '\u{0636}', '\u{0636}', '\u{0636}', '\u{0636}', // FEBD..FEC0  dad
    '\u{0637}', '\u{0637}', '\u{0637}', '\u{0637}', // FEC1..FEC4  tah
    '\u{0638}', '\u{0638}', '\u{0638}', '\u{0638}', // FEC5..FEC8  zah
    '\u{0639}', '\u{0639}', '\u{0639}', '\u{0639}', // FEC9..FECC  ain
    '\u{063A}', '\u{063A}', '\u{063A}', '\u{063A}', // FECD..FED0  ghain
    '\u{0641}', '\u{0641}', '\u{0641}', '\u{0641}', // FED1..FED4  feh
    '\u{0642}', '\u{0642}', '\u{0642}', '\u{0642}', // FED5..FED8  qaf
    '\u{0643}', '\u{0643}', '\u{0643}', '\u{0643}', // FED9..FEDC  kaf     (-> ک via fold_letter)
    '\u{0644}', '\u{0644}', '\u{0644}', '\u{0644}', // FEDD..FEE0  lam
    '\u{0645}', '\u{0645}', '\u{0645}', '\u{0645}', // FEE1..FEE4  meem
    '\u{0646}', '\u{0646}', '\u{0646}', '\u{0646}', // FEE5..FEE8  noon
    '\u{0647}', '\u{0647}', '\u{0647}', '\u{0647}', // FEE9..FEEC  heh     (-> ہ via fold_letter)
    '\u{0648}', '\u{0648}',                         // FEED..FEEE  waw
    '\u{0649}', '\u{0649}',                         // FEEF..FEF0  alef maksura (-> ی)
    '\u{064A}', '\u{064A}', '\u{064A}', '\u{064A}', // FEF1..FEF4  yeh     (-> ی via fold_letter)
];

/// Map a single-character Arabic presentation form back to its base letter.
///
/// Handles `U+FE80`–`U+FEF4`. Returns `None` for anything else, including the
/// lam-alef ligatures `U+FEF5`–`U+FEFC`, which expand to *two* characters and
/// are handled by [`expand_ligature`].
///
/// The value returned is the **Arabic** base codepoint, not the Urdu one — run
/// it through [`fold_letter`] to finish the job. [`crate::normalize`] does both.
///
/// # Examples
///
/// ```
/// use urdu_text::{fold_letter, presentation_base};
///
/// // ﻛ U+FEDB is kaf in initial form.
/// assert_eq!(presentation_base('\u{FEDB}'), Some('\u{0643}'));
/// // Folding then takes Arabic kaf to Urdu keheh.
/// assert_eq!(fold_letter('\u{0643}'), '\u{06A9}');
///
/// assert_eq!(presentation_base('ک'), None);
/// ```
#[must_use]
pub fn presentation_base(ch: char) -> Option<char> {
    if ('\u{FE80}'..='\u{FEF4}').contains(&ch) {
        PRESENTATION_FORMS.get(ch as usize - 0xFE80).copied()
    } else {
        None
    }
}

/// Expand a lam-alef presentation ligature into its two component letters.
///
/// `U+FEF5`–`U+FEFC` are the eight pre-composed lam+alef forms. They are a
/// single codepoint that renders as two letters, so any text containing one
/// will fail to match the spelled-out equivalent and will also give the wrong
/// character count.
///
/// Note that `U+FEF5`/`U+FEF6` expand to lam + **`آ`** (U+0622), not lam +
/// alef — the madda is part of the letter and is preserved.
///
/// # Examples
///
/// ```
/// use urdu_text::expand_ligature;
///
/// // ﻻ U+FEFB is the lam-alef ligature.
/// assert_eq!(expand_ligature('\u{FEFB}'), Some(('\u{0644}', '\u{0627}')));
/// // ﻵ keeps its madda.
/// assert_eq!(expand_ligature('\u{FEF5}'), Some(('\u{0644}', '\u{0622}')));
///
/// assert_eq!(expand_ligature('ل'), None);
/// ```
#[must_use]
pub fn expand_ligature(ch: char) -> Option<(char, char)> {
    // Lam is the first component of all eight; only the alef form differs.
    let second = match ch {
        '\u{FEF5}' | '\u{FEF6}' => '\u{0622}', // lam + alef with madda    ﻵ ﻶ
        '\u{FEF7}' | '\u{FEF8}' => '\u{0623}', // lam + alef w/ hamza above ﻷ ﻸ
        '\u{FEF9}' | '\u{FEFA}' => '\u{0625}', // lam + alef w/ hamza below ﻹ ﻺ
        '\u{FEFB}' | '\u{FEFC}' => '\u{0627}', // lam + plain alef          ﻻ ﻼ
        _ => return None,
    };
    Some(('\u{0644}', second))
}