urdu-text 0.1.0

Unicode normalization for Urdu: folds Arabic and Persian letterforms to their Urdu codepoints, strips aerab and tatweel, and expands PDF presentation forms.
Documentation
//! Unicode normalization for Urdu text.
//!
//! `کتاب` typed on an Urdu keyboard and `كتاب` typed on an Arabic keyboard look
//! identical on screen. They are not the same bytes: the third letter is
//! `U+06A9` in one and `U+0643` in the other. So `a == b` is `false`, your
//! search box returns nothing, and your deduplication misses half the corpus.
//!
//! This crate folds those shape-identical duplicates together, strips the marks
//! that carry no meaning, and expands the pre-shaped glyphs that PDF extraction
//! leaves behind.
//!
//! ```
//! use urdu_text::{eq_normalized, normalize};
//!
//! // Arabic kaf vs Urdu keheh — same word, different bytes.
//! assert!(eq_normalized("كتاب", "کتاب"));
//!
//! // Tatweel is decoration for line justification; it means nothing.
//! assert_eq!(normalize("کتـــاب"), "کتاب");
//!
//! // A lam-alef ligature out of a PDF, expanded back into two letters.
//! assert_eq!(normalize("\u{FEFB}ہور"), "لاہور");
//! ```
//!
//! # What this crate refuses to do
//!
//! Urdu is written in a Perso-Arabic script, so it is tempting to reach for an
//! Arabic normalizer. Every general-purpose Arabic normalizer gets at least one
//! of the following wrong, and each mistake silently changes what the text says:
//!
//! | Fold | Arabic normalizers | This crate | Why |
//! |------|--------------------|------------|-----|
//! | `آ` → `ا` | applied | **never** | `آم` "mango" ≠ `ام` |
//! | `ے` → `ی` | applied | **never** | `کے` "of" ≠ `کی` "whose" |
//! | `ھ` → `ہ` | applied | **never** | `تھا` "was" → `تہا`, not a word |
//! | strip hamza `U+0654` | applied | **never** | `کوئی` "someone" → `کوی`, not a word |
//!
//! In Arabic, `آ` really is alef with a maddah stacked on it, and folding it is
//! defensible. In Urdu it is its own letter with its own place in the alphabet.
//! Hamza is worse: `U+0654` sits inside the Unicode combining-marks range, so
//! "strip all combining marks" removes it — and in Urdu, hamza is spelling.
//!
//! # Configuring
//!
//! [`normalize`] applies a default preset. For anything else, build an
//! [`Options`] and call [`normalize_with`]:
//!
//! ```
//! use urdu_text::{normalize_with, Options};
//!
//! // Everything on, plus ASCII digits (off in the default preset).
//! let indexing = Options { ascii_digits: true, ..Options::default() };
//! assert_eq!(normalize_with("۲۰۲۴ء", &indexing), "2024ء");
//!
//! // Or start from nothing and switch on exactly one thing. Here the tatweel
//! // goes but the Arabic kaf U+0643 is deliberately left as it was.
//! let gentle = Options { strip_tatweel: true, ..Options::none() };
//! assert_eq!(normalize_with("\u{0643}ت\u{0640}اب", &gentle), "\u{0643}تاب");
//! ```
//!
//! [`Options`] is deliberately *not* `#[non_exhaustive]`, so `..Options::default()`
//! works from outside the crate. The trade-off is that adding a field is a
//! breaking change — which is the honest signal, because adding a normalization
//! step changes everyone's output anyway.
//!
//! # Properties
//!
//! - **Idempotent.** `normalize(normalize(s)) == normalize(s)` for every preset.
//! - **Single pass, one allocation.** No intermediate `String`s.
//! - **Zero dependencies**, `#![no_std]` (needs `alloc`), `#![forbid(unsafe_code)]`.
//!
//! # Where the decisions live
//!
//! Every character table is in `src/tables.rs`, one judgement call per comment.
//! The README lists the calls a native speaker should second-guess.

#![no_std]
#![forbid(unsafe_code)]
#![deny(missing_docs)]

extern crate alloc;

use alloc::string::String;

mod tables;

pub use tables::{
    expand_ligature, fold_letter, is_aerab_char, presentation_base, to_ascii_digit, TATWEEL,
};

use tables::is_format_mark;

/// Which normalization steps to apply.
///
/// Every field defaults to `true` except [`ascii_digits`](Options::ascii_digits).
/// Use `..Options::default()` to change one thing, or [`Options::none()`] to
/// start from all-off.
///
/// ```
/// use urdu_text::Options;
///
/// let o = Options { collapse_whitespace: false, ..Options::default() };
/// assert!(o.fold_letters);
/// assert!(!o.collapse_whitespace);
/// ```
#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
pub struct Options {
    /// Map Arabic Presentation Forms-B (`U+FE80`–`U+FEFC`) back to base
    /// letters, expanding the lam-alef ligatures into two characters.
    ///
    /// Turn this on for anything extracted from a PDF. Default: `true`.
    pub expand_presentation_forms: bool,

    /// Fold shape-identical Arabic and Persian letterforms to their Urdu
    /// codepoints — see [`fold_letter`] for the exact table, including the
    /// four folds this crate refuses to make.
    ///
    /// Default: `true`.
    pub fold_letters: bool,

    /// Remove aerab (harakat / diacritics), leaving hamza in place.
    ///
    /// See [`is_aerab_char`] for why hamza is excluded. Default: `true`.
    pub strip_aerab: bool,

    /// Remove tatweel (`U+0640`), the kashida elongation used to justify
    /// lines of text. Default: `true`.
    pub strip_tatweel: bool,

    /// Remove invisible formatting and bidi control characters: zero-width
    /// space/joiner/non-joiner, LRM/RLM, the embedding and isolate controls,
    /// and the BOM.
    ///
    /// Default: `true`.
    pub strip_format_marks: bool,

    /// Convert Urdu and Arabic-Indic digits to ASCII `0`–`9`.
    ///
    /// **Defaults to `false`**, unlike every other field. Rewriting `۲۰۲۴` as
    /// `2024` changes how the text reads, so it is opt-in: right for a search
    /// index, wrong for text you intend to display back to a reader.
    pub ascii_digits: bool,

    /// Collapse each run of whitespace to a single ASCII space, and trim the
    /// leading and trailing runs.
    ///
    /// Runs after format marks are stripped, so a zero-width space wedged
    /// between two blanks does not defeat the collapse. Default: `true`.
    pub collapse_whitespace: bool,
}

impl Default for Options {
    fn default() -> Self {
        Self {
            expand_presentation_forms: true,
            fold_letters: true,
            strip_aerab: true,
            strip_tatweel: true,
            strip_format_marks: true,
            ascii_digits: false,
            collapse_whitespace: true,
        }
    }
}

impl Options {
    /// Every step switched off — a starting point for building up exactly the
    /// normalization you want.
    ///
    /// `normalize_with(s, &Options::none())` returns `s` unchanged.
    ///
    /// ```
    /// use urdu_text::{normalize_with, Options};
    ///
    /// let only_ligatures = Options {
    ///     expand_presentation_forms: true,
    ///     ..Options::none()
    /// };
    /// // The ligature expands to lam + alef, but Arabic kaf is left alone.
    /// assert_eq!(
    ///     normalize_with("\u{FEFB}\u{0643}", &only_ligatures),
    ///     "\u{0644}\u{0627}\u{0643}"
    /// );
    /// ```
    #[must_use]
    pub const fn none() -> Self {
        Self {
            expand_presentation_forms: false,
            fold_letters: false,
            strip_aerab: false,
            strip_tatweel: false,
            strip_format_marks: false,
            ascii_digits: false,
            collapse_whitespace: false,
        }
    }
}

/// Normalize Urdu text with the default preset.
///
/// Equivalent to `normalize_with(input, &Options::default())`: presentation
/// forms expanded, letters folded, aerab and tatweel and invisible marks
/// stripped, whitespace collapsed, digits left as they are.
///
/// # Examples
///
/// ```
/// use urdu_text::normalize;
///
/// // Arabic kaf U+0643 folds to Urdu keheh U+06A9.
/// assert_eq!(normalize("كتاب"), "کتاب");
///
/// // Aerab removed, hamza kept — both the combining and precomposed forms.
/// assert_eq!(normalize("\u{06CC}\u{0654}"), "\u{06CC}\u{0654}");
/// assert_eq!(normalize("کِتاب"), "کتاب");
/// assert_eq!(normalize("کوئی"), "کوئی");
///
/// // Whitespace collapsed and trimmed.
/// assert_eq!(normalize("  اردو   زبان  "), "اردو زبان");
///
/// // Digits are left alone by default.
/// assert_eq!(normalize("۲۰۲۴"), "۲۰۲۴");
/// ```
#[must_use]
pub fn normalize(input: &str) -> String {
    normalize_with(input, &Options::default())
}

/// Normalize Urdu text with an explicit [`Options`].
///
/// Runs in a single pass over `input` with one allocation.
///
/// # Examples
///
/// ```
/// use urdu_text::{normalize_with, Options};
///
/// let searchable = Options { ascii_digits: true, ..Options::default() };
/// assert_eq!(normalize_with("سن ۱۹۴۷", &searchable), "سن 1947");
///
/// // Keep the diacritics, fix everything else.
/// let keep_aerab = Options { strip_aerab: false, ..Options::default() };
/// assert_eq!(normalize_with("كِتاب", &keep_aerab), "کِتاب");
/// ```
#[must_use]
pub fn normalize_with(input: &str, options: &Options) -> String {
    // One allocation. Normalization only ever shrinks or holds steady, with the
    // single exception of ligature expansion (1 char -> 2), which is rare
    // enough that reserving for it would waste more than it saves.
    let mut out = String::with_capacity(input.len());
    let mut pending_space = false;

    for ch in input.chars() {
        // Presentation-form handling is the only step that can produce two
        // characters, so it happens up front and feeds the rest of the
        // pipeline one character at a time.
        let (first, second) = if options.expand_presentation_forms {
            match expand_ligature(ch) {
                Some((a, b)) => (a, Some(b)),
                None => (presentation_base(ch).unwrap_or(ch), None),
            }
        } else {
            (ch, None)
        };

        push_normalized(&mut out, first, options, &mut pending_space);
        if let Some(b) = second {
            push_normalized(&mut out, b, options, &mut pending_space);
        }
    }

    out
}

/// Apply the per-character steps and append, if anything survives.
///
/// `pending_space` implements whitespace collapsing without a second pass: a
/// run of whitespace only sets the flag, and the flag is only spent when a real
/// character follows. Leading whitespace never sets it (`out` is still empty)
/// and trailing whitespace never spends it, so trimming falls out for free.
fn push_normalized(out: &mut String, ch: char, options: &Options, pending_space: &mut bool) {
    if options.strip_tatweel && ch == TATWEEL {
        return;
    }
    if options.strip_aerab && is_aerab_char(ch) {
        return;
    }
    if options.strip_format_marks && is_format_mark(ch) {
        return;
    }
    if options.collapse_whitespace && ch.is_whitespace() {
        if !out.is_empty() {
            *pending_space = true;
        }
        return;
    }

    let mut ch = ch;
    if options.fold_letters {
        ch = fold_letter(ch);
    }
    if options.ascii_digits {
        ch = to_ascii_digit(ch).unwrap_or(ch);
    }

    if *pending_space {
        out.push(' ');
        *pending_space = false;
    }
    out.push(ch);
}

/// Are these two strings the same once normalized?
///
/// This is what you want instead of `a == b` when comparing Urdu that came from
/// more than one source.
///
/// # Examples
///
/// ```
/// use urdu_text::eq_normalized;
///
/// assert!(eq_normalized("كتاب", "کتاب"));   // Arabic kaf vs Urdu keheh
/// assert!(eq_normalized("کتـــاب", "کتاب")); // tatweel
///
/// // Still different words, and still compare as such.
/// assert!(!eq_normalized("کے", "کی"));
/// assert!(!eq_normalized("آم", "ام"));
/// assert!(!eq_normalized("تھا", "تہا"));
/// ```
#[must_use]
pub fn eq_normalized(a: &str, b: &str) -> bool {
    normalize(a) == normalize(b)
}

/// Remove tatweel (`U+0640`) and nothing else.
///
/// ```
/// use urdu_text::strip_tatweel;
///
/// assert_eq!(strip_tatweel("کتـــاب"), "کتاب");
/// // Nothing else is touched — the Arabic kaf survives.
/// assert_eq!(strip_tatweel("\u{0643}ـ"), "\u{0643}");
/// ```
#[must_use]
pub fn strip_tatweel(input: &str) -> String {
    input.chars().filter(|&c| c != TATWEEL).collect()
}

/// Remove aerab (diacritics) and nothing else.
///
/// Hamza (`U+0654`, `U+0655`) is **kept** — see [`is_aerab_char`].
///
/// ```
/// use urdu_text::strip_aerab;
///
/// assert_eq!(strip_aerab("کِتاب"), "کتاب");
/// // A combining hamza survives, because removing it would change the word.
/// assert_eq!(strip_aerab("\u{06CC}\u{0654}"), "\u{06CC}\u{0654}");
/// // ...and so does the precomposed letter U+0626 ئ.
/// assert_eq!(strip_aerab("کوئی"), "کوئی");
/// ```
#[must_use]
pub fn strip_aerab(input: &str) -> String {
    input.chars().filter(|&c| !is_aerab_char(c)).collect()
}