Skip to main content

urdu_text/
lib.rs

1//! Unicode normalization for Urdu text.
2//!
3//! `کتاب` typed on an Urdu keyboard and `كتاب` typed on an Arabic keyboard look
4//! identical on screen. They are not the same bytes: the third letter is
5//! `U+06A9` in one and `U+0643` in the other. So `a == b` is `false`, your
6//! search box returns nothing, and your deduplication misses half the corpus.
7//!
8//! This crate folds those shape-identical duplicates together, strips the marks
9//! that carry no meaning, and expands the pre-shaped glyphs that PDF extraction
10//! leaves behind.
11//!
12//! ```
13//! use urdu_text::{eq_normalized, normalize};
14//!
15//! // Arabic kaf vs Urdu keheh — same word, different bytes.
16//! assert!(eq_normalized("كتاب", "کتاب"));
17//!
18//! // Tatweel is decoration for line justification; it means nothing.
19//! assert_eq!(normalize("کتـــاب"), "کتاب");
20//!
21//! // A lam-alef ligature out of a PDF, expanded back into two letters.
22//! assert_eq!(normalize("\u{FEFB}ہور"), "لاہور");
23//! ```
24//!
25//! # What this crate refuses to do
26//!
27//! Urdu is written in a Perso-Arabic script, so it is tempting to reach for an
28//! Arabic normalizer. Every general-purpose Arabic normalizer gets at least one
29//! of the following wrong, and each mistake silently changes what the text says:
30//!
31//! | Fold | Arabic normalizers | This crate | Why |
32//! |------|--------------------|------------|-----|
33//! | `آ` → `ا` | applied | **never** | `آم` "mango" ≠ `ام` |
34//! | `ے` → `ی` | applied | **never** | `کے` "of" ≠ `کی` "whose" |
35//! | `ھ` → `ہ` | applied | **never** | `تھا` "was" → `تہا`, not a word |
36//! | strip hamza `U+0654` | applied | **never** | `کوئی` "someone" → `کوی`, not a word |
37//!
38//! In Arabic, `آ` really is alef with a maddah stacked on it, and folding it is
39//! defensible. In Urdu it is its own letter with its own place in the alphabet.
40//! Hamza is worse: `U+0654` sits inside the Unicode combining-marks range, so
41//! "strip all combining marks" removes it — and in Urdu, hamza is spelling.
42//!
43//! # Configuring
44//!
45//! [`normalize`] applies a default preset. For anything else, build an
46//! [`Options`] and call [`normalize_with`]:
47//!
48//! ```
49//! use urdu_text::{normalize_with, Options};
50//!
51//! // Everything on, plus ASCII digits (off in the default preset).
52//! let indexing = Options { ascii_digits: true, ..Options::default() };
53//! assert_eq!(normalize_with("۲۰۲۴ء", &indexing), "2024ء");
54//!
55//! // Or start from nothing and switch on exactly one thing. Here the tatweel
56//! // goes but the Arabic kaf U+0643 is deliberately left as it was.
57//! let gentle = Options { strip_tatweel: true, ..Options::none() };
58//! assert_eq!(normalize_with("\u{0643}ت\u{0640}اب", &gentle), "\u{0643}تاب");
59//! ```
60//!
61//! [`Options`] is deliberately *not* `#[non_exhaustive]`, so `..Options::default()`
62//! works from outside the crate. The trade-off is that adding a field is a
63//! breaking change — which is the honest signal, because adding a normalization
64//! step changes everyone's output anyway.
65//!
66//! # Properties
67//!
68//! - **Idempotent.** `normalize(normalize(s)) == normalize(s)` for every preset.
69//! - **Single pass, one allocation.** No intermediate `String`s.
70//! - **Zero dependencies**, `#![no_std]` (needs `alloc`), `#![forbid(unsafe_code)]`.
71//!
72//! # Where the decisions live
73//!
74//! Every character table is in `src/tables.rs`, one judgement call per comment.
75//! The README lists the calls a native speaker should second-guess.
76
77#![no_std]
78#![forbid(unsafe_code)]
79#![deny(missing_docs)]
80
81extern crate alloc;
82
83use alloc::string::String;
84
85mod tables;
86
87pub use tables::{
88    expand_ligature, fold_letter, is_aerab_char, presentation_base, to_ascii_digit, TATWEEL,
89};
90
91use tables::is_format_mark;
92
93/// Which normalization steps to apply.
94///
95/// Every field defaults to `true` except [`ascii_digits`](Options::ascii_digits).
96/// Use `..Options::default()` to change one thing, or [`Options::none()`] to
97/// start from all-off.
98///
99/// ```
100/// use urdu_text::Options;
101///
102/// let o = Options { collapse_whitespace: false, ..Options::default() };
103/// assert!(o.fold_letters);
104/// assert!(!o.collapse_whitespace);
105/// ```
106#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
107pub struct Options {
108    /// Map Arabic Presentation Forms-B (`U+FE80`–`U+FEFC`) back to base
109    /// letters, expanding the lam-alef ligatures into two characters.
110    ///
111    /// Turn this on for anything extracted from a PDF. Default: `true`.
112    pub expand_presentation_forms: bool,
113
114    /// Fold shape-identical Arabic and Persian letterforms to their Urdu
115    /// codepoints — see [`fold_letter`] for the exact table, including the
116    /// four folds this crate refuses to make.
117    ///
118    /// Default: `true`.
119    pub fold_letters: bool,
120
121    /// Remove aerab (harakat / diacritics), leaving hamza in place.
122    ///
123    /// See [`is_aerab_char`] for why hamza is excluded. Default: `true`.
124    pub strip_aerab: bool,
125
126    /// Remove tatweel (`U+0640`), the kashida elongation used to justify
127    /// lines of text. Default: `true`.
128    pub strip_tatweel: bool,
129
130    /// Remove invisible formatting and bidi control characters: zero-width
131    /// space/joiner/non-joiner, LRM/RLM, the embedding and isolate controls,
132    /// and the BOM.
133    ///
134    /// Default: `true`.
135    pub strip_format_marks: bool,
136
137    /// Convert Urdu and Arabic-Indic digits to ASCII `0`–`9`.
138    ///
139    /// **Defaults to `false`**, unlike every other field. Rewriting `۲۰۲۴` as
140    /// `2024` changes how the text reads, so it is opt-in: right for a search
141    /// index, wrong for text you intend to display back to a reader.
142    pub ascii_digits: bool,
143
144    /// Collapse each run of whitespace to a single ASCII space, and trim the
145    /// leading and trailing runs.
146    ///
147    /// Runs after format marks are stripped, so a zero-width space wedged
148    /// between two blanks does not defeat the collapse. Default: `true`.
149    pub collapse_whitespace: bool,
150}
151
152impl Default for Options {
153    fn default() -> Self {
154        Self {
155            expand_presentation_forms: true,
156            fold_letters: true,
157            strip_aerab: true,
158            strip_tatweel: true,
159            strip_format_marks: true,
160            ascii_digits: false,
161            collapse_whitespace: true,
162        }
163    }
164}
165
166impl Options {
167    /// Every step switched off — a starting point for building up exactly the
168    /// normalization you want.
169    ///
170    /// `normalize_with(s, &Options::none())` returns `s` unchanged.
171    ///
172    /// ```
173    /// use urdu_text::{normalize_with, Options};
174    ///
175    /// let only_ligatures = Options {
176    ///     expand_presentation_forms: true,
177    ///     ..Options::none()
178    /// };
179    /// // The ligature expands to lam + alef, but Arabic kaf is left alone.
180    /// assert_eq!(
181    ///     normalize_with("\u{FEFB}\u{0643}", &only_ligatures),
182    ///     "\u{0644}\u{0627}\u{0643}"
183    /// );
184    /// ```
185    #[must_use]
186    pub const fn none() -> Self {
187        Self {
188            expand_presentation_forms: false,
189            fold_letters: false,
190            strip_aerab: false,
191            strip_tatweel: false,
192            strip_format_marks: false,
193            ascii_digits: false,
194            collapse_whitespace: false,
195        }
196    }
197}
198
199/// Normalize Urdu text with the default preset.
200///
201/// Equivalent to `normalize_with(input, &Options::default())`: presentation
202/// forms expanded, letters folded, aerab and tatweel and invisible marks
203/// stripped, whitespace collapsed, digits left as they are.
204///
205/// # Examples
206///
207/// ```
208/// use urdu_text::normalize;
209///
210/// // Arabic kaf U+0643 folds to Urdu keheh U+06A9.
211/// assert_eq!(normalize("كتاب"), "کتاب");
212///
213/// // Aerab removed, hamza kept — both the combining and precomposed forms.
214/// assert_eq!(normalize("\u{06CC}\u{0654}"), "\u{06CC}\u{0654}");
215/// assert_eq!(normalize("کِتاب"), "کتاب");
216/// assert_eq!(normalize("کوئی"), "کوئی");
217///
218/// // Whitespace collapsed and trimmed.
219/// assert_eq!(normalize("  اردو   زبان  "), "اردو زبان");
220///
221/// // Digits are left alone by default.
222/// assert_eq!(normalize("۲۰۲۴"), "۲۰۲۴");
223/// ```
224#[must_use]
225pub fn normalize(input: &str) -> String {
226    normalize_with(input, &Options::default())
227}
228
229/// Normalize Urdu text with an explicit [`Options`].
230///
231/// Runs in a single pass over `input` with one allocation.
232///
233/// # Examples
234///
235/// ```
236/// use urdu_text::{normalize_with, Options};
237///
238/// let searchable = Options { ascii_digits: true, ..Options::default() };
239/// assert_eq!(normalize_with("سن ۱۹۴۷", &searchable), "سن 1947");
240///
241/// // Keep the diacritics, fix everything else.
242/// let keep_aerab = Options { strip_aerab: false, ..Options::default() };
243/// assert_eq!(normalize_with("كِتاب", &keep_aerab), "کِتاب");
244/// ```
245#[must_use]
246pub fn normalize_with(input: &str, options: &Options) -> String {
247    // One allocation. Normalization only ever shrinks or holds steady, with the
248    // single exception of ligature expansion (1 char -> 2), which is rare
249    // enough that reserving for it would waste more than it saves.
250    let mut out = String::with_capacity(input.len());
251    let mut pending_space = false;
252
253    for ch in input.chars() {
254        // Presentation-form handling is the only step that can produce two
255        // characters, so it happens up front and feeds the rest of the
256        // pipeline one character at a time.
257        let (first, second) = if options.expand_presentation_forms {
258            match expand_ligature(ch) {
259                Some((a, b)) => (a, Some(b)),
260                None => (presentation_base(ch).unwrap_or(ch), None),
261            }
262        } else {
263            (ch, None)
264        };
265
266        push_normalized(&mut out, first, options, &mut pending_space);
267        if let Some(b) = second {
268            push_normalized(&mut out, b, options, &mut pending_space);
269        }
270    }
271
272    out
273}
274
275/// Apply the per-character steps and append, if anything survives.
276///
277/// `pending_space` implements whitespace collapsing without a second pass: a
278/// run of whitespace only sets the flag, and the flag is only spent when a real
279/// character follows. Leading whitespace never sets it (`out` is still empty)
280/// and trailing whitespace never spends it, so trimming falls out for free.
281fn push_normalized(out: &mut String, ch: char, options: &Options, pending_space: &mut bool) {
282    if options.strip_tatweel && ch == TATWEEL {
283        return;
284    }
285    if options.strip_aerab && is_aerab_char(ch) {
286        return;
287    }
288    if options.strip_format_marks && is_format_mark(ch) {
289        return;
290    }
291    if options.collapse_whitespace && ch.is_whitespace() {
292        if !out.is_empty() {
293            *pending_space = true;
294        }
295        return;
296    }
297
298    let mut ch = ch;
299    if options.fold_letters {
300        ch = fold_letter(ch);
301    }
302    if options.ascii_digits {
303        ch = to_ascii_digit(ch).unwrap_or(ch);
304    }
305
306    if *pending_space {
307        out.push(' ');
308        *pending_space = false;
309    }
310    out.push(ch);
311}
312
313/// Are these two strings the same once normalized?
314///
315/// This is what you want instead of `a == b` when comparing Urdu that came from
316/// more than one source.
317///
318/// # Examples
319///
320/// ```
321/// use urdu_text::eq_normalized;
322///
323/// assert!(eq_normalized("كتاب", "کتاب"));   // Arabic kaf vs Urdu keheh
324/// assert!(eq_normalized("کتـــاب", "کتاب")); // tatweel
325///
326/// // Still different words, and still compare as such.
327/// assert!(!eq_normalized("کے", "کی"));
328/// assert!(!eq_normalized("آم", "ام"));
329/// assert!(!eq_normalized("تھا", "تہا"));
330/// ```
331#[must_use]
332pub fn eq_normalized(a: &str, b: &str) -> bool {
333    normalize(a) == normalize(b)
334}
335
336/// Remove tatweel (`U+0640`) and nothing else.
337///
338/// ```
339/// use urdu_text::strip_tatweel;
340///
341/// assert_eq!(strip_tatweel("کتـــاب"), "کتاب");
342/// // Nothing else is touched — the Arabic kaf survives.
343/// assert_eq!(strip_tatweel("\u{0643}ـ"), "\u{0643}");
344/// ```
345#[must_use]
346pub fn strip_tatweel(input: &str) -> String {
347    input.chars().filter(|&c| c != TATWEEL).collect()
348}
349
350/// Remove aerab (diacritics) and nothing else.
351///
352/// Hamza (`U+0654`, `U+0655`) is **kept** — see [`is_aerab_char`].
353///
354/// ```
355/// use urdu_text::strip_aerab;
356///
357/// assert_eq!(strip_aerab("کِتاب"), "کتاب");
358/// // A combining hamza survives, because removing it would change the word.
359/// assert_eq!(strip_aerab("\u{06CC}\u{0654}"), "\u{06CC}\u{0654}");
360/// // ...and so does the precomposed letter U+0626 ئ.
361/// assert_eq!(strip_aerab("کوئی"), "کوئی");
362/// ```
363#[must_use]
364pub fn strip_aerab(input: &str) -> String {
365    input.chars().filter(|&c| !is_aerab_char(c)).collect()
366}