urdu_text/lib.rs
1//! Unicode normalization for Urdu text.
2//!
3//! `کتاب` typed on an Urdu keyboard and `كتاب` typed on an Arabic keyboard look
4//! identical on screen. They are not the same bytes: the third letter is
5//! `U+06A9` in one and `U+0643` in the other. So `a == b` is `false`, your
6//! search box returns nothing, and your deduplication misses half the corpus.
7//!
8//! This crate folds those shape-identical duplicates together, strips the marks
9//! that carry no meaning, and expands the pre-shaped glyphs that PDF extraction
10//! leaves behind.
11//!
12//! ```
13//! use urdu_text::{eq_normalized, normalize};
14//!
15//! // Arabic kaf vs Urdu keheh — same word, different bytes.
16//! assert!(eq_normalized("كتاب", "کتاب"));
17//!
18//! // Tatweel is decoration for line justification; it means nothing.
19//! assert_eq!(normalize("کتـــاب"), "کتاب");
20//!
21//! // A lam-alef ligature out of a PDF, expanded back into two letters.
22//! assert_eq!(normalize("\u{FEFB}ہور"), "لاہور");
23//! ```
24//!
25//! # What this crate refuses to do
26//!
27//! Urdu is written in a Perso-Arabic script, so it is tempting to reach for an
28//! Arabic normalizer. Every general-purpose Arabic normalizer gets at least one
29//! of the following wrong, and each mistake silently changes what the text says:
30//!
31//! | Fold | Arabic normalizers | This crate | Why |
32//! |------|--------------------|------------|-----|
33//! | `آ` → `ا` | applied | **never** | `آم` "mango" ≠ `ام` |
34//! | `ے` → `ی` | applied | **never** | `کے` "of" ≠ `کی` "whose" |
35//! | `ھ` → `ہ` | applied | **never** | `تھا` "was" → `تہا`, not a word |
36//! | strip hamza `U+0654` | applied | **never** | `کوئی` "someone" → `کوی`, not a word |
37//!
38//! In Arabic, `آ` really is alef with a maddah stacked on it, and folding it is
39//! defensible. In Urdu it is its own letter with its own place in the alphabet.
40//! Hamza is worse: `U+0654` sits inside the Unicode combining-marks range, so
41//! "strip all combining marks" removes it — and in Urdu, hamza is spelling.
42//!
43//! # Configuring
44//!
45//! [`normalize`] applies a default preset. For anything else, build an
46//! [`Options`] and call [`normalize_with`]:
47//!
48//! ```
49//! use urdu_text::{normalize_with, Options};
50//!
51//! // Everything on, plus ASCII digits (off in the default preset).
52//! let indexing = Options { ascii_digits: true, ..Options::default() };
53//! assert_eq!(normalize_with("۲۰۲۴ء", &indexing), "2024ء");
54//!
55//! // Or start from nothing and switch on exactly one thing. Here the tatweel
56//! // goes but the Arabic kaf U+0643 is deliberately left as it was.
57//! let gentle = Options { strip_tatweel: true, ..Options::none() };
58//! assert_eq!(normalize_with("\u{0643}ت\u{0640}اب", &gentle), "\u{0643}تاب");
59//! ```
60//!
61//! [`Options`] is deliberately *not* `#[non_exhaustive]`, so `..Options::default()`
62//! works from outside the crate. The trade-off is that adding a field is a
63//! breaking change — which is the honest signal, because adding a normalization
64//! step changes everyone's output anyway.
65//!
66//! # Properties
67//!
68//! - **Idempotent.** `normalize(normalize(s)) == normalize(s)` for every preset.
69//! - **Single pass, one allocation.** No intermediate `String`s.
70//! - **Zero dependencies**, `#![no_std]` (needs `alloc`), `#![forbid(unsafe_code)]`.
71//!
72//! # Where the decisions live
73//!
74//! Every character table is in `src/tables.rs`, one judgement call per comment.
75//! The README lists the calls a native speaker should second-guess.
76
77#![no_std]
78#![forbid(unsafe_code)]
79#![deny(missing_docs)]
80
81extern crate alloc;
82
83use alloc::string::String;
84
85mod tables;
86
87pub use tables::{
88 expand_ligature, fold_letter, is_aerab_char, presentation_base, to_ascii_digit, TATWEEL,
89};
90
91use tables::is_format_mark;
92
93/// Which normalization steps to apply.
94///
95/// Every field defaults to `true` except [`ascii_digits`](Options::ascii_digits).
96/// Use `..Options::default()` to change one thing, or [`Options::none()`] to
97/// start from all-off.
98///
99/// ```
100/// use urdu_text::Options;
101///
102/// let o = Options { collapse_whitespace: false, ..Options::default() };
103/// assert!(o.fold_letters);
104/// assert!(!o.collapse_whitespace);
105/// ```
106#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
107pub struct Options {
108 /// Map Arabic Presentation Forms-B (`U+FE80`–`U+FEFC`) back to base
109 /// letters, expanding the lam-alef ligatures into two characters.
110 ///
111 /// Turn this on for anything extracted from a PDF. Default: `true`.
112 pub expand_presentation_forms: bool,
113
114 /// Fold shape-identical Arabic and Persian letterforms to their Urdu
115 /// codepoints — see [`fold_letter`] for the exact table, including the
116 /// four folds this crate refuses to make.
117 ///
118 /// Default: `true`.
119 pub fold_letters: bool,
120
121 /// Remove aerab (harakat / diacritics), leaving hamza in place.
122 ///
123 /// See [`is_aerab_char`] for why hamza is excluded. Default: `true`.
124 pub strip_aerab: bool,
125
126 /// Remove tatweel (`U+0640`), the kashida elongation used to justify
127 /// lines of text. Default: `true`.
128 pub strip_tatweel: bool,
129
130 /// Remove invisible formatting and bidi control characters: zero-width
131 /// space/joiner/non-joiner, LRM/RLM, the embedding and isolate controls,
132 /// and the BOM.
133 ///
134 /// Default: `true`.
135 pub strip_format_marks: bool,
136
137 /// Convert Urdu and Arabic-Indic digits to ASCII `0`–`9`.
138 ///
139 /// **Defaults to `false`**, unlike every other field. Rewriting `۲۰۲۴` as
140 /// `2024` changes how the text reads, so it is opt-in: right for a search
141 /// index, wrong for text you intend to display back to a reader.
142 pub ascii_digits: bool,
143
144 /// Collapse each run of whitespace to a single ASCII space, and trim the
145 /// leading and trailing runs.
146 ///
147 /// Runs after format marks are stripped, so a zero-width space wedged
148 /// between two blanks does not defeat the collapse. Default: `true`.
149 pub collapse_whitespace: bool,
150}
151
152impl Default for Options {
153 fn default() -> Self {
154 Self {
155 expand_presentation_forms: true,
156 fold_letters: true,
157 strip_aerab: true,
158 strip_tatweel: true,
159 strip_format_marks: true,
160 ascii_digits: false,
161 collapse_whitespace: true,
162 }
163 }
164}
165
166impl Options {
167 /// Every step switched off — a starting point for building up exactly the
168 /// normalization you want.
169 ///
170 /// `normalize_with(s, &Options::none())` returns `s` unchanged.
171 ///
172 /// ```
173 /// use urdu_text::{normalize_with, Options};
174 ///
175 /// let only_ligatures = Options {
176 /// expand_presentation_forms: true,
177 /// ..Options::none()
178 /// };
179 /// // The ligature expands to lam + alef, but Arabic kaf is left alone.
180 /// assert_eq!(
181 /// normalize_with("\u{FEFB}\u{0643}", &only_ligatures),
182 /// "\u{0644}\u{0627}\u{0643}"
183 /// );
184 /// ```
185 #[must_use]
186 pub const fn none() -> Self {
187 Self {
188 expand_presentation_forms: false,
189 fold_letters: false,
190 strip_aerab: false,
191 strip_tatweel: false,
192 strip_format_marks: false,
193 ascii_digits: false,
194 collapse_whitespace: false,
195 }
196 }
197}
198
199/// Normalize Urdu text with the default preset.
200///
201/// Equivalent to `normalize_with(input, &Options::default())`: presentation
202/// forms expanded, letters folded, aerab and tatweel and invisible marks
203/// stripped, whitespace collapsed, digits left as they are.
204///
205/// # Examples
206///
207/// ```
208/// use urdu_text::normalize;
209///
210/// // Arabic kaf U+0643 folds to Urdu keheh U+06A9.
211/// assert_eq!(normalize("كتاب"), "کتاب");
212///
213/// // Aerab removed, hamza kept — both the combining and precomposed forms.
214/// assert_eq!(normalize("\u{06CC}\u{0654}"), "\u{06CC}\u{0654}");
215/// assert_eq!(normalize("کِتاب"), "کتاب");
216/// assert_eq!(normalize("کوئی"), "کوئی");
217///
218/// // Whitespace collapsed and trimmed.
219/// assert_eq!(normalize(" اردو زبان "), "اردو زبان");
220///
221/// // Digits are left alone by default.
222/// assert_eq!(normalize("۲۰۲۴"), "۲۰۲۴");
223/// ```
224#[must_use]
225pub fn normalize(input: &str) -> String {
226 normalize_with(input, &Options::default())
227}
228
229/// Normalize Urdu text with an explicit [`Options`].
230///
231/// Runs in a single pass over `input` with one allocation.
232///
233/// # Examples
234///
235/// ```
236/// use urdu_text::{normalize_with, Options};
237///
238/// let searchable = Options { ascii_digits: true, ..Options::default() };
239/// assert_eq!(normalize_with("سن ۱۹۴۷", &searchable), "سن 1947");
240///
241/// // Keep the diacritics, fix everything else.
242/// let keep_aerab = Options { strip_aerab: false, ..Options::default() };
243/// assert_eq!(normalize_with("كِتاب", &keep_aerab), "کِتاب");
244/// ```
245#[must_use]
246pub fn normalize_with(input: &str, options: &Options) -> String {
247 // One allocation. Normalization only ever shrinks or holds steady, with the
248 // single exception of ligature expansion (1 char -> 2), which is rare
249 // enough that reserving for it would waste more than it saves.
250 let mut out = String::with_capacity(input.len());
251 let mut pending_space = false;
252
253 for ch in input.chars() {
254 // Presentation-form handling is the only step that can produce two
255 // characters, so it happens up front and feeds the rest of the
256 // pipeline one character at a time.
257 let (first, second) = if options.expand_presentation_forms {
258 match expand_ligature(ch) {
259 Some((a, b)) => (a, Some(b)),
260 None => (presentation_base(ch).unwrap_or(ch), None),
261 }
262 } else {
263 (ch, None)
264 };
265
266 push_normalized(&mut out, first, options, &mut pending_space);
267 if let Some(b) = second {
268 push_normalized(&mut out, b, options, &mut pending_space);
269 }
270 }
271
272 out
273}
274
275/// Apply the per-character steps and append, if anything survives.
276///
277/// `pending_space` implements whitespace collapsing without a second pass: a
278/// run of whitespace only sets the flag, and the flag is only spent when a real
279/// character follows. Leading whitespace never sets it (`out` is still empty)
280/// and trailing whitespace never spends it, so trimming falls out for free.
281fn push_normalized(out: &mut String, ch: char, options: &Options, pending_space: &mut bool) {
282 if options.strip_tatweel && ch == TATWEEL {
283 return;
284 }
285 if options.strip_aerab && is_aerab_char(ch) {
286 return;
287 }
288 if options.strip_format_marks && is_format_mark(ch) {
289 return;
290 }
291 if options.collapse_whitespace && ch.is_whitespace() {
292 if !out.is_empty() {
293 *pending_space = true;
294 }
295 return;
296 }
297
298 let mut ch = ch;
299 if options.fold_letters {
300 ch = fold_letter(ch);
301 }
302 if options.ascii_digits {
303 ch = to_ascii_digit(ch).unwrap_or(ch);
304 }
305
306 if *pending_space {
307 out.push(' ');
308 *pending_space = false;
309 }
310 out.push(ch);
311}
312
313/// Are these two strings the same once normalized?
314///
315/// This is what you want instead of `a == b` when comparing Urdu that came from
316/// more than one source.
317///
318/// # Examples
319///
320/// ```
321/// use urdu_text::eq_normalized;
322///
323/// assert!(eq_normalized("كتاب", "کتاب")); // Arabic kaf vs Urdu keheh
324/// assert!(eq_normalized("کتـــاب", "کتاب")); // tatweel
325///
326/// // Still different words, and still compare as such.
327/// assert!(!eq_normalized("کے", "کی"));
328/// assert!(!eq_normalized("آم", "ام"));
329/// assert!(!eq_normalized("تھا", "تہا"));
330/// ```
331#[must_use]
332pub fn eq_normalized(a: &str, b: &str) -> bool {
333 normalize(a) == normalize(b)
334}
335
336/// Remove tatweel (`U+0640`) and nothing else.
337///
338/// ```
339/// use urdu_text::strip_tatweel;
340///
341/// assert_eq!(strip_tatweel("کتـــاب"), "کتاب");
342/// // Nothing else is touched — the Arabic kaf survives.
343/// assert_eq!(strip_tatweel("\u{0643}ـ"), "\u{0643}");
344/// ```
345#[must_use]
346pub fn strip_tatweel(input: &str) -> String {
347 input.chars().filter(|&c| c != TATWEEL).collect()
348}
349
350/// Remove aerab (diacritics) and nothing else.
351///
352/// Hamza (`U+0654`, `U+0655`) is **kept** — see [`is_aerab_char`].
353///
354/// ```
355/// use urdu_text::strip_aerab;
356///
357/// assert_eq!(strip_aerab("کِتاب"), "کتاب");
358/// // A combining hamza survives, because removing it would change the word.
359/// assert_eq!(strip_aerab("\u{06CC}\u{0654}"), "\u{06CC}\u{0654}");
360/// // ...and so does the precomposed letter U+0626 ئ.
361/// assert_eq!(strip_aerab("کوئی"), "کوئی");
362/// ```
363#[must_use]
364pub fn strip_aerab(input: &str) -> String {
365 input.chars().filter(|&c| !is_aerab_char(c)).collect()
366}