raqeem_core/
arabic.rs

1//! Arabic text normalization — a Rust port of scout's verified `normalize_ar`
2//! (from the `arabic-first` convention). We fold audio transcripts to a
3//! matching-stable form so downstream parsers (price extraction, commodity
4//! matching) never fuzzy-match raw, unnormalized Arabic.
5//!
6//! Faithful to the Python reference — identical output on all Arabic and ASCII
7//! input: strip diacritics / tatweel / Quranic marks, unify alef and hamza
8//! carriers, taa-marbuta → haa, Arabic-Indic & Persian digits → ASCII, and the
9//! Arabic decimal separator U+066B → `.` (so «١٢٫٥» folds to `12.5`, one number,
10//! not two). Context-sensitive Unicode case tailoring aside — Python lower-cases a
11//! word-final Greek `Σ` to `ς`, Rust to `σ` — which is out of domain for Arabic
12//! transcripts, and scout re-normalizes via the Python function downstream anyway.
13
14/// Fold Arabic text to a matching-stable form. Idempotent; ASCII passes through
15/// lower-cased, so it is safe to run unconditionally on any transcript.
16pub fn normalize_ar(text: &str) -> String {
17    let mut out = String::with_capacity(text.len());
18    for ch in text.chars() {
19        match ch {
20            // Strip: Quranic annotations, harakat/diacritics, tatweel, superscript alef.
21            '\u{0610}'..='\u{061A}'
22            | '\u{064B}'..='\u{065F}'
23            | '\u{0640}'
24            | '\u{0670}'
25            | '\u{06D6}'..='\u{06ED}' => {}
26            // Alef variants → bare alef.
27            'آ' | 'أ' | 'إ' | 'ٱ' => out.push('ا'),
28            // Hamza-carrier and taa-marbuta folds.
29            'ى' => out.push('ي'), // alef maqsura → yaa
30            'ة' => out.push('ه'), // taa marbuta → haa
31            'ؤ' => out.push('و'),
32            'ئ' => out.push('ي'),
33            // Arabic-Indic (U+0660–0669) and Persian (U+06F0–06F9) digits → ASCII.
34            '\u{0660}'..='\u{0669}' => out.push((b'0' + (ch as u32 - 0x0660) as u8) as char),
35            '\u{06F0}'..='\u{06F9}' => out.push((b'0' + (ch as u32 - 0x06F0) as u8) as char),
36            // Arabic decimal separator → ASCII point.
37            '\u{066B}' => out.push('.'),
38            // Everything else: lower-case ASCII, pass Arabic/other through unchanged.
39            other => out.extend(other.to_lowercase()),
40        }
41    }
42    // Collapse internal whitespace runs and trim.
43    out.split_whitespace().collect::<Vec<_>>().join(" ")
44}
45
46#[cfg(test)]
47mod tests {
48    use super::normalize_ar;
49
50    #[test]
51    fn folds_alef_and_hamza_carriers() {
52        assert_eq!(normalize_ar("أ"), "ا");
53        assert_eq!(normalize_ar("إ"), "ا");
54        assert_eq!(normalize_ar("آ"), "ا");
55        assert_eq!(normalize_ar("ٱ"), "ا");
56        assert_eq!(normalize_ar("مصطفى"), "مصطفي"); // alef maqsura → yaa
57        assert_eq!(normalize_ar("مسؤول"), "مسوول"); // ؤ → و
58    }
59
60    #[test]
61    fn taa_marbuta_becomes_haa() {
62        let r = normalize_ar("طماطة");
63        assert!(!r.contains('ة'));
64        assert!(r.ends_with('ه'));
65    }
66
67    #[test]
68    fn strips_tatweel_and_diacritics() {
69        assert_eq!(normalize_ar("سـلام"), "سلام"); // tatweel U+0640
70        assert_eq!(normalize_ar("سَلاَم"), "سلام"); // harakat
71    }
72
73    #[test]
74    fn digits_and_decimal_separator() {
75        assert_eq!(normalize_ar("١٢٣"), "123"); // arabic-indic
76        assert_eq!(normalize_ar("۱۲۳"), "123"); // persian
77        assert_eq!(normalize_ar("١٢٫٥"), "12.5"); // U+066B decimal → one number
78    }
79
80    #[test]
81    fn ascii_lowercased_and_whitespace_collapsed() {
82        assert_eq!(normalize_ar("Tomato"), "tomato");
83        assert_eq!(normalize_ar("  a   b  "), "a b");
84    }
85
86    #[test]
87    fn idempotent() {
88        let samples = ["أحمد", "طماطة ١٢٫٥ جنيه", "Hello  World", "سـلامٌ عليكم"];
89        for s in samples {
90            let once = normalize_ar(s);
91            assert_eq!(normalize_ar(&once), once, "not idempotent on {s:?}");
92        }
93    }
94}
raqeem_core/arabic.rs

raqeem_core/
arabic.rs
