zer_blocking/keys/
transliterated.rs

1use rphonetic::{DoubleMetaphone, Encoder};
2use zer_core::{record::Record, schema::Schema};
3
4use super::BlockingKey;
5use crate::normalize::{extract_surname_token, transliterate_and_normalize};
6
7// ── TransliteratedPhoneticKey ─────────────────────────────────────────────────
8
9/// Phonetic blocking key that first transliterates non-Latin script (Arabic,
10/// Cyrillic, Greek, etc.) to ASCII via `any_ascii`, then applies NFKD
11/// diacritic stripping and DoubleMetaphone encoding, combined with the DOB
12/// year.
13///
14/// Key format: `"PHONETIC_CODE:YEAR"`
15///
16/// Use alongside `PhoneticNameDobKey` (which only handles already-Latin
17/// input) when your dataset may contain non-Latin name entries, e.g. persons
18/// registered in Arabic script by one Schengen state and in Latin by another.
19pub struct TransliteratedPhoneticKey {
20    name_field: String,
21    dob_field: String,
22}
23
24impl TransliteratedPhoneticKey {
25    pub fn new(name_field: &str, dob_field: &str) -> Self {
26        Self {
27            name_field: name_field.into(),
28            dob_field: dob_field.into(),
29        }
30    }
31}
32
33impl BlockingKey for TransliteratedPhoneticKey {
34    fn name(&self) -> &str {
35        "transliterated_phonetic"
36    }
37
38    fn extract(&self, record: &Record, _schema: &Schema) -> Vec<String> {
39        let name_cow = record.field_as_str(&self.name_field);
40        let name_raw = match name_cow.as_deref() {
41            Some(s) => s,
42            None => return vec![],
43        };
44        let dob_cow = record.field_as_str(&self.dob_field);
45        let dob_raw = match dob_cow.as_deref() {
46            Some(s) => s,
47            None => return vec![],
48        };
49
50        let year = dob_raw.trim().get(..4).unwrap_or("");
51        if year.len() < 4 || !year.chars().all(|c| c.is_ascii_digit()) {
52            return vec![];
53        }
54
55        let norm = transliterate_and_normalize(name_raw);
56        let surname = extract_surname_token(&norm);
57        if surname.is_empty() {
58            return vec![];
59        }
60
61        let code = DoubleMetaphone::default().encode(surname);
62        if code.is_empty() {
63            return vec![];
64        }
65
66        vec![format!("{}:{}", code, year)]
67    }
68}
69
70// ── Tests ─────────────────────────────────────────────────────────────────────
71
72#[cfg(test)]
73mod tests {
74    use super::*;
75    use zer_core::{
76        record::FieldValue,
77        schema::{FieldKind, SchemaBuilder},
78    };
79
80    fn schema() -> Schema {
81        SchemaBuilder::new()
82            .field("naam", FieldKind::Name)
83            .field("dob", FieldKind::Date)
84            .build()
85            .unwrap()
86    }
87
88    fn rec(id: u64, naam: &str, dob: &str) -> Record {
89        Record::new(id)
90            .insert("naam", FieldValue::Text(naam.into()))
91            .insert("dob", FieldValue::Text(dob.into()))
92    }
93
94    #[test]
95    fn latin_diacritic_name_produces_key() {
96        let schema = schema();
97        let key = TransliteratedPhoneticKey::new("naam", "dob");
98        let r = rec(1, "Müller", "1985-03-01");
99        let keys = key.extract(&r, &schema);
100        // "Müller" → any_ascii → "Muller" → normalize → "MULLER" → phonetic + year
101        assert_eq!(keys.len(), 1);
102        assert!(keys[0].ends_with(":1985"), "key should contain DOB year");
103    }
104
105    #[test]
106    fn latin_and_arabic_transliteration_collide() {
107        let schema = schema();
108        let key = TransliteratedPhoneticKey::new("naam", "dob");
109
110        // Arabic "بن عبدالله" transliterates to approximately "bn abdallh" → surname token
111        // This is a smoke test: both produce non-empty keys for the same DOB year.
112        let r_latin = rec(1, "Benabdallah", "1999-01-01");
113        let r_arabic = rec(2, "بن عبدالله", "1999-01-01");
114
115        let k1 = key.extract(&r_latin, &schema);
116        let k2 = key.extract(&r_arabic, &schema);
117
118        // Both should produce non-empty keys, exact collision depends on any_ascii
119        assert!(!k1.is_empty(), "Latin name should produce a key");
120        assert!(
121            !k2.is_empty(),
122            "Arabic name should produce a key after transliteration"
123        );
124    }
125
126    #[test]
127    fn missing_dob_returns_empty() {
128        let schema = schema();
129        let key = TransliteratedPhoneticKey::new("naam", "dob");
130        let r = Record::new(1).insert("naam", FieldValue::Text("Jansen".into()));
131        assert!(key.extract(&r, &schema).is_empty());
132    }
133
134    #[test]
135    fn missing_name_returns_empty() {
136        let schema = schema();
137        let key = TransliteratedPhoneticKey::new("naam", "dob");
138        let r = Record::new(1).insert("dob", FieldValue::Text("1990-01-01".into()));
139        assert!(key.extract(&r, &schema).is_empty());
140    }
141
142    #[test]
143    fn tussenvoegsel_stripped_before_phonetic() {
144        let schema = schema();
145        let key = TransliteratedPhoneticKey::new("naam", "dob");
146
147        let r1 = rec(1, "van den Berg", "1990-06-15");
148        let r2 = rec(2, "Berg", "1990-06-15");
149
150        let k1 = key.extract(&r1, &schema);
151        let k2 = key.extract(&r2, &schema);
152
153        assert!(!k1.is_empty());
154        assert_eq!(
155            k1, k2,
156            "van den Berg and Berg should produce the same phonetic key"
157        );
158    }
159}
zer_blocking/keys/transliterated.rs

zer_blocking/keys/
transliterated.rs
