transliteration/
transliteration.rs

1//! Transliteration FST Example
2//!
3//! This example demonstrates how to build and use finite state transducers for
4//! transliteration between different writing systems. It shows:
5//! 1. Building character mapping FSTs for script conversion
6//! 2. Cyrillic to Latin transliteration (Russian → English)
7//! 3. Arabic to Latin transliteration
8//! 4. Greek to Latin transliteration
9//! 5. Bidirectional transliteration systems
10//! 6. Multiple transliteration schemes (BGN/PCGN, ISO, etc.)
11//! 7. Handling digraphs and context-sensitive rules
12//!
13//! Transliteration is essential for cross-language text processing, search,
14//! machine translation, and internationalization of applications.
15//!
16//! Usage:
17//! ```bash
18//! cargo run --example transliteration
19//! ```
20
21use arcweight::prelude::*;
22use std::collections::HashMap;
23
24/// Transliteration scheme types
25#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
26#[allow(dead_code)] // Some variants included for completeness but not used in this example
27enum TransliterationScheme {
28    BgnPcgn,    // Board on Geographic Names / Permanent Committee on Geographical Names
29    Iso,        // ISO transliteration standards
30    AlaLc,      // American Library Association / Library of Congress
31    Scientific, // Scientific transliteration
32    Popular,    // Popular/simplified transliteration
33}
34
35/// Script types for transliteration
36#[derive(Debug, Clone, Copy, PartialEq)]
37#[allow(dead_code)] // Script variants included for API completeness, not all used in example
38enum Script {
39    Cyrillic,
40    Latin,
41    Arabic,
42    Greek,
43    Hebrew,
44}
45
46/// Transliteration mapping entry
47#[derive(Debug, Clone)]
48struct TransliterationRule {
49    source: String,
50    target: String,
51    context_before: Option<String>,
52    context_after: Option<String>,
53    scheme: TransliterationScheme,
54}
55
56/// Transliteration system database
57struct TransliterationSystem {
58    cyrillic_to_latin: HashMap<TransliterationScheme, Vec<TransliterationRule>>,
59    arabic_to_latin: HashMap<TransliterationScheme, Vec<TransliterationRule>>,
60    greek_to_latin: HashMap<TransliterationScheme, Vec<TransliterationRule>>,
61}
62
63impl TransliterationSystem {
64    fn new() -> Self {
65        let mut system = TransliterationSystem {
66            cyrillic_to_latin: HashMap::new(),
67            arabic_to_latin: HashMap::new(),
68            greek_to_latin: HashMap::new(),
69        };
70
71        system.initialize_cyrillic_mappings();
72        system.initialize_arabic_mappings();
73        system.initialize_greek_mappings();
74
75        system
76    }
77
78    fn initialize_cyrillic_mappings(&mut self) {
79        // BGN/PCGN Russian transliteration
80        let bgn_pcgn_rules = vec![
81            // Vowels
82            TransliterationRule {
83                source: "а".to_string(),
84                target: "a".to_string(),
85                context_before: None,
86                context_after: None,
87                scheme: TransliterationScheme::BgnPcgn,
88            },
89            TransliterationRule {
90                source: "е".to_string(),
91                target: "e".to_string(),
92                context_before: None,
93                context_after: None,
94                scheme: TransliterationScheme::BgnPcgn,
95            },
96            TransliterationRule {
97                source: "ё".to_string(),
98                target: "ë".to_string(),
99                context_before: None,
100                context_after: None,
101                scheme: TransliterationScheme::BgnPcgn,
102            },
103            TransliterationRule {
104                source: "и".to_string(),
105                target: "i".to_string(),
106                context_before: None,
107                context_after: None,
108                scheme: TransliterationScheme::BgnPcgn,
109            },
110            TransliterationRule {
111                source: "о".to_string(),
112                target: "o".to_string(),
113                context_before: None,
114                context_after: None,
115                scheme: TransliterationScheme::BgnPcgn,
116            },
117            TransliterationRule {
118                source: "у".to_string(),
119                target: "u".to_string(),
120                context_before: None,
121                context_after: None,
122                scheme: TransliterationScheme::BgnPcgn,
123            },
124            TransliterationRule {
125                source: "ы".to_string(),
126                target: "y".to_string(),
127                context_before: None,
128                context_after: None,
129                scheme: TransliterationScheme::BgnPcgn,
130            },
131            TransliterationRule {
132                source: "э".to_string(),
133                target: "e".to_string(),
134                context_before: None,
135                context_after: None,
136                scheme: TransliterationScheme::BgnPcgn,
137            },
138            TransliterationRule {
139                source: "ю".to_string(),
140                target: "yu".to_string(),
141                context_before: None,
142                context_after: None,
143                scheme: TransliterationScheme::BgnPcgn,
144            },
145            TransliterationRule {
146                source: "я".to_string(),
147                target: "ya".to_string(),
148                context_before: None,
149                context_after: None,
150                scheme: TransliterationScheme::BgnPcgn,
151            },
152            // Consonants
153            TransliterationRule {
154                source: "б".to_string(),
155                target: "b".to_string(),
156                context_before: None,
157                context_after: None,
158                scheme: TransliterationScheme::BgnPcgn,
159            },
160            TransliterationRule {
161                source: "в".to_string(),
162                target: "v".to_string(),
163                context_before: None,
164                context_after: None,
165                scheme: TransliterationScheme::BgnPcgn,
166            },
167            TransliterationRule {
168                source: "г".to_string(),
169                target: "g".to_string(),
170                context_before: None,
171                context_after: None,
172                scheme: TransliterationScheme::BgnPcgn,
173            },
174            TransliterationRule {
175                source: "д".to_string(),
176                target: "d".to_string(),
177                context_before: None,
178                context_after: None,
179                scheme: TransliterationScheme::BgnPcgn,
180            },
181            TransliterationRule {
182                source: "ж".to_string(),
183                target: "zh".to_string(),
184                context_before: None,
185                context_after: None,
186                scheme: TransliterationScheme::BgnPcgn,
187            },
188            TransliterationRule {
189                source: "з".to_string(),
190                target: "z".to_string(),
191                context_before: None,
192                context_after: None,
193                scheme: TransliterationScheme::BgnPcgn,
194            },
195            TransliterationRule {
196                source: "й".to_string(),
197                target: "y".to_string(),
198                context_before: None,
199                context_after: None,
200                scheme: TransliterationScheme::BgnPcgn,
201            },
202            TransliterationRule {
203                source: "к".to_string(),
204                target: "k".to_string(),
205                context_before: None,
206                context_after: None,
207                scheme: TransliterationScheme::BgnPcgn,
208            },
209            TransliterationRule {
210                source: "л".to_string(),
211                target: "l".to_string(),
212                context_before: None,
213                context_after: None,
214                scheme: TransliterationScheme::BgnPcgn,
215            },
216            TransliterationRule {
217                source: "м".to_string(),
218                target: "m".to_string(),
219                context_before: None,
220                context_after: None,
221                scheme: TransliterationScheme::BgnPcgn,
222            },
223            TransliterationRule {
224                source: "н".to_string(),
225                target: "n".to_string(),
226                context_before: None,
227                context_after: None,
228                scheme: TransliterationScheme::BgnPcgn,
229            },
230            TransliterationRule {
231                source: "п".to_string(),
232                target: "p".to_string(),
233                context_before: None,
234                context_after: None,
235                scheme: TransliterationScheme::BgnPcgn,
236            },
237            TransliterationRule {
238                source: "р".to_string(),
239                target: "r".to_string(),
240                context_before: None,
241                context_after: None,
242                scheme: TransliterationScheme::BgnPcgn,
243            },
244            TransliterationRule {
245                source: "с".to_string(),
246                target: "s".to_string(),
247                context_before: None,
248                context_after: None,
249                scheme: TransliterationScheme::BgnPcgn,
250            },
251            TransliterationRule {
252                source: "т".to_string(),
253                target: "t".to_string(),
254                context_before: None,
255                context_after: None,
256                scheme: TransliterationScheme::BgnPcgn,
257            },
258            TransliterationRule {
259                source: "ф".to_string(),
260                target: "f".to_string(),
261                context_before: None,
262                context_after: None,
263                scheme: TransliterationScheme::BgnPcgn,
264            },
265            TransliterationRule {
266                source: "х".to_string(),
267                target: "kh".to_string(),
268                context_before: None,
269                context_after: None,
270                scheme: TransliterationScheme::BgnPcgn,
271            },
272            TransliterationRule {
273                source: "ц".to_string(),
274                target: "ts".to_string(),
275                context_before: None,
276                context_after: None,
277                scheme: TransliterationScheme::BgnPcgn,
278            },
279            TransliterationRule {
280                source: "ч".to_string(),
281                target: "ch".to_string(),
282                context_before: None,
283                context_after: None,
284                scheme: TransliterationScheme::BgnPcgn,
285            },
286            TransliterationRule {
287                source: "ш".to_string(),
288                target: "sh".to_string(),
289                context_before: None,
290                context_after: None,
291                scheme: TransliterationScheme::BgnPcgn,
292            },
293            TransliterationRule {
294                source: "щ".to_string(),
295                target: "shch".to_string(),
296                context_before: None,
297                context_after: None,
298                scheme: TransliterationScheme::BgnPcgn,
299            },
300            TransliterationRule {
301                source: "ъ".to_string(),
302                target: "\"".to_string(),
303                context_before: None,
304                context_after: None,
305                scheme: TransliterationScheme::BgnPcgn,
306            },
307            TransliterationRule {
308                source: "ь".to_string(),
309                target: "'".to_string(),
310                context_before: None,
311                context_after: None,
312                scheme: TransliterationScheme::BgnPcgn,
313            },
314        ];
315
316        // Popular/simplified transliteration
317        let popular_rules = vec![
318            TransliterationRule {
319                source: "а".to_string(),
320                target: "a".to_string(),
321                context_before: None,
322                context_after: None,
323                scheme: TransliterationScheme::Popular,
324            },
325            TransliterationRule {
326                source: "е".to_string(),
327                target: "e".to_string(),
328                context_before: None,
329                context_after: None,
330                scheme: TransliterationScheme::Popular,
331            },
332            TransliterationRule {
333                source: "ё".to_string(),
334                target: "yo".to_string(),
335                context_before: None,
336                context_after: None,
337                scheme: TransliterationScheme::Popular,
338            },
339            TransliterationRule {
340                source: "и".to_string(),
341                target: "i".to_string(),
342                context_before: None,
343                context_after: None,
344                scheme: TransliterationScheme::Popular,
345            },
346            TransliterationRule {
347                source: "о".to_string(),
348                target: "o".to_string(),
349                context_before: None,
350                context_after: None,
351                scheme: TransliterationScheme::Popular,
352            },
353            TransliterationRule {
354                source: "у".to_string(),
355                target: "u".to_string(),
356                context_before: None,
357                context_after: None,
358                scheme: TransliterationScheme::Popular,
359            },
360            TransliterationRule {
361                source: "ы".to_string(),
362                target: "y".to_string(),
363                context_before: None,
364                context_after: None,
365                scheme: TransliterationScheme::Popular,
366            },
367            TransliterationRule {
368                source: "э".to_string(),
369                target: "e".to_string(),
370                context_before: None,
371                context_after: None,
372                scheme: TransliterationScheme::Popular,
373            },
374            TransliterationRule {
375                source: "ю".to_string(),
376                target: "yu".to_string(),
377                context_before: None,
378                context_after: None,
379                scheme: TransliterationScheme::Popular,
380            },
381            TransliterationRule {
382                source: "я".to_string(),
383                target: "ya".to_string(),
384                context_before: None,
385                context_after: None,
386                scheme: TransliterationScheme::Popular,
387            },
388            TransliterationRule {
389                source: "б".to_string(),
390                target: "b".to_string(),
391                context_before: None,
392                context_after: None,
393                scheme: TransliterationScheme::Popular,
394            },
395            TransliterationRule {
396                source: "в".to_string(),
397                target: "v".to_string(),
398                context_before: None,
399                context_after: None,
400                scheme: TransliterationScheme::Popular,
401            },
402            TransliterationRule {
403                source: "г".to_string(),
404                target: "g".to_string(),
405                context_before: None,
406                context_after: None,
407                scheme: TransliterationScheme::Popular,
408            },
409            TransliterationRule {
410                source: "д".to_string(),
411                target: "d".to_string(),
412                context_before: None,
413                context_after: None,
414                scheme: TransliterationScheme::Popular,
415            },
416            TransliterationRule {
417                source: "ж".to_string(),
418                target: "zh".to_string(),
419                context_before: None,
420                context_after: None,
421                scheme: TransliterationScheme::Popular,
422            },
423            TransliterationRule {
424                source: "з".to_string(),
425                target: "z".to_string(),
426                context_before: None,
427                context_after: None,
428                scheme: TransliterationScheme::Popular,
429            },
430            TransliterationRule {
431                source: "й".to_string(),
432                target: "y".to_string(),
433                context_before: None,
434                context_after: None,
435                scheme: TransliterationScheme::Popular,
436            },
437            TransliterationRule {
438                source: "к".to_string(),
439                target: "k".to_string(),
440                context_before: None,
441                context_after: None,
442                scheme: TransliterationScheme::Popular,
443            },
444            TransliterationRule {
445                source: "л".to_string(),
446                target: "l".to_string(),
447                context_before: None,
448                context_after: None,
449                scheme: TransliterationScheme::Popular,
450            },
451            TransliterationRule {
452                source: "м".to_string(),
453                target: "m".to_string(),
454                context_before: None,
455                context_after: None,
456                scheme: TransliterationScheme::Popular,
457            },
458            TransliterationRule {
459                source: "н".to_string(),
460                target: "n".to_string(),
461                context_before: None,
462                context_after: None,
463                scheme: TransliterationScheme::Popular,
464            },
465            TransliterationRule {
466                source: "п".to_string(),
467                target: "p".to_string(),
468                context_before: None,
469                context_after: None,
470                scheme: TransliterationScheme::Popular,
471            },
472            TransliterationRule {
473                source: "р".to_string(),
474                target: "r".to_string(),
475                context_before: None,
476                context_after: None,
477                scheme: TransliterationScheme::Popular,
478            },
479            TransliterationRule {
480                source: "с".to_string(),
481                target: "s".to_string(),
482                context_before: None,
483                context_after: None,
484                scheme: TransliterationScheme::Popular,
485            },
486            TransliterationRule {
487                source: "т".to_string(),
488                target: "t".to_string(),
489                context_before: None,
490                context_after: None,
491                scheme: TransliterationScheme::Popular,
492            },
493            TransliterationRule {
494                source: "ф".to_string(),
495                target: "f".to_string(),
496                context_before: None,
497                context_after: None,
498                scheme: TransliterationScheme::Popular,
499            },
500            TransliterationRule {
501                source: "х".to_string(),
502                target: "h".to_string(),
503                context_before: None,
504                context_after: None,
505                scheme: TransliterationScheme::Popular,
506            },
507            TransliterationRule {
508                source: "ц".to_string(),
509                target: "ts".to_string(),
510                context_before: None,
511                context_after: None,
512                scheme: TransliterationScheme::Popular,
513            },
514            TransliterationRule {
515                source: "ч".to_string(),
516                target: "ch".to_string(),
517                context_before: None,
518                context_after: None,
519                scheme: TransliterationScheme::Popular,
520            },
521            TransliterationRule {
522                source: "ш".to_string(),
523                target: "sh".to_string(),
524                context_before: None,
525                context_after: None,
526                scheme: TransliterationScheme::Popular,
527            },
528            TransliterationRule {
529                source: "щ".to_string(),
530                target: "sch".to_string(),
531                context_before: None,
532                context_after: None,
533                scheme: TransliterationScheme::Popular,
534            },
535            TransliterationRule {
536                source: "ъ".to_string(),
537                target: "".to_string(),
538                context_before: None,
539                context_after: None,
540                scheme: TransliterationScheme::Popular,
541            },
542            TransliterationRule {
543                source: "ь".to_string(),
544                target: "".to_string(),
545                context_before: None,
546                context_after: None,
547                scheme: TransliterationScheme::Popular,
548            },
549        ];
550
551        self.cyrillic_to_latin
552            .insert(TransliterationScheme::BgnPcgn, bgn_pcgn_rules);
553        self.cyrillic_to_latin
554            .insert(TransliterationScheme::Popular, popular_rules);
555    }
556
557    fn initialize_arabic_mappings(&mut self) {
558        // Simplified Arabic to Latin transliteration
559        let arabic_rules = vec![
560            // Arabic letters
561            TransliterationRule {
562                source: "ا".to_string(),
563                target: "a".to_string(),
564                context_before: None,
565                context_after: None,
566                scheme: TransliterationScheme::BgnPcgn,
567            },
568            TransliterationRule {
569                source: "ب".to_string(),
570                target: "b".to_string(),
571                context_before: None,
572                context_after: None,
573                scheme: TransliterationScheme::BgnPcgn,
574            },
575            TransliterationRule {
576                source: "ت".to_string(),
577                target: "t".to_string(),
578                context_before: None,
579                context_after: None,
580                scheme: TransliterationScheme::BgnPcgn,
581            },
582            TransliterationRule {
583                source: "ث".to_string(),
584                target: "th".to_string(),
585                context_before: None,
586                context_after: None,
587                scheme: TransliterationScheme::BgnPcgn,
588            },
589            TransliterationRule {
590                source: "ج".to_string(),
591                target: "j".to_string(),
592                context_before: None,
593                context_after: None,
594                scheme: TransliterationScheme::BgnPcgn,
595            },
596            TransliterationRule {
597                source: "ح".to_string(),
598                target: "h".to_string(),
599                context_before: None,
600                context_after: None,
601                scheme: TransliterationScheme::BgnPcgn,
602            },
603            TransliterationRule {
604                source: "خ".to_string(),
605                target: "kh".to_string(),
606                context_before: None,
607                context_after: None,
608                scheme: TransliterationScheme::BgnPcgn,
609            },
610            TransliterationRule {
611                source: "د".to_string(),
612                target: "d".to_string(),
613                context_before: None,
614                context_after: None,
615                scheme: TransliterationScheme::BgnPcgn,
616            },
617            TransliterationRule {
618                source: "ذ".to_string(),
619                target: "dh".to_string(),
620                context_before: None,
621                context_after: None,
622                scheme: TransliterationScheme::BgnPcgn,
623            },
624            TransliterationRule {
625                source: "ر".to_string(),
626                target: "r".to_string(),
627                context_before: None,
628                context_after: None,
629                scheme: TransliterationScheme::BgnPcgn,
630            },
631            TransliterationRule {
632                source: "ز".to_string(),
633                target: "z".to_string(),
634                context_before: None,
635                context_after: None,
636                scheme: TransliterationScheme::BgnPcgn,
637            },
638            TransliterationRule {
639                source: "س".to_string(),
640                target: "s".to_string(),
641                context_before: None,
642                context_after: None,
643                scheme: TransliterationScheme::BgnPcgn,
644            },
645            TransliterationRule {
646                source: "ش".to_string(),
647                target: "sh".to_string(),
648                context_before: None,
649                context_after: None,
650                scheme: TransliterationScheme::BgnPcgn,
651            },
652            TransliterationRule {
653                source: "ص".to_string(),
654                target: "s".to_string(),
655                context_before: None,
656                context_after: None,
657                scheme: TransliterationScheme::BgnPcgn,
658            },
659            TransliterationRule {
660                source: "ض".to_string(),
661                target: "d".to_string(),
662                context_before: None,
663                context_after: None,
664                scheme: TransliterationScheme::BgnPcgn,
665            },
666            TransliterationRule {
667                source: "ط".to_string(),
668                target: "t".to_string(),
669                context_before: None,
670                context_after: None,
671                scheme: TransliterationScheme::BgnPcgn,
672            },
673            TransliterationRule {
674                source: "ظ".to_string(),
675                target: "z".to_string(),
676                context_before: None,
677                context_after: None,
678                scheme: TransliterationScheme::BgnPcgn,
679            },
680            TransliterationRule {
681                source: "ع".to_string(),
682                target: "'".to_string(),
683                context_before: None,
684                context_after: None,
685                scheme: TransliterationScheme::BgnPcgn,
686            },
687            TransliterationRule {
688                source: "غ".to_string(),
689                target: "gh".to_string(),
690                context_before: None,
691                context_after: None,
692                scheme: TransliterationScheme::BgnPcgn,
693            },
694            TransliterationRule {
695                source: "ف".to_string(),
696                target: "f".to_string(),
697                context_before: None,
698                context_after: None,
699                scheme: TransliterationScheme::BgnPcgn,
700            },
701            TransliterationRule {
702                source: "ق".to_string(),
703                target: "q".to_string(),
704                context_before: None,
705                context_after: None,
706                scheme: TransliterationScheme::BgnPcgn,
707            },
708            TransliterationRule {
709                source: "ك".to_string(),
710                target: "k".to_string(),
711                context_before: None,
712                context_after: None,
713                scheme: TransliterationScheme::BgnPcgn,
714            },
715            TransliterationRule {
716                source: "ل".to_string(),
717                target: "l".to_string(),
718                context_before: None,
719                context_after: None,
720                scheme: TransliterationScheme::BgnPcgn,
721            },
722            TransliterationRule {
723                source: "م".to_string(),
724                target: "m".to_string(),
725                context_before: None,
726                context_after: None,
727                scheme: TransliterationScheme::BgnPcgn,
728            },
729            TransliterationRule {
730                source: "ن".to_string(),
731                target: "n".to_string(),
732                context_before: None,
733                context_after: None,
734                scheme: TransliterationScheme::BgnPcgn,
735            },
736            TransliterationRule {
737                source: "ه".to_string(),
738                target: "h".to_string(),
739                context_before: None,
740                context_after: None,
741                scheme: TransliterationScheme::BgnPcgn,
742            },
743            TransliterationRule {
744                source: "و".to_string(),
745                target: "w".to_string(),
746                context_before: None,
747                context_after: None,
748                scheme: TransliterationScheme::BgnPcgn,
749            },
750            TransliterationRule {
751                source: "ي".to_string(),
752                target: "y".to_string(),
753                context_before: None,
754                context_after: None,
755                scheme: TransliterationScheme::BgnPcgn,
756            },
757        ];
758
759        self.arabic_to_latin
760            .insert(TransliterationScheme::BgnPcgn, arabic_rules);
761    }
762
763    fn initialize_greek_mappings(&mut self) {
764        // Greek to Latin transliteration
765        let greek_rules = vec![
766            // Vowels
767            TransliterationRule {
768                source: "α".to_string(),
769                target: "a".to_string(),
770                context_before: None,
771                context_after: None,
772                scheme: TransliterationScheme::BgnPcgn,
773            },
774            TransliterationRule {
775                source: "ε".to_string(),
776                target: "e".to_string(),
777                context_before: None,
778                context_after: None,
779                scheme: TransliterationScheme::BgnPcgn,
780            },
781            TransliterationRule {
782                source: "η".to_string(),
783                target: "i".to_string(),
784                context_before: None,
785                context_after: None,
786                scheme: TransliterationScheme::BgnPcgn,
787            },
788            TransliterationRule {
789                source: "ι".to_string(),
790                target: "i".to_string(),
791                context_before: None,
792                context_after: None,
793                scheme: TransliterationScheme::BgnPcgn,
794            },
795            TransliterationRule {
796                source: "ο".to_string(),
797                target: "o".to_string(),
798                context_before: None,
799                context_after: None,
800                scheme: TransliterationScheme::BgnPcgn,
801            },
802            TransliterationRule {
803                source: "υ".to_string(),
804                target: "y".to_string(),
805                context_before: None,
806                context_after: None,
807                scheme: TransliterationScheme::BgnPcgn,
808            },
809            TransliterationRule {
810                source: "ω".to_string(),
811                target: "o".to_string(),
812                context_before: None,
813                context_after: None,
814                scheme: TransliterationScheme::BgnPcgn,
815            },
816            // Consonants
817            TransliterationRule {
818                source: "β".to_string(),
819                target: "v".to_string(),
820                context_before: None,
821                context_after: None,
822                scheme: TransliterationScheme::BgnPcgn,
823            },
824            TransliterationRule {
825                source: "γ".to_string(),
826                target: "g".to_string(),
827                context_before: None,
828                context_after: None,
829                scheme: TransliterationScheme::BgnPcgn,
830            },
831            TransliterationRule {
832                source: "δ".to_string(),
833                target: "d".to_string(),
834                context_before: None,
835                context_after: None,
836                scheme: TransliterationScheme::BgnPcgn,
837            },
838            TransliterationRule {
839                source: "ζ".to_string(),
840                target: "z".to_string(),
841                context_before: None,
842                context_after: None,
843                scheme: TransliterationScheme::BgnPcgn,
844            },
845            TransliterationRule {
846                source: "θ".to_string(),
847                target: "th".to_string(),
848                context_before: None,
849                context_after: None,
850                scheme: TransliterationScheme::BgnPcgn,
851            },
852            TransliterationRule {
853                source: "κ".to_string(),
854                target: "k".to_string(),
855                context_before: None,
856                context_after: None,
857                scheme: TransliterationScheme::BgnPcgn,
858            },
859            TransliterationRule {
860                source: "λ".to_string(),
861                target: "l".to_string(),
862                context_before: None,
863                context_after: None,
864                scheme: TransliterationScheme::BgnPcgn,
865            },
866            TransliterationRule {
867                source: "μ".to_string(),
868                target: "m".to_string(),
869                context_before: None,
870                context_after: None,
871                scheme: TransliterationScheme::BgnPcgn,
872            },
873            TransliterationRule {
874                source: "ν".to_string(),
875                target: "n".to_string(),
876                context_before: None,
877                context_after: None,
878                scheme: TransliterationScheme::BgnPcgn,
879            },
880            TransliterationRule {
881                source: "ξ".to_string(),
882                target: "x".to_string(),
883                context_before: None,
884                context_after: None,
885                scheme: TransliterationScheme::BgnPcgn,
886            },
887            TransliterationRule {
888                source: "π".to_string(),
889                target: "p".to_string(),
890                context_before: None,
891                context_after: None,
892                scheme: TransliterationScheme::BgnPcgn,
893            },
894            TransliterationRule {
895                source: "ρ".to_string(),
896                target: "r".to_string(),
897                context_before: None,
898                context_after: None,
899                scheme: TransliterationScheme::BgnPcgn,
900            },
901            TransliterationRule {
902                source: "σ".to_string(),
903                target: "s".to_string(),
904                context_before: None,
905                context_after: None,
906                scheme: TransliterationScheme::BgnPcgn,
907            },
908            TransliterationRule {
909                source: "ς".to_string(),
910                target: "s".to_string(),
911                context_before: None,
912                context_after: None,
913                scheme: TransliterationScheme::BgnPcgn,
914            },
915            TransliterationRule {
916                source: "τ".to_string(),
917                target: "t".to_string(),
918                context_before: None,
919                context_after: None,
920                scheme: TransliterationScheme::BgnPcgn,
921            },
922            TransliterationRule {
923                source: "φ".to_string(),
924                target: "f".to_string(),
925                context_before: None,
926                context_after: None,
927                scheme: TransliterationScheme::BgnPcgn,
928            },
929            TransliterationRule {
930                source: "χ".to_string(),
931                target: "ch".to_string(),
932                context_before: None,
933                context_after: None,
934                scheme: TransliterationScheme::BgnPcgn,
935            },
936            TransliterationRule {
937                source: "ψ".to_string(),
938                target: "ps".to_string(),
939                context_before: None,
940                context_after: None,
941                scheme: TransliterationScheme::BgnPcgn,
942            },
943        ];
944
945        self.greek_to_latin
946            .insert(TransliterationScheme::BgnPcgn, greek_rules);
947    }
948
949    /// Transliterate text using simple string replacement (for demonstration)
950    fn transliterate_simple(
951        &self,
952        text: &str,
953        source_script: Script,
954        scheme: TransliterationScheme,
955    ) -> String {
956        let rules = match source_script {
957            Script::Cyrillic => self.cyrillic_to_latin.get(&scheme),
958            Script::Arabic => self.arabic_to_latin.get(&scheme),
959            Script::Greek => self.greek_to_latin.get(&scheme),
960            _ => None,
961        };
962
963        if let Some(rules) = rules {
964            let mut result = text.to_string();
965
966            // Sort rules by source length (longest first) to handle digraphs correctly
967            let mut sorted_rules = rules.clone();
968            sorted_rules.sort_by(|a, b| b.source.len().cmp(&a.source.len()));
969
970            for rule in &sorted_rules {
971                result = result.replace(&rule.source, &rule.target);
972            }
973
974            result
975        } else {
976            text.to_string()
977        }
978    }
979}
980
981/// Build a transliteration FST for a specific script and scheme
982fn build_transliteration_fst(rules: &[TransliterationRule]) -> VectorFst<TropicalWeight> {
983    let mut fst = VectorFst::new();
984    let start = fst.add_state();
985    fst.set_start(start);
986    fst.set_final(start, TropicalWeight::one());
987
988    // Sort rules by source length (longest first) to handle digraphs
989    let mut sorted_rules = rules.to_vec();
990    sorted_rules.sort_by(|a, b| b.source.len().cmp(&a.source.len()));
991
992    for rule in &sorted_rules {
993        let mut current = start;
994
995        // Process each character in the source
996        let source_chars: Vec<char> = rule.source.chars().collect();
997        for (i, &ch) in source_chars.iter().enumerate() {
998            if i == source_chars.len() - 1 {
999                // Last character - output the target
1000                let target_chars: Vec<char> = rule.target.chars().collect();
1001                let _target_state = start;
1002
1003                for &target_ch in &target_chars {
1004                    let next = fst.add_state();
1005                    fst.add_arc(
1006                        current,
1007                        Arc::new(ch as u32, target_ch as u32, TropicalWeight::one(), next),
1008                    );
1009                    current = next;
1010                }
1011
1012                // Connect back to start for more characters
1013                fst.add_arc(
1014                    current,
1015                    Arc::new(
1016                        0, // epsilon
1017                        0, // epsilon
1018                        TropicalWeight::one(),
1019                        start,
1020                    ),
1021                );
1022            } else {
1023                // Intermediate character
1024                let next = fst.add_state();
1025                fst.add_arc(
1026                    current,
1027                    Arc::new(
1028                        ch as u32,
1029                        0, // epsilon output
1030                        TropicalWeight::one(),
1031                        next,
1032                    ),
1033                );
1034                current = next;
1035            }
1036        }
1037    }
1038
1039    // Add pass-through for unknown characters
1040    for ch in 0..=127u8 {
1041        if ch.is_ascii() {
1042            fst.add_arc(
1043                start,
1044                Arc::new(ch as u32, ch as u32, TropicalWeight::one(), start),
1045            );
1046        }
1047    }
1048
1049    fst
1050}
1051
1052/// Create reverse transliteration mappings
1053fn create_reverse_mappings(rules: &[TransliterationRule]) -> Vec<TransliterationRule> {
1054    rules
1055        .iter()
1056        .map(|rule| TransliterationRule {
1057            source: rule.target.clone(),
1058            target: rule.source.clone(),
1059            context_before: rule.context_before.clone(),
1060            context_after: rule.context_after.clone(),
1061            scheme: rule.scheme,
1062        })
1063        .collect()
1064}
1065
1066fn main() -> Result<()> {
1067    println!("Transliteration FST Example");
1068    println!("===========================\n");
1069
1070    // Initialize transliteration system
1071    let transliteration_system = TransliterationSystem::new();
1072
1073    println!("Transliteration system initialized with:");
1074    let cyrillic_count = transliteration_system.cyrillic_to_latin.len();
1075    println!("  {cyrillic_count} Cyrillic schemes");
1076    let arabic_count = transliteration_system.arabic_to_latin.len();
1077    println!("  {arabic_count} Arabic schemes");
1078    let greek_count = transliteration_system.greek_to_latin.len();
1079    println!("  {greek_count} Greek schemes");
1080
1081    // Test Cyrillic transliteration
1082    println!("\n1. Cyrillic to Latin Transliteration:");
1083    println!("------------------------------------");
1084
1085    let cyrillic_tests = vec![
1086        "москва",          // Moscow
1087        "санкт-петербург", // Saint Petersburg
1088        "россия",          // Russia
1089        "владимир",        // Vladimir
1090        "екатерина",       // Ekaterina
1091        "борщ",            // Borscht
1092        "водка",           // Vodka
1093        "матрёшка",        // Matryoshka
1094        "кремль",          // Kremlin
1095        "большой",         // Bolshoy
1096    ];
1097
1098    for text in cyrillic_tests {
1099        println!("\nRussian: '{text}'");
1100
1101        // BGN/PCGN transliteration
1102        let bgn_result = transliteration_system.transliterate_simple(
1103            text,
1104            Script::Cyrillic,
1105            TransliterationScheme::BgnPcgn,
1106        );
1107        println!("  BGN/PCGN: '{bgn_result}'");
1108
1109        // Popular transliteration
1110        let popular_result = transliteration_system.transliterate_simple(
1111            text,
1112            Script::Cyrillic,
1113            TransliterationScheme::Popular,
1114        );
1115        println!("  Popular:  '{popular_result}'");
1116    }
1117
1118    // Test Arabic transliteration
1119    println!("\n2. Arabic to Latin Transliteration:");
1120    println!("----------------------------------");
1121
1122    let arabic_tests = vec![
1123        "السلام",   // Peace (as-salam)
1124        "مرحبا",   // Hello (marhaba)
1125        "شكرا",    // Thank you (shukran)
1126        "قاهرة",   // Cairo (qahirah)
1127        "بغداد",   // Baghdad
1128        "دمشق",    // Damascus
1129        "مكة",     // Mecca
1130        "المدينة", // Medina
1131    ];
1132
1133    for text in arabic_tests {
1134        println!("\nArabic: '{text}'");
1135        let result = transliteration_system.transliterate_simple(
1136            text,
1137            Script::Arabic,
1138            TransliterationScheme::BgnPcgn,
1139        );
1140        println!("  Latin: '{result}'");
1141    }
1142
1143    // Test Greek transliteration
1144    println!("\n3. Greek to Latin Transliteration:");
1145    println!("---------------------------------");
1146
1147    let greek_tests = vec![
1148        "αθήνα",       // Athens (athina)
1149        "θεσσαλονίκη", // Thessaloniki
1150        "κρήτη",       // Crete (kriti)
1151        "ελλάδα",      // Greece (ellada)
1152        "φιλοσοφία",   // Philosophy
1153        "δημοκρατία",  // Democracy
1154        "μουσική",     // Music
1155        "θέατρο",      // Theater
1156    ];
1157
1158    for text in greek_tests {
1159        println!("\nGreek: '{text}'");
1160        let result = transliteration_system.transliterate_simple(
1161            text,
1162            Script::Greek,
1163            TransliterationScheme::BgnPcgn,
1164        );
1165        println!("  Latin: '{result}'");
1166    }
1167
1168    // Demonstrate FST-based transliteration
1169    println!("\n4. FST-based Transliteration:");
1170    println!("-----------------------------");
1171
1172    if let Some(cyrillic_rules) = transliteration_system
1173        .cyrillic_to_latin
1174        .get(&TransliterationScheme::Popular)
1175    {
1176        let _transliteration_fst = build_transliteration_fst(cyrillic_rules);
1177        let state_count = _transliteration_fst.num_states();
1178        println!("Built transliteration FST with {state_count} states");
1179
1180        // Create reverse mappings for bidirectional transliteration
1181        let reverse_rules = create_reverse_mappings(cyrillic_rules);
1182        let _reverse_fst = build_transliteration_fst(&reverse_rules);
1183        let reverse_state_count = _reverse_fst.num_states();
1184        println!("Built reverse transliteration FST with {reverse_state_count} states");
1185    }
1186
1187    // Show transliteration scheme differences
1188    println!("\n5. Transliteration Scheme Comparison:");
1189    println!("------------------------------------");
1190    println!("Different schemes handle characters differently:\n");
1191
1192    let comparison_tests = vec![
1193        ("х", "Cyrillic letter KHA"),
1194        ("ё", "Cyrillic letter YO"),
1195        ("щ", "Cyrillic letter SHCHA"),
1196        ("ь", "Cyrillic soft sign"),
1197        ("ъ", "Cyrillic hard sign"),
1198    ];
1199
1200    for (char, description) in comparison_tests {
1201        println!("{char} ({description}):");
1202        let bgn = transliteration_system.transliterate_simple(
1203            char,
1204            Script::Cyrillic,
1205            TransliterationScheme::BgnPcgn,
1206        );
1207        let popular = transliteration_system.transliterate_simple(
1208            char,
1209            Script::Cyrillic,
1210            TransliterationScheme::Popular,
1211        );
1212        println!("  BGN/PCGN: '{bgn}' | Popular: '{popular}'");
1213    }
1214
1215    // Applications and use cases
1216    println!("\n6. Applications and Use Cases:");
1217    println!("-----------------------------");
1218    println!("Transliteration FSTs are essential for:");
1219    println!("  • Cross-language information retrieval");
1220    println!("  • Name matching across different scripts");
1221    println!("  • Machine translation preprocessing");
1222    println!("  • International address processing");
1223    println!("  • Search engine query expansion");
1224    println!("  • Database normalization");
1225    println!("  • Multilingual user interfaces");
1226    println!("  • Geographic information systems");
1227
1228    // Technical advantages of FSTs
1229    println!("\n7. FST Advantages for Transliteration:");
1230    println!("-------------------------------------");
1231    println!("FSTs provide several benefits over simple substitution:");
1232    println!("  • Context-sensitive rules: handle position-dependent mappings");
1233    println!("  • Bidirectional conversion: same FST for forward/reverse");
1234    println!("  • Composition: combine multiple transformation layers");
1235    println!("  • Efficiency: linear time processing");
1236    println!("  • Weighted alternatives: rank multiple transliterations");
1237    println!("  • Standardization: consistent rule application");
1238
1239    // Challenges and considerations
1240    println!("\n8. Challenges in Transliteration:");
1241    println!("--------------------------------");
1242    println!("Key challenges include:");
1243    println!("  • One-to-many mappings: single source → multiple targets");
1244    println!("  • Context sensitivity: different rules based on neighbors");
1245    println!("  • Digraphs and trigraphs: multi-character mappings");
1246    println!("  • Vowel insertion: handling consonant-only scripts");
1247    println!("  • Orthographic variations: multiple valid spellings");
1248    println!("  • Historic vs. modern conventions");
1249    println!("  • Domain-specific requirements (legal, geographic, etc.)");
1250
1251    Ok(())
1252}
transliteration/transliteration.rs

transliteration/
transliteration.rs
