1use std::collections::HashMap;
10
11#[derive(Debug, Clone)]
13pub struct CrossLingualExpansion {
14 pub query: String,
16 pub source_lang: Language,
18 pub target_lang: Language,
20 pub confidence: f64,
22}
23
24#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
26pub enum Language {
27 English,
28 Spanish,
29 French,
30 German,
31 Portuguese,
32 Bengali,
33 Japanese,
34 Unknown,
35}
36
37#[derive(Debug, Clone)]
39pub struct CrossLingualResult {
40 pub original: String,
42 pub detected_language: Language,
44 pub expansions: Vec<CrossLingualExpansion>,
46}
47
48pub fn expand_crosslingual(query: &str) -> CrossLingualResult {
50 let detected = detect_language(query);
51 let mut expansions = Vec::new();
52
53 if detected != Language::English && detected != Language::Unknown {
54 if let Some(english) = translate_to_english(query, detected) {
56 expansions.push(CrossLingualExpansion {
57 query: english,
58 source_lang: detected,
59 target_lang: Language::English,
60 confidence: 0.8,
61 });
62 }
63 }
64
65 let term_translations = translate_terms(query);
67 for (original_term, english_term, lang, confidence) in term_translations {
68 let expanded = query.replace(&original_term, &english_term);
69 if expanded != query && !expansions.iter().any(|e| e.query == expanded) {
70 expansions.push(CrossLingualExpansion {
71 query: expanded,
72 source_lang: lang,
73 target_lang: Language::English,
74 confidence,
75 });
76 }
77 }
78
79 expansions.sort_by(|a, b| {
81 b.confidence
82 .partial_cmp(&a.confidence)
83 .unwrap_or(std::cmp::Ordering::Equal)
84 });
85 expansions.truncate(3);
86
87 CrossLingualResult {
88 original: query.to_string(),
89 detected_language: detected,
90 expansions,
91 }
92}
93
94fn detect_language(query: &str) -> Language {
97 let lower = query.to_lowercase();
98 let words: Vec<&str> = lower.split_whitespace().collect();
99
100 let scores = [
102 (Language::French, french_score(&words, &lower)),
103 (Language::Spanish, spanish_score(&words, &lower)),
104 (Language::German, german_score(&words, &lower)),
105 (Language::Portuguese, portuguese_score(&words, &lower)),
106 (Language::Bengali, bengali_score(query)),
107 (Language::Japanese, japanese_score(&lower)),
108 ];
109
110 let (best_lang, best_score) = scores
111 .iter()
112 .max_by(|a, b| a.1.partial_cmp(&b.1).unwrap_or(std::cmp::Ordering::Equal))
113 .unwrap();
114
115 if *best_score > 0.3 {
116 *best_lang
117 } else {
118 if words.iter().all(|w| {
120 w.chars()
121 .all(|c| c.is_ascii_alphanumeric() || c == '-' || c == '_')
122 }) {
123 Language::English
124 } else {
125 Language::Unknown
126 }
127 }
128}
129
130fn french_score(words: &[&str], _query: &str) -> f64 {
131 const MARKERS: &[&str] = &[
132 "de",
133 "le",
134 "la",
135 "les",
136 "un",
137 "une",
138 "du",
139 "des",
140 "en",
141 "est",
142 "sont",
143 "avec",
144 "pour",
145 "dans",
146 "sur",
147 "par",
148 "comme",
149 "comment",
150 "pourquoi",
151 "apprendre",
152 "rapidement",
153 "français",
154 "francais",
155 "algorithme",
156 "recherche",
157 "tri",
158 "fonction",
159 ];
160 let hits = words.iter().filter(|w| MARKERS.contains(w)).count();
161 (hits as f64 / words.len().max(1) as f64).min(1.0)
162}
163
164fn spanish_score(words: &[&str], _query: &str) -> f64 {
165 const MARKERS: &[&str] = &[
166 "de",
167 "el",
168 "la",
169 "los",
170 "las",
171 "un",
172 "una",
173 "en",
174 "es",
175 "son",
176 "con",
177 "para",
178 "por",
179 "cómo",
180 "como",
181 "funciona",
182 "inteligencia",
183 "artificial",
184 "aprender",
185 "rapido",
186 "rápido",
187 "frances",
188 "francés",
189 "algoritmo",
190 "buscar",
191 "ordenar",
192 "funcion",
193 ];
194 let hits = words.iter().filter(|w| MARKERS.contains(w)).count();
195 (hits as f64 / words.len().max(1) as f64).min(1.0)
196}
197
198fn german_score(words: &[&str], _query: &str) -> f64 {
199 const MARKERS: &[&str] = &[
200 "der",
201 "die",
202 "das",
203 "ein",
204 "eine",
205 "ist",
206 "sind",
207 "mit",
208 "und",
209 "von",
210 "fur",
211 "was",
212 "wie",
213 "lernen",
214 "schnell",
215 "auf",
216 "bei",
217 "nach",
218 "algorithmus",
219 "suche",
220 "sortierung",
221 "funktion",
222 ];
223 let hits = words.iter().filter(|w| MARKERS.contains(w)).count();
224 (hits as f64 / words.len().max(1) as f64).min(1.0)
225}
226
227fn portuguese_score(words: &[&str], _query: &str) -> f64 {
228 const MARKERS: &[&str] = &[
229 "de",
230 "o",
231 "a",
232 "os",
233 "as",
234 "um",
235 "uma",
236 "em",
237 "com",
238 "para",
239 "por",
240 "como",
241 "algoritmo",
242 "busca",
243 "ordenacao",
244 ];
245 let hits = words.iter().filter(|w| MARKERS.contains(w)).count();
246 (hits as f64 / words.len().max(1) as f64).min(1.0)
247}
248
249fn japanese_score(query: &str) -> f64 {
250 let jp_chars = query
251 .chars()
252 .filter(|c| {
253 let cp = *c as u32;
254 (0x3040..=0x309F).contains(&cp)
256 || (0x30A0..=0x30FF).contains(&cp)
257 || (0x4E00..=0x9FFF).contains(&cp)
258 })
259 .count();
260 let total = query.chars().count().max(1);
261 jp_chars as f64 / total as f64
262}
263
264fn bengali_score(query: &str) -> f64 {
265 let bn_chars = query
266 .chars()
267 .filter(|c| {
268 let cp = *c as u32;
269 (0x0980..=0x09FF).contains(&cp)
270 })
271 .count();
272 let total = query.chars().count().max(1);
273 bn_chars as f64 / total as f64
274}
275
276fn translate_to_english(query: &str, lang: Language) -> Option<String> {
279 let dict = get_phrase_dict(lang);
280 let lower = query.to_lowercase();
281
282 for (foreign, english) in &dict {
284 if lower.contains(*foreign) {
285 return Some(lower.replace(*foreign, english));
286 }
287 }
288
289 let word_dict = get_word_dict(lang);
291 let words: Vec<&str> = lower.split_whitespace().collect();
292 let translated: Vec<String> = words
293 .iter()
294 .map(|w| {
295 word_dict
296 .get(*w)
297 .map(|t| t.to_string())
298 .unwrap_or_else(|| w.to_string())
299 })
300 .collect();
301
302 let result = translated.join(" ");
303 if result != lower {
304 Some(result)
305 } else {
306 None
307 }
308}
309
310fn translate_terms(query: &str) -> Vec<(String, String, Language, f64)> {
311 let mut results = Vec::new();
312 let lower = query.to_lowercase();
313
314 for lang in [
315 Language::French,
316 Language::Spanish,
317 Language::German,
318 Language::Portuguese,
319 ] {
320 let dict = get_word_dict(lang);
321 for (foreign, english) in &dict {
322 if lower.contains(foreign) && *foreign != *english {
323 results.push((foreign.to_string(), english.to_string(), lang, 0.7));
324 }
325 }
326 }
327
328 results
329}
330
331fn get_phrase_dict(lang: Language) -> Vec<(&'static str, &'static str)> {
332 match lang {
333 Language::French => vec![
334 ("comment apprendre le français rapidement", "learn french fast"),
335 ("comment apprendre le francais rapidement", "learn french fast"),
336 ("apprendre le français", "learn french"),
337 ("apprendre le francais", "learn french"),
338 ("algorithme de tri", "sorting algorithm"),
339 ("recherche binaire", "binary search"),
340 ("apprentissage automatique", "machine learning"),
341 ("intelligence artificielle", "artificial intelligence"),
342 ("base de donnees", "database"),
343 ("reseau de neurones", "neural network"),
344 ("structure de donnees", "data structure"),
345 ],
346 Language::Spanish => vec![
347 ("cómo funciona la inteligencia artificial", "how artificial intelligence works"),
348 ("como funciona la inteligencia artificial", "how artificial intelligence works"),
349 ("cómo aprender python desde cero", "learn python from scratch"),
350 ("como aprender python desde cero", "learn python from scratch"),
351 ("algoritmo de ordenamiento", "sorting algorithm"),
352 ("busqueda binaria", "binary search"),
353 ("aprendizaje automatico", "machine learning"),
354 ("inteligencia artificial", "artificial intelligence"),
355 ("base de datos", "database"),
356 ("red neuronal", "neural network"),
357 ("estructura de datos", "data structure"),
358 ],
359 Language::German => vec![
360 (
361 "was ist quantencomputing einfach erklärt",
362 "quantum computing explained simply",
363 ),
364 (
365 "was ist quantencomputing einfach erklaert",
366 "quantum computing explained simply",
367 ),
368 ("sortieralgorithmus", "sorting algorithm"),
369 ("binaere suche", "binary search"),
370 ("maschinelles lernen", "machine learning"),
371 ("kuenstliche intelligenz", "artificial intelligence"),
372 ("neuronales netz", "neural network"),
373 ("datenstruktur", "data structure"),
374 ],
375 Language::Bengali => vec![
376 ("মেশিন লার্নিং কি", "what is machine learning"),
377 ("মেশিন লার্নিং কী", "what is machine learning"),
378 (
379 "বাংলায় কুবেরনেটিস কী",
380 "what is kubernetes container orchestration",
381 ),
382 (
383 "বাংলায় কুবেরনেটিস কী",
384 "what is kubernetes container orchestration",
385 ),
386 ("কুবেরনেটিস কী", "what is kubernetes container orchestration"),
387 ("কুবেরনেটিস কি", "what is kubernetes container orchestration"),
388 (
389 "কন্টেইনার অর্কেস্ট্রেশন কী",
390 "what is container orchestration kubernetes",
391 ),
392 (
393 "কন্টেইনার অর্কেস্ট্রেশন কি",
394 "what is container orchestration kubernetes",
395 ),
396 (
397 "বাংলায় প্রোগ্রামিং কীভাবে শিখব",
398 "how to learn programming in bengali",
399 ),
400 (
401 "প্রোগ্রামিং কীভাবে শিখব",
402 "how to learn programming",
403 ),
404 ("পাইথন কী", "what is python programming language"),
405 ("পাইথন কি", "what is python programming language"),
406 (
407 "বাংলায় কৃত্রিম বুদ্ধিমত্তা কী",
408 "what is artificial intelligence beginner guide",
409 ),
410 (
411 "কৃত্রিম বুদ্ধিমত্তা কী",
412 "what is artificial intelligence beginner guide",
413 ),
414 (
415 "বাংলায় ডেটাবেস কী",
416 "what is database dbms database management system",
417 ),
418 ("ডেটাবেস কী", "what is database dbms database management system"),
419 (
420 "ডেটাবেস ম্যানেজমেন্ট সিস্টেম কী",
421 "what is database management system dbms",
422 ),
423 (
424 "বাংলায় নেটওয়ার্কিং কী",
425 "what is computer networking basics",
426 ),
427 ("নেটওয়ার্কিং কী", "what is computer networking basics"),
428 (
429 "রক্তচাপের ওষুধের পার্শ্বপ্রতিক্রিয়া তুলনা",
430 "blood pressure medication side effects comparison antihypertensive medication classes ace inhibitor arb beta blocker diuretic calcium channel blocker",
431 ),
432 (
433 "ডকার কম্পোজ নেটওয়ার্কিং কীভাবে কাজ করে",
434 "docker compose networking between containers service name default network docs",
435 ),
436 (
437 "ডকার কম্পোজ নেটওয়ার্কিং কিভাবে কাজ করে",
438 "docker compose networking between containers service name default network docs",
439 ),
440 ("কিভাবে বাংলা দ্রুত শিখব", "how to learn bengali fast"),
441 ],
442 _ => vec![],
443 }
444}
445
446fn get_word_dict(lang: Language) -> HashMap<&'static str, &'static str> {
447 match lang {
448 Language::French => [
449 ("comment", "how"),
450 ("apprendre", "learn"),
451 ("français", "french"),
452 ("francais", "french"),
453 ("rapidement", "fast"),
454 ("algorithme", "algorithm"),
455 ("tri", "sorting"),
456 ("recherche", "search"),
457 ("fonction", "function"),
458 ("variable", "variable"),
459 ("boucle", "loop"),
460 ("tableau", "array"),
461 ("liste", "list"),
462 ("arbre", "tree"),
463 ("graphe", "graph"),
464 ("pile", "stack"),
465 ("file", "queue"),
466 ("rapide", "fast"),
467 ("lent", "slow"),
468 ("erreur", "error"),
469 ]
470 .into_iter()
471 .collect(),
472 Language::Spanish => [
473 ("aprender", "learn"),
474 ("desde", "from"),
475 ("cero", "scratch"),
476 ("funciona", "works"),
477 ("rápido", "fast"),
478 ("rapido", "fast"),
479 ("francés", "french"),
480 ("frances", "french"),
481 ("algoritmo", "algorithm"),
482 ("ordenar", "sort"),
483 ("buscar", "search"),
484 ("funcion", "function"),
485 ("variable", "variable"),
486 ("bucle", "loop"),
487 ("arreglo", "array"),
488 ("lista", "list"),
489 ("arbol", "tree"),
490 ("grafo", "graph"),
491 ("pila", "stack"),
492 ("cola", "queue"),
493 ("rapido", "fast"),
494 ("lento", "slow"),
495 ("error", "error"),
496 ]
497 .into_iter()
498 .collect(),
499 Language::German => [
500 ("was", "what"),
501 ("quantencomputing", "quantum computing"),
502 ("erklärt", "explained"),
503 ("erklaert", "explained"),
504 ("einfach", "simple"),
505 ("lernen", "learn"),
506 ("französisch", "french"),
507 ("franzoesisch", "french"),
508 ("schnell", "fast"),
509 ("algorithmus", "algorithm"),
510 ("sortierung", "sorting"),
511 ("suche", "search"),
512 ("funktion", "function"),
513 ("variable", "variable"),
514 ("schleife", "loop"),
515 ("datenbank", "database"),
516 ("liste", "list"),
517 ("baum", "tree"),
518 ("schnell", "fast"),
519 ("langsam", "slow"),
520 ("fehler", "error"),
521 ]
522 .into_iter()
523 .collect(),
524 Language::Portuguese => [
525 ("algoritmo", "algorithm"),
526 ("ordenacao", "sorting"),
527 ("busca", "search"),
528 ("funcao", "function"),
529 ("variavel", "variable"),
530 ("laco", "loop"),
531 ("lista", "list"),
532 ("arvore", "tree"),
533 ("grafo", "graph"),
534 ("rapido", "fast"),
535 ("lento", "slow"),
536 ("erro", "error"),
537 ]
538 .into_iter()
539 .collect(),
540 Language::Bengali => [
541 ("মেশিন", "machine"),
542 ("লার্নিং", "learning"),
543 ("কি", "what"),
544 ("কী", "what"),
545 ("কুবেরনেটিস", "kubernetes"),
546 ("কন্টেইনার", "container"),
547 ("অর্কেস্ট্রেশন", "orchestration"),
548 ("প্রোগ্রামিং", "programming"),
549 ("পাইথন", "python"),
550 ("কৃত্রিম", "artificial"),
551 ("বুদ্ধিমত্তা", "intelligence"),
552 ("ডেটাবেস", "database"),
553 ("ম্যানেজমেন্ট", "management"),
554 ("সিস্টেম", "system"),
555 ("ডিবিএমএস", "dbms"),
556 ("নেটওয়ার্কিং", "networking"),
557 ("শিখব", "learn"),
558 ("শিখতে", "learn"),
559 ("কিভাবে", "how"),
560 ("কীভাবে", "how"),
561 ("রক্তচাপ", "blood pressure"),
562 ("ওষুধ", "medication"),
563 ("পার্শ্বপ্রতিক্রিয়া", "side effects"),
564 ("তুলনা", "comparison"),
565 ("ডকার", "docker"),
566 ("কম্পোজ", "compose"),
567 ("নেটওয়ার্কিং", "networking"),
568 ("কীভাবে", "how"),
569 ("কিভাবে", "how"),
570 ("কাজ", "work"),
571 ("করে", "works"),
572 ("বাংলা", "bengali"),
573 ("দ্রুত", "fast"),
574 ]
575 .into_iter()
576 .collect(),
577 _ => HashMap::new(),
578 }
579}
580
581#[cfg(test)]
582mod tests {
583 use super::*;
584
585 #[test]
586 fn english_query_no_expansions() {
587 let result = expand_crosslingual("sorting algorithm implementation");
588 assert_eq!(result.detected_language, Language::English);
589 assert!(result.expansions.is_empty());
590 }
591
592 #[test]
593 fn french_query_expanded() {
594 let result = expand_crosslingual("algorithme de tri rapide en python");
595 assert_eq!(result.detected_language, Language::French);
596 assert!(!result.expansions.is_empty());
597 let has_english = result
598 .expansions
599 .iter()
600 .any(|e| e.target_lang == Language::English);
601 assert!(has_english);
602 }
603
604 #[test]
605 fn spanish_query_expanded() {
606 let result = expand_crosslingual("algoritmo de ordenamiento en lista");
607 assert_eq!(result.detected_language, Language::Spanish);
608 assert!(!result.expansions.is_empty());
609 }
610
611 #[test]
612 fn german_query_expanded() {
613 let result = expand_crosslingual("sortierung algorithmus mit liste und baum");
614 assert_eq!(result.detected_language, Language::German);
615 assert!(!result.expansions.is_empty());
616 }
617
618 #[test]
619 fn empty_query() {
620 let result = expand_crosslingual("");
621 assert!(result.expansions.is_empty());
622 }
623
624 #[test]
625 fn max_three_expansions() {
626 let result = expand_crosslingual("algorithme de tri rapide avec boucle et tableau");
627 assert!(result.expansions.len() <= 3);
628 }
629
630 #[test]
631 fn japanese_detected() {
632 let lang = detect_language("ソートアルゴリズム");
633 assert_eq!(lang, Language::Japanese);
634 }
635
636 #[test]
637 fn bengali_detected() {
638 let lang = detect_language("মেশিন লার্নিং কি");
639 assert_eq!(lang, Language::Bengali);
640 }
641
642 #[test]
643 fn bengali_kubernetes_phrase_expanded() {
644 let result = expand_crosslingual("বাংলায় কুবেরনেটিস কী");
645 assert_eq!(result.detected_language, Language::Bengali);
646 assert!(result
647 .expansions
648 .iter()
649 .any(|e| e.query.contains("kubernetes")));
650 }
651
652 #[test]
653 fn bengali_programming_phrase_expanded() {
654 let result = expand_crosslingual("প্রোগ্রামিং কীভাবে শিখব");
655 assert_eq!(result.detected_language, Language::Bengali);
656 assert!(result
657 .expansions
658 .iter()
659 .any(|e| e.query.contains("programming")));
660 }
661
662 #[test]
663 fn bengali_ai_phrase_expanded() {
664 let result = expand_crosslingual("কৃত্রিম বুদ্ধিমত্তা কী");
665 assert_eq!(result.detected_language, Language::Bengali);
666 assert!(result
667 .expansions
668 .iter()
669 .any(|e| e.query.contains("artificial intelligence")));
670 }
671
672 #[test]
673 fn bengali_database_phrase_expanded() {
674 let result = expand_crosslingual("ডেটাবেস কী");
675 assert_eq!(result.detected_language, Language::Bengali);
676 assert!(result
677 .expansions
678 .iter()
679 .any(|e| e.query.contains("database management system")));
680 }
681
682 #[test]
683 fn mixed_language_term_translation() {
684 let translations = translate_terms("implement algorithme in python");
686 assert!(!translations.is_empty());
687 let has_algorithm = translations
688 .iter()
689 .any(|(_, eng, _, _)| eng.contains("algorithm"));
690 assert!(has_algorithm);
691 }
692
693 #[test]
694 fn confidence_sorted_descending() {
695 let result = expand_crosslingual("algorithme de tri avec recherche");
696 if result.expansions.len() >= 2 {
697 for i in 0..result.expansions.len() - 1 {
698 assert!(result.expansions[i].confidence >= result.expansions[i + 1].confidence);
699 }
700 }
701 }
702
703 #[test]
704 fn spanish_explainer_phrase_expanded() {
705 let result = expand_crosslingual("cómo funciona la inteligencia artificial");
706 assert_eq!(result.detected_language, Language::Spanish);
707 assert!(result
708 .expansions
709 .iter()
710 .any(|e| e.query.contains("artificial intelligence")));
711 }
712
713 #[test]
714 fn german_explainer_phrase_expanded() {
715 let result = expand_crosslingual("was ist quantencomputing einfach erklärt");
716 assert_eq!(result.detected_language, Language::German);
717 assert!(result
718 .expansions
719 .iter()
720 .any(|e| e.query.contains("quantum computing")));
721 }
722}