Skip to main content

fetchium_core/query/
crosslingual.rs

1//! Cross-Lingual Query Bridge (CLQB) — detects language and adds cross-lingual expansions.
2//!
3//! Automatically detects when a query contains non-English technical terms
4//! and adds English equivalents (and vice versa). Supports common programming
5//! and technical terms across 6 languages.
6//!
7//! Example: "algorithme de tri" → also searches "sorting algorithm"
8
9use std::collections::HashMap;
10
11/// A cross-lingual expansion of a query.
12#[derive(Debug, Clone)]
13pub struct CrossLingualExpansion {
14    /// The expanded query text.
15    pub query: String,
16    /// Detected source language.
17    pub source_lang: Language,
18    /// Target language of this expansion.
19    pub target_lang: Language,
20    /// Confidence in the translation (0.0–1.0).
21    pub confidence: f64,
22}
23
24/// Supported languages.
25#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
26pub enum Language {
27    English,
28    Spanish,
29    French,
30    German,
31    Portuguese,
32    Bengali,
33    Japanese,
34    Unknown,
35}
36
37/// Result of cross-lingual analysis.
38#[derive(Debug, Clone)]
39pub struct CrossLingualResult {
40    /// Original query.
41    pub original: String,
42    /// Detected language of the original query.
43    pub detected_language: Language,
44    /// Cross-lingual expansions (empty if already English with no foreign terms).
45    pub expansions: Vec<CrossLingualExpansion>,
46}
47
48/// Analyze a query and generate cross-lingual expansions.
49pub fn expand_crosslingual(query: &str) -> CrossLingualResult {
50    let detected = detect_language(query);
51    let mut expansions = Vec::new();
52
53    if detected != Language::English && detected != Language::Unknown {
54        // Translate foreign query to English
55        if let Some(english) = translate_to_english(query, detected) {
56            expansions.push(CrossLingualExpansion {
57                query: english,
58                source_lang: detected,
59                target_lang: Language::English,
60                confidence: 0.8,
61            });
62        }
63    }
64
65    // Also check for individual foreign technical terms in otherwise English queries
66    let term_translations = translate_terms(query);
67    for (original_term, english_term, lang, confidence) in term_translations {
68        let expanded = query.replace(&original_term, &english_term);
69        if expanded != query && !expansions.iter().any(|e| e.query == expanded) {
70            expansions.push(CrossLingualExpansion {
71                query: expanded,
72                source_lang: lang,
73                target_lang: Language::English,
74                confidence,
75            });
76        }
77    }
78
79    // Sort by confidence descending, limit to 3
80    expansions.sort_by(|a, b| {
81        b.confidence
82            .partial_cmp(&a.confidence)
83            .unwrap_or(std::cmp::Ordering::Equal)
84    });
85    expansions.truncate(3);
86
87    CrossLingualResult {
88        original: query.to_string(),
89        detected_language: detected,
90        expansions,
91    }
92}
93
94// ─── Language detection ──────────────────────────────────────
95
96fn detect_language(query: &str) -> Language {
97    let lower = query.to_lowercase();
98    let words: Vec<&str> = lower.split_whitespace().collect();
99
100    // Check for language-specific markers
101    let scores = [
102        (Language::French, french_score(&words, &lower)),
103        (Language::Spanish, spanish_score(&words, &lower)),
104        (Language::German, german_score(&words, &lower)),
105        (Language::Portuguese, portuguese_score(&words, &lower)),
106        (Language::Bengali, bengali_score(query)),
107        (Language::Japanese, japanese_score(&lower)),
108    ];
109
110    let (best_lang, best_score) = scores
111        .iter()
112        .max_by(|a, b| a.1.partial_cmp(&b.1).unwrap_or(std::cmp::Ordering::Equal))
113        .unwrap();
114
115    if *best_score > 0.3 {
116        *best_lang
117    } else {
118        // Default to English if no strong signal
119        if words.iter().all(|w| {
120            w.chars()
121                .all(|c| c.is_ascii_alphanumeric() || c == '-' || c == '_')
122        }) {
123            Language::English
124        } else {
125            Language::Unknown
126        }
127    }
128}
129
130fn french_score(words: &[&str], _query: &str) -> f64 {
131    const MARKERS: &[&str] = &[
132        "de",
133        "le",
134        "la",
135        "les",
136        "un",
137        "une",
138        "du",
139        "des",
140        "en",
141        "est",
142        "sont",
143        "avec",
144        "pour",
145        "dans",
146        "sur",
147        "par",
148        "comme",
149        "comment",
150        "pourquoi",
151        "apprendre",
152        "rapidement",
153        "français",
154        "francais",
155        "algorithme",
156        "recherche",
157        "tri",
158        "fonction",
159    ];
160    let hits = words.iter().filter(|w| MARKERS.contains(w)).count();
161    (hits as f64 / words.len().max(1) as f64).min(1.0)
162}
163
164fn spanish_score(words: &[&str], _query: &str) -> f64 {
165    const MARKERS: &[&str] = &[
166        "de",
167        "el",
168        "la",
169        "los",
170        "las",
171        "un",
172        "una",
173        "en",
174        "es",
175        "son",
176        "con",
177        "para",
178        "por",
179        "cómo",
180        "como",
181        "funciona",
182        "inteligencia",
183        "artificial",
184        "aprender",
185        "rapido",
186        "rápido",
187        "frances",
188        "francés",
189        "algoritmo",
190        "buscar",
191        "ordenar",
192        "funcion",
193    ];
194    let hits = words.iter().filter(|w| MARKERS.contains(w)).count();
195    (hits as f64 / words.len().max(1) as f64).min(1.0)
196}
197
198fn german_score(words: &[&str], _query: &str) -> f64 {
199    const MARKERS: &[&str] = &[
200        "der",
201        "die",
202        "das",
203        "ein",
204        "eine",
205        "ist",
206        "sind",
207        "mit",
208        "und",
209        "von",
210        "fur",
211        "was",
212        "wie",
213        "lernen",
214        "schnell",
215        "auf",
216        "bei",
217        "nach",
218        "algorithmus",
219        "suche",
220        "sortierung",
221        "funktion",
222    ];
223    let hits = words.iter().filter(|w| MARKERS.contains(w)).count();
224    (hits as f64 / words.len().max(1) as f64).min(1.0)
225}
226
227fn portuguese_score(words: &[&str], _query: &str) -> f64 {
228    const MARKERS: &[&str] = &[
229        "de",
230        "o",
231        "a",
232        "os",
233        "as",
234        "um",
235        "uma",
236        "em",
237        "com",
238        "para",
239        "por",
240        "como",
241        "algoritmo",
242        "busca",
243        "ordenacao",
244    ];
245    let hits = words.iter().filter(|w| MARKERS.contains(w)).count();
246    (hits as f64 / words.len().max(1) as f64).min(1.0)
247}
248
249fn japanese_score(query: &str) -> f64 {
250    let jp_chars = query
251        .chars()
252        .filter(|c| {
253            let cp = *c as u32;
254            // Hiragana, Katakana, or CJK Unified Ideographs
255            (0x3040..=0x309F).contains(&cp)
256                || (0x30A0..=0x30FF).contains(&cp)
257                || (0x4E00..=0x9FFF).contains(&cp)
258        })
259        .count();
260    let total = query.chars().count().max(1);
261    jp_chars as f64 / total as f64
262}
263
264fn bengali_score(query: &str) -> f64 {
265    let bn_chars = query
266        .chars()
267        .filter(|c| {
268            let cp = *c as u32;
269            (0x0980..=0x09FF).contains(&cp)
270        })
271        .count();
272    let total = query.chars().count().max(1);
273    bn_chars as f64 / total as f64
274}
275
276// ─── Translation ─────────────────────────────────────────────
277
278fn translate_to_english(query: &str, lang: Language) -> Option<String> {
279    let dict = get_phrase_dict(lang);
280    let lower = query.to_lowercase();
281
282    // Try full phrase match first
283    for (foreign, english) in &dict {
284        if lower.contains(*foreign) {
285            return Some(lower.replace(*foreign, english));
286        }
287    }
288
289    // Try word-by-word translation
290    let word_dict = get_word_dict(lang);
291    let words: Vec<&str> = lower.split_whitespace().collect();
292    let translated: Vec<String> = words
293        .iter()
294        .map(|w| {
295            word_dict
296                .get(*w)
297                .map(|t| t.to_string())
298                .unwrap_or_else(|| w.to_string())
299        })
300        .collect();
301
302    let result = translated.join(" ");
303    if result != lower {
304        Some(result)
305    } else {
306        None
307    }
308}
309
310fn translate_terms(query: &str) -> Vec<(String, String, Language, f64)> {
311    let mut results = Vec::new();
312    let lower = query.to_lowercase();
313
314    for lang in [
315        Language::French,
316        Language::Spanish,
317        Language::German,
318        Language::Portuguese,
319    ] {
320        let dict = get_word_dict(lang);
321        for (foreign, english) in &dict {
322            if lower.contains(foreign) && *foreign != *english {
323                results.push((foreign.to_string(), english.to_string(), lang, 0.7));
324            }
325        }
326    }
327
328    results
329}
330
331fn get_phrase_dict(lang: Language) -> Vec<(&'static str, &'static str)> {
332    match lang {
333        Language::French => vec![
334            ("comment apprendre le français rapidement", "learn french fast"),
335            ("comment apprendre le francais rapidement", "learn french fast"),
336            ("apprendre le français", "learn french"),
337            ("apprendre le francais", "learn french"),
338            ("algorithme de tri", "sorting algorithm"),
339            ("recherche binaire", "binary search"),
340            ("apprentissage automatique", "machine learning"),
341            ("intelligence artificielle", "artificial intelligence"),
342            ("base de donnees", "database"),
343            ("reseau de neurones", "neural network"),
344            ("structure de donnees", "data structure"),
345        ],
346        Language::Spanish => vec![
347            ("cómo funciona la inteligencia artificial", "how artificial intelligence works"),
348            ("como funciona la inteligencia artificial", "how artificial intelligence works"),
349            ("cómo aprender python desde cero", "learn python from scratch"),
350            ("como aprender python desde cero", "learn python from scratch"),
351            ("algoritmo de ordenamiento", "sorting algorithm"),
352            ("busqueda binaria", "binary search"),
353            ("aprendizaje automatico", "machine learning"),
354            ("inteligencia artificial", "artificial intelligence"),
355            ("base de datos", "database"),
356            ("red neuronal", "neural network"),
357            ("estructura de datos", "data structure"),
358        ],
359        Language::German => vec![
360            (
361                "was ist quantencomputing einfach erklärt",
362                "quantum computing explained simply",
363            ),
364            (
365                "was ist quantencomputing einfach erklaert",
366                "quantum computing explained simply",
367            ),
368            ("sortieralgorithmus", "sorting algorithm"),
369            ("binaere suche", "binary search"),
370            ("maschinelles lernen", "machine learning"),
371            ("kuenstliche intelligenz", "artificial intelligence"),
372            ("neuronales netz", "neural network"),
373            ("datenstruktur", "data structure"),
374        ],
375        Language::Bengali => vec![
376            ("মেশিন লার্নিং কি", "what is machine learning"),
377            ("মেশিন লার্নিং কী", "what is machine learning"),
378            (
379                "বাংলায় কুবেরনেটিস কী",
380                "what is kubernetes container orchestration",
381            ),
382            (
383                "বাংলায় কুবেরনেটিস কী",
384                "what is kubernetes container orchestration",
385            ),
386            ("কুবেরনেটিস কী", "what is kubernetes container orchestration"),
387            ("কুবেরনেটিস কি", "what is kubernetes container orchestration"),
388            (
389                "কন্টেইনার অর্কেস্ট্রেশন কী",
390                "what is container orchestration kubernetes",
391            ),
392            (
393                "কন্টেইনার অর্কেস্ট্রেশন কি",
394                "what is container orchestration kubernetes",
395            ),
396            (
397                "বাংলায় প্রোগ্রামিং কীভাবে শিখব",
398                "how to learn programming in bengali",
399            ),
400            (
401                "প্রোগ্রামিং কীভাবে শিখব",
402                "how to learn programming",
403            ),
404            ("পাইথন কী", "what is python programming language"),
405            ("পাইথন কি", "what is python programming language"),
406            (
407                "বাংলায় কৃত্রিম বুদ্ধিমত্তা কী",
408                "what is artificial intelligence beginner guide",
409            ),
410            (
411                "কৃত্রিম বুদ্ধিমত্তা কী",
412                "what is artificial intelligence beginner guide",
413            ),
414            (
415                "বাংলায় ডেটাবেস কী",
416                "what is database dbms database management system",
417            ),
418            ("ডেটাবেস কী", "what is database dbms database management system"),
419            (
420                "ডেটাবেস ম্যানেজমেন্ট সিস্টেম কী",
421                "what is database management system dbms",
422            ),
423            (
424                "বাংলায় নেটওয়ার্কিং কী",
425                "what is computer networking basics",
426            ),
427            ("নেটওয়ার্কিং কী", "what is computer networking basics"),
428            (
429                "রক্তচাপের ওষুধের পার্শ্বপ্রতিক্রিয়া তুলনা",
430                "blood pressure medication side effects comparison antihypertensive medication classes ace inhibitor arb beta blocker diuretic calcium channel blocker",
431            ),
432            (
433                "ডকার কম্পোজ নেটওয়ার্কিং কীভাবে কাজ করে",
434                "docker compose networking between containers service name default network docs",
435            ),
436            (
437                "ডকার কম্পোজ নেটওয়ার্কিং কিভাবে কাজ করে",
438                "docker compose networking between containers service name default network docs",
439            ),
440            ("কিভাবে বাংলা দ্রুত শিখব", "how to learn bengali fast"),
441        ],
442        _ => vec![],
443    }
444}
445
446fn get_word_dict(lang: Language) -> HashMap<&'static str, &'static str> {
447    match lang {
448        Language::French => [
449            ("comment", "how"),
450            ("apprendre", "learn"),
451            ("français", "french"),
452            ("francais", "french"),
453            ("rapidement", "fast"),
454            ("algorithme", "algorithm"),
455            ("tri", "sorting"),
456            ("recherche", "search"),
457            ("fonction", "function"),
458            ("variable", "variable"),
459            ("boucle", "loop"),
460            ("tableau", "array"),
461            ("liste", "list"),
462            ("arbre", "tree"),
463            ("graphe", "graph"),
464            ("pile", "stack"),
465            ("file", "queue"),
466            ("rapide", "fast"),
467            ("lent", "slow"),
468            ("erreur", "error"),
469        ]
470        .into_iter()
471        .collect(),
472        Language::Spanish => [
473            ("aprender", "learn"),
474            ("desde", "from"),
475            ("cero", "scratch"),
476            ("funciona", "works"),
477            ("rápido", "fast"),
478            ("rapido", "fast"),
479            ("francés", "french"),
480            ("frances", "french"),
481            ("algoritmo", "algorithm"),
482            ("ordenar", "sort"),
483            ("buscar", "search"),
484            ("funcion", "function"),
485            ("variable", "variable"),
486            ("bucle", "loop"),
487            ("arreglo", "array"),
488            ("lista", "list"),
489            ("arbol", "tree"),
490            ("grafo", "graph"),
491            ("pila", "stack"),
492            ("cola", "queue"),
493            ("rapido", "fast"),
494            ("lento", "slow"),
495            ("error", "error"),
496        ]
497        .into_iter()
498        .collect(),
499        Language::German => [
500            ("was", "what"),
501            ("quantencomputing", "quantum computing"),
502            ("erklärt", "explained"),
503            ("erklaert", "explained"),
504            ("einfach", "simple"),
505            ("lernen", "learn"),
506            ("französisch", "french"),
507            ("franzoesisch", "french"),
508            ("schnell", "fast"),
509            ("algorithmus", "algorithm"),
510            ("sortierung", "sorting"),
511            ("suche", "search"),
512            ("funktion", "function"),
513            ("variable", "variable"),
514            ("schleife", "loop"),
515            ("datenbank", "database"),
516            ("liste", "list"),
517            ("baum", "tree"),
518            ("schnell", "fast"),
519            ("langsam", "slow"),
520            ("fehler", "error"),
521        ]
522        .into_iter()
523        .collect(),
524        Language::Portuguese => [
525            ("algoritmo", "algorithm"),
526            ("ordenacao", "sorting"),
527            ("busca", "search"),
528            ("funcao", "function"),
529            ("variavel", "variable"),
530            ("laco", "loop"),
531            ("lista", "list"),
532            ("arvore", "tree"),
533            ("grafo", "graph"),
534            ("rapido", "fast"),
535            ("lento", "slow"),
536            ("erro", "error"),
537        ]
538        .into_iter()
539        .collect(),
540        Language::Bengali => [
541            ("মেশিন", "machine"),
542            ("লার্নিং", "learning"),
543            ("কি", "what"),
544            ("কী", "what"),
545            ("কুবেরনেটিস", "kubernetes"),
546            ("কন্টেইনার", "container"),
547            ("অর্কেস্ট্রেশন", "orchestration"),
548            ("প্রোগ্রামিং", "programming"),
549            ("পাইথন", "python"),
550            ("কৃত্রিম", "artificial"),
551            ("বুদ্ধিমত্তা", "intelligence"),
552            ("ডেটাবেস", "database"),
553            ("ম্যানেজমেন্ট", "management"),
554            ("সিস্টেম", "system"),
555            ("ডিবিএমএস", "dbms"),
556            ("নেটওয়ার্কিং", "networking"),
557            ("শিখব", "learn"),
558            ("শিখতে", "learn"),
559            ("কিভাবে", "how"),
560            ("কীভাবে", "how"),
561            ("রক্তচাপ", "blood pressure"),
562            ("ওষুধ", "medication"),
563            ("পার্শ্বপ্রতিক্রিয়া", "side effects"),
564            ("তুলনা", "comparison"),
565            ("ডকার", "docker"),
566            ("কম্পোজ", "compose"),
567            ("নেটওয়ার্কিং", "networking"),
568            ("কীভাবে", "how"),
569            ("কিভাবে", "how"),
570            ("কাজ", "work"),
571            ("করে", "works"),
572            ("বাংলা", "bengali"),
573            ("দ্রুত", "fast"),
574        ]
575        .into_iter()
576        .collect(),
577        _ => HashMap::new(),
578    }
579}
580
581#[cfg(test)]
582mod tests {
583    use super::*;
584
585    #[test]
586    fn english_query_no_expansions() {
587        let result = expand_crosslingual("sorting algorithm implementation");
588        assert_eq!(result.detected_language, Language::English);
589        assert!(result.expansions.is_empty());
590    }
591
592    #[test]
593    fn french_query_expanded() {
594        let result = expand_crosslingual("algorithme de tri rapide en python");
595        assert_eq!(result.detected_language, Language::French);
596        assert!(!result.expansions.is_empty());
597        let has_english = result
598            .expansions
599            .iter()
600            .any(|e| e.target_lang == Language::English);
601        assert!(has_english);
602    }
603
604    #[test]
605    fn spanish_query_expanded() {
606        let result = expand_crosslingual("algoritmo de ordenamiento en lista");
607        assert_eq!(result.detected_language, Language::Spanish);
608        assert!(!result.expansions.is_empty());
609    }
610
611    #[test]
612    fn german_query_expanded() {
613        let result = expand_crosslingual("sortierung algorithmus mit liste und baum");
614        assert_eq!(result.detected_language, Language::German);
615        assert!(!result.expansions.is_empty());
616    }
617
618    #[test]
619    fn empty_query() {
620        let result = expand_crosslingual("");
621        assert!(result.expansions.is_empty());
622    }
623
624    #[test]
625    fn max_three_expansions() {
626        let result = expand_crosslingual("algorithme de tri rapide avec boucle et tableau");
627        assert!(result.expansions.len() <= 3);
628    }
629
630    #[test]
631    fn japanese_detected() {
632        let lang = detect_language("ソートアルゴリズム");
633        assert_eq!(lang, Language::Japanese);
634    }
635
636    #[test]
637    fn bengali_detected() {
638        let lang = detect_language("মেশিন লার্নিং কি");
639        assert_eq!(lang, Language::Bengali);
640    }
641
642    #[test]
643    fn bengali_kubernetes_phrase_expanded() {
644        let result = expand_crosslingual("বাংলায় কুবেরনেটিস কী");
645        assert_eq!(result.detected_language, Language::Bengali);
646        assert!(result
647            .expansions
648            .iter()
649            .any(|e| e.query.contains("kubernetes")));
650    }
651
652    #[test]
653    fn bengali_programming_phrase_expanded() {
654        let result = expand_crosslingual("প্রোগ্রামিং কীভাবে শিখব");
655        assert_eq!(result.detected_language, Language::Bengali);
656        assert!(result
657            .expansions
658            .iter()
659            .any(|e| e.query.contains("programming")));
660    }
661
662    #[test]
663    fn bengali_ai_phrase_expanded() {
664        let result = expand_crosslingual("কৃত্রিম বুদ্ধিমত্তা কী");
665        assert_eq!(result.detected_language, Language::Bengali);
666        assert!(result
667            .expansions
668            .iter()
669            .any(|e| e.query.contains("artificial intelligence")));
670    }
671
672    #[test]
673    fn bengali_database_phrase_expanded() {
674        let result = expand_crosslingual("ডেটাবেস কী");
675        assert_eq!(result.detected_language, Language::Bengali);
676        assert!(result
677            .expansions
678            .iter()
679            .any(|e| e.query.contains("database management system")));
680    }
681
682    #[test]
683    fn mixed_language_term_translation() {
684        // English query with a foreign technical term
685        let translations = translate_terms("implement algorithme in python");
686        assert!(!translations.is_empty());
687        let has_algorithm = translations
688            .iter()
689            .any(|(_, eng, _, _)| eng.contains("algorithm"));
690        assert!(has_algorithm);
691    }
692
693    #[test]
694    fn confidence_sorted_descending() {
695        let result = expand_crosslingual("algorithme de tri avec recherche");
696        if result.expansions.len() >= 2 {
697            for i in 0..result.expansions.len() - 1 {
698                assert!(result.expansions[i].confidence >= result.expansions[i + 1].confidence);
699            }
700        }
701    }
702
703    #[test]
704    fn spanish_explainer_phrase_expanded() {
705        let result = expand_crosslingual("cómo funciona la inteligencia artificial");
706        assert_eq!(result.detected_language, Language::Spanish);
707        assert!(result
708            .expansions
709            .iter()
710            .any(|e| e.query.contains("artificial intelligence")));
711    }
712
713    #[test]
714    fn german_explainer_phrase_expanded() {
715        let result = expand_crosslingual("was ist quantencomputing einfach erklärt");
716        assert_eq!(result.detected_language, Language::German);
717        assert!(result
718            .expansions
719            .iter()
720            .any(|e| e.query.contains("quantum computing")));
721    }
722}