use std::collections::HashMap;
#[derive(Debug, Clone)]
pub struct CrossLingualExpansion {
pub query: String,
pub source_lang: Language,
pub target_lang: Language,
pub confidence: f64,
}
#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
pub enum Language {
English,
Spanish,
French,
German,
Portuguese,
Bengali,
Japanese,
Unknown,
}
#[derive(Debug, Clone)]
pub struct CrossLingualResult {
pub original: String,
pub detected_language: Language,
pub expansions: Vec<CrossLingualExpansion>,
}
pub fn expand_crosslingual(query: &str) -> CrossLingualResult {
let detected = detect_language(query);
let mut expansions = Vec::new();
if detected != Language::English && detected != Language::Unknown {
if let Some(english) = translate_to_english(query, detected) {
expansions.push(CrossLingualExpansion {
query: english,
source_lang: detected,
target_lang: Language::English,
confidence: 0.8,
});
}
}
let term_translations = translate_terms(query);
for (original_term, english_term, lang, confidence) in term_translations {
let expanded = query.replace(&original_term, &english_term);
if expanded != query && !expansions.iter().any(|e| e.query == expanded) {
expansions.push(CrossLingualExpansion {
query: expanded,
source_lang: lang,
target_lang: Language::English,
confidence,
});
}
}
expansions.sort_by(|a, b| {
b.confidence
.partial_cmp(&a.confidence)
.unwrap_or(std::cmp::Ordering::Equal)
});
expansions.truncate(3);
CrossLingualResult {
original: query.to_string(),
detected_language: detected,
expansions,
}
}
fn detect_language(query: &str) -> Language {
let lower = query.to_lowercase();
let words: Vec<&str> = lower.split_whitespace().collect();
let scores = [
(Language::French, french_score(&words, &lower)),
(Language::Spanish, spanish_score(&words, &lower)),
(Language::German, german_score(&words, &lower)),
(Language::Portuguese, portuguese_score(&words, &lower)),
(Language::Bengali, bengali_score(query)),
(Language::Japanese, japanese_score(&lower)),
];
let (best_lang, best_score) = scores
.iter()
.max_by(|a, b| a.1.partial_cmp(&b.1).unwrap_or(std::cmp::Ordering::Equal))
.unwrap();
if *best_score > 0.3 {
*best_lang
} else {
if words.iter().all(|w| {
w.chars()
.all(|c| c.is_ascii_alphanumeric() || c == '-' || c == '_')
}) {
Language::English
} else {
Language::Unknown
}
}
}
fn french_score(words: &[&str], _query: &str) -> f64 {
const MARKERS: &[&str] = &[
"de",
"le",
"la",
"les",
"un",
"une",
"du",
"des",
"en",
"est",
"sont",
"avec",
"pour",
"dans",
"sur",
"par",
"comme",
"comment",
"pourquoi",
"apprendre",
"rapidement",
"français",
"francais",
"algorithme",
"recherche",
"tri",
"fonction",
];
let hits = words.iter().filter(|w| MARKERS.contains(w)).count();
(hits as f64 / words.len().max(1) as f64).min(1.0)
}
fn spanish_score(words: &[&str], _query: &str) -> f64 {
const MARKERS: &[&str] = &[
"de",
"el",
"la",
"los",
"las",
"un",
"una",
"en",
"es",
"son",
"con",
"para",
"por",
"cómo",
"como",
"funciona",
"inteligencia",
"artificial",
"aprender",
"rapido",
"rápido",
"frances",
"francés",
"algoritmo",
"buscar",
"ordenar",
"funcion",
];
let hits = words.iter().filter(|w| MARKERS.contains(w)).count();
(hits as f64 / words.len().max(1) as f64).min(1.0)
}
fn german_score(words: &[&str], _query: &str) -> f64 {
const MARKERS: &[&str] = &[
"der",
"die",
"das",
"ein",
"eine",
"ist",
"sind",
"mit",
"und",
"von",
"fur",
"was",
"wie",
"lernen",
"schnell",
"auf",
"bei",
"nach",
"algorithmus",
"suche",
"sortierung",
"funktion",
];
let hits = words.iter().filter(|w| MARKERS.contains(w)).count();
(hits as f64 / words.len().max(1) as f64).min(1.0)
}
fn portuguese_score(words: &[&str], _query: &str) -> f64 {
const MARKERS: &[&str] = &[
"de",
"o",
"a",
"os",
"as",
"um",
"uma",
"em",
"com",
"para",
"por",
"como",
"algoritmo",
"busca",
"ordenacao",
];
let hits = words.iter().filter(|w| MARKERS.contains(w)).count();
(hits as f64 / words.len().max(1) as f64).min(1.0)
}
fn japanese_score(query: &str) -> f64 {
let jp_chars = query
.chars()
.filter(|c| {
let cp = *c as u32;
(0x3040..=0x309F).contains(&cp)
|| (0x30A0..=0x30FF).contains(&cp)
|| (0x4E00..=0x9FFF).contains(&cp)
})
.count();
let total = query.chars().count().max(1);
jp_chars as f64 / total as f64
}
fn bengali_score(query: &str) -> f64 {
let bn_chars = query
.chars()
.filter(|c| {
let cp = *c as u32;
(0x0980..=0x09FF).contains(&cp)
})
.count();
let total = query.chars().count().max(1);
bn_chars as f64 / total as f64
}
fn translate_to_english(query: &str, lang: Language) -> Option<String> {
let dict = get_phrase_dict(lang);
let lower = query.to_lowercase();
for (foreign, english) in &dict {
if lower.contains(*foreign) {
return Some(lower.replace(*foreign, english));
}
}
let word_dict = get_word_dict(lang);
let words: Vec<&str> = lower.split_whitespace().collect();
let translated: Vec<String> = words
.iter()
.map(|w| {
word_dict
.get(*w)
.map(|t| t.to_string())
.unwrap_or_else(|| w.to_string())
})
.collect();
let result = translated.join(" ");
if result != lower {
Some(result)
} else {
None
}
}
fn translate_terms(query: &str) -> Vec<(String, String, Language, f64)> {
let mut results = Vec::new();
let lower = query.to_lowercase();
for lang in [
Language::French,
Language::Spanish,
Language::German,
Language::Portuguese,
] {
let dict = get_word_dict(lang);
for (foreign, english) in &dict {
if lower.contains(foreign) && *foreign != *english {
results.push((foreign.to_string(), english.to_string(), lang, 0.7));
}
}
}
results
}
fn get_phrase_dict(lang: Language) -> Vec<(&'static str, &'static str)> {
match lang {
Language::French => vec![
("comment apprendre le français rapidement", "learn french fast"),
("comment apprendre le francais rapidement", "learn french fast"),
("apprendre le français", "learn french"),
("apprendre le francais", "learn french"),
("algorithme de tri", "sorting algorithm"),
("recherche binaire", "binary search"),
("apprentissage automatique", "machine learning"),
("intelligence artificielle", "artificial intelligence"),
("base de donnees", "database"),
("reseau de neurones", "neural network"),
("structure de donnees", "data structure"),
],
Language::Spanish => vec![
("cómo funciona la inteligencia artificial", "how artificial intelligence works"),
("como funciona la inteligencia artificial", "how artificial intelligence works"),
("cómo aprender python desde cero", "learn python from scratch"),
("como aprender python desde cero", "learn python from scratch"),
("algoritmo de ordenamiento", "sorting algorithm"),
("busqueda binaria", "binary search"),
("aprendizaje automatico", "machine learning"),
("inteligencia artificial", "artificial intelligence"),
("base de datos", "database"),
("red neuronal", "neural network"),
("estructura de datos", "data structure"),
],
Language::German => vec![
(
"was ist quantencomputing einfach erklärt",
"quantum computing explained simply",
),
(
"was ist quantencomputing einfach erklaert",
"quantum computing explained simply",
),
("sortieralgorithmus", "sorting algorithm"),
("binaere suche", "binary search"),
("maschinelles lernen", "machine learning"),
("kuenstliche intelligenz", "artificial intelligence"),
("neuronales netz", "neural network"),
("datenstruktur", "data structure"),
],
Language::Bengali => vec![
("মেশিন লার্নিং কি", "what is machine learning"),
("মেশিন লার্নিং কী", "what is machine learning"),
(
"বাংলায় কুবেরনেটিস কী",
"what is kubernetes container orchestration",
),
(
"বাংলায় কুবেরনেটিস কী",
"what is kubernetes container orchestration",
),
("কুবেরনেটিস কী", "what is kubernetes container orchestration"),
("কুবেরনেটিস কি", "what is kubernetes container orchestration"),
(
"কন্টেইনার অর্কেস্ট্রেশন কী",
"what is container orchestration kubernetes",
),
(
"কন্টেইনার অর্কেস্ট্রেশন কি",
"what is container orchestration kubernetes",
),
(
"বাংলায় প্রোগ্রামিং কীভাবে শিখব",
"how to learn programming in bengali",
),
(
"প্রোগ্রামিং কীভাবে শিখব",
"how to learn programming",
),
("পাইথন কী", "what is python programming language"),
("পাইথন কি", "what is python programming language"),
(
"বাংলায় কৃত্রিম বুদ্ধিমত্তা কী",
"what is artificial intelligence beginner guide",
),
(
"কৃত্রিম বুদ্ধিমত্তা কী",
"what is artificial intelligence beginner guide",
),
(
"বাংলায় ডেটাবেস কী",
"what is database dbms database management system",
),
("ডেটাবেস কী", "what is database dbms database management system"),
(
"ডেটাবেস ম্যানেজমেন্ট সিস্টেম কী",
"what is database management system dbms",
),
(
"বাংলায় নেটওয়ার্কিং কী",
"what is computer networking basics",
),
("নেটওয়ার্কিং কী", "what is computer networking basics"),
(
"রক্তচাপের ওষুধের পার্শ্বপ্রতিক্রিয়া তুলনা",
"blood pressure medication side effects comparison antihypertensive medication classes ace inhibitor arb beta blocker diuretic calcium channel blocker",
),
(
"ডকার কম্পোজ নেটওয়ার্কিং কীভাবে কাজ করে",
"docker compose networking between containers service name default network docs",
),
(
"ডকার কম্পোজ নেটওয়ার্কিং কিভাবে কাজ করে",
"docker compose networking between containers service name default network docs",
),
("কিভাবে বাংলা দ্রুত শিখব", "how to learn bengali fast"),
],
_ => vec![],
}
}
fn get_word_dict(lang: Language) -> HashMap<&'static str, &'static str> {
match lang {
Language::French => [
("comment", "how"),
("apprendre", "learn"),
("français", "french"),
("francais", "french"),
("rapidement", "fast"),
("algorithme", "algorithm"),
("tri", "sorting"),
("recherche", "search"),
("fonction", "function"),
("variable", "variable"),
("boucle", "loop"),
("tableau", "array"),
("liste", "list"),
("arbre", "tree"),
("graphe", "graph"),
("pile", "stack"),
("file", "queue"),
("rapide", "fast"),
("lent", "slow"),
("erreur", "error"),
]
.into_iter()
.collect(),
Language::Spanish => [
("aprender", "learn"),
("desde", "from"),
("cero", "scratch"),
("funciona", "works"),
("rápido", "fast"),
("rapido", "fast"),
("francés", "french"),
("frances", "french"),
("algoritmo", "algorithm"),
("ordenar", "sort"),
("buscar", "search"),
("funcion", "function"),
("variable", "variable"),
("bucle", "loop"),
("arreglo", "array"),
("lista", "list"),
("arbol", "tree"),
("grafo", "graph"),
("pila", "stack"),
("cola", "queue"),
("rapido", "fast"),
("lento", "slow"),
("error", "error"),
]
.into_iter()
.collect(),
Language::German => [
("was", "what"),
("quantencomputing", "quantum computing"),
("erklärt", "explained"),
("erklaert", "explained"),
("einfach", "simple"),
("lernen", "learn"),
("französisch", "french"),
("franzoesisch", "french"),
("schnell", "fast"),
("algorithmus", "algorithm"),
("sortierung", "sorting"),
("suche", "search"),
("funktion", "function"),
("variable", "variable"),
("schleife", "loop"),
("datenbank", "database"),
("liste", "list"),
("baum", "tree"),
("schnell", "fast"),
("langsam", "slow"),
("fehler", "error"),
]
.into_iter()
.collect(),
Language::Portuguese => [
("algoritmo", "algorithm"),
("ordenacao", "sorting"),
("busca", "search"),
("funcao", "function"),
("variavel", "variable"),
("laco", "loop"),
("lista", "list"),
("arvore", "tree"),
("grafo", "graph"),
("rapido", "fast"),
("lento", "slow"),
("erro", "error"),
]
.into_iter()
.collect(),
Language::Bengali => [
("মেশিন", "machine"),
("লার্নিং", "learning"),
("কি", "what"),
("কী", "what"),
("কুবেরনেটিস", "kubernetes"),
("কন্টেইনার", "container"),
("অর্কেস্ট্রেশন", "orchestration"),
("প্রোগ্রামিং", "programming"),
("পাইথন", "python"),
("কৃত্রিম", "artificial"),
("বুদ্ধিমত্তা", "intelligence"),
("ডেটাবেস", "database"),
("ম্যানেজমেন্ট", "management"),
("সিস্টেম", "system"),
("ডিবিএমএস", "dbms"),
("নেটওয়ার্কিং", "networking"),
("শিখব", "learn"),
("শিখতে", "learn"),
("কিভাবে", "how"),
("কীভাবে", "how"),
("রক্তচাপ", "blood pressure"),
("ওষুধ", "medication"),
("পার্শ্বপ্রতিক্রিয়া", "side effects"),
("তুলনা", "comparison"),
("ডকার", "docker"),
("কম্পোজ", "compose"),
("নেটওয়ার্কিং", "networking"),
("কীভাবে", "how"),
("কিভাবে", "how"),
("কাজ", "work"),
("করে", "works"),
("বাংলা", "bengali"),
("দ্রুত", "fast"),
]
.into_iter()
.collect(),
_ => HashMap::new(),
}
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn english_query_no_expansions() {
let result = expand_crosslingual("sorting algorithm implementation");
assert_eq!(result.detected_language, Language::English);
assert!(result.expansions.is_empty());
}
#[test]
fn french_query_expanded() {
let result = expand_crosslingual("algorithme de tri rapide en python");
assert_eq!(result.detected_language, Language::French);
assert!(!result.expansions.is_empty());
let has_english = result
.expansions
.iter()
.any(|e| e.target_lang == Language::English);
assert!(has_english);
}
#[test]
fn spanish_query_expanded() {
let result = expand_crosslingual("algoritmo de ordenamiento en lista");
assert_eq!(result.detected_language, Language::Spanish);
assert!(!result.expansions.is_empty());
}
#[test]
fn german_query_expanded() {
let result = expand_crosslingual("sortierung algorithmus mit liste und baum");
assert_eq!(result.detected_language, Language::German);
assert!(!result.expansions.is_empty());
}
#[test]
fn empty_query() {
let result = expand_crosslingual("");
assert!(result.expansions.is_empty());
}
#[test]
fn max_three_expansions() {
let result = expand_crosslingual("algorithme de tri rapide avec boucle et tableau");
assert!(result.expansions.len() <= 3);
}
#[test]
fn japanese_detected() {
let lang = detect_language("ソートアルゴリズム");
assert_eq!(lang, Language::Japanese);
}
#[test]
fn bengali_detected() {
let lang = detect_language("মেশিন লার্নিং কি");
assert_eq!(lang, Language::Bengali);
}
#[test]
fn bengali_kubernetes_phrase_expanded() {
let result = expand_crosslingual("বাংলায় কুবেরনেটিস কী");
assert_eq!(result.detected_language, Language::Bengali);
assert!(result
.expansions
.iter()
.any(|e| e.query.contains("kubernetes")));
}
#[test]
fn bengali_programming_phrase_expanded() {
let result = expand_crosslingual("প্রোগ্রামিং কীভাবে শিখব");
assert_eq!(result.detected_language, Language::Bengali);
assert!(result
.expansions
.iter()
.any(|e| e.query.contains("programming")));
}
#[test]
fn bengali_ai_phrase_expanded() {
let result = expand_crosslingual("কৃত্রিম বুদ্ধিমত্তা কী");
assert_eq!(result.detected_language, Language::Bengali);
assert!(result
.expansions
.iter()
.any(|e| e.query.contains("artificial intelligence")));
}
#[test]
fn bengali_database_phrase_expanded() {
let result = expand_crosslingual("ডেটাবেস কী");
assert_eq!(result.detected_language, Language::Bengali);
assert!(result
.expansions
.iter()
.any(|e| e.query.contains("database management system")));
}
#[test]
fn mixed_language_term_translation() {
let translations = translate_terms("implement algorithme in python");
assert!(!translations.is_empty());
let has_algorithm = translations
.iter()
.any(|(_, eng, _, _)| eng.contains("algorithm"));
assert!(has_algorithm);
}
#[test]
fn confidence_sorted_descending() {
let result = expand_crosslingual("algorithme de tri avec recherche");
if result.expansions.len() >= 2 {
for i in 0..result.expansions.len() - 1 {
assert!(result.expansions[i].confidence >= result.expansions[i + 1].confidence);
}
}
}
#[test]
fn spanish_explainer_phrase_expanded() {
let result = expand_crosslingual("cómo funciona la inteligencia artificial");
assert_eq!(result.detected_language, Language::Spanish);
assert!(result
.expansions
.iter()
.any(|e| e.query.contains("artificial intelligence")));
}
#[test]
fn german_explainer_phrase_expanded() {
let result = expand_crosslingual("was ist quantencomputing einfach erklärt");
assert_eq!(result.detected_language, Language::German);
assert!(result
.expansions
.iter()
.any(|e| e.query.contains("quantum computing")));
}
}