laya/
lang.rs

1//! Dependency-free language/script detection used to route between Laya checkpoints.
2//!
3//! Faithful port of `laya/lang.py`. Routing needs one decision: *is this English Latin text, or
4//! is it something the English checkpoint cannot read?* Script detection is exact; the Latin-script
5//! language guess is a stopword/diacritic heuristic and is explicitly best-effort.
6//!
7//! This is the source-of-truth port of the Python module (which carries `bn`/`az` stopword lists
8//! and the `_non_latin_words` reclassification step that the older TypeScript reference dropped).
9
10use std::collections::{HashMap, HashSet};
11use std::sync::OnceLock;
12
13use indexmap::IndexMap;
14use regex::Regex;
15use serde_json::Value;
16use unicode_general_category::{get_general_category, GeneralCategory};
17
18// Unicode blocks that the English (ModernBERT-large, 50k English BPE) checkpoint cannot read.
19const SCRIPT_RANGES: &[(&str, &[(u32, u32)])] = &[
20    ("greek", &[(0x0370, 0x03FF), (0x1F00, 0x1FFF)]),
21    (
22        "cyrillic",
23        &[(0x0400, 0x052F), (0x2DE0, 0x2DFF), (0xA640, 0xA69F)],
24    ),
25    ("armenian", &[(0x0530, 0x058F)]),
26    ("hebrew", &[(0x0590, 0x05FF)]),
27    (
28        "arabic",
29        &[
30            (0x0600, 0x06FF),
31            (0x0750, 0x077F),
32            (0x08A0, 0x08FF),
33            (0xFB50, 0xFDFF),
34            (0xFE70, 0xFEFF),
35        ],
36    ),
37    ("devanagari", &[(0x0900, 0x097F), (0xA8E0, 0xA8FF)]),
38    ("bengali", &[(0x0980, 0x09FF)]),
39    ("gurmukhi", &[(0x0A00, 0x0A7F)]),
40    ("gujarati", &[(0x0A80, 0x0AFF)]),
41    ("oriya", &[(0x0B00, 0x0B7F)]),
42    ("tamil", &[(0x0B80, 0x0BFF)]),
43    ("telugu", &[(0x0C00, 0x0C7F)]),
44    ("kannada", &[(0x0C80, 0x0CFF)]),
45    ("malayalam", &[(0x0D00, 0x0D7F)]),
46    ("sinhala", &[(0x0D80, 0x0DFF)]),
47    ("thai", &[(0x0E00, 0x0E7F)]),
48    ("lao", &[(0x0E80, 0x0EFF)]),
49    ("tibetan", &[(0x0F00, 0x0FFF)]),
50    ("myanmar", &[(0x1000, 0x109F)]),
51    ("georgian", &[(0x10A0, 0x10FF)]),
52    ("ethiopic", &[(0x1200, 0x137F)]),
53    ("khmer", &[(0x1780, 0x17FF)]),
54    (
55        "hangul",
56        &[(0x1100, 0x11FF), (0x3130, 0x318F), (0xAC00, 0xD7AF)],
57    ),
58    (
59        "kana",
60        &[(0x3040, 0x309F), (0x30A0, 0x30FF), (0x31F0, 0x31FF)],
61    ),
62    (
63        "han",
64        &[(0x3400, 0x4DBF), (0x4E00, 0x9FFF), (0xF900, 0xFAFF)],
65    ),
66];
67
68/// A diacritic rate above this is taken as evidence the text is not English.
69pub const NON_EN_DIACRITIC_RATE: f64 = 0.02;
70
71/// Non-Latin text is not for the English checkpoint even when Latin letters are the plurality.
72pub const NON_LATIN_FRACTION: f64 = 0.2;
73pub const NON_LATIN_MIN_FRACTION: f64 = 0.1;
74pub const NON_LATIN_MIN_LETTERS: i64 = 10;
75
76/// Function words per language, in the exact insertion order of the Python `_STOP` dict.
77fn stop() -> &'static IndexMap<&'static str, HashSet<&'static str>> {
78    static S: OnceLock<IndexMap<&'static str, HashSet<&'static str>>> = OnceLock::new();
79    S.get_or_init(|| {
80        let mut m: IndexMap<&'static str, HashSet<&'static str>> = IndexMap::new();
81        m.insert(
82            "en",
83            [
84                "the", "and", "is", "are", "was", "were", "to", "of", "in", "for", "with", "that",
85                "this", "it", "you", "have", "has", "not", "but", "on", "at", "be", "as", "from",
86                "will", "can", "would", "there", "their", "what", "which", "please", "we", "i",
87            ]
88            .into_iter()
89            .collect(),
90        );
91        m.insert(
92            "fr",
93            [
94                "le", "la", "les", "des", "une", "est", "pour", "dans", "que", "qui", "avec",
95                "sur", "pas", "plus", "nous", "vous", "être", "cette", "mais", "sont", "ont",
96                "aux", "ce", "et", "du", "au", "ou", "je", "tu", "il", "elle", "ils", "elles",
97                "mon", "ton", "ma", "ta", "sa", "mes", "tes", "ses", "ces", "deux", "trois",
98                "très", "bien", "tout", "tous", "toute", "fait", "veux", "veut", "peux", "peut",
99                "dois", "doit", "merci", "bonjour", "jour", "jours", "mois", "fois", "quand",
100                "comment", "pourquoi", "alors", "donc",
101            ]
102            .into_iter()
103            .collect(),
104        );
105        m.insert(
106            "de",
107            [
108                "der", "die", "das", "und", "ist", "ein", "eine", "den", "dem", "nicht", "mit",
109                "für", "auf", "von", "zu", "sich", "auch", "werden", "wurde", "haben", "sind",
110                "oder", "aber", "ich", "wir", "mir", "mich", "dir", "dich", "uns", "mein", "meine",
111                "meinen", "meinem", "meiner", "diese", "dieser", "diesen", "dieses", "einen",
112                "einem", "einer", "wie", "wo", "wann", "welche", "im", "zum", "zur", "aus", "bei",
113                "nach", "noch", "bitte", "heute", "jetzt", "kann", "kannst", "habe", "gibt",
114                "wird",
115                // shared with English on purpose: counted for English alone, they outvoted short German
116                "in", "was",
117            ]
118            .into_iter()
119            .collect(),
120        );
121        m.insert(
122            "es",
123            [
124                "el", "los", "las", "que", "por", "con", "para", "una", "es", "se", "del", "como",
125                "pero", "son", "está", "este", "esta", "todo", "más", "muy", "hay", "sus", "la",
126                "un", "y", "al", "lo", "le", "les", "su", "mi", "tu", "nos", "ni", "dos", "tres",
127                "fue", "fueron", "ser", "tiene", "tienen", "tengo", "puede", "pueden", "quiero",
128                "necesito", "hemos", "han", "sobre", "entre", "cuando", "donde", "porque",
129                "aunque", "también", "ya", "eso", "esto", "esa", "ese", "nada", "algo", "aquí",
130                "hoy", "gracias",
131            ]
132            .into_iter()
133            .collect(),
134        );
135        m.insert(
136            "pt",
137            [
138                "os", "as", "que", "em", "um", "uma", "para", "com", "não", "é", "se", "do", "da",
139                "dos", "das", "mas", "são", "está", "este", "esta", "muito", "pelo", "pela", "o",
140                "e", "na", "nas", "nos", "ao", "aos", "por", "foi", "era", "ser", "sou", "tem",
141                "tenho", "pode", "podem", "quero", "preciso", "eu", "meu", "minha", "seu", "sua",
142                "isso", "isto", "aqui", "ali", "como", "quando", "onde", "porque", "mais", "já",
143                "ainda", "agora", "hoje", "ontem", "dois", "três", "tudo", "nada", "obrigado",
144                "olá", "você", "vocês", "voce", "voces", "vc", "vcs", "nao", "sao", "ja", "até",
145                "tá", "pra", "gostaria", "obrigada", "também", "tambem", "estou", "estamos",
146                "meus", "minhas", "nosso", "nossa", "consigo", "cadê", "boa", "tarde", "noite",
147                "depois", "antes", "então", "entao", "ninguém", "ninguem", "alguém", "alguem",
148                "nenhum", "nenhuma", "estava", "ficou", "fiz", "deu",
149            ]
150            .into_iter()
151            .collect(),
152        );
153        m.insert(
154            "it",
155            [
156                "il", "lo", "gli", "che", "di", "per", "con", "non", "è", "si", "del", "della",
157                "sono", "questo", "questa", "anche", "come", "più", "sono", "nella", "alla", "la",
158                "le", "un", "uno", "una", "e", "ed", "o", "da", "su", "tra", "fra", "mi", "ci",
159                "ne", "ho", "hai", "ha", "abbiamo", "avete", "hanno", "era", "stato", "stata",
160                "devo", "deve", "devono", "voglio", "vorrei", "mio", "mia", "tuo", "sua", "quando",
161                "dove", "perche", "molto", "poco", "sempre", "mai", "già", "ancora", "adesso",
162                "oggi", "ieri", "grazie", "ciao", "scusa", "nel", "nell", "negli", "sul", "sulla",
163                "sulle", "dal", "dalla", "dallo", "dagli", "dei", "delle", "dello", "degli",
164                "agli", "alle", "col",
165            ]
166            .into_iter()
167            .collect(),
168        );
169        m.insert(
170            "nl",
171            [
172                "het", "een", "van", "is", "op", "te", "dat", "niet", "met", "voor", "zijn", "aan",
173                "door", "maar", "ook", "worden", "deze", "naar", "wordt",
174            ]
175            .into_iter()
176            .collect(),
177        );
178        m.insert(
179            "ro",
180            [
181                "și", "să", "este", "sunt", "care", "pentru", "din", "dar", "după", "până", "fără",
182                "ale", "lui", "în", "fost", "acum", "vreau", "trebuie", "foarte", "acest",
183                "această", "acesta", "aceasta", "mi", "ți", "vă", "nu",
184            ]
185            .into_iter()
186            .collect(),
187        );
188        m.insert(
189            "bn",
190            [
191                "ami",
192                "amar",
193                "amake",
194                "amra",
195                "amader",
196                "apni",
197                "apnar",
198                "apnake",
199                "apnara",
200                "tumi",
201                "tomar",
202                "tomake",
203                "tomra",
204                "tader",
205                "ota",
206                "eita",
207                "oita",
208                "ekta",
209                "ei",
210                "oi",
211                "ki",
212                "keno",
213                "kivabe",
214                "kibhabe",
215                "kothay",
216                "kokhon",
217                "kobe",
218                "koto",
219                "kintu",
220                "jodi",
221                "tahole",
222                "ar",
223                "theke",
224                "jonno",
225                "sathe",
226                "shathe",
227                "diye",
228                "niye",
229                "moddhe",
230                "kore",
231                "korte",
232                "korchi",
233                "korsi",
234                "korbo",
235                "korechi",
236                "koreche",
237                "korun",
238                "koren",
239                "korlam",
240                "hobe",
241                "hoyeche",
242                "hoise",
243                "hocche",
244                "hoyni",
245                "chai",
246                "chaina",
247                "lagbe",
248                "parchi",
249                "parbo",
250                "parchina",
251                "peyechi",
252                "paini",
253                "dite",
254                "dilam",
255                "diyechi",
256                "nai",
257                "khub",
258                "onek",
259                "ekhon",
260                "akhon",
261                "ekhono",
262                "abar",
263                "ekbar",
264                "duibar",
265                "ajke",
266                "kalke",
267                "taka",
268                "bhalo",
269                "valo",
270                "kharap",
271                "shomossa",
272                "somossa",
273                "dhonnobad",
274                "bhai",
275                "shob",
276                "keu",
277                "kichu",
278                "bolte",
279                "bolun",
280                "parben",
281                "asbe",
282                "jabe",
283                "pabo",
284                "ferot",
285                "dorkar",
286                "hoye",
287                "geche",
288                "gese",
289            ]
290            .into_iter()
291            .collect(),
292        );
293        m.insert(
294            "az",
295            [
296                "və", "ve", "bir", "bu", "üçün", "ucun", "ilə", "ile", "olan", "olub", "olmasa",
297                "var", "yox", "yoxdur", "mən", "sən", "biz", "siz", "onlar", "daha", "çox", "cox",
298                "hər", "nə", "kimi", "görə", "sonra", "əgər", "eger", "deyil", "lakin", "amma",
299                "ancaq", "artıq", "artiq", "də", "isə", "həm", "yalnız", "yalniz",
300            ]
301            .into_iter()
302            .collect(),
303        );
304        m
305    })
306}
307
308// Letters that ordinary English does not use.
309const DIACRITICS: &str = concat!(
310    "àâäãáåçéèêëíìîïñóòôöõøúùûüýÿßæœ", // Western European
311    "ăâîșțşţ",                         // Romanian
312    "ąćęłńśźż",                        // Polish
313    "čďěňřšťůž",                       // Czech / Slovak
314    "őű",                              // Hungarian
315    "ğı",                              // Turkish (text is lowercased before matching)
316    "āēģīķļņūž",                       // Baltic
317    "đ",                               // Serbo-Croatian / Vietnamese
318    "ə",                               // Azerbaijani
319);
320
321fn non_en_diacritics() -> &'static HashSet<char> {
322    static S: OnceLock<HashSet<char>> = OnceLock::new();
323    S.get_or_init(|| DIACRITICS.chars().collect())
324}
325
326/// Words that more than one list claims.
327fn shared_words() -> &'static HashSet<&'static str> {
328    static S: OnceLock<HashSet<&'static str>> = OnceLock::new();
329    S.get_or_init(|| {
330        let mut counts: HashMap<&'static str, usize> = HashMap::new();
331        for words in stop().values() {
332            for w in words {
333                *counts.entry(*w).or_insert(0) += 1;
334            }
335        }
336        counts
337            .into_iter()
338            .filter(|(_, n)| *n > 1)
339            .map(|(w, _)| w)
340            .collect()
341    })
342}
343
344fn word_re() -> &'static Regex {
345    static R: OnceLock<Regex> = OnceLock::new();
346    R.get_or_init(|| Regex::new(r"[^\W\d_]+").unwrap())
347}
348
349// A token whose dot or @ joins word characters is an identifier, not prose.
350fn identifier_re() -> &'static Regex {
351    static R: OnceLock<Regex> = OnceLock::new();
352    R.get_or_init(|| Regex::new(r"[\w-]*(?:[.@][\w-]+)+").unwrap())
353}
354
355/// A combining mark (Python `unicodedata.combining(ch) != 0`): Mn / Mc / Me.
356fn is_combining(ch: char) -> bool {
357    matches!(
358        get_general_category(ch),
359        GeneralCategory::NonspacingMark
360            | GeneralCategory::SpacingMark
361            | GeneralCategory::EnclosingMark
362    )
363}
364
365/// `detect_script` / `script_profile` Latin test (`cp < 0x02B0`).
366fn is_latin_detect(cp: u32) -> bool {
367    cp < 0x02B0
368        || (0x1E00..=0x1EFF).contains(&cp)
369        || (0xFF21..=0xFF3A).contains(&cp)
370        || (0xFF41..=0xFF5A).contains(&cp)
371}
372
373/// `_script_of` Latin test (`cp < 0x0250` — deliberately distinct from `detect_script`).
374fn is_latin_script_of(cp: u32) -> bool {
375    cp < 0x0250
376        || (0x1E00..=0x1EFF).contains(&cp)
377        || (0xFF21..=0xFF3A).contains(&cp)
378        || (0xFF41..=0xFF5A).contains(&cp)
379}
380
381fn in_ranges(cp: u32, ranges: &[(u32, u32)]) -> bool {
382    ranges.iter().any(|&(lo, hi)| lo <= cp && cp <= hi)
383}
384
385/// Round to 4 decimals, half-to-even (Python `round(x, 4)`).
386fn round4(x: f64) -> f64 {
387    round_half_even(x * 10000.0) / 10000.0
388}
389
390/// Round to the nearest integer, half-to-even (Python `round(x)`); valid for `x >= 0`.
391fn round_half_even(x: f64) -> f64 {
392    let floor = x.floor();
393    let diff = x - floor;
394    if diff < 0.5 {
395        floor
396    } else if diff > 0.5 {
397        floor + 1.0
398    } else {
399        // exactly .5 -> round to even
400        if (floor as i64) % 2 == 0 {
401            floor
402        } else {
403            floor + 1.0
404        }
405    }
406}
407
408// --- text flattening -------------------------------------------------------
409
410fn iter_text(v: &Value, depth: usize, out: &mut Vec<String>) {
411    if depth > 6 {
412        return;
413    }
414    match v {
415        Value::Null => {}
416        Value::String(s) => out.push(s.clone()),
417        Value::Array(a) => {
418            for x in a {
419                iter_text(x, depth + 1, out);
420            }
421        }
422        Value::Object(m) => {
423            for x in m.values() {
424                iter_text(x, depth + 1, out);
425            }
426        }
427        _ => {}
428    }
429}
430
431/// Flatten a state into the text used for detection (keys are ignored). `max_chars` default 4000.
432pub fn state_text(state: &Value, max_chars: usize) -> String {
433    let mut leaves: Vec<String> = Vec::new();
434    iter_text(state, 0, &mut leaves);
435    // Bound the join work before truncation: only `max_chars` are ever returned, so stop
436    // materializing leaves once the budget is spent (parity with upstream `state_text`).
437    let mut parts: Vec<String> = Vec::new();
438    let mut budget = max_chars as isize;
439    for leaf in leaves {
440        if budget <= 0 {
441            break;
442        }
443        let leaf_len = leaf.chars().count() as isize;
444        if leaf_len > budget {
445            parts.push(leaf.chars().take(budget as usize).collect());
446            break;
447        }
448        parts.push(leaf);
449        // Account for the joining space without materializing the full text first.
450        budget -= leaf_len + 1;
451    }
452    parts.join(" ").chars().take(max_chars).collect()
453}
454
455// --- script detection ------------------------------------------------------
456
457/// Dominant script of `text`: `"latin"`, `"han"`, ... or `"unknown"` if there are no letters.
458pub fn detect_script(text: &str) -> String {
459    let mut counts: IndexMap<&'static str, i64> = IndexMap::new();
460    let mut latin: i64 = 0;
461    for ch in text.chars() {
462        if !ch.is_alphabetic() {
463            continue;
464        }
465        let cp = ch as u32;
466        if is_latin_detect(cp) {
467            latin += 1;
468            continue;
469        }
470        let mut found: Option<&'static str> = None;
471        for (name, ranges) in SCRIPT_RANGES {
472            if in_ranges(cp, ranges) {
473                found = Some(name);
474                break;
475            }
476        }
477        let key = found.unwrap_or("other");
478        *counts.entry(key).or_insert(0) += 1;
479    }
480    counts.insert("latin", latin);
481    let total: i64 = counts.values().sum();
482    if total == 0 {
483        return "unknown".to_string();
484    }
485    // Python `max(..., key=value)` returns the first item on a tie (insertion order).
486    let mut best: &str = "";
487    let mut best_n: i64 = i64::MIN;
488    for (k, v) in &counts {
489        if *v > best_n {
490            best_n = *v;
491            best = k;
492        }
493    }
494    best.to_string()
495}
496
497/// Fraction of alphabetic characters belonging to each detected script (nonzero entries only).
498pub fn script_profile(text: &str) -> IndexMap<String, f64> {
499    let mut counts: IndexMap<&'static str, i64> = IndexMap::new();
500    counts.insert("latin", 0);
501    for ch in text.chars() {
502        if !ch.is_alphabetic() {
503            continue;
504        }
505        let cp = ch as u32;
506        if is_latin_detect(cp) {
507            *counts.get_mut("latin").unwrap() += 1;
508            continue;
509        }
510        let mut found: Option<&'static str> = None;
511        for (name, ranges) in SCRIPT_RANGES {
512            if in_ranges(cp, ranges) {
513                found = Some(name);
514                break;
515            }
516        }
517        let key = found.unwrap_or("other");
518        *counts.entry(key).or_insert(0) += 1;
519    }
520    let total: i64 = counts.values().sum();
521    let mut out: IndexMap<String, f64> = IndexMap::new();
522    if total == 0 {
523        return out;
524    }
525    for (k, v) in &counts {
526        if *v != 0 {
527            out.insert((*k).to_string(), *v as f64 / total as f64);
528        }
529    }
530    out
531}
532
533fn script_of(ch: char) -> Option<&'static str> {
534    let cp = ch as u32;
535    if is_latin_script_of(cp) {
536        return None;
537    }
538    for (name, ranges) in SCRIPT_RANGES {
539        if in_ranges(cp, ranges) {
540            return Some(name);
541        }
542    }
543    None
544}
545
546/// Non-Latin runs that read as words rather than as annotation inside English prose.
547fn non_latin_words(text: &str) -> Vec<String> {
548    let mut runs: Vec<String> = Vec::new();
549    let mut cur = String::new();
550    let mut script: Option<&'static str> = None;
551    for ch in text.chars() {
552        if is_combining(ch) {
553            continue;
554        }
555        let s = script_of(ch);
556        if s.is_some() && s == script {
557            cur.push(ch);
558            continue;
559        }
560        if !cur.is_empty() {
561            runs.push(std::mem::take(&mut cur));
562        }
563        match s {
564            Some(_) => {
565                cur = ch.to_string();
566                script = s;
567            }
568            None => {
569                cur = String::new();
570                script = None;
571            }
572        }
573    }
574    if !cur.is_empty() {
575        runs.push(cur);
576    }
577    runs.into_iter()
578        .filter(|w| {
579            w.chars().count() >= 2 && !w.chars().next().map(|c| c.is_uppercase()).unwrap_or(false)
580        })
581        .collect()
582}
583
584// --- Latin-script language guess ------------------------------------------
585
586/// Evidence behind the Latin-script language guess.
587#[derive(Debug, Clone)]
588pub struct LatinProfile {
589    pub language: Option<String>,
590    pub english_hits: usize,
591    pub diacritic_rate: f64,
592    pub looks_non_english: bool,
593}
594
595pub fn latin_profile(text: &str) -> LatinProfile {
596    // Order matches Python: identifier-strip the ORIGINAL text, replace 'İ'->'i', lower, then findall.
597    let stripped = identifier_re().replace_all(text, " ");
598    let normalized = stripped.replace('\u{0130}', "i").to_lowercase();
599    let words: Vec<&str> = word_re()
600        .find_iter(&normalized)
601        .map(|m| m.as_str())
602        .collect();
603
604    let lowered = text.to_lowercase();
605    let diac_set = non_en_diacritics();
606    let diac = lowered.chars().filter(|ch| diac_set.contains(ch)).count();
607    let len = lowered.chars().count();
608    let diac_rate = diac as f64 / (std::cmp::max(1, len) as f64);
609    let non_english = diac_rate >= NON_EN_DIACRITIC_RATE;
610
611    if words.len() < 4 {
612        return LatinProfile {
613            language: None,
614            english_hits: 0,
615            diacritic_rate: diac_rate,
616            looks_non_english: non_english,
617        };
618    }
619
620    let stop = stop();
621    let shared = shared_words();
622
623    // Per-language raw hit counts.
624    let mut scores: IndexMap<&'static str, i64> = IndexMap::new();
625    for (lg, sw) in stop {
626        let s = words.iter().filter(|w| sw.contains(*w)).count() as i64;
627        scores.insert(lg, s);
628    }
629    let en = scores.get("en").copied().unwrap_or(0);
630
631    let word_set: HashSet<&str> = words.iter().copied().collect();
632
633    // Only a language that matched at least one word no other list claims may be named.
634    let mut best_lg: Option<&'static str> = None;
635    let mut best: i64 = 0;
636    for (lg, sw) in stop {
637        if *lg == "en" {
638            continue;
639        }
640        let has_evidence = word_set
641            .iter()
642            .any(|w| sw.contains(*w) && !shared.contains(*w));
643        if !has_evidence {
644            continue;
645        }
646        let s = *scores.get(lg).unwrap();
647        if s > best {
648            best = s;
649            best_lg = Some(lg);
650        }
651    }
652
653    let mut language: Option<String> = None;
654    if let Some(bl) = best_lg {
655        // A non-English language needs a clear margin over English function words, or (with
656        // non-English letters present) at least a two-hit tie. Mirrors the elif ladder in
657        // laya/lang.py, with the two "name best_lg" branches combined.
658        if best >= std::cmp::max(2, en + 2) || (non_english && best >= std::cmp::max(2, en)) {
659            language = Some(bl.to_string());
660        } else if en > 0 && !non_english {
661            language = Some("en".to_string());
662        }
663    } else if en > 0 && !non_english {
664        language = Some("en".to_string());
665    }
666
667    LatinProfile {
668        language,
669        english_hits: en as usize,
670        diacritic_rate: diac_rate,
671        looks_non_english: non_english,
672    }
673}
674
675/// Best-effort language code for Latin-script text, or `None` when undecided.
676pub fn guess_latin_language(text: &str) -> Option<String> {
677    latin_profile(text).language
678}
679
680// --- top-level analysis ----------------------------------------------------
681
682/// Result of [`analyse`]. Field names/semantics mirror the Python dict keys.
683#[derive(Debug, Clone)]
684pub struct Analysis {
685    pub script: String,
686    pub script_profile: IndexMap<String, f64>,
687    pub language: Option<String>,
688    pub is_english: bool,
689    pub language_undecided: bool,
690    pub diacritic_rate: f64,
691    pub non_latin_fraction: f64,
692}
693
694pub fn analyse(state: &Value) -> Analysis {
695    let text = state_text(state, 4000);
696    let prof = script_profile(&text);
697    let mut script = detect_script(&text);
698
699    let non_latin = if !prof.is_empty() {
700        round4(1.0 - prof.get("latin").copied().unwrap_or(0.0))
701    } else {
702        0.0
703    };
704    let sum_alpha = text.chars().filter(|c| c.is_alphabetic()).count() as f64;
705    let n_non_latin = round_half_even(non_latin * sum_alpha) as i64;
706
707    if script == "latin"
708        && !non_latin_words(&text).is_empty()
709        && (non_latin >= NON_LATIN_FRACTION
710            || (non_latin >= NON_LATIN_MIN_FRACTION && n_non_latin >= NON_LATIN_MIN_LETTERS))
711    {
712        // reclassify to the dominant non-latin script (first max on a tie, insertion order)
713        let mut best: Option<&str> = None;
714        let mut best_v = f64::NEG_INFINITY;
715        for (k, v) in &prof {
716            if k == "latin" {
717                continue;
718            }
719            if *v > best_v {
720                best_v = *v;
721                best = Some(k);
722            }
723        }
724        if let Some(b) = best {
725            script = b.to_string();
726        }
727    }
728
729    if script == "unknown" {
730        return Analysis {
731            script: "unknown".to_string(),
732            script_profile: prof,
733            language: None,
734            is_english: true,
735            language_undecided: true,
736            diacritic_rate: 0.0,
737            non_latin_fraction: 0.0,
738        };
739    }
740    if script != "latin" {
741        return Analysis {
742            script,
743            script_profile: prof,
744            language: None,
745            is_english: false,
746            language_undecided: true,
747            diacritic_rate: 0.0,
748            non_latin_fraction: non_latin,
749        };
750    }
751
752    let prof_lat = latin_profile(&text);
753    let lang = prof_lat.language.clone();
754    let undecided = lang.is_none();
755    let english = lang.as_deref() == Some("en") || (undecided && !prof_lat.looks_non_english);
756    Analysis {
757        script: "latin".to_string(),
758        script_profile: prof,
759        language: lang,
760        is_english: english,
761        language_undecided: undecided,
762        diacritic_rate: round4(prof_lat.diacritic_rate),
763        non_latin_fraction: non_latin,
764    }
765}
766
767/// True when the English checkpoint can be expected to read this state.
768pub fn is_english(state: &Value) -> bool {
769    analyse(state).is_english
770}
771
772#[cfg(test)]
773mod tests {
774    use super::*;
775    use serde_json::json;
776
777    fn s(t: &str) -> Value {
778        Value::String(t.to_string())
779    }
780
781    #[test]
782    fn detects_devanagari_as_non_latin() {
783        assert_eq!(detect_script("मुझसे दो बार शुल्क लिया गया"), "devanagari");
784    }
785
786    #[test]
787    fn routes_english_latin_to_english() {
788        let a = analyse(&s("Please refund the duplicate charge"));
789        assert_eq!(a.script, "latin");
790        assert!(a.is_english);
791        assert_eq!(a.language.as_deref(), Some("en"));
792    }
793
794    #[test]
795    fn routes_german_latin_to_non_english() {
796        assert!(!is_english(&s("Der Kunde wurde zweimal belastet")));
797    }
798
799    #[test]
800    fn detects_french_multilingual_signals() {
801        let a = analyse(&s("Je voudrais un remboursement pour la commande"));
802        assert!(!a.is_english);
803        assert_eq!(a.language.as_deref(), Some("fr"));
804    }
805
806    #[test]
807    fn unknown_no_letters_is_english_and_undecided() {
808        let a = analyse(&s("123 !!!"));
809        assert_eq!(a.script, "unknown");
810        assert!(a.is_english);
811        assert!(a.language_undecided);
812    }
813
814    #[test]
815    fn cjk_han_is_not_english() {
816        let a = analyse(&s("请退还重复的费用"));
817        assert_eq!(a.script, "han");
818        assert!(!a.is_english);
819    }
820
821    #[test]
822    fn hindi_devanagari_is_not_english() {
823        let a = analyse(&s("मुझसे दो बार शुल्क लिया गया"));
824        assert_eq!(a.script, "devanagari");
825        assert!(!a.is_english);
826    }
827
828    #[test]
829    fn romanized_bangla_detected() {
830        let a = analyse(&s("ami ekta refund chai amar order er jonno"));
831        assert_eq!(a.script, "latin");
832        assert_eq!(a.language.as_deref(), Some("bn"));
833        assert!(!a.is_english);
834    }
835
836    #[test]
837    fn url_only_not_misdetected_as_foreign() {
838        // Without identifier stripping, four "com" tokens would score Portuguese/Italian.
839        let a = analyse(&s("acme.com foo.com bar.com test.com"));
840        assert_eq!(a.language, None);
841        assert!(a.is_english);
842    }
843
844    #[test]
845    fn script_profile_latin_first_and_fractions() {
846        let p = script_profile("abc देव");
847        // latin inserted first; both scripts present.
848        let keys: Vec<&String> = p.keys().collect();
849        assert_eq!(keys[0], "latin");
850        assert!(p.contains_key("devanagari"));
851        let sum: f64 = p.values().sum();
852        assert!((sum - 1.0).abs() < 1e-9);
853    }
854
855    #[test]
856    fn nested_state_is_flattened() {
857        let state = json!({"a": "Please refund", "b": ["the", "duplicate", "charge"]});
858        assert!(is_english(&state));
859    }
860
861    #[test]
862    fn round4_is_half_to_even() {
863        assert_eq!(round_half_even(2.5), 2.0);
864        assert_eq!(round_half_even(3.5), 4.0);
865    }
866}
laya/lang.rs

laya/
lang.rs
