laya/
lang.rs

1//! Dependency-free language/script detection used to route between Laya checkpoints.
2//!
3//! Faithful port of `laya/lang.py`. Routing needs one decision: *is this English Latin text, or
4//! is it something the English checkpoint cannot read?* Script detection is exact; the Latin-script
5//! language guess is a stopword/diacritic heuristic and is explicitly best-effort.
6//!
7//! This is the source-of-truth port of the Python module (which carries `bn`/`az` stopword lists
8//! and the `_non_latin_words` reclassification step that the older TypeScript reference dropped).
9
10use std::collections::{HashMap, HashSet};
11use std::sync::OnceLock;
12
13use indexmap::IndexMap;
14use regex::Regex;
15use serde_json::Value;
16use unicode_general_category::{get_general_category, GeneralCategory};
17
18// Unicode blocks that the English (ModernBERT-large, 50k English BPE) checkpoint cannot read.
19const SCRIPT_RANGES: &[(&str, &[(u32, u32)])] = &[
20    ("greek", &[(0x0370, 0x03FF), (0x1F00, 0x1FFF)]),
21    (
22        "cyrillic",
23        &[(0x0400, 0x052F), (0x2DE0, 0x2DFF), (0xA640, 0xA69F)],
24    ),
25    ("armenian", &[(0x0530, 0x058F)]),
26    ("hebrew", &[(0x0590, 0x05FF)]),
27    (
28        "arabic",
29        &[
30            (0x0600, 0x06FF),
31            (0x0750, 0x077F),
32            (0x08A0, 0x08FF),
33            (0xFB50, 0xFDFF),
34            (0xFE70, 0xFEFF),
35        ],
36    ),
37    ("devanagari", &[(0x0900, 0x097F), (0xA8E0, 0xA8FF)]),
38    ("bengali", &[(0x0980, 0x09FF)]),
39    ("gurmukhi", &[(0x0A00, 0x0A7F)]),
40    ("gujarati", &[(0x0A80, 0x0AFF)]),
41    ("oriya", &[(0x0B00, 0x0B7F)]),
42    ("tamil", &[(0x0B80, 0x0BFF)]),
43    ("telugu", &[(0x0C00, 0x0C7F)]),
44    ("kannada", &[(0x0C80, 0x0CFF)]),
45    ("malayalam", &[(0x0D00, 0x0D7F)]),
46    ("sinhala", &[(0x0D80, 0x0DFF)]),
47    ("thai", &[(0x0E00, 0x0E7F)]),
48    ("lao", &[(0x0E80, 0x0EFF)]),
49    ("tibetan", &[(0x0F00, 0x0FFF)]),
50    ("myanmar", &[(0x1000, 0x109F)]),
51    ("georgian", &[(0x10A0, 0x10FF)]),
52    ("ethiopic", &[(0x1200, 0x137F)]),
53    ("khmer", &[(0x1780, 0x17FF)]),
54    (
55        "hangul",
56        &[(0x1100, 0x11FF), (0x3130, 0x318F), (0xAC00, 0xD7AF)],
57    ),
58    (
59        "kana",
60        &[(0x3040, 0x309F), (0x30A0, 0x30FF), (0x31F0, 0x31FF)],
61    ),
62    (
63        "han",
64        &[(0x3400, 0x4DBF), (0x4E00, 0x9FFF), (0xF900, 0xFAFF)],
65    ),
66];
67
68/// A diacritic rate above this is taken as evidence the text is not English.
69pub const NON_EN_DIACRITIC_RATE: f64 = 0.02;
70
71/// Non-Latin text is not for the English checkpoint even when Latin letters are the plurality.
72pub const NON_LATIN_FRACTION: f64 = 0.2;
73pub const NON_LATIN_MIN_FRACTION: f64 = 0.1;
74pub const NON_LATIN_MIN_LETTERS: i64 = 10;
75
76/// Function words per language, in the exact insertion order of the Python `_STOP` dict.
77fn stop() -> &'static IndexMap<&'static str, HashSet<&'static str>> {
78    static S: OnceLock<IndexMap<&'static str, HashSet<&'static str>>> = OnceLock::new();
79    S.get_or_init(|| {
80        let mut m: IndexMap<&'static str, HashSet<&'static str>> = IndexMap::new();
81        m.insert(
82            "en",
83            [
84                "the", "and", "is", "are", "was", "were", "to", "of", "in", "for", "with", "that",
85                "this", "it", "you", "have", "has", "not", "but", "on", "at", "be", "as", "from",
86                "will", "can", "would", "there", "their", "what", "which", "please", "we", "i",
87            ]
88            .into_iter()
89            .collect(),
90        );
91        m.insert(
92            "fr",
93            [
94                "le", "la", "les", "des", "une", "est", "pour", "dans", "que", "qui", "avec",
95                "sur", "pas", "plus", "nous", "vous", "être", "cette", "mais", "sont", "ont",
96                "aux", "ce", "et", "du", "au", "ou", "je", "tu", "il", "elle", "ils", "elles",
97                "mon", "ton", "ma", "ta", "sa", "mes", "tes", "ses", "ces", "deux", "trois",
98                "très", "bien", "tout", "tous", "toute", "fait", "veux", "veut", "peux", "peut",
99                "dois", "doit", "merci", "bonjour", "jour", "jours", "mois", "fois", "quand",
100                "comment", "pourquoi", "alors", "donc",
101            ]
102            .into_iter()
103            .collect(),
104        );
105        m.insert(
106            "de",
107            [
108                "der", "die", "das", "und", "ist", "ein", "eine", "den", "dem", "nicht", "mit",
109                "für", "auf", "von", "zu", "sich", "auch", "werden", "wurde", "haben", "sind",
110                "oder", "aber",
111            ]
112            .into_iter()
113            .collect(),
114        );
115        m.insert(
116            "es",
117            [
118                "el", "los", "las", "que", "por", "con", "para", "una", "es", "se", "del", "como",
119                "pero", "son", "está", "este", "esta", "todo", "más", "muy", "hay", "sus", "la",
120                "un", "y", "al", "lo", "le", "les", "su", "mi", "tu", "nos", "ni", "dos", "tres",
121                "fue", "fueron", "ser", "tiene", "tienen", "tengo", "puede", "pueden", "quiero",
122                "necesito", "hemos", "han", "sobre", "entre", "cuando", "donde", "porque",
123                "aunque", "también", "ya", "eso", "esto", "esa", "ese", "nada", "algo", "aquí",
124                "hoy", "gracias",
125            ]
126            .into_iter()
127            .collect(),
128        );
129        m.insert(
130            "pt",
131            [
132                "os", "as", "que", "em", "um", "uma", "para", "com", "não", "é", "se", "do", "da",
133                "dos", "das", "mas", "são", "está", "este", "esta", "muito", "pelo", "pela", "o",
134                "e", "na", "nas", "nos", "ao", "aos", "por", "foi", "era", "ser", "sou", "tem",
135                "tenho", "pode", "podem", "quero", "preciso", "eu", "meu", "minha", "seu", "sua",
136                "isso", "isto", "aqui", "ali", "como", "quando", "onde", "porque", "mais", "já",
137                "ainda", "agora", "hoje", "ontem", "dois", "três", "tudo", "nada", "obrigado",
138                "olá", "você", "vocês", "voce", "voces", "vc", "vcs", "nao", "sao", "ja", "até",
139                "tá", "pra", "gostaria", "obrigada", "também", "tambem", "estou", "estamos",
140                "meus", "minhas", "nosso", "nossa", "consigo", "cadê", "boa", "tarde", "noite",
141                "depois", "antes", "então", "entao", "ninguém", "ninguem", "alguém", "alguem",
142                "nenhum", "nenhuma", "estava", "ficou", "fiz", "deu",
143            ]
144            .into_iter()
145            .collect(),
146        );
147        m.insert(
148            "it",
149            [
150                "il", "lo", "gli", "che", "di", "per", "con", "non", "è", "si", "del", "della",
151                "sono", "questo", "questa", "anche", "come", "più", "sono", "nella", "alla", "la",
152                "le", "un", "uno", "una", "e", "ed", "o", "da", "su", "tra", "fra", "mi", "ci",
153                "ne", "ho", "hai", "ha", "abbiamo", "avete", "hanno", "era", "stato", "stata",
154                "devo", "deve", "devono", "voglio", "vorrei", "mio", "mia", "tuo", "sua", "quando",
155                "dove", "perche", "molto", "poco", "sempre", "mai", "già", "ancora", "adesso",
156                "oggi", "ieri", "grazie", "ciao", "scusa", "nel", "nell", "negli", "sul", "sulla",
157                "sulle", "dal", "dalla", "dallo", "dagli", "dei", "delle", "dello", "degli",
158                "agli", "alle", "col",
159            ]
160            .into_iter()
161            .collect(),
162        );
163        m.insert(
164            "nl",
165            [
166                "het", "een", "van", "is", "op", "te", "dat", "niet", "met", "voor", "zijn", "aan",
167                "door", "maar", "ook", "worden", "deze", "naar", "wordt",
168            ]
169            .into_iter()
170            .collect(),
171        );
172        m.insert(
173            "ro",
174            [
175                "și", "să", "este", "sunt", "care", "pentru", "din", "dar", "după", "până", "fără",
176                "ale", "lui", "în", "fost", "acum", "vreau", "trebuie", "foarte", "acest",
177                "această", "acesta", "aceasta", "mi", "ți", "vă", "nu",
178            ]
179            .into_iter()
180            .collect(),
181        );
182        m.insert(
183            "bn",
184            [
185                "ami",
186                "amar",
187                "amake",
188                "amra",
189                "amader",
190                "apni",
191                "apnar",
192                "apnake",
193                "apnara",
194                "tumi",
195                "tomar",
196                "tomake",
197                "tomra",
198                "tader",
199                "ota",
200                "eita",
201                "oita",
202                "ekta",
203                "ei",
204                "oi",
205                "ki",
206                "keno",
207                "kivabe",
208                "kibhabe",
209                "kothay",
210                "kokhon",
211                "kobe",
212                "koto",
213                "kintu",
214                "jodi",
215                "tahole",
216                "ar",
217                "theke",
218                "jonno",
219                "sathe",
220                "shathe",
221                "diye",
222                "niye",
223                "moddhe",
224                "kore",
225                "korte",
226                "korchi",
227                "korsi",
228                "korbo",
229                "korechi",
230                "koreche",
231                "korun",
232                "koren",
233                "korlam",
234                "hobe",
235                "hoyeche",
236                "hoise",
237                "hocche",
238                "hoyni",
239                "chai",
240                "chaina",
241                "lagbe",
242                "parchi",
243                "parbo",
244                "parchina",
245                "peyechi",
246                "paini",
247                "dite",
248                "dilam",
249                "diyechi",
250                "nai",
251                "khub",
252                "onek",
253                "ekhon",
254                "akhon",
255                "ekhono",
256                "abar",
257                "ekbar",
258                "duibar",
259                "ajke",
260                "kalke",
261                "taka",
262                "bhalo",
263                "valo",
264                "kharap",
265                "shomossa",
266                "somossa",
267                "dhonnobad",
268                "bhai",
269                "shob",
270                "keu",
271                "kichu",
272                "bolte",
273                "bolun",
274                "parben",
275                "asbe",
276                "jabe",
277                "pabo",
278                "ferot",
279                "dorkar",
280                "hoye",
281                "geche",
282                "gese",
283            ]
284            .into_iter()
285            .collect(),
286        );
287        m.insert(
288            "az",
289            [
290                "və", "ve", "bir", "bu", "üçün", "ucun", "ilə", "ile", "olan", "olub", "olmasa",
291                "var", "yox", "yoxdur", "mən", "sən", "biz", "siz", "onlar", "daha", "çox", "cox",
292                "hər", "nə", "kimi", "görə", "sonra", "əgər", "eger", "deyil", "lakin", "amma",
293                "ancaq", "artıq", "artiq", "də", "isə", "həm", "yalnız", "yalniz",
294            ]
295            .into_iter()
296            .collect(),
297        );
298        m
299    })
300}
301
302// Letters that ordinary English does not use.
303const DIACRITICS: &str = concat!(
304    "àâäãáåçéèêëíìîïñóòôöõøúùûüýÿßæœ", // Western European
305    "ăâîșțşţ",                         // Romanian
306    "ąćęłńśźż",                        // Polish
307    "čďěňřšťůž",                       // Czech / Slovak
308    "őű",                              // Hungarian
309    "ğı",                              // Turkish (text is lowercased before matching)
310    "āēģīķļņūž",                       // Baltic
311    "đ",                               // Serbo-Croatian / Vietnamese
312    "ə",                               // Azerbaijani
313);
314
315fn non_en_diacritics() -> &'static HashSet<char> {
316    static S: OnceLock<HashSet<char>> = OnceLock::new();
317    S.get_or_init(|| DIACRITICS.chars().collect())
318}
319
320/// Words that more than one list claims.
321fn shared_words() -> &'static HashSet<&'static str> {
322    static S: OnceLock<HashSet<&'static str>> = OnceLock::new();
323    S.get_or_init(|| {
324        let mut counts: HashMap<&'static str, usize> = HashMap::new();
325        for words in stop().values() {
326            for w in words {
327                *counts.entry(*w).or_insert(0) += 1;
328            }
329        }
330        counts
331            .into_iter()
332            .filter(|(_, n)| *n > 1)
333            .map(|(w, _)| w)
334            .collect()
335    })
336}
337
338fn word_re() -> &'static Regex {
339    static R: OnceLock<Regex> = OnceLock::new();
340    R.get_or_init(|| Regex::new(r"[^\W\d_]+").unwrap())
341}
342
343// A token whose dot or @ joins word characters is an identifier, not prose.
344fn identifier_re() -> &'static Regex {
345    static R: OnceLock<Regex> = OnceLock::new();
346    R.get_or_init(|| Regex::new(r"[\w-]*(?:[.@][\w-]+)+").unwrap())
347}
348
349/// A combining mark (Python `unicodedata.combining(ch) != 0`): Mn / Mc / Me.
350fn is_combining(ch: char) -> bool {
351    matches!(
352        get_general_category(ch),
353        GeneralCategory::NonspacingMark
354            | GeneralCategory::SpacingMark
355            | GeneralCategory::EnclosingMark
356    )
357}
358
359/// `detect_script` / `script_profile` Latin test (`cp < 0x02B0`).
360fn is_latin_detect(cp: u32) -> bool {
361    cp < 0x02B0
362        || (0x1E00..=0x1EFF).contains(&cp)
363        || (0xFF21..=0xFF3A).contains(&cp)
364        || (0xFF41..=0xFF5A).contains(&cp)
365}
366
367/// `_script_of` Latin test (`cp < 0x0250` — deliberately distinct from `detect_script`).
368fn is_latin_script_of(cp: u32) -> bool {
369    cp < 0x0250
370        || (0x1E00..=0x1EFF).contains(&cp)
371        || (0xFF21..=0xFF3A).contains(&cp)
372        || (0xFF41..=0xFF5A).contains(&cp)
373}
374
375fn in_ranges(cp: u32, ranges: &[(u32, u32)]) -> bool {
376    ranges.iter().any(|&(lo, hi)| lo <= cp && cp <= hi)
377}
378
379/// Round to 4 decimals, half-to-even (Python `round(x, 4)`).
380fn round4(x: f64) -> f64 {
381    round_half_even(x * 10000.0) / 10000.0
382}
383
384/// Round to the nearest integer, half-to-even (Python `round(x)`); valid for `x >= 0`.
385fn round_half_even(x: f64) -> f64 {
386    let floor = x.floor();
387    let diff = x - floor;
388    if diff < 0.5 {
389        floor
390    } else if diff > 0.5 {
391        floor + 1.0
392    } else {
393        // exactly .5 -> round to even
394        if (floor as i64) % 2 == 0 {
395            floor
396        } else {
397            floor + 1.0
398        }
399    }
400}
401
402// --- text flattening -------------------------------------------------------
403
404fn iter_text(v: &Value, depth: usize, out: &mut Vec<String>) {
405    if depth > 6 {
406        return;
407    }
408    match v {
409        Value::Null => {}
410        Value::String(s) => out.push(s.clone()),
411        Value::Array(a) => {
412            for x in a {
413                iter_text(x, depth + 1, out);
414            }
415        }
416        Value::Object(m) => {
417            for x in m.values() {
418                iter_text(x, depth + 1, out);
419            }
420        }
421        _ => {}
422    }
423}
424
425/// Flatten a state into the text used for detection (keys are ignored). `max_chars` default 4000.
426pub fn state_text(state: &Value, max_chars: usize) -> String {
427    let mut parts: Vec<String> = Vec::new();
428    iter_text(state, 0, &mut parts);
429    let joined = parts.join(" ");
430    joined.chars().take(max_chars).collect()
431}
432
433// --- script detection ------------------------------------------------------
434
435/// Dominant script of `text`: `"latin"`, `"han"`, ... or `"unknown"` if there are no letters.
436pub fn detect_script(text: &str) -> String {
437    let mut counts: IndexMap<&'static str, i64> = IndexMap::new();
438    let mut latin: i64 = 0;
439    for ch in text.chars() {
440        if !ch.is_alphabetic() {
441            continue;
442        }
443        let cp = ch as u32;
444        if is_latin_detect(cp) {
445            latin += 1;
446            continue;
447        }
448        let mut found: Option<&'static str> = None;
449        for (name, ranges) in SCRIPT_RANGES {
450            if in_ranges(cp, ranges) {
451                found = Some(name);
452                break;
453            }
454        }
455        let key = found.unwrap_or("other");
456        *counts.entry(key).or_insert(0) += 1;
457    }
458    counts.insert("latin", latin);
459    let total: i64 = counts.values().sum();
460    if total == 0 {
461        return "unknown".to_string();
462    }
463    // Python `max(..., key=value)` returns the first item on a tie (insertion order).
464    let mut best: &str = "";
465    let mut best_n: i64 = i64::MIN;
466    for (k, v) in &counts {
467        if *v > best_n {
468            best_n = *v;
469            best = k;
470        }
471    }
472    best.to_string()
473}
474
475/// Fraction of alphabetic characters belonging to each detected script (nonzero entries only).
476pub fn script_profile(text: &str) -> IndexMap<String, f64> {
477    let mut counts: IndexMap<&'static str, i64> = IndexMap::new();
478    counts.insert("latin", 0);
479    for ch in text.chars() {
480        if !ch.is_alphabetic() {
481            continue;
482        }
483        let cp = ch as u32;
484        if is_latin_detect(cp) {
485            *counts.get_mut("latin").unwrap() += 1;
486            continue;
487        }
488        let mut found: Option<&'static str> = None;
489        for (name, ranges) in SCRIPT_RANGES {
490            if in_ranges(cp, ranges) {
491                found = Some(name);
492                break;
493            }
494        }
495        let key = found.unwrap_or("other");
496        *counts.entry(key).or_insert(0) += 1;
497    }
498    let total: i64 = counts.values().sum();
499    let mut out: IndexMap<String, f64> = IndexMap::new();
500    if total == 0 {
501        return out;
502    }
503    for (k, v) in &counts {
504        if *v != 0 {
505            out.insert((*k).to_string(), *v as f64 / total as f64);
506        }
507    }
508    out
509}
510
511fn script_of(ch: char) -> Option<&'static str> {
512    let cp = ch as u32;
513    if is_latin_script_of(cp) {
514        return None;
515    }
516    for (name, ranges) in SCRIPT_RANGES {
517        if in_ranges(cp, ranges) {
518            return Some(name);
519        }
520    }
521    None
522}
523
524/// Non-Latin runs that read as words rather than as annotation inside English prose.
525fn non_latin_words(text: &str) -> Vec<String> {
526    let mut runs: Vec<String> = Vec::new();
527    let mut cur = String::new();
528    let mut script: Option<&'static str> = None;
529    for ch in text.chars() {
530        if is_combining(ch) {
531            continue;
532        }
533        let s = script_of(ch);
534        if s.is_some() && s == script {
535            cur.push(ch);
536            continue;
537        }
538        if !cur.is_empty() {
539            runs.push(std::mem::take(&mut cur));
540        }
541        match s {
542            Some(_) => {
543                cur = ch.to_string();
544                script = s;
545            }
546            None => {
547                cur = String::new();
548                script = None;
549            }
550        }
551    }
552    if !cur.is_empty() {
553        runs.push(cur);
554    }
555    runs.into_iter()
556        .filter(|w| {
557            w.chars().count() >= 2 && !w.chars().next().map(|c| c.is_uppercase()).unwrap_or(false)
558        })
559        .collect()
560}
561
562// --- Latin-script language guess ------------------------------------------
563
564/// Evidence behind the Latin-script language guess.
565#[derive(Debug, Clone)]
566pub struct LatinProfile {
567    pub language: Option<String>,
568    pub english_hits: usize,
569    pub diacritic_rate: f64,
570    pub looks_non_english: bool,
571}
572
573pub fn latin_profile(text: &str) -> LatinProfile {
574    // Order matches Python: identifier-strip the ORIGINAL text, replace 'İ'->'i', lower, then findall.
575    let stripped = identifier_re().replace_all(text, " ");
576    let normalized = stripped.replace('\u{0130}', "i").to_lowercase();
577    let words: Vec<&str> = word_re()
578        .find_iter(&normalized)
579        .map(|m| m.as_str())
580        .collect();
581
582    let lowered = text.to_lowercase();
583    let diac_set = non_en_diacritics();
584    let diac = lowered.chars().filter(|ch| diac_set.contains(ch)).count();
585    let len = lowered.chars().count();
586    let diac_rate = diac as f64 / (std::cmp::max(1, len) as f64);
587    let non_english = diac_rate >= NON_EN_DIACRITIC_RATE;
588
589    if words.len() < 4 {
590        return LatinProfile {
591            language: None,
592            english_hits: 0,
593            diacritic_rate: diac_rate,
594            looks_non_english: non_english,
595        };
596    }
597
598    let stop = stop();
599    let shared = shared_words();
600
601    // Per-language raw hit counts.
602    let mut scores: IndexMap<&'static str, i64> = IndexMap::new();
603    for (lg, sw) in stop {
604        let s = words.iter().filter(|w| sw.contains(*w)).count() as i64;
605        scores.insert(lg, s);
606    }
607    let en = scores.get("en").copied().unwrap_or(0);
608
609    let word_set: HashSet<&str> = words.iter().copied().collect();
610
611    // Only a language that matched at least one word no other list claims may be named.
612    let mut best_lg: Option<&'static str> = None;
613    let mut best: i64 = 0;
614    for (lg, sw) in stop {
615        if *lg == "en" {
616            continue;
617        }
618        let has_evidence = word_set
619            .iter()
620            .any(|w| sw.contains(*w) && !shared.contains(*w));
621        if !has_evidence {
622            continue;
623        }
624        let s = *scores.get(lg).unwrap();
625        if s > best {
626            best = s;
627            best_lg = Some(lg);
628        }
629    }
630
631    let mut language: Option<String> = None;
632    if let Some(bl) = best_lg {
633        // A non-English language needs a clear margin over English function words, or (with
634        // non-English letters present) at least a two-hit tie. Mirrors the elif ladder in
635        // laya/lang.py, with the two "name best_lg" branches combined.
636        if best >= std::cmp::max(2, en + 2) || (non_english && best >= std::cmp::max(2, en)) {
637            language = Some(bl.to_string());
638        } else if en > 0 && !non_english {
639            language = Some("en".to_string());
640        }
641    } else if en > 0 && !non_english {
642        language = Some("en".to_string());
643    }
644
645    LatinProfile {
646        language,
647        english_hits: en as usize,
648        diacritic_rate: diac_rate,
649        looks_non_english: non_english,
650    }
651}
652
653/// Best-effort language code for Latin-script text, or `None` when undecided.
654pub fn guess_latin_language(text: &str) -> Option<String> {
655    latin_profile(text).language
656}
657
658// --- top-level analysis ----------------------------------------------------
659
660/// Result of [`analyse`]. Field names/semantics mirror the Python dict keys.
661#[derive(Debug, Clone)]
662pub struct Analysis {
663    pub script: String,
664    pub script_profile: IndexMap<String, f64>,
665    pub language: Option<String>,
666    pub is_english: bool,
667    pub language_undecided: bool,
668    pub diacritic_rate: f64,
669    pub non_latin_fraction: f64,
670}
671
672pub fn analyse(state: &Value) -> Analysis {
673    let text = state_text(state, 4000);
674    let prof = script_profile(&text);
675    let mut script = detect_script(&text);
676
677    let non_latin = if !prof.is_empty() {
678        round4(1.0 - prof.get("latin").copied().unwrap_or(0.0))
679    } else {
680        0.0
681    };
682    let sum_alpha = text.chars().filter(|c| c.is_alphabetic()).count() as f64;
683    let n_non_latin = round_half_even(non_latin * sum_alpha) as i64;
684
685    if script == "latin"
686        && !non_latin_words(&text).is_empty()
687        && (non_latin >= NON_LATIN_FRACTION
688            || (non_latin >= NON_LATIN_MIN_FRACTION && n_non_latin >= NON_LATIN_MIN_LETTERS))
689    {
690        // reclassify to the dominant non-latin script (first max on a tie, insertion order)
691        let mut best: Option<&str> = None;
692        let mut best_v = f64::NEG_INFINITY;
693        for (k, v) in &prof {
694            if k == "latin" {
695                continue;
696            }
697            if *v > best_v {
698                best_v = *v;
699                best = Some(k);
700            }
701        }
702        if let Some(b) = best {
703            script = b.to_string();
704        }
705    }
706
707    if script == "unknown" {
708        return Analysis {
709            script: "unknown".to_string(),
710            script_profile: prof,
711            language: None,
712            is_english: true,
713            language_undecided: true,
714            diacritic_rate: 0.0,
715            non_latin_fraction: 0.0,
716        };
717    }
718    if script != "latin" {
719        return Analysis {
720            script,
721            script_profile: prof,
722            language: None,
723            is_english: false,
724            language_undecided: true,
725            diacritic_rate: 0.0,
726            non_latin_fraction: non_latin,
727        };
728    }
729
730    let prof_lat = latin_profile(&text);
731    let lang = prof_lat.language.clone();
732    let undecided = lang.is_none();
733    let english = lang.as_deref() == Some("en") || (undecided && !prof_lat.looks_non_english);
734    Analysis {
735        script: "latin".to_string(),
736        script_profile: prof,
737        language: lang,
738        is_english: english,
739        language_undecided: undecided,
740        diacritic_rate: round4(prof_lat.diacritic_rate),
741        non_latin_fraction: non_latin,
742    }
743}
744
745/// True when the English checkpoint can be expected to read this state.
746pub fn is_english(state: &Value) -> bool {
747    analyse(state).is_english
748}
749
750#[cfg(test)]
751mod tests {
752    use super::*;
753    use serde_json::json;
754
755    fn s(t: &str) -> Value {
756        Value::String(t.to_string())
757    }
758
759    #[test]
760    fn detects_devanagari_as_non_latin() {
761        assert_eq!(detect_script("मुझसे दो बार शुल्क लिया गया"), "devanagari");
762    }
763
764    #[test]
765    fn routes_english_latin_to_english() {
766        let a = analyse(&s("Please refund the duplicate charge"));
767        assert_eq!(a.script, "latin");
768        assert!(a.is_english);
769        assert_eq!(a.language.as_deref(), Some("en"));
770    }
771
772    #[test]
773    fn routes_german_latin_to_non_english() {
774        assert!(!is_english(&s("Der Kunde wurde zweimal belastet")));
775    }
776
777    #[test]
778    fn detects_french_multilingual_signals() {
779        let a = analyse(&s("Je voudrais un remboursement pour la commande"));
780        assert!(!a.is_english);
781        assert_eq!(a.language.as_deref(), Some("fr"));
782    }
783
784    #[test]
785    fn unknown_no_letters_is_english_and_undecided() {
786        let a = analyse(&s("123 !!!"));
787        assert_eq!(a.script, "unknown");
788        assert!(a.is_english);
789        assert!(a.language_undecided);
790    }
791
792    #[test]
793    fn cjk_han_is_not_english() {
794        let a = analyse(&s("请退还重复的费用"));
795        assert_eq!(a.script, "han");
796        assert!(!a.is_english);
797    }
798
799    #[test]
800    fn hindi_devanagari_is_not_english() {
801        let a = analyse(&s("मुझसे दो बार शुल्क लिया गया"));
802        assert_eq!(a.script, "devanagari");
803        assert!(!a.is_english);
804    }
805
806    #[test]
807    fn romanized_bangla_detected() {
808        let a = analyse(&s("ami ekta refund chai amar order er jonno"));
809        assert_eq!(a.script, "latin");
810        assert_eq!(a.language.as_deref(), Some("bn"));
811        assert!(!a.is_english);
812    }
813
814    #[test]
815    fn url_only_not_misdetected_as_foreign() {
816        // Without identifier stripping, four "com" tokens would score Portuguese/Italian.
817        let a = analyse(&s("acme.com foo.com bar.com test.com"));
818        assert_eq!(a.language, None);
819        assert!(a.is_english);
820    }
821
822    #[test]
823    fn script_profile_latin_first_and_fractions() {
824        let p = script_profile("abc देव");
825        // latin inserted first; both scripts present.
826        let keys: Vec<&String> = p.keys().collect();
827        assert_eq!(keys[0], "latin");
828        assert!(p.contains_key("devanagari"));
829        let sum: f64 = p.values().sum();
830        assert!((sum - 1.0).abs() < 1e-9);
831    }
832
833    #[test]
834    fn nested_state_is_flattened() {
835        let state = json!({"a": "Please refund", "b": ["the", "duplicate", "charge"]});
836        assert!(is_english(&state));
837    }
838
839    #[test]
840    fn round4_is_half_to_even() {
841        assert_eq!(round_half_even(2.5), 2.0);
842        assert_eq!(round_half_even(3.5), 4.0);
843    }
844}
laya/lang.rs

laya/
lang.rs
