Skip to main content

cortex_router/
fold.rs

1//! Unicode-aware folding for lexical classifiers.
2//!
3//! ASCII lowercase treats Cyrillic, Hebrew, and German ß as opaque, so a
4//! Russian mutation cue never fires. NFKC plus Unicode case fold is the
5//! minimum that makes mixed-language tasks comparable.
6
7use unicode_normalization::UnicodeNormalization;
8
9/// Compatibility-normalize, case-fold, and collapse whitespace.
10#[must_use]
11pub fn fold_text(value: &str) -> String {
12    value
13        .nfkc()
14        .collect::<String>()
15        .to_lowercase()
16        .split_whitespace()
17        .collect::<Vec<_>>()
18        .join(" ")
19}
20
21/// True when the task mixes Latin letters with another script.
22#[must_use]
23pub fn mixed_script(value: &str) -> bool {
24    let mut latin = false;
25    let mut other = false;
26    for ch in value.chars() {
27        if ch.is_ascii_alphabetic() {
28            latin = true;
29        } else if ch.is_alphabetic() {
30            other = true;
31        }
32        if latin && other {
33            return true;
34        }
35    }
36    false
37}
38
39/// Fold, then keep letters/digits and turn everything else into spaces so
40/// phrase matching can use word boundaries.
41#[must_use]
42pub fn fold_words(value: &str) -> String {
43    fold_text(value)
44        .chars()
45        .map(|character| {
46            if character.is_alphanumeric() {
47                character
48            } else {
49                ' '
50            }
51        })
52        .collect::<String>()
53        .split_whitespace()
54        .collect::<Vec<_>>()
55        .join(" ")
56}
57
58#[cfg(test)]
59mod tests {
60    use super::*;
61
62    #[test]
63    fn cyrillic_case_folds() {
64        assert_eq!(fold_text("  ДОБАВЬ  Обработку "), "добавь обработку");
65    }
66
67    #[test]
68    fn nfkc_composes_compatibility_forms() {
69        assert_eq!(fold_text("fix"), "fix");
70    }
71}