remem-ai 0.5.141

use super::translations::CJK_EN_TRANSLATIONS;

pub(super) fn is_cjk(c: char) -> bool {
    matches!(
        c,
        '\u{4E00}'..='\u{9FFF}' |
        '\u{3400}'..='\u{4DBF}' |
        '\u{F900}'..='\u{FAFF}'
    )
}

pub(super) fn tokenize_mixed(raw: &str) -> Vec<String> {
    let mut tokens = Vec::new();
    for part in raw.split_whitespace() {
        let chars: Vec<char> = part.chars().collect();
        if chars.is_empty() {
            continue;
        }
        let mut i = 0;
        while i < chars.len() {
            if is_cjk(chars[i]) {
                let start = i;
                while i < chars.len() && is_cjk(chars[i]) {
                    i += 1;
                }
                tokens.push(chars[start..i].iter().collect());
            } else {
                let start = i;
                while i < chars.len() && !is_cjk(chars[i]) {
                    i += 1;
                }
                let segment: String = chars[start..i].iter().collect();
                let trimmed = segment.trim();
                if !trimmed.is_empty() {
                    tokens.push(trimmed.to_string());
                }
            }
        }
    }
    tokens
}

pub(super) fn segment_cjk(text: &str) -> Vec<String> {
    let chars: Vec<char> = text.chars().collect();
    let mut segments = Vec::new();
    let mut i = 0;

    while i < chars.len() {
        let mut best_len = 0;
        for len in (2..=4).rev() {
            if i + len <= chars.len() {
                let candidate: String = chars[i..i + len].iter().collect();
                if CJK_EN_TRANSLATIONS.contains_key(candidate.as_str()) {
                    best_len = len;
                    break;
                }
            }
        }
        if best_len > 0 {
            segments.push(chars[i..i + best_len].iter().collect());
            i += best_len;
        } else {
            segments.push(chars[i..i + 1].iter().collect());
            i += 1;
        }
    }

    segments
}