use unicode_segmentation::UnicodeSegmentation;
#[inline]
pub fn is_cjk(c: char) -> bool {
matches!(c as u32,
0x1100..=0x11FF | 0x3040..=0x30FF | 0x3400..=0x4DBF | 0x4E00..=0x9FFF | 0xA960..=0xA97F | 0xAC00..=0xD7AF | 0xF900..=0xFAFF | 0xFF66..=0xFF9D | 0x20000..=0x2FA1F
)
}
#[inline]
fn is_meaningful(w: &str) -> bool {
w.chars().any(|c| c.is_alphanumeric())
}
pub fn tokenize(text: &str) -> Vec<String> {
let mut out = Vec::with_capacity(text.len() / 5);
for word in text.unicode_words() {
if !is_meaningful(word) {
continue;
}
let cjk_chars: Vec<char> = word.chars().filter(|c| is_cjk(*c)).collect();
if cjk_chars.len() * 2 >= word.chars().count() && !cjk_chars.is_empty() {
for c in &cjk_chars {
out.push(c.to_lowercase().collect());
}
for pair in cjk_chars.windows(2) {
out.push(pair.iter().collect());
}
} else {
out.push(word.to_lowercase());
}
}
out
}
pub fn estimate_tokens(text: &str) -> usize {
let mut cjk = 0usize;
let mut other_bytes = 0usize;
for c in text.chars() {
if is_cjk(c) {
cjk += 1;
} else {
other_bytes += c.len_utf8();
}
}
let latin = other_bytes.div_ceil(4);
let cjk_tokens = (cjk as f64 / 1.4).ceil() as usize;
(latin + cjk_tokens).max(usize::from(!text.trim().is_empty()))
}
pub fn normalize_ws(s: &str) -> String {
let mut out = String::with_capacity(s.len());
let mut pending_space = false;
for c in s.chars() {
if c.is_whitespace() {
pending_space = !out.is_empty();
} else {
if pending_space {
out.push(' ');
pending_space = false;
}
out.push(c);
}
}
out
}
pub fn visible_len(s: &str) -> usize {
s.chars().count()
}
#[cfg(test)]
mod visible_len_tests {
use super::*;
use unicode_segmentation::UnicodeSegmentation;
#[test]
fn agrees_with_grapheme_counting_on_ordinary_text() {
for s in ["hello world", "한국어 텍스트입니다", "日本語のテキスト", "Ελληνικά", ""]
{
assert_eq!(visible_len(s), s.graphemes(true).count(), "{s:?}");
}
}
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn latin_tokens_are_lowercased_words() {
assert_eq!(tokenize("Hello, World! 42"), ["hello", "world", "42"]);
}
#[test]
fn korean_yields_unigrams_and_bigrams() {
let t = tokenize("한국어");
assert!(t.contains(&"한".to_string()));
assert!(t.contains(&"한국".to_string()));
assert!(t.contains(&"국어".to_string()));
}
#[test]
fn cjk_costs_more_tokens_per_char_than_latin() {
assert!(estimate_tokens("데이터 수집") > estimate_tokens("data"));
assert_eq!(estimate_tokens(""), 0);
}
#[test]
fn whitespace_collapses() {
assert_eq!(normalize_ws(" a \n\t b "), "a b");
}
}