#[cfg(feature = "icu_segmenter")]
use icu_segmenter::WordSegmenter;
#[cfg(feature = "icu_segmenter")]
pub fn icu_segment(text: &str, _locale: &str) -> String {
let segmenter = WordSegmenter::new_auto();
let mut tokens = Vec::new();
let mut start = 0;
for boundary in segmenter.segment_str(text) {
if boundary > start {
let token = &text[start..boundary];
if !token.trim().is_empty() {
tokens.push(token.to_string());
}
}
start = boundary;
}
if start < text.len() {
let token = &text[start..];
if !token.trim().is_empty() {
tokens.push(token.to_string());
}
}
if tokens.is_empty() {
return text.to_string();
}
tokens.join(" ")
}
#[cfg(not(feature = "icu_segmenter"))]
pub fn icu_segment(text: &str, _locale: &str) -> String {
text.to_string()
}