#[cfg(feature = "lindera")]
use lindera::dictionary::{load_embedded_dictionary, DictionaryKind};
#[cfg(feature = "lindera")]
use lindera::mode::{Mode, Penalty};
#[cfg(feature = "lindera")]
use lindera::segmenter::Segmenter;
#[cfg(feature = "lindera")]
use std::sync::OnceLock;
#[cfg(feature = "lindera")]
fn segmenter() -> Option<&'static Segmenter> {
static SEGMENTER: OnceLock<Option<Segmenter>> = OnceLock::new();
SEGMENTER
.get_or_init(|| {
let dictionary = load_embedded_dictionary(DictionaryKind::IPADIC).ok()?;
Some(Segmenter::new(
Mode::Decompose(Penalty::default()),
dictionary,
None,
))
})
.as_ref()
}
#[cfg(feature = "lindera")]
pub fn lindera_segment(text: &str) -> String {
let Some(segmenter) = segmenter() else {
return text.to_string();
};
let Ok(tokens) = segmenter.segment(std::borrow::Cow::Borrowed(text)) else {
return text.to_string();
};
let tokens: Vec<&str> = tokens
.iter()
.map(|token| token.surface.as_ref())
.filter(|t| !t.trim().is_empty())
.collect();
if tokens.is_empty() {
return text.to_string();
}
tokens.join(" ")
}
#[cfg(not(feature = "lindera"))]
pub fn lindera_segment(text: &str) -> String {
text.to_string()
}