use unicode_normalization::UnicodeNormalization;
#[cfg(feature = "icu_segmenter")]
mod icu_segmenter;
#[cfg(feature = "icu_segmenter")]
use icu_segmenter::icu_segment;
#[cfg(feature = "lindera")]
mod lindera_segmenter;
#[cfg(feature = "lindera")]
use lindera_segmenter::lindera_segment;
pub trait Segmenter: Send + Sync {
fn segment(&self, text: &str) -> Box<dyn Iterator<Item = String> + '_>;
fn name(&self) -> &'static str;
}
impl Segmenter for Box<dyn Segmenter> {
fn segment(&self, text: &str) -> Box<dyn Iterator<Item = String> + '_> {
self.as_ref().segment(text)
}
fn name(&self) -> &'static str {
self.as_ref().name()
}
}
pub struct BigramFirstCharSegmenter;
impl Segmenter for BigramFirstCharSegmenter {
fn segment(&self, text: &str) -> Box<dyn Iterator<Item = String> + '_> {
let tokens = collect_bigram_first_char(text);
Box::new(tokens.into_iter())
}
fn name(&self) -> &'static str {
"bigram-first-char"
}
}
#[cfg(feature = "jieba-rs")]
pub struct JiebaSegmenter {
jieba: jieba_rs::Jieba,
}
#[cfg(feature = "jieba-rs")]
impl Default for JiebaSegmenter {
fn default() -> Self {
Self::new()
}
}
#[cfg(feature = "jieba-rs")]
impl JiebaSegmenter {
pub fn new() -> Self {
Self {
jieba: jieba_rs::Jieba::new(),
}
}
}
#[cfg(feature = "jieba-rs")]
impl Segmenter for JiebaSegmenter {
fn segment(&self, text: &str) -> Box<dyn Iterator<Item = String> + '_> {
let tokens: Vec<String> = self
.jieba
.cut(text, false)
.into_iter()
.map(String::from)
.collect();
Box::new(tokens.into_iter())
}
fn name(&self) -> &'static str {
"jieba"
}
}
static SEGMENTER: std::sync::OnceLock<Box<dyn Segmenter>> = std::sync::OnceLock::new();
pub fn init_segmenter() -> &'static dyn Segmenter {
SEGMENTER.get_or_init(|| {
#[cfg(feature = "jieba-rs")]
{
return Box::new(JiebaSegmenter::new()) as Box<dyn Segmenter>;
}
#[allow(unreachable_code)]
Box::new(BigramFirstCharSegmenter)
})
}
pub fn segmenter() -> &'static dyn Segmenter {
init_segmenter()
}
fn dense_script(c: char) -> Option<u8> {
match c as u32 {
0x0E00..=0x0E7F | 0x0E80..=0x0EFF | 0x1000..=0x109F | 0x1780..=0x17FF => Some(2),
0x2E80..=0x2FFF | 0x3000..=0x303F | 0x3040..=0x30FF | 0x3400..=0x4DBF | 0x4E00..=0x9FFF | 0xAC00..=0xD7AF | 0xF900..=0xFAFF | 0x20000..=0x2FA1F => Some(1),
_ => None,
}
}
pub fn is_dense_script(c: char) -> bool {
dense_script(c).is_some()
}
pub fn is_searchable_query(query: &str) -> bool {
let mut chars = query.chars().filter(|c| !c.is_whitespace());
match chars.next() {
None => false,
Some(c) if is_dense_script(c) => true,
Some(_) => query.chars().filter(|c| !c.is_whitespace()).count() >= 3,
}
}
fn collect_bigram_first_char(text: &str) -> Vec<String> {
let normalised: String = text.nfkc().collect();
let mut tokens = Vec::new();
let mut run: Vec<char> = Vec::new();
let mut word: String = String::new();
fn flush_run(run: &mut Vec<char>, tokens: &mut Vec<String>) {
if run.is_empty() {
return;
}
if let Some(kind) = run.first().and_then(|c| dense_script(*c)) {
match kind {
2 => {
for c in run.iter() {
tokens.push(c.to_string());
}
}
_ => {
if let Some(first) = run.first() {
tokens.push(first.to_string());
}
for pair in run.windows(2) {
let mut s = String::new();
s.push(pair[0]);
s.push(pair[1]);
tokens.push(s);
}
}
}
}
run.clear();
}
for c in normalised.chars() {
if dense_script(c).is_some() {
if !word.is_empty() {
tokens.push(std::mem::take(&mut word));
}
run.push(c);
} else {
flush_run(&mut run, &mut tokens);
if c.is_whitespace() {
if !word.is_empty() {
tokens.push(std::mem::take(&mut word));
}
} else {
word.push(c);
}
}
}
if !word.is_empty() {
tokens.push(std::mem::take(&mut word));
}
flush_run(&mut run, &mut tokens);
tokens
}
pub fn detect_language(text: &str) -> Option<&'static str> {
let mut script_counts: std::collections::HashMap<&'static str, usize> =
std::collections::HashMap::new();
for c in text.chars() {
let script = match c as u32 {
0x2E80..=0x2FFF | 0x3000..=0x303F | 0x3400..=0x4DBF | 0x4E00..=0x9FFF | 0xF900..=0xFAFF | 0x20000..=0x2FA1F => "zh",
0x3040..=0x30FF
=> "ja",
0xAC00..=0xD7AF
=> "ko",
0x0E00..=0x0E7F
=> "th",
0x0E80..=0x0EFF
=> "lo",
0x1000..=0x109F
=> "my",
0x1780..=0x17FF
=> "km",
0x0600..=0x06FF | 0x0750..=0x077F | 0x08A0..=0x08FF | 0xFB50..=0xFDFF | 0xFE70..=0xFEFF => "ar",
0x0590..=0x05FF
=> "he",
0x0900..=0x097F
=> "hi",
_ => continue,
};
*script_counts.entry(script).or_insert(0) += 1;
}
script_counts
.into_iter()
.max_by_key(|(_, count)| *count)
.map(|(script, _)| script)
}
#[cfg(feature = "lindera")]
fn is_japanese(text: &str) -> bool {
text.chars().any(|c| {
let cp = c as u32;
(0x3040..=0x309F).contains(&cp) || (0x30A0..=0x30FF).contains(&cp) })
}
pub fn fts_segment(text: &str) -> String {
let seg = segmenter();
match seg.name() {
"bigram-first-char" => {
#[cfg(feature = "icu_segmenter")]
if let Some(lang) = detect_language(text) {
return icu_segment(text, lang);
}
#[cfg(feature = "lindera")]
if is_japanese(text) {
return lindera_segment(text);
}
let tokens = collect_bigram_first_char(text);
tokens.join(" ")
}
_ => {
let tokens = seg.segment(text);
tokens.collect::<Vec<_>>().join(" ")
}
}
}
pub fn same_tokens(a: &str, b: &str) -> bool {
a.split_whitespace().eq(b.split_whitespace())
}
pub fn query_terms(query: &str) -> Vec<String> {
fts_segment(query)
.split_whitespace()
.map(|t| t.to_string())
.collect()
}
pub const SNIPPET_CHARS: usize = 240;
pub fn excerpt(text: &str, terms: &[String], max_chars: usize) -> String {
let collapsed = collapse_whitespace(text);
let chars: Vec<char> = collapsed.chars().collect();
if chars.len() <= max_chars {
return collapsed;
}
let lower: Vec<char> = chars
.iter()
.map(|c| c.to_lowercase().next().unwrap_or(*c))
.collect();
let mut hit: Option<usize> = None;
for term in terms {
let needle: Vec<char> = term
.chars()
.map(|c| c.to_lowercase().next().unwrap_or(c))
.collect();
if needle.is_empty() {
continue;
}
if let Some(pos) = lower
.windows(needle.len())
.position(|window| window == needle.as_slice())
{
hit = Some(hit.map_or(pos, |best: usize| best.min(pos)));
}
}
let start = hit
.map(|pos| pos.saturating_sub(max_chars / 3))
.unwrap_or(0);
let end = (start + max_chars).min(chars.len());
let mut out: String = chars[start..end].iter().collect();
if start > 0 {
out.insert(0, '…');
}
if end < chars.len() {
out.push('…');
}
out
}
fn collapse_whitespace(text: &str) -> String {
let mut out = String::with_capacity(text.len());
let mut prev_space = false;
for c in text.chars() {
if c.is_whitespace() {
if !prev_space {
out.push(' ');
}
prev_space = true;
} else {
out.push(c);
prev_space = false;
}
}
out.trim().to_string()
}
#[cfg(test)]
mod tests {
use super::*;
#[cfg(not(feature = "icu_segmenter"))]
#[test]
fn cjk_runs_become_bigrams() {
let s = fts_segment("全文檢索");
let tokens: Vec<&str> = s.split_whitespace().collect();
assert_eq!(tokens, ["全", "全文", "文檢", "檢索"]);
}
#[cfg(feature = "icu_segmenter")]
#[test]
fn cjk_runs_become_words() {
let s = fts_segment("全文檢索");
let tokens: Vec<&str> = s.split_whitespace().collect();
assert!(tokens.contains(&"檢索"), "{tokens:?}");
assert!(
!tokens.contains(&"文檢"),
"no cross-word bigrams under ICU: {tokens:?}"
);
}
#[test]
fn two_character_words_survive() {
let doc = fts_segment("南投縣魚池鄉");
let query = fts_segment("魚池");
let query_tokens: Vec<&str> = query.split_whitespace().collect();
let doc_tokens: Vec<&str> = doc.split_whitespace().collect();
assert!(
query_tokens.iter().any(|qt| doc_tokens.contains(qt)),
"query tokens: {:?}, doc tokens: {:?}",
query_tokens,
doc_tokens
);
}
#[test]
fn leaves_latin_alone() {
assert_eq!(fts_segment("fridge warranty 2027"), "fridge warranty 2027");
}
#[test]
fn normalises_full_width() {
assert_eq!(fts_segment("WARRANTY"), "WARRANTY");
}
#[cfg(not(feature = "icu_segmenter"))]
#[test]
fn thai_is_indexed_per_character() {
let s = fts_segment("การรับประกัน");
assert!(s.split_whitespace().count() > 3, "{s}");
}
#[cfg(feature = "icu_segmenter")]
#[test]
fn thai_is_indexed_per_word() {
let s = fts_segment("การรับประกัน");
let tokens: Vec<&str> = s.split_whitespace().collect();
assert!(tokens.contains(&"การ"), "{tokens:?}");
assert!(tokens.contains(&"รับประกัน"), "{tokens:?}");
}
#[test]
fn short_dense_queries_are_searchable() {
assert!(is_searchable_query("検索"));
assert!(is_searchable_query("法"));
assert!(!is_searchable_query("ab"));
assert!(!is_searchable_query(" "));
assert!(is_searchable_query("tax"));
}
#[test]
fn excerpt_is_a_verbatim_window() {
let text = "alpha beta gamma delta epsilon zeta eta theta";
let out = excerpt(text, &["delta".to_string()], 24);
assert!(out.contains("delta"), "{out}");
assert!(text.contains(out.trim_matches('…')), "{out}");
}
#[test]
fn excerpt_leaves_short_text_whole() {
assert_eq!(
excerpt("fridge warranty", &["warranty".to_string()], 240),
"fridge warranty"
);
}
#[test]
fn segmenter_name_default() {
assert_eq!(segmenter().name(), "bigram-first-char");
}
}