use crate::unicode_data::{contains, is_letter, is_mark, is_number};
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub struct Segment<'a> {
pub text: &'a str,
pub is_word: bool,
}
pub trait Segmenter: Send + Sync {
fn segment_words<'a>(&self, text: &'a str, locale: Option<&str>) -> Vec<Segment<'a>>;
fn graphemes<'a>(&self, text: &'a str, locale: Option<&str>) -> Vec<&'a str>;
}
#[derive(Debug, Clone, Copy, Default, PartialEq, Eq)]
pub struct SpecSegmenter;
#[rustfmt::skip]
const RUN_RANGES: &[(u32, u32)] = &[
(0x0E00, 0x0E7F), (0x0E80, 0x0EFF), (0x1000, 0x109F), (0x1100, 0x11FF), (0x1780, 0x17FF), (0x3005, 0x3007), (0x3041, 0x30FF), (0x3130, 0x318F), (0x31F0, 0x31FF), (0x3400, 0x4DBF), (0x4E00, 0x9FFF), (0xA960, 0xA97F), (0xA9E0, 0xA9FF), (0xAA60, 0xAA7F), (0xAC00, 0xD7A3), (0xD7B0, 0xD7FF), (0xF900, 0xFAFF), (0xFF66, 0xFF9D), (0xFFA0, 0xFFDC), (0x20000, 0x2EBEF), (0x2F800, 0x2FA1F), (0x30000, 0x323AF), ];
const JAMO_L: &[(u32, u32)] = &[(0x1100, 0x115F), (0xA960, 0xA97C)];
const JAMO_V: &[(u32, u32)] = &[(0x1160, 0x11A7), (0xD7B0, 0xD7C6)];
const JAMO_T: &[(u32, u32)] = &[(0x11A8, 0x11FF), (0xD7CB, 0xD7FB)];
const SYLLABLE_FIRST: u32 = 0xAC00;
const SYLLABLE_LAST: u32 = 0xD7A3;
const INDIC_LINKERS: [char; 6] = [
'\u{094D}', '\u{09CD}', '\u{0ACD}', '\u{0B4D}', '\u{0C4D}', '\u{0D4D}',
];
#[rustfmt::skip]
const INDIC_CONSONANTS: &[(u32, u32)] = &[
(0x0915, 0x0939), (0x0958, 0x095F), (0x0978, 0x097F), (0x0995, 0x09A8), (0x09AA, 0x09B0), (0x09B2, 0x09B2), (0x09B6, 0x09B9), (0x09DC, 0x09DD), (0x09DF, 0x09DF), (0x09F0, 0x09F1),
(0x0A95, 0x0AA8), (0x0AAA, 0x0AB0), (0x0AB2, 0x0AB3), (0x0AB5, 0x0AB9), (0x0AF9, 0x0AF9),
(0x0B15, 0x0B28), (0x0B2A, 0x0B30), (0x0B32, 0x0B33), (0x0B35, 0x0B39), (0x0B5C, 0x0B5D), (0x0B5F, 0x0B5F), (0x0B71, 0x0B71),
(0x0C15, 0x0C28), (0x0C2A, 0x0C39), (0x0C58, 0x0C5A), (0x0D15, 0x0D3A), ];
const ZWJ: char = '\u{200D}';
fn is_word_char(c: char) -> bool {
if c.is_ascii() {
c.is_ascii_alphanumeric()
} else {
is_letter(c) || is_number(c) || is_mark(c)
}
}
fn starts_word(c: char) -> bool {
if c.is_ascii() {
c.is_ascii_alphanumeric()
} else {
is_letter(c) || is_number(c)
}
}
fn is_mark_char(c: char) -> bool {
!c.is_ascii() && is_mark(c)
}
fn is_run_char(c: char) -> bool {
!c.is_ascii() && contains(RUN_RANGES, u32::from(c)) && is_word_char(c)
}
fn is_apostrophe(c: char) -> bool {
c == '\'' || c == '\u{2019}'
}
#[derive(Clone, Copy, PartialEq, Eq)]
enum Jamo {
L,
V,
T,
Lv,
Lvt,
}
fn jamo_class(c: char) -> Option<Jamo> {
let code = u32::from(c);
if (SYLLABLE_FIRST..=SYLLABLE_LAST).contains(&code) {
return Some(if (code - SYLLABLE_FIRST) % 28 == 0 {
Jamo::Lv
} else {
Jamo::Lvt
});
}
if contains(JAMO_L, code) {
Some(Jamo::L)
} else if contains(JAMO_V, code) {
Some(Jamo::V)
} else if contains(JAMO_T, code) {
Some(Jamo::T)
} else {
None
}
}
fn hangul_joins(previous: Option<Jamo>, following: Option<Jamo>) -> bool {
match (previous, following) {
(None, _) | (_, None) => false,
(Some(Jamo::L), _) => true,
(Some(Jamo::Lv | Jamo::V), Some(next)) => matches!(next, Jamo::V | Jamo::T),
(Some(Jamo::Lvt | Jamo::T), Some(next)) => next == Jamo::T,
}
}
fn is_indic_consonant(c: char) -> bool {
contains(INDIC_CONSONANTS, u32::from(c))
}
fn is_indic_linker(c: char) -> bool {
INDIC_LINKERS.contains(&c)
}
struct Chars<'a> {
text: &'a str,
chars: Vec<(usize, char)>,
}
impl<'a> Chars<'a> {
fn new(text: &'a str) -> Self {
Self {
text,
chars: text.char_indices().collect(),
}
}
fn len(&self) -> usize {
self.chars.len()
}
fn at(&self, index: usize) -> char {
self.chars[index].1
}
fn slice(&self, start: usize, end: usize) -> &'a str {
let byte = |index: usize| self.chars.get(index).map_or(self.text.len(), |c| c.0);
&self.text[byte(start)..byte(end)]
}
}
impl Segmenter for SpecSegmenter {
fn segment_words<'a>(&self, text: &'a str, _locale: Option<&str>) -> Vec<Segment<'a>> {
let chars = Chars::new(text);
let len = chars.len();
let mut out = Vec::new();
let mut index = 0;
while index < len {
let start = index;
let is_word = if !starts_word(chars.at(index)) {
while index < len && !starts_word(chars.at(index)) {
index += 1;
}
false
} else if is_run_char(chars.at(index)) {
while index < len && (is_run_char(chars.at(index)) || is_mark_char(chars.at(index)))
{
index += 1;
}
true
} else {
loop {
while index < len
&& is_word_char(chars.at(index))
&& !is_run_char(chars.at(index))
{
index += 1;
}
let joins = index + 1 < len
&& is_apostrophe(chars.at(index))
&& is_word_char(chars.at(index + 1))
&& !is_run_char(chars.at(index + 1));
if !joins {
break;
}
index += 1;
}
true
};
out.push(Segment {
text: chars.slice(start, index),
is_word,
});
}
out
}
fn graphemes<'a>(&self, text: &'a str, _locale: Option<&str>) -> Vec<&'a str> {
let chars = Chars::new(text);
let len = chars.len();
let mut out = Vec::new();
let mut start = 0;
while start < len {
let mut end = start + 1;
if chars.at(start) == '\r' && end < len && chars.at(end) == '\n' {
end += 1;
}
let saw_consonant = is_indic_consonant(chars.at(start));
let mut linked = false;
while end < len {
let c = chars.at(end);
if is_mark_char(c) {
if saw_consonant && is_indic_linker(c) {
linked = true;
}
end += 1;
} else if c == ZWJ {
end = (end + 2).min(len); } else if hangul_joins(jamo_class(chars.at(end - 1)), jamo_class(c)) {
end += 1;
} else if linked && is_indic_consonant(c) {
linked = false;
end += 1;
} else {
break;
}
}
out.push(chars.slice(start, end));
start = end;
}
out
}
}