use super::grapheme_tables::GRAPHEME_PROPERTIES;
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
enum Gcb {
Other,
CR,
LF,
Control,
Extend,
Zwj,
RegionalIndicator,
Prepend,
SpacingMark,
L,
V,
T,
Lv,
Lvt,
}
const GCB: [Gcb; 14] = [
Gcb::Other,
Gcb::CR,
Gcb::LF,
Gcb::Control,
Gcb::Extend,
Gcb::Zwj,
Gcb::RegionalIndicator,
Gcb::Prepend,
Gcb::SpacingMark,
Gcb::L,
Gcb::V,
Gcb::T,
Gcb::Lv,
Gcb::Lvt,
];
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
enum InCB {
None,
Consonant,
Extend,
Linker,
}
const INCB: [InCB; 4] = [InCB::None, InCB::Consonant, InCB::Extend, InCB::Linker];
#[derive(Clone, Copy, Debug)]
struct Props {
gcb: Gcb,
extended_pictographic: bool,
incb: InCB,
}
fn props(c: char) -> Props {
let c = u32::from(c);
let bits = match GRAPHEME_PROPERTIES.binary_search_by(|&(lo, hi, _)| {
if hi < c {
std::cmp::Ordering::Less
} else if lo > c {
std::cmp::Ordering::Greater
} else {
std::cmp::Ordering::Equal
}
}) {
Ok(i) => GRAPHEME_PROPERTIES[i].2,
Err(_) => 0,
};
Props {
gcb: GCB[usize::from(bits & 0xf)],
extended_pictographic: bits & 0x10 != 0,
incb: INCB[usize::from((bits >> 5) & 0x3)],
}
}
#[derive(Clone, Copy, Debug, Default)]
struct Context {
emoji: Option<bool>,
regional_indicators: usize,
conjunct: Option<bool>,
}
impl Context {
fn push(self, c: Props) -> Context {
Context {
emoji: if c.extended_pictographic {
Some(false)
} else {
match (self.emoji, c.gcb) {
(Some(false), Gcb::Extend) => Some(false),
(Some(false), Gcb::Zwj) => Some(true),
_ => None,
}
},
regional_indicators: if c.gcb == Gcb::RegionalIndicator {
self.regional_indicators + 1
} else {
0
},
conjunct: match (c.incb, self.conjunct) {
(InCB::Consonant, _) => Some(false),
(InCB::Linker, Some(_)) => Some(true),
(InCB::Extend, Some(linked)) => Some(linked),
_ => None,
},
}
}
}
fn is_boundary(prev: Props, next: Props, context: Context) -> bool {
use Gcb::*;
match (prev.gcb, next.gcb) {
(CR, LF) => false,
(Control | CR | LF, _) | (_, Control | CR | LF) => true,
(L, L | V | Lv | Lvt) | (Lv | V, V | T) | (Lvt | T, T) => false,
(_, Extend | Zwj | SpacingMark) | (Prepend, _) => false,
_ if next.incb == InCB::Consonant && context.conjunct == Some(true) => false,
(Zwj, _) if next.extended_pictographic && context.emoji == Some(true) => false,
(RegionalIndicator, RegionalIndicator) => context.regional_indicators.is_multiple_of(2),
_ => true,
}
}
pub(crate) fn graphemes(text: &str) -> Graphemes<'_> {
Graphemes { rest: text }
}
pub(crate) struct Graphemes<'a> {
rest: &'a str,
}
impl<'a> Iterator for Graphemes<'a> {
type Item = &'a str;
fn next(&mut self) -> Option<&'a str> {
let mut chars = self.rest.char_indices();
let (_, first) = chars.next()?;
let mut prev = props(first);
let mut context = Context::default().push(prev);
let mut end = self.rest.len();
for (i, c) in chars {
let next = props(c);
if is_boundary(prev, next, context) {
end = i;
break;
}
context = context.push(next);
prev = next;
}
let (segment, rest) = self.rest.split_at(end);
self.rest = rest;
Some(segment)
}
}
#[cfg(test)]
mod tests {
use super::super::grapheme_tables::GRAPHEME_BREAK_TEST;
use super::*;
#[test]
fn grapheme_break_test() {
let mut cases = 0;
for line in GRAPHEME_BREAK_TEST.lines() {
let mut text = String::new();
let mut want = Vec::new();
for token in line.split(' ') {
match token {
"÷" => want.push(text.len()),
"×" => {}
hex => {
text.push(char::from_u32(u32::from_str_radix(hex, 16).unwrap()).unwrap())
}
}
}
let mut got = vec![0];
for segment in graphemes(&text) {
got.push(got.last().unwrap() + segment.len());
}
assert_eq!(got, want, "{line}");
cases += 1;
}
assert_eq!(cases, 766);
}
#[test]
fn segments() {
fn split(s: &str) -> Vec<&str> {
graphemes(s).collect()
}
assert_eq!(split(""), Vec::<&str>::new());
assert_eq!(split("a\r\nb"), ["a", "\r\n", "b"]);
assert_eq!(
split("\u{1F1FA}\u{1F1F8}\u{1F1EB}"),
["\u{1F1FA}\u{1F1F8}", "\u{1F1EB}"]
);
assert_eq!(
split("\u{0915}\u{094D}\u{0937}\u{093F}x"),
["\u{0915}\u{094D}\u{0937}\u{093F}", "x"]
);
}
}