use regex_automata::{dfa::Automaton, Anchored, Input};
use crate::{
ext_slice::ByteSlice,
unicode::fsm::{
grapheme_break_fwd::GRAPHEME_BREAK_FWD,
grapheme_break_rev::GRAPHEME_BREAK_REV,
regional_indicator_rev::REGIONAL_INDICATOR_REV,
},
utf8,
};
#[derive(Clone, Debug)]
pub struct Graphemes<'a> {
bs: &'a [u8],
}
impl<'a> Graphemes<'a> {
pub(crate) fn new(bs: &'a [u8]) -> Graphemes<'a> {
Graphemes { bs }
}
#[inline]
pub fn as_bytes(&self) -> &'a [u8] {
self.bs
}
}
impl<'a> Iterator for Graphemes<'a> {
type Item = &'a str;
#[inline]
fn next(&mut self) -> Option<&'a str> {
let (grapheme, size) = decode_grapheme(self.bs);
if size == 0 {
return None;
}
self.bs = &self.bs[size..];
Some(grapheme)
}
}
impl<'a> DoubleEndedIterator for Graphemes<'a> {
#[inline]
fn next_back(&mut self) -> Option<&'a str> {
let (grapheme, size) = decode_last_grapheme(self.bs);
if size == 0 {
return None;
}
self.bs = &self.bs[..self.bs.len() - size];
Some(grapheme)
}
}
#[derive(Clone, Debug)]
pub struct GraphemeIndices<'a> {
bs: &'a [u8],
forward_index: usize,
reverse_index: usize,
}
impl<'a> GraphemeIndices<'a> {
pub(crate) fn new(bs: &'a [u8]) -> GraphemeIndices<'a> {
GraphemeIndices { bs, forward_index: 0, reverse_index: bs.len() }
}
#[inline]
pub fn as_bytes(&self) -> &'a [u8] {
self.bs
}
}
impl<'a> Iterator for GraphemeIndices<'a> {
type Item = (usize, usize, &'a str);
#[inline]
fn next(&mut self) -> Option<(usize, usize, &'a str)> {
let index = self.forward_index;
let (grapheme, size) = decode_grapheme(self.bs);
if size == 0 {
return None;
}
self.bs = &self.bs[size..];
self.forward_index += size;
Some((index, index + size, grapheme))
}
}
impl<'a> DoubleEndedIterator for GraphemeIndices<'a> {
#[inline]
fn next_back(&mut self) -> Option<(usize, usize, &'a str)> {
let (grapheme, size) = decode_last_grapheme(self.bs);
if size == 0 {
return None;
}
self.bs = &self.bs[..self.bs.len() - size];
self.reverse_index -= size;
Some((self.reverse_index, self.reverse_index + size, grapheme))
}
}
pub fn decode_grapheme(bs: &[u8]) -> (&str, usize) {
if bs.is_empty() {
("", 0)
} else if bs.len() >= 2
&& bs[0].is_ascii()
&& bs[1].is_ascii()
&& !bs[0].is_ascii_whitespace()
{
let grapheme = unsafe { bs[..1].to_str_unchecked() };
(grapheme, 1)
} else if let Some(hm) = {
let input = Input::new(bs).anchored(Anchored::Yes);
GRAPHEME_BREAK_FWD.try_search_fwd(&input).unwrap()
} {
let grapheme = unsafe { bs[..hm.offset()].to_str_unchecked() };
(grapheme, grapheme.len())
} else {
const INVALID: &'static str = "\u{FFFD}";
let (_, size) = utf8::decode_lossy(bs);
(INVALID, size)
}
}
fn decode_last_grapheme(bs: &[u8]) -> (&str, usize) {
if bs.is_empty() {
("", 0)
} else if let Some(hm) = {
let input = Input::new(bs).anchored(Anchored::Yes);
GRAPHEME_BREAK_REV.try_search_rev(&input).unwrap()
} {
let start = adjust_rev_for_regional_indicator(bs, hm.offset());
let grapheme = unsafe { bs[start..].to_str_unchecked() };
(grapheme, grapheme.len())
} else {
const INVALID: &'static str = "\u{FFFD}";
let (_, size) = utf8::decode_last_lossy(bs);
(INVALID, size)
}
}
fn adjust_rev_for_regional_indicator(mut bs: &[u8], i: usize) -> usize {
if bs.len() - i != 8 {
return i;
}
let mut count = 0;
while let Some(hm) = {
let input = Input::new(bs).anchored(Anchored::Yes);
REGIONAL_INDICATOR_REV.try_search_rev(&input).unwrap()
} {
bs = &bs[..hm.offset()];
count += 1;
}
if count % 2 == 0 {
i
} else {
i + 4
}
}
#[cfg(all(test, feature = "std"))]
mod tests {
use alloc::{
string::{String, ToString},
vec,
vec::Vec,
};
#[cfg(not(miri))]
use ucd_parse::GraphemeClusterBreakTest;
use crate::tests::LOSSY_TESTS;
use super::*;
#[test]
#[cfg(not(miri))]
fn forward_ucd() {
for (i, test) in ucdtests().into_iter().enumerate() {
let given = test.grapheme_clusters.concat();
let got: Vec<String> = Graphemes::new(given.as_bytes())
.map(|cluster| cluster.to_string())
.collect();
assert_eq!(
test.grapheme_clusters,
got,
"\ngrapheme forward break test {} failed:\n\
given: {:?}\n\
expected: {:?}\n\
got: {:?}\n",
i,
uniescape(&given),
uniescape_vec(&test.grapheme_clusters),
uniescape_vec(&got),
);
}
}
#[test]
#[cfg(not(miri))]
fn reverse_ucd() {
for (i, test) in ucdtests().into_iter().enumerate() {
let given = test.grapheme_clusters.concat();
let mut got: Vec<String> = Graphemes::new(given.as_bytes())
.rev()
.map(|cluster| cluster.to_string())
.collect();
got.reverse();
assert_eq!(
test.grapheme_clusters,
got,
"\n\ngrapheme reverse break test {} failed:\n\
given: {:?}\n\
expected: {:?}\n\
got: {:?}\n",
i,
uniescape(&given),
uniescape_vec(&test.grapheme_clusters),
uniescape_vec(&got),
);
}
}
#[test]
fn forward_lossy() {
for &(expected, input) in LOSSY_TESTS {
let got = Graphemes::new(input.as_bytes()).collect::<String>();
assert_eq!(expected, got);
}
}
#[test]
fn reverse_lossy() {
for &(expected, input) in LOSSY_TESTS {
let expected: String = expected.chars().rev().collect();
let got =
Graphemes::new(input.as_bytes()).rev().collect::<String>();
assert_eq!(expected, got);
}
}
#[cfg(not(miri))]
fn uniescape(s: &str) -> String {
s.chars().flat_map(|c| c.escape_unicode()).collect::<String>()
}
#[cfg(not(miri))]
fn uniescape_vec(strs: &[String]) -> Vec<String> {
strs.iter().map(|s| uniescape(s)).collect()
}
#[cfg(not(miri))]
fn ucdtests() -> Vec<GraphemeClusterBreakTest> {
const TESTDATA: &'static str =
include_str!("data/GraphemeBreakTest.txt");
let mut tests = vec![];
for mut line in TESTDATA.lines() {
line = line.trim();
if line.starts_with("#") || line.contains("surrogate") {
continue;
}
tests.push(line.parse().unwrap());
}
tests
}
}