use regex_automata::{dfa::Automaton, Anchored, Input};
use crate::{
ext_slice::ByteSlice,
unicode::fsm::{
simple_word_fwd::SIMPLE_WORD_FWD, word_break_fwd::WORD_BREAK_FWD,
},
utf8,
};
#[derive(Clone, Debug)]
pub struct Words<'a>(WordsWithBreaks<'a>);
impl<'a> Words<'a> {
pub(crate) fn new(bs: &'a [u8]) -> Words<'a> {
Words(WordsWithBreaks::new(bs))
}
#[inline]
pub fn as_bytes(&self) -> &'a [u8] {
self.0.as_bytes()
}
}
impl<'a> Iterator for Words<'a> {
type Item = &'a str;
#[inline]
fn next(&mut self) -> Option<&'a str> {
while let Some(word) = self.0.next() {
let input =
Input::new(word).anchored(Anchored::Yes).earliest(true);
if SIMPLE_WORD_FWD.try_search_fwd(&input).unwrap().is_some() {
return Some(word);
}
}
None
}
}
#[derive(Clone, Debug)]
pub struct WordIndices<'a>(WordsWithBreakIndices<'a>);
impl<'a> WordIndices<'a> {
pub(crate) fn new(bs: &'a [u8]) -> WordIndices<'a> {
WordIndices(WordsWithBreakIndices::new(bs))
}
#[inline]
pub fn as_bytes(&self) -> &'a [u8] {
self.0.as_bytes()
}
}
impl<'a> Iterator for WordIndices<'a> {
type Item = (usize, usize, &'a str);
#[inline]
fn next(&mut self) -> Option<(usize, usize, &'a str)> {
while let Some((start, end, word)) = self.0.next() {
let input =
Input::new(word).anchored(Anchored::Yes).earliest(true);
if SIMPLE_WORD_FWD.try_search_fwd(&input).unwrap().is_some() {
return Some((start, end, word));
}
}
None
}
}
#[derive(Clone, Debug)]
pub struct WordsWithBreaks<'a> {
bs: &'a [u8],
}
impl<'a> WordsWithBreaks<'a> {
pub(crate) fn new(bs: &'a [u8]) -> WordsWithBreaks<'a> {
WordsWithBreaks { bs }
}
#[inline]
pub fn as_bytes(&self) -> &'a [u8] {
self.bs
}
}
impl<'a> Iterator for WordsWithBreaks<'a> {
type Item = &'a str;
#[inline]
fn next(&mut self) -> Option<&'a str> {
let (word, size) = decode_word(self.bs);
if size == 0 {
return None;
}
self.bs = &self.bs[size..];
Some(word)
}
}
#[derive(Clone, Debug)]
pub struct WordsWithBreakIndices<'a> {
bs: &'a [u8],
forward_index: usize,
}
impl<'a> WordsWithBreakIndices<'a> {
pub(crate) fn new(bs: &'a [u8]) -> WordsWithBreakIndices<'a> {
WordsWithBreakIndices { bs, forward_index: 0 }
}
#[inline]
pub fn as_bytes(&self) -> &'a [u8] {
self.bs
}
}
impl<'a> Iterator for WordsWithBreakIndices<'a> {
type Item = (usize, usize, &'a str);
#[inline]
fn next(&mut self) -> Option<(usize, usize, &'a str)> {
let index = self.forward_index;
let (word, size) = decode_word(self.bs);
if size == 0 {
return None;
}
self.bs = &self.bs[size..];
self.forward_index += size;
Some((index, index + size, word))
}
}
fn decode_word(bs: &[u8]) -> (&str, usize) {
if bs.is_empty() {
("", 0)
} else if let Some(hm) = {
let input = Input::new(bs).anchored(Anchored::Yes);
WORD_BREAK_FWD.try_search_fwd(&input).unwrap()
} {
let word = unsafe { bs[..hm.offset()].to_str_unchecked() };
(word, word.len())
} else {
const INVALID: &'static str = "\u{FFFD}";
let (_, size) = utf8::decode_lossy(bs);
(INVALID, size)
}
}
#[cfg(all(test, feature = "std"))]
mod tests {
use alloc::{vec, vec::Vec};
#[cfg(not(miri))]
use ucd_parse::WordBreakTest;
use crate::ext_slice::ByteSlice;
#[test]
#[cfg(not(miri))]
fn forward_ucd() {
for (i, test) in ucdtests().into_iter().enumerate() {
let given = test.words.concat();
let got = words(given.as_bytes());
assert_eq!(
test.words,
got,
"\n\nword forward break test {} failed:\n\
given: {:?}\n\
expected: {:?}\n\
got: {:?}\n",
i,
given,
strs_to_bstrs(&test.words),
strs_to_bstrs(&got),
);
}
}
#[test]
fn forward_additional() {
assert_eq!(vec!["a", ".", " ", "Y"], words(b"a. Y"));
assert_eq!(vec!["r", ".", " ", "Yo"], words(b"r. Yo"));
assert_eq!(
vec!["whatsoever", ".", " ", "You", " ", "may"],
words(b"whatsoever. You may")
);
assert_eq!(
vec!["21stcentury'syesterday"],
words(b"21stcentury'syesterday")
);
assert_eq!(vec!["Bonta_", "'", "s"], words(b"Bonta_'s"));
assert_eq!(vec!["_vhat's"], words(b"_vhat's"));
assert_eq!(vec!["__on'anima"], words(b"__on'anima"));
assert_eq!(vec!["123_", "'", "4"], words(b"123_'4"));
assert_eq!(vec!["_123'4"], words(b"_123'4"));
assert_eq!(vec!["__12'345"], words(b"__12'345"));
assert_eq!(
vec!["tomorrowat4", ":", "00", ","],
words(b"tomorrowat4:00,")
);
assert_eq!(vec!["RS1", "'", "s"], words(b"RS1's"));
assert_eq!(vec!["X38"], words(b"X38"));
assert_eq!(vec!["4abc", ":", "00", ","], words(b"4abc:00,"));
assert_eq!(vec!["12S", "'", "1"], words(b"12S'1"));
assert_eq!(vec!["1XY"], words(b"1XY"));
assert_eq!(vec!["\u{FEFF}", "Ты"], words("\u{FEFF}Ты".as_bytes()));
assert_eq!(
vec!["\u{10570}\u{10597}"],
words("\u{10570}\u{10597}".as_bytes())
);
}
fn words(bytes: &[u8]) -> Vec<&str> {
bytes.words_with_breaks().collect()
}
#[cfg(not(miri))]
fn strs_to_bstrs<S: AsRef<str>>(strs: &[S]) -> Vec<&[u8]> {
strs.iter().map(|s| s.as_ref().as_bytes()).collect()
}
#[cfg(not(miri))]
fn ucdtests() -> Vec<WordBreakTest> {
const TESTDATA: &'static str = include_str!("data/WordBreakTest.txt");
let mut tests = vec![];
for mut line in TESTDATA.lines() {
line = line.trim();
if line.starts_with("#") || line.contains("surrogate") {
continue;
}
tests.push(line.parse().unwrap());
}
tests
}
}