use regex_automata::{dfa::Automaton, Anchored, Input};
use crate::{
ext_slice::ByteSlice,
unicode::fsm::sentence_break_fwd::SENTENCE_BREAK_FWD, utf8,
};
#[derive(Clone, Debug)]
pub struct Sentences<'a> {
bs: &'a [u8],
}
impl<'a> Sentences<'a> {
pub(crate) fn new(bs: &'a [u8]) -> Sentences<'a> {
Sentences { bs }
}
#[inline]
pub fn as_bytes(&self) -> &'a [u8] {
self.bs
}
}
impl<'a> Iterator for Sentences<'a> {
type Item = &'a str;
#[inline]
fn next(&mut self) -> Option<&'a str> {
let (sentence, size) = decode_sentence(self.bs);
if size == 0 {
return None;
}
self.bs = &self.bs[size..];
Some(sentence)
}
}
#[derive(Clone, Debug)]
pub struct SentenceIndices<'a> {
bs: &'a [u8],
forward_index: usize,
}
impl<'a> SentenceIndices<'a> {
pub(crate) fn new(bs: &'a [u8]) -> SentenceIndices<'a> {
SentenceIndices { bs, forward_index: 0 }
}
#[inline]
pub fn as_bytes(&self) -> &'a [u8] {
self.bs
}
}
impl<'a> Iterator for SentenceIndices<'a> {
type Item = (usize, usize, &'a str);
#[inline]
fn next(&mut self) -> Option<(usize, usize, &'a str)> {
let index = self.forward_index;
let (word, size) = decode_sentence(self.bs);
if size == 0 {
return None;
}
self.bs = &self.bs[size..];
self.forward_index += size;
Some((index, index + size, word))
}
}
fn decode_sentence(bs: &[u8]) -> (&str, usize) {
if bs.is_empty() {
("", 0)
} else if let Some(hm) = {
let input = Input::new(bs).anchored(Anchored::Yes);
SENTENCE_BREAK_FWD.try_search_fwd(&input).unwrap()
} {
let sentence = unsafe { bs[..hm.offset()].to_str_unchecked() };
(sentence, sentence.len())
} else {
const INVALID: &'static str = "\u{FFFD}";
let (_, size) = utf8::decode_lossy(bs);
(INVALID, size)
}
}
#[cfg(all(test, feature = "std"))]
mod tests {
use alloc::{vec, vec::Vec};
#[cfg(not(miri))]
use ucd_parse::SentenceBreakTest;
use crate::ext_slice::ByteSlice;
#[test]
#[cfg(not(miri))]
fn forward_ucd() {
for (i, test) in ucdtests().into_iter().enumerate() {
let given = test.sentences.concat();
let got = sentences(given.as_bytes());
assert_eq!(
test.sentences,
got,
"\n\nsentence forward break test {} failed:\n\
given: {:?}\n\
expected: {:?}\n\
got: {:?}\n",
i,
given,
strs_to_bstrs(&test.sentences),
strs_to_bstrs(&got),
);
}
}
#[test]
fn forward_additional() {
assert_eq!(vec!["a.. ", "A"], sentences(b"a.. A"));
assert_eq!(vec!["a.. a"], sentences(b"a.. a"));
assert_eq!(vec!["a... ", "A"], sentences(b"a... A"));
assert_eq!(vec!["a... a"], sentences(b"a... a"));
assert_eq!(vec!["a...,..., a"], sentences(b"a...,..., a"));
}
fn sentences(bytes: &[u8]) -> Vec<&str> {
bytes.sentences().collect()
}
#[cfg(not(miri))]
fn strs_to_bstrs<S: AsRef<str>>(strs: &[S]) -> Vec<&[u8]> {
strs.iter().map(|s| s.as_ref().as_bytes()).collect()
}
#[cfg(not(miri))]
fn ucdtests() -> Vec<SentenceBreakTest> {
const TESTDATA: &'static str =
include_str!("data/SentenceBreakTest.txt");
let mut tests = vec![];
for mut line in TESTDATA.lines() {
line = line.trim();
if line.starts_with("#") || line.contains("surrogate") {
continue;
}
tests.push(line.parse().unwrap());
}
tests
}
}