#![deny(missing_docs)]
#![forbid(unsafe_code)]
use std::collections::HashSet;
const DEFAULT_ABBREVIATIONS: &[&str] = &[
"mr", "mrs", "ms", "dr", "prof", "sr", "jr", "st", "rev", "hon", "gen", "col", "sgt", "capt",
"lt", "gov", "sen", "rep", "pres", "supt", "det", "vs", "etc", "e.g", "i.e", "cf", "al",
"inc", "ltd", "co", "corp", "dept", "univ", "assn", "bros", "no", "vol", "pp", "fig", "figs",
"approx", "est", "min", "max", "misc", "avg", "dist", "ed", "eds", "trans", "jan", "feb",
"mar", "apr", "jun", "jul", "aug", "sep", "sept", "oct", "nov", "dec", "mon", "tue", "wed",
"thu", "fri", "sat", "sun", "a.m", "p.m", "u.s", "u.k", "u.n", "ph.d", "b.a", "m.a", "d.c",
];
#[derive(Clone, Debug)]
pub struct Segmenter {
abbreviations: HashSet<String>,
}
impl Default for Segmenter {
fn default() -> Self {
Self {
abbreviations: DEFAULT_ABBREVIATIONS
.iter()
.map(|s| (*s).to_owned())
.collect(),
}
}
}
impl Segmenter {
#[must_use]
pub fn new<I, S>(abbreviations: I) -> Self
where
I: IntoIterator<Item = S>,
S: Into<String>,
{
Self {
abbreviations: abbreviations.into_iter().map(Into::into).collect(),
}
}
pub fn add_abbreviation(&mut self, abbr: impl Into<String>) -> &mut Self {
self.abbreviations.insert(abbr.into());
self
}
#[must_use]
pub fn spans(&self, text: &str) -> Vec<(usize, usize)> {
let bytes = text.as_bytes();
let mut spans = Vec::new();
let mut start = 0usize;
let mut i = 0usize;
while i < bytes.len() {
let b = bytes[i];
let is_terminator = b == b'.' || b == b'!' || b == b'?';
let is_unicode_ellipsis =
text.is_char_boundary(i) && text[i..].starts_with('\u{2026}');
if !is_terminator && !is_unicode_ellipsis {
i += 1;
continue;
}
let mut end = i;
if is_unicode_ellipsis {
end += '\u{2026}'.len_utf8();
}
while end < bytes.len() && matches!(bytes[end], b'.' | b'!' | b'?') {
end += 1;
}
if bytes[i] == b'.' && end - i == 1 && !self.period_ends_sentence(text, i) {
i = end;
continue;
}
while end < bytes.len() {
let rest = &text[end..];
let close = ['"', '\'', ')', ']', '\u{201d}', '\u{2019}', '\u{00bb}']
.iter()
.find(|&&c| rest.starts_with(c));
match close {
Some(c) => end += c.len_utf8(),
None => break,
}
}
if end >= bytes.len() {
push_span(&mut spans, text, start, end);
start = end;
i = end;
continue;
}
let next_non_ws = text[end..]
.char_indices()
.find(|(_, c)| !c.is_whitespace());
let had_ws = text[end..]
.chars()
.next()
.is_some_and(char::is_whitespace);
match next_non_ws {
Some((off, c))
if had_ws
&& (c.is_uppercase()
|| c.is_ascii_digit()
|| "\"'([\u{201c}\u{2018}\u{00ab}".contains(c)) =>
{
push_span(&mut spans, text, start, end);
start = end + off;
i = start;
}
_ => {
i = end;
}
}
}
if start < bytes.len() {
push_span(&mut spans, text, start, bytes.len());
}
spans
}
pub fn sentences<'a>(&self, text: &'a str) -> impl Iterator<Item = &'a str> + 'a {
self.spans(text)
.into_iter()
.map(move |(s, e)| &text[s..e])
}
fn period_ends_sentence(&self, text: &str, pos: usize) -> bool {
let before = &text[..pos];
let token_start = before
.rfind(|c: char| c.is_whitespace() || "\"'([\u{201c}\u{2018}".contains(c))
.map_or(0, |i| i + text[i..].chars().next().map_or(1, char::len_utf8));
let token = &before[token_start..];
if token.is_empty() {
return false;
}
let after = &text[pos + 1..];
if token.chars().last().is_some_and(|c| c.is_ascii_digit())
&& after.chars().next().is_some_and(|c| c.is_ascii_digit())
{
return false;
}
let mut token_chars = token.chars();
if let (Some(c), None) = (token_chars.next(), token_chars.next()) {
if c.is_uppercase() {
return false;
}
}
let lowered = token.trim_end_matches('.').to_lowercase();
if self.abbreviations.contains(lowered.as_str()) {
return false;
}
true
}
}
fn push_span(spans: &mut Vec<(usize, usize)>, text: &str, start: usize, end: usize) {
let slice = &text[start..end];
let trimmed_start = start + (slice.len() - slice.trim_start().len());
let trimmed_end = end - (slice.len() - slice.trim_end().len());
if trimmed_start < trimmed_end {
spans.push((trimmed_start, trimmed_end));
}
}
#[must_use]
pub fn split(text: &str) -> Vec<&str> {
let seg = Segmenter::default();
let spans = seg.spans(text);
spans.into_iter().map(|(s, e)| &text[s..e]).collect()
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn basic_split() {
assert_eq!(
split("Hello world. How are you? I am fine!"),
["Hello world.", "How are you?", "I am fine!"]
);
}
#[test]
fn abbreviations_do_not_split() {
assert_eq!(
split("Dr. Smith met Mr. Jones. They talked."),
["Dr. Smith met Mr. Jones.", "They talked."]
);
assert_eq!(
split("Fruits, e.g. apples, are good. Yes."),
["Fruits, e.g. apples, are good.", "Yes."]
);
}
#[test]
fn decimals_and_versions() {
assert_eq!(
split("Pi is 3.14 roughly. Use version 1.2.3 now."),
["Pi is 3.14 roughly.", "Use version 1.2.3 now."]
);
}
#[test]
fn initials() {
assert_eq!(
split("J. R. R. Tolkien wrote it. I read it."),
["J. R. R. Tolkien wrote it.", "I read it."]
);
}
#[test]
fn ellipsis() {
assert_eq!(
split("Well... I guess so. Then we left..."),
["Well...", "I guess so.", "Then we left..."]
);
assert_eq!(
split("Then we began... or tried to."),
["Then we began... or tried to."]
);
assert_eq!(split("Wait\u{2026} What?"), ["Wait\u{2026}", "What?"]);
}
#[test]
fn quotes_after_terminator() {
assert_eq!(
split("He said \"Stop.\" She refused."),
["He said \"Stop.\"", "She refused."]
);
}
#[test]
fn multiple_terminators() {
assert_eq!(split("Really?! Yes!!! Ok."), ["Really?!", "Yes!!!", "Ok."]);
}
#[test]
fn spans_index_into_original() {
let text = " One. Two! ";
let seg = Segmenter::default();
let spans = seg.spans(text);
assert_eq!(spans.len(), 2);
assert_eq!(&text[spans[0].0..spans[0].1], "One.");
assert_eq!(&text[spans[1].0..spans[1].1], "Two!");
}
#[test]
fn no_terminator_yields_whole_text() {
assert_eq!(split("no punctuation here"), ["no punctuation here"]);
assert!(split(" ").is_empty());
assert!(split("").is_empty());
}
#[test]
fn custom_abbreviation() {
let mut seg = Segmenter::default();
seg.add_abbreviation("blvd");
let s: Vec<&str> = seg.sentences("Turn at Sunset Blvd. Then stop.").collect();
assert_eq!(s, ["Turn at Sunset Blvd. Then stop."]);
assert_eq!(
split("Turn at Sunset Blvd. Then stop."),
["Turn at Sunset Blvd.", "Then stop."]
);
}
#[test]
fn unicode_text() {
assert_eq!(
split("C'est fini. Très bien!"),
["C'est fini.", "Très bien!"]
);
}
}