sentencize 0.1.0

Rule-based sentence boundary detection: split text into sentences with abbreviation, decimal, ellipsis and quote handling. Zero dependencies.
Documentation
//! # sentencize
//!
//! Rule-based sentence boundary detection for English text — a pure-Rust,
//! zero-dependency answer to the segmentation step usually delegated to
//! Python (nltk / pysbd) in document, RAG and LLM-chunking pipelines.
//!
//! Handles the classic pitfalls of naive `split('.')`:
//! - abbreviations (`Dr.`, `e.g.`, `U.S.`), with a customizable list
//! - decimal numbers (`3.14`) and version-like tokens (`v1.2.3`)
//! - ellipses (`...` / `…`)
//! - closing quotes and brackets after terminators (`He said "Go."`)
//! - initials (`J. R. R. Tolkien`)
//!
//! The splitter returns byte spans into the original text, so it is lossless
//! and allocation-light.
//!
//! # Quick start
//!
//! [`split`] applies the built-in English rules in one call and borrows from
//! the input:
//!
//! ```
//! let sentences = sentencize::split("Hello world. How are you? I am fine!");
//! assert_eq!(sentences, ["Hello world.", "How are you?", "I am fine!"]);
//! ```
//!
//! # Reusing a segmenter
//!
//! [`Segmenter`] holds the abbreviation set, so build it once and reuse it
//! across documents rather than calling [`split`] in a hot loop:
//!
//! ```
//! use sentencize::Segmenter;
//!
//! let seg = Segmenter::default();
//! let text = "Dr. Smith arrived at 3.5 p.m. sharp. He said \"Sit down.\" Then we began... or tried to.";
//! let sentences: Vec<&str> = seg.sentences(text).collect();
//! assert_eq!(sentences, [
//!     "Dr. Smith arrived at 3.5 p.m. sharp.",
//!     "He said \"Sit down.\"",
//!     "Then we began... or tried to.",
//! ]);
//! ```
//!
//! # Byte spans
//!
//! [`Segmenter::spans`] returns whitespace-trimmed, non-overlapping `(start,
//! end)` offsets, which map each sentence back to its exact position in the
//! source document:
//!
//! ```
//! use sentencize::Segmenter;
//!
//! let text = "  One.  Two!  ";
//! let spans = Segmenter::default().spans(text);
//! assert_eq!(&text[spans[0].0..spans[0].1], "One.");
//! assert_eq!(&text[spans[1].0..spans[1].1], "Two!");
//! ```
//!
//! # Custom abbreviations
//!
//! Extend the built-in list with [`Segmenter::add_abbreviation`], or replace it
//! wholesale with [`Segmenter::new`]. Entries are lowercase and written without
//! their trailing period:
//!
//! ```
//! use sentencize::Segmenter;
//!
//! let mut seg = Segmenter::default();
//! seg.add_abbreviation("blvd");
//! let sentences: Vec<&str> = seg.sentences("Turn at Sunset Blvd. Then stop.").collect();
//! assert_eq!(sentences, ["Turn at Sunset Blvd. Then stop."]);
//! ```
//!
//! # Scope
//!
//! These are deterministic heuristics tuned for English prose — no model, no
//! training data, no dependencies. Other Latin-script languages generally work,
//! but the default abbreviation list is English.

#![deny(missing_docs)]
#![forbid(unsafe_code)]

use std::collections::HashSet;

/// Common English abbreviations that do not end a sentence.
const DEFAULT_ABBREVIATIONS: &[&str] = &[
    "mr", "mrs", "ms", "dr", "prof", "sr", "jr", "st", "rev", "hon", "gen", "col", "sgt", "capt",
    "lt", "gov", "sen", "rep", "pres", "supt", "det", "vs", "etc", "e.g", "i.e", "cf", "al",
    "inc", "ltd", "co", "corp", "dept", "univ", "assn", "bros", "no", "vol", "pp", "fig", "figs",
    "approx", "est", "min", "max", "misc", "avg", "dist", "ed", "eds", "trans", "jan", "feb",
    "mar", "apr", "jun", "jul", "aug", "sep", "sept", "oct", "nov", "dec", "mon", "tue", "wed",
    "thu", "fri", "sat", "sun", "a.m", "p.m", "u.s", "u.k", "u.n", "ph.d", "b.a", "m.a", "d.c",
];

/// A configurable sentence segmenter.
///
/// Construct via [`Segmenter::default`] for the built-in English rules, or
/// customize the abbreviation list with [`Segmenter::new`] /
/// [`Segmenter::add_abbreviation`].
#[derive(Clone, Debug)]
pub struct Segmenter {
    /// Lowercased abbreviations, stored without a trailing period.
    abbreviations: HashSet<String>,
}

impl Default for Segmenter {
    fn default() -> Self {
        Self {
            abbreviations: DEFAULT_ABBREVIATIONS
                .iter()
                .map(|s| (*s).to_owned())
                .collect(),
        }
    }
}

impl Segmenter {
    /// Creates a segmenter with a caller-supplied abbreviation list
    /// (lowercase, without trailing periods). Use `Segmenter::default()` for
    /// the built-in English list.
    #[must_use]
    pub fn new<I, S>(abbreviations: I) -> Self
    where
        I: IntoIterator<Item = S>,
        S: Into<String>,
    {
        Self {
            abbreviations: abbreviations.into_iter().map(Into::into).collect(),
        }
    }

    /// Adds an abbreviation (lowercase, without the trailing period).
    pub fn add_abbreviation(&mut self, abbr: impl Into<String>) -> &mut Self {
        self.abbreviations.insert(abbr.into());
        self
    }

    /// Returns the byte spans `(start, end)` of each sentence in `text`.
    /// Spans are trimmed of surrounding whitespace and never overlap.
    #[must_use]
    pub fn spans(&self, text: &str) -> Vec<(usize, usize)> {
        let bytes = text.as_bytes();
        let mut spans = Vec::new();
        let mut start = 0usize;

        let mut i = 0usize;
        while i < bytes.len() {
            let b = bytes[i];
            let is_terminator = b == b'.' || b == b'!' || b == b'?';
            // U+2026 HORIZONTAL ELLIPSIS (e2 80 a6)
            let is_unicode_ellipsis =
                text.is_char_boundary(i) && text[i..].starts_with('\u{2026}');

            if !is_terminator && !is_unicode_ellipsis {
                i += 1;
                continue;
            }

            // Consume runs of terminators: "...", "?!", "!!!".
            let mut end = i;
            if is_unicode_ellipsis {
                end += '\u{2026}'.len_utf8();
            }
            while end < bytes.len() && matches!(bytes[end], b'.' | b'!' | b'?') {
                end += 1;
            }

            if bytes[i] == b'.' && end - i == 1 && !self.period_ends_sentence(text, i) {
                i = end;
                continue;
            }

            // Include closing quotes/brackets that follow the terminator.
            while end < bytes.len() {
                let rest = &text[end..];
                let close = ['"', '\'', ')', ']', '\u{201d}', '\u{2019}', '\u{00bb}']
                    .iter()
                    .find(|&&c| rest.starts_with(c));
                match close {
                    Some(c) => end += c.len_utf8(),
                    None => break,
                }
            }

            // A boundary requires end-of-text, or whitespace followed by a
            // plausible sentence opener (uppercase, digit, quote, bracket).
            if end >= bytes.len() {
                push_span(&mut spans, text, start, end);
                start = end;
                i = end;
                continue;
            }
            let next_non_ws = text[end..]
                .char_indices()
                .find(|(_, c)| !c.is_whitespace());
            let had_ws = text[end..]
                .chars()
                .next()
                .is_some_and(char::is_whitespace);
            match next_non_ws {
                Some((off, c))
                    if had_ws
                        && (c.is_uppercase()
                            || c.is_ascii_digit()
                            || "\"'([\u{201c}\u{2018}\u{00ab}".contains(c)) =>
                {
                    // Ellipsis followed by a lowercase word was already
                    // excluded by the uppercase check above.
                    push_span(&mut spans, text, start, end);
                    start = end + off;
                    i = start;
                }
                _ => {
                    i = end;
                }
            }
        }
        if start < bytes.len() {
            push_span(&mut spans, text, start, bytes.len());
        }
        spans
    }

    /// Iterates over sentences as `&str` slices of `text`.
    pub fn sentences<'a>(&self, text: &'a str) -> impl Iterator<Item = &'a str> + 'a {
        self.spans(text)
            .into_iter()
            .map(move |(s, e)| &text[s..e])
    }

    /// Decides whether the single period at byte `pos` can end a sentence.
    fn period_ends_sentence(&self, text: &str, pos: usize) -> bool {
        let before = &text[..pos];

        // Token immediately preceding the period.
        let token_start = before
            .rfind(|c: char| c.is_whitespace() || "\"'([\u{201c}\u{2018}".contains(c))
            .map_or(0, |i| i + text[i..].chars().next().map_or(1, char::len_utf8));
        let token = &before[token_start..];

        if token.is_empty() {
            return false;
        }

        // Decimal number or dotted version: "3.14", "1.2.3".
        let after = &text[pos + 1..];
        if token.chars().last().is_some_and(|c| c.is_ascii_digit())
            && after.chars().next().is_some_and(|c| c.is_ascii_digit())
        {
            return false;
        }

        // Single initial: "J." in "J. R. R. Tolkien".
        let mut token_chars = token.chars();
        if let (Some(c), None) = (token_chars.next(), token_chars.next()) {
            if c.is_uppercase() {
                return false;
            }
        }

        // Known abbreviation (case-insensitive, dot-internal ones included).
        let lowered = token.trim_end_matches('.').to_lowercase();
        if self.abbreviations.contains(lowered.as_str()) {
            return false;
        }

        true
    }
}

fn push_span(spans: &mut Vec<(usize, usize)>, text: &str, start: usize, end: usize) {
    let slice = &text[start..end];
    let trimmed_start = start + (slice.len() - slice.trim_start().len());
    let trimmed_end = end - (slice.len() - slice.trim_end().len());
    if trimmed_start < trimmed_end {
        spans.push((trimmed_start, trimmed_end));
    }
}

/// Convenience: split with the default English segmenter.
#[must_use]
pub fn split(text: &str) -> Vec<&str> {
    let seg = Segmenter::default();
    let spans = seg.spans(text);
    spans.into_iter().map(|(s, e)| &text[s..e]).collect()
}

#[cfg(test)]
mod tests {
    use super::*;

    #[test]
    fn basic_split() {
        assert_eq!(
            split("Hello world. How are you? I am fine!"),
            ["Hello world.", "How are you?", "I am fine!"]
        );
    }

    #[test]
    fn abbreviations_do_not_split() {
        assert_eq!(
            split("Dr. Smith met Mr. Jones. They talked."),
            ["Dr. Smith met Mr. Jones.", "They talked."]
        );
        assert_eq!(
            split("Fruits, e.g. apples, are good. Yes."),
            ["Fruits, e.g. apples, are good.", "Yes."]
        );
    }

    #[test]
    fn decimals_and_versions() {
        assert_eq!(
            split("Pi is 3.14 roughly. Use version 1.2.3 now."),
            ["Pi is 3.14 roughly.", "Use version 1.2.3 now."]
        );
    }

    #[test]
    fn initials() {
        assert_eq!(
            split("J. R. R. Tolkien wrote it. I read it."),
            ["J. R. R. Tolkien wrote it.", "I read it."]
        );
    }

    #[test]
    fn ellipsis() {
        // Ellipsis followed by an uppercase opener is a boundary (matches
        // pysbd's golden rules); followed by lowercase it is not.
        assert_eq!(
            split("Well... I guess so. Then we left..."),
            ["Well...", "I guess so.", "Then we left..."]
        );
        assert_eq!(
            split("Then we began... or tried to."),
            ["Then we began... or tried to."]
        );
        assert_eq!(split("Wait\u{2026} What?"), ["Wait\u{2026}", "What?"]);
    }

    #[test]
    fn quotes_after_terminator() {
        assert_eq!(
            split("He said \"Stop.\" She refused."),
            ["He said \"Stop.\"", "She refused."]
        );
    }

    #[test]
    fn multiple_terminators() {
        assert_eq!(split("Really?! Yes!!! Ok."), ["Really?!", "Yes!!!", "Ok."]);
    }

    #[test]
    fn spans_index_into_original() {
        let text = "  One.  Two!  ";
        let seg = Segmenter::default();
        let spans = seg.spans(text);
        assert_eq!(spans.len(), 2);
        assert_eq!(&text[spans[0].0..spans[0].1], "One.");
        assert_eq!(&text[spans[1].0..spans[1].1], "Two!");
    }

    #[test]
    fn no_terminator_yields_whole_text() {
        assert_eq!(split("no punctuation here"), ["no punctuation here"]);
        assert!(split("   ").is_empty());
        assert!(split("").is_empty());
    }

    #[test]
    fn custom_abbreviation() {
        let mut seg = Segmenter::default();
        seg.add_abbreviation("blvd");
        let s: Vec<&str> = seg.sentences("Turn at Sunset Blvd. Then stop.").collect();
        assert_eq!(s, ["Turn at Sunset Blvd. Then stop."]);
        // Default segmenter (without it) splits:
        assert_eq!(
            split("Turn at Sunset Blvd. Then stop."),
            ["Turn at Sunset Blvd.", "Then stop."]
        );
    }

    #[test]
    fn unicode_text() {
        assert_eq!(
            split("C'est fini. Très bien!"),
            ["C'est fini.", "Très bien!"]
        );
    }
}