sentencex 0.1.30

use regex::Regex;
use rustc_hash::FxHashSet;
use std::sync::LazyLock;

use crate::SentenceBoundary;
use crate::constants::EMAIL_REGEX;
use crate::constants::EXCLAMATION_WORDS;
use crate::constants::GLOBAL_SENTENCE_TERMINATORS;
use crate::constants::PARENS_REGEX;
use crate::constants::QuotePair;
use crate::constants::is_sentence_terminator;

use super::quotes::{
    OrphanCloserPositions, QuoteMispairing, collect_quote_ranges, extend_past_orphan_closer,
    inner_terminator_boundary, is_symmetric_quote_closer, is_symmetric_quote_mispairing,
    peel_leading_symmetric_quote, tag_quote_mispairing,
};

use super::trailing_markers::{MarkerTable, classify_trailing_marker, marker_bypasses_suppression};

static DEFAULT_SENTENCE_BREAK_REGEX: LazyLock<Regex> = LazyLock::new(|| {
    // Branch 1 (`\.(?:[ \t]+\.){2,}`) coalesces three-or-more spaced dots
    // (`. . .`, `. . . .`) into one match. Two-dot `. .` is excluded so a
    // period followed by a leading ellipsis (`raak. ...en`) is not eaten as a
    // single run. `[ \t]` (not `\s`) keeps newlines intact for paragraph splits.
    //
    // Branch 2 (`[!?…](?:[ \t]+[!?…])+`) coalesces two-or-more spaced runs,
    // mixed or homogeneous (`! !`, `? ? ?`, `! ?`, `… !`). `+` rather than
    // `{2,}` is safe here — there's no leading-ellipsis equivalent for `!`/`?`.
    // Both branches must precede the class for leftmost-first alternation.
    let pattern = format!(
        r"\.(?:[ \t]+\.){{2,}}|[!?…](?:[ \t]+[!?…])+|[{}]+",
        GLOBAL_SENTENCE_TERMINATORS.iter().collect::<String>()
    );

    Regex::new(&pattern).unwrap()
});

// Matches a lowercase letter or digit, optionally preceded by non-word characters
// (e.g. a space or punctuation). Used by languages that extend the base continuation
// check with their own month lists.
static CONTINUE_AFTER_NONWORD_REGEX: LazyLock<Regex> =
    LazyLock::new(|| Regex::new(r"^\W*[0-9a-z]").unwrap());

// Ellipsis continuation: treat a multi-char terminator run as mid-sentence when the follow-up is
// whitespace + a lowercase letter or digit (`... no`, `. . . what`). Languages with a
// capitalized word that is ambiguous with a sentence start (English standalone `I`) extend
// this via the `is_ellipsis_continuation` trait method.
pub(crate) static ELLIPSIS_CONTINUE_REGEX: LazyLock<Regex> =
    LazyLock::new(|| Regex::new(r"^\s+[0-9a-z]").unwrap());

/// Equivalent to regex `^[0-9a-z]`. Direct byte check is faster for simple pattern.
fn starts_with_ascii_lowercase_or_digit(s: &str) -> bool {
    s.as_bytes()
        .first()
        .is_some_and(|b| matches!(b, b'a'..=b'z' | b'0'..=b'9'))
}

/// If the bytes at `at` start a `\n[\r]*\n` paragraph separator, return
/// the byte range of the entire separator, else `None`.
fn paragraph_break_at(bytes: &[u8], at: usize) -> Option<(usize, usize)> {
    if bytes.get(at) != Some(&b'\n') {
        return None;
    }

    let mut end = at + 1;
    while bytes.get(end) == Some(&b'\r') {
        end += 1;
    }

    (bytes.get(end) == Some(&b'\n')).then_some((at, end + 1))
}

/// Replaces the previous paragraph split regex `\n[\r]*\n` with memchr scan for performance.
/// Iterate over `\n[\r]*\n` paragraph separators in `text` as `(start, end)` byte ranges.
pub(crate) fn paragraph_breaks(text: &str) -> impl Iterator<Item = (usize, usize)> + '_ {
    let bytes = text.as_bytes();
    let mut cursor = 0;

    std::iter::from_fn(move || {
        loop {
            let newline = cursor + memchr::memchr(b'\n', &bytes[cursor..])?;
            if let Some((start, end)) = paragraph_break_at(bytes, newline) {
                cursor = end;
                return Some((start, end));
            }

            // Lone `\n` — advance past it and keep scanning.
            cursor = newline + 1;
        }
    })
}

/// True when a `.` sits inside a code-like numbered token rather than ending a
/// sentence. A digit immediately before, and an alphanumeric token with a digit
/// after with no space, e.g. the chess move `7.Bg5`.
fn is_code_like_numbered_token(head: &str, next_word_approx: &str) -> bool {
    head.bytes().next_back().is_some_and(|b| b.is_ascii_digit())
        && next_word_approx.starts_with(|c: char| c.is_alphabetic())
        && next_word_approx.bytes().any(|b| b.is_ascii_digit())
}

fn is_single_ascii_upper(s: &str) -> bool {
    s.len() == 1 && s.as_bytes()[0].is_ascii_uppercase()
}

pub(crate) fn abbreviation_set_contains(set: &FxHashSet<String>, word: &str) -> bool {
    if word.bytes().all(|b| b < 128 && !b.is_ascii_uppercase()) {
        return set.contains(word);
    }

    // For small words a stack buffer avoids a string allocation
    if word.is_ascii() && word.len() <= 32 {
        let mut buf = [0u8; 32];
        for (i, b) in word.bytes().enumerate() {
            buf[i] = b.to_ascii_lowercase();
        }

        // SAFETY: ASCII is valid UTF-8
        let lower = unsafe { std::str::from_utf8_unchecked(&buf[..word.len()]) };
        return set.contains(lower);
    }

    set.contains(word.to_lowercase().as_str())
}

/// True iff `s` (after leading whitespace) begins with a name-initial token:
/// a single uppercase ASCII letter, a `.`, then end-of-string or whitespace.
/// `J. R. Tolkien` triggers. `Jones`, `J.R.R.`, and `A.B` do not.
fn starts_with_initial(s: &str) -> bool {
    let mut chars = s.trim_start().chars();
    let Some(first) = chars.next() else {
        return false;
    };

    first.is_ascii_uppercase()
        && chars.next() == Some('.')
        && chars.next().is_none_or(char::is_whitespace)
}

/// Push `boundary` only if it advances past the last recorded position.
/// Assumes the caller is passing a non empty list of boundaries.
fn push_if_increasing(boundaries: &mut Vec<usize>, boundary: usize) {
    debug_assert!(!boundaries.is_empty());

    if boundary > *boundaries.last().unwrap() {
        boundaries.push(boundary);
    }
}

/// Find terminator-run matches in `text`, folding a whitespace-separated
/// dot-only follow-up onto a preceding `!`/`?`/`…` run so `Bravo ! .` and
/// `Happy! . . . no one …` surface as one coalesced terminator.
///
/// The regex already coalesces homogeneous runs (`! !`, `. . .`) and
/// contiguous mixed runs like `! ...` (matched by the contiguous-class
/// branch as `!...`). Spaced mixed runs like `! . . .` can't be expressed
/// without lookahead - `[!?…][ \t]+\.` would eat the first dot of an
/// ellipsis - so they arrive here as two matches and are folded only when
/// the follow-up is pure `.`s separated by whitespace.
fn find_terminator_matches(text: &str, regex: &Regex, out: &mut Vec<(usize, usize)>) {
    out.clear();

    // Faster path for ASCII + DEFAULT_SENTENCE_BREAK_REGEX
    if std::ptr::eq(regex, &*DEFAULT_SENTENCE_BREAK_REGEX) && text.is_ascii() {
        scan_ascii_matches(text, out);
        return;
    }

    for m in regex.find_iter(text) {
        fold_match(out, text, m.start(), m.end());
    }
}

/// Push `[start, end)` onto `out`. If `should_fold` says the two form a single run
/// like `Happy ! . . .`, fold it into the previous match.
fn fold_match(out: &mut Vec<(usize, usize)>, text: &str, start: usize, end: usize) {
    if let Some(last) = out.last_mut()
        && should_fold(text, *last, start, end)
    {
        last.1 = end;
        return;
    }

    out.push((start, end));
}

/// True when match `[start, end)` should fold onto the previous match `[prev_start, prev_end)`,
/// i.e., whitespace separated dot only run like `. . .` following a `!`/`?`/`…` ending.
fn should_fold(
    text: &str,
    (prev_start, prev_end): (usize, usize),
    start: usize,
    end: usize,
) -> bool {
    let prev = &text[prev_start..prev_end];
    let candidate = &text[start..end];
    let gap = &text[prev_end..start];

    let is_blank = |c: char| matches!(c, ' ' | '\t');
    let prev_is_emphatic = prev.ends_with(['!', '?', '…']);
    let candidate_is_dot_run =
        candidate.starts_with('.') && candidate.chars().all(|c| c == '.' || is_blank(c));
    let separated_by_blanks = !gap.is_empty() && gap.chars().all(is_blank);

    prev_is_emphatic && candidate_is_dot_run && separated_by_blanks
}

/// ASCII fast path for `find_terminator_matches`.
/// Only valid for the default regex.
/// branch 1 `\.(?:[ \t]+\.){2,}` (3+ blank separated dots).
/// branch 2 `[!?](?:[ \t]+[!?])+` (2+ blank separated `!`/`?`)
/// branch 3 (`[.!?]+`) via `contiguous_run`
fn scan_ascii_matches(text: &str, out: &mut Vec<(usize, usize)>) {
    let bytes = text.as_bytes();

    let mut cursor = 0;
    while let Some(rel) = memchr::memchr3(b'.', b'!', b'?', &bytes[cursor..]) {
        let p = cursor + rel;

        let spaced = if bytes[p] == b'.' {
            spaced_run(bytes, p, |b| b == b'.', 3)
        } else {
            spaced_run(bytes, p, |b| b == b'!' || b == b'?', 2)
        };

        let end = spaced.unwrap_or_else(|| contiguous_run(bytes, p));

        fold_match(out, text, p, end);
        cursor = end;
    }
}

/// Scans a space/tab-separated run of `is_member` bytes starting at `p`.
/// Returns the offset just past the last member,
/// or `None` if the run has fewer than `min_total` members.
fn spaced_run(
    bytes: &[u8],
    p: usize,
    is_member: impl Fn(u8) -> bool,
    min_total: usize,
) -> Option<usize> {
    let mut count = 1;
    let mut end = p + 1;

    loop {
        let mut k = end;
        while matches!(bytes.get(k), Some(b' ' | b'\t')) {
            k += 1;
        }

        if k > end && bytes.get(k).is_some_and(|&b| is_member(b)) {
            count += 1;
            end = k + 1;
        } else {
            break;
        }
    }

    (count >= min_total).then_some(end)
}

/// End offset of the contiguous terminator run starting at `p`
fn contiguous_run(bytes: &[u8], p: usize) -> usize {
    let mut end = p + 1;

    while matches!(bytes.get(end), Some(b'.' | b'!' | b'?')) {
        end += 1;
    }

    end
}

/// Scratch buffer
#[derive(Default)]
struct ParagraphScratch {
    sentence_boundaries: Vec<usize>,
    matches: Vec<(usize, usize)>,
    skippable_ranges: Vec<SkippableRange>,
    orphan_closers: OrphanCloserPositions,
}

impl ParagraphScratch {
    fn with_capacity(capacity: usize) -> Self {
        Self {
            sentence_boundaries: Vec::with_capacity(capacity),
            matches: Vec::with_capacity(capacity),
            skippable_ranges: Vec::with_capacity(capacity),
            orphan_closers: OrphanCloserPositions::default(),
        }
    }
}

fn boundary_symbol(paragraph: &str, end: usize) -> Option<&str> {
    let trimmed = paragraph[..end].trim_end();
    trimmed
        .char_indices()
        .next_back()
        .and_then(|(idx, ch)| is_sentence_terminator(ch).then(|| &trimmed[idx..]))
}

fn push_separator_boundary<'a>(
    boundaries: &mut Vec<SentenceBoundary<'a>>,
    separator: &'a str,
    start_byte: usize,
    end_byte: usize,
    char_offset: &mut usize,
) {
    let separator_chars = separator.chars().count();

    boundaries.push(SentenceBoundary {
        start_index: *char_offset,
        end_index: *char_offset + separator_chars,
        start_byte,
        end_byte,
        text: separator,
        boundary_symbol: None,
        is_paragraph_break: true,
    });

    *char_offset += separator_chars;
}

/// Convert the paragraph's sentence-break offsets into `SentenceBoundary` values,
/// advancing the running character cursor `char_offset`.
fn push_paragraph_sentences<'a>(
    paragraph: &'a str,
    para_start: usize,
    sentence_boundaries: &[usize],
    char_offset: &mut usize,
    boundaries: &mut Vec<SentenceBoundary<'a>>,
) {
    debug_assert_eq!(sentence_boundaries.first().copied(), Some(0));
    debug_assert_eq!(sentence_boundaries.last().copied(), Some(paragraph.len()));

    for window in sentence_boundaries.windows(2) {
        let seg_start = window[0];
        let seg_end = window[1];
        let sentence_text = &paragraph[seg_start..seg_end];
        let end_offset = *char_offset + sentence_text.chars().count();

        boundaries.push(SentenceBoundary {
            start_index: *char_offset,
            end_index: end_offset,
            start_byte: para_start + seg_start,
            end_byte: para_start + seg_end,
            text: sentence_text,
            boundary_symbol: boundary_symbol(paragraph, seg_end),
            is_paragraph_break: false,
        });

        *char_offset = end_offset;
    }
}

/// Properties of the sorted non list region of a paragraph's skippable ranges.
/// `len` is the number of skippable ranges.
/// `binary_search` is whether to use binary search when searching in the ranges, i.e., is there enough data to
/// justify the overhead of binary search.
#[derive(Clone, Copy)]
pub(crate) struct NonListRegion {
    pub(crate) len: usize,
    pub(crate) binary_search: bool,
}

/// Minimum number of ranges required before it's worth while to use binary search (rather than a linear scan).
const BINARY_SEARCH_MIN_RANGES: usize = 64;

/// Get the byte offsets in `paragraph` where sentences break. Skip terminators inside quotes, parens, and lists.
fn collect_sentence_breaks<L: Language + ?Sized>(
    lang: &L,
    paragraph: &str,
    sentence_break_regex: &Regex,
    scratch: &mut ParagraphScratch,
) {
    let ParagraphScratch {
        sentence_boundaries,
        matches,
        skippable_ranges,
        orphan_closers,
    } = scratch;

    orphan_closers.reset();

    sentence_boundaries.clear();
    sentence_boundaries.push(0);

    find_terminator_matches(paragraph, sentence_break_regex, matches);
    lang.get_skippable_ranges(paragraph, skippable_ranges);

    let non_list_len = skippable_ranges.len();
    let non_list_region = NonListRegion {
        len: non_list_len,
        binary_search: non_list_len > BINARY_SEARCH_MIN_RANGES && !ranges_overlap(skippable_ranges),
    };

    let list_starts = super::list_markers::detect_list_items(paragraph);
    add_list_item_ranges(skippable_ranges, &list_starts, paragraph.len());

    for &(match_start, match_end) in matches.iter() {
        let Some(boundary) = lang.find_boundary(paragraph, match_start, match_end) else {
            continue;
        };

        let break_at = match containing_range(
            lang,
            paragraph,
            boundary,
            match_start,
            match_end,
            skippable_ranges,
            non_list_region,
        ) {
            Some(range) => inner_terminator_boundary(lang, paragraph, range, boundary),
            None => Some(extend_past_orphan_closer(
                lang,
                paragraph,
                boundary,
                skippable_ranges,
                non_list_region,
                orphan_closers,
            )),
        };

        if let Some(break_at) = break_at {
            push_if_increasing(sentence_boundaries, break_at);
        }
    }

    merge_list_item_boundaries(sentence_boundaries, &list_starts);

    if *sentence_boundaries.last().unwrap() != paragraph.len() {
        sentence_boundaries.push(paragraph.len());
    }
}

/// True iff a range overlaps another one.
fn ranges_overlap(start_sorted_ranges: &[SkippableRange]) -> bool {
    let mut max_end = 0;

    for r in start_sorted_ranges {
        if r.start < max_end {
            return true;
        }

        max_end = max_end.max(r.end);
    }

    false
}

/// Binary search the non list region of `ranges` for the range satisfying `is_break`.
/// Fall back to the appended list ranges.
/// `ranges` needs to be sorted and non overlapping.
fn select_containing_binary(
    ranges: &[SkippableRange],
    non_list_len: usize,
    boundary: usize,
    is_break: impl Fn(&SkippableRange) -> bool,
) -> Option<&SkippableRange> {
    let (non_list, list) = ranges.split_at(non_list_len);

    non_list
        .partition_point(|r| r.start < boundary)
        .checked_sub(1)
        .map(|i| &non_list[i])
        .filter(|&r| is_break(r))
        .or_else(|| list.iter().find(|&r| is_break(r)))
}

/// The first skippable range that genuinely encloses `boundary`: it contains the
/// offset and is not a symmetric-quote mispairing (which only looks like containment).
/// `Some` means the terminator at `boundary` should be suppressed rather than split on.
fn containing_range<'r, L: Language + ?Sized>(
    lang: &L,
    paragraph: &str,
    boundary: usize,
    match_start: usize,
    match_end: usize,
    ranges: &'r [SkippableRange],
    region: NonListRegion,
) -> Option<&'r SkippableRange> {
    let is_break = |range: &SkippableRange| {
        range.contains(boundary)
            && !is_symmetric_quote_mispairing(lang, paragraph, range, match_start, match_end)
    };

    if region.binary_search {
        select_containing_binary(ranges, region.len, boundary, is_break)
    } else {
        ranges.iter().find(|&r| is_break(r))
    }
}

/// Push a skippable range for each list-item line span (the last to `paragraph_len`)
/// so a terminator inside an item does not split it.
fn add_list_item_ranges(
    skippable_ranges: &mut Vec<SkippableRange>,
    list_starts: &[usize],
    paragraph_len: usize,
) {
    for pair in list_starts.windows(2) {
        skippable_ranges.push(SkippableRange::new(
            pair[0],
            pair[1],
            SkippableRangeType::ListItem,
        ));
    }

    if let Some(&last) = list_starts.last() {
        skippable_ranges.push(SkippableRange::new(
            last,
            paragraph_len,
            SkippableRangeType::ListItem,
        ));
    }
}

/// Add each list item line start as a sentence boundary, then sort and dedup.
fn merge_list_item_boundaries(sentence_boundaries: &mut Vec<usize>, list_starts: &[usize]) {
    if list_starts.is_empty() {
        return;
    }

    for &start in list_starts {
        if start > 0 {
            sentence_boundaries.push(start);
        }
    }

    sentence_boundaries.sort_unstable();
    sentence_boundaries.dedup();
}

/// Shared helper for languages that continue sentences before month names.
///
/// Returns `true` if `text` starts with a lowercase letter/digit (after optional
/// non-word characters), or if its first whitespace-delimited word (case-insensitively
/// capitalised) is one of the supplied `months`.
pub fn continues_after_boundary(text: &str, months: &[&str]) -> bool {
    if CONTINUE_AFTER_NONWORD_REGEX.is_match(text) {
        return true;
    }

    let next_word = text
        .split_whitespace()
        .next()
        .unwrap_or("")
        .trim_matches(['.', '!', '?']);

    if next_word.is_empty() {
        return false;
    }

    // Build a version with the first character upper-cased (handles non-ASCII safely).
    let capitalized: String = next_word
        .chars()
        .enumerate()
        .map(|(i, c)| {
            if i == 0 {
                c.to_uppercase().to_string()
            } else {
                c.to_string()
            }
        })
        .collect();

    months.contains(&next_word) || months.contains(&capitalized.as_str())
}

#[derive(Debug, Clone, Copy, PartialEq)]
pub enum SkippableRangeType {
    Quote,
    Parentheses,
    Email,
    ListItem,
}

#[derive(Debug, Clone, Copy)]
pub struct SkippableRange {
    pub start: usize,
    pub end: usize,
    pub range_type: SkippableRangeType,
    pub quote_pair: Option<&'static QuotePair>,
    pub quote_mispairing: QuoteMispairing,
}

impl SkippableRange {
    pub fn new(start: usize, end: usize, range_type: SkippableRangeType) -> Self {
        Self {
            start,
            end,
            range_type,
            quote_pair: None,
            quote_mispairing: QuoteMispairing::None,
        }
    }

    pub fn new_quote(start: usize, end: usize, pair: &'static QuotePair) -> Self {
        Self {
            start,
            end,
            range_type: SkippableRangeType::Quote,
            quote_pair: Some(pair),
            quote_mispairing: QuoteMispairing::None,
        }
    }

    pub fn contains(&self, position: usize) -> bool {
        position > self.start && position < self.end
    }

    pub fn is_quote(&self) -> bool {
        self.range_type == SkippableRangeType::Quote
    }
}

pub trait Language {
    /// Returns a reference to the compiled regex pattern that matches sentence terminating
    /// punctuation. The default implementation uses a static LazyLock for zero-cost access.
    fn get_sentence_break_regex(&self) -> &'static Regex {
        &DEFAULT_SENTENCE_BREAK_REGEX
    }

    /// Analyzes the input text and returns a vector of sentence boundaries.
    /// This is the main method for sentence segmentation that:
    /// 1. Splits text into paragraphs at double newlines
    /// 2. Identifies potential sentence breaks using regex patterns
    /// 3. Filters out false positives (abbreviations, quotes, etc.)
    /// 4. Returns structured boundary information including start/end positions and boundary symbols
    ///
    /// Each boundary contains the sentence text, position indices, and metadata about the boundary type.
    fn get_sentence_boundaries<'a>(&self, text: &'a str) -> Vec<SentenceBoundary<'a>> {
        let text_len = text.len();
        let capacity = (text_len / 50).max(1);
        let mut boundaries = Vec::with_capacity(capacity);
        let mut scratch = ParagraphScratch::with_capacity(capacity);
        let regex = self.get_sentence_break_regex();

        // Walk each paragraph paired with its trailing separator (`None` after the last
        // paragraph). `para_start` / `char_offset` are the running byte / character
        // cursors, tracked separately for correct multi-byte UTF-8 handling ("日本語"
        // is 3 characters but 9 bytes).
        let (mut para_start, mut char_offset) = (0usize, 0usize);
        let trailing_separators = paragraph_breaks(text)
            .map(Some)
            .chain(std::iter::once(None));

        for trailing_separator in trailing_separators {
            let para_end = trailing_separator.map_or(text_len, |(sep_start, _)| sep_start);
            let paragraph = &text[para_start..para_end];

            collect_sentence_breaks(self, paragraph, regex, &mut scratch);
            push_paragraph_sentences(
                paragraph,
                para_start,
                &scratch.sentence_boundaries,
                &mut char_offset,
                &mut boundaries,
            );

            // Emit the separator that follows this paragraph (none after the last).
            if let Some((sep_start, sep_end)) = trailing_separator {
                push_separator_boundary(
                    &mut boundaries,
                    &text[sep_start..sep_end],
                    sep_start,
                    sep_end,
                    &mut char_offset,
                );

                para_start = sep_end;
            }
        }

        boundaries
    }

    /// Segments `text` into sentence and paragraph separator slices.
    /// Emits slices directly instead of building per-sentence index/symbol metadata.
    fn segment<'a>(&self, text: &'a str) -> Vec<&'a str> {
        let text_len = text.len();
        let capacity = (text_len / 50).max(1);
        let mut sentences = Vec::with_capacity(capacity);
        let mut scratch = ParagraphScratch::with_capacity(capacity);
        let regex = self.get_sentence_break_regex();

        let mut para_start = 0usize;
        let trailing_separators = paragraph_breaks(text)
            .map(Some)
            .chain(std::iter::once(None));

        for trailing_separator in trailing_separators {
            let para_end = trailing_separator.map_or(text_len, |(sep_start, _)| sep_start);
            let paragraph = &text[para_start..para_end];

            collect_sentence_breaks(self, paragraph, regex, &mut scratch);

            for window in scratch.sentence_boundaries.windows(2) {
                let sentence = &paragraph[window[0]..window[1]];
                if !sentence.is_empty() {
                    sentences.push(sentence);
                }
            }

            if let Some((sep_start, sep_end)) = trailing_separator {
                let separator = &text[sep_start..sep_end];
                if !separator.is_empty() {
                    sentences.push(separator);
                }

                para_start = sep_end;
            }
        }

        sentences
    }

    /// Returns the character used to mark abbreviations in this language.
    /// By default returns "." (period), but should be overridden by specific languages
    /// that use different abbreviation markers. Used by the abbreviation detection logic
    /// to determine if a potential sentence boundary is actually an abbreviation.
    fn get_abbreviation_char(&self) -> &str {
        "."
    }

    /// Returns a list of known abbreviations for this language.
    /// These are used to prevent false sentence breaks at abbreviation periods.
    /// For example, "Dr." or "etc." should not trigger a sentence boundary.
    /// Languages should override this to provide their specific abbreviation lists.
    /// Returns an empty set by default.
    fn get_abbreviations(&self) -> &FxHashSet<String> {
        static EMPTY_ABBREVS: LazyLock<FxHashSet<String>> = LazyLock::new(FxHashSet::default);
        &EMPTY_ABBREVS
    }

    /// Returns a set of safe sentence-opener words for this language.
    /// Restrict the list to function words and auxiliaries that almost never appear
    /// capitalized mid-sentence. Never include proper nouns.
    /// Returns an empty set to preserve default behaviour. Languages opt in by overriding.
    fn get_sentence_starters(&self) -> &FxHashSet<String> {
        static EMPTY_STARTERS: LazyLock<FxHashSet<String>> = LazyLock::new(FxHashSet::default);
        &EMPTY_STARTERS
    }

    /// Words permitted in fronted adverbial phrases. Used in `prefix_is_purely_fronting`.
    /// Languages must opt in.
    /// Returns an empty set by default.
    fn get_fronting_words(&self) -> &FxHashSet<String> {
        static EMPTY_FRONTING: LazyLock<FxHashSet<String>> = LazyLock::new(FxHashSet::default);
        &EMPTY_FRONTING
    }

    /// Trailing-marker lookup table.
    /// Languages must opt in.
    /// Returns an empty MarkerTable by default.
    #[inline]
    fn get_trailing_markers(&self) -> &'static MarkerTable {
        MarkerTable::empty()
    }

    /// Byte offset past the leading run of whitespace/terminators in `word`,
    /// or `None` when `word` continues the current sentence.
    fn get_boundary_extend(&self, word: &str) -> Option<usize> {
        if self.continue_in_next_word(word.trim()) || CONTINUE_AFTER_NONWORD_REGEX.is_match(word) {
            return None;
        }

        let mut count = 0;
        for ch in word.chars() {
            if ch.is_whitespace() || is_sentence_terminator(ch) {
                count += ch.len_utf8();
            } else {
                break;
            }
        }

        Some(count)
    }

    /// Checks if a potential sentence boundary is actually part of an abbreviation.
    /// Examines the text before the separator to see if it ends with a known abbreviation.
    /// Returns true if this appears to be an abbreviation (and thus not a sentence boundary),
    /// false if it's likely a genuine sentence end. Used to prevent breaking sentences
    /// at abbreviations like "Dr. Smith" or "etc."
    fn is_abbreviation(&self, head: &str, _tail: &str, separator: &str) -> bool {
        let last_word = self.get_last_word(head);
        self.is_abbreviation_for(last_word, separator)
    }

    /// Same check as `is_abbreviation` but skips the `get_last_word(head)` call
    /// when the caller already has the trailing word. Used on the hot path in
    /// `find_boundary`, which computes `last_word` once and shares it with
    /// `is_name_initial`/`next_word_is_sentence_starter`.
    fn is_abbreviation_for(&self, last_word: &str, separator: &str) -> bool {
        if self.get_abbreviation_char() != separator || last_word.is_empty() {
            return false;
        }
        abbreviation_set_contains(self.get_abbreviations(), last_word)
    }

    /// Detects a name initial: a single uppercase ASCII letter followed by a
    /// period in a position that looks like part of a name. Returns true when
    /// the immediately preceding token in `head` starts with an uppercase
    /// ASCII letter (`Albert I.`, `George W.`) or the immediately following
    /// token is itself an initial (`J. R. R. Tolkien`, including the
    /// sentence-initial position where there is no preceding token).
    ///
    /// Conservative: ASCII-only on both sides, so non-Latin scripts are
    /// unaffected. Caller is expected to gate this on the matched terminator
    /// being a single `.` and on `last_word` being a single uppercase letter.
    /// The helper re-checks the latter so it is safe to call standalone.
    fn is_name_initial(&self, head: &str, next_word_approx: &str) -> bool {
        let last_word = self.get_last_word(head);
        self.is_name_initial_for(head, last_word, next_word_approx)
    }

    /// Same check as `is_name_initial` but skips the `get_last_word(head)` call
    /// when the caller already has the trailing word. Used on the hot path in
    /// `find_boundary`.
    fn is_name_initial_for(&self, head: &str, last_word: &str, next_word_approx: &str) -> bool {
        if !is_single_ascii_upper(last_word) {
            return false;
        }

        // Preceding-token rule: trim the initial and any separators
        // get_last_word splits on (whitespace, `.`, `/`), then take the
        // trailing word of what's left.
        let prefix = head[..head.len() - last_word.len()]
            .trim_end_matches(|c: char| c.is_whitespace() || c == '.' || c == '/');

        if self
            .get_last_word(prefix)
            .chars()
            .next()
            .is_some_and(|c| c.is_ascii_uppercase())
        {
            return true;
        }

        starts_with_initial(next_word_approx)
    }

    /// Returns true if the next non-space token in `next_word_approx` is a known
    /// sentence opener for this language. Overrides sentence break suppression of
    /// abbreviation or name-initial paths. A listed starter word strongly signals
    /// the start of a new sentence.
    fn next_word_is_sentence_starter(&self, next_word_approx: &str) -> bool {
        let starters = self.get_sentence_starters();
        if starters.is_empty() {
            return false;
        }

        let trimmed = next_word_approx.trim_start();

        let word_end = trimmed
            .find(|c: char| c.is_whitespace() || c == ',' || is_sentence_terminator(c))
            .unwrap_or(trimmed.len());

        if word_end == 0 {
            return false;
        }

        let starter_candidate = &trimmed[..word_end];
        starters.contains(starter_candidate)
    }

    /// One way override that lets `find_boundary` keep a boundary the abbreviation / name-initial path would
    /// otherwise suppress. Fires when the next word is a registered sentence starter and the trailing token:
    /// - Starts with an uppercase letter: initials (`I.`), names (`Penn.`), acronyms (`BART.`).
    /// - A known multi dot abbreviation (`w.e.f.`).
    /// - A multi character lowercase abbreviation (`etc.`, `man.`).
    fn should_override_abbrev_suppression_for(
        &self,
        head: &str,
        last_word: &str,
        next_is_starter: bool,
    ) -> bool {
        if !next_is_starter {
            return false;
        }

        let tail_starts_uppercase = last_word
            .chars()
            .next()
            .is_some_and(|c| c.is_ascii_uppercase());

        if tail_starts_uppercase {
            return true;
        }

        if self.is_multi_dot_abbreviation(head, last_word.len()) {
            return true;
        }

        last_word.chars().nth(1).is_some() && self.get_abbreviations().contains(last_word)
    }

    /// True when `head`'s trailing token is a multi-dot abbreviation listed
    /// in this language's abbreviation table (`w.e.f`, `U.S.`, ...).
    fn is_multi_dot_abbreviation(&self, head: &str, tail_len: usize) -> bool {
        let last_word_full = self.get_last_word_full(head);
        if last_word_full.len() <= tail_len {
            return false;
        }

        abbreviation_set_contains(self.get_abbreviations(), last_word_full)
    }

    /// Like `get_last_word`, but keeps internal `.`s so multi-dot
    /// abbreviations (`w.e.f`, `U.S`, `p.m`) are returned whole. Splits only
    /// on whitespace and `/`. Used by abbreviation lookup so the full token
    /// can be matched against the abbreviation table.
    fn get_last_word_full<'a>(&self, text: &'a str) -> &'a str {
        text.trim_end()
            .rsplit(|c: char| c.is_whitespace() || c == '/')
            .next()
            .expect("str::rsplit always yields at least one element")
    }

    /// Extracts the last word from the given text by splitting on whitespace, periods, and slashes.
    /// Used primarily by abbreviation detection to check if the word before a potential
    /// sentence boundary is a known abbreviation. Returns an empty string if no words
    /// are found. This is a performance-optimized version that avoids collecting all words.
    fn get_last_word<'a>(&self, text: &'a str) -> &'a str {
        // Trim trailing whitespace so a stray space before the terminator
        // (`U.S .`) doesn't blank out the last word. `/` joins route names
        // to abbreviations (`171/U.S`) without being a real word boundary,
        // so split on it too.
        text.trim_end()
            .rsplit(|c: char| c.is_whitespace() || c == '.' || c == '/')
            .next()
            .expect("str::rsplit always yields at least one element")
    }

    /// Checks if a potential sentence boundary is actually an exclamation word that shouldn't
    /// trigger a sentence break. Examines the last word before the boundary and checks if
    /// it's in the list of known exclamation words (like "Hey!" or "Wow!").
    /// Returns true if this is an exclamation that should not break the sentence.
    fn is_exclamation(&self, head: &str, _tail: &str) -> bool {
        let last_word = self.get_last_word(head);
        self.is_exclamation_for(last_word)
    }

    /// Same check as `is_exclamation` but skips the `get_last_word(head)`
    /// call when the caller already has the trailing word. Used on the hot
    /// path in `find_boundary`.
    fn is_exclamation_for(&self, last_word: &str) -> bool {
        if last_word.is_empty() {
            return false;
        }

        EXCLAMATION_WORDS
            .iter()
            .any(|w| w.strip_suffix('!').is_some_and(|p| p == last_word))
    }

    /// True when the terminator at `[start, end)` looks like a confident
    /// sentence end: a single `.` whose preceding word is a symmetric quote
    /// closer (`''`, `"`, …) and whose follower starts with a capital letter
    /// — the `closer + . + UpperWord` shape. Used as a structural escape
    /// valve for symmetric-pair quote ranges (`''…''`, `'…'`, `"…"`) that the
    /// non-greedy `QUOTES_REGEX` may have mispaired across a real boundary.
    fn has_strong_sentence_break(&self, paragraph: &str, start: usize, end: usize) -> bool {
        if end - start != 1 || paragraph.as_bytes()[start] != b'.' {
            return false;
        }

        debug_assert!(paragraph.is_char_boundary(start + 1));
        let next_word_approx = self.get_next_word_approx(paragraph, start + 1);
        let trimmed_next = next_word_approx.trim_start();
        if !trimmed_next
            .chars()
            .next()
            .is_some_and(|c| c.is_ascii_uppercase())
        {
            return false;
        }

        let head = &paragraph[..start];
        let last_word = self.get_last_word(head);
        if last_word.is_empty() || is_single_ascii_upper(last_word) {
            return false;
        }

        if self.is_abbreviation(head, last_word, ".")
            || self.is_multi_dot_abbreviation(head, last_word.len())
        {
            return false;
        }

        if is_symmetric_quote_closer(last_word) {
            return true;
        }

        self.next_word_is_sentence_starter(trimmed_next)
    }

    /// Returns an approximate substring of the next word(s) starting from the given position.
    /// Limited to a maximum of 30 bytes (+ codepoint rounding) for performance. Used to analyze context
    /// after a potential sentence boundary to determine if the boundary should be created.
    /// Handles UTF-8 character boundaries safely to avoid panics on non-ASCII text.
    /// NOTE: If the longest special words (abbreviations, starters, etc) ever exceed 30 bytes, then
    /// consider raising the cap, or making the cap max special word length aware.
    fn get_next_word_approx<'a>(&self, text: &'a str, start: usize) -> &'a str {
        if start >= text.len() {
            return "";
        }

        // The current call site guarantees a character boundary. Catch any future drift.
        debug_assert!(text.is_char_boundary(start));

        let max_bytes = 30;
        let end_pos = (start + max_bytes).min(text.len());
        &text[start..text.ceil_char_boundary(end_pos)]
    }

    /// When a lowercase/digit (or comma) follower, an ellipsis continuation, or a spaced `!`/`?`
    /// before a lowercase word occurs after a terminator, suppress the sentence break.
    fn terminator_continues(&self, matched: &str, head: &str, next_word_approx: &str) -> bool {
        if matched.chars().nth(1).is_some() {
            return self.is_ellipsis_continuation(next_word_approx)
                || (head.chars().next_back().is_some_and(|c| !c.is_whitespace())
                    && starts_with_ascii_lowercase_or_digit(next_word_approx));
        }

        self.continue_in_next_word(next_word_approx)
            // e.g., "Father Came Too ! is a British comedy film".
            || (matches!(matched, "!" | "?")
                && matches!(head.as_bytes().last(), Some(b' ' | b'\t'))
                && CONTINUE_AFTER_NONWORD_REGEX.is_match(next_word_approx))
    }

    /// Whether a `.` terminator should be suppressed.
    fn period_suppresses_boundary(
        &self,
        head: &str,
        last_word: &str,
        next_word_approx: &str,
    ) -> bool {
        let suppress = self.is_name_initial_for(head, last_word, next_word_approx)
            || self.is_abbreviation_for(last_word, ".");

        let marker = classify_trailing_marker(head, self.get_trailing_markers());
        if !suppress && marker.is_none() {
            return false;
        }

        let next_is_starter = self.next_word_is_sentence_starter(next_word_approx);
        let marker_bypass = marker.as_ref().is_some_and(|m| {
            marker_bypasses_suppression(m, next_word_approx, next_is_starter, self)
        });

        !marker_bypass
            && !self.should_override_abbrev_suppression_for(head, last_word, next_is_starter)
    }

    /// Analyzes a potential sentence boundary and determines the exact position where
    /// the sentence should end, or returns None if this shouldn't be a boundary.
    /// Considers abbreviations, exclamations, numbered references, and continuation patterns.
    /// This is the core logic that distinguishes true sentence boundaries from false positives
    /// like abbreviations or mid-sentence punctuation.
    fn find_boundary(&self, text: &str, start: usize, end: usize) -> Option<usize> {
        let head = &text[..start];
        let matched = &text[start..end];
        let next_word_approx = self.get_next_word_approx(text, end);

        // Gate the regex to only run if `[` is found
        if memchr::memchr(b'[', next_word_approx.as_bytes()).is_some()
            && let Some(m) = crate::constants::NUMBERED_REFERENCE_REGEX.find(next_word_approx)
        {
            return Some(end + m.end());
        }

        if self.terminator_continues(matched, head, next_word_approx) {
            return None;
        }

        let last_word = self.get_last_word(head);

        if matched == "." {
            if is_code_like_numbered_token(head, next_word_approx) {
                return None;
            }

            if self.period_suppresses_boundary(head, last_word, next_word_approx) {
                return None;
            }
        }

        if self.is_exclamation_for(last_word) {
            return None;
        }

        // Swallow any whitespace after the terminator into the boundary.
        // Replaces the `^\s+` regex.
        let trailing_ws = next_word_approx.len() - next_word_approx.trim_start().len();
        Some(end + trailing_ws)
    }

    /// True when text following a multi-char terminator run (`...`, `! ?`,
    /// `. . .`) continues the current sentence rather than starting a new one.
    /// The default treats only whitespace + a lowercase letter/digit as
    /// continuation.
    fn is_ellipsis_continuation(&self, text_after_run: &str) -> bool {
        ELLIPSIS_CONTINUE_REGEX.is_match(text_after_run)
    }

    /// Determines if the text after a potential boundary indicates the sentence should continue.
    /// Returns true if the next word starts with a lowercase letter or number, suggesting
    /// the sentence is continuing rather than starting a new one. This helps avoid breaking
    /// sentences at abbreviations or in the middle of compound sentences.
    fn continue_in_next_word(&self, text_after_boundary: &str) -> bool {
        if starts_with_ascii_lowercase_or_digit(text_after_boundary) {
            return true;
        }

        peel_leading_symmetric_quote(text_after_boundary).starts_with(',')
    }

    /// Identifies ranges of text that should be skipped during sentence boundary detection.
    /// This includes quoted text, parenthetical expressions, and email addresses where
    /// internal punctuation should not trigger sentence breaks. Returns a sorted vector
    /// of ranges that can be efficiently checked during boundary detection to avoid
    /// false positives within these special text regions.
    fn get_skippable_ranges(&self, text: &str, out: &mut Vec<SkippableRange>) {
        out.clear();

        collect_quote_ranges(text, out);

        for mat in PARENS_REGEX.find_iter(text) {
            out.push(SkippableRange::new(
                mat.start(),
                mat.end(),
                SkippableRangeType::Parentheses,
            ));
        }

        for mat in EMAIL_REGEX.find_iter(text) {
            out.push(SkippableRange::new(
                mat.start(),
                mat.end(),
                SkippableRangeType::Email,
            ));
        }

        // Sort ranges by start position for more efficient lookups
        out.sort_unstable_by_key(|r| r.start);

        // Cache mispairing on each quote range for re-use
        tag_quote_mispairing(text, out);
    }
}

#[cfg(test)]
mod tests {
    use super::Language;
    use crate::languages::Japanese;

    #[test]
    fn get_boundary_extend_sums_run_in_bytes() {
        let lang = Japanese {};

        assert_eq!(lang.get_boundary_extend(". X"), Some(2));
        assert_eq!(lang.get_boundary_extend("。次"), Some(3));
        assert_eq!(lang.get_boundary_extend("。。次"), Some(6));
        assert_eq!(lang.get_boundary_extend("　　X"), Some(6));
        assert_eq!(lang.get_boundary_extend(""), Some(0));
        assert_eq!(lang.get_boundary_extend(" foo"), None);
    }
}