varar_core/
sentences.rs

1//! Splits a block of text into sentence-level spans so the matcher can try each
2//! sentence independently — port of `sentences.ts` / `Sentences.java`. Operates
3//! on `char`s with a running UTF-16 offset table (the Python-port approach): the
4//! split decisions use BMP terminators, and emitted offsets are UTF-16.
5
6use crate::offsets::{java_strip, java_strip_leading, utf16_len};
7
8/// A sentence: the trimmed text plus its UTF-16 offsets into the input.
9#[derive(Clone, Debug, PartialEq, Eq)]
10pub struct Sentence {
11    pub text: String,
12    pub start_offset: usize,
13    pub end_offset: usize,
14}
15
16impl Sentence {
17    pub fn new(text: impl Into<String>, start_offset: usize, end_offset: usize) -> Sentence {
18        Sentence {
19            text: text.into(),
20            start_offset,
21            end_offset,
22        }
23    }
24}
25
26const ABBREVIATIONS: [&str; 5] = ["e.g.", "i.e.", "etc.", "cf.", "vs."];
27
28/// Splits `text` on `.`/`!`/`?`/newline terminators, skipping backtick code-span
29/// and double-quoted interiors, and treating decimals and a fixed abbreviation
30/// list as non-terminating dots.
31pub fn split_sentences(text: &str) -> Vec<Sentence> {
32    let chars: Vec<char> = text.chars().collect();
33    let n = chars.len();
34
35    // Prefix table: cp_to_u16[i] = UTF-16 offset of char i; cp_to_u16[n] = total.
36    let mut cp_to_u16 = vec![0usize; n + 1];
37    for i in 0..n {
38        cp_to_u16[i + 1] = cp_to_u16[i] + chars[i].len_utf16();
39    }
40
41    // Mark backtick code spans and double-quoted strings as no-split zones.
42    let mut skip = vec![false; n];
43    let mut j = 0;
44    while j < n {
45        let c = chars[j];
46        if c == '`' || c == '"' {
47            match find_char(&chars, j + 1, c) {
48                Some(close) => {
49                    for entry in skip.iter_mut().take(close + 1).skip(j) {
50                        *entry = true;
51                    }
52                    j = close;
53                }
54                None => break,
55            }
56        }
57        j += 1;
58    }
59
60    let mut out = Vec::new();
61    let mut segment_start = 0usize;
62    let mut i = 0;
63    while i < n {
64        if skip[i] {
65            i += 1;
66            continue;
67        }
68        let ch = chars[i];
69        if ch == '\n' || ch == '.' || ch == '!' || ch == '?' {
70            if ch == '.' && is_inside_number_or_abbrev(&chars, i) {
71                i += 1;
72                continue;
73            }
74            let end = i + 1;
75            push_segment(&mut out, &chars, &cp_to_u16, segment_start, end);
76            i = end;
77            // Skip following whitespace so the next sentence starts at content.
78            while i < n && (chars[i] == ' ' || chars[i] == '\n') {
79                i += 1;
80            }
81            segment_start = i;
82            continue;
83        }
84        i += 1;
85    }
86    push_segment(&mut out, &chars, &cp_to_u16, segment_start, n);
87    out
88}
89
90fn find_char(chars: &[char], from: usize, target: char) -> Option<usize> {
91    (from..chars.len()).find(|&k| chars[k] == target)
92}
93
94fn push_segment(
95    out: &mut Vec<Sentence>,
96    chars: &[char],
97    cp_to_u16: &[usize],
98    start: usize,
99    end: usize,
100) {
101    if end <= start {
102        return;
103    }
104    let raw: String = chars[start..end].iter().collect();
105    let slice = java_strip(&raw);
106    if slice.is_empty() {
107        return;
108    }
109    let leading = utf16_len(&raw) - utf16_len(java_strip_leading(&raw));
110    let trimmed_start = cp_to_u16[start] + leading;
111    let trimmed_end = trimmed_start + utf16_len(slice);
112    out.push(Sentence {
113        text: slice.to_string(),
114        start_offset: trimmed_start,
115        end_offset: trimmed_end,
116    });
117}
118
119fn is_inside_number_or_abbrev(chars: &[char], dot_pos: usize) -> bool {
120    let prev = if dot_pos > 0 {
121        chars[dot_pos - 1]
122    } else {
123        '\0'
124    };
125    let next = if dot_pos + 1 < chars.len() {
126        chars[dot_pos + 1]
127    } else {
128        '\0'
129    };
130    if prev.is_ascii_digit() && next.is_ascii_digit() {
131        return true;
132    }
133    // Known abbreviations ending at dot_pos+1.
134    for abbrev in ABBREVIATIONS {
135        let len = abbrev.chars().count();
136        let from = (dot_pos + 1).saturating_sub(len);
137        let candidate: String = chars[from..dot_pos + 1].iter().collect();
138        if candidate == abbrev {
139            return true;
140        }
141    }
142    // Lowercase letter following → likely intra-word.
143    next.is_ascii_lowercase()
144}
varar_core/sentences.rs

varar_core/
sentences.rs
