use smallvec::SmallVec;
#[derive(Debug, Clone)]
pub struct EnclosureContext<'a> {
pub position: usize,
pub preceding_chars: SmallVec<[char; 3]>,
pub following_chars: SmallVec<[char; 3]>,
pub line_offset: usize,
pub chunk_text: &'a str,
}
pub trait EnclosureSuppressor: Send + Sync {
fn should_suppress_enclosure(&self, ch: char, context: &EnclosureContext) -> bool;
fn description(&self) -> &str {
"Generic enclosure suppressor"
}
}
#[derive(Debug, Clone, Default)]
pub struct EnglishEnclosureSuppressor;
impl EnglishEnclosureSuppressor {
pub fn new() -> Self {
Self
}
fn is_contraction_apostrophe(&self, context: &EnclosureContext) -> bool {
let prev_is_alpha = context
.preceding_chars
.last()
.map(|c| c.is_alphabetic())
.unwrap_or(false);
let next_is_alpha = context
.following_chars
.first()
.map(|c| c.is_alphabetic())
.unwrap_or(false);
prev_is_alpha && next_is_alpha
}
fn is_possessive_or_plural(&self, context: &EnclosureContext) -> bool {
let prev_is_alnum = context
.preceding_chars
.last()
.map(|c| c.is_alphanumeric())
.unwrap_or(false);
let next_is_space_or_punct = context
.following_chars
.first()
.map(|c| c.is_whitespace() || c.is_ascii_punctuation())
.unwrap_or(true);
prev_is_alnum && next_is_space_or_punct
}
fn is_measurement_mark(&self, _ch: char, context: &EnclosureContext) -> bool {
let prev_char = context.preceding_chars.last();
if let Some(prev) = prev_char {
if prev.is_numeric() {
return true;
}
if *prev == '°' || *prev == '\'' {
return true;
}
if prev.is_whitespace() && context.preceding_chars.len() >= 2 {
if let Some(prev_prev) = context
.preceding_chars
.get(context.preceding_chars.len() - 2)
{
if prev_prev.is_numeric() {
return true;
}
}
}
}
false
}
fn is_list_item_paren(&self, context: &EnclosureContext) -> bool {
if context.line_offset > 10 {
return false;
}
let mut found_alnum = false;
let mut found_only_space_before_alnum = true;
for ch in context.preceding_chars.iter().rev() {
if ch.is_alphanumeric() {
found_alnum = true;
} else if !ch.is_whitespace() && ch != &'.' && !found_alnum {
found_only_space_before_alnum = false;
break;
}
}
found_alnum && found_only_space_before_alnum
}
fn is_year_abbreviation(&self, context: &EnclosureContext) -> bool {
let prev_is_space_or_start = context
.preceding_chars
.last()
.map(|c| c.is_whitespace())
.unwrap_or(true);
if context.following_chars.len() >= 3 {
let follows_year_pattern = context.following_chars[0].is_ascii_digit()
&& context.following_chars[1].is_ascii_digit()
&& context.following_chars[2] == 's';
prev_is_space_or_start && follows_year_pattern
} else {
false
}
}
}
impl EnclosureSuppressor for EnglishEnclosureSuppressor {
fn should_suppress_enclosure(&self, ch: char, context: &EnclosureContext) -> bool {
match ch {
'\'' | '\u{2019}' => {
self.is_contraction_apostrophe(context)
|| self.is_possessive_or_plural(context)
|| self.is_measurement_mark(ch, context)
|| self.is_year_abbreviation(context)
}
'"' => self.is_measurement_mark(ch, context),
')' => self.is_list_item_paren(context),
_ => false,
}
}
fn description(&self) -> &str {
"English enclosure suppressor (contractions, measurements, list items)"
}
}
#[derive(Debug, Clone, Default)]
pub struct NoOpEnclosureSuppressor;
impl EnclosureSuppressor for NoOpEnclosureSuppressor {
fn should_suppress_enclosure(&self, _ch: char, _context: &EnclosureContext) -> bool {
false
}
fn description(&self) -> &str {
"No-op enclosure suppressor"
}
}
#[cfg(test)]
mod tests {
use super::*;
fn create_context<'a>(
preceding: &str,
following: &str,
line_offset: usize,
chunk_text: &'a str,
) -> EnclosureContext<'a> {
let preceding_chars: SmallVec<[char; 3]> = preceding
.chars()
.rev()
.take(3)
.collect::<Vec<_>>()
.into_iter()
.rev()
.collect();
let following_chars: SmallVec<[char; 3]> = following.chars().take(3).collect();
EnclosureContext {
position: 0,
preceding_chars,
following_chars,
line_offset,
chunk_text,
}
}
#[test]
fn test_contraction_detection() {
let suppressor = EnglishEnclosureSuppressor::new();
let context = create_context("isn", "t", 10, "isn't");
assert!(suppressor.should_suppress_enclosure('\'', &context));
let context = create_context("don", "t", 10, "don't");
assert!(suppressor.should_suppress_enclosure('\'', &context));
let context = create_context("I", "m", 5, "I'm");
assert!(suppressor.should_suppress_enclosure('\'', &context));
let context = create_context("", "Hello", 0, "'Hello");
assert!(!suppressor.should_suppress_enclosure('\'', &context));
let context = create_context("said ", "", 10, "said '");
assert!(!suppressor.should_suppress_enclosure('\'', &context));
}
#[test]
fn test_possessive_detection() {
let suppressor = EnglishEnclosureSuppressor::new();
let context = create_context("James", " ", 10, "James' ");
assert!(suppressor.should_suppress_enclosure('\'', &context));
let context = create_context("90s", ".", 10, "90s'.");
assert!(suppressor.should_suppress_enclosure('\'', &context));
let context = create_context("", "90s", 0, "'90s");
assert!(!suppressor.is_possessive_or_plural(&context));
}
#[test]
fn test_year_abbreviations() {
let suppressor = EnglishEnclosureSuppressor::new();
let context = create_context(" ", "90s", 10, " '90s");
assert!(suppressor.should_suppress_enclosure('\'', &context));
let context = create_context("", "60s", 0, "'60s");
assert!(suppressor.should_suppress_enclosure('\'', &context));
let context = create_context("\t", "20s", 5, "\t'20s");
assert!(suppressor.should_suppress_enclosure('\'', &context));
let context = create_context(" ", "90s", 10, " '90s");
assert!(suppressor.should_suppress_enclosure('\u{2019}', &context));
let context = create_context("a", "90s", 10, "a'90s");
assert!(!suppressor.should_suppress_enclosure('\'', &context));
let context = create_context(" ", "9s", 10, " '9s");
assert!(!suppressor.should_suppress_enclosure('\'', &context));
let context = create_context(" ", "90a", 10, " '90a");
assert!(!suppressor.should_suppress_enclosure('\'', &context));
let context = create_context(" ", "900s", 10, " '900s");
assert!(!suppressor.should_suppress_enclosure('\'', &context));
}
#[test]
fn test_measurement_marks() {
let suppressor = EnglishEnclosureSuppressor::new();
let context = create_context("5", "9", 10, "5'9");
assert!(suppressor.should_suppress_enclosure('\'', &context));
let context = create_context("9", "", 10, "5'9\"");
assert!(suppressor.should_suppress_enclosure('"', &context));
let context = create_context("45°", "30", 10, "45°'30");
assert!(suppressor.should_suppress_enclosure('\'', &context));
let context = create_context("30'", "", 10, "45°30'\"");
assert!(suppressor.should_suppress_enclosure('"', &context));
}
#[test]
fn test_list_item_parentheses() {
let suppressor = EnglishEnclosureSuppressor::new();
let context = create_context("1", " ", 1, "1) ");
assert!(suppressor.should_suppress_enclosure(')', &context));
let context = create_context(" a", " ", 3, " a) ");
assert!(suppressor.should_suppress_enclosure(')', &context));
let context = create_context("1.", " ", 2, "1.) ");
assert!(suppressor.should_suppress_enclosure(')', &context));
let context = create_context("(text", " ", 20, "(text) ");
assert!(!suppressor.should_suppress_enclosure(')', &context));
let context = create_context("1", " ", 20, "some text 1) ");
assert!(!suppressor.should_suppress_enclosure(')', &context));
}
#[test]
fn test_unicode_apostrophes() {
let suppressor = EnglishEnclosureSuppressor::new();
let context = create_context("isn", "t", 10, "isn't");
assert!(suppressor.should_suppress_enclosure('\u{2019}', &context));
let context = create_context("I", "m", 5, "I'm");
assert!(suppressor.should_suppress_enclosure('\u{2019}', &context));
}
}