use super::{NFAChar, NFA};
#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
pub enum BoundaryKind {
WordStart,
WordEnd,
}
impl BoundaryKind {
pub fn matches_at(&self, text: &str, pos: usize) -> bool {
match self {
BoundaryKind::WordStart => {
pos == 0 || Self::is_word_boundary_char(text.chars().nth(pos.saturating_sub(1)))
}
BoundaryKind::WordEnd => {
pos >= text.len() || Self::is_word_boundary_char(text.chars().nth(pos))
}
}
}
pub fn matches_at_bytes(&self, text: &[u8], pos: usize) -> bool {
match self {
BoundaryKind::WordStart => {
pos == 0 || Self::is_word_boundary_byte(text.get(pos.saturating_sub(1)).copied())
}
BoundaryKind::WordEnd => {
pos >= text.len() || Self::is_word_boundary_byte(text.get(pos).copied())
}
}
}
fn is_word_boundary_char(c: Option<char>) -> bool {
match c {
None => true, Some(c) => !c.is_alphanumeric() && c != '_',
}
}
fn is_word_boundary_byte(b: Option<u8>) -> bool {
match b {
None => true,
Some(b) => !b.is_ascii_alphanumeric() && b != b'_',
}
}
}
#[derive(Debug, Clone)]
pub struct ContextMatcherChar {
pub left: Option<ContextPatternChar>,
pub right: Option<ContextPatternChar>,
}
#[derive(Debug, Clone)]
pub enum ContextPatternChar {
Nfa(NFAChar),
Boundary(BoundaryKind),
And(Box<ContextPatternChar>, Box<ContextPatternChar>),
Or(Box<ContextPatternChar>, Box<ContextPatternChar>),
Not(Box<ContextPatternChar>),
}
impl ContextPatternChar {
pub fn accepts(&self, input: &str) -> bool {
match self {
ContextPatternChar::Nfa(nfa) => nfa.accepts(input),
ContextPatternChar::Boundary(kind) => match kind {
BoundaryKind::WordStart => input.is_empty(),
BoundaryKind::WordEnd => input.is_empty(),
},
ContextPatternChar::And(a, b) => a.accepts(input) && b.accepts(input),
ContextPatternChar::Or(a, b) => a.accepts(input) || b.accepts(input),
ContextPatternChar::Not(inner) => !inner.accepts(input),
}
}
}
impl ContextMatcherChar {
pub fn new(left: Option<NFAChar>, right: Option<NFAChar>) -> Self {
Self {
left: left.map(ContextPatternChar::Nfa),
right: right.map(ContextPatternChar::Nfa),
}
}
pub fn word_start() -> Self {
Self {
left: Some(ContextPatternChar::Boundary(BoundaryKind::WordStart)),
right: None,
}
}
pub fn word_end() -> Self {
Self {
left: None,
right: Some(ContextPatternChar::Boundary(BoundaryKind::WordEnd)),
}
}
pub fn none() -> Self {
Self {
left: None,
right: None,
}
}
pub fn from_compiled(left_nfa: Option<NFAChar>, right_nfa: Option<NFAChar>) -> Self {
let left = left_nfa.map(|nfa| {
if Self::is_word_boundary_nfa(&nfa) {
ContextPatternChar::Boundary(BoundaryKind::WordStart)
} else {
ContextPatternChar::Nfa(nfa)
}
});
let right = right_nfa.map(|nfa| {
if Self::is_word_boundary_nfa(&nfa) {
ContextPatternChar::Boundary(BoundaryKind::WordEnd)
} else {
ContextPatternChar::Nfa(nfa)
}
});
Self { left, right }
}
fn is_word_boundary_nfa(nfa: &NFAChar) -> bool {
nfa.accepts("") && nfa.transitions().iter().all(|t| t.label.is_epsilon())
}
pub fn matches_at(&self, text: &str, match_start: usize, match_end: usize) -> bool {
if let Some(ref left_pattern) = self.left {
if !self.matches_left_context(left_pattern, text, match_start) {
return false;
}
}
if let Some(ref right_pattern) = self.right {
if !self.matches_right_context(right_pattern, text, match_end) {
return false;
}
}
true
}
fn matches_left_context(
&self,
pattern: &ContextPatternChar,
text: &str,
match_start: usize,
) -> bool {
match pattern {
ContextPatternChar::Boundary(kind) => kind.matches_at(text, match_start),
ContextPatternChar::Nfa(nfa) => {
let prefix: String = text.chars().take(match_start).collect();
for start in 0..=prefix.chars().count() {
let suffix: String = prefix.chars().skip(start).collect();
if nfa.accepts(&suffix) {
return true;
}
}
false
}
ContextPatternChar::And(a, b) => {
self.matches_left_context(a, text, match_start)
&& self.matches_left_context(b, text, match_start)
}
ContextPatternChar::Or(a, b) => {
self.matches_left_context(a, text, match_start)
|| self.matches_left_context(b, text, match_start)
}
ContextPatternChar::Not(inner) => !self.matches_left_context(inner, text, match_start),
}
}
fn matches_right_context(
&self,
pattern: &ContextPatternChar,
text: &str,
match_end: usize,
) -> bool {
match pattern {
ContextPatternChar::Boundary(kind) => kind.matches_at(text, match_end),
ContextPatternChar::Nfa(nfa) => {
let suffix: String = text.chars().skip(match_end).collect();
for len in 0..=suffix.chars().count() {
let prefix: String = suffix.chars().take(len).collect();
if nfa.accepts(&prefix) {
return true;
}
}
false
}
ContextPatternChar::And(a, b) => {
self.matches_right_context(a, text, match_end)
&& self.matches_right_context(b, text, match_end)
}
ContextPatternChar::Or(a, b) => {
self.matches_right_context(a, text, match_end)
|| self.matches_right_context(b, text, match_end)
}
ContextPatternChar::Not(inner) => !self.matches_right_context(inner, text, match_end),
}
}
}
#[derive(Debug, Clone)]
pub struct ContextMatcher {
pub left: Option<ContextPattern>,
pub right: Option<ContextPattern>,
}
#[derive(Debug, Clone)]
pub enum ContextPattern {
Nfa(NFA),
Boundary(BoundaryKind),
And(Box<ContextPattern>, Box<ContextPattern>),
Or(Box<ContextPattern>, Box<ContextPattern>),
Not(Box<ContextPattern>),
}
impl ContextMatcher {
pub fn new(left: Option<NFA>, right: Option<NFA>) -> Self {
Self {
left: left.map(ContextPattern::Nfa),
right: right.map(ContextPattern::Nfa),
}
}
pub fn word_start() -> Self {
Self {
left: Some(ContextPattern::Boundary(BoundaryKind::WordStart)),
right: None,
}
}
pub fn word_end() -> Self {
Self {
left: None,
right: Some(ContextPattern::Boundary(BoundaryKind::WordEnd)),
}
}
pub fn none() -> Self {
Self {
left: None,
right: None,
}
}
pub fn from_compiled(left_nfa: Option<NFA>, right_nfa: Option<NFA>) -> Self {
let left = left_nfa.map(|nfa| {
if Self::is_word_boundary_nfa(&nfa) {
ContextPattern::Boundary(BoundaryKind::WordStart)
} else {
ContextPattern::Nfa(nfa)
}
});
let right = right_nfa.map(|nfa| {
if Self::is_word_boundary_nfa(&nfa) {
ContextPattern::Boundary(BoundaryKind::WordEnd)
} else {
ContextPattern::Nfa(nfa)
}
});
Self { left, right }
}
fn is_word_boundary_nfa(nfa: &NFA) -> bool {
nfa.accepts(b"") && nfa.transitions().iter().all(|t| t.label.is_epsilon())
}
pub fn matches_at(&self, text: &[u8], match_start: usize, match_end: usize) -> bool {
if let Some(ref left_pattern) = self.left {
if !self.matches_left_context(left_pattern, text, match_start) {
return false;
}
}
if let Some(ref right_pattern) = self.right {
if !self.matches_right_context(right_pattern, text, match_end) {
return false;
}
}
true
}
fn matches_left_context(
&self,
pattern: &ContextPattern,
text: &[u8],
match_start: usize,
) -> bool {
match pattern {
ContextPattern::Boundary(kind) => kind.matches_at_bytes(text, match_start),
ContextPattern::Nfa(nfa) => {
let prefix = &text[..match_start];
for start in 0..=prefix.len() {
let suffix = &prefix[start..];
if nfa.accepts(suffix) {
return true;
}
}
false
}
ContextPattern::And(a, b) => {
self.matches_left_context(a, text, match_start)
&& self.matches_left_context(b, text, match_start)
}
ContextPattern::Or(a, b) => {
self.matches_left_context(a, text, match_start)
|| self.matches_left_context(b, text, match_start)
}
ContextPattern::Not(inner) => !self.matches_left_context(inner, text, match_start),
}
}
fn matches_right_context(
&self,
pattern: &ContextPattern,
text: &[u8],
match_end: usize,
) -> bool {
match pattern {
ContextPattern::Boundary(kind) => kind.matches_at_bytes(text, match_end),
ContextPattern::Nfa(nfa) => {
let suffix = &text[match_end..];
for len in 0..=suffix.len() {
let prefix = &suffix[..len];
if nfa.accepts(prefix) {
return true;
}
}
false
}
ContextPattern::And(a, b) => {
self.matches_right_context(a, text, match_end)
&& self.matches_right_context(b, text, match_end)
}
ContextPattern::Or(a, b) => {
self.matches_right_context(a, text, match_end)
|| self.matches_right_context(b, text, match_end)
}
ContextPattern::Not(inner) => !self.matches_right_context(inner, text, match_end),
}
}
}
#[derive(Debug, Clone)]
pub struct ContextualRewriteRuleChar {
pub source: NFAChar,
pub replacement: Vec<char>,
pub context: ContextMatcherChar,
pub weight: f64,
}
impl ContextualRewriteRuleChar {
pub fn new(
source: NFAChar,
replacement: Vec<char>,
left_context: Option<NFAChar>,
right_context: Option<NFAChar>,
weight: f64,
) -> Self {
Self {
source,
replacement,
context: ContextMatcherChar::from_compiled(left_context, right_context),
weight,
}
}
pub fn can_apply_at(&self, text: &str, match_start: usize, match_end: usize) -> bool {
let substring: String = text
.chars()
.skip(match_start)
.take(match_end - match_start)
.collect();
if !self.source.accepts(&substring) {
return false;
}
self.context.matches_at(text, match_start, match_end)
}
pub fn apply_at(&self, text: &str, match_start: usize, match_end: usize) -> String {
let chars: Vec<char> = text.chars().collect();
let mut result: Vec<char> = chars[..match_start].to_vec();
result.extend(&self.replacement);
result.extend(&chars[match_end..]);
result.into_iter().collect()
}
pub fn find_first_match(&self, text: &str, start_from: usize) -> Option<(usize, usize)> {
let chars: Vec<char> = text.chars().collect();
for start in start_from..chars.len() {
for end in (start + 1)..=chars.len() {
let substring: String = chars[start..end].iter().collect();
if self.source.accepts(&substring) && self.context.matches_at(text, start, end) {
return Some((start, end));
}
}
}
None
}
}
#[derive(Debug, Clone)]
pub struct ContextualRewriteRule {
pub source: NFA,
pub replacement: Vec<u8>,
pub context: ContextMatcher,
pub weight: f64,
}
impl ContextualRewriteRule {
pub fn new(
source: NFA,
replacement: Vec<u8>,
left_context: Option<NFA>,
right_context: Option<NFA>,
weight: f64,
) -> Self {
Self {
source,
replacement,
context: ContextMatcher::from_compiled(left_context, right_context),
weight,
}
}
pub fn can_apply_at(&self, text: &[u8], match_start: usize, match_end: usize) -> bool {
let substring = &text[match_start..match_end];
if !self.source.accepts(substring) {
return false;
}
self.context.matches_at(text, match_start, match_end)
}
pub fn apply_at(&self, text: &[u8], match_start: usize, match_end: usize) -> Vec<u8> {
let mut result = text[..match_start].to_vec();
result.extend(&self.replacement);
result.extend(&text[match_end..]);
result
}
pub fn find_first_match(&self, text: &[u8], start_from: usize) -> Option<(usize, usize)> {
for start in start_from..text.len() {
for end in (start + 1)..=text.len() {
let substring = &text[start..end];
if self.source.accepts(substring) && self.context.matches_at(text, start, end) {
return Some((start, end));
}
}
}
None
}
}
#[cfg(test)]
mod tests {
use super::*;
use crate::phonetic::nfa::compiler::{compile, compile_bytes};
use crate::phonetic::regex::{parse, parse_bytes};
#[test]
fn test_word_start_at_beginning() {
assert!(BoundaryKind::WordStart.matches_at("hello", 0));
}
#[test]
fn test_word_start_after_space() {
assert!(BoundaryKind::WordStart.matches_at("hello world", 6));
}
#[test]
fn test_word_start_mid_word() {
assert!(!BoundaryKind::WordStart.matches_at("hello", 2));
}
#[test]
fn test_word_end_at_end() {
assert!(BoundaryKind::WordEnd.matches_at("hello", 5));
}
#[test]
fn test_word_end_before_space() {
assert!(BoundaryKind::WordEnd.matches_at("hello world", 5));
}
#[test]
fn test_word_end_mid_word() {
assert!(!BoundaryKind::WordEnd.matches_at("hello", 2));
}
#[test]
fn test_context_matcher_none() {
let matcher = ContextMatcherChar::none();
assert!(matcher.matches_at("anything", 0, 8));
}
#[test]
fn test_context_matcher_word_start() {
let matcher = ContextMatcherChar::word_start();
assert!(matcher.matches_at("hello", 0, 1));
assert!(!matcher.matches_at("hello", 2, 3));
assert!(matcher.matches_at("hello world", 6, 7));
}
#[test]
fn test_context_matcher_word_end() {
let matcher = ContextMatcherChar::word_end();
assert!(matcher.matches_at("hello", 4, 5));
assert!(!matcher.matches_at("hello", 1, 2));
assert!(matcher.matches_at("hello world", 4, 5));
}
#[test]
fn test_context_matcher_lookahead() {
let nfa = compile(&parse("[ei]").expect("test fixture: parse must be Ok"))
.expect("test fixture: compile must be Ok");
let matcher = ContextMatcherChar::new(None, Some(nfa));
assert!(matcher.matches_at("city", 0, 1));
assert!(matcher.matches_at("cent", 0, 1));
assert!(!matcher.matches_at("cat", 0, 1));
}
#[test]
fn test_context_matcher_lookbehind() {
let nfa = compile(&parse("[aeiou]").expect("test fixture: parse must be Ok"))
.expect("test fixture: compile must be Ok");
let matcher = ContextMatcherChar::new(Some(nfa), None);
assert!(matcher.matches_at("roses", 2, 3));
assert!(!matcher.matches_at("star", 0, 1));
}
#[test]
fn test_context_matcher_both_contexts() {
let left_nfa = compile(&parse("[aeiou]").expect("test fixture: parse must be Ok"))
.expect("test fixture: compile must be Ok");
let right_nfa = compile(&parse("[aeiou]").expect("test fixture: parse must be Ok"))
.expect("test fixture: compile must be Ok");
let matcher = ContextMatcherChar::new(Some(left_nfa), Some(right_nfa));
assert!(matcher.matches_at("roses", 2, 3));
assert!(!matcher.matches_at("star", 0, 1));
assert!(!matcher.matches_at("fast", 2, 3));
}
#[test]
fn test_contextual_rule_simple() {
let source = compile(&parse("ph").expect("test fixture: parse must be Ok"))
.expect("test fixture: compile must be Ok");
let rule = ContextualRewriteRuleChar::new(source, vec!['f'], None, None, 0.0);
assert!(rule.can_apply_at("phone", 0, 2));
assert_eq!(rule.apply_at("phone", 0, 2), "fone");
}
#[test]
fn test_contextual_rule_with_lookahead() {
let source = compile(&parse("c").expect("test fixture: parse must be Ok"))
.expect("test fixture: compile must be Ok");
let right = compile(&parse("[ei]").expect("test fixture: parse must be Ok"))
.expect("test fixture: compile must be Ok");
let rule = ContextualRewriteRuleChar::new(source, vec!['s'], None, Some(right), 0.0);
assert!(rule.can_apply_at("city", 0, 1));
assert_eq!(rule.apply_at("city", 0, 1), "sity");
assert!(!rule.can_apply_at("cat", 0, 1));
}
#[test]
fn test_contextual_rule_find_first_match() {
let source = compile(&parse("c").expect("test fixture: parse must be Ok"))
.expect("test fixture: compile must be Ok");
let right = compile(&parse("[ei]").expect("test fixture: parse must be Ok"))
.expect("test fixture: compile must be Ok");
let rule = ContextualRewriteRuleChar::new(source, vec!['s'], None, Some(right), 0.0);
let result = rule.find_first_match("soccer", 0);
assert_eq!(result, Some((3, 4))); }
#[test]
fn test_contextual_rule_word_boundary() {
let source = compile(&parse("e").expect("test fixture: parse must be Ok"))
.expect("test fixture: compile must be Ok");
let rule = ContextualRewriteRuleChar {
source,
replacement: vec![],
context: ContextMatcherChar::word_end(),
weight: 0.0,
};
assert!(rule.can_apply_at("phone", 4, 5));
assert_eq!(rule.apply_at("phone", 4, 5), "phon");
assert!(!rule.can_apply_at("phonetic", 4, 5));
}
#[test]
fn test_byte_context_matcher_none() {
let matcher = ContextMatcher::none();
assert!(matcher.matches_at(b"anything", 0, 8));
}
#[test]
fn test_byte_context_matcher_word_start() {
let matcher = ContextMatcher::word_start();
assert!(matcher.matches_at(b"hello", 0, 1));
assert!(!matcher.matches_at(b"hello", 2, 3));
}
#[test]
fn test_byte_context_matcher_lookahead() {
let nfa = compile_bytes(&parse_bytes(b"[ei]").expect("test fixture: parse must be Ok"))
.expect("test fixture: compile must be Ok");
let matcher = ContextMatcher::new(None, Some(nfa));
assert!(matcher.matches_at(b"city", 0, 1));
assert!(!matcher.matches_at(b"cat", 0, 1));
}
#[test]
fn test_byte_contextual_rule() {
let source = compile_bytes(&parse_bytes(b"ph").expect("test fixture: parse must be Ok"))
.expect("test fixture: compile must be Ok");
let rule = ContextualRewriteRule::new(source, vec![b'f'], None, None, 0.0);
assert!(rule.can_apply_at(b"phone", 0, 2));
assert_eq!(rule.apply_at(b"phone", 0, 2), b"fone");
}
}