pub mod compound;
pub mod contraction;
pub mod engine;
pub(crate) mod korean_context;
pub mod parser;
pub mod pronunciation;
pub mod rule_10_1;
pub mod rule_10_11;
pub mod rule_10_13;
pub mod rule_10_2;
pub mod rule_10_3;
pub mod rule_10_4;
pub mod rule_10_5;
pub mod rule_10_6;
pub mod rule_10_6_8;
pub mod rule_10_6_middle;
pub mod rule_10_6_restricted;
pub mod rule_10_7;
pub mod rule_10_7_pron;
pub mod rule_10_7_struct;
pub mod rule_10_8;
pub mod rule_10_9;
pub mod rule_10_9_list;
pub mod rule_11;
pub mod rule_12;
pub mod rule_13;
pub mod rule_14;
pub mod rule_15;
pub mod rule_16;
pub mod rule_3;
pub mod rule_3_24;
pub mod rule_4;
pub mod rule_5_7;
pub mod rule_6;
pub mod rule_7;
pub mod rule_9;
pub mod span;
pub mod standing_alone;
pub mod token;
use engine::EnglishUebEngine;
pub fn try_encode(text: &str) -> Option<Vec<u8>> {
use unicode_normalization::UnicodeNormalization;
let composed: String = text.nfc().collect();
if !is_ueb_eligible(&composed) {
return None;
}
encode_english(&composed, content_route_uses_document_english(&composed))
}
pub fn encode_forced(text: &str) -> Option<Vec<u8>> {
encode_english(text, true)
}
fn encode_english(text: &str, explicit_english: bool) -> Option<Vec<u8>> {
use unicode_normalization::UnicodeNormalization;
let composed: String = text.nfc().collect();
if let Some(cells) = encode_struck_ligature_text(&composed) {
return Some(cells);
}
if let Some(cells) = encode_single_caron_word(&composed, explicit_english) {
return Some(cells);
}
if let Some(cells) = rule_14::table_language_identifier(&composed) {
return Some(cells);
}
if let Some(cells) = rule_14::encode_with_code_switches(&composed, |segment| {
let tokens = parser::parse_english(segment);
if tokens.is_empty() {
Some(Vec::new())
} else {
EnglishUebEngine::new().encode(&tokens, explicit_english)
}
}) {
return Some(cells);
}
let tokens = parser::parse_english(&composed);
if tokens.is_empty() {
return None;
}
EnglishUebEngine::new().encode(&tokens, explicit_english)
}
fn content_route_uses_document_english(text: &str) -> bool {
has_inline_dollar_math_in_prose(text)
|| parenthesized_digit_group_before_number(&text.chars().collect::<Vec<_>>())
}
fn encode_struck_ligature_text(text: &str) -> Option<Vec<u8>> {
if !text.contains('\u{0336}') {
return None;
}
let chars: Vec<char> = text.chars().collect();
let mut out = Vec::new();
let mut i = 0;
while i < chars.len() {
if is_ligature_letter(chars[i]) && chars.get(i + 1) == Some(&'\u{0336}') {
let second = *chars.get(i + 2)?;
if !is_ligature_letter(second) || chars.get(i + 3) != Some(&'\u{0336}') {
return None;
}
push_rule4_letter(chars[i], &mut out)?;
if second.is_uppercase() {
out.push(crate::unicode::decode_unicode('⠠'));
}
out.extend([
crate::unicode::decode_unicode('⠘'),
crate::unicode::decode_unicode('⠖'),
]);
push_rule4_letter_without_leading_cap(second, &mut out)?;
i += 4;
continue;
}
match chars[i] {
' ' => out.push(0),
'?' => out.push(crate::unicode::decode_unicode('⠦')),
c if is_ligature_letter(c) => push_rule4_letter(c, &mut out)?,
_ => return None,
}
i += 1;
}
Some(out)
}
fn encode_single_caron_word(text: &str, _explicit_english: bool) -> Option<Vec<u8>> {
let has_caron = text
.chars()
.any(|c| matches!(c, 'č' | 'Č' | 'ě' | 'Ě' | 'ř' | 'Ř' | 'š' | 'Š' | 'ž' | 'Ž'));
if !has_caron || !text.chars().all(is_ligature_letter) {
return None;
}
let mut out = Vec::new();
for c in text.chars() {
push_rule4_letter(c, &mut out)?;
}
Some(out)
}
fn is_ligature_letter(c: char) -> bool {
c.is_ascii_alphabetic() || rule_4::is_modified_letter(c)
}
fn push_rule4_letter(c: char, out: &mut Vec<u8>) -> Option<()> {
if let Some(cells) = rule_4::accent_cells(c) {
out.extend(cells);
} else {
if c.is_uppercase() {
out.push(crate::unicode::decode_unicode('⠠'));
}
out.push(crate::english::encode_english(c.to_ascii_lowercase()).ok()?);
}
Some(())
}
fn push_rule4_letter_without_leading_cap(c: char, out: &mut Vec<u8>) -> Option<()> {
if let Some(cells) = rule_4::accent_cells(c) {
let cells =
if c.is_uppercase() && cells.first() == Some(&crate::unicode::decode_unicode('⠠')) {
&cells[1..]
} else {
cells.as_slice()
};
out.extend(cells);
} else {
out.push(crate::english::encode_english(c.to_ascii_lowercase()).ok()?);
}
Some(())
}
pub fn is_ueb_eligible(text: &str) -> bool {
text.chars().any(|c| {
c.is_ascii_alphabetic()
|| c == '\u{0332}'
|| rule_9::decode_styled(c).is_some()
|| rule_9::decode_small_cap(c).is_some()
|| matches!(c, '\u{00A2}' | '\u{00A3}' | '\u{00A5}')
|| matches!(
c,
'\u{00A9}' | '\u{00AE}' | '\u{2122}' | '\u{0E3F}' | '\u{2713}' | '\u{2740}'
)
})
|| (text.chars().any(|c| matches!(c, '\u{20AC}' | '\u{20A3}'))
&& text.chars().any(|c| c.is_ascii_digit())
&& text.contains(" = "))
|| {
let chars: Vec<char> = text.chars().collect();
chars
.windows(2)
.any(|w| rule_16::is_line_char(w[0]) && rule_16::is_line_char(w[1]))
|| chars
.windows(2)
.any(|w| w[0].is_ascii_digit() && matches!(w[1], '\'' | '"'))
|| chars
.windows(3)
.any(|w| w[0].is_ascii_digit() && w[1] == ' ' && w[2].is_ascii_digit())
|| chars
.windows(3)
.any(|w| w[0] == '$' && w[1] == ' ' && w[2].is_ascii_digit())
|| parenthesized_digit_group_before_number(&chars)
}
}
fn parenthesized_digit_group_before_number(chars: &[char]) -> bool {
matches!(chars.first(), Some('('))
&& chars.iter().position(|c| *c == ')').is_some_and(|close| {
close > 1
&& chars[1..close].iter().all(|c| c.is_ascii_digit())
&& matches!(chars.get(close + 1), Some(' '))
&& chars.get(close + 2).is_some_and(|c| c.is_ascii_digit())
})
}
pub fn is_math_owned(text: &str) -> bool {
if text.len() >= 2 && text.starts_with('$') && text.ends_with('$') {
return true;
}
if has_inline_dollar_math_in_prose(text) {
return false;
}
if text.chars().any(|c| {
matches!(
c,
'\u{2295}'
| '\u{21D2}'
| '\u{2194}'
| '→'
| '←'
| '↗'
| '↘'
| '↑'
| '↓'
| '△'
| '□'
| '′'
| '″'
| '|'
| '‖'
| '\u{0304}'
| '\u{0302}'
)
}) {
let chars: Vec<char> = text.chars().collect();
let has_line_run = chars
.windows(2)
.any(|w| rule_16::is_line_char(w[0]) && rule_16::is_line_char(w[1]));
let mut run = 0usize;
let mut longest_lower_run = 0usize;
for c in text.chars() {
if c.is_ascii_lowercase() {
run += 1;
longest_lower_run = longest_lower_run.max(run);
} else {
run = 0;
}
}
if !has_line_run && longest_lower_run < 3 {
return true;
}
}
let cells: Vec<char> = text.chars().collect();
if cells.iter().enumerate().any(|(i, &c)| {
matches!(c, '=' | '<' | '>')
&& !is_angle_bracket_prose(&cells, i)
&& (i.checked_sub(1).is_some_and(|j| cells[j] != ' ')
|| cells.get(i + 1).is_some_and(|&n| n != ' '))
}) {
return true;
}
if !text.contains(' ')
&& text.chars().any(rule_3_24::is_script_char)
&& longest_ascii_letter_run(text) < 3
{
return true;
}
let after_coeff = text.trim_start_matches(|c: char| c.is_ascii_digit());
if let Some((name, _)) = crate::rules::math::function::match_function_prefix(after_coeff) {
let rest = &after_coeff[name.len()..];
let rest_is_math_arg = rest.is_empty()
|| !rest.starts_with(|c: char| c.is_ascii_alphabetic())
|| rest.starts_with(|c: char| c.is_ascii_uppercase())
|| (!rest.contains(' ') && rest.chars().any(rule_3_24::is_script_char))
|| rest.chars().all(|c| {
c.is_ascii_alphabetic()
&& !matches!(c.to_ascii_lowercase(), 'a' | 'e' | 'i' | 'o' | 'u')
});
if rest_is_math_arg {
return true;
}
}
if let Some(open) = text.find('(') {
let inner = &text[open + 1..];
let close = inner.find(')').unwrap_or(inner.len());
let bracketed = &inner[..close];
let before_math = text[..open]
.chars()
.next_back()
.is_some_and(|c| c.is_ascii_alphanumeric());
let after_math = inner
.get(close + ')'.len_utf8()..)
.and_then(|tail| tail.chars().next())
.is_some_and(|c| !c.is_whitespace());
if bracketed.chars().any(|c| c.is_ascii_digit())
&& !bracketed.contains(' ')
&& !bracketed.contains('$')
&& (before_math || after_math)
{
return true;
}
}
if text.contains("://") || text.contains('\\') {
return true;
}
if text.contains(' ')
|| text.contains('-')
|| text.contains('@')
|| longest_ascii_letter_run(text) >= 4
{
return false;
}
let lower = text.to_ascii_lowercase();
let is_ordinal = ["st", "nd", "rd", "th"].iter().any(|suf| {
lower.ends_with(suf) && lower[..lower.len() - 2].chars().all(|c| c.is_ascii_digit())
});
if is_ordinal {
return false;
}
let chars: Vec<char> = text.chars().collect();
chars
.windows(3)
.any(|w| w[0].is_ascii_digit() && w[1].is_ascii_alphabetic() && w[2].is_ascii_alphabetic())
}
fn longest_ascii_letter_run(text: &str) -> usize {
let mut run = 0usize;
let mut longest = 0usize;
for c in text.chars() {
if c.is_ascii_alphabetic() {
run += 1;
longest = longest.max(run);
} else {
run = 0;
}
}
longest
}
fn has_inline_dollar_math_in_prose(text: &str) -> bool {
let trimmed = text.trim();
if trimmed.starts_with('$') && trimmed.ends_with('$') && trimmed.matches('$').count() == 2 {
return false;
}
let mut in_span = false;
let mut has_span = false;
let mut outside_letters = 0usize;
for c in text.chars() {
if c == '$' {
if in_span {
has_span = true;
}
in_span = !in_span;
} else if !in_span && c.is_ascii_alphabetic() {
outside_letters += 1;
}
}
has_span && outside_letters >= 3
}
fn is_angle_bracket_prose(chars: &[char], index: usize) -> bool {
match chars[index] {
'<' => {
let starts_after_boundary = index == 0 || chars[index - 1].is_whitespace();
starts_after_boundary && matching_prose_angle_close(chars, index).is_some()
}
'>' => chars[..index]
.iter()
.enumerate()
.rev()
.find(|(_, c)| **c == '<')
.is_some_and(|(open, _)| is_angle_bracket_prose(chars, open)),
_ => false,
}
}
fn matching_prose_angle_close(chars: &[char], open: usize) -> Option<usize> {
let close = chars
.iter()
.enumerate()
.skip(open + 1)
.find_map(|(i, c)| if *c == '>' { Some(i) } else { None })?;
let before_close = chars.get(close.checked_sub(1)?).copied()?;
let after_close = chars.get(close + 1).copied();
let closes_before_boundary =
after_close.is_none_or(|c| c.is_whitespace() || c.is_ascii_punctuation());
if before_close != '<' && closes_before_boundary {
Some(close)
} else {
None
}
}
#[cfg(test)]
mod is_math_owned_tests {
use super::{
encode_english, encode_struck_ligature_text, has_inline_dollar_math_in_prose, is_math_owned,
};
use crate::unicode::decode_unicode;
#[rstest::rstest]
#[case::sin("sin")]
#[case::cos("cos")]
#[case::sinh("sinh")]
#[case::log2("log2")]
#[case::two_log7("2log7")]
#[case::sin3x("sin3x")]
#[case::sinxy("sinxy")] #[case::two_cosx("2cosx")] #[case::three_ab("3ab")] #[case::f_paren("f(x-1)")] #[case::factorial("(3n)!")]
#[case::eq_relation("ax=b")]
#[case::gt_relation("a>b")]
#[case::lt_relation("x<0")]
#[case::eq_func("y=f(x)")]
#[case::set_eq("A={2, 4, 6, ...}")] #[case::interval("-1<x<3")]
#[case::vars_equal("VarsEqual=(x==y);")]
#[case::script_c_squared("c\u{00B2}")]
#[case::script_x_sub2("x\u{2082}")]
#[case::script_chemical("H\u{2082}O")]
#[case::script_unit("4m\u{00B2}")]
#[case::script_cube_root("\u{00B3}\u{221A}x\u{00B3}")]
#[case::script_log_sub2("log\u{2082}(x+1)")]
#[case::spaced_right_arrow("p → q")]
#[case::prime("x′")]
#[case::absolute_value("|x|")]
#[case::triangle_name("△ABC")]
#[case::combining_hat("p\u{0302}")]
fn math_owned_inputs_are_blocked(#[case] text: &str) {
assert!(is_math_owned(text), "{text:?} should be math-owned");
}
#[rstest::rstest]
#[case::singe("singe")] #[case::singeing("singeing")]
#[case::arccosine("arccosine")] #[case::ordinal_2nd("2nd")]
#[case::ordinal_3rd("3rd")]
#[case::unit_3b("3b")] #[case::cents_99c("99c")]
#[case::hyphenated("child-ish-ly")]
#[case::sentence("That is quite fair.")]
#[case::plain_word("cat")]
#[case::spaced_eq("a = b")]
#[case::spaced_lt("positron < posi")]
#[case::spaced_sum("as easy as 2 + 2 = 4")]
#[case::paren_phrase_billion("$2bn (2 billion dollars)")]
#[case::paren_phrase_escudos("20$00 (20 escudos)")]
#[case::paren_phrase_pounds("\u{00A3}3m (3 million pounds)")]
#[case::paren_phrase_enough("Buy meat (enough for 2).")]
#[case::script_footnote("knowledge.\u{00B3}")]
#[case::script_word_subscripts("mass\u{209B}\u{1D64}\u{2099}")]
#[case::angle_phrase("<in file>")]
#[case::angle_variables("<x, y>")]
#[case::angle_email("<J.Child@children.net>")]
#[case::angle_email_after_name("Jan Swan <swanj@iafrica.com>")]
#[case::phone_area("phone: (61) 3 1234 5678")]
#[case::currency_paren("Balance: ($52.68)")]
#[case::phone_number("(416) 486-2500")]
#[case::shopping4you("shopping4you")]
#[case::address_digit_word("4starhotel@webnet.com")]
#[case::inline_nemeth_prose("The result will be in the form $(ax+by)(cx+dy)$, where $ac=12$.")]
fn english_inputs_are_not_blocked(#[case] text: &str) {
assert!(!is_math_owned(text), "{text:?} should NOT be math-owned");
}
#[test]
fn struck_ligature_handles_uppercase_second_letter() {
assert_eq!(
encode_struck_ligature_text("a\u{0336}B\u{0336}"),
Some(vec![
decode_unicode('⠁'),
decode_unicode('⠠'),
decode_unicode('⠘'),
decode_unicode('⠖'),
decode_unicode('⠃'),
])
);
}
#[test]
fn struck_ligature_rejects_unmarked_second_letter() {
assert_eq!(encode_struck_ligature_text("a\u{0336}B"), None);
}
#[test]
fn struck_ligature_keeps_capital_for_accented_second_letter() {
assert_eq!(
encode_struck_ligature_text("a\u{0336}É\u{0336}"),
Some(vec![
decode_unicode('⠁'),
decode_unicode('⠠'),
decode_unicode('⠘'),
decode_unicode('⠖'),
decode_unicode('⠘'),
decode_unicode('⠌'),
decode_unicode('⠑'),
])
);
}
#[test]
fn code_switch_closure_encodes_non_empty_english_segment() {
let encoded = encode_english("word قُ", true).expect("Arabic code switch should encode");
assert!(encoded.contains(&decode_unicode('⠺')));
assert!(encoded.iter().any(|cell| *cell != 0));
}
#[test]
fn plain_english_route_uses_engine_after_parsing_tokens() {
let input = std::hint::black_box("cat");
assert_eq!(
encode_english(input, true),
Some(vec![
decode_unicode('⠉'),
decode_unicode('⠁'),
decode_unicode('⠞')
])
);
}
#[test]
fn full_dollar_span_is_not_inline_prose_math() {
assert!(!has_inline_dollar_math_in_prose("$x+1$"));
}
}
#[cfg(test)]
mod is_ueb_eligible_tests {
use super::is_ueb_eligible;
#[rstest::rstest]
#[case::cent_amount("10\u{00A2}")]
#[case::pound_amount("\u{00A3}24")]
#[case::yen_amount("\u{00A5}360")]
#[case::euro_franc_equation("1 \u{20AC} = 6.55957\u{20A3}")]
#[case::ascii_letter("cat")]
#[case::styled_digit("3\u{0332}4")] #[case::phone_number("(416) 486-2500")]
fn ueb_owned_inputs_are_eligible(#[case] text: &str) {
assert!(is_ueb_eligible(text), "{text:?} should be UEB-eligible");
}
#[rstest::rstest]
#[case::dollar("$50")]
#[case::euro("\u{20AC}75")]
#[case::franc("\u{20A3}1")]
#[case::fullwidth_cent("25\u{FFE0}")]
#[case::fullwidth_pound("\u{FFE1}88")]
#[case::fullwidth_yen("\u{FFE5}1")]
#[case::fullwidth_won("\u{FFE6}100")]
fn korean_owned_currency_is_not_eligible(#[case] text: &str) {
assert!(
!is_ueb_eligible(text),
"{text:?} must stay in the legacy path"
);
}
}
#[cfg(test)]
mod encode_pipeline_tests {
use super::encode_forced;
#[test]
fn forced_empty_input_yields_none() {
assert_eq!(encode_forced(""), None);
}
}