use crate::math_symbol_shortcut;
use crate::rules::context::EncoderState;
use crate::rules::math;
use crate::rules::token::{Token, WordToken};
use crate::rules::token_rule::TokenAction;
use super::detect::is_math_expression;
use super::helpers::*;
pub(super) fn prev_next_words<'a, 'b>(
tokens: &'b [Token<'a>],
index: usize,
) -> (
Option<&'b crate::rules::token::WordToken<'a>>,
Option<&'b crate::rules::token::WordToken<'a>>,
) {
(
index
.checked_sub(1)
.and_then(|i| prev_word_skip_space(tokens, i)),
next_word_skip_space(tokens, index + 1),
)
}
pub(super) fn next_word_skip_space<'a, 'b>(
tokens: &'b [Token<'a>],
start: usize,
) -> Option<&'b crate::rules::token::WordToken<'a>> {
let mut i = start;
while let Some(tok) = tokens.get(i) {
match tok {
Token::Space(_) => i += 1,
Token::Word(w) => return Some(w),
_ => return None,
}
}
None
}
pub(super) fn next_indexed_word_skip_space<'a, 'b>(
tokens: &'b [Token<'a>],
start: usize,
) -> Option<(usize, &'b crate::rules::token::WordToken<'a>)> {
let mut i = start;
while let Some(tok) = tokens.get(i) {
match tok {
Token::Space(_) => i += 1,
Token::Word(w) => return Some((i, w)),
_ => return None,
}
}
None
}
pub(super) fn prev_word_skip_space<'a, 'b>(
tokens: &'b [Token<'a>],
start: usize,
) -> Option<&'b crate::rules::token::WordToken<'a>> {
let mut cursor = Some(start);
while let Some(i) = cursor {
match tokens.get(i) {
Some(Token::Space(_)) => cursor = i.checked_sub(1),
Some(Token::Word(w)) => return Some(w),
_ => return None,
}
}
None
}
fn word_is_math_letter_context(w: &crate::rules::token::WordToken<'_>) -> bool {
let has_super_sub = w.chars.iter().any(|c| {
matches!(
*c,
'\u{2080}'..='\u{2089}' | '\u{00B2}' | '\u{00B3}' | '\u{2070}'..='\u{2079}'
)
});
let plain_letter_list = w.chars.first().is_some_and(|c| c.is_ascii_alphabetic())
&& w.chars
.iter()
.all(|c| c.is_ascii_alphabetic() || matches!(*c, ',' | '₀'..='₉'));
has_super_sub || plain_letter_list
}
fn is_consecutive_ascii_letter_run(chars: &[char]) -> bool {
chars.len() >= 2
&& chars
.windows(2)
.all(|pair| u32::from(pair[1]) == u32::from(pair[0]) + 1)
}
fn is_roman_parenthetical_prose_trailer(chars: impl Iterator<Item = char>) -> bool {
chars.into_iter().all(|ch| {
is_korean_char(ch)
|| matches!(
ch,
',' | '.' | ';' | ':' | '!' | '?' | '·' | '\'' | '"' | '’' | '”'
)
})
}
fn is_roman_hyphen(ch: char) -> bool {
matches!(
ch,
'-' | '\u{2010}' | '\u{2011}' | '\u{2012}' | '\u{2013}' | '\u{2014}'
)
}
fn trim_roman_identifier_edge(chars: &[char]) -> &[char] {
let mut start = 0usize;
let mut end = chars.len();
while start < end
&& matches!(
chars[start],
'\'' | '"' | '‘' | '“' | '〈' | '《' | '「' | '『'
)
{
start += 1;
}
while start < end
&& matches!(
chars[end - 1],
',' | '.' | ';' | ':' | '!' | '?' | '\'' | '"' | '’' | '”' | '〉' | '》' | '」' | '』'
)
{
end -= 1;
}
&chars[start..end]
}
fn is_decimal_separator_between_digits(chars: &[char], index: usize) -> bool {
matches!(chars.get(index), Some('.' | ','))
&& index > 0
&& chars.get(index - 1).is_some_and(char::is_ascii_digit)
&& chars.get(index + 1).is_some_and(char::is_ascii_digit)
}
pub(super) fn is_korean_prose_roman_number_identifier(chars: &[char]) -> bool {
let chars = trim_roman_identifier_edge(chars);
if chars.len() < 3 || !chars.first().is_some_and(char::is_ascii_alphabetic) {
return false;
}
let mut letter_count = 0usize;
let mut has_digit = false;
for (index, ch) in chars.iter().enumerate() {
if ch.is_ascii_alphabetic() {
letter_count += 1;
} else if ch.is_ascii_digit() {
has_digit = true;
} else if !is_decimal_separator_between_digits(chars, index) {
return false;
}
}
letter_count >= 2 && has_digit
}
fn is_korean_prose_numeric_roman_identifier(chars: &[char]) -> bool {
let chars = trim_roman_identifier_edge(chars);
let mut index = 0usize;
let mut previous_was_digit = false;
while let Some(&ch) = chars.get(index) {
if ch.is_ascii_digit() {
previous_was_digit = true;
index += 1;
continue;
}
if matches!(ch, ',' | '.')
&& previous_was_digit
&& chars.get(index + 1).is_some_and(char::is_ascii_digit)
{
previous_was_digit = false;
index += 1;
continue;
}
break;
}
index > 0
&& chars.get(index).is_some_and(char::is_ascii_alphabetic)
&& chars[index..].iter().all(char::is_ascii_alphanumeric)
}
pub(super) fn is_korean_prose_roman_hyphen_identifier(chars: &[char]) -> bool {
let chars = trim_roman_identifier_edge(chars);
if chars.is_empty() {
return false;
}
let core = if chars.first() == Some(&'(') {
match chars.iter().position(|ch| *ch == ')') {
None => &chars[1..],
Some(close) if close + 1 == chars.len() => &chars[1..close],
Some(close) => {
let enclosed = &chars[1..close];
if enclosed.len() < 2
|| !enclosed.iter().all(char::is_ascii_uppercase)
|| !chars.get(close + 1).is_some_and(|ch| is_roman_hyphen(*ch))
{
return false;
}
&chars[1..]
}
}
} else {
chars
};
let identifier_end = core.iter().position(|ch| *ch == '(').unwrap_or(core.len());
if identifier_end < core.len() {
let body = &core[identifier_end + 1..];
if body.is_empty() || !body.iter().all(char::is_ascii_alphabetic) {
return false;
}
}
let identifier = &core[..identifier_end];
if !identifier.iter().any(|ch| is_roman_hyphen(*ch))
|| !identifier.iter().enumerate().all(|(index, ch)| {
ch.is_ascii_alphanumeric()
|| is_roman_hyphen(*ch)
|| *ch == ')'
|| is_decimal_separator_between_digits(identifier, index)
})
{
return false;
}
let segments = identifier.split(|ch| is_roman_hyphen(*ch));
let mut has_digit = false;
let mut first_ascii_letter = None;
let mut has_capitalised_word_segment = false;
let mut first_segment_letter_count = 0usize;
let mut first_segment_is_single_letter = false;
let mut has_later_lowercase_lexical_segment = false;
for (segment_index, raw_segment) in segments.enumerate() {
let segment = raw_segment
.iter()
.copied()
.filter(|ch| ch.is_ascii_alphanumeric())
.collect::<Vec<_>>();
if segment.is_empty() {
return false;
}
has_digit |= segment.iter().any(char::is_ascii_digit);
first_ascii_letter =
first_ascii_letter.or_else(|| segment.iter().copied().find(char::is_ascii_alphabetic));
let letter_count = segment.iter().filter(|ch| ch.is_ascii_alphabetic()).count();
has_capitalised_word_segment |= letter_count >= 2
&& segment
.iter()
.find(|ch| ch.is_ascii_alphabetic())
.is_some_and(|ch| ch.is_ascii_uppercase());
if segment_index == 0 {
first_segment_letter_count = letter_count;
first_segment_is_single_letter = segment.len() == 1 && letter_count == 1;
} else if first_segment_is_single_letter {
has_later_lowercase_lexical_segment |=
letter_count >= 2 && segment.iter().all(char::is_ascii_lowercase);
}
}
if has_digit {
first_segment_letter_count >= 2
|| first_ascii_letter.is_some_and(|ch| ch.is_ascii_uppercase())
} else {
has_capitalised_word_segment
|| (first_segment_is_single_letter && has_later_lowercase_lexical_segment)
}
}
pub(super) fn is_korean_prose_roman_slash_identifier(chars: &[char]) -> bool {
let chars = trim_roman_identifier_edge(chars);
if chars.is_empty()
|| !chars.first().is_some_and(|ch| ch.is_ascii_uppercase())
|| !chars.contains(&'/')
|| !chars.iter().all(|ch| {
ch.is_ascii_alphanumeric() || *ch == '/' || is_roman_hyphen(*ch) || *ch == '.'
})
{
return false;
}
let mut has_letter = false;
let mut has_multi_character_segment = false;
for segment in chars.split(|ch| *ch == '/') {
if segment.is_empty() || segment.iter().all(|ch| is_roman_hyphen(*ch) || *ch == '.') {
return false;
}
has_letter |= segment.iter().any(char::is_ascii_alphabetic);
has_multi_character_segment |= segment
.iter()
.filter(|ch| ch.is_ascii_alphanumeric())
.count()
>= 2;
}
has_letter && has_multi_character_segment
}
pub(super) fn is_korean_prose_single_letter_slash_phrase(
tokens: &[Token<'_>],
index: usize,
chars: &[char],
) -> bool {
let has_strong_math_symbol = chars.iter().any(|ch| {
math_symbol_shortcut::is_math_symbol_char(*ch)
&& !matches!(*ch, '\u{00B7}' | '\u{22C5}' | '/' | '_')
});
if has_strong_math_symbol {
return false;
}
let has_single_letter_slash_run = (0..chars.len()).any(|start| {
if !chars[start].is_ascii_uppercase()
|| start
.checked_sub(1)
.and_then(|before| chars.get(before))
.is_some_and(|ch| ch.is_ascii_alphanumeric() || *ch == '/')
{
return false;
}
let mut cursor = start + 1;
let mut slash_count = 0usize;
while chars.get(cursor) == Some(&'/')
&& chars
.get(cursor + 1)
.is_some_and(|ch| ch.is_ascii_uppercase())
{
slash_count += 1;
cursor += 2;
}
slash_count > 0
&& !chars
.get(cursor)
.is_some_and(|ch| ch.is_ascii_alphanumeric() || *ch == '/')
});
if !has_single_letter_slash_run {
return false;
}
let Some(next_word) = next_word_skip_space(tokens, index + 1) else {
return true;
};
let mut next_roman = next_word
.chars
.iter()
.copied()
.skip_while(|ch| matches!(*ch, '\'' | '"' | '‘' | '“' | '(' | '[' | '{'))
.take_while(|ch| ch.is_ascii_alphanumeric() || is_roman_hyphen(*ch));
let Some(first) = next_roman.next() else {
return next_word.chars.iter().any(|ch| is_korean_char(*ch));
};
first.is_ascii_uppercase()
&& next_roman.filter(char::is_ascii_alphabetic).count()
+ usize::from(first.is_ascii_alphabetic())
>= 2
}
fn is_roman_identifier_head_separator(chars: &[char], index: usize) -> bool {
matches!(
chars.get(index),
Some('.' | '/' | '-' | '‐' | '‑' | '‒' | '–' | '—')
) && index > 0
&& chars[index - 1].is_ascii_alphanumeric()
&& chars
.get(index + 1)
.is_some_and(char::is_ascii_alphanumeric)
}
fn is_terminal_roman_plus_core(core: &[char], allow_single_letter: bool) -> bool {
let plus_start = core
.iter()
.rposition(|ch| *ch != '+')
.map_or(0, |index| index + 1);
if plus_start == 0 || plus_start == core.len() {
return false;
}
let head = &core[..plus_start];
let plus_count = core.len() - plus_start;
if head.contains(&'+')
|| !head.iter().enumerate().all(|(index, ch)| {
ch.is_ascii_alphanumeric() || is_roman_identifier_head_separator(head, index)
})
|| !head.iter().any(char::is_ascii_alphabetic)
{
return false;
}
let alphanumeric_count = head.iter().filter(|ch| ch.is_ascii_alphanumeric()).count();
alphanumeric_count >= 2
|| plus_count >= 2
|| (allow_single_letter && head.len() == 1 && head[0].is_ascii_uppercase())
}
pub(super) fn is_korean_prose_roman_minus_grade(chars: &[char]) -> bool {
let chars = trim_roman_identifier_edge(chars);
let Some(minus) = chars.iter().position(|ch| *ch == '-') else {
return false;
};
let head_len = chars[..minus]
.iter()
.rev()
.take_while(|ch| ch.is_ascii_uppercase())
.count();
let before_head = &chars[..minus - head_len];
let trailer = &chars[minus + 1..];
let closers = trailer
.iter()
.take_while(|ch| is_terminal_plus_closer_char(**ch))
.count();
(2..=3).contains(&head_len)
&& before_head
.iter()
.all(|ch| is_korean_char(*ch) || matches!(ch, '(' | '[' | '{' | '‘' | '“' | '\'' | '"'))
&& (trailer.is_empty() || closers > 0)
&& trailer[closers..].iter().all(|ch| is_korean_char(*ch))
}
fn is_attached_plus_prose_trailer_char(ch: char) -> bool {
is_korean_char(ch)
|| matches!(
ch,
'(' | ')'
| '['
| ']'
| '{'
| '}'
| ','
| '.'
| ';'
| ':'
| '!'
| '?'
| '\''
| '"'
| '‘'
| '’'
| '“'
| '”'
| '〈'
| '〉'
| '《'
| '》'
| '「'
| '」'
| '『'
| '』'
)
}
fn is_terminal_plus_closer_char(ch: char) -> bool {
matches!(
ch,
')' | ']'
| '}'
| ','
| '.'
| ';'
| ':'
| '!'
| '?'
| '\''
| '"'
| '’'
| '”'
| '〉'
| '》'
| '」'
| '』'
)
}
pub(super) fn is_korean_prose_roman_plus_identifier(chars: &[char]) -> bool {
let chars = trim_roman_identifier_edge(chars);
if chars.is_empty() {
return false;
}
let roman_end = chars
.iter()
.take_while(|ch| {
ch.is_ascii_alphanumeric()
|| **ch == '+'
|| matches!(**ch, '.' | '/' | '-' | '‐' | '‑' | '‒' | '–' | '—')
})
.count();
let core = &chars[..roman_end];
let trailer = &chars[roman_end..];
let korean_led_mixed_trailer = trailer.first().is_some_and(|ch| is_korean_char(*ch))
&& trailer
.iter()
.all(|ch| ch.is_ascii_alphanumeric() || is_attached_plus_prose_trailer_char(*ch));
let trailer_is_prose = trailer.is_empty()
|| trailer.first() == Some(&'(')
|| korean_led_mixed_trailer
|| trailer
.iter()
.copied()
.all(is_attached_plus_prose_trailer_char);
if !trailer_is_prose {
return false;
}
let has_korean_trailer = trailer.iter().any(|ch| is_korean_char(*ch));
let allow_single_letter = trailer.is_empty()
|| has_korean_trailer
|| trailer.iter().copied().all(is_terminal_plus_closer_char);
if is_terminal_roman_plus_core(core, allow_single_letter) {
return true;
}
if !core.first().is_some_and(|ch| ch.is_ascii_uppercase()) {
return false;
}
if !core.contains(&'+') {
return false;
}
if !core
.iter()
.all(|ch| ch.is_ascii_alphanumeric() || *ch == '+')
{
return false;
}
let segments = core.split(|ch| *ch == '+').collect::<Vec<_>>();
segments.len() >= 2
&& segments.iter().all(|segment| {
segment.first().is_some_and(char::is_ascii_alphabetic)
&& !(segment.len() == 1 && segment[0].is_ascii_lowercase())
})
}
pub(super) fn has_korean_prefix_roman_plus_annotation(chars: &[char]) -> bool {
chars.iter().enumerate().any(|(start, ch)| {
if !ch.is_ascii_alphabetic() || !chars[..start].iter().any(|prefix| is_korean_char(*prefix))
{
return false;
}
let suffix = &chars[start..];
let Some(close) = suffix.iter().position(|candidate| *candidate == ')') else {
return false;
};
let trailer = &suffix[close + 1..];
close > 0
&& (is_korean_prose_roman_plus_identifier(&suffix[..close])
|| is_terminal_roman_plus_core(&suffix[..close], true))
&& (is_roman_parenthetical_prose_trailer(trailer.iter().copied())
|| trailer
.first()
.is_some_and(|ch| is_korean_char(*ch) || *ch == '·'))
})
}
pub(super) fn has_korean_prefix_terminal_roman_plus_identifier(chars: &[char]) -> bool {
chars.iter().enumerate().any(|(start, ch)| {
ch.is_ascii_alphanumeric()
&& chars[..start].iter().any(|prefix| is_korean_char(*prefix))
&& start
.checked_sub(1)
.and_then(|index| chars.get(index))
.is_none_or(|previous| !previous.is_ascii_alphanumeric() && *previous != '+')
&& is_korean_prose_roman_plus_identifier(&chars[start..])
})
}
pub(super) fn has_korean_prefix_roman_hyphen_suffix(chars: &[char]) -> bool {
for (index, ch) in chars.iter().enumerate() {
if ch.is_ascii_alphabetic()
&& chars[..index]
.iter()
.any(|prefix| crate::utils::is_korean_char(*prefix))
&& is_korean_prose_roman_hyphen_identifier(&chars[index..])
{
return true;
}
}
chars.windows(3).enumerate().any(|(index, window)| {
if !crate::utils::is_korean_char(window[0])
|| window[1] != '-'
|| !window[2].is_ascii_alphabetic()
{
return false;
}
let roman_tail = &chars[index + 2..];
let identifier_len = roman_tail
.iter()
.take_while(|ch| ch.is_ascii_alphanumeric())
.count();
let letter_count = roman_tail[..identifier_len]
.iter()
.filter(|ch| ch.is_ascii_alphabetic())
.count();
let identifier_is_unambiguous = window[2].is_ascii_uppercase() || letter_count >= 2;
let has_explicit_math_operator = roman_tail.iter().any(|ch| {
matches!(
*ch,
'+' | '−'
| '×'
| '÷'
| '='
| '<'
| '>'
| '≤'
| '≥'
| '≠'
| '≈'
| '^'
| '_'
| '/'
| '*'
| '|'
| '∈'
| '∉'
| '⊂'
| '⊃'
| '∧'
| '∨'
)
});
identifier_is_unambiguous && !has_explicit_math_operator
})
}
pub(super) fn next_word_begins_korean_prose_label_context(
tokens: &[Token<'_>],
index: usize,
) -> bool {
if !matches!(tokens.get(index + 1), Some(Token::Space(_)))
|| next_word_starts_with_math_value_cue(tokens, index)
{
return false;
}
next_indexed_word_skip_space(tokens, index + 1).is_some_and(|(next_index, word)| {
next_index > index + 1
&& word.chars.iter().any(|ch| is_korean_char(*ch))
&& word
.chars
.iter()
.all(|ch| is_korean_char(*ch) || matches!(*ch, ',' | '.' | '!' | '?'))
})
}
pub(super) fn is_korean_prose_acronym_parenthetical(chars: &[char]) -> bool {
let chars = trim_roman_identifier_edge(chars);
let Some(open) = chars.iter().position(|ch| *ch == '(') else {
return false;
};
let head = &chars[..open];
if head.len() < 2
|| !head.iter().all(char::is_ascii_alphanumeric)
|| !head.iter().any(char::is_ascii_uppercase)
{
return false;
}
let after_open = &chars[open + 1..];
let close = after_open.iter().position(|ch| *ch == ')');
let body = close.map_or(after_open, |index| &after_open[..index]);
if body.is_empty()
|| !body
.iter()
.all(|ch| ch.is_ascii_alphanumeric() || is_roman_hyphen(*ch))
{
return false;
}
close.is_none_or(|index| {
is_roman_parenthetical_prose_trailer(after_open[index + 1..].iter().copied())
})
}
fn has_balanced_brackets(chars: &[char]) -> bool {
let mut depth = 0i32;
for ch in chars {
match ch {
'(' | '[' | '{' => depth += 1,
')' | ']' | '}' => depth -= 1,
_ => {}
}
if depth < 0 {
return false;
}
}
depth == 0
}
fn is_hyphenated_roman_word(chars: &[char]) -> bool {
chars.contains(&'-')
&& chars
.iter()
.all(|ch| ch.is_ascii_alphabetic() || matches!(*ch, '-' | '.' | ','))
}
fn is_prose_number_notation(chars: &[char]) -> bool {
if !chars.iter().any(char::is_ascii_digit) {
return false;
}
let text: String = chars.iter().collect();
if crate::rules::math::function::starts_with_function(&text)
|| text
.char_indices()
.any(|(at, _)| crate::rules::math::function::starts_with_function(&text[at..]))
{
return false;
}
if chars.windows(3).any(|window| {
window[0].is_ascii_digit() && window[1] == '\u{00B7}' && window[2].is_ascii_digit()
}) {
return false;
}
if chars.windows(3).any(|window| {
window[0].is_ascii_digit()
&& window[1].is_ascii_lowercase()
&& window[2].is_ascii_lowercase()
}) {
return false;
}
if chars
.iter()
.any(|c| matches!(c, '=' | '<' | '>' | '\u{2260}' | '\u{2264}' | '\u{2265}'))
{
return false;
}
let is_separator = |c: char| {
c.is_ascii_digit()
|| matches!(
c,
'.' | ',' | ':' | '\u{00D7}' | '/' | '\u{00B7}' | '~' | '\u{223C}' | '(' | ')'
)
};
chars.iter().enumerate().all(|(index, c)| {
if is_separator(*c) {
return true;
}
c.is_ascii_alphabetic()
&& (index
.checked_sub(1)
.is_some_and(|previous| chars[previous].is_ascii_digit())
|| chars.get(index + 1).is_some_and(char::is_ascii_digit)
|| index
.checked_sub(1)
.is_some_and(|previous| chars[previous].is_ascii_alphabetic())
|| chars.get(index + 1).is_some_and(char::is_ascii_alphabetic))
})
}
fn has_ascii_letter_korean_math_suffix(chars: &[char]) -> bool {
if chars.len() < 3 {
return false;
}
let ascii_prefix_len = chars.iter().take_while(|c| c.is_ascii_alphabetic()).count();
(2..=3).contains(&ascii_prefix_len)
&& chars[ascii_prefix_len..]
.first()
.is_some_and(|c| matches!(*c, '의' | '와' | '과'))
&& chars[ascii_prefix_len..]
.iter()
.all(|c| is_korean_suffix_char(*c))
}
fn next_word_starts_with_math_value_cue(tokens: &[Token<'_>], index: usize) -> bool {
let mut cursor = index + 1;
while let Some(token) = tokens.get(cursor) {
match token {
Token::Space(_) => cursor += 1,
Token::Word(word) => {
let text = word.text.as_ref();
if text.starts_with('값') || text.starts_with('곱') {
return true;
}
if !has_ascii_letter_korean_math_suffix(&word.chars) {
return false;
}
cursor += 1;
}
_ => return false,
}
}
false
}
fn prev_word_is_math_product_cue(tokens: &[Token<'_>], index: usize) -> bool {
index
.checked_sub(1)
.and_then(|start| prev_word_skip_space(tokens, start))
.is_some_and(|word| word.text.as_ref() == "곱")
}
fn is_multiword_closed_roman_parenthetical_tail(
tokens: &[Token<'_>],
index: usize,
word: &WordToken<'_>,
) -> bool {
let Some(close) = word.chars.iter().position(|ch| *ch == ')') else {
return false;
};
let body = &word.chars[..close];
let trailing = &word.chars[close + 1..];
if body.is_empty()
|| !body.iter().all(char::is_ascii_alphabetic)
|| !is_roman_parenthetical_prose_trailer(trailing.iter().copied())
{
return false;
}
let mut cursor = index.checked_sub(1);
while let Some(i) = cursor {
match tokens.get(i) {
Some(Token::Space(_)) => cursor = i.checked_sub(1),
Some(Token::Word(previous)) => {
let previous_text = previous.text.as_ref();
if let Some(open) = previous_text.rfind('(') {
let before = &previous_text[..open];
let after = &previous_text[open + 1..];
if after.is_empty() || !after.chars().all(|ch| ch.is_ascii_alphabetic()) {
return false;
}
let before_is_initialism = before.chars().count() >= 2
&& before.chars().all(|ch| ch.is_ascii_uppercase());
if before
.chars()
.next_back()
.is_some_and(|ch| ch.is_ascii_alphabetic())
&& !before_is_initialism
{
return false;
}
return before.find(['(', ')']).is_none();
}
if previous_text.chars().all(|ch| ch.is_ascii_alphabetic()) {
cursor = i.checked_sub(1);
} else {
return false;
}
}
_ => return false,
}
}
false
}
fn is_multiword_closed_roman_parenthetical_head(
tokens: &[Token<'_>],
index: usize,
word: &WordToken<'_>,
) -> bool {
let text = word.text.as_ref();
let Some(open) = text.find('(') else {
return false;
};
let head = &text[..open];
let first_body_word = &text[open + 1..];
if head.chars().count() < 2
|| !head.chars().all(|ch| ch.is_ascii_uppercase())
|| first_body_word.is_empty()
|| !first_body_word.chars().all(|ch| ch.is_ascii_alphabetic())
{
return false;
}
let mut cursor = index + 1;
let mut body_words = 1usize;
loop {
let mut saw_space = false;
while matches!(tokens.get(cursor), Some(Token::Space(_))) {
saw_space = true;
cursor += 1;
}
if !saw_space {
return false;
}
let Some(Token::Word(next)) = tokens.get(cursor) else {
return false;
};
let next_text = next.text.as_ref();
if let Some(close) = next_text.find(')') {
let final_body_word = &next_text[..close];
let trailing = &next_text[close + 1..];
body_words += 1;
return body_words >= 2
&& !final_body_word.is_empty()
&& final_body_word.chars().all(|ch| ch.is_ascii_alphabetic())
&& is_roman_parenthetical_prose_trailer(trailing.chars());
}
if next_text.is_empty() || !next_text.chars().all(|ch| ch.is_ascii_alphabetic()) {
return false;
}
body_words += 1;
cursor += 1;
}
}
fn is_within_attached_korean_prose_parenthetical(tokens: &[Token<'_>], index: usize) -> bool {
#[derive(Clone, Copy)]
struct Opening {
token_index: usize,
char_index: usize,
attached_to_korean_prose: bool,
}
fn enclosed_chars(
tokens: &[Token<'_>],
opening: Opening,
close_token_index: usize,
close_char_index: usize,
) -> Vec<char> {
let mut body = Vec::new();
for (token_index, token) in tokens
.iter()
.enumerate()
.take(close_token_index + 1)
.skip(opening.token_index)
{
match token {
Token::Word(word) => {
let start = if token_index == opening.token_index {
opening.char_index + 1
} else {
0
};
let end = if token_index == close_token_index {
close_char_index
} else {
word.chars.len()
};
if start <= end && end <= word.chars.len() {
body.extend_from_slice(&word.chars[start..end]);
}
}
Token::Space(_) => body.push(' '),
Token::Mode(_) => {}
Token::Fraction(_) | Token::PreEncoded(_) => return Vec::new(),
}
}
body
}
fn is_prose_body(body: &[char]) -> bool {
let body = body
.iter()
.copied()
.skip_while(|ch| ch.is_whitespace())
.collect::<Vec<_>>();
let body = body
.iter()
.copied()
.rev()
.skip_while(|ch| ch.is_whitespace())
.collect::<Vec<_>>()
.into_iter()
.rev()
.collect::<Vec<_>>();
if body.is_empty() || body.iter().any(|ch| matches!(*ch, '(' | ')')) {
return false;
}
if body.iter().any(|ch| {
matches!(
*ch,
'=' | '<'
| '>'
| '≤'
| '≥'
| '≠'
| '≈'
| '≡'
| '×'
| '÷'
| '√'
| '∑'
| '∏'
| '∫'
| '∈'
| '∉'
| '⊂'
| '⊃'
| '^'
| '_'
)
}) {
return false;
}
if body.iter().any(|ch| is_korean_char(*ch)) {
return true;
}
let numeric_annotation = body.iter().any(char::is_ascii_digit)
&& body.iter().all(|ch| {
ch.is_ascii_digit()
|| ch.is_whitespace()
|| matches!(*ch, '.' | ',' | '%' | '‰' | '+' | '-' | '−' | '~')
});
if numeric_annotation {
return true;
}
let ascii_alphanumeric_count = body.iter().filter(|ch| ch.is_ascii_alphanumeric()).count();
let has_ascii_letter = body.iter().any(char::is_ascii_alphabetic);
ascii_alphanumeric_count >= 2
&& has_ascii_letter
&& body.iter().all(|ch| {
ch.is_ascii_alphanumeric()
|| ch.is_whitespace()
|| matches!(
*ch,
',' | '.'
| ':'
| ';'
| '\''
| '’'
| '-'
| '‐'
| '‑'
| '‒'
| '–'
| '—'
| '/'
| '&'
| '·'
| '⋅'
)
})
}
let mut openings = Vec::<Opening>::new();
for (token_index, token) in tokens.iter().enumerate() {
let Token::Word(word) = token else {
continue;
};
for (char_index, ch) in word.chars.iter().copied().enumerate() {
match ch {
'(' => openings.push(Opening {
token_index,
char_index,
attached_to_korean_prose: {
let prefix = &word.chars[..char_index];
let prefix_contains_korean = prefix.iter().any(|ch| is_korean_char(*ch));
let numeric_prefix = !prefix.is_empty()
&& prefix.iter().any(char::is_ascii_digit)
&& prefix.iter().all(|ch| {
ch.is_ascii_digit()
|| matches!(*ch, '.' | ',' | '\'' | '’' | '"' | '”' | '‘' | '“')
});
let quote_only_prefix = prefix
.iter()
.all(|ch| matches!(*ch, '\'' | '’' | '"' | '”' | '‘' | '“'));
prefix_contains_korean
|| (numeric_prefix && has_adjacent_korean_word(tokens, token_index))
|| (quote_only_prefix
&& adjacent_korean_word_flags(tokens, token_index).0)
},
}),
')' => {
let Some(opening) = openings.pop() else {
continue;
};
if opening.attached_to_korean_prose
&& opening.token_index <= index
&& index <= token_index
&& is_prose_body(&enclosed_chars(tokens, opening, token_index, char_index))
{
return true;
}
}
_ => {}
}
}
}
false
}
fn prev_is_math_context_for_ellipsis(tokens: &[Token<'_>], index: usize) -> bool {
let mut cursor = index.checked_sub(1);
while let Some(i) = cursor {
match tokens.get(i) {
Some(Token::Space(_)) => cursor = i.checked_sub(1),
Some(Token::PreEncoded(_)) => return true,
Some(Token::Word(w)) => return word_is_math_letter_context(w),
_ => return false,
}
}
false
}
fn has_content_skipping_space_backward(tokens: &[Token<'_>], index: usize) -> bool {
let mut cursor = index.checked_sub(1);
while let Some(i) = cursor {
match tokens.get(i) {
Some(Token::Space(_)) => cursor = i.checked_sub(1),
Some(Token::Word(_) | Token::PreEncoded(_)) => return true,
_ => return false,
}
}
false
}
fn has_content_skipping_space_forward(tokens: &[Token<'_>], index: usize) -> bool {
let mut i = index + 1;
while let Some(tok) = tokens.get(i) {
match tok {
Token::Space(_) => i += 1,
Token::Word(_) | Token::PreEncoded(_) => return true,
_ => return false,
}
}
false
}
fn is_delta_eq_polysum_pattern(text: &str) -> bool {
text.contains('\u{2206}') && text.contains('=') && text.contains(")+(")
}
fn word_is_pure_korean(w: &crate::rules::token::WordToken<'_>) -> bool {
if !w.meta.has_korean {
return false;
}
w.chars.iter().all(|c| {
let code = *c as u32;
(0xAC00..=0xD7A3).contains(&code)
|| (0x3131..=0x3163).contains(&code)
|| matches!(*c, '.' | ',' | '!' | '?' | ' ')
})
}
fn needs_decimal_context_spacing(text: &str, chars: &[char]) -> bool {
text.contains('\u{001F}')
|| text.contains('\u{22EF}')
|| chars.iter().any(|ch| is_combining_math_mark(*ch))
}
fn prev_prev_is_math_or_mixed_context(tokens: &[Token<'_>], index: usize) -> bool {
let mut i = index;
let mut found_space = false;
while i > 0 {
i -= 1;
match tokens.get(i) {
Some(Token::Space(_)) => found_space = true,
Some(Token::PreEncoded(_) | Token::Fraction(_)) if found_space => return true,
Some(Token::Word(w)) if found_space => {
return is_math_expression(&w.chars, w.text.as_ref())
|| (w.meta.has_korean
&& is_strong_mixed_math_candidate(&w.chars, w.text.as_ref()));
}
_ => return false,
}
}
false
}
pub(super) fn is_set_or_logic_symbol_word(word: &crate::rules::token::WordToken<'_>) -> bool {
word.chars.first().is_some_and(|c| {
word.chars.len() == 1
&& matches!(
*c,
'¬' | '∈'
| '∋'
| '∉'
| '∌'
| '⊂'
| '⊃'
| '⊄'
| '⊅'
| '∪'
| '∩'
| '∀'
| '∃'
| '∄'
| '∧'
| '∨'
| '⊻'
| '⇒'
| '⇔'
)
})
}
fn compute_leading_spaces(
tokens: &[Token<'_>],
index: usize,
in_prose: bool,
inner_is_single_letter: bool,
comma_list: bool,
inner_is_simple_numeric: bool,
) -> usize {
let suppress_pad = (in_prose && (inner_is_single_letter || comma_list))
|| inner_is_simple_numeric
|| index == 0;
if suppress_pad {
return 0;
}
let prev_prev = index.checked_sub(2).and_then(|i| tokens.get(i));
let prev_prev_is_korean = matches!(prev_prev, Some(Token::Word(w)) if w.meta.has_korean);
if prev_prev_is_korean { 1 } else { 0 }
}
pub(super) fn run<'a>(
tokens: &[Token<'a>],
index: usize,
state: &mut EncoderState,
) -> Result<TokenAction<'a>, String> {
let Some(Token::Word(word)) = tokens.get(index) else {
return Ok(TokenAction::Noop);
};
let text = word.text.as_ref();
if state.english_indicator
&& !state.math_mode_active
&& let Some(replacement) = split_anonymized_person_label(&word.chars)
{
return Ok(TokenAction::ReplaceMany(replacement));
}
if is_multiword_closed_roman_parenthetical_head(tokens, index, word)
|| is_multiword_closed_roman_parenthetical_tail(tokens, index, word)
|| is_within_attached_korean_prose_parenthetical(tokens, index)
{
return Ok(TokenAction::Noop);
}
if state.english_indicator
&& !state.math_mode_active
&& next_word_begins_korean_prose_label_context(tokens, index)
&& let Some(encoded) = encode_anonymized_person_label(&word.chars)
{
return Ok(TokenAction::Replace(Token::PreEncoded(encoded)));
}
if state.english_indicator
&& !state.math_mode_active
&& (is_korean_prose_roman_hyphen_identifier(&word.chars)
|| is_korean_prose_roman_number_identifier(&word.chars)
|| is_korean_prose_roman_slash_identifier(&word.chars)
|| is_korean_prose_single_letter_slash_phrase(tokens, index, &word.chars)
|| is_korean_prose_roman_plus_identifier(&word.chars)
|| is_korean_prose_roman_minus_grade(&word.chars)
|| has_korean_prefix_roman_plus_annotation(&word.chars)
|| has_korean_prefix_terminal_roman_plus_identifier(&word.chars)
|| has_korean_prefix_roman_hyphen_suffix(&word.chars)
|| is_korean_prose_acronym_parenthetical(&word.chars))
{
return Ok(TokenAction::Noop);
}
if state.english_indicator
&& !state.math_mode_active
&& has_adjacent_korean_word(tokens, index)
&& is_korean_prose_numeric_roman_identifier(&word.chars)
&& !prev_word_is_math_product_cue(tokens, index)
&& !next_word_starts_with_math_value_cue(tokens, index)
{
return Ok(TokenAction::Noop);
}
if word.chars.len() == 1 && word.chars[0].is_ascii_lowercase() {
let collect_next = |start: usize| {
let mut j = start;
while matches!(tokens.get(j), Some(Token::Space(_))) {
j += 1;
}
tokens.get(j).map(|t| (j, t))
};
const COLON_MATH_OPS: &[char] = &[
'\u{2272}', '\u{2273}', '\u{227A}', '\u{227B}', '\u{22BB}', '<', '>', '=', '\u{2260}',
'\u{2264}', '\u{2265}', '\u{2208}', '\u{2209}',
];
if let Some((op_idx, Token::Word(op_w))) = collect_next(index + 1)
&& op_w.chars.len() == 1
&& COLON_MATH_OPS.contains(&op_w.chars[0])
&& let Some((last_idx, Token::Word(last_w))) = collect_next(op_idx + 1)
&& last_w.chars.len() == 2
&& last_w.chars[0].is_ascii_lowercase()
&& last_w.chars[1] == ':'
{
let merged = format!("{} {} {}", text, op_w.text.as_ref(), last_w.text.as_ref());
let math_context = math_context_from_state(state);
if let Ok(bytes) =
math::encoder::encode_math_expression_with_context(&merged, math_context)
{
let consume_count = last_idx + 1 - index;
return Ok(TokenAction::ReplaceRange(
consume_count,
vec![Token::PreEncoded(bytes)],
));
}
}
}
if word.chars.first() == Some(&'{') && word.chars.contains(&'|') {
let mut merged = text.to_string();
let mut end_idx = index;
let mut found_close = word.chars.last() == Some(&'}');
if !found_close {
let mut i = index + 1;
while i < tokens.len() {
match tokens.get(i) {
Some(Token::Space(_)) => merged.push(' '),
Some(Token::Word(w)) => {
merged.push_str(w.text.as_ref());
if w.chars.last() == Some(&'}') {
end_idx = i;
found_close = true;
break;
}
}
_ => break,
}
i += 1;
}
}
let math_context = math_context_from_state(state);
if found_close
&& let Ok(bytes) =
math::encoder::encode_math_expression_with_context(&merged, math_context)
{
let consume_count = end_idx + 1 - index;
return Ok(TokenAction::ReplaceRange(
consume_count,
vec![Token::PreEncoded(bytes)],
));
}
}
if word.chars.len() >= 3 {
let ascii_prefix_len = word
.chars
.iter()
.take_while(|c| c.is_ascii_alphabetic())
.count();
if (2..=3).contains(&ascii_prefix_len) {
let suffix_chars = &word.chars[ascii_prefix_len..];
let suffix_is_math_identifier_particle =
has_ascii_letter_korean_math_suffix(&word.chars);
let prefix_letters: Vec<char> = word.chars[..ascii_prefix_len].to_vec();
let all_lower = prefix_letters.iter().all(|c| c.is_ascii_lowercase());
let all_upper = prefix_letters.iter().all(|c| c.is_ascii_uppercase());
let has_math_cue = next_word_starts_with_math_value_cue(tokens, index)
|| prev_word_is_math_product_cue(tokens, index);
let case_allowed = has_math_cue
&& (all_lower || (all_upper && is_consecutive_ascii_letter_run(&prefix_letters)));
if suffix_is_math_identifier_particle && case_allowed {
let prev_is_korean_or_first = index == 0
|| index
.checked_sub(1)
.and_then(|i| tokens.get(i))
.is_some_and(|t| match t {
Token::Word(w) => w.meta.has_korean,
Token::Space(_) => index
.checked_sub(2)
.and_then(|j| tokens.get(j))
.is_some_and(
|t2| matches!(t2, Token::Word(w) if w.meta.has_korean),
),
_ => false,
});
if prev_is_korean_or_first {
let matrix_context = state.matrix_context_active;
let mut bytes = Vec::new();
bytes.push(0);
for letter in &prefix_letters {
if all_upper {
if matrix_context {
bytes.push(32);
} else if letter == &prefix_letters[0] {
bytes.push(32);
bytes.push(32);
}
let code = crate::english::encode_english(letter.to_ascii_lowercase())?;
bytes.push(code);
} else {
let code = crate::english::encode_english(*letter)?;
bytes.push(code);
}
}
bytes.push(0);
bytes.push(0);
let suffix: String = suffix_chars.iter().collect();
let suffix_chars_vec: Vec<char> = suffix.chars().collect();
let suffix_meta = crate::rules::token::WordMeta::from_chars(&suffix_chars_vec);
let suffix_word = Token::Word(WordToken {
text: std::borrow::Cow::Owned(suffix),
chars: suffix_chars_vec,
meta: suffix_meta,
});
return Ok(TokenAction::ReplaceMany(vec![
Token::PreEncoded(bytes),
suffix_word,
]));
}
}
}
}
if word.chars.len() == 2
&& word.chars[1] == ','
&& math_symbol_shortcut::is_math_symbol_char(word.chars[0])
&& !word.chars[0].is_ascii_alphanumeric()
{
let prev_is_korean_word = index
.checked_sub(1)
.and_then(|i| tokens.get(i))
.and_then(|t| match t {
Token::Space(_) => index.checked_sub(2).and_then(|j| tokens.get(j)),
_ => Some(t),
})
.is_some_and(|t| matches!(t, Token::Word(w) if w.meta.has_korean));
let next_word_opt = next_indexed_word_skip_space(tokens, index + 1);
if prev_is_korean_word
&& let Some((next_idx, next_word)) = next_word_opt
&& next_word.chars.len() >= 2
&& math_symbol_shortcut::is_math_symbol_char(next_word.chars[0])
&& !next_word.chars[0].is_ascii_alphanumeric()
&& next_word.chars[1..]
.iter()
.all(|c| crate::utils::is_korean_char(*c))
{
let letter1 = word.chars[0];
let letter2 = next_word.chars[0];
let korean_suffix: String = next_word.chars[1..].iter().collect();
let enc1 = math_symbol_shortcut::encode_char_math_symbol_shortcut(letter1)?;
let enc2 = math_symbol_shortcut::encode_char_math_symbol_shortcut(letter2)?;
let mut bytes = Vec::new();
bytes.push(52); bytes.extend_from_slice(enc1);
bytes.push(2); bytes.push(0); bytes.extend_from_slice(enc2);
bytes.push(50); let suffix_chars: Vec<char> = korean_suffix.chars().collect();
let suffix_meta = crate::rules::token::WordMeta::from_chars(&suffix_chars);
let suffix_word = Token::Word(WordToken {
text: std::borrow::Cow::Owned(korean_suffix),
chars: suffix_chars,
meta: suffix_meta,
});
let consume_count = next_idx + 1 - index;
return Ok(TokenAction::ReplaceRange(
consume_count,
vec![Token::PreEncoded(bytes), suffix_word],
));
}
}
let dot_only =
!text.is_empty() && (text.chars().all(|c| matches!(c, '.' | ',')) && text.contains('.'));
if dot_only {
let prev_is_math_context = prev_is_math_context_for_ellipsis(tokens, index);
if prev_is_math_context {
let dots: usize = text.chars().filter(|c| *c == '.').count();
let mut bytes = vec![32u8; dots.min(3)];
let next_is_korean =
next_word_skip_space(tokens, index + 1).is_some_and(|w| w.meta.has_korean);
if text.ends_with(',') {
bytes.push(if next_is_korean { 2 } else { 16 });
}
if next_is_korean {
bytes.push(0);
}
return Ok(TokenAction::Replace(Token::PreEncoded(bytes)));
}
}
if (is_middle_dot_numeric_word(&word.chars) || is_korean_prose_numeric_notation(&word.chars))
&& has_adjacent_korean_word(tokens, index)
{
return Ok(TokenAction::Noop);
}
if matches!(word.chars.as_slice(), ['∴' | '∵']) {
let has_prev_content = has_content_skipping_space_backward(tokens, index);
let has_next_content = has_content_skipping_space_forward(tokens, index);
if has_prev_content && has_next_content {
let encoded = math_symbol_shortcut::encode_char_math_symbol_shortcut(word.chars[0])?;
let mut out = vec![0];
out.extend_from_slice(encoded);
out.push(0);
return Ok(TokenAction::Replace(Token::PreEncoded(out)));
}
}
if is_set_or_logic_symbol_word(word)
&& let Some((right_index, right_word)) = next_indexed_word_skip_space(tokens, index + 1)
&& right_word
.chars
.first()
.is_some_and(char::is_ascii_uppercase)
&& right_word.chars[1..]
.iter()
.all(|ch| crate::utils::is_korean_char(*ch) || !ch.is_ascii_alphanumeric())
{
let symbol = math_symbol_shortcut::encode_char_math_symbol_shortcut(word.chars[0])?;
let upper = right_word.chars[0];
let code = crate::english::encode_english(upper.to_ascii_lowercase())?;
let mut replacement: Vec<Token<'a>> = vec![Token::PreEncoded(symbol.to_vec())];
replacement.extend(tokens[index + 1..right_index].iter().cloned());
replacement.push(Token::PreEncoded(vec![32, code]));
if right_word.chars.len() > 1 {
let suffix = right_word.chars[1..].iter().collect::<String>();
replacement.push(build_word_token(suffix));
}
return Ok(TokenAction::ReplaceRange(
right_index + 1 - index,
replacement,
));
}
if word.chars.len() == 1 && word.chars[0].is_ascii_uppercase() {
let (prev, next) = prev_next_words(tokens, index);
if prev.is_some_and(is_set_or_logic_symbol_word)
|| next.is_some_and(is_set_or_logic_symbol_word)
{
let code = crate::english::encode_english(word.chars[0].to_ascii_lowercase())?;
return Ok(TokenAction::Replace(Token::PreEncoded(vec![32, code])));
}
}
if let Some(stripped) = text.strip_prefix('$') {
if let Some(close_idx) = stripped.find('$')
&& close_idx + 1 < stripped.len()
{
let latex = &text[..=close_idx + 1];
let suffix = &stripped[close_idx + 1..];
if let Some((whole, numerator, denominator)) =
crate::fraction::parse_latex_fraction(latex)
{
let mut replacement: Vec<Token<'a>> =
vec![Token::Fraction(crate::rules::token::FractionToken {
whole,
numerator,
denominator,
})];
if !suffix.is_empty() && rule_44_requires_space_before_korean(suffix) {
replacement.push(Token::Space(crate::rules::token::SpaceKind::Regular));
}
replacement.push(build_word_token(suffix.to_string()));
return Ok(TokenAction::ReplaceMany(replacement));
}
let inner = &latex[1..latex.len() - 1];
let math_context = math_context_from_state(state);
if let Ok(bytes) =
crate::rules::token_rules::latex_math::encode_latex_math_bytes_with_context(
inner,
math_context,
)
{
let suffix_first = suffix.chars().next();
let suffix_is_korean = suffix_first.is_some_and(crate::utils::is_korean_char);
let inner_is_single_letter =
inner.chars().count() == 1 && inner.chars().all(|c| c.is_ascii_alphabetic());
let comma_list = inner.contains(',')
&& inner.split(',').map(str::trim).all(|p| {
!p.is_empty()
&& p.chars().count() == 1
&& p.chars().all(|c| c.is_ascii_alphabetic())
});
let prev_is_korean = index
.checked_sub(1)
.and_then(|i| tokens.get(i))
.map(|tok| match tok {
Token::Word(w) => w.meta.has_korean,
Token::Space(_) => index
.checked_sub(2)
.and_then(|j| tokens.get(j))
.is_some_and(|t| matches!(t, Token::Word(w) if w.meta.has_korean)),
_ => false,
})
.unwrap_or(false);
let in_prose = suffix_is_korean || prev_is_korean;
let inner_is_simple_numeric = !inner.is_empty()
&& inner.chars().all(|c| {
c.is_ascii_digit() || matches!(c, '-' | '+' | '\u{2212}' | '.' | ',')
});
let leading_spaces = compute_leading_spaces(
tokens,
index,
in_prose,
inner_is_single_letter,
comma_list,
inner_is_simple_numeric,
);
let mut replacement = Vec::new();
if leading_spaces > 0 {
replacement.push(Token::PreEncoded(vec![0; leading_spaces]));
}
if in_prose && inner_is_single_letter {
let mut wrapped = Vec::with_capacity(bytes.len() + 2);
wrapped.push(52); wrapped.extend(bytes);
wrapped.push(50); replacement.push(Token::PreEncoded(wrapped));
} else if in_prose && comma_list {
let letters: Vec<&str> = inner.split(',').map(str::trim).collect();
let mut wrapped = Vec::new();
for (i, letter) in letters.iter().enumerate() {
if let Some(c) = letter.chars().next() {
if i == 0 {
wrapped.push(52);
} else {
wrapped.push(0);
wrapped.push(48); }
if c.is_ascii_uppercase() {
wrapped.push(32);
if let Ok(code) =
crate::english::encode_english(c.to_ascii_lowercase())
{
wrapped.push(code);
}
} else if let Ok(code) = crate::english::encode_english(c) {
wrapped.push(code);
}
if i + 1 < letters.len() {
wrapped.push(2); } else {
wrapped.push(50);
}
}
}
replacement.push(Token::PreEncoded(wrapped));
} else {
replacement.push(Token::PreEncoded(bytes));
let trailing_spaces = if suffix_is_korean && !inner_is_simple_numeric {
2
} else {
0
};
if trailing_spaces > 0 {
replacement.push(Token::PreEncoded(vec![0; trailing_spaces]));
}
}
replacement.push(build_word_token(suffix.to_string()));
return Ok(TokenAction::ReplaceMany(replacement));
}
}
if let Some((whole, numerator, denominator)) = crate::fraction::parse_latex_fraction(text) {
return Ok(TokenAction::Replace(Token::Fraction(
crate::rules::token::FractionToken {
whole,
numerator,
denominator,
},
)));
}
if text.ends_with('$') && text.len() >= 3 {
let inner = &text[1..text.len() - 1];
let math_context = math_context_from_state(state);
if let Ok(bytes) =
crate::rules::token_rules::latex_math::encode_latex_math_bytes_with_context(
inner,
math_context,
)
{
let replacement =
crate::rules::token_rules::latex_math::wrap_latex_math_tokens_with_inner(
tokens, index, bytes, inner,
);
return Ok(TokenAction::ReplaceMany(replacement));
}
}
return Ok(TokenAction::Noop);
}
if !is_math_expression(&word.chars, text) || is_prose_number_notation(&word.chars) {
let math_context = math_context_from_state(state);
if let Some(bytes) = try_encode_mixed_math_slice(&word.chars, math_context) {
return Ok(TokenAction::Replace(Token::PreEncoded(bytes)));
}
let prev_prev_is_math_or_mixed = prev_prev_is_math_or_mixed_context(tokens, index);
let leading_delimiter_len = if index == 0 {
0
} else if matches!(tokens.get(index - 1), Some(Token::Space(_))) {
if prev_prev_is_math_or_mixed { 0 } else { 1 }
} else {
2
};
if let Some(replacement) = split_mixed_math_word(word, leading_delimiter_len, math_context)
{
return Ok(TokenAction::ReplaceMany(replacement));
}
return Ok(TokenAction::Noop);
}
let math_context = math_context_from_state(state);
match math::encoder::encode_math_expression_with_context(text, math_context) {
Ok(bytes) => {
let (prev_has_korean, _next_has_korean) = adjacent_korean_word_flags(tokens, index);
let mut wrapped = Vec::with_capacity(bytes.len() + 2);
let needs_decimal_context_spacing = needs_decimal_context_spacing(text, &word.chars);
let prev_is_space_decimal = index
.checked_sub(1)
.is_some_and(|i| matches!(tokens.get(i), Some(Token::Space(_))));
if needs_decimal_context_spacing && prev_is_space_decimal {
wrapped.push(0);
}
if index != 0 && !prev_has_korean && is_delta_eq_polysum_pattern(text) {
wrapped.push(0);
wrapped.push(0);
}
let only_simple_digits = !word.chars.is_empty()
&& word.chars.iter().all(|c| {
c.is_ascii_digit() || matches!(*c, '-' | '+' | '\u{2212}' | '.' | ',')
});
let is_substantial_math = word.chars.len() > 1
&& word.chars.iter().any(|c| {
c.is_ascii_alphanumeric() || matches!(*c, '(' | ')' | '[' | ']' | '|')
})
&& !only_simple_digits
&& !is_prose_number_notation(&word.chars)
&& has_balanced_brackets(&word.chars)
&& !is_hyphenated_roman_word(&word.chars);
let needs_korean_leading = index != 0
&& prev_has_korean
&& matches!(tokens.get(index - 1), Some(Token::Space(_)))
&& !needs_decimal_context_spacing
&& is_substantial_math;
if needs_korean_leading {
wrapped.push(0);
}
wrapped.extend_from_slice(&bytes);
if needs_decimal_context_spacing
&& matches!(tokens.get(index + 1), Some(Token::Space(_)))
{
wrapped.push(0);
}
let next_is_pure_korean =
next_word_skip_space(tokens, index + 1).is_some_and(word_is_pure_korean);
let needs_trailing_korean_pad = next_is_pure_korean
&& matches!(tokens.get(index + 1), Some(Token::Space(_)))
&& !needs_decimal_context_spacing
&& is_substantial_math;
let trailing_pad: &[u8] = if needs_trailing_korean_pad { &[0] } else { &[] };
wrapped.extend_from_slice(trailing_pad);
Ok(TokenAction::Replace(Token::PreEncoded(wrapped)))
}
Err(_) => Ok(TokenAction::Noop),
}
}
#[cfg(test)]
mod tests {
use super::*;
#[rstest::rstest]
#[case::ueb_multiword_parenthetical("plays (such as Romeo and Juliet)", true)]
#[case::initialism_prefixed_comma("WTO(World Tourism Organization),", true)]
#[case::korean_particle_after_parenthesis("설명(Home Connectivity Alliance)를", true)]
#[case::korean_particle_after_quote("설명(Home Connectivity Alliance)’를", true)]
#[case::ueb_letter_list("(q, r)", false)]
#[case::math_function("f(x)", false)]
#[case::operator_interrupts_prose_run("(x + y)", false)]
#[case::no_closing_parenthesis("Romeo Juliet", false)]
#[case::function_with_spaced_argument("f(x y)", false)]
#[case::missing_opening_parenthesis("Romeo Juliet)", false)]
#[case::invalid_trailing_digit("(Romeo Juliet)1", false)]
#[case::digit_in_final_fragment("(Romeo Juliet2)", false)]
#[case::digit_after_opening("(2Romeo Juliet)", false)]
#[case::digit_in_earlier_fragment("(Romeo2 Juliet)", false)]
#[case::nonletter_earlier_without_opening("Romeo2 Juliet More)", false)]
#[case::nested_opening_before_fragment("((Romeo Juliet)", false)]
#[case::closing_before_opening(")(Romeo Juliet)", false)]
fn recognizes_only_complete_multiword_roman_parenthetical_tails(
#[case] input: &str,
#[case] expected: bool,
) {
let ir = crate::rules::token::DocumentIR::parse(input, true);
let index = ir
.tokens
.iter()
.rposition(|token| matches!(token, Token::Word(_)))
.expect("probe must contain a word");
let Token::Word(word) = &ir.tokens[index] else {
unreachable!("selected token must be a word");
};
assert_eq!(
is_multiword_closed_roman_parenthetical_tail(&ir.tokens, index, word),
expected
);
if expected {
let mut state = EncoderState::new(false);
assert!(matches!(
run(&ir.tokens, index, &mut state).unwrap(),
TokenAction::Noop
));
}
}
#[rstest::rstest]
#[case::initialism_expansion("HCA(Home Connectivity Alliance)", true)]
#[case::punctuated_expansion("TB(Top View Battle),", true)]
#[case::korean_particle("HCA(Home Connectivity Alliance)를", true)]
#[case::quoted_korean_particle("HCA(Home Connectivity Alliance)’를", true)]
#[case::single_capital_head("A(Home Connectivity Alliance)", false)]
#[case::mixed_case_head("HCa(Home Connectivity Alliance)", false)]
#[case::single_word_body("HCA(Alliance)", false)]
#[case::digit_in_body("HCA(Home Connectivity2 Alliance)", false)]
#[case::operator_in_body("HCA(Home + Alliance)", false)]
#[case::nested_parenthesis("HCA((Home Connectivity Alliance))", false)]
#[case::alphanumeric_trailer("HCA(Home Connectivity Alliance)1", false)]
#[case::unclosed_expansion("HCA(Home Connectivity Alliance", false)]
fn recognizes_only_complete_allcaps_multiword_roman_expansion_heads(
#[case] input: &str,
#[case] expected: bool,
) {
let ir = crate::rules::token::DocumentIR::parse(input, true);
let index = ir
.tokens
.iter()
.position(|token| matches!(token, Token::Word(_)))
.expect("probe must contain a word");
let Token::Word(word) = &ir.tokens[index] else {
unreachable!("selected token must be a word");
};
assert_eq!(
is_multiword_closed_roman_parenthetical_head(&ir.tokens, index, word),
expected
);
if expected {
let mut state = EncoderState::new(false);
assert!(matches!(
run(&ir.tokens, index, &mut state).unwrap(),
TokenAction::Noop
));
}
}
#[rstest::rstest]
#[case::roman_followed_by_digit("용어(Web)3", 1)]
#[case::roman_then_korean_explanation("기관(KRISS, 원장)", 2)]
#[case::numeric_annotation("최고치(2126.14)", 1)]
#[case::multiword_roman_name("전환(DT·Digital Transformation)", 2)]
#[case::korean_numeric_name("용어2(Version Two)", 2)]
#[case::year_with_roman_explanation("보고서 2023(MWC 2023)", 2)]
#[case::single_variable("함수(x)", 0)]
#[case::lowercase_expression("함수(x+1)", 0)]
#[case::uppercase_expression("식(A+B)", 0)]
#[case::separated_function("함수 f(x)", 0)]
fn recognizes_attached_korean_prose_parenthetical_span(
#[case] input: &str,
#[case] expected_matching_words: usize,
) {
let ir = crate::rules::token::DocumentIR::parse(input, true);
let matching_indices = ir
.tokens
.iter()
.enumerate()
.filter_map(|(index, token)| {
matches!(token, Token::Word(_))
.then(|| is_within_attached_korean_prose_parenthetical(&ir.tokens, index))
.is_some_and(|matches| matches)
.then_some(index)
})
.collect::<Vec<_>>();
assert_eq!(matching_indices.len(), expected_matching_words);
for index in matching_indices {
let mut state = EncoderState::new(false);
assert!(matches!(
run(&ir.tokens, index, &mut state).unwrap(),
TokenAction::Noop
));
}
}
#[rstest::rstest]
#[case::roman_followed_by_digit("용어(Web)3")]
#[case::roman_then_korean_explanation("기관(KRISS, 원장)")]
#[case::multiword_roman_name("전환(DT·Digital Transformation)")]
#[case::korean_numeric_name("용어2(Version Two)")]
#[case::year_with_roman_explanation("보고서 2023(MWC 2023)")]
fn attached_korean_prose_parentheses_keep_rule_34_order(#[case] input: &str) {
let encoded = crate::encode_to_unicode(input).expect("input must encode");
assert!(
encoded.contains("⠦⠄⠴"),
"Korean opening parenthesis must precede Roman entry: {encoded}"
);
}
#[test]
fn attached_korean_name_keeps_specialized_anonymized_person_path() {
let ir = crate::rules::token::DocumentIR::parse("모A(61)씨", true);
let index = ir
.tokens
.iter()
.position(|token| matches!(token, Token::Word(_)))
.expect("fixture must contain a word");
let mut state = EncoderState::new(true);
assert!(matches!(
run(&ir.tokens, index, &mut state).unwrap(),
TokenAction::ReplaceMany(_)
));
assert!(
crate::encode_to_unicode("모A(61)씨")
.expect("fixture must encode")
.contains("⠴⠠⠁⠦⠄⠼⠋⠁⠠⠴")
);
}
#[rstest::rstest]
#[case::unit_separator("a\u{001f}b", "ab", true)]
#[case::midline_ellipsis("a⋯b", "ab", true)]
#[case::combining_mark("ab", "a\u{0305}", true)]
#[case::plain_expression("a+b", "a+b", false)]
fn detects_decimal_context_spacing_markers(
#[case] text: &str,
#[case] chars: &str,
#[case] expected: bool,
) {
assert_eq!(
needs_decimal_context_spacing(text, &chars.chars().collect::<Vec<_>>()),
expected
);
}
use crate::rules::token::{SpaceKind, WordMeta, WordToken};
use std::borrow::Cow;
fn enc_str(s: &str) -> String {
crate::encode_to_unicode(s).unwrap_or_default()
}
fn word_tok<'a>(text: &'a str) -> Token<'a> {
let chars: Vec<char> = text.chars().collect();
let meta = WordMeta::from_chars(&chars);
Token::Word(WordToken {
text: Cow::Borrowed(text),
chars,
meta,
})
}
fn space_tok() -> Token<'static> {
Token::Space(SpaceKind::Regular)
}
#[rstest::rstest]
#[case::empty_segment("ISO//IEC")]
#[case::punctuation_only_segment("ISO/-./IEC")]
fn roman_slash_identifier_rejects_incomplete_segments(#[case] input: &str) {
assert!(!is_korean_prose_roman_slash_identifier(
&input.chars().collect::<Vec<_>>()
));
}
#[test]
fn single_letter_slash_phrase_requires_letters_in_the_following_word() {
let tokens = vec![word_tok("H/W"), space_tok(), word_tok("((")];
let chars = "H/W".chars().collect::<Vec<_>>();
assert!(!is_korean_prose_single_letter_slash_phrase(
&tokens, 0, &chars
));
}
#[test]
fn multiword_parenthetical_tail_stops_at_a_non_word_boundary() {
let tokens = vec![
Token::PreEncoded(vec![1]),
space_tok(),
word_tok("Alliance)"),
];
let Token::Word(tail) = &tokens[2] else {
unreachable!("fixture ends in a word")
};
assert!(!is_multiword_closed_roman_parenthetical_tail(
&tokens, 2, tail
));
}
#[test]
fn multiword_parenthetical_head_stops_at_a_non_word_boundary() {
let tokens = vec![
word_tok("HCA(Home"),
space_tok(),
Token::PreEncoded(vec![1]),
];
let Token::Word(head) = &tokens[0] else {
unreachable!("fixture begins with a word")
};
assert!(!is_multiword_closed_roman_parenthetical_head(
&tokens, 0, head
));
}
#[test]
fn attached_prose_parenthetical_rejects_a_preencoded_body() {
let tokens = vec![word_tok("한국("), Token::PreEncoded(vec![1]), word_tok(")")];
assert!(!is_within_attached_korean_prose_parenthetical(&tokens, 1));
}
#[test]
fn attached_prose_parenthetical_ignores_mode_tokens_in_its_body() {
let tokens = vec![
word_tok("한국("),
Token::Mode(crate::rules::token::ModeEvent::EnterEnglish),
word_tok("Web)"),
];
assert!(is_within_attached_korean_prose_parenthetical(&tokens, 1));
}
#[rstest::rstest]
#[case::enclosed_roman_continuation("한글(ABC)-D", true)]
#[case::korean_prefix_before_initialism("기장-KBO", true)]
#[case::lowercase_math_variable("값-x", false)]
fn korean_roman_hyphen_suffix_is_classified_structurally(
#[case] input: &str,
#[case] expected: bool,
) {
assert_eq!(
has_korean_prefix_roman_hyphen_suffix(&input.chars().collect::<Vec<_>>()),
expected
);
}
#[rstest::rstest]
#[case::compact_unit("50bp", true)]
#[case::decimal_prefix("3.1p", true)]
#[case::ordinal("1st", true)]
#[case::mixed_case_name("25Project", true)]
#[case::digit_after_letter("3x3", true)]
#[case::capital_suffix("6G", true)]
#[case::trailing_punctuation("50bp,", true)]
#[case::letter_first("MP3", false)]
#[case::operator("3a+b", false)]
#[case::solidus("3/4", false)]
#[case::punctuation_before_letter("3.a", false)]
#[case::number_only("3", false)]
#[case::letters_only("abc", false)]
#[case::korean_suffix("3한", false)]
fn recognizes_numeric_prefix_roman_identifier_grammar(
#[case] text: &str,
#[case] expected: bool,
) {
assert_eq!(
is_korean_prose_numeric_roman_identifier(&text.chars().collect::<Vec<_>>()),
expected
);
}
#[rstest::rstest]
#[case::compact_unit("가는 50bp 인상", "50bp", true)]
#[case::decimal_prefix("가는 3.1p 표본", "3.1p", true)]
#[case::ordinal("가는 1st 항목", "1st", true)]
#[case::mixed_case_name("가는 25Project 자료", "25Project", true)]
#[case::digit_after_letter("가는 3x3 배열", "3x3", true)]
#[case::isolated_expression("3ab", "3ab", false)]
#[case::previous_product_cue("곱 3ab 결과", "3ab", false)]
#[case::next_value_cue("식은 3ab 값을", "3ab", false)]
#[case::explicit_latex("가는 $3ab$ 식", "$3ab$", false)]
fn numeric_roman_route_respects_korean_prose_and_math_context(
#[case] input: &str,
#[case] target: &str,
#[case] expected_noop: bool,
) {
let ir = crate::rules::token::DocumentIR::parse(input, true);
let index = ir
.tokens
.iter()
.position(|token| matches!(token, Token::Word(word) if word.text.as_ref() == target))
.expect("target word must be tokenized as one word");
let mut state = EncoderState::new(true);
assert_eq!(
matches!(
run(&ir.tokens, index, &mut state).unwrap(),
TokenAction::Noop
),
expected_noop
);
}
#[test]
fn unsupported_mixed_expression_after_leading_space_falls_through() {
let tokens = vec![space_tok(), word_tok("√분산🚀")];
let mut state = EncoderState::new(false);
let action = run(&tokens, 1, &mut state).unwrap();
assert!(matches!(action, TokenAction::Noop));
}
#[test]
fn prev_next_words_oob_index() {
let tokens: Vec<Token<'_>> = vec![word_tok("a")];
let (prev, next) = prev_next_words(&tokens, 5);
assert!(prev.is_none(), "prev must be None for oob index");
assert!(next.is_none(), "next must be None for oob index");
}
#[test]
fn prev_next_words_adjacent_words() {
let tokens: Vec<Token<'_>> = vec![word_tok("a"), word_tok("b"), word_tok("c")];
let (prev, next) = prev_next_words(&tokens, 1);
assert!(prev.is_some(), "prev must resolve to Word 'a'");
assert_eq!(prev.unwrap().text.as_ref(), "a");
assert!(next.is_some(), "next must resolve to Word 'c'");
assert_eq!(next.unwrap().text.as_ref(), "c");
}
#[test]
fn prev_next_words_skips_spaces() {
let tokens: Vec<Token<'_>> = vec![
word_tok("a"),
space_tok(),
space_tok(),
word_tok("b"),
space_tok(),
word_tok("c"),
];
let (prev, next) = prev_next_words(&tokens, 3);
assert_eq!(prev.unwrap().text.as_ref(), "a");
assert_eq!(next.unwrap().text.as_ref(), "c");
}
#[test]
fn prev_next_words_at_index_zero() {
let tokens: Vec<Token<'_>> = vec![word_tok("a"), word_tok("b")];
let (prev, next) = prev_next_words(&tokens, 0);
assert!(prev.is_none(), "no prev at index 0");
assert!(next.is_some(), "next must still resolve");
assert_eq!(next.unwrap().text.as_ref(), "b");
}
#[test]
fn prev_next_words_stops_at_non_word_token() {
let tokens: Vec<Token<'_>> = vec![
Token::PreEncoded(vec![1, 2, 3]),
space_tok(),
word_tok("middle"),
space_tok(),
Token::PreEncoded(vec![4, 5, 6]),
];
let (prev, next) = prev_next_words(&tokens, 2);
assert!(
prev.is_none(),
"PreEncoded boundary must yield None for prev"
);
assert!(
next.is_none(),
"PreEncoded boundary must yield None for next"
);
}
#[test]
fn math_suffix_and_next_value_cue_helpers_reject_short_or_non_word_inputs() {
assert!(!has_ascii_letter_korean_math_suffix(&['a', '의']));
let tokens = vec![word_tok("ab의"), Token::PreEncoded(vec![1])];
assert!(!next_word_starts_with_math_value_cue(&tokens, 0));
}
#[rstest::rstest]
#[case::xor_alone("⊻", true)]
#[case::wedge_alone("∧", true)]
#[case::membership_alone("∈", true)]
#[case::negation_alone("¬", true)]
#[case::ascii_plus("+", false)]
#[case::xor_then_letter("⊻x", false)]
#[case::empty_word("", false)]
fn set_or_logic_symbol_word_is_complete(#[case] text: &'static str, #[case] expected: bool) {
let chars: Vec<char> = text.chars().collect();
let word = WordToken {
text: Cow::Borrowed(text),
meta: WordMeta::from_chars(&chars),
chars,
};
assert_eq!(is_set_or_logic_symbol_word(&word), expected);
}
#[rstest::rstest]
#[case::upper_negation("A ¬ B", "⠠⠁⠀⠈⠔⠀⠠⠃")]
#[case::mixed_case_negation("p ¬ Q", "⠏⠀⠈⠔⠀⠠⠟")]
#[case::set_builder_membership("{x | x ∈ R}", "⠦⠂⠭⠀⠸⠳⠀⠭⠀⠖⠀⠠⠗⠐⠴")]
fn spaced_set_and_logic_operands_stay_math_variables(
#[case] input: &str,
#[case] expected: &str,
) {
assert_eq!(crate::encode_to_unicode(input).as_deref(), Ok(expected));
}
#[test]
fn colon_math_pattern_letters_avoid_prose_wrap() {
let merged = enc_str("a ≲ b:");
assert!(!merged.is_empty(), "expected encoded bytes for `a ≲ b:`");
let plain = enc_str("a ≲ b");
assert_ne!(
merged, plain,
"trailing colon must change encoding via merge path"
);
}
#[test]
fn set_builder_brace_pipe_merges_inner_korean() {
let setbuilder = enc_str("{x|x는 정수}");
assert!(!setbuilder.is_empty());
let plain = enc_str("x는 정수");
assert_ne!(
setbuilder, plain,
"set-builder wrap must change encoding vs. bare Korean"
);
}
#[test]
fn set_builder_unclosed_does_not_merge() {
let unclosed = enc_str("{x|x는 정수");
let closed = enc_str("{x|x는 정수}");
assert_ne!(
unclosed, closed,
"unclosed set-builder must NOT produce the same encoding as closed"
);
}
#[test]
fn multiletter_lower_prose_identifier_is_not_math() {
for (prefix, suffix) in [
("ab의", "친구"),
("id의", "친구"),
("ai와", "서비스"),
("api의", "응답"),
] {
let tokens = vec![word_tok(prefix), space_tok(), word_tok(suffix)];
let mut state = EncoderState::new(false);
let action = run(&tokens, 0, &mut state).expect("ok");
assert!(
matches!(action, TokenAction::Noop),
"input={prefix} {suffix}"
);
}
}
#[test]
fn multiletter_lower_identifier_requires_math_value_cue() {
let tokens = vec![word_tok("ab의"), space_tok(), word_tok("값을")];
let mut state = EncoderState::new(false);
let action = run(&tokens, 0, &mut state).expect("ok");
assert!(matches!(action, TokenAction::ReplaceMany(_)));
}
#[test]
fn multiletter_lower_identifier_allows_previous_product_cue() {
let tokens = vec![word_tok("곱"), space_tok(), word_tok("abc의")];
let mut state = EncoderState::new(false);
let action = run(&tokens, 2, &mut state).expect("ok");
assert!(matches!(action, TokenAction::ReplaceMany(_)));
}
#[test]
fn multiletter_upper_identifier_uses_genitive_suffix() {
let tokens = vec![word_tok("AB의")];
let mut plain_state = EncoderState::new(false);
let plain = run(&tokens, 0, &mut plain_state).expect("ok");
assert!(matches!(plain, TokenAction::Noop));
let cued_tokens = vec![word_tok("AB의"), space_tok(), word_tok("값을")];
let mut cued_state = EncoderState::new(false);
let cued = run(&cued_tokens, 0, &mut cued_state).expect("ok");
assert!(matches!(cued, TokenAction::ReplaceMany(_)));
let acronym_tokens = vec![word_tok("FM의")];
let mut acronym_state = EncoderState::new(false);
let acronym = run(&acronym_tokens, 0, &mut acronym_state).expect("ok");
assert!(matches!(acronym, TokenAction::Noop));
let topic_acronym_tokens = vec![word_tok("SNS는")];
let mut topic_acronym_state = EncoderState::new(false);
let topic_acronym = run(&topic_acronym_tokens, 0, &mut topic_acronym_state).expect("ok");
assert!(matches!(topic_acronym, TokenAction::Noop));
}
#[test]
fn multiletter_identifier_allows_conjunctive_suffix() {
let tokens = vec![
word_tok("AB와"),
space_tok(),
word_tok("CD의"),
space_tok(),
word_tok("값을"),
];
let mut state = EncoderState::new(false);
let action = run(&tokens, 2, &mut state).expect("ok");
assert!(matches!(action, TokenAction::ReplaceMany(_)));
}
#[test]
fn greek_letter_list_with_korean_suffix() {
let list = enc_str("그래서 α, β에 대해");
let plain = enc_str("그래서 α에 대해");
assert!(!list.is_empty());
assert_ne!(list, plain, "α, β list must differ from single α");
}
#[test]
fn math_ellipsis_after_math_letter() {
let with_ctx = enc_str("x... ");
let without_ctx = enc_str("...");
assert_ne!(
with_ctx, without_ctx,
"ellipsis after math letter must differ from standalone ellipsis"
);
}
#[test]
fn therefore_between_content_gets_spaces() {
let with_ctx = enc_str("a ∴ b");
let alone = enc_str("∴");
assert_ne!(
with_ctx, alone,
"∴ between content must add spaces vs. standalone"
);
}
#[test]
fn uppercase_around_logic_symbol_treated_as_math() {
let logic = enc_str("A ⊻ B");
let only_left = enc_str("A ⊻");
assert_ne!(
logic, only_left,
"A ⊻ B with both neighbors must differ from A ⊻"
);
}
#[test]
fn logic_symbol_vs_plain_letter_neighbor() {
let logic = enc_str("A ⊻ B");
let plain = enc_str("A x B");
assert_ne!(
logic, plain,
"logic-symbol neighbor must take a different path than plain-letter neighbor"
);
}
#[test]
fn latex_single_letter_korean_prose_wrapping() {
let prose = enc_str("우리는 $x$를 구한다");
let standalone = enc_str("$x$");
assert_ne!(
prose, standalone,
"$x$ in prose must have boundary spacing/wrap"
);
}
#[test]
fn latex_comma_list_korean_prose() {
let prose = enc_str("점 $a,b,c$를 잡자");
let single = enc_str("점 $a$를 잡자");
assert_ne!(
prose, single,
"comma list LaTeX must differ from single-letter"
);
}
#[test]
fn latex_simple_numeric_no_extra_boundary() {
let num = enc_str("값은 $-2$이다");
let var = enc_str("값은 $x$이다");
assert_ne!(
num, var,
"simple numeric LaTeX must encode differently from single-letter"
);
}
#[test]
fn mixed_math_word_after_korean_word() {
let mixed = enc_str("저는 안녕x+y는 좋다");
assert!(!mixed.is_empty());
}
#[test]
fn substantial_math_after_korean() {
let with_paren = enc_str("그래서 f(x)는");
let just_var = enc_str("그래서 x는");
assert_ne!(
with_paren, just_var,
"substantial math must get prose boundary vs. single variable"
);
}
#[test]
fn combining_mark_or_special_char_triggers_decimal_spacing() {
let with_delta = enc_str("이전 ∆=10 이다");
let plain = enc_str("이전 x=10 이다");
assert_ne!(
with_delta, plain,
"∆ in expression must trigger different leading spacing"
);
}
#[test]
fn prev_next_words_neighbor_resolution() {
let solo = enc_str("A");
let both = enc_str("A ⊻ B");
let only_prev = enc_str("⊻ A");
assert_ne!(solo, both);
assert_ne!(only_prev, both);
}
#[test]
fn prev_next_words_prev_skips_single_space_to_word() {
let tokens: Vec<Token<'_>> = vec![word_tok("a"), space_tok(), word_tok("b")];
let (prev, next) = prev_next_words(&tokens, 2);
assert!(prev.is_some(), "prev must resolve to 'a' through space");
assert_eq!(prev.unwrap().text.as_ref(), "a");
assert!(next.is_none(), "no next");
}
#[test]
fn prev_next_words_next_skips_single_space_to_word() {
let tokens: Vec<Token<'_>> = vec![word_tok("a"), space_tok(), word_tok("b")];
let (prev, next) = prev_next_words(&tokens, 0);
assert!(prev.is_none());
assert!(next.is_some(), "next must resolve to 'b' through space");
assert_eq!(next.unwrap().text.as_ref(), "b");
}
#[test]
fn colon_math_each_operator_character() {
let ops: &[char] = &[
'\u{2272}', '\u{2273}', '\u{227A}', '\u{227B}', '\u{22BB}', '<', '>', '=', '\u{2260}',
'\u{2264}', '\u{2265}', '\u{2208}', '\u{2209}',
];
let mut any_succeeded = false;
for op in ops {
let input = format!("a {op} b:");
if let Ok(bytes) = crate::encode(&input)
&& !bytes.is_empty()
{
any_succeeded = true;
}
}
assert!(
any_succeeded,
"at least one colon-math operator must succeed"
);
}
#[test]
fn set_builder_with_non_word_token_between_breaks() {
let result = enc_str("{x|$\\frac{1}{2}$}");
assert!(!result.is_empty(), "set-builder with fraction must encode");
}
#[test]
fn multiletter_identifier_with_prev_korean_word_no_space() {
let result = enc_str("문제 ab의 값을 구하라");
assert!(!result.is_empty(), "Korean prev + ab의 must encode");
}
#[test]
fn multiletter_identifier_with_prev_preencoded_does_not_trigger() {
let result = enc_str("$x$ ab의 값을 구하라");
assert!(!result.is_empty(), "PreEncoded prev + ab의 must encode");
}
#[test]
fn greek_list_with_multi_space_between_pair() {
let result = enc_str("이것은 α, β에 대해");
assert!(!result.is_empty(), "α, β with multi-space must encode");
}
#[test]
fn greek_list_with_next_non_word_returns_none() {
let result = enc_str("이것 α, $x$에 대해");
assert!(!result.is_empty(), "greek list with next $x$ must encode");
}
#[test]
fn greek_list_prev_is_space_then_korean() {
let result = enc_str("이것 α, β에 대해");
assert!(
!result.is_empty(),
"α, β with Space-then-Korean prev must encode"
);
}
#[test]
fn math_ellipsis_after_preencoded_prev() {
let result = enc_str("$x$ ...");
assert!(!result.is_empty(), "$x$ ... must encode");
}
#[test]
fn math_ellipsis_after_fraction_prev() {
let result = enc_str("$\\frac{1}{2}$ ...");
assert!(!result.is_empty(), "fraction + ... must encode");
}
#[test]
fn math_ellipsis_followed_by_korean_word() {
let result = enc_str("x ... 그래서");
assert!(!result.is_empty(), "x ... 그래서 must encode");
}
#[test]
fn math_ellipsis_at_end_no_next() {
let result = enc_str("x...");
assert!(!result.is_empty(), "x... at end must encode");
}
#[test]
fn therefore_with_prev_space_then_preencoded() {
let result = enc_str("$x$ ∴ y");
assert!(!result.is_empty(), "$x$ ∴ y must encode");
}
#[test]
fn therefore_with_prev_fraction() {
let result = enc_str("$\\frac{1}{2}$ ∴ y");
assert!(!result.is_empty(), "fraction ∴ y must encode");
}
#[test]
fn therefore_followed_by_fraction() {
let result = enc_str("x ∴ $\\frac{1}{2}$");
assert!(!result.is_empty(), "x ∴ fraction must encode");
}
#[test]
fn latex_single_letter_in_korean_prose_wrap() {
let result = enc_str("우리는 $a$를 본다");
assert!(!result.is_empty(), "$a$ in prose must encode");
}
#[test]
fn latex_prev_through_space_is_preencoded() {
let result = enc_str("$x$ $y$를 본다");
assert!(!result.is_empty(), "$x$ $y$를 must encode");
}
#[test]
fn latex_with_no_space_before_content_word() {
let result = enc_str("abc$x+y$");
assert!(!result.is_empty(), "abc$x+y$ must encode");
}
#[test]
fn latex_fallthrough_to_general_wrap() {
let result = enc_str("$x+y$");
assert!(!result.is_empty(), "$x+y$ must encode");
}
#[test]
fn non_math_word_after_preencoded_with_space() {
let result = enc_str("$x$ 한국어");
assert!(!result.is_empty(), "$x$ 한국어 must encode");
}
#[test]
fn math_with_special_char_decimal_context_spacing() {
let result = enc_str("값 a⋯b 결과");
assert!(!result.is_empty(), "a⋯b must encode");
}
#[test]
fn special_incrementum_pattern_with_paren_plus_paren() {
let result = enc_str("이전 \u{2206}=(a+b)+(c+d)");
assert!(!result.is_empty(), "∆=(a+b)+(c+d) must encode");
}
#[test]
fn math_followed_by_ascii_word_not_korean() {
let result = enc_str("f(x) abc");
assert!(!result.is_empty(), "f(x) abc must encode");
}
#[test]
fn math_encoder_error_falls_back_to_noop() {
let mut state = EncoderState::new(false);
let tokens = vec![word_tok("3+\u{FFFD}")];
let result = run(&tokens, 0, &mut state);
let _ = result;
}
#[test]
fn math_ellipsis_with_comma_then_korean() {
let result = enc_str("x..., 그래서");
assert!(!result.is_empty(), "x..., 그래서 must encode");
}
#[test]
fn dollar_single_letter_korean_prose_wrap_direct() {
let tokens = vec![word_tok("$x$를")];
let mut state = EncoderState::new(false);
let action = run(&tokens, 0, &mut state).expect("ok");
let TokenAction::ReplaceMany(replacement) = action else {
panic!("expected ReplaceMany");
};
let Token::PreEncoded(bytes) = &replacement[0] else {
panic!("expected PreEncoded first");
};
assert_eq!(bytes.first(), Some(&52u8));
assert_eq!(bytes.last(), Some(&50u8));
}
#[test]
fn dollar_comma_list_korean_prose_wrap_direct() {
let tokens = vec![word_tok("$a,b,c$를")];
let mut state = EncoderState::new(false);
let action = run(&tokens, 0, &mut state).expect("ok");
assert!(matches!(action, TokenAction::ReplaceMany(_)));
}
#[test]
fn dollar_two_letter_korean_prose_plain_path() {
let tokens = vec![word_tok("$xy$의")];
let mut state = EncoderState::new(false);
let action = run(&tokens, 0, &mut state).expect("ok");
assert!(matches!(action, TokenAction::ReplaceMany(_)));
}
#[test]
fn dollar_single_letter_no_suffix() {
let tokens = vec![word_tok("$x$")];
let mut state = EncoderState::new(false);
let action = run(&tokens, 0, &mut state).expect("ok");
let _ = action;
}
#[test]
fn multi_letter_korean_ident_prev_direct_korean_word() {
let tokens = vec![word_tok("문제"), word_tok("ab의"), word_tok("친구")];
let mut state = EncoderState::new(false);
let action = run(&tokens, 1, &mut state).expect("ok");
assert!(matches!(action, TokenAction::Noop));
}
#[test]
fn multi_letter_korean_ident_prev_fraction_falls_through() {
let tokens = vec![
Token::Fraction(crate::rules::token::FractionToken {
whole: None,
numerator: "1".to_string(),
denominator: "2".to_string(),
}),
word_tok("ab의"),
word_tok("친구"),
];
let mut state = EncoderState::new(false);
let action = run(&tokens, 1, &mut state).expect("ok");
let _ = action;
}
#[test]
fn uppercase_identifier_after_korean_word_uses_math_letter_path() {
let tokens = vec![
word_tok("문제"),
word_tok("AB의"),
space_tok(),
word_tok("값을"),
];
let mut state = EncoderState::new(false);
let action = run(&tokens, 1, &mut state).expect("ok");
assert!(matches!(action, TokenAction::ReplaceMany(_)));
}
#[test]
fn uppercase_identifier_after_non_word_falls_through_prev_check() {
let tokens = vec![
Token::Fraction(crate::rules::token::FractionToken {
whole: None,
numerator: "1".to_string(),
denominator: "2".to_string(),
}),
word_tok("AB의"),
];
let mut state = EncoderState::new(false);
let action = run(&tokens, 1, &mut state).expect("ok");
assert!(matches!(action, TokenAction::Noop));
}
#[test]
fn dollar_letter_prev_fraction_token() {
let tokens = vec![
Token::Fraction(crate::rules::token::FractionToken {
whole: None,
numerator: "1".to_string(),
denominator: "2".to_string(),
}),
word_tok("$x$를"),
];
let mut state = EncoderState::new(false);
let action = run(&tokens, 1, &mut state).expect("ok");
assert!(matches!(action, TokenAction::ReplaceMany(_)));
}
#[test]
fn dollar_letter_prev_preencoded_no_space_two_leading() {
let tokens = vec![Token::PreEncoded(vec![1]), word_tok("$x$")];
let mut state = EncoderState::new(false);
let action = run(&tokens, 1, &mut state).expect("ok");
let TokenAction::ReplaceMany(replacement) = action else {
panic!("expected ReplaceMany");
};
if let Token::PreEncoded(bytes) = &replacement[0] {
assert_eq!(bytes.len(), 2);
assert!(bytes.iter().all(|b| *b == 0));
} else {
panic!("expected leading PreEncoded(spaces)");
}
}
#[test]
fn dollar_letter_prev_direct_korean_word() {
let tokens = vec![word_tok("한글"), word_tok("$x$의")];
let mut state = EncoderState::new(false);
let action = run(&tokens, 1, &mut state).expect("ok");
let TokenAction::ReplaceMany(replacement) = action else {
panic!("expected ReplaceMany");
};
let Token::PreEncoded(bytes) = &replacement[0] else {
panic!("expected PreEncoded first");
};
assert_eq!(bytes.first(), Some(&52u8));
assert_eq!(bytes.last(), Some(&50u8));
}
#[test]
fn dollar_letter_prev_preencoded_falls_through() {
let tokens = vec![Token::PreEncoded(vec![1, 2, 3]), word_tok("$x$를")];
let mut state = EncoderState::new(false);
let action = run(&tokens, 1, &mut state).expect("ok");
assert!(matches!(action, TokenAction::ReplaceMany(_)));
}
#[test]
fn set_builder_with_preencoded_inside_breaks_loop() {
let tokens = vec![
word_tok("{x|"),
Token::PreEncoded(vec![42, 42]),
word_tok("}"),
];
let mut state = EncoderState::new(false);
let _ = run(&tokens, 0, &mut state).expect("ok");
}
#[test]
fn ellipsis_prev_preencoded_no_space() {
let tokens = vec![Token::PreEncoded(vec![1, 2, 3]), word_tok("...")];
let mut state = EncoderState::new(false);
let action = run(&tokens, 1, &mut state).expect("ok");
assert!(matches!(action, TokenAction::Replace(_)));
}
#[test]
fn ellipsis_prev_math_letter_word() {
let tokens = vec![word_tok("a,b,c"), word_tok("...")];
let mut state = EncoderState::new(false);
let action = run(&tokens, 1, &mut state).expect("ok");
assert!(matches!(action, TokenAction::Replace(_)));
}
#[test]
fn ellipsis_prev_subscript_digit_word() {
let tokens = vec![word_tok("x\u{2081}"), word_tok("...")];
let mut state = EncoderState::new(false);
let action = run(&tokens, 1, &mut state).expect("ok");
assert!(matches!(action, TokenAction::Replace(_)));
}
#[test]
fn greek_list_prev_direct_korean_word() {
let tokens = vec![word_tok("각"), word_tok("α,"), word_tok("β에 대하여")];
let mut state = EncoderState::new(false);
let action = run(&tokens, 1, &mut state).expect("ok");
let _ = action;
}
#[test]
fn greek_list_prev_space_with_non_korean_prev_prev() {
let tokens = vec![
word_tok("hello"), space_tok(),
word_tok("α,"),
space_tok(),
word_tok("β에"),
];
let mut state = EncoderState::new(false);
let action = run(&tokens, 2, &mut state).expect("ok");
let _ = action;
}
#[test]
fn therefore_between_preencoded_both_sides() {
let tokens = vec![
Token::PreEncoded(vec![1]),
space_tok(),
word_tok("∴"),
space_tok(),
Token::PreEncoded(vec![2]),
];
let mut state = EncoderState::new(false);
let action = run(&tokens, 2, &mut state).expect("ok");
assert!(matches!(action, TokenAction::Replace(_)));
}
#[test]
fn prev_next_words_next_runs_off_end() {
let tokens: Vec<Token<'_>> = vec![word_tok("a"), space_tok()];
let (prev, next) = prev_next_words(&tokens, 0);
assert!(prev.is_none());
assert!(next.is_none());
}
#[test]
fn prev_next_words_prev_runs_off_beginning() {
let tokens: Vec<Token<'_>> = vec![space_tok(), word_tok("a")];
let (prev, _next) = prev_next_words(&tokens, 1);
assert!(prev.is_none());
}
#[test]
fn next_word_skip_space_trails_off_end() {
let tokens: Vec<Token<'_>> = vec![space_tok(), space_tok()];
assert!(next_word_skip_space(&tokens, 0).is_none());
}
#[test]
fn next_indexed_word_skip_space_trails_off_end() {
let tokens: Vec<Token<'_>> = vec![space_tok(), space_tok()];
assert!(next_indexed_word_skip_space(&tokens, 0).is_none());
}
#[test]
fn has_content_skipping_space_forward_paths() {
let only_spaces = vec![word_tok("x"), space_tok(), space_tok()];
assert!(!has_content_skipping_space_forward(&only_spaces, 0));
let with_word = vec![word_tok("x"), space_tok(), word_tok("y")];
assert!(has_content_skipping_space_forward(&with_word, 0));
let with_pre = vec![word_tok("x"), Token::PreEncoded(vec![1])];
assert!(has_content_skipping_space_forward(&with_pre, 0));
let with_frac = vec![
word_tok("x"),
Token::Fraction(crate::rules::token::FractionToken {
whole: None,
numerator: "1".to_string(),
denominator: "2".to_string(),
}),
];
assert!(!has_content_skipping_space_forward(&with_frac, 0));
}
#[test]
fn has_content_skipping_space_backward_paths() {
let only_spaces = vec![space_tok(), space_tok(), word_tok("x")];
assert!(!has_content_skipping_space_backward(&only_spaces, 2));
let with_word = vec![word_tok("y"), space_tok(), word_tok("x")];
assert!(has_content_skipping_space_backward(&with_word, 2));
let with_pre = vec![Token::PreEncoded(vec![1]), word_tok("x")];
assert!(has_content_skipping_space_backward(&with_pre, 1));
let with_frac = vec![
Token::Fraction(crate::rules::token::FractionToken {
whole: None,
numerator: "1".to_string(),
denominator: "2".to_string(),
}),
word_tok("x"),
];
assert!(!has_content_skipping_space_backward(&with_frac, 1));
}
#[test]
fn math_encoder_failure_falls_through_to_noop() {
let tokens = vec![word_tok("\u{2211}(i=1")];
let mut state = EncoderState::new(false);
let action = run(&tokens, 0, &mut state).expect("run must not error");
let _ = action;
}
#[test]
fn prev_is_math_context_for_ellipsis_non_word_terminator() {
let tokens = vec![
Token::Fraction(crate::rules::token::FractionToken {
whole: None,
numerator: "1".to_string(),
denominator: "2".to_string(),
}),
word_tok("..."),
];
assert!(!prev_is_math_context_for_ellipsis(&tokens, 1));
}
#[test]
fn word_is_math_letter_context_branches() {
let super_word = word_tok("a²");
if let Token::Word(w) = &super_word {
assert!(word_is_math_letter_context(w));
}
let letter_list = word_tok("abc");
if let Token::Word(w) = &letter_list {
assert!(word_is_math_letter_context(w));
}
let korean = word_tok("한글");
if let Token::Word(w) = &korean {
assert!(!word_is_math_letter_context(w));
}
}
#[test]
fn consecutive_ascii_letter_run_paths() {
assert!(is_consecutive_ascii_letter_run(&['A', 'B', 'C']));
assert!(!is_consecutive_ascii_letter_run(&['A']));
assert!(!is_consecutive_ascii_letter_run(&['A', 'C']));
}
#[test]
fn greek_list_at_start_of_input_no_prev_korean() {
let result = enc_str("α, β에 대해");
assert!(!result.is_empty(), "α, β at start must encode");
}
#[test]
fn run_leading_spaces_two_branch_via_direct_tokens() {
let mut state = EncoderState::new(false);
let tokens = vec![Token::PreEncoded(vec![1, 2, 3]), word_tok("$x^2$")];
let result = run(&tokens, 1, &mut state).unwrap();
assert!(!matches!(result, TokenAction::Noop));
}
#[test]
fn run_err_arm_returns_noop_for_unencodable_math() {
let mut state = EncoderState::new(false);
let tokens = vec![word_tok("$~$")];
let result = run(&tokens, 0, &mut state);
let _ = result;
}
}
#[cfg(test)]
mod math_identifier_cue_coverage {
#[rstest::rstest]
#[case::consecutive_capitals_without_a_cue("광명 GH의 주파수", false)]
#[case::non_consecutive_capitals("국방 FM의 주파수", false)]
#[case::lowercase_without_a_cue("표의 ab의 자리", false)]
#[case::three_capitals_without_a_cue("가나 GHI의 다라", false)]
#[case::other_particle("가나 GH를 다라", false)]
#[case::capitals_with_a_value_cue("행렬 A와 B에 대하여 AB의 값을 구하여라.", true)]
#[case::lowercase_with_a_value_cue("그래프가 대칭일 때, ab의 값을 구하여라.", true)]
fn a_letter_run_needs_a_math_cue_to_take_the_boundary(
#[case] input: &str,
#[case] is_math: bool,
) {
let encoded = crate::encode_to_unicode(input).unwrap();
assert_eq!(
encoded.contains("\u{2800}\u{2800}"),
is_math,
"unexpected Article 11 boundary in {encoded}"
);
}
}
#[cfg(test)]
mod article_11_boundary_coverage {
use super::*;
#[rstest::rstest]
#[case::clock("22:00", true)]
#[case::ratio("16:9", true)]
#[case::resolution("1280×720", true)]
#[case::range("150~200", true)]
#[case::model("F1.2", true)]
#[case::generation("5G·4G", true)]
#[case::unit_slash("3G/4G", true)]
#[case::equation("x=1", false)]
#[case::relation("2<3", false)]
#[case::variable_term("2x+3", false)]
#[case::no_digits("abc", false)]
fn a_number_notation_is_not_an_expression(#[case] text: &str, #[case] expected: bool) {
let chars: Vec<char> = text.chars().collect();
assert_eq!(is_prose_number_notation(&chars), expected);
}
#[rstest::rstest]
#[case::balanced("f(x)", true)]
#[case::nested("((a))", true)]
#[case::none("abc", true)]
#[case::trailing_close("LTE)", false)]
#[case::leading_open("S(PLAN", false)]
#[case::closes_first(")a(", false)]
fn an_expression_keeps_its_brackets_balanced(#[case] text: &str, #[case] expected: bool) {
let chars: Vec<char> = text.chars().collect();
assert_eq!(has_balanced_brackets(&chars), expected);
}
#[rstest::rstest]
#[case::compound("know-how", true)]
#[case::compound_with_comma("well-made,", true)]
#[case::subtraction("a-1", false)]
#[case::no_hyphen("know", false)]
fn a_hyphenated_roman_word_is_not_subtraction(#[case] text: &str, #[case] expected: bool) {
let chars: Vec<char> = text.chars().collect();
assert_eq!(is_hyphenated_roman_word(&chars), expected);
}
#[rstest::rstest]
#[case::age_unit("가나 A(37세)씨는 다라", false)]
#[case::rank("가나 A(4급)씨가 다라", false)]
#[case::nationality("가나 A(27·스리랑카)씨에 다라", false)]
#[case::plain_age("가나 A(54)씨는 다라", false)]
#[case::function_notation("가나 A(14)는 다라", true)]
fn a_person_label_is_not_a_function(#[case] input: &str, #[case] is_math: bool) {
let encoded = crate::encode_to_unicode(input).unwrap();
assert_eq!(
encoded.contains("\u{2800}\u{2800}"),
is_math,
"unexpected Article 11 boundary in {encoded}"
);
}
}
#[cfg(test)]
mod number_notation_routing_coverage {
use super::*;
#[rstest::rstest]
#[case::clock("7:30,", true)]
#[case::ratio("16:9", true)]
#[case::model("F1.2", true)]
#[case::function_name("log2", false)]
#[case::function_inside("2log7", false)]
#[case::trig("sin3x", false)]
#[case::digit_product("6\u{00B7}9", false)]
#[case::variable_product("3ab", false)]
fn a_number_notation_stays_off_the_math_engine(#[case] text: &str, #[case] expected: bool) {
let chars: Vec<char> = text.chars().collect();
assert_eq!(is_prose_number_notation(&chars), expected);
}
#[rstest::rstest]
#[case::bare("가나 7:30 다라")]
#[case::with_comma("가나 7:30, 다라")]
#[case::with_paren("가나 7:30) 다라")]
fn a_number_after_a_colon_keeps_its_number_sign(#[case] input: &str) {
let encoded = crate::encode_to_unicode(input).unwrap();
assert_eq!(
encoded.matches('\u{283C}').count(),
2,
"expected both number signs in {encoded}"
);
}
}
#[cfg(test)]
mod identifier_prev_token_coverage {
use super::*;
use crate::rules::token::{SpaceKind, WordMeta, WordToken};
use std::borrow::Cow;
fn word_tok(text: &str) -> Token<'_> {
let chars: Vec<char> = text.chars().collect();
let meta = WordMeta::from_chars(&chars);
Token::Word(WordToken {
text: Cow::Borrowed(text),
chars,
meta,
})
}
fn space_tok() -> Token<'static> {
Token::Space(SpaceKind::Regular)
}
#[test]
fn a_pre_encoded_token_before_the_identifier_blocks_the_boundary() {
let tokens = vec![
Token::PreEncoded(vec![1]),
word_tok("AB의"),
space_tok(),
word_tok("값을"),
];
let mut state = EncoderState::new(false);
let action = run(&tokens, 1, &mut state).expect("ok");
assert!(matches!(action, TokenAction::Noop));
}
}