use std::{borrow::Cow, cell::RefCell};
#[doc(hidden)]
pub mod corpus_analysis {
pub fn is_sentence_corpus_shard_name(name: &str) -> bool {
name.starts_with("sentence_") && name.ends_with(".json")
}
pub fn has_ascii_alphanumeric_before(input: &str, byte_index: usize) -> bool {
input
.get(..byte_index)
.unwrap_or_default()
.chars()
.next_back()
.is_some_and(|previous| previous.is_ascii_alphanumeric())
}
#[cfg(test)]
mod tests {
use super::*;
#[rstest::rstest]
#[case::sentence_json("sentence_000.json", true)]
#[case::wrong_prefix("document_000.json", false)]
#[case::wrong_extension("sentence_000.txt", false)]
fn classifies_sentence_corpus_shard_names(#[case] name: &str, #[case] expected: bool) {
assert_eq!(is_sentence_corpus_shard_name(name), expected);
}
#[rstest::rstest]
#[case::start_of_input("A(14)", 0, false)]
#[case::ascii_letter("BA(14)", 1, true)]
#[case::ascii_digit("1A(14)", 1, true)]
#[case::korean_scalar("가A(14)", 3, false)]
#[case::non_scalar_boundary("가A(14)", 1, false)]
fn detects_ascii_alphanumeric_immediately_before_boundary(
#[case] input: &str,
#[case] byte_index: usize,
#[case] expected: bool,
) {
assert_eq!(has_ascii_alphanumeric_before(input, byte_index), expected);
}
}
}
mod char_shortcut;
pub(crate) mod char_struct;
#[cfg(feature = "cli")]
pub mod cli;
mod encoder;
pub(crate) mod english;
pub(crate) mod english_logic;
pub(crate) mod fraction;
mod hanja;
mod ipa;
mod jauem;
mod korean_char;
mod korean_part;
mod math_symbol_shortcut;
mod moeum;
pub(crate) mod number;
mod rule;
pub(crate) mod rules;
mod split;
pub(crate) mod symbol_shortcut;
pub(crate) mod unicode;
pub(crate) mod utils;
pub(crate) mod word_shortcut;
use ipa::{detect_ipa_context, encode_ipa, is_ipa_phonetic_symbol};
#[cfg(test)]
mod test_helpers {
use crate::char_struct::CharType;
use crate::rules::context::{EncoderState, RuleContext};
pub(crate) struct CtxOwned {
pub word_chars: Vec<char>,
pub char_types: Vec<CharType>,
pub skip_count: usize,
pub state: EncoderState,
pub result: Vec<u8>,
pub prev_word: String,
pub remaining_words: Vec<String>,
pub roman_section_continues_from_previous_word: bool,
}
impl CtxOwned {
pub(crate) fn for_text(text: &str, english_indicator: bool) -> Self {
let word_chars: Vec<char> = text.chars().collect();
let char_types: Vec<CharType> = word_chars
.iter()
.map(|c| CharType::new(*c).expect("CharType::new should not fail in tests"))
.collect();
Self {
word_chars,
char_types,
skip_count: 0,
state: EncoderState::new(english_indicator),
result: Vec::new(),
prev_word: String::new(),
remaining_words: Vec::new(),
roman_section_continues_from_previous_word: false,
}
}
pub(crate) fn with_prev_word(mut self, prev_word: impl Into<String>) -> Self {
self.prev_word = prev_word.into();
self
}
pub(crate) fn with_roman_section_continuation(mut self) -> Self {
self.roman_section_continues_from_previous_word = true;
self
}
pub(crate) fn with_remaining_words<I, S>(mut self, words: I) -> Self
where
I: IntoIterator<Item = S>,
S: Into<String>,
{
self.remaining_words = words.into_iter().map(Into::into).collect();
self
}
pub(crate) fn ctx_at<'a>(&'a mut self, index: usize) -> RuleContext<'a> {
let remaining: Vec<&str> = self.remaining_words.iter().map(String::as_str).collect();
let leaked: &'a [&'a str] = Box::leak(remaining.into_boxed_slice());
RuleContext {
word_chars: &self.word_chars,
index,
char_type: &self.char_types[index],
prev_word: &self.prev_word,
remaining_words: leaked,
has_korean_char: self.word_chars.iter().any(|c| {
let cp = *c as u32;
(0xAC00..=0xD7A3).contains(&cp)
}),
is_all_uppercase: false,
ascii_starts_at_beginning: false,
roman_section_continues_from_previous_word: self
.roman_section_continues_from_previous_word,
skip_count: &mut self.skip_count,
state: &mut self.state,
result: &mut self.result,
}
}
}
}
pub use encoder::Encoder;
thread_local! {
static ENCODER_CACHE: RefCell<Option<Encoder>> = const { RefCell::new(None) };
}
fn with_encoder<F, R>(english_indicator: bool, f: F) -> R
where
F: FnOnce(&mut Encoder) -> R,
{
ENCODER_CACHE.with(|cell| {
let Ok(mut cached) = cell.try_borrow_mut() else {
let mut encoder = Encoder::new(english_indicator);
encoder.reset_state();
return f(&mut encoder);
};
if !matches!(&*cached, Some(encoder) if encoder.english_indicator() == english_indicator) {
*cached = Some(Encoder::new(english_indicator));
}
let encoder = cached.as_mut().expect("encoder cache just initialized");
encoder.reset_state();
f(encoder)
})
}
#[derive(Debug, Clone, Default)]
pub struct EncodeOptions {
pub default_mode: Option<crate::rules::context::EncodingMode>,
}
#[derive(Debug, Clone)]
pub struct FormattingSpan {
pub range: std::ops::Range<usize>,
pub kind: FormattingKind,
}
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub enum FormattingKind {
Emphasis,
Bold,
Custom1,
Custom2,
}
impl FormattingKind {
pub(crate) fn markers(self) -> ([u8; 2], [u8; 2]) {
match self {
Self::Emphasis => ([32, 36], [36, 4]),
Self::Bold => ([48, 36], [36, 6]),
Self::Custom1 => ([16, 36], [36, 2]),
Self::Custom2 => ([8, 36], [36, 1]),
}
}
}
pub fn encode(text: &str) -> Result<Vec<u8>, String> {
encode_with_options(text, &EncodeOptions::default())
}
fn normalize_math_alphanumeric_char(c: char) -> char {
let cp = c as u32;
if cp == 0x210E {
return 'h';
}
const BLOCKS: &[(u32, char)] = &[
(0x1D400, 'A'),
(0x1D41A, 'a'),
(0x1D434, 'A'),
(0x1D44E, 'a'),
(0x1D468, 'A'),
(0x1D482, 'a'),
(0x1D49C, 'A'),
(0x1D4B6, 'a'),
(0x1D4D0, 'A'),
(0x1D4EA, 'a'),
(0x1D504, 'A'),
(0x1D51E, 'a'),
(0x1D538, 'A'),
(0x1D552, 'a'),
(0x1D56C, 'A'),
(0x1D586, 'a'),
(0x1D5A0, 'A'),
(0x1D5BA, 'a'),
(0x1D5D4, 'A'),
(0x1D5EE, 'a'),
(0x1D608, 'A'),
(0x1D622, 'a'),
(0x1D63C, 'A'),
(0x1D656, 'a'),
(0x1D670, 'A'),
(0x1D68A, 'a'),
];
for &(start, base) in BLOCKS {
if cp >= start && cp < start + 26 {
return char::from(base as u8 + (cp - start) as u8);
}
}
const DIGIT_BLOCKS: &[u32] = &[0x1D7CE, 0x1D7D8, 0x1D7E2, 0x1D7EC, 0x1D7F6];
for &start in DIGIT_BLOCKS {
if cp >= start && cp < start + 10 {
return char::from(b'0' + (cp - start) as u8);
}
}
c
}
fn may_normalize_math_alphanumeric(c: char) -> bool {
let cp = c as u32;
cp == 0x210E || (0x1D400..=0x1D7FF).contains(&cp)
}
fn normalize_math_alphanumeric_string(text: &str) -> Cow<'_, str> {
if !text.chars().any(may_normalize_math_alphanumeric) {
return Cow::Borrowed(text);
}
Cow::Owned(text.chars().map(normalize_math_alphanumeric_char).collect())
}
fn may_normalize_roman_numeral_presentation(c: char) -> bool {
(0x2160..=0x217f).contains(&(c as u32))
}
fn may_normalize_parenthesized_hangul_presentation(c: char) -> bool {
(0x3200..=0x321e).contains(&(c as u32))
}
fn may_normalize_word_separator_middle_dot(c: char) -> bool {
c == '\u{2e31}'
}
fn pure_roman_compatibility_unit_decomposition(c: char) -> Option<Vec<char>> {
use unicode_normalization::UnicodeNormalization;
let parts =
crate::rules::korean::rule_69::compatibility_unit_decomposition(c).or_else(|| {
crate::rules::korean::rule_68::is_rule_68_symbol(c)
.then(|| std::iter::once(c).nfkc().collect())
})?;
parts.iter().all(char::is_ascii_alphabetic).then_some(parts)
}
fn normalize_roman_numeral_presentation<'a>(text: Cow<'a, str>) -> Cow<'a, str> {
use unicode_normalization::UnicodeNormalization;
let mut out = String::with_capacity(text.len());
for ch in text.chars() {
if may_normalize_roman_numeral_presentation(ch) {
out.extend(std::iter::once(ch).nfkc());
} else {
out.push(ch);
}
}
Cow::Owned(out)
}
fn normalize_parenthesized_hangul_presentation<'a>(text: Cow<'a, str>) -> Cow<'a, str> {
use unicode_normalization::UnicodeNormalization;
let mut out = String::with_capacity(text.len());
for ch in text.chars() {
if may_normalize_parenthesized_hangul_presentation(ch) {
out.extend(std::iter::once(ch).nfkc());
} else {
out.push(ch);
}
}
Cow::Owned(out)
}
fn normalize_word_separator_middle_dot<'a>(text: Cow<'a, str>) -> Cow<'a, str> {
let mut out = String::with_capacity(text.len());
for ch in text.chars() {
if may_normalize_word_separator_middle_dot(ch) {
out.push(' ');
} else {
out.push(ch);
}
}
Cow::Owned(out)
}
fn normalize_pure_roman_compatibility_units<'a>(text: Cow<'a, str>) -> Cow<'a, str> {
let chars = text.chars().collect::<Vec<_>>();
let mut out = String::with_capacity(text.len());
for (index, ch) in chars.iter().copied().enumerate() {
let is_roman_unit_component = |candidate: char| {
candidate.is_ascii_alphabetic()
|| candidate == 'μ'
|| pure_roman_compatibility_unit_decomposition(candidate).is_some()
};
let directly_joined = index
.checked_sub(1)
.and_then(|previous| chars.get(previous))
.is_some_and(|previous| is_roman_unit_component(*previous))
|| chars
.get(index + 1)
.is_some_and(|next| is_roman_unit_component(*next));
let joined_through_slash = (index >= 2
&& matches!(chars[index - 1], '/' | '\u{2044}' | '\u{2215}')
&& is_roman_unit_component(chars[index - 2]))
|| (index + 2 < chars.len()
&& matches!(chars[index + 1], '/' | '\u{2044}' | '\u{2215}')
&& is_roman_unit_component(chars[index + 2]));
if (directly_joined || joined_through_slash)
&& let Some(parts) = pure_roman_compatibility_unit_decomposition(ch)
{
out.extend(parts);
} else {
out.push(ch);
}
}
Cow::Owned(out)
}
fn is_foldable_fullwidth(c: char) -> bool {
matches!(c, '\u{FF01}'..='\u{FF5E}') && !matches!(c, '\u{FF03}' | '\u{FF1A}')
}
fn parenthesized_number_expansion(c: char) -> Option<String> {
let value = (c as u32).checked_sub(0x2473)?;
(1..=20).contains(&value).then(|| format!("({value})"))
}
fn may_normalize_print_variant(c: char) -> bool {
matches!(
c,
'\u{02DA}' | '\u{2010}' | '\u{2011}' | '\u{2043}' | '\u{00AD}' | '\u{00B0}' | '²' | '³'
) || is_foldable_fullwidth(c)
|| parenthesized_number_expansion(c).is_some()
|| matches!(
c,
'\u{30FB}'
| '\u{FF65}'
| '\u{2027}'
| '\u{2024}'
| '\u{2A2F}'
| '\u{301C}'
| '\u{00B4}'
| '\u{200B}'
| '\u{200C}'
| '\u{200D}'
| '\u{FEFF}'
)
}
fn square_unit_presentation(letters: &[char], exponent: char) -> Option<char> {
use unicode_normalization::UnicodeNormalization;
let spelled: String = letters.iter().chain(std::iter::once(&exponent)).collect();
(0x3371..=0x33DF)
.filter_map(char::from_u32)
.find(|candidate| std::iter::once(*candidate).nfkc().eq(spelled.chars()))
}
fn degree_unit_glyph(ch: char, next: Option<char>) -> Option<char> {
if !matches!(ch, '\u{00B0}' | '\u{02DA}') {
return None;
}
match next {
Some('C') => Some('\u{2103}'),
Some('F') => Some('\u{2109}'),
_ => None,
}
}
fn normalize_print_variants<'a>(text: Cow<'a, str>) -> Cow<'a, str> {
let chars = text.chars().collect::<Vec<_>>();
let mut out = String::with_capacity(text.len());
let mut index = 0usize;
while index < chars.len() {
let ch = chars[index];
if let Some(glyph) = degree_unit_glyph(ch, chars.get(index + 1).copied()) {
out.push(glyph);
index += 2;
continue;
}
let starts_letter_run = ch.is_ascii_alphabetic()
&& index
.checked_sub(1)
.is_none_or(|previous| !chars[previous].is_ascii_alphabetic());
if starts_letter_run {
let run_end = index
+ chars[index..]
.iter()
.take_while(|c| c.is_ascii_alphabetic())
.count();
let exponent = match chars.get(run_end) {
Some('²') => Some('2'),
Some('³') => Some('3'),
_ => None,
};
if let Some(exponent) = exponent
&& let Some(unit) = square_unit_presentation(&chars[index..run_end], exponent)
{
out.push(unit);
index = run_end + 1;
continue;
}
}
match ch {
'\u{02DA}' => out.push('\u{00B0}'),
'\u{2010}' | '\u{2011}' | '\u{2043}' => out.push('-'),
'\u{30FB}' | '\u{FF65}' | '\u{2027}' | '\u{2024}' => {
out.push('\u{00B7}');
}
'\u{2A2F}' => out.push('\u{00D7}'),
_ if parenthesized_number_expansion(ch).is_some() => {
out.push_str(&parenthesized_number_expansion(ch).unwrap_or_default());
}
'\u{301C}' => out.push('~'),
'\u{00B4}' => out.push('\''),
'\u{00AD}' | '\u{200B}' | '\u{200C}' | '\u{200D}' | '\u{FEFF}' => {}
_ if is_foldable_fullwidth(ch) => {
out.push(char::from_u32(ch as u32 - 0xFEE0).unwrap_or(ch));
}
_ => out.push(ch),
}
index += 1;
}
Cow::Owned(out)
}
fn is_middle_korean_hanja_context(chars: &[char]) -> bool {
let has_old_jamo = chars.iter().any(|c| {
matches!(*c, '\u{3131}'..='\u{318E}' | '\u{1100}'..='\u{11FF}' | '\u{E000}'..='\u{F8FF}')
});
let has_tone_mark = chars.iter().any(|c| matches!(*c, '\u{00B7}' | '\u{FF1A}'));
let has_modern_hangul = chars.iter().any(|c| matches!(*c, '\u{AC00}'..='\u{D7A3}'));
let has_gloss = chars.iter().enumerate().any(|(index, c)| {
hanja::reading(*c).is_some_and(|reading| {
chars[index + 1..]
.iter()
.take(reading.chars().count())
.copied()
.eq(reading.chars())
})
});
has_old_jamo || has_tone_mark || has_gloss || !has_modern_hangul
}
fn expand_hanja_readings<'a>(text: Cow<'a, str>) -> Cow<'a, str> {
let chars: Vec<char> = text.chars().collect();
if is_middle_korean_hanja_context(&chars) {
return text;
}
let mut out = String::with_capacity(text.len());
for ch in chars {
match hanja::reading(ch) {
Some(reading) => out.push_str(reading),
None => out.push(ch),
}
}
Cow::Owned(out)
}
fn may_expand_pictograph(c: char) -> bool {
matches!(c, '\u{260E}' | '\u{260F}')
}
fn expand_pictographs<'a>(text: Cow<'a, str>) -> Cow<'a, str> {
let mut out = String::with_capacity(text.len());
for ch in text.chars() {
if may_expand_pictograph(ch) {
out.push_str("Tel");
} else {
out.push(ch);
}
}
Cow::Owned(out)
}
fn default_math_expression_needs_whole_route(text: &str) -> bool {
let chars: Vec<char> = text.chars().collect();
if chars.len() < 2 {
return false;
}
let has_operand = chars.iter().any(|c| c.is_ascii_alphanumeric());
has_operand
&& chars.iter().enumerate().any(|(i, c)| match *c {
'→' | '←' | '↗' | '↘' | '↑' | '↓' | '△' | '□' | '≅' | '▷' | '◁' => {
true
}
'\u{0304}' | '\u{0302}' => combining_mark_on_single_letter(&chars, i),
_ => false,
})
}
fn combining_mark_on_single_letter(chars: &[char], i: usize) -> bool {
let is_combining = |c: char| ('\u{0300}'..='\u{036F}').contains(&c);
let mut letters = 0usize;
let mut j = i;
while j > 0 {
let c = chars[j - 1];
if c.is_alphabetic() {
letters += 1;
} else if !is_combining(c) {
break;
}
j -= 1;
}
let mut j = i + 1;
while j < chars.len() {
let c = chars[j];
if c.is_alphabetic() {
letters += 1;
} else if !is_combining(c) {
break;
}
j += 1;
}
letters < 3
}
#[derive(Clone, Copy, Default)]
struct NormalizationTriggers {
has_math_alphanumeric: bool,
has_roman_numeral_presentation: bool,
has_parenthesized_hangul_presentation: bool,
has_word_separator_middle_dot: bool,
has_pure_roman_compatibility_unit: bool,
has_print_variant: bool,
has_hanja: bool,
has_pictograph: bool,
has_decomposable_latin: bool,
has_negation_combiner: bool,
has_vector_mark: bool,
has_formatting_mark_or_sentinel: bool,
has_ipa_group_start: bool,
has_ipa_symbol: bool,
}
impl NormalizationTriggers {
fn scan(text: &str) -> Self {
let mut triggers = Self::default();
for c in text.chars() {
triggers.has_math_alphanumeric |= may_normalize_math_alphanumeric(c);
triggers.has_roman_numeral_presentation |= may_normalize_roman_numeral_presentation(c);
triggers.has_parenthesized_hangul_presentation |=
may_normalize_parenthesized_hangul_presentation(c);
triggers.has_word_separator_middle_dot |= may_normalize_word_separator_middle_dot(c);
triggers.has_pure_roman_compatibility_unit |=
pure_roman_compatibility_unit_decomposition(c).is_some();
triggers.has_print_variant |= may_normalize_print_variant(c);
triggers.has_hanja |= hanja::is_hanja(c);
triggers.has_pictograph |= may_expand_pictograph(c);
triggers.has_decomposable_latin |= may_decompose_accented_latin(c);
triggers.has_negation_combiner |= c == '\u{0338}';
triggers.has_vector_mark |= is_vector_mark(c);
triggers.has_formatting_mark_or_sentinel |=
is_formatting_mark(c) || is_formatting_sentinel(c);
triggers.has_ipa_group_start |= matches!(c, '[' | '/');
triggers.has_ipa_symbol |= is_ipa_phonetic_symbol(c);
}
triggers
}
fn may_need_emphasis_expansion(self) -> bool {
self.has_formatting_mark_or_sentinel || self.has_decomposable_latin
}
fn may_contain_ipa_context(self) -> bool {
self.has_ipa_group_start && self.has_ipa_symbol
}
}
fn move_negation_combiner_before_base<'a>(text: Cow<'a, str>) -> Cow<'a, str> {
if !text.as_ref().contains('\u{0338}') {
return text;
}
let source = text.as_ref();
let chars: Vec<char> = source.chars().collect();
let mut out = String::with_capacity(source.len());
let mut i = 0;
while i < chars.len() {
if i + 1 < chars.len() && chars[i + 1] == '\u{0338}' {
out.push(chars[i + 1]);
out.push(chars[i]);
i += 2;
} else {
out.push(chars[i]);
i += 1;
}
}
Cow::Owned(out)
}
fn expand_emphasis_marks<'a>(text: Cow<'a, str>) -> Cow<'a, str> {
const FORMATTING_MARKS: &[(char, char, char)] = &[
('\u{0307}', '\u{E000}', '\u{E001}'), ('\u{0331}', '\u{E002}', '\u{E003}'), ('\u{0332}', '\u{E004}', '\u{E005}'), ('\u{0333}', '\u{E006}', '\u{E007}'), ];
if !text
.as_ref()
.chars()
.any(|c| is_formatting_sentinel(c) || is_formatting_mark(c))
{
return text;
}
let source = text.as_ref();
let chars: Vec<char> = source.chars().collect();
let mut token_has_korean = vec![false; chars.len()];
{
let mut i = 0;
while i < chars.len() {
if chars[i] == ' ' {
i += 1;
continue;
}
let start = i;
while i < chars.len() && chars[i] != ' ' {
i += 1;
}
let has = chars[start..i].iter().any(|c| utils::is_korean_char(*c));
for slot in token_has_korean.iter_mut().take(i).skip(start) {
*slot = has;
}
}
}
let mut out: Vec<char> = Vec::with_capacity(chars.len());
let mut i = 0;
while i < chars.len() {
let mark_entry = FORMATTING_MARKS
.iter()
.find(|(mark, _, _)| *mark == chars[i]);
let Some(&(mark_char, start_sentinel, end_sentinel)) = mark_entry else {
out.push(chars[i]);
i += 1;
continue;
};
if !token_has_korean[i] {
out.push(chars[i]);
i += 1;
continue;
}
let mut count = 1;
let mut last = i;
let mut j = i + 1;
while j < chars.len() {
if chars[j] == mark_char {
count += 1;
last = j;
j += 1;
} else if chars[j] == ' ' && j + 1 < chars.len() && chars[j + 1] == mark_char {
j += 1;
} else {
break;
}
}
let mut units = 0;
let mut start_in_out = out.len();
while start_in_out > 0 && units < count {
let c = out[start_in_out - 1];
if c == ' ' || is_formatting_sentinel(c) {
start_in_out -= 1;
} else {
units += 1;
start_in_out -= 1;
}
}
if units == count {
while start_in_out > 0 {
let c = out[start_in_out - 1];
if c.is_ascii_digit() || matches!(c, ',' | '.') {
start_in_out -= 1;
} else {
break;
}
}
out.insert(start_in_out, start_sentinel);
out.push(end_sentinel);
} else {
for _ in 0..count {
out.push(mark_char);
}
}
i = last + 1;
}
merge_adjacent_formatting_wraps(Cow::Owned(out.into_iter().collect()))
}
fn is_formatting_sentinel(c: char) -> bool {
matches!(c as u32, 0xE000..=0xE007)
}
fn is_formatting_mark(c: char) -> bool {
matches!(c, '\u{0307}' | '\u{0331}' | '\u{0332}' | '\u{0333}')
}
fn merge_adjacent_formatting_wraps<'a>(text: Cow<'a, str>) -> Cow<'a, str> {
const SENTINEL_PAIRS: &[(char, char)] = &[
('\u{E000}', '\u{E001}'),
('\u{E002}', '\u{E003}'),
('\u{E004}', '\u{E005}'),
('\u{E006}', '\u{E007}'),
];
if !text.as_ref().chars().any(is_formatting_sentinel) {
return text;
}
let mut chars: Vec<char> = text.as_ref().chars().collect();
let mut any_changed = false;
let mut changed = true;
while changed {
changed = false;
for &(open, close) in SENTINEL_PAIRS {
let mut i = 0;
while i < chars.len() {
if chars[i] != close {
i += 1;
continue;
}
let mut j = i + 1;
while j < chars.len() && chars[j] == ' ' {
j += 1;
}
if j < chars.len() && chars[j] == open {
chars.remove(j);
chars.remove(i);
changed = true;
any_changed = true;
} else {
i += 1;
}
}
}
}
if any_changed {
Cow::Owned(chars.into_iter().collect())
} else {
text
}
}
fn is_vector_mark(c: char) -> bool {
matches!(c, '\u{20D6}' | '\u{20D7}' | '\u{20E1}' | '\u{20D1}')
}
fn collapse_repeated_vector_marks<'a>(text: Cow<'a, str>) -> Cow<'a, str> {
debug_assert!(text.as_ref().chars().any(is_vector_mark));
let source = text.as_ref();
let chars: Vec<char> = source.chars().collect();
let mut out = String::with_capacity(source.len());
let mut i = 0;
let mut changed = false;
while i < chars.len() {
if chars[i].is_ascii_alphabetic() && i + 1 < chars.len() && is_vector_mark(chars[i + 1]) {
changed = true;
let mark = chars[i + 1];
let mut letters = vec![chars[i]];
let mut j = i + 2;
while j + 1 < chars.len() && chars[j].is_ascii_alphabetic() && chars[j + 1] == mark {
letters.push(chars[j]);
j += 2;
}
out.push(mark);
for l in letters {
out.push(l);
}
i = j;
continue;
}
out.push(chars[i]);
i += 1;
}
if changed { Cow::Owned(out) } else { text }
}
fn may_decompose_accented_latin(c: char) -> bool {
let cp = c as u32;
!matches!(c, '\u{00C5}' | '\u{00E5}')
&& ((0x00C0..=0x024F).contains(&cp) || (0x1E00..=0x1EFF).contains(&cp))
}
fn decompose_accented_latin<'a>(text: Cow<'a, str>) -> Cow<'a, str> {
use unicode_normalization::UnicodeNormalization;
let mut out = String::new();
for c in text.as_ref().chars() {
if may_decompose_accented_latin(c) {
for d in std::iter::once(c).nfd() {
out.push(d);
}
} else {
out.push(c);
}
}
Cow::Owned(out)
}
fn is_isolated_roman_section(text: &str) -> bool {
let mut has_letter = false;
for ch in text.chars() {
if ch == ' ' {
continue;
}
if ch.is_ascii_alphabetic() {
has_letter = true;
} else {
return false;
}
}
has_letter
}
pub fn encode_with_options(text: &str, options: &EncodeOptions) -> Result<Vec<u8>, String> {
use crate::rules::context::EncodingMode;
let normalization_triggers = NormalizationTriggers::scan(text);
if options.default_mode.is_none()
&& !text.chars().any(crate::utils::is_korean_char)
&& crate::rules::english_ueb::is_ueb_eligible(text)
&& !crate::rules::english_ueb::is_math_owned(text)
&& let Some(bytes) = crate::rules::english_ueb::try_encode(text)
{
return Ok(bytes);
}
if matches!(options.default_mode, Some(EncodingMode::English))
&& let Some(bytes) = crate::rules::english_ueb::encode_forced(text)
{
return Ok(bytes);
}
let normalized_text = if normalization_triggers.has_math_alphanumeric {
normalize_math_alphanumeric_string(text)
} else {
Cow::Borrowed(text)
};
let normalized_text = if normalization_triggers.has_roman_numeral_presentation {
normalize_roman_numeral_presentation(normalized_text)
} else {
normalized_text
};
let normalized_text = if normalization_triggers.has_parenthesized_hangul_presentation {
normalize_parenthesized_hangul_presentation(normalized_text)
} else {
normalized_text
};
let normalized_text = if normalization_triggers.has_word_separator_middle_dot {
normalize_word_separator_middle_dot(normalized_text)
} else {
normalized_text
};
let normalized_text = if normalization_triggers.has_pure_roman_compatibility_unit {
normalize_pure_roman_compatibility_units(normalized_text)
} else {
normalized_text
};
let normalized_text = if normalization_triggers.has_print_variant {
normalize_print_variants(normalized_text)
} else {
normalized_text
};
let normalized_text = if normalization_triggers.has_hanja {
expand_hanja_readings(normalized_text)
} else {
normalized_text
};
let normalized_text = if normalization_triggers.has_pictograph {
expand_pictographs(normalized_text)
} else {
normalized_text
};
let normalized_text = if normalization_triggers.has_decomposable_latin {
decompose_accented_latin(normalized_text)
} else {
normalized_text
};
let normalized_text = if normalization_triggers.has_negation_combiner {
move_negation_combiner_before_base(normalized_text)
} else {
normalized_text
};
let normalized_text = if normalization_triggers.has_vector_mark {
collapse_repeated_vector_marks(normalized_text)
} else {
normalized_text
};
let normalized_text = if normalization_triggers.may_need_emphasis_expansion() {
expand_emphasis_marks(normalized_text)
} else {
normalized_text
};
let text: &str = normalized_text.as_ref();
let matrix_context = text.contains("행렬");
let math_mode = matches!(options.default_mode, Some(EncodingMode::Math));
let math_context = crate::rules::math::math_token_rule::MathContext {
matrix_context_active: matrix_context,
math_mode_active: math_mode,
};
let ipa_auto = options.default_mode.is_none()
&& normalization_triggers.may_contain_ipa_context()
&& detect_ipa_context(text);
if ipa_auto || matches!(options.default_mode, Some(EncodingMode::Ipa)) {
return encode_ipa(text);
}
if let Some(EncodingMode::ObjectSymbol) = options.default_mode {
let chars: Vec<char> = text.chars().collect();
if chars.len() == 1 {
let mark = match chars[0] {
'○' => Some(52u8), '×' => Some(45u8), '△' => Some(44u8), '□' => Some(54u8), _ => None,
};
if let Some(m) = mark {
return Ok(vec![56, m, 7]); }
}
}
if let Some(EncodingMode::Number) = options.default_mode {
let chars: Vec<char> = text.chars().collect();
if !chars.is_empty()
&& chars.iter().all(|c| {
matches!(
c.to_ascii_uppercase(),
'I' | 'V' | 'X' | 'L' | 'C' | 'D' | 'M'
)
})
{
let mut out = vec![52u8]; if chars.iter().all(|c| c.is_ascii_uppercase()) {
out.push(32); if chars.len() >= 2 {
out.push(32); }
}
for ch in &chars {
out.push(crate::english::encode_english(ch.to_ascii_lowercase())?);
}
out.push(50); return Ok(out);
}
}
let default_math_owned = options.default_mode.is_none()
&& default_math_expression_needs_whole_route(text)
&& !text.chars().any(crate::utils::is_korean_char);
if matches!(options.default_mode, Some(EncodingMode::Math)) || default_math_owned {
if math_mode
&& text.len() >= 3
&& text.starts_with('$')
&& text.ends_with('$')
&& text.matches('$').count() == 2
{
let inner = &text[1..text.len() - 1];
return crate::rules::token_rules::latex_math::encode_latex_math_bytes_with_context(
inner,
math_context,
);
}
let chars: Vec<char> = text.chars().collect();
if chars.len() == 1 && chars[0].is_ascii_lowercase() {
return Ok(vec![52, crate::english::encode_english(chars[0])?]);
}
if chars.len() == 1 {
match chars[0] {
'(' => return Ok(vec![38]), ')' => return Ok(vec![52]), '{' => return Ok(vec![54]), '}' => return Ok(vec![54]), '[' => return Ok(vec![55, 4]), ']' => return Ok(vec![32, 62]), _ => {}
}
}
if chars.len() == 1
&& let Ok(code) =
crate::math_symbol_shortcut::encode_char_math_symbol_shortcut(chars[0])
{
return Ok(code.to_vec());
}
let cleaned: String = {
let mut s = String::with_capacity(text.len());
let chs: Vec<char> = text.chars().collect();
let mut i = 0;
while i < chs.len() {
let c = chs[i];
if c == ' '
&& i + 1 < chs.len()
&& matches!(chs[i + 1], '=' | '+' | '-' | '<' | '>')
{
i += 1;
continue;
}
if matches!(c, '=' | '+' | '-' | '<' | '>')
&& i + 1 < chs.len()
&& chs[i + 1] == ' '
{
s.push(c);
i += 2;
continue;
}
s.push(c);
i += 1;
}
s
};
if let Ok(bytes) =
rules::math::encoder::encode_math_expression_with_context(&cleaned, math_context)
{
return Ok(bytes);
}
}
let english_indicator = text
.split(' ')
.filter(|w| !w.is_empty())
.any(|word| word.chars().any(utils::is_korean_char));
with_encoder(english_indicator, |encoder| {
encoder.set_matrix_context_active(matrix_context);
encoder.set_math_mode_active(math_mode);
if let Some(mode) = options.default_mode {
encoder.set_default_mode(mode);
}
let mut result = Vec::new();
encoder.encode(text, &mut result)?;
let wrap_roman_section = matches!(options.default_mode, Some(EncodingMode::English))
|| (matches!(options.default_mode, Some(EncodingMode::Korean))
&& is_isolated_roman_section(text));
if wrap_roman_section && !result.is_empty() {
result.insert(0, 52);
result.push(50);
}
Ok(result)
})
}
pub fn encode_with_formatting(text: &str, spans: &[FormattingSpan]) -> Result<Vec<u8>, String> {
if spans.is_empty() {
return encode(text);
}
let english_indicator = text
.split(' ')
.filter(|w| !w.is_empty())
.any(|word| word.chars().any(utils::is_korean_char));
with_encoder(english_indicator, |encoder| {
let mut result = Vec::new();
encoder.encode_with_formatting(text, spans, &mut result)?;
Ok(result)
})
}
pub fn encode_to_unicode(text: &str) -> Result<String, String> {
let result = encode(text)?;
Ok(result
.iter()
.map(|c| unicode::encode_unicode(*c))
.collect::<String>())
}
pub fn encode_to_unicode_with_formatting(
text: &str,
spans: &[FormattingSpan],
) -> Result<String, String> {
let result = encode_with_formatting(text, spans)?;
Ok(result
.iter()
.map(|c| unicode::encode_unicode(*c))
.collect::<String>())
}
pub fn encode_to_braille_font(text: &str) -> Result<String, String> {
let result = encode(text)?;
Ok(result
.iter()
.map(|c| unicode::encode_unicode(*c))
.collect::<String>())
}
#[cfg(test)]
mod state_bleed_tests {
use super::encode;
#[test]
fn cached_encoder_resets_between_different_contexts() {
let before = encode("안녕").unwrap();
let _english = encode("hello").unwrap();
let after = encode("안녕").unwrap();
assert_eq!(before, after);
}
}
#[cfg(test)]
mod test {
use std::{collections::HashMap, fs::File};
use crate::{symbol_shortcut, unicode::encode_unicode};
use proptest::prelude::*;
use super::*;
fn find_nth_range(text: &str, needle: &str, _nth: usize) -> std::ops::Range<usize> {
let start = text
.find(needle)
.unwrap_or_else(|| panic!("substring '{needle}' not found in '{text}'"));
start..start + needle.len()
}
#[test]
fn english_continuation_after_inline_number() {
let output = encode("가 a1a").unwrap();
assert!(
output.contains(&48),
"inline number should trigger english continuation indicator"
);
}
#[test]
fn symbol_triggers_english_segment_at_start() {
let output = encode("(A 가").unwrap();
let english_symbol = symbol_shortcut::encode_english_char_symbol_shortcut('(').unwrap();
assert_eq!(output[0], 52);
assert!(output.len() > english_symbol.len());
assert_eq!(
&output[1..1 + english_symbol.len()],
english_symbol,
"opening english symbol should use english shortcut"
);
}
#[rstest::rstest]
#[case::slash_forced_symbol("가 a/")]
#[case::underscore_leave_english("가 a_b")]
fn english_symbol_terminator_variants(#[case] input: &str) {
let output = encode(input).unwrap();
assert!(
output.contains(&50),
"english terminator (50) absent for {input:?}"
);
}
#[test]
fn comma_prefix_variants_and_korean_following() {
let output = encode("가 A,가").unwrap();
let comma = symbol_shortcut::encode_char_symbol_shortcut(',').unwrap();
assert!(
output.windows(comma.len()).any(|window| window == comma),
"comma before Korean should use Korean punctuation mapping"
);
assert!(encode("가 A!,가").is_ok());
}
#[test]
fn next_word_single_letter_sets_continuation_flag() {
let output = encode("가 a b").unwrap();
assert!(
output.contains(&48),
"single-letter following word should trigger continuation marker"
);
}
#[rstest::rstest]
#[case::forced_symbol_inserts_terminator("가 a /", 50)]
#[case::skip_symbol_requests_continuation("가 a . b", 48)]
fn next_word_symbol_rules_apply(#[case] input: &str, #[case] expected_byte: u8) {
let output = encode(input).unwrap();
assert!(
output.contains(&expected_byte),
"expected byte {expected_byte} not in output for {input:?}"
);
}
#[test]
fn next_word_with_invalid_char_returns_error() {
let err = encode("가 a 😀");
assert!(err.is_err());
}
#[test]
fn encode_with_formatting_wraps_markers() {
let text = "다음 보기에서 명사가 아닌 것은?";
let spans = vec![FormattingSpan {
range: find_nth_range(text, "아닌", 0),
kind: FormattingKind::Emphasis,
}];
let unicode = encode_to_unicode_with_formatting(text, &spans).unwrap();
assert!(unicode.contains("⠠⠤⠣⠉⠟⠤⠄"));
}
#[test]
fn encode_with_formatting_rejects_non_boundary_range() {
let text = "왜";
let spans = [FormattingSpan {
range: 1..3,
kind: FormattingKind::Emphasis,
}];
let err = encode_with_formatting(text, &spans);
assert!(err.is_err());
}
#[derive(serde::Deserialize, Default)]
#[serde(rename_all = "camelCase")]
struct TestCaseRuleConfig {
#[serde(default)]
benchmark: bool,
#[serde(default)]
shards: bool,
}
type TestCaseRuleMap = HashMap<String, TestCaseRuleConfig>;
fn load_test_case_rule_map() -> TestCaseRuleMap {
serde_json::from_str(
&std::fs::read_to_string(concat!(env!("CARGO_MANIFEST_DIR"), "/../../rule_map.json"))
.unwrap(),
)
.unwrap()
}
fn logical_test_case_key(physical_key: &str, rule_map: &TestCaseRuleMap) -> String {
if rule_map.contains_key(physical_key) {
return physical_key.to_string();
}
let mut matches = rule_map
.iter()
.filter(|(_, config)| config.shards)
.filter_map(|(key, _)| {
physical_key
.strip_prefix(key)
.and_then(|suffix| suffix.strip_prefix('_'))
.filter(|suffix| {
!suffix.is_empty() && suffix.bytes().all(|byte| byte.is_ascii_digit())
})
.map(|_| key.clone())
})
.collect::<Vec<_>>();
matches.sort();
assert!(
matches.len() <= 1,
"fixture {physical_key:?} matches multiple sharded rule-map entries: {matches:?}"
);
matches.pop().unwrap_or_else(|| physical_key.to_string())
}
fn collect_test_files(rule_map: &TestCaseRuleMap) -> Vec<(std::path::PathBuf, String)> {
let test_cases_dir =
std::path::Path::new(concat!(env!("CARGO_MANIFEST_DIR"), "/../../test_cases"));
let mut files = Vec::new();
for entry in std::fs::read_dir(test_cases_dir).unwrap() {
let entry = entry.unwrap();
let path = entry.path();
if path.is_dir() {
let subdir = path.file_name().unwrap().to_string_lossy().to_string();
for sub_entry in std::fs::read_dir(&path).unwrap() {
let sub_entry = sub_entry.unwrap();
let sub_path = sub_entry.path();
if sub_path.extension().unwrap_or_default() == "json" {
let stem = sub_path.file_stem().unwrap().to_string_lossy().to_string();
let physical_key = format!("{}/{}", subdir, stem);
let key = logical_test_case_key(&physical_key, rule_map);
files.push((sub_path, key));
}
}
}
}
files.sort_by(|a, b| a.1.cmp(&b.1).then_with(|| a.0.cmp(&b.0)));
files
}
#[rstest::rstest]
#[case::exact_rule("korean/rule_1", "korean/rule_1")]
#[case::numbered_shard("2025_corpus/sentence_04", "2025_corpus/sentence")]
#[case::unregistered_file("2025_corpus/other_01", "2025_corpus/other_01")]
fn resolves_physical_fixture_to_logical_rule_key(
#[case] physical_key: &str,
#[case] expected: &str,
) {
let rule_map = HashMap::from([
("korean/rule_1".to_string(), TestCaseRuleConfig::default()),
(
"2025_corpus/sentence".to_string(),
TestCaseRuleConfig {
shards: true,
..TestCaseRuleConfig::default()
},
),
]);
assert_eq!(logical_test_case_key(physical_key, &rule_map), expected);
}
fn testcase_answer_forms(
record: &serde_json::Value,
filename: &str,
line_num: usize,
) -> Vec<(String, String, String)> {
if let Some(serde_json::Value::Array(alternatives)) = record.get("alternatives") {
return alternatives
.iter()
.map(|alternative| {
let internal = alternative["internal"].as_str().unwrap_or_else(|| {
panic!(
"'alternatives.internal' 필드를 읽는 중 오류 발생: at {} in {}",
line_num, filename
)
});
let expected = alternative["expected"].as_str().unwrap_or_else(|| {
panic!(
"'alternatives.expected' 필드를 읽는 중 오류 발생: at {} in {}",
line_num, filename
)
});
let unicode = alternative["unicode"].as_str().unwrap_or_else(|| {
panic!(
"'alternatives.unicode' 필드를 읽는 중 오류 발생: at {} in {}",
line_num, filename
)
});
(
internal.to_string(),
expected.to_string(),
unicode.to_string(),
)
})
.collect();
}
let internal = record["internal"].as_str().unwrap_or_else(|| {
panic!(
"'internal' 필드를 읽는 중 오류 발생: at {} in {}",
line_num, filename
)
});
let expected = record["expected"].as_str().unwrap_or_else(|| {
panic!(
"'expected' 필드를 읽는 중 오류 발생: at {} in {}",
line_num, filename
)
});
let unicode = record["unicode"].as_str().unwrap_or_else(|| {
panic!(
"'unicode' 필드를 읽는 중 오류 발생: at {} in {}",
line_num, filename
)
});
vec![(
internal.to_string(),
expected.to_string(),
unicode.to_string(),
)]
}
#[derive(serde::Deserialize)]
struct NiklCorpusCase {
input: String,
unicode: String,
}
fn load_test_case_group<T: serde::de::DeserializeOwned>(key: &str) -> Vec<T> {
let rule_map = load_test_case_rule_map();
assert!(rule_map.contains_key(key), "unknown test-case group: {key}");
let paths = collect_test_files(&rule_map)
.into_iter()
.filter_map(|(path, logical_key)| (logical_key == key).then_some(path))
.collect::<Vec<_>>();
assert!(!paths.is_empty(), "test-case group {key} has no JSON files");
let mut cases = Vec::new();
for path in paths {
let mut shard: Vec<T> = serde_json::from_reader(
File::open(&path).expect("test-case JSON must be readable"),
)
.unwrap_or_else(|error| panic!("{} must be valid JSON: {error}", path.display()));
cases.append(&mut shard);
}
cases
}
fn load_nikl_2025_corpus_cases() -> Vec<NiklCorpusCase> {
load_test_case_group("2025_corpus/sentence")
}
type TestStatusRow = (
String,
String,
String,
String,
bool,
String,
bool,
String,
bool,
);
const REPORT_PAGE_SIZE: usize = 250;
#[derive(serde::Serialize)]
#[serde(rename_all = "camelCase")]
struct ReportPageInfo {
page_size: usize,
page_count: usize,
}
fn paginate_rows(rows: Vec<TestStatusRow>) -> Vec<Vec<TestStatusRow>> {
if rows.is_empty() {
return vec![Vec::new()];
}
rows.chunks(REPORT_PAGE_SIZE)
.map(<[TestStatusRow]>::to_vec)
.collect()
}
#[rstest::rstest]
#[case::no_rows(0, 1)]
#[case::partial_page(1, 1)]
#[case::exact_page(REPORT_PAGE_SIZE, 1)]
#[case::page_overflow(REPORT_PAGE_SIZE + 1, 2)]
#[case::many_pages(REPORT_PAGE_SIZE * 3, 3)]
fn paginate_rows_publishes_every_row(#[case] row_count: usize, #[case] expected_pages: usize) {
let rows = (0..row_count)
.map(|index| -> TestStatusRow {
(
format!("가{index}"),
String::new(),
"⠈⠣".to_string(),
"⠈⠣".to_string(),
index % 2 == 0,
String::new(),
false,
String::new(),
false,
)
})
.collect::<Vec<TestStatusRow>>();
let pages = paginate_rows(rows.clone());
assert_eq!(pages.len(), expected_pages);
assert_eq!(pages.concat(), rows);
assert!(pages.iter().all(|page| page.len() <= REPORT_PAGE_SIZE));
}
#[derive(Default)]
struct NiklFailureStats {
encoding_errors: usize,
contains_latin: usize,
contains_digits: usize,
contains_delimiters: usize,
korean_text_only: usize,
}
fn classify_nikl_failure(input: &str, is_encoding_error: bool, stats: &mut NiklFailureStats) {
if is_encoding_error {
stats.encoding_errors += 1;
}
let contains_latin = input.chars().any(|ch| ch.is_ascii_alphabetic());
let contains_digits = input.chars().any(|ch| ch.is_ascii_digit());
let contains_delimiters = input.chars().any(|ch| {
matches!(
ch,
'(' | ')' | '[' | ']' | '{' | '}' | '“' | '”' | '‘' | '’' | '"' | '\''
)
});
stats.contains_latin += usize::from(contains_latin);
stats.contains_digits += usize::from(contains_digits);
stats.contains_delimiters += usize::from(contains_delimiters);
stats.korean_text_only +=
usize::from(!contains_latin && !contains_digits && !contains_delimiters);
}
#[test]
#[ignore = "NIKL 2025 corpus support is tracked as a benchmark until it reaches 100%"]
fn test_nikl_2025_parallel_corpus() {
let cases = load_nikl_2025_corpus_cases();
assert!(
!cases.is_empty(),
"NIKL 2025 corpus fixture must not be empty"
);
let mut failures = Vec::new();
let mut failure_stats = NiklFailureStats::default();
for case in &cases {
match encode_to_unicode(&case.input) {
Ok(actual) if actual == case.unicode => {}
Ok(actual) => {
classify_nikl_failure(&case.input, false, &mut failure_stats);
failures.push((
case.input.as_str(),
case.input.as_str(),
case.unicode.as_str(),
"mismatch",
actual,
));
}
Err(error) => {
classify_nikl_failure(&case.input, true, &mut failure_stats);
failures.push((
case.input.as_str(),
case.input.as_str(),
case.unicode.as_str(),
"encoding error",
error,
));
}
}
}
if !failures.is_empty() {
let preview = failures
.iter()
.take(20)
.map(|(id, input, expected, kind, actual)| {
format!(
"{id} ({kind})\n input: {input}\n expected: {expected}\n actual: {actual}"
)
})
.collect::<Vec<_>>()
.join("\n");
panic!(
"NIKL 2025 corpus: {}/{} cases differ from the reference.\n\
Failure traits (overlapping): encoding errors={}, Latin={}, digits={}, delimiters={}, Korean-text-only={}.\n\
First {}:\n{}",
failures.len(),
cases.len(),
failure_stats.encoding_errors,
failure_stats.contains_latin,
failure_stats.contains_digits,
failure_stats.contains_delimiters,
failure_stats.korean_text_only,
failures.len().min(20),
preview
);
}
}
#[test]
pub fn test_by_testcase() {
let rule_map = load_test_case_rule_map();
let files = collect_test_files(&rule_map);
let mut total = 0;
let mut failed = 0;
let mut failed_cases = Vec::new();
let mut skipped_cases: Vec<(String, usize, String, String)> = Vec::new();
let mut file_stats = std::collections::BTreeMap::new();
let rule_map_keys: std::collections::HashSet<String> = rule_map.keys().cloned().collect();
let file_keys: std::collections::HashSet<_> =
files.iter().map(|(_, key)| key.clone()).collect();
let missing_keys = rule_map_keys.difference(&file_keys).collect::<Vec<_>>();
let extra_keys = file_keys.difference(&rule_map_keys).collect::<Vec<_>>();
if !missing_keys.is_empty() || !extra_keys.is_empty() {
panic!(
"rule_map.json 파일이 올바르지 않습니다. missing: {:?}, extra: {:?}",
missing_keys, extra_keys
);
}
for (path, file_stem) in &files {
let config = rule_map
.get(file_stem)
.unwrap_or_else(|| panic!("missing rule-map config for {file_stem}"));
let content = std::fs::read_to_string(path).unwrap();
let filename = path.file_name().unwrap().to_string_lossy();
let records: Vec<serde_json::Value> = serde_json::from_str(&content)
.unwrap_or_else(|e| panic!("JSON 파일을 읽는 중 오류 발생: {} in {}", e, filename));
let mut file_total = 0;
let mut file_failed = 0;
let mut file_world_total = 0;
let mut file_world_failed = 0;
let mut file_jeomsarang_total = 0;
let mut file_jeomsarang_failed = 0;
let mut test_status: Vec<TestStatusRow> = Vec::new();
for (line_num, record) in records.iter().enumerate() {
if let Some(reason) = record.get("limitation").and_then(|v| v.as_str()) {
let input = record["input"].as_str().unwrap_or("");
let expected_values = testcase_answer_forms(record, &filename, line_num)
.into_iter()
.map(|(_, _, unicode)| unicode)
.collect::<Vec<_>>();
if let Ok(actual) = crate::encode_to_unicode(input)
&& expected_values.contains(&actual)
{
panic!(
"STALE limitation in {} line {}: input={:?} passes but is marked limitation: {:?}",
filename, line_num, input, reason
);
}
skipped_cases.push((
filename.to_string(),
line_num + 1,
input.to_string(),
reason.to_string(),
));
continue;
}
if !config.benchmark {
total += 1;
}
file_total += 1;
let input = record["input"].as_str().unwrap_or_else(|| {
panic!(
"'input' 필드를 읽는 중 오류 발생: at {} in {}",
line_num, filename
)
});
let context = record["context"].as_str().unwrap_or("");
let note = record["note"].as_str().unwrap_or("").to_string();
let (world, jeomsarang) = if config.benchmark {
(String::new(), String::new())
} else {
(
record["world"].as_str().unwrap_or("").to_string(),
record["jeomsarang"].as_str().unwrap_or("").to_string(),
)
};
if !config.benchmark {
file_world_total += 1;
file_jeomsarang_total += 1;
}
let answer_forms = testcase_answer_forms(record, &filename, line_num);
let expected_forms = answer_forms
.iter()
.map(|(_, expected, _)| expected)
.map(|expected| expected.trim().replace(" ", "⠀"))
.collect::<Vec<_>>();
let unicode_forms = answer_forms
.into_iter()
.map(|(_, _, unicode)| unicode)
.collect::<Vec<_>>();
let expected_display = expected_forms.join(" / ");
let unicode_display = unicode_forms.join(" / ");
let input_for_encoding: String =
if let Some(prefix) = context.strip_prefix("strip_prefix:") {
input.strip_prefix(prefix).unwrap_or(input).to_string()
} else {
input.to_string()
};
let encoding_result = match context.parse::<crate::rules::context::EncodingMode>() {
Ok(mode) => encode_with_options(
&input_for_encoding,
&EncodeOptions {
default_mode: Some(mode),
},
),
Err(_) => encode(&input_for_encoding),
};
match encoding_result {
Ok(actual) => {
let braille_expected = actual
.iter()
.map(|c| unicode::encode_unicode(*c))
.collect::<String>();
let actual_str = actual
.iter()
.map(|c| {
if *c == 255 {
"\n".to_string()
} else {
c.to_string()
}
})
.collect::<String>();
let case_matches = expected_forms.contains(&actual_str);
if !case_matches {
file_failed += 1;
if !config.benchmark {
failed += 1;
failed_cases.push((
filename.to_string(),
line_num + 1,
input.to_string(),
expected_display.clone(),
actual_str.clone(),
braille_expected.clone(),
unicode_display.clone(),
));
}
}
let world_is_success = !world.is_empty() && unicode_forms.contains(&world);
if !config.benchmark && !world_is_success {
file_world_failed += 1;
}
let jeomsarang_is_success =
!jeomsarang.is_empty() && unicode_forms.contains(&jeomsarang);
if !config.benchmark && !jeomsarang_is_success {
file_jeomsarang_failed += 1;
}
test_status.push((
input.to_string(),
note.clone(),
unicode_display.clone(),
braille_expected.clone(),
case_matches,
world.clone(),
world_is_success,
jeomsarang.clone(),
jeomsarang_is_success,
));
}
Err(e) => {
if !config.benchmark {
println!("Error: {}", e);
}
file_failed += 1;
if !config.benchmark {
failed += 1;
failed_cases.push((
filename.to_string(),
line_num + 1,
input.to_string(),
expected_display.clone(),
"".to_string(),
e.to_string(),
unicode_display.clone(),
));
}
let world_is_success = !world.is_empty() && unicode_forms.contains(&world);
if !config.benchmark && !world_is_success {
file_world_failed += 1;
}
let jeomsarang_is_success =
!jeomsarang.is_empty() && unicode_forms.contains(&jeomsarang);
if !config.benchmark && !jeomsarang_is_success {
file_jeomsarang_failed += 1;
}
test_status.push((
input.to_string(),
note.clone(),
unicode_display.clone(),
e.to_string(),
false,
world.clone(),
world_is_success,
jeomsarang.clone(),
jeomsarang_is_success,
));
}
}
}
let stats = file_stats
.entry(file_stem.clone())
.or_insert_with(|| (0, 0, 0, 0, 0, 0, Vec::<TestStatusRow>::new()));
stats.0 += file_total;
stats.1 += file_failed;
stats.2 += file_world_total;
stats.3 += file_world_failed;
stats.4 += file_jeomsarang_total;
stats.5 += file_jeomsarang_failed;
stats.6.append(&mut test_status);
}
if !failed_cases.is_empty() {
println!("\n실패한 케이스:");
println!("=================");
for (filename, line_num, input, expected, actual, unicode, braille) in failed_cases {
let diff = {
let unicode_words: Vec<&str> = unicode.split(encode_unicode(0)).collect();
let braille_words: Vec<&str> = braille.split(encode_unicode(0)).collect();
let mut diff = Vec::new();
for (i, (u, b)) in unicode_words.iter().zip(braille_words.iter()).enumerate() {
if u != b {
diff.push(i);
}
}
diff
};
let input_words: Vec<&str> = input.split(' ').collect();
let unicode_words: Vec<&str> = unicode.split(encode_unicode(0)).collect();
if input_words.len() != unicode_words.len() {
println!("파일: {}, 라인 {}: '{}'", filename, line_num, input);
println!(" 예상: {}", expected);
println!(" 실제: {}", actual);
println!(" 유니코드 Result: {}", unicode);
println!(" 유니코드 Expected: {}", braille);
} else {
let mut colored_input = String::new();
let mut colored_unicode = String::new();
for (i, word) in input_words.iter().enumerate() {
if diff.contains(&i) {
colored_input.push_str(&format!("\x1b[31m{}\x1b[0m", word));
colored_unicode
.push_str(&format!("\x1b[31m{}\x1b[0m", unicode_words[i]));
} else {
colored_input.push_str(word);
colored_unicode.push_str(unicode_words[i]);
}
if i < input_words.len() - 1 {
colored_input.push(' ');
colored_unicode.push(' ');
}
}
println!("파일: {}, 라인 {}: '{}'", filename, line_num, colored_input);
println!(" 예상: {}", expected);
println!(" 실제: {}", actual);
println!(" 유니코드 Result: {}", colored_unicode);
println!(" 유니코드 Expected: {}", braille);
}
println!();
}
}
if !skipped_cases.is_empty() {
println!("\nSkip된 케이스 (limitation):");
println!("=================");
for (filename, line_num, input, reason) in &skipped_cases {
println!(
"\x1b[33m파일: {}, 라인 {}: '{}'\x1b[0m",
filename, line_num, input
);
println!(" 사유: {}", reason);
println!();
}
println!("총 Skip: {}건", skipped_cases.len());
}
let report_root = std::path::PathBuf::from(concat!(
env!("CARGO_MANIFEST_DIR"),
"/../../apps/landing/public/test-status"
));
if report_root.exists() {
std::fs::remove_dir_all(&report_root).unwrap_or_else(|error| {
panic!(
"failed to clear report directory {}: {error}",
report_root.display()
)
});
}
let mut report_manifest = std::collections::BTreeMap::new();
for (key, stats) in &mut file_stats {
let pages = paginate_rows(std::mem::take(&mut stats.6));
let group_dir = report_root.join(key);
std::fs::create_dir_all(&group_dir).unwrap_or_else(|error| {
panic!(
"failed to create report directory {}: {error}",
group_dir.display()
)
});
for (index, page) in pages.iter().enumerate() {
let page_path = group_dir.join(format!("page-{}.json", index + 1));
serde_json::to_writer(File::create(&page_path).unwrap(), page).unwrap();
}
report_manifest.insert(
key.clone(),
ReportPageInfo {
page_size: REPORT_PAGE_SIZE,
page_count: pages.len(),
},
);
}
std::fs::create_dir_all(&report_root).unwrap();
serde_json::to_writer_pretty(
File::create(report_root.join("manifest.json")).unwrap(),
&report_manifest,
)
.unwrap();
let status_path = concat!(env!("CARGO_MANIFEST_DIR"), "/../../test_status.json");
serde_json::to_writer_pretty(File::create(status_path).unwrap(), &file_stats).unwrap();
let mut category_stats: std::collections::BTreeMap<String, (usize, usize)> =
std::collections::BTreeMap::new();
for (key, value) in &file_stats {
let category = key.split('/').next().unwrap_or(key.as_str()).to_string();
let entry = category_stats.entry(category).or_insert((0, 0));
entry.0 += value.0;
entry.1 += value.1;
}
println!("\n파일별 테스트 결과:");
println!("=================");
for (filename, (file_total, file_failed, _, _, _, _, _)) in file_stats {
let success_rate =
((file_total - file_failed) as f64 / file_total as f64 * 100.0) as i32;
let color = if success_rate == 100 {
"\x1b[32m" } else if success_rate == 0 {
"\x1b[31m" } else {
"\x1b[33m" };
println!(
"{}: {}개 중 {}개 성공 (성공률: {}{}%\x1b[0m)",
filename,
file_total,
file_total - file_failed,
color,
success_rate
);
}
println!("\n카테고리별 결과:");
println!("=================");
for (category, (cat_total, cat_failed)) in &category_stats {
println!(
"{}: {}/{} 성공",
category,
cat_total - cat_failed,
cat_total
);
}
println!("\n전체 테스트 결과 요약:");
println!("=================");
println!("총 테스트 케이스: {}", total);
println!("성공: {}", total - failed);
println!("실패: {}", failed);
println!("Skip (limitation): {}", skipped_cases.len());
if failed > 0 {
panic!("{} test cases failed.", failed);
}
}
proptest! {
#[test]
fn test_encode_proptest(s: String) {
let result = encode(&s);
let _encoded = match result {
Ok(encoded) => {
let is_only_nonemitting = s.chars().all(|c| {
c == ' '
|| matches!(
crate::char_struct::CharType::new(c),
Ok(crate::char_struct::CharType::CombiningMark)
)
});
assert!(!encoded.is_empty() || s.is_empty() || is_only_nonemitting);
let unicode_result = encode_to_unicode(&s);
assert!(unicode_result.is_ok());
let unicode_string = unicode_result.unwrap();
assert!(!unicode_string.is_empty() || s.is_empty() || is_only_nonemitting);
encoded
}
Err(_) => {
return Ok(()); }
};
}
}
#[test]
fn test_accuracy_report() {
let rule_map = load_test_case_rule_map();
let files = collect_test_files(&rule_map)
.into_iter()
.filter(|(_, key)| !rule_map[key].benchmark)
.collect::<Vec<_>>();
let mut total = 0usize;
let mut passed = 0usize;
let mut per_file: Vec<(String, usize, usize)> = Vec::new();
for (path, filename) in &files {
let content = std::fs::read_to_string(path).unwrap();
let records: Vec<serde_json::Value> = serde_json::from_str(&content).unwrap();
let mut file_total = 0;
let mut file_passed = 0;
for (line_num, record) in records.iter().enumerate() {
let input = record["input"].as_str().unwrap();
let expected_forms = testcase_answer_forms(record, filename, line_num)
.into_iter()
.map(|(_, expected, _)| expected.trim().replace(" ", "⠀"))
.collect::<Vec<_>>();
if expected_forms
.iter()
.any(|expected| expected.chars().any(|c| !c.is_ascii_digit()))
{
continue;
}
total += 1;
file_total += 1;
if let Ok(actual) = encode(input) {
let actual_str = actual.iter().map(|c| c.to_string()).collect::<String>();
if expected_forms.contains(&actual_str) {
passed += 1;
file_passed += 1;
}
}
}
per_file.push((filename.clone(), file_total, file_passed));
}
per_file.sort();
println!("\n═══════════════════════════════════════════════");
println!(" BRAILLIFY ACCURACY REPORT (engine-driven)");
println!("═══════════════════════════════════════════════");
for (name, ft, fp) in &per_file {
let pct = (*fp * 100).checked_div(*ft).unwrap_or(100);
let status = if pct == 100 { "✓" } else { "✗" };
if pct < 100 {
println!(" {} {:20} {:>3}/{:<3} ({:>3}%)", status, name, fp, ft, pct);
}
}
let all_pass: usize = per_file.iter().filter(|(_, t, p)| t == p).count();
let some_fail: usize = per_file.len() - all_pass;
println!("───────────────────────────────────────────────");
println!(
" Files: {} total, {} all-pass, {} with failures",
per_file.len(),
all_pass,
some_fail
);
println!(
" Cases: {}/{} passed ({:.1}%)",
passed,
total,
passed as f64 / total as f64 * 100.0
);
println!("═══════════════════════════════════════════════\n");
}
#[test]
fn test_encoder_streaming() {
let mut encoder = Encoder::new(false); let mut buffer = Vec::new();
encoder.encode("test", &mut buffer).unwrap();
encoder.encode("ing", &mut buffer).unwrap();
let mut expected = encode("test").unwrap();
expected.extend(encode("ing").unwrap());
assert_eq!(buffer, expected);
}
}
#[cfg(test)]
mod coverage_targeted_tests {
use super::*;
use crate::rules::context::EncodingMode;
#[rstest::rstest]
#[case::emphasis(FormattingKind::Emphasis, [32, 36], [36, 4])]
#[case::bold(FormattingKind::Bold, [48, 36], [36, 6])]
#[case::custom1(FormattingKind::Custom1, [16, 36], [36, 2])]
#[case::custom2(FormattingKind::Custom2, [8, 36], [36, 1])]
fn formatting_kind_markers_all_variants(
#[case] kind: FormattingKind,
#[case] start: [u8; 2],
#[case] end: [u8; 2],
) {
assert_eq!(kind.markers(), (start, end));
}
#[test]
fn normalize_math_planck_h() {
assert_eq!(normalize_math_alphanumeric_char('\u{210E}'), 'h');
}
#[rstest::rstest]
#[case::bold_capital_a('\u{1D400}', 'A')]
#[case::bold_small_a('\u{1D41A}', 'a')]
#[case::bold_digit_zero('\u{1D7CE}', '0')]
#[case::passthrough_ascii('Z', 'Z')]
fn normalize_math_alphanumeric_block_mapping(#[case] input: char, #[case] expected: char) {
assert_eq!(normalize_math_alphanumeric_char(input), expected);
}
#[rstest::rstest]
#[case::upper_one("Ⅰ", "I")]
#[case::upper_two("Ⅱ", "II")]
#[case::upper_seven("Ⅶ", "VII")]
#[case::lower_four("ⅳ", "iv")]
#[case::embedded("제Ⅲ장", "제III장")]
fn normalizes_roman_numeral_presentation(#[case] input: &str, #[case] expected: &str) {
assert_eq!(
normalize_roman_numeral_presentation(Cow::Borrowed(input)),
expected
);
}
#[rstest::rstest]
#[case::pdf_sentence(
"가영이는 미적분학 Ⅱ 과목을 수강하고 있다.",
"가영이는 미적분학 II 과목을 수강하고 있다."
)]
#[case::attached_chapter("제Ⅲ장", "제III장")]
#[case::adjacent_particle("Ⅶ을", "VII을")]
#[case::lowercase_indicator("ⅳ를", "iv를")]
fn unicode_roman_numeral_matches_ascii_rule_36_path(
#[case] presentation: &str,
#[case] ascii: &str,
) {
assert_eq!(encode_to_unicode(presentation), encode_to_unicode(ascii));
}
#[test]
fn roman_numeral_normalization_leaves_other_nfkc_characters_unchanged() {
let input = "ↀ㈜";
assert_eq!(
normalize_roman_numeral_presentation(Cow::Borrowed(input)),
input
);
}
#[rstest::rstest]
#[case::parenthesized_jamo("㈀", "(ᄀ)")]
#[case::parenthesized_syllable("㈎", "(가)")]
#[case::incorporated_association("㈔", "(사)")]
#[case::incorporated_company("㈜", "(주)")]
#[case::afternoon("㈞", "(오후)")]
fn normalizes_parenthesized_hangul_presentation(#[case] input: &str, #[case] expected: &str) {
assert_eq!(
normalize_parenthesized_hangul_presentation(Cow::Borrowed(input)),
expected
);
}
#[rstest::rstest]
#[case::association_prefix("㈔한국", "(사)한국")]
#[case::company_prefix("㈜한빛", "(주)한빛")]
#[case::attached_company_suffix("한빛㈜", "한빛(주)")]
fn parenthesized_hangul_presentation_matches_expanded_print(
#[case] presentation: &str,
#[case] expanded: &str,
) {
assert_eq!(encode_to_unicode(presentation), encode_to_unicode(expanded));
}
#[test]
fn normalizes_word_separator_middle_dot_to_print_space() {
assert_eq!(
normalize_word_separator_middle_dot(Cow::Borrowed("인증⸱실천⸱교육")),
"인증 실천 교육"
);
}
#[test]
fn word_separator_middle_dot_matches_visible_word_spacing() {
assert_eq!(
encode_to_unicode("인증⸱실천⸱교육"),
encode_to_unicode("인증 실천 교육")
);
}
#[rstest::rstest]
#[case::kilowatt_hour("㎾h", "kWh")]
#[case::milli_sievert("m㏜", "mSv")]
#[case::watt_per_kilogram("W/㎏", "W/kg")]
#[case::kilogram_carbon_equivalent("㎏CO2eq", "kgCO2eq")]
#[case::milligram_per_gram("㎎/g", "mg/g")]
fn normalizes_pure_roman_compatibility_unit_components(
#[case] input: &str,
#[case] expected: &str,
) {
assert_eq!(
normalize_pure_roman_compatibility_units(Cow::Borrowed(input)),
expected
);
}
#[rstest::rstest]
#[case::superscript("㎥")]
#[case::quotient("㎧")]
#[case::standalone_hectare("㏊")]
#[case::non_unit_compatibility_abbreviation("㏚")]
fn pure_roman_unit_normalization_preserves_other_compatibility_forms(#[case] input: &str) {
assert_eq!(
normalize_pure_roman_compatibility_units(Cow::Borrowed(input)),
input
);
}
#[rstest::rstest]
#[case::kilowatt_hour("용량은 1㎾h이다", "용량은 1kWh이다")]
#[case::milli_sievert("선량은 1m㏜보다 낮다", "선량은 1mSv보다 낮다")]
#[case::watt_per_kilogram("기준은 4.0W/㎏이다", "기준은 4.0W/kg이다")]
fn compound_compatibility_units_match_semantic_roman_spelling(
#[case] presentation: &str,
#[case] expanded: &str,
) {
assert_eq!(
encode_to_unicode(presentation),
encode_to_unicode(expanded),
"presentation={presentation:?}"
);
}
#[test]
fn normalize_math_alphanumeric_runtime_block_offset() {
let input = std::hint::black_box('\u{1D44F}');
assert_eq!(normalize_math_alphanumeric_char(input), 'b');
}
#[test]
fn normalize_math_alphanumeric_runtime_digit_offset() {
let input = std::hint::black_box('\u{1D7D9}');
assert_eq!(normalize_math_alphanumeric_char(input), '1');
}
#[test]
fn normalize_math_string_no_trigger() {
let result = normalize_math_alphanumeric_string("plain ASCII");
assert!(matches!(result, Cow::Borrowed(_)));
}
#[test]
fn normalize_math_string_with_trigger() {
let result = normalize_math_alphanumeric_string("X = \u{1D400}");
assert!(matches!(result, Cow::Owned(_)));
assert_eq!(result.as_ref(), "X = A");
}
#[test]
fn encode_decomposes_runtime_accented_latin_when_triggered() {
let input = std::hint::black_box("café");
let encoded = encode_with_options(input, &EncodeOptions::default()).unwrap();
assert_eq!(
encoded,
encode_with_options("cafe\u{301}", &EncodeOptions::default()).unwrap()
);
}
#[rstest::rstest]
#[case::plain_subscript("B₆")]
#[case::latex_subscript("$B_6$")]
fn korean_rule68_compact_subscript_default_routes_to_korean(#[case] input: &str) {
let expected = vec![52, 32, 3, 48, 60, 11];
let plain = encode("B₆").unwrap();
let encoded = encode(input).unwrap();
assert_eq!(plain, expected);
assert_eq!(encoded, plain);
}
#[test]
fn negation_combiner_absent_short_circuits() {
let input: Cow<'_, str> = Cow::Borrowed("no combiner here");
let result = move_negation_combiner_before_base(input);
assert_eq!(result.as_ref(), "no combiner here");
}
#[rstest::rstest]
#[case::circle("○", &[56, 52, 7])]
#[case::cross("×", &[56, 45, 7])]
#[case::triangle("△", &[56, 44, 7])]
#[case::square("□", &[56, 54, 7])]
fn encode_object_symbol_mode_each_glyph(#[case] input: &str, #[case] expected: &[u8]) {
let opts = EncodeOptions {
default_mode: Some(EncodingMode::ObjectSymbol),
};
assert_eq!(encode_with_options(input, &opts).unwrap(), expected);
}
#[test]
fn encode_object_symbol_mode_non_matching_falls_through() {
let opts = EncodeOptions {
default_mode: Some(EncodingMode::ObjectSymbol),
};
let result = encode_with_options("A", &opts);
assert!(result.is_ok());
}
#[test]
fn encode_number_mode_roman_uppercase() {
let opts = EncodeOptions {
default_mode: Some(EncodingMode::Number),
};
let single = encode_with_options("I", &opts).unwrap();
assert!(single.starts_with(&[52, 32]));
assert!(single.ends_with(&[50]));
let multi = encode_with_options("IV", &opts).unwrap();
assert_eq!(multi[0], 52);
assert_eq!(multi[1], 32);
assert_eq!(multi[2], 32);
assert_eq!(multi[multi.len() - 1], 50);
}
#[test]
fn encode_number_mode_roman_lowercase() {
let opts = EncodeOptions {
default_mode: Some(EncodingMode::Number),
};
let result = encode_with_options("ix", &opts).unwrap();
assert_eq!(result[0], 52); assert_ne!(result[1], 32); assert_eq!(result[result.len() - 1], 50); }
#[test]
fn encode_number_mode_non_roman_falls_through() {
let opts = EncodeOptions {
default_mode: Some(EncodingMode::Number),
};
let result = encode_with_options("Z", &opts);
assert!(result.is_ok());
}
#[test]
fn encode_math_mode_single_lowercase() {
let opts = EncodeOptions {
default_mode: Some(EncodingMode::Math),
};
let result = encode_with_options("x", &opts).unwrap();
assert_eq!(result[0], 52); assert_eq!(result.len(), 2);
}
#[test]
fn encode_math_mode_single_brackets() {
let opts = EncodeOptions {
default_mode: Some(EncodingMode::Math),
};
assert_eq!(encode_with_options("(", &opts).unwrap(), vec![38]);
assert_eq!(encode_with_options(")", &opts).unwrap(), vec![52]);
assert_eq!(encode_with_options("{", &opts).unwrap(), vec![54]);
assert_eq!(encode_with_options("}", &opts).unwrap(), vec![54]);
assert_eq!(encode_with_options("[", &opts).unwrap(), vec![55, 4]);
assert_eq!(encode_with_options("]", &opts).unwrap(), vec![32, 62]);
}
#[test]
fn encode_math_mode_single_math_symbol() {
let opts = EncodeOptions {
default_mode: Some(EncodingMode::Math),
};
let result = encode_with_options("+", &opts);
assert!(result.is_ok());
}
#[rstest::rstest]
#[case::congruence("A ≅ B", "⠠⠁⠀⠈⠔⠒⠒⠀⠠⠃")]
#[case::right_geometric_operation("G ▷ N", "⠠⠛⠀⠸⠜⠀⠠⠝")]
#[case::left_geometric_operation("N ◁ G", "⠠⠝⠀⠸⠣⠀⠠⠛")]
fn default_route_keeps_math_relation_operands_in_one_expression(
#[case] input: &str,
#[case] expected: &str,
) {
assert_eq!(encode_to_unicode(input).as_deref(), Ok(expected));
}
#[test]
fn encode_math_mode_multichar_strips_spaces() {
let opts = EncodeOptions {
default_mode: Some(EncodingMode::Math),
};
let a = encode_with_options("x = y", &opts).unwrap();
let b = encode_with_options("x=y", &opts).unwrap();
assert_eq!(a, b, "Spaces around '=' must be stripped in math mode");
let c = encode_with_options("a + b", &opts).unwrap();
let d = encode_with_options("a+b", &opts).unwrap();
assert_eq!(c, d);
}
#[test]
fn encode_with_options_explicit_default_mode() {
let opts = EncodeOptions {
default_mode: Some(EncodingMode::English),
};
let result = encode_with_options("hello", &opts);
assert!(result.is_ok());
}
#[test]
fn korean_context_wraps_isolated_roman_section() {
let opts = EncodeOptions {
default_mode: Some(EncodingMode::Korean),
};
let wrapped = encode_with_options("but", &opts).unwrap();
assert_eq!(
wrapped.first(),
Some(&52),
"고립 로마자는 로마자표 ⠴로 시작"
);
assert_eq!(wrapped.last(), Some(&50), "고립 로마자는 종료표 ⠲로 끝남");
let phrase = encode_with_options("Table of Contents", &opts).unwrap();
assert_eq!(phrase.first(), Some(&52));
assert_eq!(phrase.last(), Some(&50));
let unit = encode_with_options("%p", &opts).unwrap();
assert_ne!(unit.last(), Some(&50), "%p(제69항)는 종료표로 감싸지 않음");
}
#[rstest::rstest]
#[case::word("but", true)]
#[case::phrase_with_spaces("Table of Contents", true)]
#[case::percent_unit("%p", false)]
#[case::has_digit("abc123", false)]
#[case::empty("", false)]
#[case::only_space(" ", false)]
fn is_isolated_roman_section_paths(#[case] input: &str, #[case] expected: bool) {
assert_eq!(is_isolated_roman_section(input), expected);
}
#[test]
fn encode_with_formatting_empty_spans_delegates() {
let plain = encode("hello").unwrap();
let formatted = encode_with_formatting("hello", &[]).unwrap();
assert_eq!(plain, formatted);
}
#[test]
fn encode_to_braille_font_basic() {
let result = encode_to_braille_font("a").unwrap();
assert!(!result.is_empty());
for ch in result.chars() {
let cp = ch as u32;
assert!((0x2800..=0x28FF).contains(&cp), "non-braille char {:?}", ch);
}
}
#[test]
fn encode_to_unicode_with_formatting_empty() {
let result = encode_to_unicode_with_formatting("a", &[]).unwrap();
assert!(!result.is_empty());
}
#[rstest::rstest]
#[case::no_markers("plain text", false)]
#[case::brackets_ipa("[əbaut]", true)]
#[case::brackets_without_ipa_then_slashes_ipa("[abc] /əb/", true)]
#[case::slashes_with_ipa("/əb/", true)]
fn detect_ipa_context_variants(#[case] input: &str, #[case] expected: bool) {
assert_eq!(detect_ipa_context(input), expected, "input={input:?}");
}
#[test]
fn detect_ipa_context_slashes_without_ipa() {
let s = "abc // \u{0259} xyz";
let _ = detect_ipa_context(s);
}
#[test]
fn latex_math_comprehensive_sweep() {
let inputs: &[&str] = &[
"1+2",
"x = 1",
"a + b - c",
"x \\times y",
"$x$",
"$x = 1$",
"$x + y$",
"$\\frac{1}{2}$",
"$\\frac{a+b}{c-d}$",
"$x^2$",
"$x^{n+1}$",
"$x_n$",
"$x_{i+1}$",
"$\\sqrt{2}$",
"$\\sqrt[3]{x}$",
"$\\sum_{i=1}^{n} i$",
"$\\int_0^1 f(x) dx$",
"$\\lim_{x \\to 0} f(x)$",
"$f(x) = x^2 + 1$",
"$y \\neq 0$",
"$x \\geq 0$",
"$x \\leq 1$",
"$A \\cup B$",
"$A \\cap B$",
"$A \\subset B$",
"$\\emptyset$",
"$\\forall x$",
"$\\exists y$",
"$\\alpha$",
"$\\beta$",
"$\\pi$",
"$\\theta$",
"$x + $ $y$",
"1 + $x$ = 2",
"$x$ and $y$",
"$\\sin x$",
"$\\cos x$",
"$\\log x$",
"$\\ln x$",
"$\\begin{matrix} 1 & 2 \\\\ 3 & 4 \\end{matrix}$",
"$\\begin{pmatrix} a & b \\\\ c & d \\end{pmatrix}$",
"$\\begin{bmatrix} 1 \\\\ 2 \\end{bmatrix}$",
"$\\begin{array}{cc} x & y \\\\ z & w \\end{array}$",
"수식 $x + 1$ 입니다",
"함수 $f(x)$",
"$a_1$",
"$a_{12}$",
"$x_n y_n$",
"$x^2 + y^2$",
"$2^{10}$",
"$x_i^j$",
"$a^b_c$",
"1+2=3",
"10×5=50",
"x/y",
"1<2",
"3>2",
"x≥0",
"1/2",
"3/4 cup",
"x1/2y",
"$(x+y)$",
"$[a,b]$",
"$\\{x | x > 0\\}$",
"$$",
"$x = ",
];
for input in inputs {
let _ = encode(input);
let _ = encode_to_unicode(input);
}
}
#[test]
fn math_mode_comprehensive_sweep() {
let inputs: &[&str] = &[
"1+2", "x=1", "a+b-c", "x*y", "x/y", "(a+b)", "{c}", "[d]", "x^2", "x_n", "x≥0", "y≤1",
"a≠b", "+", "-", "*", "/", "=", "<", ">", "≠", "≥", "≤", "π", "α", "β", "∞", "∂",
"f(x)", "1 + 2", "x = y",
];
let opts = EncodeOptions {
default_mode: Some(EncodingMode::Math),
};
for input in inputs {
let _ = encode_with_options(input, &opts);
}
}
#[test]
fn formatting_mark_wrap_absorbs_leading_digits() {
let _ = encode("5강\u{0307}");
let _ = encode("1,000원\u{0307}");
let _ = encode("3.14를\u{0307}");
}
#[test]
fn formatting_mark_preserved_when_units_insufficient() {
let _ = encode("한\u{0307}\u{0307}\u{0307}\u{0307}");
let _ = encode("한글\u{0307}\u{0307}\u{0307}\u{0307}\u{0307}");
let _ = encode("\u{0307}\u{0307}");
}
#[test]
fn decompose_accented_latin_not_called_for_plain_input() {
let _ = encode("안녕하세요");
let _ = encode("hello");
}
#[test]
fn decompose_accented_latin_called_for_accented_input() {
let _ = encode(std::hint::black_box("café"));
let _ = encode(std::hint::black_box("piñata"));
let _ = encode(std::hint::black_box("ão"));
}
#[test]
fn decompose_accented_latin_directly_expands_latin_marks() {
assert_eq!(
decompose_accented_latin(Cow::Borrowed("café Å")),
Cow::<str>::Owned("cafe\u{0301} Å".to_string())
);
}
#[test]
fn default_mode_routes_styled_english_and_inline_nemeth_to_ueb() {
assert!(encode("𝐡𝐢𝐬 𝐡𝐞𝐫𝐬 𝐢𝐭𝐬").is_ok());
assert!(encode("solve $x+1$ now").is_ok());
}
#[rstest::rstest]
#[case::bold_zero('\u{1D7CE}', '0')]
#[case::bold_one('\u{1D7CF}', '1')]
#[case::bold_nine('\u{1D7D7}', '9')]
#[case::double_struck_zero('\u{1D7D8}', '0')]
#[case::sans_serif_zero('\u{1D7E2}', '0')]
#[case::sans_serif_bold_zero('\u{1D7EC}', '0')]
#[case::monospace_zero('\u{1D7F6}', '0')]
#[case::monospace_nine('\u{1D7FF}', '9')]
fn normalize_math_alphanumeric_digits(#[case] input: char, #[case] expected: char) {
assert_eq!(
normalize_math_alphanumeric_char(std::hint::black_box(input)),
expected
);
}
#[test]
fn encode_normalizes_math_alphanumeric_digit_blocks() {
assert!(encode(std::hint::black_box("𝟘+𝟙=𝟙")).is_ok());
}
#[rstest::rstest]
#[case::bold_capital_a('\u{1D400}', 'A')]
#[case::bold_lower_a('\u{1D41A}', 'a')]
#[case::italic_lower_h('\u{210E}', 'h')]
fn normalize_math_alphanumeric_letters(#[case] input: char, #[case] expected: char) {
assert_eq!(normalize_math_alphanumeric_char(input), expected);
}
#[test]
fn may_normalize_math_alphanumeric_detects_supported_ranges() {
assert!(may_normalize_math_alphanumeric('\u{210E}'));
assert!(may_normalize_math_alphanumeric('\u{1D400}'));
assert!(!may_normalize_math_alphanumeric('A'));
}
}
#[cfg(test)]
mod debug_reader {
use crate::rules::english_ueb;
#[test]
fn debug_reader() {
for input in ["reader", "READER", "READER'S", "(READER'S DIGEST)"] {
if let Some(result) = english_ueb::try_encode(input) {
let unicode: String = result
.iter()
.map(|c| crate::unicode::encode_unicode(*c))
.collect();
eprintln!("[{}] result: {:?}", input, result);
eprintln!("[{}] unicode: {}", input, unicode);
} else {
eprintln!("[{}] returned None", input);
}
}
}
}
#[cfg(test)]
mod print_variant_coverage {
use super::*;
#[rstest::rstest]
#[case::celsius("25\u{00B0}C", "25\u{2103}")]
#[case::fahrenheit("77\u{00B0}F", "77\u{2109}")]
#[case::ring_celsius("25\u{02DA}C", "25\u{2103}")]
#[case::ring_fahrenheit("77\u{02DA}F", "77\u{2109}")]
#[case::letter_without_a_degree("25C", "25C")]
#[case::degree_at_the_end("25\u{00B0}", "25\u{00B0}")]
#[case::degree_before_another_letter("25\u{00B0}K", "25\u{00B0}K")]
fn degree_letter_pair_folds_to_the_unit_glyph(#[case] input: &str, #[case] expected: &str) {
assert_eq!(
normalize_print_variants(std::borrow::Cow::Borrowed(input)).as_ref(),
expected
);
}
#[rstest::rstest]
#[case::fullwidth_percent('\u{FF05}', true)]
#[case::fullwidth_letter('\u{FF4D}', true)]
#[case::fullwidth_hash_is_the_math_cardinal('\u{FF03}', false)]
#[case::fullwidth_colon_is_the_old_hangul_mark('\u{FF1A}', false)]
#[case::ascii_is_not_a_variant('m', false)]
fn fullwidth_folding_excludes_the_two_reserved_glyphs(
#[case] input: char,
#[case] expected: bool,
) {
assert_eq!(is_foldable_fullwidth(input), expected);
}
#[rstest::rstest]
#[case::before_slash("\u{338F}/h")]
#[case::after_slash("h/\u{338F}")]
fn a_square_unit_joined_through_a_slash_decomposes(#[case] input: &str) {
let folded = normalize_pure_roman_compatibility_units(std::borrow::Cow::Borrowed(input));
assert!(
folded.contains("kg"),
"expected the unit to spell out, got {folded:?}"
);
}
#[test]
fn a_detached_square_unit_keeps_its_glyph() {
let folded =
normalize_pure_roman_compatibility_units(std::borrow::Cow::Borrowed("\u{338F}"));
assert_eq!(folded.as_ref(), "\u{338F}");
}
}
#[cfg(test)]
mod print_variant_fold_coverage {
use super::*;
use std::borrow::Cow;
#[rstest::rstest]
#[case::ring_above_alone("\u{02DA}", "\u{00B0}")]
#[case::unicode_hyphen("\u{2010}", "-")]
#[case::non_breaking_hyphen("\u{2011}", "-")]
#[case::katakana_middle_dot("\u{30FB}", "\u{00B7}")]
#[case::halfwidth_middle_dot("\u{FF65}", "\u{00B7}")]
#[case::hyphenation_point("\u{2027}", "\u{00B7}")]
#[case::one_dot_leader("\u{2024}", "\u{00B7}")]
#[case::vector_cross("\u{2A2F}", "\u{00D7}")]
#[case::parenthesised_five("\u{2478}", "(5)")]
#[case::parenthesised_twenty("\u{2487}", "(20)")]
#[case::wave_dash("\u{301C}", "~")]
#[case::acute_accent("\u{00B4}", "'")]
#[case::soft_hyphen("\u{00AD}", "")]
#[case::zero_width_space("\u{200B}", "")]
#[case::zero_width_joiner("\u{200D}", "")]
#[case::byte_order_mark("\u{FEFF}", "")]
#[case::fullwidth_percent("\u{FF05}", "%")]
#[case::fullwidth_letter("\u{FF4D}", "m")]
#[case::plain_char_is_kept("m", "m")]
fn a_print_variant_folds_to_the_character_the_standard_defines(
#[case] input: &str,
#[case] expected: &str,
) {
assert_eq!(
normalize_print_variants(Cow::Borrowed(input)).as_ref(),
expected
);
}
#[rstest::rstest]
#[case::celsius("25\u{00B0}C", "25\u{2103}")]
#[case::fahrenheit("77\u{00B0}F", "77\u{2109}")]
#[case::letter_without_a_degree("25C", "25C")]
#[case::degree_at_the_end("25\u{00B0}", "25\u{00B0}")]
fn a_degree_letter_pair_folds_to_the_unit_glyph(#[case] input: &str, #[case] expected: &str) {
assert_eq!(
normalize_print_variants(Cow::Borrowed(input)).as_ref(),
expected
);
}
#[rstest::rstest]
#[case::black_telephone("\u{260E}051", "Tel051")]
#[case::white_telephone("\u{260F}051", "Tel051")]
#[case::other_char_is_kept("051", "051")]
fn a_telephone_sign_expands_to_its_printed_meaning(
#[case] input: &str,
#[case] expected: &str,
) {
assert_eq!(expand_pictographs(Cow::Borrowed(input)).as_ref(), expected);
}
#[rstest::rstest]
#[case::before_slash("\u{338F}/h")]
#[case::after_slash("h/\u{338F}")]
fn a_square_unit_joined_through_a_slash_decomposes(#[case] input: &str) {
let folded = normalize_pure_roman_compatibility_units(Cow::Borrowed(input));
assert!(folded.contains("kg"), "expected kg in {folded:?}");
}
#[test]
fn a_detached_square_unit_keeps_its_glyph() {
assert_eq!(
normalize_pure_roman_compatibility_units(Cow::Borrowed("\u{338F}")).as_ref(),
"\u{338F}"
);
}
}