use std::{borrow::Cow, cell::RefCell};
mod char_shortcut;
pub(crate) mod char_struct;
#[cfg(feature = "cli")]
pub mod cli;
mod encoder;
pub(crate) mod english;
pub(crate) mod english_logic;
pub(crate) mod fraction;
mod ipa;
mod jauem;
mod korean_char;
mod korean_part;
mod math_symbol_shortcut;
mod moeum;
pub(crate) mod number;
mod rule;
pub(crate) mod rules;
mod split;
pub(crate) mod symbol_shortcut;
pub(crate) mod unicode;
pub(crate) mod utils;
pub(crate) mod word_shortcut;
use ipa::{detect_ipa_context, encode_ipa, is_ipa_phonetic_symbol};
#[cfg(test)]
mod test_helpers {
use crate::char_struct::CharType;
use crate::rules::context::{EncoderState, RuleContext};
pub(crate) struct CtxOwned {
pub word_chars: Vec<char>,
pub char_types: Vec<CharType>,
pub skip_count: usize,
pub state: EncoderState,
pub result: Vec<u8>,
pub prev_word: String,
pub remaining_words: Vec<String>,
}
impl CtxOwned {
pub(crate) fn for_text(text: &str, english_indicator: bool) -> Self {
let word_chars: Vec<char> = text.chars().collect();
let char_types: Vec<CharType> = word_chars
.iter()
.map(|c| CharType::new(*c).expect("CharType::new should not fail in tests"))
.collect();
Self {
word_chars,
char_types,
skip_count: 0,
state: EncoderState::new(english_indicator),
result: Vec::new(),
prev_word: String::new(),
remaining_words: Vec::new(),
}
}
pub(crate) fn with_prev_word(mut self, prev_word: impl Into<String>) -> Self {
self.prev_word = prev_word.into();
self
}
pub(crate) fn with_remaining_words<I, S>(mut self, words: I) -> Self
where
I: IntoIterator<Item = S>,
S: Into<String>,
{
self.remaining_words = words.into_iter().map(Into::into).collect();
self
}
pub(crate) fn ctx_at<'a>(&'a mut self, index: usize) -> RuleContext<'a> {
let remaining: Vec<&str> = self.remaining_words.iter().map(String::as_str).collect();
let leaked: &'a [&'a str] = Box::leak(remaining.into_boxed_slice());
RuleContext {
word_chars: &self.word_chars,
index,
char_type: &self.char_types[index],
prev_word: &self.prev_word,
remaining_words: leaked,
has_korean_char: self.word_chars.iter().any(|c| {
let cp = *c as u32;
(0xAC00..=0xD7A3).contains(&cp)
}),
is_all_uppercase: false,
ascii_starts_at_beginning: false,
skip_count: &mut self.skip_count,
state: &mut self.state,
result: &mut self.result,
}
}
}
}
pub use encoder::Encoder;
thread_local! {
static ENCODER_CACHE: RefCell<Option<Encoder>> = const { RefCell::new(None) };
}
fn with_encoder<F, R>(english_indicator: bool, f: F) -> R
where
F: FnOnce(&mut Encoder) -> R,
{
ENCODER_CACHE.with(|cell| {
let Ok(mut cached) = cell.try_borrow_mut() else {
let mut encoder = Encoder::new(english_indicator);
encoder.reset_state();
return f(&mut encoder);
};
if !matches!(&*cached, Some(encoder) if encoder.english_indicator() == english_indicator) {
*cached = Some(Encoder::new(english_indicator));
}
let encoder = cached.as_mut().expect("encoder cache just initialized");
encoder.reset_state();
f(encoder)
})
}
#[derive(Debug, Clone, Default)]
pub struct EncodeOptions {
pub default_mode: Option<crate::rules::context::EncodingMode>,
}
#[derive(Debug, Clone)]
pub struct FormattingSpan {
pub range: std::ops::Range<usize>,
pub kind: FormattingKind,
}
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub enum FormattingKind {
Emphasis,
Bold,
Custom1,
Custom2,
}
impl FormattingKind {
pub(crate) fn markers(self) -> ([u8; 2], [u8; 2]) {
match self {
Self::Emphasis => ([32, 36], [36, 4]),
Self::Bold => ([48, 36], [36, 6]),
Self::Custom1 => ([16, 36], [36, 2]),
Self::Custom2 => ([8, 36], [36, 1]),
}
}
}
pub fn encode(text: &str) -> Result<Vec<u8>, String> {
encode_with_options(text, &EncodeOptions::default())
}
fn normalize_math_alphanumeric_char(c: char) -> char {
let cp = c as u32;
if cp == 0x210E {
return 'h';
}
const BLOCKS: &[(u32, char)] = &[
(0x1D400, 'A'),
(0x1D41A, 'a'),
(0x1D434, 'A'),
(0x1D44E, 'a'),
(0x1D468, 'A'),
(0x1D482, 'a'),
(0x1D49C, 'A'),
(0x1D4B6, 'a'),
(0x1D4D0, 'A'),
(0x1D4EA, 'a'),
(0x1D504, 'A'),
(0x1D51E, 'a'),
(0x1D538, 'A'),
(0x1D552, 'a'),
(0x1D56C, 'A'),
(0x1D586, 'a'),
(0x1D5A0, 'A'),
(0x1D5BA, 'a'),
(0x1D5D4, 'A'),
(0x1D5EE, 'a'),
(0x1D608, 'A'),
(0x1D622, 'a'),
(0x1D63C, 'A'),
(0x1D656, 'a'),
(0x1D670, 'A'),
(0x1D68A, 'a'),
];
for &(start, base) in BLOCKS {
if cp >= start && cp < start + 26 {
return char::from(base as u8 + (cp - start) as u8);
}
}
const DIGIT_BLOCKS: &[u32] = &[0x1D7CE, 0x1D7D8, 0x1D7E2, 0x1D7EC, 0x1D7F6];
for &start in DIGIT_BLOCKS {
if cp >= start && cp < start + 10 {
return char::from(b'0' + (cp - start) as u8);
}
}
c
}
fn may_normalize_math_alphanumeric(c: char) -> bool {
let cp = c as u32;
cp == 0x210E || (0x1D400..=0x1D7FF).contains(&cp)
}
fn normalize_math_alphanumeric_string(text: &str) -> Cow<'_, str> {
if !text.chars().any(may_normalize_math_alphanumeric) {
return Cow::Borrowed(text);
}
Cow::Owned(text.chars().map(normalize_math_alphanumeric_char).collect())
}
fn default_math_expression_needs_whole_route(text: &str) -> bool {
let chars: Vec<char> = text.chars().collect();
if chars.len() < 2 {
return false;
}
let has_operand = chars.iter().any(|c| c.is_ascii_alphanumeric());
has_operand
&& chars.iter().enumerate().any(|(i, c)| match *c {
'→' | '←' | '↗' | '↘' | '↑' | '↓' | '△' | '□' => true,
'\u{0304}' | '\u{0302}' => combining_mark_on_single_letter(&chars, i),
_ => false,
})
}
fn combining_mark_on_single_letter(chars: &[char], i: usize) -> bool {
let is_combining = |c: char| ('\u{0300}'..='\u{036F}').contains(&c);
let mut letters = 0usize;
let mut j = i;
while j > 0 {
let c = chars[j - 1];
if c.is_alphabetic() {
letters += 1;
} else if !is_combining(c) {
break;
}
j -= 1;
}
let mut j = i + 1;
while j < chars.len() {
let c = chars[j];
if c.is_alphabetic() {
letters += 1;
} else if !is_combining(c) {
break;
}
j += 1;
}
letters < 3
}
#[derive(Clone, Copy, Default)]
struct NormalizationTriggers {
has_math_alphanumeric: bool,
has_decomposable_latin: bool,
has_negation_combiner: bool,
has_vector_mark: bool,
has_formatting_mark_or_sentinel: bool,
has_ipa_group_start: bool,
has_ipa_symbol: bool,
}
impl NormalizationTriggers {
fn scan(text: &str) -> Self {
let mut triggers = Self::default();
for c in text.chars() {
triggers.has_math_alphanumeric |= may_normalize_math_alphanumeric(c);
triggers.has_decomposable_latin |= may_decompose_accented_latin(c);
triggers.has_negation_combiner |= c == '\u{0338}';
triggers.has_vector_mark |= is_vector_mark(c);
triggers.has_formatting_mark_or_sentinel |=
is_formatting_mark(c) || is_formatting_sentinel(c);
triggers.has_ipa_group_start |= matches!(c, '[' | '/');
triggers.has_ipa_symbol |= is_ipa_phonetic_symbol(c);
}
triggers
}
fn may_need_emphasis_expansion(self) -> bool {
self.has_formatting_mark_or_sentinel || self.has_decomposable_latin
}
fn may_contain_ipa_context(self) -> bool {
self.has_ipa_group_start && self.has_ipa_symbol
}
}
fn move_negation_combiner_before_base<'a>(text: Cow<'a, str>) -> Cow<'a, str> {
if !text.as_ref().contains('\u{0338}') {
return text;
}
let source = text.as_ref();
let chars: Vec<char> = source.chars().collect();
let mut out = String::with_capacity(source.len());
let mut i = 0;
while i < chars.len() {
if i + 1 < chars.len() && chars[i + 1] == '\u{0338}' {
out.push(chars[i + 1]);
out.push(chars[i]);
i += 2;
} else {
out.push(chars[i]);
i += 1;
}
}
Cow::Owned(out)
}
fn expand_emphasis_marks<'a>(text: Cow<'a, str>) -> Cow<'a, str> {
const FORMATTING_MARKS: &[(char, char, char)] = &[
('\u{0307}', '\u{E000}', '\u{E001}'), ('\u{0331}', '\u{E002}', '\u{E003}'), ('\u{0332}', '\u{E004}', '\u{E005}'), ('\u{0333}', '\u{E006}', '\u{E007}'), ];
if !text
.as_ref()
.chars()
.any(|c| is_formatting_sentinel(c) || is_formatting_mark(c))
{
return text;
}
let source = text.as_ref();
let chars: Vec<char> = source.chars().collect();
let mut token_has_korean = vec![false; chars.len()];
{
let mut i = 0;
while i < chars.len() {
if chars[i] == ' ' {
i += 1;
continue;
}
let start = i;
while i < chars.len() && chars[i] != ' ' {
i += 1;
}
let has = chars[start..i].iter().any(|c| utils::is_korean_char(*c));
for slot in token_has_korean.iter_mut().take(i).skip(start) {
*slot = has;
}
}
}
let mut out: Vec<char> = Vec::with_capacity(chars.len());
let mut i = 0;
while i < chars.len() {
let mark_entry = FORMATTING_MARKS
.iter()
.find(|(mark, _, _)| *mark == chars[i]);
let Some(&(mark_char, start_sentinel, end_sentinel)) = mark_entry else {
out.push(chars[i]);
i += 1;
continue;
};
if !token_has_korean[i] {
out.push(chars[i]);
i += 1;
continue;
}
let mut count = 1;
let mut last = i;
let mut j = i + 1;
while j < chars.len() {
if chars[j] == mark_char {
count += 1;
last = j;
j += 1;
} else if chars[j] == ' ' && j + 1 < chars.len() && chars[j + 1] == mark_char {
j += 1;
} else {
break;
}
}
let mut units = 0;
let mut start_in_out = out.len();
while start_in_out > 0 && units < count {
let c = out[start_in_out - 1];
if c == ' ' || is_formatting_sentinel(c) {
start_in_out -= 1;
} else {
units += 1;
start_in_out -= 1;
}
}
if units == count {
while start_in_out > 0 {
let c = out[start_in_out - 1];
if c.is_ascii_digit() || matches!(c, ',' | '.') {
start_in_out -= 1;
} else {
break;
}
}
out.insert(start_in_out, start_sentinel);
out.push(end_sentinel);
} else {
for _ in 0..count {
out.push(mark_char);
}
}
i = last + 1;
}
merge_adjacent_formatting_wraps(Cow::Owned(out.into_iter().collect()))
}
fn is_formatting_sentinel(c: char) -> bool {
matches!(c as u32, 0xE000..=0xE007)
}
fn is_formatting_mark(c: char) -> bool {
matches!(c, '\u{0307}' | '\u{0331}' | '\u{0332}' | '\u{0333}')
}
fn merge_adjacent_formatting_wraps<'a>(text: Cow<'a, str>) -> Cow<'a, str> {
const SENTINEL_PAIRS: &[(char, char)] = &[
('\u{E000}', '\u{E001}'),
('\u{E002}', '\u{E003}'),
('\u{E004}', '\u{E005}'),
('\u{E006}', '\u{E007}'),
];
if !text.as_ref().chars().any(is_formatting_sentinel) {
return text;
}
let mut chars: Vec<char> = text.as_ref().chars().collect();
let mut any_changed = false;
let mut changed = true;
while changed {
changed = false;
for &(open, close) in SENTINEL_PAIRS {
let mut i = 0;
while i < chars.len() {
if chars[i] != close {
i += 1;
continue;
}
let mut j = i + 1;
while j < chars.len() && chars[j] == ' ' {
j += 1;
}
if j < chars.len() && chars[j] == open {
chars.remove(j);
chars.remove(i);
changed = true;
any_changed = true;
} else {
i += 1;
}
}
}
}
if any_changed {
Cow::Owned(chars.into_iter().collect())
} else {
text
}
}
fn is_vector_mark(c: char) -> bool {
matches!(c, '\u{20D6}' | '\u{20D7}' | '\u{20E1}' | '\u{20D1}')
}
fn collapse_repeated_vector_marks<'a>(text: Cow<'a, str>) -> Cow<'a, str> {
debug_assert!(text.as_ref().chars().any(is_vector_mark));
let source = text.as_ref();
let chars: Vec<char> = source.chars().collect();
let mut out = String::with_capacity(source.len());
let mut i = 0;
let mut changed = false;
while i < chars.len() {
if chars[i].is_ascii_alphabetic() && i + 1 < chars.len() && is_vector_mark(chars[i + 1]) {
changed = true;
let mark = chars[i + 1];
let mut letters = vec![chars[i]];
let mut j = i + 2;
while j + 1 < chars.len() && chars[j].is_ascii_alphabetic() && chars[j + 1] == mark {
letters.push(chars[j]);
j += 2;
}
out.push(mark);
for l in letters {
out.push(l);
}
i = j;
continue;
}
out.push(chars[i]);
i += 1;
}
if changed { Cow::Owned(out) } else { text }
}
fn may_decompose_accented_latin(c: char) -> bool {
let cp = c as u32;
!matches!(c, '\u{00C5}' | '\u{00E5}')
&& ((0x00C0..=0x024F).contains(&cp) || (0x1E00..=0x1EFF).contains(&cp))
}
fn decompose_accented_latin<'a>(text: Cow<'a, str>) -> Cow<'a, str> {
use unicode_normalization::UnicodeNormalization;
let mut out = String::new();
for c in text.as_ref().chars() {
if may_decompose_accented_latin(c) {
for d in std::iter::once(c).nfd() {
out.push(d);
}
} else {
out.push(c);
}
}
Cow::Owned(out)
}
fn is_isolated_roman_section(text: &str) -> bool {
let mut has_letter = false;
for ch in text.chars() {
if ch == ' ' {
continue;
}
if ch.is_ascii_alphabetic() {
has_letter = true;
} else {
return false;
}
}
has_letter
}
pub fn encode_with_options(text: &str, options: &EncodeOptions) -> Result<Vec<u8>, String> {
use crate::rules::context::EncodingMode;
let normalization_triggers = NormalizationTriggers::scan(text);
if options.default_mode.is_none()
&& !text.chars().any(crate::utils::is_korean_char)
&& crate::rules::english_ueb::is_ueb_eligible(text)
&& !crate::rules::english_ueb::is_math_owned(text)
&& let Some(bytes) = crate::rules::english_ueb::try_encode(text)
{
return Ok(bytes);
}
if matches!(options.default_mode, Some(EncodingMode::English))
&& let Some(bytes) = crate::rules::english_ueb::encode_forced(text)
{
return Ok(bytes);
}
let normalized_text = if normalization_triggers.has_math_alphanumeric {
normalize_math_alphanumeric_string(text)
} else {
Cow::Borrowed(text)
};
let normalized_text = if normalization_triggers.has_decomposable_latin {
decompose_accented_latin(normalized_text)
} else {
normalized_text
};
let normalized_text = if normalization_triggers.has_negation_combiner {
move_negation_combiner_before_base(normalized_text)
} else {
normalized_text
};
let normalized_text = if normalization_triggers.has_vector_mark {
collapse_repeated_vector_marks(normalized_text)
} else {
normalized_text
};
let normalized_text = if normalization_triggers.may_need_emphasis_expansion() {
expand_emphasis_marks(normalized_text)
} else {
normalized_text
};
let text: &str = normalized_text.as_ref();
let matrix_context = text.contains("행렬");
let math_mode = matches!(options.default_mode, Some(EncodingMode::Math));
let math_context = crate::rules::math::math_token_rule::MathContext {
matrix_context_active: matrix_context,
math_mode_active: math_mode,
};
let ipa_auto = options.default_mode.is_none()
&& normalization_triggers.may_contain_ipa_context()
&& detect_ipa_context(text);
if ipa_auto || matches!(options.default_mode, Some(EncodingMode::Ipa)) {
return encode_ipa(text);
}
if let Some(EncodingMode::ObjectSymbol) = options.default_mode {
let chars: Vec<char> = text.chars().collect();
if chars.len() == 1 {
let mark = match chars[0] {
'○' => Some(52u8), '×' => Some(45u8), '△' => Some(44u8), '□' => Some(54u8), _ => None,
};
if let Some(m) = mark {
return Ok(vec![56, m, 7]); }
}
}
if let Some(EncodingMode::Number) = options.default_mode {
let chars: Vec<char> = text.chars().collect();
if !chars.is_empty()
&& chars.iter().all(|c| {
matches!(
c.to_ascii_uppercase(),
'I' | 'V' | 'X' | 'L' | 'C' | 'D' | 'M'
)
})
{
let mut out = vec![52u8]; if chars.iter().all(|c| c.is_ascii_uppercase()) {
out.push(32); if chars.len() >= 2 {
out.push(32); }
}
for ch in &chars {
out.push(crate::english::encode_english(ch.to_ascii_lowercase())?);
}
out.push(50); return Ok(out);
}
}
let default_math_owned = options.default_mode.is_none()
&& default_math_expression_needs_whole_route(text)
&& !text.chars().any(crate::utils::is_korean_char);
if matches!(options.default_mode, Some(EncodingMode::Math)) || default_math_owned {
let chars: Vec<char> = text.chars().collect();
if chars.len() == 1 && chars[0].is_ascii_lowercase() {
return Ok(vec![52, crate::english::encode_english(chars[0])?]);
}
if chars.len() == 1 {
match chars[0] {
'(' => return Ok(vec![38]), ')' => return Ok(vec![52]), '{' => return Ok(vec![54]), '}' => return Ok(vec![54]), '[' => return Ok(vec![55, 4]), ']' => return Ok(vec![32, 62]), _ => {}
}
}
if chars.len() == 1
&& let Ok(code) =
crate::math_symbol_shortcut::encode_char_math_symbol_shortcut(chars[0])
{
return Ok(code.to_vec());
}
let cleaned: String = {
let mut s = String::with_capacity(text.len());
let chs: Vec<char> = text.chars().collect();
let mut i = 0;
while i < chs.len() {
let c = chs[i];
if c == ' '
&& i + 1 < chs.len()
&& matches!(chs[i + 1], '=' | '+' | '-' | '<' | '>')
{
i += 1;
continue;
}
if matches!(c, '=' | '+' | '-' | '<' | '>')
&& i + 1 < chs.len()
&& chs[i + 1] == ' '
{
s.push(c);
i += 2;
continue;
}
s.push(c);
i += 1;
}
s
};
if let Ok(bytes) =
rules::math::encoder::encode_math_expression_with_context(&cleaned, math_context)
{
return Ok(bytes);
}
}
let english_indicator = text
.split(' ')
.filter(|w| !w.is_empty())
.any(|word| word.chars().any(utils::is_korean_char));
with_encoder(english_indicator, |encoder| {
encoder.set_matrix_context_active(matrix_context);
encoder.set_math_mode_active(math_mode);
if let Some(mode) = options.default_mode {
encoder.set_default_mode(mode);
}
let mut result = Vec::new();
encoder.encode(text, &mut result)?;
let wrap_roman_section = matches!(options.default_mode, Some(EncodingMode::English))
|| (matches!(options.default_mode, Some(EncodingMode::Korean))
&& is_isolated_roman_section(text));
if wrap_roman_section && !result.is_empty() {
result.insert(0, 52);
result.push(50);
}
Ok(result)
})
}
pub fn encode_with_formatting(text: &str, spans: &[FormattingSpan]) -> Result<Vec<u8>, String> {
if spans.is_empty() {
return encode(text);
}
let english_indicator = text
.split(' ')
.filter(|w| !w.is_empty())
.any(|word| word.chars().any(utils::is_korean_char));
with_encoder(english_indicator, |encoder| {
let mut result = Vec::new();
encoder.encode_with_formatting(text, spans, &mut result)?;
Ok(result)
})
}
pub fn encode_to_unicode(text: &str) -> Result<String, String> {
let result = encode(text)?;
Ok(result
.iter()
.map(|c| unicode::encode_unicode(*c))
.collect::<String>())
}
pub fn encode_to_unicode_with_formatting(
text: &str,
spans: &[FormattingSpan],
) -> Result<String, String> {
let result = encode_with_formatting(text, spans)?;
Ok(result
.iter()
.map(|c| unicode::encode_unicode(*c))
.collect::<String>())
}
pub fn encode_to_braille_font(text: &str) -> Result<String, String> {
let result = encode(text)?;
Ok(result
.iter()
.map(|c| unicode::encode_unicode(*c))
.collect::<String>())
}
#[cfg(test)]
mod state_bleed_tests {
use super::encode;
#[test]
fn cached_encoder_resets_between_different_contexts() {
let before = encode("안녕").unwrap();
let _english = encode("hello").unwrap();
let after = encode("안녕").unwrap();
assert_eq!(before, after);
}
}
#[cfg(test)]
mod test {
use std::{collections::HashMap, fs::File};
use crate::{symbol_shortcut, unicode::encode_unicode};
use proptest::prelude::*;
use super::*;
fn find_nth_range(text: &str, needle: &str, _nth: usize) -> std::ops::Range<usize> {
let start = text
.find(needle)
.unwrap_or_else(|| panic!("substring '{needle}' not found in '{text}'"));
start..start + needle.len()
}
#[test]
fn english_continuation_after_inline_number() {
let output = encode("가 a1a").unwrap();
assert!(
output.contains(&48),
"inline number should trigger english continuation indicator"
);
}
#[test]
fn symbol_triggers_english_segment_at_start() {
let output = encode("(A 가").unwrap();
let english_symbol = symbol_shortcut::encode_english_char_symbol_shortcut('(').unwrap();
assert_eq!(output[0], 52);
assert!(output.len() > english_symbol.len());
assert_eq!(
&output[1..1 + english_symbol.len()],
english_symbol,
"opening english symbol should use english shortcut"
);
}
#[rstest::rstest]
#[case::slash_forced_symbol("가 a/")]
#[case::underscore_leave_english("가 a_b")]
fn english_symbol_terminator_variants(#[case] input: &str) {
let output = encode(input).unwrap();
assert!(
output.contains(&50),
"english terminator (50) absent for {input:?}"
);
}
#[test]
fn comma_prefix_variants_and_korean_following() {
let output = encode("가 A,가").unwrap();
let comma = symbol_shortcut::encode_char_symbol_shortcut(',').unwrap();
assert!(
output.windows(comma.len()).any(|window| window == comma),
"comma before Korean should use Korean punctuation mapping"
);
assert!(encode("가 A!,가").is_ok());
}
#[test]
fn next_word_single_letter_sets_continuation_flag() {
let output = encode("가 a b").unwrap();
assert!(
output.contains(&48),
"single-letter following word should trigger continuation marker"
);
}
#[rstest::rstest]
#[case::forced_symbol_inserts_terminator("가 a /", 50)]
#[case::skip_symbol_requests_continuation("가 a . b", 48)]
fn next_word_symbol_rules_apply(#[case] input: &str, #[case] expected_byte: u8) {
let output = encode(input).unwrap();
assert!(
output.contains(&expected_byte),
"expected byte {expected_byte} not in output for {input:?}"
);
}
#[test]
fn next_word_with_invalid_char_returns_error() {
let err = encode("가 a 😀");
assert!(err.is_err());
}
#[test]
fn encode_with_formatting_wraps_markers() {
let text = "다음 보기에서 명사가 아닌 것은?";
let spans = vec![FormattingSpan {
range: find_nth_range(text, "아닌", 0),
kind: FormattingKind::Emphasis,
}];
let unicode = encode_to_unicode_with_formatting(text, &spans).unwrap();
assert!(unicode.contains("⠠⠤⠣⠉⠟⠤⠄"));
}
#[test]
fn encode_with_formatting_rejects_non_boundary_range() {
let text = "왜";
let spans = [FormattingSpan {
range: 1..3,
kind: FormattingKind::Emphasis,
}];
let err = encode_with_formatting(text, &spans);
assert!(err.is_err());
}
fn collect_test_files() -> Vec<(std::path::PathBuf, String)> {
let test_cases_dir =
std::path::Path::new(concat!(env!("CARGO_MANIFEST_DIR"), "/../../test_cases"));
let mut files = Vec::new();
for entry in std::fs::read_dir(test_cases_dir).unwrap() {
let entry = entry.unwrap();
let path = entry.path();
if path.is_dir() {
let subdir = path.file_name().unwrap().to_string_lossy().to_string();
for sub_entry in std::fs::read_dir(&path).unwrap() {
let sub_entry = sub_entry.unwrap();
let sub_path = sub_entry.path();
if sub_path.extension().unwrap_or_default() == "json" {
let stem = sub_path.file_stem().unwrap().to_string_lossy().to_string();
let key = format!("{}/{}", subdir, stem);
files.push((sub_path, key));
}
}
}
}
files.sort_by(|a, b| a.1.cmp(&b.1));
files
}
fn testcase_answer_forms(
record: &serde_json::Value,
filename: &str,
line_num: usize,
) -> Vec<(String, String, String)> {
if let Some(serde_json::Value::Array(alternatives)) = record.get("alternatives") {
return alternatives
.iter()
.map(|alternative| {
let internal = alternative["internal"].as_str().unwrap_or_else(|| {
panic!(
"'alternatives.internal' 필드를 읽는 중 오류 발생: at {} in {}",
line_num, filename
)
});
let expected = alternative["expected"].as_str().unwrap_or_else(|| {
panic!(
"'alternatives.expected' 필드를 읽는 중 오류 발생: at {} in {}",
line_num, filename
)
});
let unicode = alternative["unicode"].as_str().unwrap_or_else(|| {
panic!(
"'alternatives.unicode' 필드를 읽는 중 오류 발생: at {} in {}",
line_num, filename
)
});
(
internal.to_string(),
expected.to_string(),
unicode.to_string(),
)
})
.collect();
}
let internal = record["internal"].as_str().unwrap_or_else(|| {
panic!(
"'internal' 필드를 읽는 중 오류 발생: at {} in {}",
line_num, filename
)
});
let expected = record["expected"].as_str().unwrap_or_else(|| {
panic!(
"'expected' 필드를 읽는 중 오류 발생: at {} in {}",
line_num, filename
)
});
let unicode = record["unicode"].as_str().unwrap_or_else(|| {
panic!(
"'unicode' 필드를 읽는 중 오류 발생: at {} in {}",
line_num, filename
)
});
vec![(
internal.to_string(),
expected.to_string(),
unicode.to_string(),
)]
}
#[test]
pub fn test_by_testcase() {
let files = collect_test_files();
let mut total = 0;
let mut failed = 0;
let mut failed_cases = Vec::new();
let mut skipped_cases: Vec<(String, usize, String, String)> = Vec::new();
let mut file_stats = std::collections::BTreeMap::new();
let rule_map: HashMap<String, HashMap<String, String>> = serde_json::from_str(
&std::fs::read_to_string(concat!(env!("CARGO_MANIFEST_DIR"), "/../../rule_map.json"))
.unwrap(),
)
.unwrap();
let rule_map_keys: std::collections::HashSet<String> = rule_map.keys().cloned().collect();
let file_keys: std::collections::HashSet<_> =
files.iter().map(|(_, key)| key.clone()).collect();
let missing_keys = rule_map_keys.difference(&file_keys).collect::<Vec<_>>();
let extra_keys = file_keys.difference(&rule_map_keys).collect::<Vec<_>>();
if !missing_keys.is_empty() || !extra_keys.is_empty() {
panic!(
"rule_map.json 파일이 올바르지 않습니다. missing: {:?}, extra: {:?}",
missing_keys, extra_keys
);
}
for (path, file_stem) in &files {
let content = std::fs::read_to_string(path).unwrap();
let filename = path.file_name().unwrap().to_string_lossy();
let records: Vec<serde_json::Value> = serde_json::from_str(&content)
.unwrap_or_else(|e| panic!("JSON 파일을 읽는 중 오류 발생: {} in {}", e, filename));
let mut file_total = 0;
let mut file_failed = 0;
let mut file_world_total = 0;
let mut file_world_failed = 0;
let mut file_jeomsarang_total = 0;
let mut file_jeomsarang_failed = 0;
type TestStatusRow = (
String,
String,
String,
String,
bool,
String,
bool,
String,
bool,
);
let mut test_status: Vec<TestStatusRow> = Vec::new();
for (line_num, record) in records.iter().enumerate() {
if let Some(reason) = record.get("limitation").and_then(|v| v.as_str()) {
let input = record["input"].as_str().unwrap_or("");
let expected_values = testcase_answer_forms(record, &filename, line_num)
.into_iter()
.map(|(_, _, unicode)| unicode)
.collect::<Vec<_>>();
if let Ok(actual) = crate::encode_to_unicode(input)
&& expected_values.contains(&actual)
{
panic!(
"STALE limitation in {} line {}: input={:?} passes but is marked limitation: {:?}",
filename, line_num, input, reason
);
}
skipped_cases.push((
filename.to_string(),
line_num + 1,
input.to_string(),
reason.to_string(),
));
continue;
}
total += 1;
file_total += 1;
let input = record["input"].as_str().unwrap_or_else(|| {
panic!(
"'input' 필드를 읽는 중 오류 발생: at {} in {}",
line_num, filename
)
});
let context = record["context"].as_str().unwrap_or("");
let note = record["note"].as_str().unwrap_or("").to_string();
let world = record["world"].as_str().unwrap_or("").to_string();
file_world_total += 1;
let jeomsarang = record["jeomsarang"].as_str().unwrap_or("").to_string();
file_jeomsarang_total += 1;
let answer_forms = testcase_answer_forms(record, &filename, line_num);
let expected_forms = answer_forms
.iter()
.map(|(_, expected, _)| expected)
.map(|expected| expected.trim().replace(" ", "⠀"))
.collect::<Vec<_>>();
let unicode_forms = answer_forms
.into_iter()
.map(|(_, _, unicode)| unicode)
.collect::<Vec<_>>();
let expected_display = expected_forms.join(" / ");
let unicode_display = unicode_forms.join(" / ");
let input_for_encoding: String =
if let Some(prefix) = context.strip_prefix("strip_prefix:") {
input.strip_prefix(prefix).unwrap_or(input).to_string()
} else {
input.to_string()
};
let encoding_result = match context.parse::<crate::rules::context::EncodingMode>() {
Ok(mode) => encode_with_options(
&input_for_encoding,
&EncodeOptions {
default_mode: Some(mode),
},
),
Err(_) => encode(&input_for_encoding),
};
match encoding_result {
Ok(actual) => {
let braille_expected = actual
.iter()
.map(|c| unicode::encode_unicode(*c))
.collect::<String>();
let actual_str = actual
.iter()
.map(|c| {
if *c == 255 {
"\n".to_string()
} else {
c.to_string()
}
})
.collect::<String>();
let case_matches = expected_forms.contains(&actual_str);
if !case_matches {
failed += 1;
file_failed += 1;
failed_cases.push((
filename.to_string(),
line_num + 1,
input.to_string(),
expected_display.clone(),
actual_str.clone(),
braille_expected.clone(),
unicode_display.clone(),
));
}
let world_is_success = !world.is_empty() && unicode_forms.contains(&world);
if !world_is_success {
file_world_failed += 1;
}
let jeomsarang_is_success =
!jeomsarang.is_empty() && unicode_forms.contains(&jeomsarang);
if !jeomsarang_is_success {
file_jeomsarang_failed += 1;
}
test_status.push((
input.to_string(),
note.clone(),
unicode_display.clone(),
braille_expected.clone(),
case_matches,
world.clone(),
world_is_success,
jeomsarang.clone(),
jeomsarang_is_success,
));
}
Err(e) => {
println!("Error: {}", e);
failed += 1;
file_failed += 1;
failed_cases.push((
filename.to_string(),
line_num + 1,
input.to_string(),
expected_display.clone(),
"".to_string(),
e.to_string(),
unicode_display.clone(),
));
let world_is_success = !world.is_empty() && unicode_forms.contains(&world);
if !world_is_success {
file_world_failed += 1;
}
let jeomsarang_is_success =
!jeomsarang.is_empty() && unicode_forms.contains(&jeomsarang);
if !jeomsarang_is_success {
file_jeomsarang_failed += 1;
}
test_status.push((
input.to_string(),
note.clone(),
unicode_display.clone(),
e.to_string(),
false,
world.clone(),
world_is_success,
jeomsarang.clone(),
jeomsarang_is_success,
));
}
}
}
file_stats.insert(
file_stem.clone(),
(
file_total,
file_failed,
file_world_total,
file_world_failed,
file_jeomsarang_total,
file_jeomsarang_failed,
test_status,
),
);
}
if !failed_cases.is_empty() {
println!("\n실패한 케이스:");
println!("=================");
for (filename, line_num, input, expected, actual, unicode, braille) in failed_cases {
let diff = {
let unicode_words: Vec<&str> = unicode.split(encode_unicode(0)).collect();
let braille_words: Vec<&str> = braille.split(encode_unicode(0)).collect();
let mut diff = Vec::new();
for (i, (u, b)) in unicode_words.iter().zip(braille_words.iter()).enumerate() {
if u != b {
diff.push(i);
}
}
diff
};
let input_words: Vec<&str> = input.split(' ').collect();
let unicode_words: Vec<&str> = unicode.split(encode_unicode(0)).collect();
if input_words.len() != unicode_words.len() {
println!("파일: {}, 라인 {}: '{}'", filename, line_num, input);
println!(" 예상: {}", expected);
println!(" 실제: {}", actual);
println!(" 유니코드 Result: {}", unicode);
println!(" 유니코드 Expected: {}", braille);
} else {
let mut colored_input = String::new();
let mut colored_unicode = String::new();
for (i, word) in input_words.iter().enumerate() {
if diff.contains(&i) {
colored_input.push_str(&format!("\x1b[31m{}\x1b[0m", word));
colored_unicode
.push_str(&format!("\x1b[31m{}\x1b[0m", unicode_words[i]));
} else {
colored_input.push_str(word);
colored_unicode.push_str(unicode_words[i]);
}
if i < input_words.len() - 1 {
colored_input.push(' ');
colored_unicode.push(' ');
}
}
println!("파일: {}, 라인 {}: '{}'", filename, line_num, colored_input);
println!(" 예상: {}", expected);
println!(" 실제: {}", actual);
println!(" 유니코드 Result: {}", colored_unicode);
println!(" 유니코드 Expected: {}", braille);
}
println!();
}
}
if !skipped_cases.is_empty() {
println!("\nSkip된 케이스 (limitation):");
println!("=================");
for (filename, line_num, input, reason) in &skipped_cases {
println!(
"\x1b[33m파일: {}, 라인 {}: '{}'\x1b[0m",
filename, line_num, input
);
println!(" 사유: {}", reason);
println!();
}
println!("총 Skip: {}건", skipped_cases.len());
}
let status_path = concat!(env!("CARGO_MANIFEST_DIR"), "/../../test_status.json");
serde_json::to_writer_pretty(File::create(status_path).unwrap(), &file_stats).unwrap();
let mut category_stats: std::collections::BTreeMap<String, (usize, usize)> =
std::collections::BTreeMap::new();
for (key, value) in &file_stats {
let category = key.split('/').next().unwrap_or(key.as_str()).to_string();
let entry = category_stats.entry(category).or_insert((0, 0));
entry.0 += value.0;
entry.1 += value.1;
}
println!("\n파일별 테스트 결과:");
println!("=================");
for (filename, (file_total, file_failed, _, _, _, _, _)) in file_stats {
let success_rate =
((file_total - file_failed) as f64 / file_total as f64 * 100.0) as i32;
let color = if success_rate == 100 {
"\x1b[32m" } else if success_rate == 0 {
"\x1b[31m" } else {
"\x1b[33m" };
println!(
"{}: {}개 중 {}개 성공 (성공률: {}{}%\x1b[0m)",
filename,
file_total,
file_total - file_failed,
color,
success_rate
);
}
println!("\n카테고리별 결과:");
println!("=================");
for (category, (cat_total, cat_failed)) in &category_stats {
println!(
"{}: {}/{} 성공",
category,
cat_total - cat_failed,
cat_total
);
}
println!("\n전체 테스트 결과 요약:");
println!("=================");
println!("총 테스트 케이스: {}", total);
println!("성공: {}", total - failed);
println!("실패: {}", failed);
println!("Skip (limitation): {}", skipped_cases.len());
if failed > 0 {
panic!("{} test cases failed.", failed);
}
}
proptest! {
#[test]
fn test_encode_proptest(s: String) {
let result = encode(&s);
let _encoded = match result {
Ok(encoded) => {
let is_only_nonemitting = s.chars().all(|c| {
c == ' '
|| matches!(
crate::char_struct::CharType::new(c),
Ok(crate::char_struct::CharType::CombiningMark)
)
});
assert!(!encoded.is_empty() || s.is_empty() || is_only_nonemitting);
let unicode_result = encode_to_unicode(&s);
assert!(unicode_result.is_ok());
let unicode_string = unicode_result.unwrap();
assert!(!unicode_string.is_empty() || s.is_empty() || is_only_nonemitting);
encoded
}
Err(_) => {
return Ok(()); }
};
}
}
#[test]
fn test_accuracy_report() {
let files = collect_test_files();
let mut total = 0usize;
let mut passed = 0usize;
let mut per_file: Vec<(String, usize, usize)> = Vec::new();
for (path, filename) in &files {
let content = std::fs::read_to_string(path).unwrap();
let records: Vec<serde_json::Value> = serde_json::from_str(&content).unwrap();
let mut file_total = 0;
let mut file_passed = 0;
for (line_num, record) in records.iter().enumerate() {
let input = record["input"].as_str().unwrap();
let expected_forms = testcase_answer_forms(record, filename, line_num)
.into_iter()
.map(|(_, expected, _)| expected.trim().replace(" ", "⠀"))
.collect::<Vec<_>>();
if expected_forms
.iter()
.any(|expected| expected.chars().any(|c| !c.is_ascii_digit()))
{
continue;
}
total += 1;
file_total += 1;
if let Ok(actual) = encode(input) {
let actual_str = actual.iter().map(|c| c.to_string()).collect::<String>();
if expected_forms.contains(&actual_str) {
passed += 1;
file_passed += 1;
}
}
}
per_file.push((filename.clone(), file_total, file_passed));
}
per_file.sort();
println!("\n═══════════════════════════════════════════════");
println!(" BRAILLIFY ACCURACY REPORT (engine-driven)");
println!("═══════════════════════════════════════════════");
for (name, ft, fp) in &per_file {
let pct = (*fp * 100).checked_div(*ft).unwrap_or(100);
let status = if pct == 100 { "✓" } else { "✗" };
if pct < 100 {
println!(" {} {:20} {:>3}/{:<3} ({:>3}%)", status, name, fp, ft, pct);
}
}
let all_pass: usize = per_file.iter().filter(|(_, t, p)| t == p).count();
let some_fail: usize = per_file.len() - all_pass;
println!("───────────────────────────────────────────────");
println!(
" Files: {} total, {} all-pass, {} with failures",
per_file.len(),
all_pass,
some_fail
);
println!(
" Cases: {}/{} passed ({:.1}%)",
passed,
total,
passed as f64 / total as f64 * 100.0
);
println!("═══════════════════════════════════════════════\n");
}
#[test]
fn test_encoder_streaming() {
let mut encoder = Encoder::new(false); let mut buffer = Vec::new();
encoder.encode("test", &mut buffer).unwrap();
encoder.encode("ing", &mut buffer).unwrap();
let mut expected = encode("test").unwrap();
expected.extend(encode("ing").unwrap());
assert_eq!(buffer, expected);
}
}
#[cfg(test)]
mod coverage_targeted_tests {
use super::*;
use crate::rules::context::EncodingMode;
#[rstest::rstest]
#[case::emphasis(FormattingKind::Emphasis, [32, 36], [36, 4])]
#[case::bold(FormattingKind::Bold, [48, 36], [36, 6])]
#[case::custom1(FormattingKind::Custom1, [16, 36], [36, 2])]
#[case::custom2(FormattingKind::Custom2, [8, 36], [36, 1])]
fn formatting_kind_markers_all_variants(
#[case] kind: FormattingKind,
#[case] start: [u8; 2],
#[case] end: [u8; 2],
) {
assert_eq!(kind.markers(), (start, end));
}
#[test]
fn normalize_math_planck_h() {
assert_eq!(normalize_math_alphanumeric_char('\u{210E}'), 'h');
}
#[rstest::rstest]
#[case::bold_capital_a('\u{1D400}', 'A')]
#[case::bold_small_a('\u{1D41A}', 'a')]
#[case::bold_digit_zero('\u{1D7CE}', '0')]
#[case::passthrough_ascii('Z', 'Z')]
fn normalize_math_alphanumeric_block_mapping(#[case] input: char, #[case] expected: char) {
assert_eq!(normalize_math_alphanumeric_char(input), expected);
}
#[test]
fn normalize_math_alphanumeric_runtime_block_offset() {
let input = std::hint::black_box('\u{1D44F}');
assert_eq!(normalize_math_alphanumeric_char(input), 'b');
}
#[test]
fn normalize_math_alphanumeric_runtime_digit_offset() {
let input = std::hint::black_box('\u{1D7D9}');
assert_eq!(normalize_math_alphanumeric_char(input), '1');
}
#[test]
fn normalize_math_string_no_trigger() {
let result = normalize_math_alphanumeric_string("plain ASCII");
assert!(matches!(result, Cow::Borrowed(_)));
}
#[test]
fn normalize_math_string_with_trigger() {
let result = normalize_math_alphanumeric_string("X = \u{1D400}");
assert!(matches!(result, Cow::Owned(_)));
assert_eq!(result.as_ref(), "X = A");
}
#[test]
fn encode_decomposes_runtime_accented_latin_when_triggered() {
let input = std::hint::black_box("café");
let encoded = encode_with_options(input, &EncodeOptions::default()).unwrap();
assert_eq!(
encoded,
encode_with_options("cafe\u{301}", &EncodeOptions::default()).unwrap()
);
}
#[rstest::rstest]
#[case::plain_subscript("B₆")]
#[case::latex_subscript("$B_6$")]
fn korean_rule68_compact_subscript_default_routes_to_korean(#[case] input: &str) {
let expected = vec![52, 32, 3, 48, 60, 11];
let plain = encode("B₆").unwrap();
let encoded = encode(input).unwrap();
assert_eq!(plain, expected);
assert_eq!(encoded, plain);
}
#[test]
fn negation_combiner_absent_short_circuits() {
let input: Cow<'_, str> = Cow::Borrowed("no combiner here");
let result = move_negation_combiner_before_base(input);
assert_eq!(result.as_ref(), "no combiner here");
}
#[rstest::rstest]
#[case::circle("○", &[56, 52, 7])]
#[case::cross("×", &[56, 45, 7])]
#[case::triangle("△", &[56, 44, 7])]
#[case::square("□", &[56, 54, 7])]
fn encode_object_symbol_mode_each_glyph(#[case] input: &str, #[case] expected: &[u8]) {
let opts = EncodeOptions {
default_mode: Some(EncodingMode::ObjectSymbol),
};
assert_eq!(encode_with_options(input, &opts).unwrap(), expected);
}
#[test]
fn encode_object_symbol_mode_non_matching_falls_through() {
let opts = EncodeOptions {
default_mode: Some(EncodingMode::ObjectSymbol),
};
let result = encode_with_options("A", &opts);
assert!(result.is_ok());
}
#[test]
fn encode_number_mode_roman_uppercase() {
let opts = EncodeOptions {
default_mode: Some(EncodingMode::Number),
};
let single = encode_with_options("I", &opts).unwrap();
assert!(single.starts_with(&[52, 32]));
assert!(single.ends_with(&[50]));
let multi = encode_with_options("IV", &opts).unwrap();
assert_eq!(multi[0], 52);
assert_eq!(multi[1], 32);
assert_eq!(multi[2], 32);
assert_eq!(multi[multi.len() - 1], 50);
}
#[test]
fn encode_number_mode_roman_lowercase() {
let opts = EncodeOptions {
default_mode: Some(EncodingMode::Number),
};
let result = encode_with_options("ix", &opts).unwrap();
assert_eq!(result[0], 52); assert_ne!(result[1], 32); assert_eq!(result[result.len() - 1], 50); }
#[test]
fn encode_number_mode_non_roman_falls_through() {
let opts = EncodeOptions {
default_mode: Some(EncodingMode::Number),
};
let result = encode_with_options("Z", &opts);
assert!(result.is_ok());
}
#[test]
fn encode_math_mode_single_lowercase() {
let opts = EncodeOptions {
default_mode: Some(EncodingMode::Math),
};
let result = encode_with_options("x", &opts).unwrap();
assert_eq!(result[0], 52); assert_eq!(result.len(), 2);
}
#[test]
fn encode_math_mode_single_brackets() {
let opts = EncodeOptions {
default_mode: Some(EncodingMode::Math),
};
assert_eq!(encode_with_options("(", &opts).unwrap(), vec![38]);
assert_eq!(encode_with_options(")", &opts).unwrap(), vec![52]);
assert_eq!(encode_with_options("{", &opts).unwrap(), vec![54]);
assert_eq!(encode_with_options("}", &opts).unwrap(), vec![54]);
assert_eq!(encode_with_options("[", &opts).unwrap(), vec![55, 4]);
assert_eq!(encode_with_options("]", &opts).unwrap(), vec![32, 62]);
}
#[test]
fn encode_math_mode_single_math_symbol() {
let opts = EncodeOptions {
default_mode: Some(EncodingMode::Math),
};
let result = encode_with_options("+", &opts);
assert!(result.is_ok());
}
#[test]
fn encode_math_mode_multichar_strips_spaces() {
let opts = EncodeOptions {
default_mode: Some(EncodingMode::Math),
};
let a = encode_with_options("x = y", &opts).unwrap();
let b = encode_with_options("x=y", &opts).unwrap();
assert_eq!(a, b, "Spaces around '=' must be stripped in math mode");
let c = encode_with_options("a + b", &opts).unwrap();
let d = encode_with_options("a+b", &opts).unwrap();
assert_eq!(c, d);
}
#[test]
fn encode_with_options_explicit_default_mode() {
let opts = EncodeOptions {
default_mode: Some(EncodingMode::English),
};
let result = encode_with_options("hello", &opts);
assert!(result.is_ok());
}
#[test]
fn korean_context_wraps_isolated_roman_section() {
let opts = EncodeOptions {
default_mode: Some(EncodingMode::Korean),
};
let wrapped = encode_with_options("but", &opts).unwrap();
assert_eq!(
wrapped.first(),
Some(&52),
"고립 로마자는 로마자표 ⠴로 시작"
);
assert_eq!(wrapped.last(), Some(&50), "고립 로마자는 종료표 ⠲로 끝남");
let phrase = encode_with_options("Table of Contents", &opts).unwrap();
assert_eq!(phrase.first(), Some(&52));
assert_eq!(phrase.last(), Some(&50));
let unit = encode_with_options("%p", &opts).unwrap();
assert_ne!(unit.last(), Some(&50), "%p(제69항)는 종료표로 감싸지 않음");
}
#[rstest::rstest]
#[case::word("but", true)]
#[case::phrase_with_spaces("Table of Contents", true)]
#[case::percent_unit("%p", false)]
#[case::has_digit("abc123", false)]
#[case::empty("", false)]
#[case::only_space(" ", false)]
fn is_isolated_roman_section_paths(#[case] input: &str, #[case] expected: bool) {
assert_eq!(is_isolated_roman_section(input), expected);
}
#[test]
fn encode_with_formatting_empty_spans_delegates() {
let plain = encode("hello").unwrap();
let formatted = encode_with_formatting("hello", &[]).unwrap();
assert_eq!(plain, formatted);
}
#[test]
fn encode_to_braille_font_basic() {
let result = encode_to_braille_font("a").unwrap();
assert!(!result.is_empty());
for ch in result.chars() {
let cp = ch as u32;
assert!((0x2800..=0x28FF).contains(&cp), "non-braille char {:?}", ch);
}
}
#[test]
fn encode_to_unicode_with_formatting_empty() {
let result = encode_to_unicode_with_formatting("a", &[]).unwrap();
assert!(!result.is_empty());
}
#[rstest::rstest]
#[case::no_markers("plain text", false)]
#[case::brackets_ipa("[əbaut]", true)]
#[case::brackets_without_ipa_then_slashes_ipa("[abc] /əb/", true)]
#[case::slashes_with_ipa("/əb/", true)]
fn detect_ipa_context_variants(#[case] input: &str, #[case] expected: bool) {
assert_eq!(detect_ipa_context(input), expected, "input={input:?}");
}
#[test]
fn detect_ipa_context_slashes_without_ipa() {
let s = "abc // \u{0259} xyz";
let _ = detect_ipa_context(s);
}
#[test]
fn latex_math_comprehensive_sweep() {
let inputs: &[&str] = &[
"1+2",
"x = 1",
"a + b - c",
"x \\times y",
"$x$",
"$x = 1$",
"$x + y$",
"$\\frac{1}{2}$",
"$\\frac{a+b}{c-d}$",
"$x^2$",
"$x^{n+1}$",
"$x_n$",
"$x_{i+1}$",
"$\\sqrt{2}$",
"$\\sqrt[3]{x}$",
"$\\sum_{i=1}^{n} i$",
"$\\int_0^1 f(x) dx$",
"$\\lim_{x \\to 0} f(x)$",
"$f(x) = x^2 + 1$",
"$y \\neq 0$",
"$x \\geq 0$",
"$x \\leq 1$",
"$A \\cup B$",
"$A \\cap B$",
"$A \\subset B$",
"$\\emptyset$",
"$\\forall x$",
"$\\exists y$",
"$\\alpha$",
"$\\beta$",
"$\\pi$",
"$\\theta$",
"$x + $ $y$",
"1 + $x$ = 2",
"$x$ and $y$",
"$\\sin x$",
"$\\cos x$",
"$\\log x$",
"$\\ln x$",
"$\\begin{matrix} 1 & 2 \\\\ 3 & 4 \\end{matrix}$",
"$\\begin{pmatrix} a & b \\\\ c & d \\end{pmatrix}$",
"$\\begin{bmatrix} 1 \\\\ 2 \\end{bmatrix}$",
"$\\begin{array}{cc} x & y \\\\ z & w \\end{array}$",
"수식 $x + 1$ 입니다",
"함수 $f(x)$",
"$a_1$",
"$a_{12}$",
"$x_n y_n$",
"$x^2 + y^2$",
"$2^{10}$",
"$x_i^j$",
"$a^b_c$",
"1+2=3",
"10×5=50",
"x/y",
"1<2",
"3>2",
"x≥0",
"1/2",
"3/4 cup",
"x1/2y",
"$(x+y)$",
"$[a,b]$",
"$\\{x | x > 0\\}$",
"$$",
"$x = ",
];
for input in inputs {
let _ = encode(input);
let _ = encode_to_unicode(input);
}
}
#[test]
fn math_mode_comprehensive_sweep() {
let inputs: &[&str] = &[
"1+2", "x=1", "a+b-c", "x*y", "x/y", "(a+b)", "{c}", "[d]", "x^2", "x_n", "x≥0", "y≤1",
"a≠b", "+", "-", "*", "/", "=", "<", ">", "≠", "≥", "≤", "π", "α", "β", "∞", "∂",
"f(x)", "1 + 2", "x = y",
];
let opts = EncodeOptions {
default_mode: Some(EncodingMode::Math),
};
for input in inputs {
let _ = encode_with_options(input, &opts);
}
}
#[test]
fn formatting_mark_wrap_absorbs_leading_digits() {
let _ = encode("5강\u{0307}");
let _ = encode("1,000원\u{0307}");
let _ = encode("3.14를\u{0307}");
}
#[test]
fn formatting_mark_preserved_when_units_insufficient() {
let _ = encode("한\u{0307}\u{0307}\u{0307}\u{0307}");
let _ = encode("한글\u{0307}\u{0307}\u{0307}\u{0307}\u{0307}");
let _ = encode("\u{0307}\u{0307}");
}
#[test]
fn decompose_accented_latin_not_called_for_plain_input() {
let _ = encode("안녕하세요");
let _ = encode("hello");
}
#[test]
fn decompose_accented_latin_called_for_accented_input() {
let _ = encode(std::hint::black_box("café"));
let _ = encode(std::hint::black_box("piñata"));
let _ = encode(std::hint::black_box("ão"));
}
#[test]
fn decompose_accented_latin_directly_expands_latin_marks() {
assert_eq!(
decompose_accented_latin(Cow::Borrowed("café Å")),
Cow::<str>::Owned("cafe\u{0301} Å".to_string())
);
}
#[test]
fn default_mode_routes_styled_english_and_inline_nemeth_to_ueb() {
assert!(encode("𝐡𝐢𝐬 𝐡𝐞𝐫𝐬 𝐢𝐭𝐬").is_ok());
assert!(encode("solve $x+1$ now").is_ok());
}
#[rstest::rstest]
#[case::bold_zero('\u{1D7CE}', '0')]
#[case::bold_one('\u{1D7CF}', '1')]
#[case::bold_nine('\u{1D7D7}', '9')]
#[case::double_struck_zero('\u{1D7D8}', '0')]
#[case::sans_serif_zero('\u{1D7E2}', '0')]
#[case::sans_serif_bold_zero('\u{1D7EC}', '0')]
#[case::monospace_zero('\u{1D7F6}', '0')]
#[case::monospace_nine('\u{1D7FF}', '9')]
fn normalize_math_alphanumeric_digits(#[case] input: char, #[case] expected: char) {
assert_eq!(
normalize_math_alphanumeric_char(std::hint::black_box(input)),
expected
);
}
#[test]
fn encode_normalizes_math_alphanumeric_digit_blocks() {
assert!(encode(std::hint::black_box("𝟘+𝟙=𝟙")).is_ok());
}
#[rstest::rstest]
#[case::bold_capital_a('\u{1D400}', 'A')]
#[case::bold_lower_a('\u{1D41A}', 'a')]
#[case::italic_lower_h('\u{210E}', 'h')]
fn normalize_math_alphanumeric_letters(#[case] input: char, #[case] expected: char) {
assert_eq!(normalize_math_alphanumeric_char(input), expected);
}
#[test]
fn may_normalize_math_alphanumeric_detects_supported_ranges() {
assert!(may_normalize_math_alphanumeric('\u{210E}'));
assert!(may_normalize_math_alphanumeric('\u{1D400}'));
assert!(!may_normalize_math_alphanumeric('A'));
}
}
#[cfg(test)]
mod debug_reader {
use crate::rules::english_ueb;
#[test]
fn debug_reader() {
for input in ["reader", "READER", "READER'S", "(READER'S DIGEST)"] {
if let Some(result) = english_ueb::try_encode(input) {
let unicode: String = result
.iter()
.map(|c| crate::unicode::encode_unicode(*c))
.collect();
eprintln!("[{}] result: {:?}", input, result);
eprintln!("[{}] unicode: {}", input, unicode);
} else {
eprintln!("[{}] returned None", input);
}
}
}
}