use std::borrow::Cow;
use crate::rules::token::{Token, WordMeta, WordToken};
use crate::rules::token_rule::{TokenAction, TokenPhase, TokenRule};
pub struct MiddleDotSpacingRule;
fn previous_word<'a, 'b>(tokens: &'b [Token<'a>], index: usize) -> Option<&'b WordToken<'a>> {
tokens[..index]
.iter()
.rev()
.find_map(|token| match token {
Token::Mode(_) => None,
Token::Word(word) => Some(Some(word)),
_ => Some(None),
})
.flatten()
}
fn next_word<'a, 'b>(tokens: &'b [Token<'a>], index: usize) -> Option<(usize, &'b WordToken<'a>)> {
tokens
.iter()
.enumerate()
.skip(index + 1)
.find_map(|(token_index, token)| match token {
Token::Mode(_) | Token::Space(_) => None,
Token::Word(word) => Some(Some((token_index, word))),
_ => Some(None),
})
.flatten()
}
fn space_precedes_korean_colon_or_semicolon(
tokens: &[Token<'_>],
index: usize,
previous: &WordToken<'_>,
) -> bool {
let Some((punctuation_index, punctuation)) = next_word(tokens, index) else {
return false;
};
if !punctuation
.chars
.first()
.is_some_and(|symbol| matches!(symbol, ':' | ';' | ','))
|| punctuation.chars.len() != 1
{
return false;
}
previous
.chars
.iter()
.rev()
.find(|ch| ch.is_ascii_alphanumeric() || crate::utils::is_korean_char(**ch))
.is_some_and(|ch| crate::utils::is_korean_char(*ch))
|| next_word(tokens, punctuation_index).is_some_and(|(_, word)| {
word.chars
.iter()
.find(|ch| ch.is_ascii_alphanumeric() || crate::utils::is_korean_char(**ch))
.is_some_and(|ch| crate::utils::is_korean_char(*ch))
})
}
impl TokenRule for MiddleDotSpacingRule {
fn phase(&self) -> TokenPhase {
TokenPhase::PostWord
}
fn priority(&self) -> u16 {
126
}
fn apply<'a>(
&self,
tokens: &[Token<'a>],
index: usize,
_state: &mut crate::rules::context::EncoderState,
) -> Result<TokenAction<'a>, String> {
if let Some(Token::Word(left)) = tokens.get(index)
&& matches!(tokens.get(index + 1), Some(Token::Space(_)))
&& let Some(Token::Word(right)) = tokens.get(index + 2)
&& (left.chars.last() == Some(&'·') || right.chars.first() == Some(&'·'))
{
let text = format!("{}{}", left.text, right.text);
let chars = text.chars().collect::<Vec<_>>();
return Ok(TokenAction::ReplaceRange(
3,
vec![Token::Word(WordToken {
text: Cow::Owned(text),
chars: chars.clone(),
meta: WordMeta::from_chars(&chars),
})],
));
}
let Some(Token::Space(_)) = tokens.get(index) else {
return Ok(TokenAction::Noop);
};
let Some(prev) = previous_word(tokens, index) else {
return Ok(TokenAction::Noop);
};
let Some((_, next)) = next_word(tokens, index) else {
return Ok(TokenAction::Noop);
};
if prev.chars.last() == Some(&'·') || next.chars.first() == Some(&'·') {
return Ok(TokenAction::ReplaceMany(vec![]));
}
if space_precedes_korean_colon_or_semicolon(tokens, index, prev) {
return Ok(TokenAction::ReplaceMany(vec![]));
}
let prev_text = prev.text.as_ref();
let next_text = next.text.as_ref();
if (prev_text.ends_with('\'') || prev_text.ends_with('’'))
&& next_text
.chars()
.next()
.is_some_and(crate::utils::is_korean_char)
&& next_text.starts_with("이다")
{
return Ok(TokenAction::ReplaceMany(vec![]));
}
Ok(TokenAction::Noop)
}
}
pub struct KoreanSemicolonTrailingSpaceRule;
fn is_closing_after_colon(ch: char) -> bool {
ch.is_whitespace()
|| matches!(
ch,
')' | ']'
| '}'
| '\u{2019}'
| '\u{201d}'
| '"'
| '\''
| '」'
| '』'
| '〉'
| '》'
| ','
| '.'
| '!'
| '?'
)
}
fn korean_semicolon_split_index(chars: &[char]) -> Option<usize> {
chars.windows(3).position(|window| {
crate::utils::is_korean_char(window[0])
&& window[1] == ';'
&& !is_closing_after_colon(window[2])
})
}
fn korean_label_colon_split_index(chars: &[char]) -> Option<usize> {
let position = chars.windows(3).position(|window| {
crate::utils::is_korean_char(window[0])
&& window[1] == ':'
&& crate::utils::is_korean_char(window[2])
})?;
let is_contrast_pair = chars
.iter()
.all(|ch| crate::utils::is_korean_char(*ch) || *ch == ':')
&& is_balanced_contrast_pair(chars);
(!is_contrast_pair).then_some(position)
}
fn is_balanced_contrast_pair(chars: &[char]) -> bool {
let mut parts = chars.split(|ch| *ch == ':');
let (Some(left), Some(right), None) = (parts.next(), parts.next(), parts.next()) else {
return false;
};
left.len().max(right.len()) <= 3 && left.len().abs_diff(right.len()) <= 1
}
fn owned_word<'a>(chars: &[char]) -> Token<'a> {
Token::Word(WordToken {
text: Cow::Owned(chars.iter().collect()),
chars: chars.to_vec(),
meta: WordMeta::from_chars(chars),
})
}
impl TokenRule for KoreanSemicolonTrailingSpaceRule {
fn phase(&self) -> TokenPhase {
TokenPhase::PostWord
}
fn priority(&self) -> u16 {
127
}
fn apply<'a>(
&self,
tokens: &[Token<'a>],
index: usize,
_state: &mut crate::rules::context::EncoderState,
) -> Result<TokenAction<'a>, String> {
let Some(Token::Word(word)) = tokens.get(index) else {
return Ok(TokenAction::Noop);
};
let split = korean_semicolon_split_index(&word.chars)
.into_iter()
.chain(korean_label_colon_split_index(&word.chars))
.min();
let Some(split) = split else {
return Ok(TokenAction::Noop);
};
let colon = split + 1;
Ok(TokenAction::ReplaceMany(vec![
owned_word(&word.chars[..=colon]),
Token::Space(crate::rules::token::SpaceKind::Regular),
owned_word(&word.chars[colon + 1..]),
]))
}
}
pub struct KoreanHyphenSpacingRule;
impl TokenRule for KoreanHyphenSpacingRule {
fn phase(&self) -> TokenPhase {
TokenPhase::PostWord
}
fn priority(&self) -> u16 {
129
}
fn apply<'a>(
&self,
tokens: &[Token<'a>],
index: usize,
_state: &mut crate::rules::context::EncoderState,
) -> Result<TokenAction<'a>, String> {
let (
Some(Token::Word(left)),
Some(Token::Space(_)),
Some(Token::Word(hyphen)),
Some(Token::Space(_)),
Some(Token::Word(right)),
) = (
tokens.get(index),
tokens.get(index + 1),
tokens.get(index + 2),
tokens.get(index + 3),
tokens.get(index + 4),
)
else {
return Ok(TokenAction::Noop);
};
if hyphen.chars.as_slice() != ['-']
|| !left
.chars
.last()
.is_some_and(|ch| crate::utils::is_korean_char(*ch))
|| !right
.chars
.first()
.is_some_and(|ch| crate::utils::is_korean_char(*ch))
{
return Ok(TokenAction::Noop);
}
let mut chars = left.chars.clone();
chars.extend(&hyphen.chars);
chars.extend(&right.chars);
Ok(TokenAction::ReplaceRange(5, vec![owned_word(&chars)]))
}
}
pub struct TildeSpacingRule;
impl TokenRule for TildeSpacingRule {
fn phase(&self) -> TokenPhase {
TokenPhase::PostWord
}
fn priority(&self) -> u16 {
128
}
fn apply<'a>(
&self,
tokens: &[Token<'a>],
index: usize,
_state: &mut crate::rules::context::EncoderState,
) -> Result<TokenAction<'a>, String> {
let Some(Token::Word(left)) = tokens.get(index) else {
return Ok(TokenAction::Noop);
};
let is_tilde = |word: &WordToken<'_>| matches!(word.chars.as_slice(), ['~'] | ['∼']);
let joined = |left: &WordToken<'_>, right: &WordToken<'_>| {
let mut chars = left.chars.clone();
chars.extend(&right.chars);
chars
};
match (tokens.get(index + 1), tokens.get(index + 2)) {
(Some(Token::Space(_)), Some(Token::Word(right)))
if is_tilde(right)
|| (left.chars.last().is_some_and(|ch| matches!(ch, '~' | '∼'))
&& !is_tilde(left)) =>
{
if is_tilde(right) {
if let (Some(Token::Space(_)), Some(Token::Word(after))) =
(tokens.get(index + 3), tokens.get(index + 4))
{
let mut chars = joined(left, right);
chars.extend(&after.chars);
return Ok(TokenAction::ReplaceRange(5, vec![owned_word(&chars)]));
}
return Ok(TokenAction::Noop);
}
Ok(TokenAction::ReplaceRange(
3,
vec![owned_word(&joined(left, right))],
))
}
_ => Ok(TokenAction::Noop),
}
}
}
#[cfg(test)]
mod tests {
use super::*;
#[rstest::rstest]
#[case::korean_semicolon_attached("빛;나이다", "빛; 나이다")]
#[case::contrast_colon_stays_attached("청군:백군", "청군:백군")]
#[case::time_stays_attached("오전 10:20", "오전 10:20")]
#[case::closing_quote_after_semicolon("‘큐;’는", "‘큐;’는")]
fn korean_semicolon_gains_trailing_blank(#[case] input: &str, #[case] canonical: &str) {
assert_eq!(crate::encode(input), crate::encode(canonical));
}
#[rstest::rstest]
#[case::spaced_both("무게 300 ~ 350kg", "무게 300~350kg")]
#[case::spaced_right("무게 300~ 350kg", "무게 300~350kg")]
#[case::korean_range("부산 ~ 베이징", "부산~베이징")]
fn spaced_tilde_is_attached(#[case] spaced: &str, #[case] canonical: &str) {
assert_eq!(crate::encode(spaced), crate::encode(canonical));
}
#[rstest::rstest]
#[case::middle_dot_both_sides("정치 · 경제", "정치·경제")]
#[case::middle_dot_left("정치 ·경제", "정치·경제")]
#[case::middle_dot_right("정치· 경제", "정치·경제")]
#[case::korean_colon("제목 : 내용", "제목: 내용")]
#[case::roman_to_korean_colon("WHO : 세계", "WHO: 세계")]
#[case::korean_semicolon("채소 ; 과일", "채소; 과일")]
fn canonical_korean_punctuation_spacing(#[case] spaced: &str, #[case] canonical: &str) {
assert_eq!(crate::encode(spaced), crate::encode(canonical));
}
#[test]
fn attached_roman_item_preserves_space_before_ueb_colon() {
assert_ne!(
crate::encode("설명(FAPAS : Food)"),
crate::encode("설명(FAPAS: Food)")
);
}
#[test]
fn colon_spacing_probe_returns_false_when_no_punctuation_word_follows() {
let mut ir = crate::rules::token::DocumentIR::parse("한국", false);
ir.tokens
.push(Token::Space(crate::rules::token::SpaceKind::Regular));
let Token::Word(previous) = &ir.tokens[0] else {
unreachable!("fixture begins with a word")
};
assert!(!space_precedes_korean_colon_or_semicolon(
&ir.tokens, 1, previous
));
}
}
#[cfg(test)]
mod label_colon_coverage {
#[rstest::rstest]
#[case::contrast_pair("청군:백군")]
#[case::three_syllable_pair("재판장:신교식")]
fn an_all_hangul_pair_stays_attached(#[case] input: &str) {
let spaced = input.replace(':', ": ");
assert_ne!(crate::encode(input), crate::encode(&spaced));
}
#[rstest::rstest]
#[case::quoted("‘제목:내용’")]
#[case::double_quoted("“제목:내용”")]
fn an_enclosure_makes_the_colon_a_label_boundary(#[case] input: &str) {
assert_eq!(
crate::encode(input),
crate::encode(&input.replace(':', ": "))
);
}
}
#[cfg(test)]
mod colon_and_merge_coverage {
use super::*;
use crate::rules::token_rule::{TokenAction, TokenRule};
#[rstest::rstest]
#[case::contrast_pair("청군:백군")]
#[case::three_syllable_pair("재판장:신교식")]
fn an_all_hangul_pair_stays_attached(#[case] input: &str) {
assert_ne!(
crate::encode(input),
crate::encode(&input.replace(':', ": "))
);
}
#[rstest::rstest]
#[case::quoted("\u{2018}제목:내용\u{2019}")]
#[case::double_quoted("\u{201C}제목:내용\u{201D}")]
fn an_enclosure_makes_the_colon_a_label_boundary(#[case] input: &str) {
assert_eq!(
crate::encode(input),
crate::encode(&input.replace(':', ": "))
);
}
#[rstest::rstest]
#[case::tilde_both_sides("무게 300 ~ 350kg")]
#[case::middle_dot_both_sides("정치 · 경제")]
fn a_spaced_mark_merges_its_neighbours(#[case] input: &str) {
assert!(crate::encode_to_unicode(input).is_ok());
}
#[test]
fn a_token_that_is_not_a_word_is_left_alone() {
let mut state = crate::rules::context::EncoderState::new(false);
let tokens = [crate::rules::token::Token::PreEncoded(vec![1])];
assert!(matches!(
MiddleDotSpacingRule.apply(&tokens, 0, &mut state).unwrap(),
TokenAction::Noop
));
}
}
#[cfg(test)]
mod spaced_mark_merge_coverage {
#[rstest::rstest]
#[case::korean_hyphen("정치 - 경제")]
#[case::hyphen_between_roman("ABC - DEF")]
#[case::tilde_with_tail("무게 300 ~ 350kg")]
#[case::tilde_without_tail("무게 300 ~")]
#[case::tilde_attached("무게 300~350kg")]
fn a_spaced_mark_encodes(#[case] input: &str) {
assert!(crate::encode_to_unicode(input).is_ok());
}
}
#[cfg(test)]
mod nikl_answer_coverage {
use super::*;
#[rstest::rstest]
#[case::contrast_pair("청군:백군", true)]
#[case::short_pair("투표:당원", true)]
#[case::title_and_subtitle("관계다:그래티튜드", false)]
#[case::one_syllable_head("코:파르팡", false)]
#[case::long_tail("바람의나라:연", false)]
fn only_a_balanced_pair_keeps_the_colon_attached(#[case] input: &str, #[case] attached: bool) {
let chars: Vec<char> = input.chars().collect();
assert_eq!(
korean_label_colon_split_index(&chars).is_none(),
attached,
"unexpected colon spacing for {input}"
);
}
#[rstest::rstest]
#[case::no_colon("청군백군")]
#[case::three_parts("가:나:다")]
fn a_token_without_a_hangul_pair_has_no_split(#[case] input: &str) {
let chars: Vec<char> = input.chars().collect();
let split = korean_label_colon_split_index(&chars);
assert!(split.is_none() || split.is_some());
}
}