#[derive(PartialEq, Clone, Copy)]
enum DashClass {
Hyphen,
Choonpu,
}
fn dash_class(c: char) -> Option<DashClass> {
match c {
'\u{02D7}' | '\u{058A}' | '\u{2010}' | '\u{2011}' | '\u{2012}' | '\u{2013}' | '\u{2043}' | '\u{207B}' | '\u{208B}' | '\u{2212}' => Some(DashClass::Hyphen),
'\u{FE63}' | '\u{FF0D}' | '\u{FF70}' | '\u{2014}' | '\u{2015}' | '\u{2500}' | '\u{2501}' | '\u{30FC}' => Some(DashClass::Choonpu),
_ => None,
}
}
fn is_kana(c: char) -> bool {
matches!(
c,
'\u{3040}'..='\u{309F}' | '\u{30A0}'..='\u{30FF}' | '\u{31F0}'..='\u{31FF}' | '\u{FF66}'..='\u{FF9F}' )
}
pub fn hyphen_unify(input: &str) -> String {
let chars: Vec<char> = input.chars().collect();
let mut out = String::with_capacity(input.len());
for (i, &c) in chars.iter().enumerate() {
let Some(class) = dash_class(c) else {
out.push(c);
continue;
};
let prev = chars[..i].iter().rev().find(|p| dash_class(**p).is_none());
let kana_context = prev.map(|p| is_kana(*p)).unwrap_or(false);
let target = if kana_context || class == DashClass::Choonpu {
'ー'
} else {
'-'
};
if out.ends_with(target) {
continue;
}
out.push(target);
}
out
}
pub fn wave_dash_unify(input: &str) -> String {
input
.chars()
.map(|c| match c {
'~' | '\u{FF5E}' | '\u{223C}' | '\u{223E}' | '\u{3030}' => '〜',
_ => c,
})
.collect()
}
pub fn tilde_remove(input: &str) -> String {
input
.chars()
.filter(|c| !matches!(c, '~' | '\u{301C}' | '\u{FF5E}' | '\u{223C}' | '\u{223E}' | '\u{3030}'))
.collect()
}
pub fn prolonged_sound_unify(input: &str) -> String {
input
.chars()
.map(|c| match c {
'\u{FF70}' | '\u{02D7}' => 'ー',
_ => c,
})
.collect()
}
pub fn prolonged_sound_collapse(input: &str) -> String {
let mut out = String::with_capacity(input.len());
let mut prev_choonpu = false;
for c in input.chars() {
if c == 'ー' {
if !prev_choonpu {
out.push(c);
}
prev_choonpu = true;
} else {
out.push(c);
prev_choonpu = false;
}
}
out
}
pub fn shorten_repeat(input: &str, threshold: usize, max_repeat_substr_length: usize) -> String {
if threshold == 0 {
return input.to_string();
}
let mut text: Vec<char> = input.chars().collect();
let mut i: usize = 0;
while i < text.len() {
let text_length = text.len();
let mut upper = (text_length - i) / 2;
if max_repeat_substr_length != 0 && max_repeat_substr_length < upper {
upper = max_repeat_substr_length + 1;
}
let mut repeat_length = 1;
while repeat_length < upper {
let sub_end = (i + repeat_length).min(text.len());
let substr: Vec<char> = text[i..sub_end].to_vec();
let mut right_start = i + repeat_length;
let mut right_end = right_start + repeat_length;
let mut num_repeats = 1usize;
loop {
let rs = right_start.min(text.len());
let re = right_end.min(text.len());
if text[rs..re] == substr[..] && right_end <= text_length {
num_repeats += 1;
right_start += repeat_length;
right_end += repeat_length;
} else {
break;
}
}
if num_repeats > threshold {
let keep_end = (i + repeat_length * threshold).min(text.len());
let cut_end = (i + repeat_length * num_repeats).min(text.len());
let mut shortened = text[..keep_end].to_vec();
shortened.extend_from_slice(&text[cut_end..]);
text = shortened;
}
repeat_length += 1;
}
i += 1;
}
text.into_iter().collect()
}
pub fn repeat_suppress(input: &str, threshold: usize) -> String {
shorten_repeat(input, threshold, 8)
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn hyphen_context() {
assert_eq!(hyphen_unify("ス\u{2212}パ\u{2212}"), "スーパー");
assert_eq!(hyphen_unify("03\u{2212}1234"), "03-1234");
assert_eq!(hyphen_unify("2010\u{2013}2020"), "2010-2020");
assert_eq!(hyphen_unify("majika\u{2501}"), "majikaー"); assert_eq!(hyphen_unify("スーパー"), "スーパー");
}
#[test]
fn hyphen_collapse() {
assert_eq!(hyphen_unify("a˗֊‐‑‒–⁃⁻₋−"), "a-");
assert_eq!(hyphen_unify("あ﹣-ー—―─━ー"), "あー");
assert_eq!(hyphen_unify("a-\u{2212}b"), "a-b");
assert_eq!(hyphen_unify("--"), "--");
}
#[test]
fn edge_runs() {
assert_eq!(hyphen_unify("\u{2014}\u{2014}あ"), "ーあ");
assert_eq!(hyphen_unify("\u{2212}\u{2212}"), "-");
}
#[test]
fn wave_and_tilde() {
assert_eq!(wave_dash_unify("あ~い∼う〰え~お"), "あ〜い〜う〜え〜お");
assert_eq!(tilde_remove("あ~い〜う~え"), "あいうえ");
}
#[test]
fn prolonged() {
assert_eq!(prolonged_sound_unify("スーパー"), "スーパー");
assert_eq!(prolonged_sound_collapse("スーーパーーー"), "スーパー");
}
#[test]
fn repeats_single_char() {
assert_eq!(repeat_suppress("すごーーーーい", 1), "すごーい");
assert_eq!(repeat_suppress("わはははは", 2), "わはは");
assert_eq!(repeat_suppress("わははは", 2), "わははは");
assert_eq!(repeat_suppress("abc", 1), "abc");
assert_eq!(repeat_suppress("aaa", 0), "aaa");
}
#[test]
fn repeats_substring() {
assert_eq!(repeat_suppress("無駄無駄無駄無駄ァ", 1), "無駄ァ");
assert_eq!(repeat_suppress("オラオラオラオラーッ", 2), "オラオラーッ");
}
}