pub fn space_unify(input: &str) -> String {
input
.chars()
.map(|c| match c {
'\u{3000}' | '\t' => ' ',
_ => c,
})
.collect()
}
pub fn space_collapse(input: &str) -> String {
let mut out = String::with_capacity(input.len());
let mut in_run = false;
for c in input.chars() {
if matches!(c, ' ' | '\t' | '\u{3000}') {
if !in_run {
out.push(' ');
in_run = true;
}
} else {
out.push(c);
in_run = false;
}
}
out
}
pub fn newline_unify(input: &str) -> String {
let mut out = String::with_capacity(input.len());
let mut chars = input.chars().peekable();
while let Some(c) = chars.next() {
if c == '\r' {
if chars.peek() == Some(&'\n') {
chars.next();
}
out.push('\n');
} else {
out.push(c);
}
}
out
}
pub fn newline_collapse(input: &str, keep_paragraph: bool) -> String {
let cap = if keep_paragraph { 2 } else { 1 };
let mut out = String::with_capacity(input.len());
let mut run = 0usize;
for c in input.chars() {
if c == '\n' {
run += 1;
if run <= cap {
out.push('\n');
}
} else {
run = 0;
out.push(c);
}
}
out
}
pub fn trim_lines(input: &str) -> String {
let trim_set: &[char] = &[' ', '\t', '\u{3000}'];
let mut out = String::with_capacity(input.len());
let mut first = true;
for line in input.split('\n') {
if !first {
out.push('\n');
}
out.push_str(line.trim_matches(|c| trim_set.contains(&c)));
first = false;
}
out
}
pub(crate) fn is_japanese(c: char) -> bool {
matches!(c,
'\u{3040}'..='\u{309F}' | '\u{30A0}'..='\u{30FF}' | '\u{31F0}'..='\u{31FF}' | '\u{3400}'..='\u{4DBF}' | '\u{4E00}'..='\u{9FFF}' | '\u{F900}'..='\u{FAFF}' | '\u{3000}'..='\u{303F}' | '\u{FF00}'..='\u{FFEF}' )
}
pub fn ja_space_remove(input: &str, keep_latin_boundary: bool) -> String {
let chars: Vec<char> = input.chars().collect();
let mut out = String::with_capacity(input.len());
let mut i = 0;
while i < chars.len() {
let c = chars[i];
if matches!(c, ' ' | '\u{3000}') {
let prev = out.chars().rev().find(|p| !matches!(p, ' ' | '\u{3000}'));
let mut j = i;
while j < chars.len() && matches!(chars[j], ' ' | '\u{3000}') {
j += 1;
}
let next = chars.get(j).copied();
let remove = match (prev, next) {
(Some(p), Some(n)) => {
let next_ja = is_japanese(n) && n != 'ー';
if keep_latin_boundary {
is_japanese(p) && next_ja
} else {
p == '*' || is_japanese(p) || next_ja
}
}
_ => false,
};
if remove {
i = j; continue;
}
out.push(c);
i += 1;
} else {
out.push(c);
i += 1;
}
}
out
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn unify_and_collapse() {
assert_eq!(space_unify("a\u{3000}b\tc"), "a b c");
assert_eq!(space_collapse("a \u{3000}\t b"), "a b");
}
#[test]
fn newlines() {
assert_eq!(newline_unify("a\r\nb\rc\nd"), "a\nb\nc\nd");
assert_eq!(newline_collapse("a\n\n\n\nb", true), "a\n\nb");
assert_eq!(newline_collapse("a\n\n\nb", false), "a\nb");
}
#[test]
fn trim() {
assert_eq!(trim_lines(" a \n\u{3000}b\u{3000}\n\tc"), "a\nb\nc");
}
#[test]
fn ja_space_conservative() {
assert_eq!(ja_space_remove("検索 エンジン", true), "検索エンジン");
assert_eq!(ja_space_remove("Rust 入門", true), "Rust 入門");
assert_eq!(ja_space_remove("Coding the Matrix", true), "Coding the Matrix");
}
#[test]
fn ja_space_aggressive() {
assert_eq!(ja_space_remove("PRML 副読本", false), "PRML副読本");
assert_eq!(ja_space_remove("検索 エンジン", false), "検索エンジン");
assert_eq!(ja_space_remove("Coding the Matrix", false), "Coding the Matrix");
assert_eq!(
ja_space_remove("南アルプスの 天然水 スパークリング", false),
"南アルプスの天然水スパークリング"
);
}
#[test]
fn ja_space_multi_space_run() {
assert_eq!(ja_space_remove("副\u{3000}\u{3000}読\u{3000}本", true), "副読本");
}
#[test]
fn ja_space_neologdn_quirks() {
assert_eq!(ja_space_remove("Sparking* Lemon", false), "Sparking*Lemon");
assert_eq!(ja_space_remove("Sparking* Lemon", true), "Sparking* Lemon");
assert_eq!(ja_space_remove("A ーあ", false), "A ーあ");
assert_eq!(ja_space_remove("ラー メン", false), "ラーメン");
}
}