use unicode_normalization::UnicodeNormalization;
use unicode_properties::{GeneralCategory, UnicodeGeneralCategory};
const INVALID_RENDERING: &[&str] = &["🕵️", "☝"];
fn in_block(ch: char, lo: u32, hi: u32) -> bool {
let cp = ch as u32;
cp >= lo && cp <= hi
}
fn is_strip_block_char(ch: char) -> bool {
in_block(ch, 0x1F600, 0x1F64F)
|| in_block(ch, 0x1F300, 0x1F5FF)
|| in_block(ch, 0x1F680, 0x1F6FF)
|| in_block(ch, 0x1F700, 0x1F77F)
|| in_block(ch, 0x1F780, 0x1F7FF)
|| in_block(ch, 0x1F800, 0x1F8FF)
|| in_block(ch, 0x1F900, 0x1F9FF)
|| in_block(ch, 0x1FA00, 0x1FA6F)
|| in_block(ch, 0x1FA70, 0x1FAFF)
|| in_block(ch, 0x1F1E0, 0x1F1FF)
|| in_block(ch, 0x2600, 0x26FF)
|| in_block(ch, 0x2700, 0x27BF)
|| in_block(ch, 0xFE00, 0xFE0F)
|| in_block(ch, 0xE0100, 0xE01EF)
|| in_block(ch, 0x1F3FB, 0x1F3FF)
}
fn is_control_char(ch: char) -> bool {
in_block(ch, 0x00, 0x08)
|| in_block(ch, 0x0B, 0x0C)
|| in_block(ch, 0x0E, 0x1F)
|| in_block(ch, 0x7F, 0x9F)
|| in_block(ch, 0x200B, 0x200F)
|| in_block(ch, 0x202A, 0x202E)
|| in_block(ch, 0x2060, 0x206F)
|| ch == '\u{FEFF}'
|| in_block(ch, 0xFFF0, 0xFFF8)
}
fn is_private_char(ch: char) -> bool {
in_block(ch, 0xD800, 0xDFFF)
|| in_block(ch, 0xE000, 0xF8FF)
|| in_block(ch, 0xF900, 0xFAFF)
|| in_block(ch, 0xFE10, 0xFE1F)
|| in_block(ch, 0xFE20, 0xFE2F)
|| in_block(ch, 0xF0000, 0xFFFFF)
|| in_block(ch, 0x100000, 0x10FFFF)
}
fn general_category_group(gc: GeneralCategory) -> char {
use GeneralCategory::*;
match gc {
UppercaseLetter | LowercaseLetter | TitlecaseLetter | ModifierLetter | OtherLetter => 'L',
DecimalNumber | LetterNumber | OtherNumber => 'N',
ConnectorPunctuation | DashPunctuation | OpenPunctuation | ClosePunctuation
| InitialPunctuation | FinalPunctuation | OtherPunctuation => 'P',
MathSymbol | CurrencySymbol | ModifierSymbol | OtherSymbol => 'S',
SpaceSeparator | LineSeparator | ParagraphSeparator => 'Z',
NonspacingMark | EnclosingMark | SpacingMark => 'M',
Control | Format | Surrogate | PrivateUse | Unassigned => 'C',
}
}
pub fn strip_modifiers(text: Option<&str>) -> Option<String> {
let text = text?;
let mut cleaned = text.to_string();
for invalid in INVALID_RENDERING {
cleaned = cleaned.replace(invalid, " ");
}
let mut result = String::with_capacity(cleaned.len());
for ch in cleaned.chars() {
let gc = ch.general_category();
let group = general_category_group(gc);
if matches!(group, 'L' | 'N' | 'P' | 'S') {
result.push(ch);
} else if matches!(
gc,
GeneralCategory::ModifierSymbol
| GeneralCategory::Format
| GeneralCategory::NonspacingMark
| GeneralCategory::EnclosingMark
) || ch == '\u{200D}'
|| ch == '\u{200C}'
{
continue;
} else {
result.push(ch);
}
}
let mut result = strip_variation_selectors(&result);
result = strip_zero_width_joiners(&result);
result = normalize_crlf(&result);
let trimmed = result.trim();
if trimmed.is_empty() {
None
} else {
Some(trimmed.replace('\x00', ""))
}
}
fn strip_variation_selectors(s: &str) -> String {
let mut out = String::with_capacity(s.len());
for ch in s.chars() {
if in_block(ch, 0xFE00, 0xFE0F)
|| in_block(ch, 0xE0100, 0xE01EF)
|| in_block(ch, 0x1F3FB, 0x1F3FF)
{
continue;
}
out.push(ch);
}
out
}
fn strip_zero_width_joiners(s: &str) -> String {
let mut out = String::with_capacity(s.len());
for ch in s.chars() {
if ch == '\u{200D}' || ch == '\u{200C}' {
continue;
}
out.push(ch);
}
out
}
fn normalize_crlf(s: &str) -> String {
let mut out = String::with_capacity(s.len());
let mut chars = s.chars().peekable();
while let Some(ch) = chars.next() {
if ch == '\r' {
if chars.peek() == Some(&'\n') {
chars.next();
}
out.push('\n');
} else {
out.push(ch);
}
}
out
}
pub fn sanitize_name(name: Option<&str>) -> Option<String> {
let name = name?;
let normalized: String = name.nfkc().collect();
let mut result = String::with_capacity(normalized.len());
for ch in normalized.chars() {
let gc = ch.general_category();
let group = general_category_group(gc);
if matches!(group, 'L' | 'N' | 'P') {
result.push(ch);
} else if matches!(
gc,
GeneralCategory::SpaceSeparator
| GeneralCategory::LineSeparator
| GeneralCategory::ParagraphSeparator
) {
result.push(' ');
} else if gc == GeneralCategory::SpacingMark {
result.push(ch);
}
}
result = strip_blocks(&result);
result = strip_control(&result);
result = strip_private(&result);
result = collapse_whitespace(&result);
let trimmed = result.trim();
if trimmed.is_empty() {
None
} else {
Some(trimmed.to_string())
}
}
fn strip_blocks(s: &str) -> String {
let mut out = String::with_capacity(s.len());
for ch in s.chars() {
if is_strip_block_char(ch) {
continue;
}
out.push(ch);
}
out
}
fn strip_control(s: &str) -> String {
let mut out = String::with_capacity(s.len());
for ch in s.chars() {
if is_control_char(ch) {
continue;
}
out.push(ch);
}
out
}
fn strip_private(s: &str) -> String {
let mut out = String::with_capacity(s.len());
for ch in s.chars() {
if is_private_char(ch) {
continue;
}
out.push(ch);
}
out
}
fn collapse_whitespace(s: &str) -> String {
let mut result = String::with_capacity(s.len());
let mut prev_ws = false;
for ch in s.chars() {
if ch.is_whitespace() {
if !prev_ws {
result.push(' ');
prev_ws = true;
}
} else {
result.push(ch);
prev_ws = false;
}
}
result
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn sanitize_normal_name() {
assert_eq!(sanitize_name(Some("Alice")), Some("Alice".to_string()));
assert_eq!(
sanitize_name(Some("Bob Smith")),
Some("Bob Smith".to_string())
);
}
#[test]
fn sanitize_none_returns_none() {
assert_eq!(sanitize_name(None), None);
}
#[test]
fn sanitize_empty_returns_none() {
assert_eq!(sanitize_name(Some("")), None);
assert_eq!(sanitize_name(Some(" ")), None);
}
#[test]
fn sanitize_strips_emoji() {
assert_eq!(
sanitize_name(Some("Hello 🎉 World")),
Some("Hello World".to_string())
);
}
#[test]
fn sanitize_strips_zalgo() {
let zalgo = "Z̸̧a̸l̵g̶o̸";
let result = sanitize_name(Some(zalgo)).unwrap();
assert_eq!(result, "Zalgo");
}
#[test]
fn sanitize_strips_control_chars() {
let with_ctrl = "Hel\x00lo\x1FWorld";
let result = sanitize_name(Some(with_ctrl)).unwrap();
assert_eq!(result, "HelloWorld");
}
#[test]
fn sanitize_cjk_preserved() {
assert_eq!(
sanitize_name(Some("田中太郎")),
Some("田中太郎".to_string())
);
}
#[test]
fn sanitize_nfkc_normalization() {
assert_eq!(sanitize_name(Some("①")), Some("1".to_string()));
assert_eq!(sanitize_name(Some("Ⅰ")), Some("I".to_string()));
assert_eq!(sanitize_name(Some("fi")), Some("fi".to_string()));
}
#[test]
fn sanitize_collapse_whitespace() {
assert_eq!(
sanitize_name(Some("Hello World !")),
Some("Hello World !".to_string())
);
assert_eq!(
sanitize_name(Some(" leading and trailing ")),
Some("leading and trailing".to_string())
);
}
#[test]
fn sanitize_symbols_stripped() {
assert_eq!(sanitize_name(Some("★Hello★")), Some("Hello".to_string()));
assert_eq!(sanitize_name(Some("$100")), Some("100".to_string()));
}
#[test]
fn sanitize_preserves_punctuation() {
assert_eq!(
sanitize_name(Some("Hello-World")),
Some("Hello-World".to_string())
);
assert_eq!(
sanitize_name(Some("test@example")),
Some("test@example".to_string())
);
assert_eq!(sanitize_name(Some("O'Brien")), Some("O'Brien".to_string()));
}
#[test]
fn sanitize_flags_stripped() {
assert_eq!(sanitize_name(Some("🇺🇸")), None);
}
#[test]
fn sanitize_mixed_content() {
let input = "🎉 Hello 🌍 World 🎊";
let result = sanitize_name(Some(input)).unwrap();
assert_eq!(result, "Hello World");
}
#[test]
fn strip_modifiers_none_returns_none() {
assert_eq!(strip_modifiers(None), None);
}
#[test]
fn strip_modifiers_basic() {
assert_eq!(
strip_modifiers(Some("Hello World")),
Some("Hello World".to_string())
);
}
#[test]
fn strip_modifiers_strips_combining() {
let zalgo = "H̸e̸l̸l̸o̸";
let result = strip_modifiers(Some(zalgo)).unwrap();
assert_eq!(result, "Hello");
}
#[test]
fn strip_modifiers_preserves_symbols() {
assert_eq!(strip_modifiers(Some("$100")), Some("$100".to_string()));
assert_eq!(strip_modifiers(Some("★test★")), Some("★test★".to_string()));
}
#[test]
fn strip_modifiers_strips_zwj() {
let input = "a\u{200D}b";
assert_eq!(strip_modifiers(Some(input)), Some("ab".to_string()));
}
#[test]
fn strip_modifiers_null_bytes() {
let input = "Hel\x00lo";
assert_eq!(strip_modifiers(Some(input)), Some("Hello".to_string()));
}
#[test]
fn strip_modifiers_empty_returns_none() {
assert_eq!(strip_modifiers(Some("")), None);
assert_eq!(strip_modifiers(Some(" ")), None);
}
#[test]
fn strip_modifiers_carriage_return() {
assert_eq!(strip_modifiers(Some("a\r\nb")), Some("a\nb".to_string()));
assert_eq!(strip_modifiers(Some("a\rb")), Some("a\nb".to_string()));
}
}