pub static CONFUSABLES: &[(u32, &str)] = &include!("confusables_generated.rs");
pub const CONFUSABLES_UNICODE_VERSION: &str = "18.0.0";
pub const CONFUSABLES_SOURCE_SHA256: &str =
"6ed3ee967c9dfdf6677d563c9985182fbc50a2efb7d6059cd57b2e2ce18f5b92";
pub const CONFUSABLES_ENTRY_COUNT: usize = 6712;
pub fn lookup(c: char) -> Option<&'static str> {
let cp = c as u32;
CONFUSABLES
.binary_search_by_key(&cp, |(code_point, _)| *code_point)
.ok()
.map(|idx| CONFUSABLES[idx].1)
}
pub fn has_confusables(text: &str) -> bool {
text.chars().any(|c| lookup(c).is_some())
}
pub fn find_confusables(text: &str) -> Vec<(char, &'static str)> {
text.chars()
.filter_map(|c| lookup(c).map(|sub| (c, sub)))
.collect()
}
fn expand_substitution(sub: &str) -> Vec<char> {
let mut out = Vec::new();
for part in sub.split_whitespace() {
let hex = match part.strip_prefix("U+") {
Some(hex) => hex,
None => continue,
};
let cp = match u32::from_str_radix(hex, 16) {
Ok(cp) => cp,
Err(_) => continue,
};
if let Some(c) = char::from_u32(cp) {
out.push(c);
}
}
out
}
pub fn internal_skeleton(text: &str) -> String {
use unicode_normalization::UnicodeNormalization;
let mut out = String::with_capacity(text.len());
for c in text.nfd().collect::<String>().chars() {
if crate::text::unicode_properties::is_default_ignorable(c) {
continue;
}
match lookup(c) {
Some(sub) => {
let expanded = expand_substitution(sub);
if expanded.is_empty() {
out.push(c);
} else {
out.extend(expanded);
}
}
None => out.push(c),
}
}
out.nfd().collect()
}
pub fn bidi_skeleton_ltr(text: &str) -> String {
use unicode_bidi::{BidiInfo, Level};
use unicode_normalization::UnicodeNormalization as _;
if !text
.chars()
.any(crate::text::unicode_properties::is_bidi_r_or_al)
{
return internal_skeleton(text);
}
let data = crate::text::unicode_properties::Unicode18BidiData;
let info = BidiInfo::new_with_data_source(&data, text, Some(Level::ltr()));
if info.paragraphs.is_empty() {
return internal_skeleton(text);
}
let mut reordered = String::with_capacity(text.len());
let mut last_end = 0usize;
for para in &info.paragraphs {
if para.range.start > last_end {
reordered.push_str(&text[last_end..para.range.start]);
}
let para_text = &text[para.range.clone()];
let para_chars: Vec<char> = para_text.chars().collect();
if para_chars.is_empty() {
last_end = para.range.end;
continue;
}
let sub_info = BidiInfo::new_with_data_source(&data, para_text, Some(Level::ltr()));
assert_eq!(
sub_info.paragraphs.len(),
1,
"invariant: paragraph substring must yield exactly one paragraph"
);
let sub_para = &sub_info.paragraphs[0];
let levels_per_char: Vec<Level> =
sub_info.reordered_levels_per_char(sub_para, sub_para.range.clone());
assert_eq!(
levels_per_char.len(),
para_chars.len(),
"invariant: paragraph-local levels must match paragraph chars"
);
let index_map = BidiInfo::reorder_visual(&levels_per_char);
let mut visual: Vec<(char, Level)> = Vec::with_capacity(para_chars.len());
for &logical_idx in &index_map {
if let Some(&c) = para_chars.get(logical_idx) {
let lvl = levels_per_char
.get(logical_idx)
.copied()
.unwrap_or_else(Level::ltr);
visual.push((c, lvl));
}
}
let is_mark = |c: char| {
unicode_general_category::get_general_category(c)
.abbreviation()
.starts_with('M')
};
let mut i = 0usize;
while i + 1 < visual.len() {
let (c, _) = visual[i];
let (nxt, _) = visual[i + 1];
if is_mark(c) && !is_mark(nxt) {
visual.swap(i, i + 1);
i = i.saturating_sub(1);
} else {
i += 1;
}
}
for (c, lvl) in visual.iter_mut() {
if lvl.is_rtl() {
if let Some(m) = crate::text::unicode_properties::bidi_mirror(*c) {
*c = m;
}
}
}
for (c, _) in visual {
reordered.push(c);
}
last_end = para.range.end;
}
if last_end < text.len() {
reordered.push_str(&text[last_end..]);
}
let _ = || {
let _: String = "".nfd().collect();
};
internal_skeleton(&reordered)
}
pub fn confusable_skeleton(text: &str) -> String {
bidi_skeleton_ltr(text)
}
pub fn are_confusable(a: &str, b: &str) -> bool {
a != b && confusable_skeleton(a) == confusable_skeleton(b)
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn test_confusables_loaded() {
assert!(CONFUSABLES.len() > 1400, "Confusables should have entries");
}
#[test]
fn test_cyrillic_a_confusable() {
assert_eq!(lookup('А'), Some("U+0041"));
}
#[test]
fn test_sorted_no_duplicates() {
for window in CONFUSABLES.windows(2) {
assert!(
window[0].0 < window[1].0,
"Table must be strictly sorted: 0x{:04X} >= 0x{:04X}",
window[0].0,
window[1].0
);
}
}
#[test]
fn test_entry_count() {
assert_eq!(
CONFUSABLES.len(),
CONFUSABLES_ENTRY_COUNT,
"Confusables table drifted from the pinned epoch count"
);
}
#[test]
fn test_provenance_matches_generated_header() {
let generated = include_str!("confusables_generated.rs");
assert!(
generated.starts_with(&format!(
"// Unicode version: {}",
CONFUSABLES_UNICODE_VERSION
)),
"generated header version must match CONFUSABLES_UNICODE_VERSION"
);
assert!(
generated.contains(CONFUSABLES_SOURCE_SHA256),
"generated header checksum must match CONFUSABLES_SOURCE_SHA256"
);
assert_eq!(CONFUSABLES_UNICODE_VERSION, "18.0.0");
}
#[test]
fn test_skeleton_cyrillic_spoof_collides() {
assert_eq!(confusable_skeleton("apple"), "apple");
assert_eq!(confusable_skeleton("аpple"), "apple");
assert!(are_confusable("apple", "аpple"));
}
#[test]
fn test_skeleton_multi_code_point_target() {
assert_eq!(confusable_skeleton("Æ"), "AE");
assert!(are_confusable("Æ", "AE"));
}
#[test]
fn test_skeleton_shared_component_is_not_collision() {
assert!(!are_confusable("АX", "ΑY"));
assert!(!are_confusable("apple", "orange"));
}
#[test]
fn test_skeleton_identical_inputs_are_not_confusable() {
assert!(!are_confusable("apple", "apple"));
assert!(!are_confusable("", ""));
}
#[test]
fn test_skeleton_deterministic_and_idempotent() {
for input in ["hello", "apple", "аpple", "Æ", "ß", "", "café", "日本語"] {
assert_eq!(confusable_skeleton(input), confusable_skeleton(input));
let once = confusable_skeleton(input);
assert_eq!(
confusable_skeleton(&once),
once,
"skeleton must be idempotent for {input:?}"
);
}
}
#[test]
fn test_representative_substitutions() {
assert_eq!(lookup('"'), Some("U+0027 U+0027"));
assert_eq!(lookup('µ'), Some("U+03BC"));
assert_eq!(lookup('А'), Some("U+0041"));
assert_eq!(lookup('\u{2C09B}'), Some("U+5341"));
assert_eq!(lookup('Æ'), Some("U+0041 U+0045"));
}
}