use core::iter::once;
use unicode_normalization::char::{
canonical_combining_class, is_combining_mark,
};
use unicode_normalization::{IsNormalized, UnicodeNormalization, is_nfc_quick};
use unicode_security::mixed_script::AugmentedScriptSet;
use unicode_security::skeleton;
use crate::scan::{Hit, located};
pub(super) fn compat(character: char) -> bool {
!once(character).nfkc().eq(once(character).nfc())
}
pub(super) fn lookalike(character: char) -> bool {
let mut buffer = [0_u8; 4];
let text = character.encode_utf8(&mut buffer);
let mut drawn = skeleton(text).peekable();
!character.is_ascii()
&& drawn.peek().is_some()
&& drawn.all(|c| c.is_ascii())
}
pub(super) fn denormal(hits: &[Hit]) -> Vec<Hit> {
let characters = hits.iter().map(|hit| hit.character);
if is_nfc_quick(characters) == IsNormalized::Yes {
return Vec::new();
}
hits.chunk_by(|_, next| !boundary(next.character))
.filter_map(divergence)
.collect()
}
fn boundary(character: char) -> bool {
canonical_combining_class(character) == 0
&& is_nfc_quick(once(character)) == IsNormalized::Yes
}
fn divergence(segment: &[Hit]) -> Option<Hit> {
let original = || segment.iter().map(|hit| hit.character);
let normal: Vec<char> = original().nfc().collect();
let differs = normal
.iter()
.copied()
.zip(original())
.position(|(a, b)| a != b);
let longer = (normal.len() != segment.len()).then_some(normal.len());
let index = differs.or(longer)?;
segment.get(index).or(segment.last()).copied()
}
#[cfg(test)]
pub(crate) fn mixed(hits: &[Hit]) -> Vec<Hit> {
mixed_in(hits.iter().copied())
}
pub(super) fn mixed_in(hits: impl IntoIterator<Item = Hit>) -> Vec<Hit> {
let mut words = Words::default();
hits.into_iter().filter_map(|hit| words.step(hit)).collect()
}
#[derive(Default)]
struct Words {
scripts: AugmentedScriptSet,
pointed: bool,
last: Option<char>,
}
impl Words {
fn step(&mut self, hit: Hit) -> Option<Hit> {
let joins = self.last.is_some_and(in_word) && in_word(hit.character);
self.last = Some(hit.character);
if !joins {
self.scripts = AugmentedScriptSet::default();
self.pointed = false;
}
if self.pointed {
return None;
}
self.scripts
.intersect_with(AugmentedScriptSet::for_char(hit.character));
self.pointed = self.scripts.is_empty();
self.pointed.then_some(hit)
}
}
fn in_word(character: char) -> bool {
character.is_alphanumeric() || is_combining_mark(character)
}
pub(super) fn denormal_in(text: &str) -> Vec<Hit> {
if is_nfc_quick(text.chars()) == IsNormalized::Yes {
return Vec::new();
}
let hits: Vec<Hit> = located(text).collect();
denormal(&hits)
}
#[cfg(test)]
mod tests {
use super::{compat, denormal, lookalike, mixed};
use crate::scan::{Hit, located};
type Finder = fn(&[Hit]) -> Vec<Hit>;
fn at(text: &str, find: Finder) -> Vec<(usize, char)> {
let hits: Vec<_> = located(text).collect();
find(&hits)
.into_iter()
.map(|hit| (hit.position.column, hit.character))
.collect()
}
#[test]
fn compat_is_what_nfkc_folds_and_nfc_keeps() {
for c in ['\u{FF21}', '\u{FB01}', '\u{00B2}', '\u{2460}'] {
assert!(compat(c), "{c:?}");
}
for c in ['a', '\u{00E9}', '\u{0416}', '\u{212B}'] {
assert!(!compat(c), "{c:?}");
}
}
#[test]
fn a_lookalike_is_non_ascii_drawn_as_ascii() {
for c in ['\u{0430}', '\u{03BF}', '\u{0421}'] {
assert!(lookalike(c), "{c:?}");
}
for c in ['a', 'm', '\u{0416}', '\u{00E9}', '\u{4E00}'] {
assert!(!lookalike(c), "{c:?}");
}
}
#[test]
fn nfc_text_is_not_denormal() {
assert_eq!(at("caf\u{00E9} \u{AC00}\n", denormal), vec![]);
}
#[test]
fn denormal_points_where_nfc_first_diverges() {
let decomposed = at("cafe\u{0301} x\u{212B}\n", denormal);
assert_eq!(decomposed, vec![(4, 'e'), (8, '\u{212B}')]);
}
#[test]
fn conjoining_jamo_are_one_segment() {
assert_eq!(at("\u{1100}\u{1161}\n", denormal), vec![(1, '\u{1100}')]);
}
#[test]
fn misordered_marks_diverge_at_the_first_mark() {
let text = "x\u{0301}\u{0323}\n";
assert_eq!(at(text, denormal), vec![(2, '\u{0301}')]);
}
#[test]
fn a_word_in_one_script_is_not_mixed() {
let text = "hello \u{043C}\u{0438}\u{0440} \u{6F22}\u{3042} x2\n";
assert_eq!(at(text, mixed), vec![]);
}
#[test]
fn mixed_points_at_the_first_foreign_letter_once_per_word() {
let text = "p\u{0430}yp\u{0430}l ok \u{0430}b\n";
assert_eq!(at(text, mixed), vec![(2, '\u{0430}'), (12, 'b')]);
}
}
#[cfg(test)]
mod streamed {
use super::{denormal, denormal_in, in_word, mixed_in};
use crate::scan::{Hit, located};
use unicode_security::mixed_script::AugmentedScriptSet;
fn chunked(hits: &[Hit]) -> Vec<Hit> {
hits.chunk_by(|a, b| in_word(a.character) && in_word(b.character))
.filter_map(|word| {
let mut scripts = AugmentedScriptSet::default();
word.iter().copied().find(|hit| {
let one = AugmentedScriptSet::for_char(hit.character);
scripts.intersect_with(one);
scripts.is_empty()
})
})
.collect()
}
const TEXTS: [&str; 6] = [
"p\u{0430}yp\u{0430}l ok \u{0430}b\n",
"\u{0378} a\u{0378}b \u{0378}\u{0378}",
"x\u{0301}\u{0430} 12\u{0430}3 \u{6F22}\u{3042}a",
"\u{043C}\u{0438}\u{0440}hello",
"",
"a\u{0300}\u{0300}\u{0410}\u{0410}a",
];
#[test]
fn the_streamed_word_walk_answers_as_the_chunked_one() {
for text in TEXTS {
let hits: Vec<Hit> = located(text).collect();
assert_eq!(
mixed_in(hits.iter().copied()),
chunked(&hits),
"{text:?}"
);
}
}
#[test]
fn the_whole_text_nfc_walk_answers_as_the_slice_one() {
for text in [
"cafe\u{0301} x\u{212B}\n",
"caf\u{00E9}\n",
"\u{1100}\u{1161}",
] {
let hits: Vec<Hit> = located(text).collect();
assert_eq!(denormal_in(text), denormal(&hits), "{text:?}");
}
}
#[test]
fn no_ascii_text_is_denormal_or_mixed() {
let every: String = (0_u8..=0x7F).map(char::from).collect();
let words: String = every.chars().flat_map(|c| ['a', c, '1']).collect();
for text in [every, words] {
let hits: Vec<Hit> = located(&text).collect();
assert!(denormal(&hits).is_empty());
assert!(chunked(&hits).is_empty());
}
}
}