use crate::compat::{MiniToString, String, Vec};
#[derive(Debug, Clone, PartialEq)]
pub struct GraphemeCluster {
pub content: String,
pub char_count: usize,
pub width: f32,
}
pub struct GraphemeProcessor;
impl GraphemeProcessor {
pub fn split_graphemes(text: &str) -> Vec<GraphemeCluster> {
let chars: Vec<char> = text.chars().collect();
let mut clusters: Vec<GraphemeCluster> = Vec::new();
let mut i = 0;
let len = chars.len();
while i < len {
let mut cluster_chars = Vec::new();
cluster_chars.push(chars[i]);
let mut j = i + 1;
while j < len {
let c = chars[j];
if Self::is_combining_mark(c) {
cluster_chars.push(c);
j += 1;
continue;
}
if Self::is_emoji_modifier(c) && Self::is_emoji(chars[j - 1]) {
cluster_chars.push(c);
j += 1;
continue;
}
if c == '\u{200D}' {
cluster_chars.push(c);
j += 1;
if j < len {
cluster_chars.push(chars[j]);
j += 1;
}
continue;
}
if Self::is_variation_selector(c) {
cluster_chars.push(c);
j += 1;
continue;
}
if Self::is_regional_indicator(chars[i])
&& Self::is_regional_indicator(c)
&& cluster_chars.len() == 1
{
cluster_chars.push(c);
j += 1;
break;
}
if c == '\u{20E3}' && cluster_chars.len() == 1 {
cluster_chars.push(c);
j += 1;
break;
}
break;
}
let content: String = cluster_chars.iter().collect();
let char_count = cluster_chars.len();
let width = Self::cluster_display_width(&cluster_chars);
clusters.push(GraphemeCluster { content, char_count, width });
i = j;
}
clusters
}
pub fn grapheme_count(text: &str) -> usize {
Self::split_graphemes(text).len()
}
pub fn truncate_to_graphemes(text: &str, max: usize) -> String {
let clusters = Self::split_graphemes(text);
if clusters.len() <= max {
return text.to_string();
}
clusters.into_iter().take(max).map(|c| c.content).collect()
}
pub fn is_emoji(c: char) -> bool {
let code = c as u32;
matches!(code,
0x1F300..=0x1F5FF
| 0x1F600..=0x1F64F
| 0x1F680..=0x1F6FF
| 0x2600..=0x26FF
| 0x2700..=0x27BF
| 0x1F900..=0x1F9FF
| 0x1FA00..=0x1FA6F
| 0x1FA70..=0x1FAFF
| 0x1F200..=0x1F2FF
| 0x1F000..=0x1F02F
| 0x1F030..=0x1F09F
| 0x1F0A0..=0x1F0FF
| 0x231A..=0x23FF
| 0x24C2..=0x24C2
| 0x25AA..=0x25AB
| 0x25B6..=0x25B6
| 0x25C0..=0x25C0
| 0x25FB..=0x25FE
| 0x2934..=0x2935
| 0x2B05..=0x2B07
| 0x2B1B..=0x2B1C
| 0x2B50..=0x2B50
| 0x2B55..=0x2B55
| 0x3030..=0x3030
| 0x303D..=0x303D
| 0x3297..=0x3297
| 0x3299..=0x3299
| 0x1F1E6..=0x1F1FF
)
}
pub fn is_emoji_modifier(c: char) -> bool {
let code = c as u32;
matches!(code, 0x1F3FB..=0x1F3FF)
}
pub fn is_zwj_sequence(chars: &[char]) -> bool {
if chars.len() < 3 {
return false;
}
let has_zwj = chars.contains(&'\u{200D}');
if !has_zwj {
return false;
}
let segments: Vec<&[char]> = chars.split(|&c| c == '\u{200D}').collect();
if segments.len() < 2 {
return false;
}
segments.iter().all(|seg| !seg.is_empty() && seg.iter().all(|&c| Self::is_emoji(c)))
}
pub fn emoji_width(text: &str) -> u32 {
let chars: Vec<char> = text.chars().collect();
if chars.is_empty() {
return 0;
}
if Self::is_zwj_sequence(&chars) {
return 2;
}
if chars.len() == 2
&& Self::is_regional_indicator(chars[0])
&& Self::is_regional_indicator(chars[1])
{
return 2;
}
if Self::is_emoji(chars[0]) {
return 2;
}
1
}
fn is_combining_mark(c: char) -> bool {
is_combining_mark(c)
}
fn is_variation_selector(c: char) -> bool {
is_variation_selector(c)
}
fn is_regional_indicator(c: char) -> bool {
let code = c as u32;
matches!(code, 0x1F1E6..=0x1F1FF)
}
fn cluster_display_width(chars: &[char]) -> f32 {
if chars.is_empty() {
return 0.0;
}
let first = chars[0];
if Self::is_emoji(first) {
return 2.0; }
if Self::is_regional_indicator(first) && chars.len() >= 2 {
return 2.0;
}
1.0
}
}
pub(crate) fn is_combining_mark(c: char) -> bool {
let code = c as u32;
matches!(code,
0x0300..=0x036F
| 0x1AB0..=0x1AFF
| 0x1DC0..=0x1DFF
| 0xFE20..=0xFE2F
| 0x0901..=0x0903
| 0x093E..=0x094D
| 0x0E31..=0x0E3A
| 0x0E47..=0x0E4E
| 0x0981..=0x0983
| 0x09BE..=0x09CD
| 0x0A01..=0x0A03
| 0x0A3E..=0x0A4D
| 0x0B01..=0x0B03
| 0x0B3E..=0x0B4D
| 0x0F82..=0x0F84
| 0x0F86..=0x0F8B
)
}
pub(crate) fn is_variation_selector(c: char) -> bool {
let code = c as u32;
matches!(code, 0xFE00..=0xFE0F)
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn test_basic_ascii_graphemes() {
let clusters = GraphemeProcessor::split_graphemes("Hello");
assert_eq!(clusters.len(), 5);
for (i, cluster) in clusters.iter().enumerate() {
let expected = ["H", "e", "l", "l", "o"];
assert_eq!(cluster.content, expected[i]);
assert_eq!(cluster.char_count, 1);
}
}
#[test]
fn test_empty_string() {
let clusters = GraphemeProcessor::split_graphemes("");
assert!(clusters.is_empty());
assert_eq!(GraphemeProcessor::grapheme_count(""), 0);
}
#[test]
fn test_grapheme_count() {
assert_eq!(GraphemeProcessor::grapheme_count("Hello"), 5);
assert_eq!(GraphemeProcessor::grapheme_count("世界"), 2);
assert_eq!(GraphemeProcessor::grapheme_count(""), 0);
}
#[test]
fn test_truncate_to_graphemes() {
assert_eq!(GraphemeProcessor::truncate_to_graphemes("Hello World", 5), "Hello");
assert_eq!(GraphemeProcessor::truncate_to_graphemes("Hi", 5), "Hi");
assert_eq!(GraphemeProcessor::truncate_to_graphemes("", 3), "");
}
#[test]
fn test_simple_emoji() {
let clusters = GraphemeProcessor::split_graphemes("a😀b");
assert_eq!(clusters.len(), 3);
assert_eq!(clusters[0].content, "a");
assert_eq!(clusters[1].content, "😀");
assert_eq!(clusters[1].char_count, 1);
assert_eq!(clusters[2].content, "b");
assert!(GraphemeProcessor::is_emoji('😀'));
}
#[test]
fn test_emoji_with_modifier_skin_tone() {
let text = "\u{1F44D}\u{1F3FD}";
let clusters = GraphemeProcessor::split_graphemes(text);
assert_eq!(clusters.len(), 1, "emoji + skin tone should be one cluster");
assert_eq!(clusters[0].char_count, 2);
assert!(GraphemeProcessor::is_emoji('\u{1F44D}'));
assert!(GraphemeProcessor::is_emoji_modifier('\u{1F3FD}'));
}
#[test]
fn test_zwj_sequences() {
let chars: Vec<char> = [
'\u{1F468}',
'\u{200D}',
'\u{1F469}',
'\u{200D}',
'\u{1F467}',
'\u{200D}',
'\u{1F466}',
]
.to_vec();
assert!(
GraphemeProcessor::is_zwj_sequence(&chars),
"family emoji should be a ZWJ sequence"
);
let non_zwj: Vec<char> = "abc".chars().collect();
assert!(!GraphemeProcessor::is_zwj_sequence(&non_zwj));
}
#[test]
fn test_flag_sequences() {
let text = "\u{1F1EB}\u{1F1F7}";
let clusters = GraphemeProcessor::split_graphemes(text);
assert_eq!(clusters.len(), 1, "flag pair should be one cluster");
assert_eq!(clusters[0].char_count, 2);
assert_eq!(GraphemeProcessor::emoji_width(text), 2);
}
#[test]
fn test_emoji_with_zwj_sequence_in_text() {
let text = "Hello\u{1F468}\u{200D}\u{1F469}\u{200D}\u{1F467}\u{200D}\u{1F466}World";
let clusters = GraphemeProcessor::split_graphemes(text);
assert_eq!(clusters.len(), 11);
assert!(clusters[5].content.contains('\u{200D}'));
}
#[test]
fn test_is_emoji() {
assert!(GraphemeProcessor::is_emoji('😀')); assert!(GraphemeProcessor::is_emoji('❤')); assert!(GraphemeProcessor::is_emoji('🚀')); assert!(!GraphemeProcessor::is_emoji('a'));
assert!(!GraphemeProcessor::is_emoji(' '));
}
#[test]
fn test_is_emoji_modifier() {
assert!(GraphemeProcessor::is_emoji_modifier('\u{1F3FB}')); assert!(GraphemeProcessor::is_emoji_modifier('\u{1F3FF}')); assert!(!GraphemeProcessor::is_emoji_modifier('a'));
}
#[test]
fn test_emoji_width() {
assert_eq!(GraphemeProcessor::emoji_width("😀"), 2);
assert_eq!(GraphemeProcessor::emoji_width("a"), 1);
assert_eq!(GraphemeProcessor::emoji_width(""), 0);
}
#[test]
fn test_combining_mark_graphemes() {
let text = "e\u{0301}";
let clusters = GraphemeProcessor::split_graphemes(text);
assert_eq!(clusters.len(), 1, "e + combining accent should be one cluster");
assert_eq!(clusters[0].char_count, 2);
assert_eq!(clusters[0].content.chars().count(), 2);
}
#[test]
fn test_emoji_keycap_sequence() {
let text = "1\u{20E3}";
let clusters = GraphemeProcessor::split_graphemes(text);
assert_eq!(clusters.len(), 1, "digit + keycap should be one cluster");
}
#[test]
fn test_grapheme_truncate_respects_boundary() {
let text = "ab😀cd";
let truncated = GraphemeProcessor::truncate_to_graphemes(text, 3);
assert_eq!(truncated, "ab😀");
}
#[test]
fn test_variation_selector() {
let text = "\u{00A9}\u{FE0F}";
let clusters = GraphemeProcessor::split_graphemes(text);
assert_eq!(clusters.len(), 1, "copyright + VS16 should be one cluster");
}
}