lexical-ui 0.1.2

A Rust port of Lexical's model and headless packages, at parity with the JS implementation.
Documentation
//! What `findOffset` (`@lexical/react`'s `shared/useCharacterLimit.ts`)
//! gets from `new Intl.Segmenter().segment(text)`: the text's extended
//! grapheme clusters, by UAX #29's rules (GB3 to GB999), on the properties
//! `grapheme_tables.rs` has from the Unicode version of Node's ICU
//! (generated by `rust/scripts/generate-grapheme-tables.mjs`, which checks
//! that `Intl.Segmenter` agrees with every line of GraphemeBreakTest.txt).
//! The default locale's segmenter is ICU's root rules, which are UAX #29's.

use super::grapheme_tables::GRAPHEME_PROPERTIES;

/// `Grapheme_Cluster_Break`, in the order of the generated tables.
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
enum Gcb {
    Other,
    CR,
    LF,
    Control,
    Extend,
    Zwj,
    RegionalIndicator,
    Prepend,
    SpacingMark,
    L,
    V,
    T,
    Lv,
    Lvt,
}

const GCB: [Gcb; 14] = [
    Gcb::Other,
    Gcb::CR,
    Gcb::LF,
    Gcb::Control,
    Gcb::Extend,
    Gcb::Zwj,
    Gcb::RegionalIndicator,
    Gcb::Prepend,
    Gcb::SpacingMark,
    Gcb::L,
    Gcb::V,
    Gcb::T,
    Gcb::Lv,
    Gcb::Lvt,
];

/// `Indic_Conjunct_Break`, in the order of the generated tables.
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
enum InCB {
    None,
    Consonant,
    Extend,
    Linker,
}

const INCB: [InCB; 4] = [InCB::None, InCB::Consonant, InCB::Extend, InCB::Linker];

/// The properties of a code point the rules read.
#[derive(Clone, Copy, Debug)]
struct Props {
    gcb: Gcb,
    extended_pictographic: bool,
    incb: InCB,
}

fn props(c: char) -> Props {
    let c = u32::from(c);
    let bits = match GRAPHEME_PROPERTIES.binary_search_by(|&(lo, hi, _)| {
        if hi < c {
            std::cmp::Ordering::Less
        } else if lo > c {
            std::cmp::Ordering::Greater
        } else {
            std::cmp::Ordering::Equal
        }
    }) {
        Ok(i) => GRAPHEME_PROPERTIES[i].2,
        Err(_) => 0,
    };
    Props {
        gcb: GCB[usize::from(bits & 0xf)],
        extended_pictographic: bits & 0x10 != 0,
        incb: INCB[usize::from((bits >> 5) & 0x3)],
    }
}

/// What the rules that look further back than one code point have seen.
#[derive(Clone, Copy, Debug, Default)]
struct Context {
    /// GB11: the text so far ends with `\p{ExtPict} Extend*` (`Some(false)`)
    /// or `\p{ExtPict} Extend* Zwj` (`Some(true)`).
    emoji: Option<bool>,
    /// GB12 and GB13: how many regional indicators the text ends with.
    regional_indicators: usize,
    /// GB9c: the text ends with `\p{InCB=Consonant} [\p{InCB=Extend}
    /// \p{InCB=Linker}]*` (`Some(false)`), with a linker among them
    /// (`Some(true)`).
    conjunct: Option<bool>,
}

impl Context {
    /// The context once `c` follows.
    fn push(self, c: Props) -> Context {
        Context {
            emoji: if c.extended_pictographic {
                Some(false)
            } else {
                match (self.emoji, c.gcb) {
                    (Some(false), Gcb::Extend) => Some(false),
                    (Some(false), Gcb::Zwj) => Some(true),
                    _ => None,
                }
            },
            regional_indicators: if c.gcb == Gcb::RegionalIndicator {
                self.regional_indicators + 1
            } else {
                0
            },
            conjunct: match (c.incb, self.conjunct) {
                (InCB::Consonant, _) => Some(false),
                (InCB::Linker, Some(_)) => Some(true),
                (InCB::Extend, Some(linked)) => Some(linked),
                _ => None,
            },
        }
    }
}

/// Whether there is a grapheme cluster boundary between `prev` and `next`,
/// where `context` is what the text up to `prev` (included) ends with.
fn is_boundary(prev: Props, next: Props, context: Context) -> bool {
    use Gcb::*;
    match (prev.gcb, next.gcb) {
        // GB3.
        (CR, LF) => false,
        // GB4 and GB5.
        (Control | CR | LF, _) | (_, Control | CR | LF) => true,
        // GB6, GB7 and GB8.
        (L, L | V | Lv | Lvt) | (Lv | V, V | T) | (Lvt | T, T) => false,
        // GB9, GB9a and GB9b.
        (_, Extend | Zwj | SpacingMark) | (Prepend, _) => false,
        // GB9c.
        _ if next.incb == InCB::Consonant && context.conjunct == Some(true) => false,
        // GB11.
        (Zwj, _) if next.extended_pictographic && context.emoji == Some(true) => false,
        // GB12 and GB13.
        (RegionalIndicator, RegionalIndicator) => context.regional_indicators.is_multiple_of(2),
        // GB999.
        _ => true,
    }
}

/// The extended grapheme clusters of `text`, in order: the segments of
/// `new Intl.Segmenter().segment(text)`.
pub(crate) fn graphemes(text: &str) -> Graphemes<'_> {
    Graphemes { rest: text }
}

/// The iterator of [`graphemes`].
pub(crate) struct Graphemes<'a> {
    rest: &'a str,
}

impl<'a> Iterator for Graphemes<'a> {
    type Item = &'a str;

    fn next(&mut self) -> Option<&'a str> {
        let mut chars = self.rest.char_indices();
        let (_, first) = chars.next()?;
        let mut prev = props(first);
        let mut context = Context::default().push(prev);
        let mut end = self.rest.len();
        for (i, c) in chars {
            let next = props(c);
            if is_boundary(prev, next, context) {
                end = i;
                break;
            }
            context = context.push(next);
            prev = next;
        }
        let (segment, rest) = self.rest.split_at(end);
        self.rest = rest;
        Some(segment)
    }
}

#[cfg(test)]
mod tests {
    use super::super::grapheme_tables::GRAPHEME_BREAK_TEST;
    use super::*;

    /// Every line of GraphemeBreakTest.txt, which the generator checked
    /// `Intl.Segmenter` splits the same way.
    #[test]
    fn grapheme_break_test() {
        let mut cases = 0;
        for line in GRAPHEME_BREAK_TEST.lines() {
            let mut text = String::new();
            let mut want = Vec::new();
            for token in line.split(' ') {
                match token {
                    "÷" => want.push(text.len()),
                    "×" => {}
                    hex => {
                        text.push(char::from_u32(u32::from_str_radix(hex, 16).unwrap()).unwrap())
                    }
                }
            }
            let mut got = vec![0];
            for segment in graphemes(&text) {
                got.push(got.last().unwrap() + segment.len());
            }
            assert_eq!(got, want, "{line}");
            cases += 1;
        }
        assert_eq!(cases, 766);
    }

    #[test]
    fn segments() {
        fn split(s: &str) -> Vec<&str> {
            graphemes(s).collect()
        }
        assert_eq!(split(""), Vec::<&str>::new());
        assert_eq!(split("a\r\nb"), ["a", "\r\n", "b"]);
        assert_eq!(
            split("\u{1F1FA}\u{1F1F8}\u{1F1EB}"),
            ["\u{1F1FA}\u{1F1F8}", "\u{1F1EB}"]
        );
        assert_eq!(
            split("\u{0915}\u{094D}\u{0937}\u{093F}x"),
            ["\u{0915}\u{094D}\u{0937}\u{093F}", "x"]
        );
    }
}