Skip to main content

textflow/
unicode.rs

1use core::iter::Peekable;
2use core::ops::Range;
3use core::str::CharIndices;
4
5use crate::properties::{is_close_punctuation, is_open_punctuation, is_wide};
6
7#[cfg(feature = "unicode")]
8#[derive(Clone, Copy, Debug, Eq, PartialEq)]
9pub enum Script {
10    Common,
11    Inherited,
12    Latin,
13    Greek,
14    Cyrillic,
15    Hebrew,
16    Arabic,
17    Thai,
18    Devanagari,
19    Han,
20    Hiragana,
21    Katakana,
22    Hangul,
23    Unknown,
24}
25
26#[cfg(feature = "unicode")]
27impl Script {
28    pub fn of(character: char) -> Self {
29        script(character)
30    }
31
32    pub const fn iso15924_tag(self) -> [u8; 4] {
33        match self {
34            Self::Common => *b"Zyyy",
35            Self::Inherited => *b"Zinh",
36            Self::Latin => *b"Latn",
37            Self::Greek => *b"Grek",
38            Self::Cyrillic => *b"Cyrl",
39            Self::Hebrew => *b"Hebr",
40            Self::Arabic => *b"Arab",
41            Self::Thai => *b"Thai",
42            Self::Devanagari => *b"Deva",
43            Self::Han => *b"Hani",
44            Self::Hiragana => *b"Hira",
45            Self::Katakana => *b"Kana",
46            Self::Hangul => *b"Hang",
47            Self::Unknown => *b"Zzzz",
48        }
49    }
50}
51
52#[derive(Clone, Debug, Eq, PartialEq)]
53pub struct Grapheme<'a> {
54    pub text: &'a str,
55    pub range: Range<usize>,
56}
57
58pub struct Graphemes<'a> {
59    text: &'a str,
60    cursor: usize,
61}
62
63pub fn graphemes(text: &str) -> Graphemes<'_> {
64    Graphemes { text, cursor: 0 }
65}
66
67impl<'a> Iterator for Graphemes<'a> {
68    type Item = Grapheme<'a>;
69
70    fn next(&mut self) -> Option<Self::Item> {
71        if self.cursor == self.text.len() {
72            return None;
73        }
74        let start = self.cursor;
75        let mut chars = self.text[start..].char_indices();
76        let (_, first) = chars.next()?;
77        let mut previous = grapheme_class(first);
78        let mut regional_count = usize::from(previous == GraphemeClass::Regional);
79        let mut indic_consonant = devanagari_consonant(first);
80        let mut indic_linked = false;
81        let mut end = start + first.len_utf8();
82
83        for (offset, character) in chars {
84            let next = grapheme_class(character);
85            let indic_conjunct = indic_consonant && indic_linked && devanagari_consonant(character);
86            if grapheme_break(previous, next, regional_count) && !indic_conjunct {
87                break;
88            }
89            end = start + offset + character.len_utf8();
90            if next == GraphemeClass::Regional {
91                regional_count += 1;
92            } else if !matches!(next, GraphemeClass::Extend | GraphemeClass::Zwj) {
93                regional_count = 0;
94            }
95            if character == '\u{094D}' && indic_consonant {
96                indic_linked = true;
97            } else if devanagari_consonant(character) {
98                indic_consonant = true;
99                indic_linked = false;
100            } else if !matches!(next, GraphemeClass::Extend | GraphemeClass::Zwj) {
101                indic_consonant = false;
102                indic_linked = false;
103            }
104            previous = next;
105        }
106
107        self.cursor = end;
108        Some(Grapheme {
109            text: &self.text[start..end],
110            range: start..end,
111        })
112    }
113}
114
115fn devanagari_consonant(character: char) -> bool {
116    matches!(
117        character as u32,
118        0x0915..=0x0939 | 0x0958..=0x095F | 0x0978..=0x097F
119    )
120}
121
122#[derive(Clone, Copy, Debug, Eq, PartialEq)]
123pub enum LineBreakKind {
124    Allowed,
125    Mandatory,
126}
127
128#[derive(Clone, Copy, Debug, Eq, PartialEq)]
129pub struct LineBreak {
130    pub offset: usize,
131    pub kind: LineBreakKind,
132}
133
134/// Supplies additional line breaks at validated grapheme boundaries.
135#[cfg(feature = "shaping")]
136pub trait LineBreakProvider {
137    /// Returns an additional break for `offset`, or `None` to preserve the built-in result.
138    fn break_at(&self, text: &str, offset: usize) -> Option<LineBreakKind>;
139}
140
141pub struct LineBreaks<'a> {
142    text: &'a str,
143    chars: Peekable<CharIndices<'a>>,
144    emitted_end: bool,
145}
146
147pub fn line_breaks(text: &str) -> LineBreaks<'_> {
148    LineBreaks {
149        text,
150        chars: text.char_indices().peekable(),
151        emitted_end: false,
152    }
153}
154
155impl Iterator for LineBreaks<'_> {
156    type Item = LineBreak;
157
158    fn next(&mut self) -> Option<Self::Item> {
159        while let Some((start, character)) = self.chars.next() {
160            let offset = start + character.len_utf8();
161            let next = self.chars.peek().map(|(_, next)| *next);
162            let kind = if character == '\n' || character == '\r' && next != Some('\n') {
163                Some(LineBreakKind::Mandatory)
164            } else if character == '\u{200B}' {
165                Some(LineBreakKind::Allowed)
166            } else if is_space(character) {
167                (!next.is_some_and(is_space)).then_some(LineBreakKind::Allowed)
168            } else if matches!(character, '-' | '/' | '\u{2010}' | '\u{2013}')
169                || next.is_some_and(|next| allows_wide_break(character, next))
170            {
171                Some(LineBreakKind::Allowed)
172            } else {
173                None
174            };
175            if let Some(kind) = kind {
176                if offset == self.text.len() {
177                    self.emitted_end = true;
178                    return Some(LineBreak {
179                        offset,
180                        kind: LineBreakKind::Mandatory,
181                    });
182                }
183                return Some(LineBreak { offset, kind });
184            }
185        }
186        if self.emitted_end {
187            None
188        } else {
189            self.emitted_end = true;
190            Some(LineBreak {
191                offset: self.text.len(),
192                kind: LineBreakKind::Mandatory,
193            })
194        }
195    }
196}
197
198#[cfg(feature = "unicode")]
199#[derive(Clone, Debug, Eq, PartialEq)]
200pub struct ScriptRun {
201    pub text: Range<usize>,
202    pub script: Script,
203}
204
205#[cfg(feature = "unicode")]
206pub fn script_runs(text: &str) -> ScriptRuns<'_> {
207    ScriptRuns {
208        text,
209        graphemes: graphemes(text),
210        pending: None,
211    }
212}
213
214#[cfg(feature = "unicode")]
215pub struct ScriptRuns<'a> {
216    text: &'a str,
217    graphemes: Graphemes<'a>,
218    pending: Option<Grapheme<'a>>,
219}
220
221#[cfg(feature = "unicode")]
222impl Iterator for ScriptRuns<'_> {
223    type Item = ScriptRun;
224
225    fn next(&mut self) -> Option<Self::Item> {
226        let first = self.pending.take().or_else(|| self.graphemes.next())?;
227        let start = first.range.start;
228        let mut run_script = strong_script(first.text);
229
230        for cluster in self.graphemes.by_ref() {
231            let next_script = strong_script(cluster.text);
232            if run_script.is_none() {
233                run_script = next_script;
234            } else if next_script.is_some() && next_script != run_script {
235                self.pending = Some(cluster);
236                break;
237            }
238        }
239
240        let end = self
241            .pending
242            .as_ref()
243            .map_or(self.text.len(), |cluster| cluster.range.start);
244        Some(ScriptRun {
245            text: start..end,
246            script: run_script.unwrap_or(Script::Common),
247        })
248    }
249}
250
251#[derive(Clone, Copy, Eq, PartialEq)]
252enum GraphemeClass {
253    Other,
254    Cr,
255    Lf,
256    Control,
257    Extend,
258    Zwj,
259    Regional,
260    Prepend,
261    SpacingMark,
262    L,
263    V,
264    T,
265    Lv,
266    Lvt,
267    ExtendedPictographic,
268}
269
270fn grapheme_break(previous: GraphemeClass, next: GraphemeClass, regional_count: usize) -> bool {
271    if previous == GraphemeClass::Cr && next == GraphemeClass::Lf {
272        return false;
273    }
274    if matches!(
275        previous,
276        GraphemeClass::Cr | GraphemeClass::Lf | GraphemeClass::Control
277    ) || matches!(
278        next,
279        GraphemeClass::Cr | GraphemeClass::Lf | GraphemeClass::Control
280    ) {
281        return true;
282    }
283    if matches!(
284        (previous, next),
285        (
286            GraphemeClass::L,
287            GraphemeClass::L | GraphemeClass::V | GraphemeClass::Lv | GraphemeClass::Lvt
288        ) | (
289            GraphemeClass::Lv | GraphemeClass::V,
290            GraphemeClass::V | GraphemeClass::T
291        ) | (GraphemeClass::Lvt | GraphemeClass::T, GraphemeClass::T)
292    ) {
293        return false;
294    }
295    if matches!(
296        next,
297        GraphemeClass::Extend | GraphemeClass::Zwj | GraphemeClass::SpacingMark
298    ) || previous == GraphemeClass::Prepend
299        || previous == GraphemeClass::Zwj && next == GraphemeClass::ExtendedPictographic
300    {
301        return false;
302    }
303    !(previous == GraphemeClass::Regional
304        && next == GraphemeClass::Regional
305        && regional_count % 2 == 1)
306}
307
308fn grapheme_class(character: char) -> GraphemeClass {
309    let value = character as u32;
310    match value {
311        0x000D => GraphemeClass::Cr,
312        0x000A => GraphemeClass::Lf,
313        0x0000..=0x001F | 0x007F..=0x009F => GraphemeClass::Control,
314        0x200D => GraphemeClass::Zwj,
315        0x1F1E6..=0x1F1FF => GraphemeClass::Regional,
316        0x0600..=0x0605 | 0x06DD | 0x070F | 0x08E2 => GraphemeClass::Prepend,
317        0x0903 | 0x093B..=0x0940 | 0x0949..=0x094C | 0x0982..=0x0983 => GraphemeClass::SpacingMark,
318        0x1100..=0x115F | 0xA960..=0xA97C => GraphemeClass::L,
319        0x1160..=0x11A7 | 0xD7B0..=0xD7C6 => GraphemeClass::V,
320        0x11A8..=0x11FF | 0xD7CB..=0xD7FB => GraphemeClass::T,
321        0xAC00..=0xD7A3 if (value - 0xAC00) % 28 == 0 => GraphemeClass::Lv,
322        0xAC00..=0xD7A3 => GraphemeClass::Lvt,
323        0x1F000..=0x1FAFF | 0x2600..=0x27BF => GraphemeClass::ExtendedPictographic,
324        _ if is_extend(value) => GraphemeClass::Extend,
325        _ => GraphemeClass::Other,
326    }
327}
328
329fn is_extend(value: u32) -> bool {
330    matches!(
331        value,
332        0x0300..=0x036F
333            | 0x0483..=0x0489
334            | 0x0591..=0x05BD
335            | 0x05BF
336            | 0x05C1..=0x05C2
337            | 0x0610..=0x061A
338            | 0x064B..=0x065F
339            | 0x0670
340            | 0x06D6..=0x06ED
341            | 0x0E31
342            | 0x0E34..=0x0E3A
343            | 0x0E47..=0x0E4E
344            | 0x0900..=0x0902
345            | 0x093A
346            | 0x093C
347            | 0x0941..=0x0948
348            | 0x094D
349            | 0x0951..=0x0957
350            | 0x1AB0..=0x1AFF
351            | 0x1DC0..=0x1DFF
352            | 0x20D0..=0x20FF
353            | 0xFE00..=0xFE0F
354            | 0xFE20..=0xFE2F
355            | 0x1F3FB..=0x1F3FF
356            | 0xE0100..=0xE01EF
357    )
358}
359
360#[cfg(feature = "unicode")]
361fn script(character: char) -> Script {
362    let value = character as u32;
363    match value {
364        _ if is_extend(value) => Script::Inherited,
365        0x0041..=0x005A | 0x0061..=0x007A | 0x00C0..=0x024F | 0x1E00..=0x1EFF => Script::Latin,
366        0x0370..=0x03FF | 0x1F00..=0x1FFF => Script::Greek,
367        0x0400..=0x052F | 0x2DE0..=0x2DFF | 0xA640..=0xA69F => Script::Cyrillic,
368        0x0590..=0x05FF => Script::Hebrew,
369        0x0600..=0x06FF | 0x0750..=0x077F | 0x08A0..=0x08FF | 0xFB50..=0xFDFF | 0xFE70..=0xFEFF => {
370            Script::Arabic
371        }
372        0x0E00..=0x0E7F => Script::Thai,
373        0x0900..=0x097F | 0xA8E0..=0xA8FF => Script::Devanagari,
374        0x3040..=0x309F => Script::Hiragana,
375        0x30A0..=0x30FF | 0x31F0..=0x31FF => Script::Katakana,
376        0x1100..=0x11FF | 0x3130..=0x318F | 0xAC00..=0xD7AF => Script::Hangul,
377        0x3400..=0x4DBF | 0x4E00..=0x9FFF | 0xF900..=0xFAFF | 0x20000..=0x323AF => Script::Han,
378        0x0000..=0x0040
379        | 0x005B..=0x0060
380        | 0x007B..=0x00BF
381        | 0x2000..=0x206F
382        | 0x20A0..=0x20CF
383        | 0x2100..=0x214F
384        | 0x2190..=0x2BFF
385        | 0x1F000..=0x1FAFF => Script::Common,
386        _ => Script::Unknown,
387    }
388}
389
390#[cfg(feature = "unicode")]
391fn strong_script(grapheme: &str) -> Option<Script> {
392    grapheme
393        .chars()
394        .map(script)
395        .find(|script| !matches!(script, Script::Common | Script::Inherited | Script::Unknown))
396}
397
398fn is_space(character: char) -> bool {
399    matches!(
400        character,
401        ' ' | '\t' | '\u{00A0}' | '\u{2000}'..='\u{200A}' | '\u{3000}'
402    )
403}
404
405fn allows_wide_break(previous: char, next: char) -> bool {
406    !is_open_punctuation(previous)
407        && !is_close_punctuation(next)
408        && (is_wide(previous) || is_wide(next))
409}
410
411#[cfg(test)]
412mod tests {
413    use super::*;
414    use std::prelude::v1::*;
415
416    #[test]
417    fn grapheme_ranges_preserve_common_clusters() {
418        let text = "a\u{301}👩‍🚀🇨🇳";
419        let clusters = graphemes(text).collect::<Vec<_>>();
420
421        assert_eq!(clusters.len(), 3);
422        assert_eq!(clusters[0].text, "a\u{301}");
423        assert_eq!(clusters[1].text, "👩‍🚀");
424        assert_eq!(clusters[2].text, "🇨🇳");
425    }
426
427    #[test]
428    fn thai_marks_stay_with_their_base() {
429        let text = "กิ่";
430        let clusters = graphemes(text).collect::<Vec<_>>();
431
432        assert_eq!(clusters.len(), 1);
433        assert_eq!(clusters[0].text, text);
434    }
435
436    #[test]
437    fn devanagari_linkers_keep_conjuncts_in_one_cluster() {
438        let text = "क्षत्रिय";
439        let clusters = graphemes(text).collect::<Vec<_>>();
440
441        assert_eq!(clusters.len(), 3);
442        assert_eq!(clusters[0].text, "क्ष");
443        assert_eq!(clusters[1].text, "त्रि");
444        assert_eq!(clusters[2].text, "य");
445    }
446
447    #[cfg(feature = "unicode")]
448    #[test]
449    fn thai_has_a_stable_script_identity() {
450        assert_eq!(Script::of('ก'), Script::Thai);
451        assert_eq!(Script::Thai.iso15924_tag(), *b"Thai");
452
453        let runs = script_runs("กิ่ ภาษา").collect::<Vec<_>>();
454        assert_eq!(runs.len(), 1);
455        assert_eq!(runs[0].script, Script::Thai);
456    }
457
458    #[test]
459    fn line_breaks_preserve_words_and_cjk_punctuation() {
460        let latin = line_breaks("hello world").collect::<Vec<_>>();
461        let cjk = line_breaks("你好,世界").collect::<Vec<_>>();
462
463        assert!(latin.contains(&LineBreak {
464            offset: 6,
465            kind: LineBreakKind::Allowed,
466        }));
467        assert!(!cjk
468            .iter()
469            .any(|line_break| line_break.offset == "你好".len()));
470        assert_eq!(cjk.last().unwrap().kind, LineBreakKind::Mandatory);
471    }
472
473    #[test]
474    fn zero_width_space_is_an_explicit_break() {
475        let text = "ภาษา\u{200B}ไทย";
476        let breaks = line_breaks(text).collect::<Vec<_>>();
477
478        assert!(breaks.contains(&LineBreak {
479            offset: "ภาษา\u{200B}".len(),
480            kind: LineBreakKind::Allowed,
481        }));
482    }
483
484    #[test]
485    fn line_breaks_keep_bracket_edges_together() {
486        let text = "《文字》〉";
487        let offsets = line_breaks(text)
488            .filter(|line_break| line_break.kind == LineBreakKind::Allowed)
489            .map(|line_break| line_break.offset)
490            .collect::<Vec<_>>();
491
492        assert_eq!(offsets, ["《文".len()]);
493    }
494
495    #[cfg(feature = "unicode")]
496    #[test]
497    fn neutral_prefix_joins_the_first_strong_script() {
498        let text = "(العربية) Latin";
499        let runs = script_runs(text).collect::<Vec<_>>();
500
501        assert_eq!(runs.len(), 2);
502        assert_eq!(runs[0].script, Script::Arabic);
503        assert_eq!(&text[runs[0].text.clone()], "(العربية) ");
504        assert_eq!(runs[1].script, Script::Latin);
505    }
506}