Skip to main content

textflow/
unicode.rs

1use core::iter::Peekable;
2use core::ops::Range;
3use core::str::CharIndices;
4
5use crate::properties::{is_close_punctuation, is_open_punctuation, is_wide};
6
7#[cfg(feature = "unicode")]
8#[derive(Clone, Copy, Debug, Eq, PartialEq)]
9pub enum Script {
10    Common,
11    Inherited,
12    Latin,
13    Greek,
14    Cyrillic,
15    Hebrew,
16    Arabic,
17    Thai,
18    Devanagari,
19    Han,
20    Hiragana,
21    Katakana,
22    Hangul,
23    Unknown,
24}
25
26#[cfg(feature = "unicode")]
27impl Script {
28    pub fn of(character: char) -> Self {
29        script(character)
30    }
31
32    pub const fn iso15924_tag(self) -> [u8; 4] {
33        match self {
34            Self::Common => *b"Zyyy",
35            Self::Inherited => *b"Zinh",
36            Self::Latin => *b"Latn",
37            Self::Greek => *b"Grek",
38            Self::Cyrillic => *b"Cyrl",
39            Self::Hebrew => *b"Hebr",
40            Self::Arabic => *b"Arab",
41            Self::Thai => *b"Thai",
42            Self::Devanagari => *b"Deva",
43            Self::Han => *b"Hani",
44            Self::Hiragana => *b"Hira",
45            Self::Katakana => *b"Kana",
46            Self::Hangul => *b"Hang",
47            Self::Unknown => *b"Zzzz",
48        }
49    }
50}
51
52#[derive(Clone, Debug, Eq, PartialEq)]
53pub struct Grapheme<'a> {
54    pub text: &'a str,
55    pub range: Range<usize>,
56}
57
58pub struct Graphemes<'a> {
59    text: &'a str,
60    cursor: usize,
61}
62
63pub fn graphemes(text: &str) -> Graphemes<'_> {
64    Graphemes { text, cursor: 0 }
65}
66
67impl<'a> Iterator for Graphemes<'a> {
68    type Item = Grapheme<'a>;
69
70    fn next(&mut self) -> Option<Self::Item> {
71        if self.cursor == self.text.len() {
72            return None;
73        }
74        let start = self.cursor;
75        let mut chars = self.text[start..].char_indices();
76        let (_, first) = chars.next()?;
77        let mut previous = grapheme_class(first);
78        let mut regional_count = usize::from(previous == GraphemeClass::Regional);
79        let mut indic_consonant = devanagari_consonant(first);
80        let mut indic_linked = false;
81        let mut end = start + first.len_utf8();
82
83        for (offset, character) in chars {
84            let next = grapheme_class(character);
85            let indic_conjunct = indic_consonant && indic_linked && devanagari_consonant(character);
86            if grapheme_break(previous, next, regional_count) && !indic_conjunct {
87                break;
88            }
89            end = start + offset + character.len_utf8();
90            if next == GraphemeClass::Regional {
91                regional_count += 1;
92            } else if !matches!(next, GraphemeClass::Extend | GraphemeClass::Zwj) {
93                regional_count = 0;
94            }
95            if character == '\u{094D}' && indic_consonant {
96                indic_linked = true;
97            } else if devanagari_consonant(character) {
98                indic_consonant = true;
99                indic_linked = false;
100            } else if !matches!(next, GraphemeClass::Extend | GraphemeClass::Zwj) {
101                indic_consonant = false;
102                indic_linked = false;
103            }
104            previous = next;
105        }
106
107        self.cursor = end;
108        Some(Grapheme {
109            text: &self.text[start..end],
110            range: start..end,
111        })
112    }
113}
114
115fn devanagari_consonant(character: char) -> bool {
116    matches!(
117        character as u32,
118        0x0915..=0x0939 | 0x0958..=0x095F | 0x0978..=0x097F
119    )
120}
121
122#[derive(Clone, Copy, Debug, Eq, PartialEq)]
123pub enum LineBreakKind {
124    Allowed,
125    Mandatory,
126}
127
128#[derive(Clone, Copy, Debug, Eq, PartialEq)]
129pub struct LineBreak {
130    pub offset: usize,
131    pub kind: LineBreakKind,
132}
133
134/// Supplies additional line breaks at validated grapheme boundaries.
135#[cfg(feature = "shaping")]
136pub trait LineBreakProvider {
137    /// Returns an additional break for `offset`, or `None` to preserve the built-in result.
138    fn break_at(&self, text: &str, offset: usize) -> Option<LineBreakKind>;
139}
140
141pub struct LineBreaks<'a> {
142    text: &'a str,
143    chars: Peekable<CharIndices<'a>>,
144    emitted_end: bool,
145}
146
147pub fn line_breaks(text: &str) -> LineBreaks<'_> {
148    LineBreaks {
149        text,
150        chars: text.char_indices().peekable(),
151        emitted_end: false,
152    }
153}
154
155pub(crate) fn hard_break_len(text: &str) -> usize {
156    match text.as_bytes() {
157        [b'\r', b'\n', ..] => 2,
158        [b'\r' | b'\n', ..] => 1,
159        _ => 0,
160    }
161}
162
163#[cfg(feature = "shaping")]
164pub(crate) fn is_hard_break(text: &str) -> bool {
165    let len = hard_break_len(text);
166    len != 0 && len == text.len()
167}
168
169impl Iterator for LineBreaks<'_> {
170    type Item = LineBreak;
171
172    fn next(&mut self) -> Option<Self::Item> {
173        while let Some((start, character)) = self.chars.next() {
174            let mut offset = start + character.len_utf8();
175            let next = self.chars.peek().map(|(_, next)| *next);
176            let separator = hard_break_len(&self.text[start..]);
177            let kind = if separator != 0 {
178                if separator == 2 {
179                    self.chars.next();
180                    offset = start + separator;
181                }
182                Some(LineBreakKind::Mandatory)
183            } else if character == '\u{200B}' {
184                Some(LineBreakKind::Allowed)
185            } else if is_space(character) {
186                (!next.is_some_and(is_space)).then_some(LineBreakKind::Allowed)
187            } else if matches!(character, '-' | '/' | '\u{2010}' | '\u{2013}')
188                || next.is_some_and(|next| allows_wide_break(character, next))
189            {
190                Some(LineBreakKind::Allowed)
191            } else {
192                None
193            };
194            if let Some(kind) = kind {
195                if offset == self.text.len() {
196                    self.emitted_end = true;
197                    return Some(LineBreak {
198                        offset,
199                        kind: LineBreakKind::Mandatory,
200                    });
201                }
202                return Some(LineBreak { offset, kind });
203            }
204        }
205        if self.emitted_end {
206            None
207        } else {
208            self.emitted_end = true;
209            Some(LineBreak {
210                offset: self.text.len(),
211                kind: LineBreakKind::Mandatory,
212            })
213        }
214    }
215}
216
217#[cfg(feature = "unicode")]
218#[derive(Clone, Debug, Eq, PartialEq)]
219pub struct ScriptRun {
220    pub text: Range<usize>,
221    pub script: Script,
222}
223
224#[cfg(feature = "unicode")]
225pub fn script_runs(text: &str) -> ScriptRuns<'_> {
226    ScriptRuns {
227        text,
228        graphemes: graphemes(text),
229        pending: None,
230    }
231}
232
233#[cfg(feature = "unicode")]
234pub struct ScriptRuns<'a> {
235    text: &'a str,
236    graphemes: Graphemes<'a>,
237    pending: Option<Grapheme<'a>>,
238}
239
240#[cfg(feature = "unicode")]
241impl Iterator for ScriptRuns<'_> {
242    type Item = ScriptRun;
243
244    fn next(&mut self) -> Option<Self::Item> {
245        let first = self.pending.take().or_else(|| self.graphemes.next())?;
246        let start = first.range.start;
247        let mut run_script = strong_script(first.text);
248
249        for cluster in self.graphemes.by_ref() {
250            let next_script = strong_script(cluster.text);
251            if run_script.is_none() {
252                run_script = next_script;
253            } else if next_script.is_some() && next_script != run_script {
254                self.pending = Some(cluster);
255                break;
256            }
257        }
258
259        let end = self
260            .pending
261            .as_ref()
262            .map_or(self.text.len(), |cluster| cluster.range.start);
263        Some(ScriptRun {
264            text: start..end,
265            script: run_script.unwrap_or(Script::Common),
266        })
267    }
268}
269
270#[derive(Clone, Copy, Eq, PartialEq)]
271enum GraphemeClass {
272    Other,
273    Cr,
274    Lf,
275    Control,
276    Extend,
277    Zwj,
278    Regional,
279    Prepend,
280    SpacingMark,
281    L,
282    V,
283    T,
284    Lv,
285    Lvt,
286    ExtendedPictographic,
287}
288
289fn grapheme_break(previous: GraphemeClass, next: GraphemeClass, regional_count: usize) -> bool {
290    if previous == GraphemeClass::Cr && next == GraphemeClass::Lf {
291        return false;
292    }
293    if matches!(
294        previous,
295        GraphemeClass::Cr | GraphemeClass::Lf | GraphemeClass::Control
296    ) || matches!(
297        next,
298        GraphemeClass::Cr | GraphemeClass::Lf | GraphemeClass::Control
299    ) {
300        return true;
301    }
302    if matches!(
303        (previous, next),
304        (
305            GraphemeClass::L,
306            GraphemeClass::L | GraphemeClass::V | GraphemeClass::Lv | GraphemeClass::Lvt
307        ) | (
308            GraphemeClass::Lv | GraphemeClass::V,
309            GraphemeClass::V | GraphemeClass::T
310        ) | (GraphemeClass::Lvt | GraphemeClass::T, GraphemeClass::T)
311    ) {
312        return false;
313    }
314    if matches!(
315        next,
316        GraphemeClass::Extend | GraphemeClass::Zwj | GraphemeClass::SpacingMark
317    ) || previous == GraphemeClass::Prepend
318        || previous == GraphemeClass::Zwj && next == GraphemeClass::ExtendedPictographic
319    {
320        return false;
321    }
322    !(previous == GraphemeClass::Regional
323        && next == GraphemeClass::Regional
324        && regional_count % 2 == 1)
325}
326
327fn grapheme_class(character: char) -> GraphemeClass {
328    let value = character as u32;
329    match value {
330        0x000D => GraphemeClass::Cr,
331        0x000A => GraphemeClass::Lf,
332        0x0000..=0x001F | 0x007F..=0x009F => GraphemeClass::Control,
333        0x200D => GraphemeClass::Zwj,
334        0x1F1E6..=0x1F1FF => GraphemeClass::Regional,
335        0x0600..=0x0605 | 0x06DD | 0x070F | 0x08E2 => GraphemeClass::Prepend,
336        0x0903 | 0x093B..=0x0940 | 0x0949..=0x094C | 0x0982..=0x0983 => GraphemeClass::SpacingMark,
337        0x1100..=0x115F | 0xA960..=0xA97C => GraphemeClass::L,
338        0x1160..=0x11A7 | 0xD7B0..=0xD7C6 => GraphemeClass::V,
339        0x11A8..=0x11FF | 0xD7CB..=0xD7FB => GraphemeClass::T,
340        0xAC00..=0xD7A3 if (value - 0xAC00) % 28 == 0 => GraphemeClass::Lv,
341        0xAC00..=0xD7A3 => GraphemeClass::Lvt,
342        0x1F000..=0x1FAFF | 0x2600..=0x27BF => GraphemeClass::ExtendedPictographic,
343        _ if is_extend(value) => GraphemeClass::Extend,
344        _ => GraphemeClass::Other,
345    }
346}
347
348fn is_extend(value: u32) -> bool {
349    matches!(
350        value,
351        0x0300..=0x036F
352            | 0x0483..=0x0489
353            | 0x0591..=0x05BD
354            | 0x05BF
355            | 0x05C1..=0x05C2
356            | 0x0610..=0x061A
357            | 0x064B..=0x065F
358            | 0x0670
359            | 0x06D6..=0x06ED
360            | 0x0E31
361            | 0x0E34..=0x0E3A
362            | 0x0E47..=0x0E4E
363            | 0x0900..=0x0902
364            | 0x093A
365            | 0x093C
366            | 0x0941..=0x0948
367            | 0x094D
368            | 0x0951..=0x0957
369            | 0x1AB0..=0x1AFF
370            | 0x1DC0..=0x1DFF
371            | 0x20D0..=0x20FF
372            | 0xFE00..=0xFE0F
373            | 0xFE20..=0xFE2F
374            | 0x1F3FB..=0x1F3FF
375            | 0xE0100..=0xE01EF
376    )
377}
378
379#[cfg(feature = "unicode")]
380fn script(character: char) -> Script {
381    let value = character as u32;
382    match value {
383        _ if is_extend(value) => Script::Inherited,
384        0x0041..=0x005A | 0x0061..=0x007A | 0x00C0..=0x024F | 0x1E00..=0x1EFF => Script::Latin,
385        0x0370..=0x03FF | 0x1F00..=0x1FFF => Script::Greek,
386        0x0400..=0x052F | 0x2DE0..=0x2DFF | 0xA640..=0xA69F => Script::Cyrillic,
387        0x0590..=0x05FF => Script::Hebrew,
388        0x0600..=0x06FF | 0x0750..=0x077F | 0x08A0..=0x08FF | 0xFB50..=0xFDFF | 0xFE70..=0xFEFF => {
389            Script::Arabic
390        }
391        0x0E00..=0x0E7F => Script::Thai,
392        0x0900..=0x097F | 0xA8E0..=0xA8FF => Script::Devanagari,
393        0x3040..=0x309F => Script::Hiragana,
394        0x30A0..=0x30FF | 0x31F0..=0x31FF => Script::Katakana,
395        0x1100..=0x11FF | 0x3130..=0x318F | 0xAC00..=0xD7AF => Script::Hangul,
396        0x3400..=0x4DBF | 0x4E00..=0x9FFF | 0xF900..=0xFAFF | 0x20000..=0x323AF => Script::Han,
397        0x0000..=0x0040
398        | 0x005B..=0x0060
399        | 0x007B..=0x00BF
400        | 0x2000..=0x206F
401        | 0x20A0..=0x20CF
402        | 0x2100..=0x214F
403        | 0x2190..=0x2BFF
404        | 0x1F000..=0x1FAFF => Script::Common,
405        _ => Script::Unknown,
406    }
407}
408
409#[cfg(feature = "unicode")]
410fn strong_script(grapheme: &str) -> Option<Script> {
411    grapheme
412        .chars()
413        .map(script)
414        .find(|script| !matches!(script, Script::Common | Script::Inherited | Script::Unknown))
415}
416
417fn is_space(character: char) -> bool {
418    matches!(
419        character,
420        ' ' | '\t' | '\u{00A0}' | '\u{2000}'..='\u{200A}' | '\u{3000}'
421    )
422}
423
424fn allows_wide_break(previous: char, next: char) -> bool {
425    !is_open_punctuation(previous)
426        && !is_close_punctuation(next)
427        && (is_wide(previous) || is_wide(next))
428}
429
430#[cfg(test)]
431mod tests {
432    use super::*;
433    use std::prelude::v1::*;
434
435    #[test]
436    fn grapheme_ranges_preserve_common_clusters() {
437        let text = "a\u{301}👩‍🚀🇨🇳";
438        let clusters = graphemes(text).collect::<Vec<_>>();
439
440        assert_eq!(clusters.len(), 3);
441        assert_eq!(clusters[0].text, "a\u{301}");
442        assert_eq!(clusters[1].text, "👩‍🚀");
443        assert_eq!(clusters[2].text, "🇨🇳");
444    }
445
446    #[test]
447    fn thai_marks_stay_with_their_base() {
448        let text = "กิ่";
449        let clusters = graphemes(text).collect::<Vec<_>>();
450
451        assert_eq!(clusters.len(), 1);
452        assert_eq!(clusters[0].text, text);
453    }
454
455    #[test]
456    fn devanagari_linkers_keep_conjuncts_in_one_cluster() {
457        let text = "क्षत्रिय";
458        let clusters = graphemes(text).collect::<Vec<_>>();
459
460        assert_eq!(clusters.len(), 3);
461        assert_eq!(clusters[0].text, "क्ष");
462        assert_eq!(clusters[1].text, "त्रि");
463        assert_eq!(clusters[2].text, "य");
464    }
465
466    #[cfg(feature = "unicode")]
467    #[test]
468    fn thai_has_a_stable_script_identity() {
469        assert_eq!(Script::of('ก'), Script::Thai);
470        assert_eq!(Script::Thai.iso15924_tag(), *b"Thai");
471
472        let runs = script_runs("กิ่ ภาษา").collect::<Vec<_>>();
473        assert_eq!(runs.len(), 1);
474        assert_eq!(runs[0].script, Script::Thai);
475    }
476
477    #[test]
478    fn line_breaks_preserve_words_and_cjk_punctuation() {
479        let latin = line_breaks("hello world").collect::<Vec<_>>();
480        let cjk = line_breaks("你好,世界").collect::<Vec<_>>();
481
482        assert!(latin.contains(&LineBreak {
483            offset: 6,
484            kind: LineBreakKind::Allowed,
485        }));
486        assert!(!cjk
487            .iter()
488            .any(|line_break| line_break.offset == "你好".len()));
489        assert_eq!(cjk.last().unwrap().kind, LineBreakKind::Mandatory);
490    }
491
492    #[test]
493    fn zero_width_space_is_an_explicit_break() {
494        let text = "ภาษา\u{200B}ไทย";
495        let breaks = line_breaks(text).collect::<Vec<_>>();
496
497        assert!(breaks.contains(&LineBreak {
498            offset: "ภาษา\u{200B}".len(),
499            kind: LineBreakKind::Allowed,
500        }));
501    }
502
503    #[test]
504    fn line_breaks_keep_bracket_edges_together() {
505        let text = "《文字》〉";
506        let offsets = line_breaks(text)
507            .filter(|line_break| line_break.kind == LineBreakKind::Allowed)
508            .map(|line_break| line_break.offset)
509            .collect::<Vec<_>>();
510
511        assert_eq!(offsets, ["《文".len()]);
512    }
513
514    #[cfg(feature = "unicode")]
515    #[test]
516    fn neutral_prefix_joins_the_first_strong_script() {
517        let text = "(العربية) Latin";
518        let runs = script_runs(text).collect::<Vec<_>>();
519
520        assert_eq!(runs.len(), 2);
521        assert_eq!(runs[0].script, Script::Arabic);
522        assert_eq!(&text[runs[0].text.clone()], "(العربية) ");
523        assert_eq!(runs[1].script, Script::Latin);
524    }
525}