Skip to main content

textflow/
unicode.rs

1use core::iter::Peekable;
2use core::ops::Range;
3use core::str::CharIndices;
4
5use crate::properties::{is_close_punctuation, is_open_punctuation, is_wide};
6
7#[cfg(feature = "unicode")]
8#[derive(Clone, Copy, Debug, Eq, PartialEq)]
9pub enum Script {
10    Common,
11    Inherited,
12    Latin,
13    Greek,
14    Cyrillic,
15    Hebrew,
16    Arabic,
17    Thai,
18    Devanagari,
19    Han,
20    Hiragana,
21    Katakana,
22    Hangul,
23    Unknown,
24}
25
26#[cfg(feature = "unicode")]
27impl Script {
28    pub fn of(character: char) -> Self {
29        script(character)
30    }
31
32    pub const fn iso15924_tag(self) -> [u8; 4] {
33        match self {
34            Self::Common => *b"Zyyy",
35            Self::Inherited => *b"Zinh",
36            Self::Latin => *b"Latn",
37            Self::Greek => *b"Grek",
38            Self::Cyrillic => *b"Cyrl",
39            Self::Hebrew => *b"Hebr",
40            Self::Arabic => *b"Arab",
41            Self::Thai => *b"Thai",
42            Self::Devanagari => *b"Deva",
43            Self::Han => *b"Hani",
44            Self::Hiragana => *b"Hira",
45            Self::Katakana => *b"Kana",
46            Self::Hangul => *b"Hang",
47            Self::Unknown => *b"Zzzz",
48        }
49    }
50}
51
52#[derive(Clone, Debug, Eq, PartialEq)]
53pub struct Grapheme<'a> {
54    pub text: &'a str,
55    pub range: Range<usize>,
56}
57
58pub struct Graphemes<'a> {
59    text: &'a str,
60    cursor: usize,
61}
62
63pub fn graphemes(text: &str) -> Graphemes<'_> {
64    Graphemes { text, cursor: 0 }
65}
66
67impl<'a> Iterator for Graphemes<'a> {
68    type Item = Grapheme<'a>;
69
70    fn next(&mut self) -> Option<Self::Item> {
71        if self.cursor == self.text.len() {
72            return None;
73        }
74        let start = self.cursor;
75        let mut chars = self.text[start..].char_indices();
76        let (_, first) = chars.next()?;
77        let mut previous = grapheme_class(first);
78        let mut regional_count = usize::from(previous == GraphemeClass::Regional);
79        let mut end = start + first.len_utf8();
80
81        for (offset, character) in chars {
82            let next = grapheme_class(character);
83            if grapheme_break(previous, next, regional_count) {
84                break;
85            }
86            end = start + offset + character.len_utf8();
87            if next == GraphemeClass::Regional {
88                regional_count += 1;
89            } else if !matches!(next, GraphemeClass::Extend | GraphemeClass::Zwj) {
90                regional_count = 0;
91            }
92            previous = next;
93        }
94
95        self.cursor = end;
96        Some(Grapheme {
97            text: &self.text[start..end],
98            range: start..end,
99        })
100    }
101}
102
103#[derive(Clone, Copy, Debug, Eq, PartialEq)]
104pub enum LineBreakKind {
105    Allowed,
106    Mandatory,
107}
108
109#[derive(Clone, Copy, Debug, Eq, PartialEq)]
110pub struct LineBreak {
111    pub offset: usize,
112    pub kind: LineBreakKind,
113}
114
115/// Supplies additional line breaks at validated grapheme boundaries.
116#[cfg(feature = "shaping")]
117pub trait LineBreakProvider {
118    /// Returns an additional break for `offset`, or `None` to preserve the built-in result.
119    fn break_at(&self, text: &str, offset: usize) -> Option<LineBreakKind>;
120}
121
122pub struct LineBreaks<'a> {
123    text: &'a str,
124    chars: Peekable<CharIndices<'a>>,
125    emitted_end: bool,
126}
127
128pub fn line_breaks(text: &str) -> LineBreaks<'_> {
129    LineBreaks {
130        text,
131        chars: text.char_indices().peekable(),
132        emitted_end: false,
133    }
134}
135
136impl Iterator for LineBreaks<'_> {
137    type Item = LineBreak;
138
139    fn next(&mut self) -> Option<Self::Item> {
140        while let Some((start, character)) = self.chars.next() {
141            let offset = start + character.len_utf8();
142            let next = self.chars.peek().map(|(_, next)| *next);
143            let kind = if character == '\n' || character == '\r' && next != Some('\n') {
144                Some(LineBreakKind::Mandatory)
145            } else if character == '\u{200B}' {
146                Some(LineBreakKind::Allowed)
147            } else if is_space(character) {
148                (!next.is_some_and(is_space)).then_some(LineBreakKind::Allowed)
149            } else if matches!(character, '-' | '/' | '\u{2010}' | '\u{2013}')
150                || next.is_some_and(|next| allows_wide_break(character, next))
151            {
152                Some(LineBreakKind::Allowed)
153            } else {
154                None
155            };
156            if let Some(kind) = kind {
157                if offset == self.text.len() {
158                    self.emitted_end = true;
159                    return Some(LineBreak {
160                        offset,
161                        kind: LineBreakKind::Mandatory,
162                    });
163                }
164                return Some(LineBreak { offset, kind });
165            }
166        }
167        if self.emitted_end {
168            None
169        } else {
170            self.emitted_end = true;
171            Some(LineBreak {
172                offset: self.text.len(),
173                kind: LineBreakKind::Mandatory,
174            })
175        }
176    }
177}
178
179#[cfg(feature = "unicode")]
180#[derive(Clone, Debug, Eq, PartialEq)]
181pub struct ScriptRun {
182    pub text: Range<usize>,
183    pub script: Script,
184}
185
186#[cfg(feature = "unicode")]
187pub fn script_runs(text: &str) -> ScriptRuns<'_> {
188    ScriptRuns {
189        text,
190        graphemes: graphemes(text),
191        pending: None,
192    }
193}
194
195#[cfg(feature = "unicode")]
196pub struct ScriptRuns<'a> {
197    text: &'a str,
198    graphemes: Graphemes<'a>,
199    pending: Option<Grapheme<'a>>,
200}
201
202#[cfg(feature = "unicode")]
203impl Iterator for ScriptRuns<'_> {
204    type Item = ScriptRun;
205
206    fn next(&mut self) -> Option<Self::Item> {
207        let first = self.pending.take().or_else(|| self.graphemes.next())?;
208        let start = first.range.start;
209        let mut run_script = strong_script(first.text);
210
211        for cluster in self.graphemes.by_ref() {
212            let next_script = strong_script(cluster.text);
213            if run_script.is_none() {
214                run_script = next_script;
215            } else if next_script.is_some() && next_script != run_script {
216                self.pending = Some(cluster);
217                break;
218            }
219        }
220
221        let end = self
222            .pending
223            .as_ref()
224            .map_or(self.text.len(), |cluster| cluster.range.start);
225        Some(ScriptRun {
226            text: start..end,
227            script: run_script.unwrap_or(Script::Common),
228        })
229    }
230}
231
232#[derive(Clone, Copy, Eq, PartialEq)]
233enum GraphemeClass {
234    Other,
235    Cr,
236    Lf,
237    Control,
238    Extend,
239    Zwj,
240    Regional,
241    Prepend,
242    SpacingMark,
243    L,
244    V,
245    T,
246    Lv,
247    Lvt,
248    ExtendedPictographic,
249}
250
251fn grapheme_break(previous: GraphemeClass, next: GraphemeClass, regional_count: usize) -> bool {
252    if previous == GraphemeClass::Cr && next == GraphemeClass::Lf {
253        return false;
254    }
255    if matches!(
256        previous,
257        GraphemeClass::Cr | GraphemeClass::Lf | GraphemeClass::Control
258    ) || matches!(
259        next,
260        GraphemeClass::Cr | GraphemeClass::Lf | GraphemeClass::Control
261    ) {
262        return true;
263    }
264    if matches!(
265        (previous, next),
266        (
267            GraphemeClass::L,
268            GraphemeClass::L | GraphemeClass::V | GraphemeClass::Lv | GraphemeClass::Lvt
269        ) | (
270            GraphemeClass::Lv | GraphemeClass::V,
271            GraphemeClass::V | GraphemeClass::T
272        ) | (GraphemeClass::Lvt | GraphemeClass::T, GraphemeClass::T)
273    ) {
274        return false;
275    }
276    if matches!(
277        next,
278        GraphemeClass::Extend | GraphemeClass::Zwj | GraphemeClass::SpacingMark
279    ) || previous == GraphemeClass::Prepend
280        || previous == GraphemeClass::Zwj && next == GraphemeClass::ExtendedPictographic
281    {
282        return false;
283    }
284    !(previous == GraphemeClass::Regional
285        && next == GraphemeClass::Regional
286        && regional_count % 2 == 1)
287}
288
289fn grapheme_class(character: char) -> GraphemeClass {
290    let value = character as u32;
291    match value {
292        0x000D => GraphemeClass::Cr,
293        0x000A => GraphemeClass::Lf,
294        0x0000..=0x001F | 0x007F..=0x009F => GraphemeClass::Control,
295        0x200D => GraphemeClass::Zwj,
296        0x1F1E6..=0x1F1FF => GraphemeClass::Regional,
297        0x0600..=0x0605 | 0x06DD | 0x070F | 0x08E2 => GraphemeClass::Prepend,
298        0x0903 | 0x093B..=0x0940 | 0x0949..=0x094C | 0x0982..=0x0983 => GraphemeClass::SpacingMark,
299        0x1100..=0x115F | 0xA960..=0xA97C => GraphemeClass::L,
300        0x1160..=0x11A7 | 0xD7B0..=0xD7C6 => GraphemeClass::V,
301        0x11A8..=0x11FF | 0xD7CB..=0xD7FB => GraphemeClass::T,
302        0xAC00..=0xD7A3 if (value - 0xAC00) % 28 == 0 => GraphemeClass::Lv,
303        0xAC00..=0xD7A3 => GraphemeClass::Lvt,
304        0x1F000..=0x1FAFF | 0x2600..=0x27BF => GraphemeClass::ExtendedPictographic,
305        _ if is_extend(value) => GraphemeClass::Extend,
306        _ => GraphemeClass::Other,
307    }
308}
309
310fn is_extend(value: u32) -> bool {
311    matches!(
312        value,
313        0x0300..=0x036F
314            | 0x0483..=0x0489
315            | 0x0591..=0x05BD
316            | 0x05BF
317            | 0x05C1..=0x05C2
318            | 0x0610..=0x061A
319            | 0x064B..=0x065F
320            | 0x0670
321            | 0x06D6..=0x06ED
322            | 0x0E31
323            | 0x0E34..=0x0E3A
324            | 0x0E47..=0x0E4E
325            | 0x0900..=0x0902
326            | 0x093A
327            | 0x093C
328            | 0x0941..=0x0948
329            | 0x094D
330            | 0x0951..=0x0957
331            | 0x1AB0..=0x1AFF
332            | 0x1DC0..=0x1DFF
333            | 0x20D0..=0x20FF
334            | 0xFE00..=0xFE0F
335            | 0xFE20..=0xFE2F
336            | 0x1F3FB..=0x1F3FF
337            | 0xE0100..=0xE01EF
338    )
339}
340
341#[cfg(feature = "unicode")]
342fn script(character: char) -> Script {
343    let value = character as u32;
344    match value {
345        _ if is_extend(value) => Script::Inherited,
346        0x0041..=0x005A | 0x0061..=0x007A | 0x00C0..=0x024F | 0x1E00..=0x1EFF => Script::Latin,
347        0x0370..=0x03FF | 0x1F00..=0x1FFF => Script::Greek,
348        0x0400..=0x052F | 0x2DE0..=0x2DFF | 0xA640..=0xA69F => Script::Cyrillic,
349        0x0590..=0x05FF => Script::Hebrew,
350        0x0600..=0x06FF | 0x0750..=0x077F | 0x08A0..=0x08FF | 0xFB50..=0xFDFF | 0xFE70..=0xFEFF => {
351            Script::Arabic
352        }
353        0x0E00..=0x0E7F => Script::Thai,
354        0x0900..=0x097F | 0xA8E0..=0xA8FF => Script::Devanagari,
355        0x3040..=0x309F => Script::Hiragana,
356        0x30A0..=0x30FF | 0x31F0..=0x31FF => Script::Katakana,
357        0x1100..=0x11FF | 0x3130..=0x318F | 0xAC00..=0xD7AF => Script::Hangul,
358        0x3400..=0x4DBF | 0x4E00..=0x9FFF | 0xF900..=0xFAFF | 0x20000..=0x323AF => Script::Han,
359        0x0000..=0x0040
360        | 0x005B..=0x0060
361        | 0x007B..=0x00BF
362        | 0x2000..=0x206F
363        | 0x20A0..=0x20CF
364        | 0x2100..=0x214F
365        | 0x2190..=0x2BFF
366        | 0x1F000..=0x1FAFF => Script::Common,
367        _ => Script::Unknown,
368    }
369}
370
371#[cfg(feature = "unicode")]
372fn strong_script(grapheme: &str) -> Option<Script> {
373    grapheme
374        .chars()
375        .map(script)
376        .find(|script| !matches!(script, Script::Common | Script::Inherited | Script::Unknown))
377}
378
379fn is_space(character: char) -> bool {
380    matches!(
381        character,
382        ' ' | '\t' | '\u{00A0}' | '\u{2000}'..='\u{200A}' | '\u{3000}'
383    )
384}
385
386fn allows_wide_break(previous: char, next: char) -> bool {
387    !is_open_punctuation(previous)
388        && !is_close_punctuation(next)
389        && (is_wide(previous) || is_wide(next))
390}
391
392#[cfg(test)]
393mod tests {
394    use super::*;
395    use std::prelude::v1::*;
396
397    #[test]
398    fn grapheme_ranges_preserve_common_clusters() {
399        let text = "a\u{301}👩‍🚀🇨🇳";
400        let clusters = graphemes(text).collect::<Vec<_>>();
401
402        assert_eq!(clusters.len(), 3);
403        assert_eq!(clusters[0].text, "a\u{301}");
404        assert_eq!(clusters[1].text, "👩‍🚀");
405        assert_eq!(clusters[2].text, "🇨🇳");
406    }
407
408    #[test]
409    fn thai_marks_stay_with_their_base() {
410        let text = "กิ่";
411        let clusters = graphemes(text).collect::<Vec<_>>();
412
413        assert_eq!(clusters.len(), 1);
414        assert_eq!(clusters[0].text, text);
415    }
416
417    #[cfg(feature = "unicode")]
418    #[test]
419    fn thai_has_a_stable_script_identity() {
420        assert_eq!(Script::of('ก'), Script::Thai);
421        assert_eq!(Script::Thai.iso15924_tag(), *b"Thai");
422
423        let runs = script_runs("กิ่ ภาษา").collect::<Vec<_>>();
424        assert_eq!(runs.len(), 1);
425        assert_eq!(runs[0].script, Script::Thai);
426    }
427
428    #[test]
429    fn line_breaks_preserve_words_and_cjk_punctuation() {
430        let latin = line_breaks("hello world").collect::<Vec<_>>();
431        let cjk = line_breaks("你好,世界").collect::<Vec<_>>();
432
433        assert!(latin.contains(&LineBreak {
434            offset: 6,
435            kind: LineBreakKind::Allowed,
436        }));
437        assert!(!cjk
438            .iter()
439            .any(|line_break| line_break.offset == "你好".len()));
440        assert_eq!(cjk.last().unwrap().kind, LineBreakKind::Mandatory);
441    }
442
443    #[test]
444    fn zero_width_space_is_an_explicit_break() {
445        let text = "ภาษา\u{200B}ไทย";
446        let breaks = line_breaks(text).collect::<Vec<_>>();
447
448        assert!(breaks.contains(&LineBreak {
449            offset: "ภาษา\u{200B}".len(),
450            kind: LineBreakKind::Allowed,
451        }));
452    }
453
454    #[test]
455    fn line_breaks_keep_bracket_edges_together() {
456        let text = "《文字》〉";
457        let offsets = line_breaks(text)
458            .filter(|line_break| line_break.kind == LineBreakKind::Allowed)
459            .map(|line_break| line_break.offset)
460            .collect::<Vec<_>>();
461
462        assert_eq!(offsets, ["《文".len()]);
463    }
464
465    #[cfg(feature = "unicode")]
466    #[test]
467    fn neutral_prefix_joins_the_first_strong_script() {
468        let text = "(العربية) Latin";
469        let runs = script_runs(text).collect::<Vec<_>>();
470
471        assert_eq!(runs.len(), 2);
472        assert_eq!(runs[0].script, Script::Arabic);
473        assert_eq!(&text[runs[0].text.clone()], "(العربية) ");
474        assert_eq!(runs[1].script, Script::Latin);
475    }
476}