Skip to main content

textflow/
unicode.rs

1use core::iter::Peekable;
2use core::ops::Range;
3use core::str::CharIndices;
4
5use crate::properties::{is_close_punctuation, is_open_punctuation, is_wide};
6
7#[cfg(feature = "unicode")]
8#[derive(Clone, Copy, Debug, Eq, PartialEq)]
9pub enum Script {
10    Common,
11    Inherited,
12    Latin,
13    Greek,
14    Cyrillic,
15    Hebrew,
16    Arabic,
17    Devanagari,
18    Han,
19    Hiragana,
20    Katakana,
21    Hangul,
22    Unknown,
23}
24
25#[cfg(feature = "unicode")]
26impl Script {
27    pub fn of(character: char) -> Self {
28        script(character)
29    }
30
31    pub const fn iso15924_tag(self) -> [u8; 4] {
32        match self {
33            Self::Common => *b"Zyyy",
34            Self::Inherited => *b"Zinh",
35            Self::Latin => *b"Latn",
36            Self::Greek => *b"Grek",
37            Self::Cyrillic => *b"Cyrl",
38            Self::Hebrew => *b"Hebr",
39            Self::Arabic => *b"Arab",
40            Self::Devanagari => *b"Deva",
41            Self::Han => *b"Hani",
42            Self::Hiragana => *b"Hira",
43            Self::Katakana => *b"Kana",
44            Self::Hangul => *b"Hang",
45            Self::Unknown => *b"Zzzz",
46        }
47    }
48}
49
50#[derive(Clone, Debug, Eq, PartialEq)]
51pub struct Grapheme<'a> {
52    pub text: &'a str,
53    pub range: Range<usize>,
54}
55
56pub struct Graphemes<'a> {
57    text: &'a str,
58    cursor: usize,
59}
60
61pub fn graphemes(text: &str) -> Graphemes<'_> {
62    Graphemes { text, cursor: 0 }
63}
64
65impl<'a> Iterator for Graphemes<'a> {
66    type Item = Grapheme<'a>;
67
68    fn next(&mut self) -> Option<Self::Item> {
69        if self.cursor == self.text.len() {
70            return None;
71        }
72        let start = self.cursor;
73        let mut chars = self.text[start..].char_indices();
74        let (_, first) = chars.next()?;
75        let mut previous = grapheme_class(first);
76        let mut regional_count = usize::from(previous == GraphemeClass::Regional);
77        let mut end = start + first.len_utf8();
78
79        for (offset, character) in chars {
80            let next = grapheme_class(character);
81            if grapheme_break(previous, next, regional_count) {
82                break;
83            }
84            end = start + offset + character.len_utf8();
85            if next == GraphemeClass::Regional {
86                regional_count += 1;
87            } else if !matches!(next, GraphemeClass::Extend | GraphemeClass::Zwj) {
88                regional_count = 0;
89            }
90            previous = next;
91        }
92
93        self.cursor = end;
94        Some(Grapheme {
95            text: &self.text[start..end],
96            range: start..end,
97        })
98    }
99}
100
101#[derive(Clone, Copy, Debug, Eq, PartialEq)]
102pub enum LineBreakKind {
103    Allowed,
104    Mandatory,
105}
106
107#[derive(Clone, Copy, Debug, Eq, PartialEq)]
108pub struct LineBreak {
109    pub offset: usize,
110    pub kind: LineBreakKind,
111}
112
113pub struct LineBreaks<'a> {
114    text: &'a str,
115    chars: Peekable<CharIndices<'a>>,
116    emitted_end: bool,
117}
118
119pub fn line_breaks(text: &str) -> LineBreaks<'_> {
120    LineBreaks {
121        text,
122        chars: text.char_indices().peekable(),
123        emitted_end: false,
124    }
125}
126
127impl Iterator for LineBreaks<'_> {
128    type Item = LineBreak;
129
130    fn next(&mut self) -> Option<Self::Item> {
131        while let Some((start, character)) = self.chars.next() {
132            let offset = start + character.len_utf8();
133            let next = self.chars.peek().map(|(_, next)| *next);
134            let kind = if character == '\n' || character == '\r' && next != Some('\n') {
135                Some(LineBreakKind::Mandatory)
136            } else if is_space(character) {
137                (!next.is_some_and(is_space)).then_some(LineBreakKind::Allowed)
138            } else if matches!(character, '-' | '/' | '\u{2010}' | '\u{2013}')
139                || next.is_some_and(|next| allows_wide_break(character, next))
140            {
141                Some(LineBreakKind::Allowed)
142            } else {
143                None
144            };
145            if let Some(kind) = kind {
146                if offset == self.text.len() {
147                    self.emitted_end = true;
148                    return Some(LineBreak {
149                        offset,
150                        kind: LineBreakKind::Mandatory,
151                    });
152                }
153                return Some(LineBreak { offset, kind });
154            }
155        }
156        if self.emitted_end {
157            None
158        } else {
159            self.emitted_end = true;
160            Some(LineBreak {
161                offset: self.text.len(),
162                kind: LineBreakKind::Mandatory,
163            })
164        }
165    }
166}
167
168#[cfg(feature = "unicode")]
169#[derive(Clone, Debug, Eq, PartialEq)]
170pub struct ScriptRun {
171    pub text: Range<usize>,
172    pub script: Script,
173}
174
175#[cfg(feature = "unicode")]
176pub fn script_runs(text: &str) -> ScriptRuns<'_> {
177    ScriptRuns {
178        text,
179        graphemes: graphemes(text),
180        pending: None,
181    }
182}
183
184#[cfg(feature = "unicode")]
185pub struct ScriptRuns<'a> {
186    text: &'a str,
187    graphemes: Graphemes<'a>,
188    pending: Option<Grapheme<'a>>,
189}
190
191#[cfg(feature = "unicode")]
192impl Iterator for ScriptRuns<'_> {
193    type Item = ScriptRun;
194
195    fn next(&mut self) -> Option<Self::Item> {
196        let first = self.pending.take().or_else(|| self.graphemes.next())?;
197        let start = first.range.start;
198        let mut run_script = strong_script(first.text);
199
200        for cluster in self.graphemes.by_ref() {
201            let next_script = strong_script(cluster.text);
202            if run_script.is_none() {
203                run_script = next_script;
204            } else if next_script.is_some() && next_script != run_script {
205                self.pending = Some(cluster);
206                break;
207            }
208        }
209
210        let end = self
211            .pending
212            .as_ref()
213            .map_or(self.text.len(), |cluster| cluster.range.start);
214        Some(ScriptRun {
215            text: start..end,
216            script: run_script.unwrap_or(Script::Common),
217        })
218    }
219}
220
221#[derive(Clone, Copy, Eq, PartialEq)]
222enum GraphemeClass {
223    Other,
224    Cr,
225    Lf,
226    Control,
227    Extend,
228    Zwj,
229    Regional,
230    Prepend,
231    SpacingMark,
232    L,
233    V,
234    T,
235    Lv,
236    Lvt,
237    ExtendedPictographic,
238}
239
240fn grapheme_break(previous: GraphemeClass, next: GraphemeClass, regional_count: usize) -> bool {
241    if previous == GraphemeClass::Cr && next == GraphemeClass::Lf {
242        return false;
243    }
244    if matches!(
245        previous,
246        GraphemeClass::Cr | GraphemeClass::Lf | GraphemeClass::Control
247    ) || matches!(
248        next,
249        GraphemeClass::Cr | GraphemeClass::Lf | GraphemeClass::Control
250    ) {
251        return true;
252    }
253    if matches!(
254        (previous, next),
255        (
256            GraphemeClass::L,
257            GraphemeClass::L | GraphemeClass::V | GraphemeClass::Lv | GraphemeClass::Lvt
258        ) | (
259            GraphemeClass::Lv | GraphemeClass::V,
260            GraphemeClass::V | GraphemeClass::T
261        ) | (GraphemeClass::Lvt | GraphemeClass::T, GraphemeClass::T)
262    ) {
263        return false;
264    }
265    if matches!(
266        next,
267        GraphemeClass::Extend | GraphemeClass::Zwj | GraphemeClass::SpacingMark
268    ) || previous == GraphemeClass::Prepend
269        || previous == GraphemeClass::Zwj && next == GraphemeClass::ExtendedPictographic
270    {
271        return false;
272    }
273    !(previous == GraphemeClass::Regional
274        && next == GraphemeClass::Regional
275        && regional_count % 2 == 1)
276}
277
278fn grapheme_class(character: char) -> GraphemeClass {
279    let value = character as u32;
280    match value {
281        0x000D => GraphemeClass::Cr,
282        0x000A => GraphemeClass::Lf,
283        0x0000..=0x001F | 0x007F..=0x009F => GraphemeClass::Control,
284        0x200D => GraphemeClass::Zwj,
285        0x1F1E6..=0x1F1FF => GraphemeClass::Regional,
286        0x0600..=0x0605 | 0x06DD | 0x070F | 0x08E2 => GraphemeClass::Prepend,
287        0x0903 | 0x093B..=0x0940 | 0x0949..=0x094C | 0x0982..=0x0983 => GraphemeClass::SpacingMark,
288        0x1100..=0x115F | 0xA960..=0xA97C => GraphemeClass::L,
289        0x1160..=0x11A7 | 0xD7B0..=0xD7C6 => GraphemeClass::V,
290        0x11A8..=0x11FF | 0xD7CB..=0xD7FB => GraphemeClass::T,
291        0xAC00..=0xD7A3 if (value - 0xAC00) % 28 == 0 => GraphemeClass::Lv,
292        0xAC00..=0xD7A3 => GraphemeClass::Lvt,
293        0x1F000..=0x1FAFF | 0x2600..=0x27BF => GraphemeClass::ExtendedPictographic,
294        _ if is_extend(value) => GraphemeClass::Extend,
295        _ => GraphemeClass::Other,
296    }
297}
298
299fn is_extend(value: u32) -> bool {
300    matches!(
301        value,
302        0x0300..=0x036F
303            | 0x0483..=0x0489
304            | 0x0591..=0x05BD
305            | 0x05BF
306            | 0x05C1..=0x05C2
307            | 0x0610..=0x061A
308            | 0x064B..=0x065F
309            | 0x0670
310            | 0x06D6..=0x06ED
311            | 0x0900..=0x0902
312            | 0x093A
313            | 0x093C
314            | 0x0941..=0x0948
315            | 0x094D
316            | 0x0951..=0x0957
317            | 0x1AB0..=0x1AFF
318            | 0x1DC0..=0x1DFF
319            | 0x20D0..=0x20FF
320            | 0xFE00..=0xFE0F
321            | 0xFE20..=0xFE2F
322            | 0x1F3FB..=0x1F3FF
323            | 0xE0100..=0xE01EF
324    )
325}
326
327#[cfg(feature = "unicode")]
328fn script(character: char) -> Script {
329    let value = character as u32;
330    match value {
331        _ if is_extend(value) => Script::Inherited,
332        0x0041..=0x005A | 0x0061..=0x007A | 0x00C0..=0x024F | 0x1E00..=0x1EFF => Script::Latin,
333        0x0370..=0x03FF | 0x1F00..=0x1FFF => Script::Greek,
334        0x0400..=0x052F | 0x2DE0..=0x2DFF | 0xA640..=0xA69F => Script::Cyrillic,
335        0x0590..=0x05FF => Script::Hebrew,
336        0x0600..=0x06FF | 0x0750..=0x077F | 0x08A0..=0x08FF | 0xFB50..=0xFDFF | 0xFE70..=0xFEFF => {
337            Script::Arabic
338        }
339        0x0900..=0x097F | 0xA8E0..=0xA8FF => Script::Devanagari,
340        0x3040..=0x309F => Script::Hiragana,
341        0x30A0..=0x30FF | 0x31F0..=0x31FF => Script::Katakana,
342        0x1100..=0x11FF | 0x3130..=0x318F | 0xAC00..=0xD7AF => Script::Hangul,
343        0x3400..=0x4DBF | 0x4E00..=0x9FFF | 0xF900..=0xFAFF | 0x20000..=0x323AF => Script::Han,
344        0x0000..=0x0040
345        | 0x005B..=0x0060
346        | 0x007B..=0x00BF
347        | 0x2000..=0x206F
348        | 0x20A0..=0x20CF
349        | 0x2100..=0x214F
350        | 0x2190..=0x2BFF
351        | 0x1F000..=0x1FAFF => Script::Common,
352        _ => Script::Unknown,
353    }
354}
355
356#[cfg(feature = "unicode")]
357fn strong_script(grapheme: &str) -> Option<Script> {
358    grapheme
359        .chars()
360        .map(script)
361        .find(|script| !matches!(script, Script::Common | Script::Inherited | Script::Unknown))
362}
363
364fn is_space(character: char) -> bool {
365    matches!(
366        character,
367        ' ' | '\t' | '\u{00A0}' | '\u{2000}'..='\u{200A}' | '\u{3000}'
368    )
369}
370
371fn allows_wide_break(previous: char, next: char) -> bool {
372    !is_open_punctuation(previous)
373        && !is_close_punctuation(next)
374        && (is_wide(previous) || is_wide(next))
375}
376
377#[cfg(test)]
378mod tests {
379    use super::*;
380    use std::prelude::v1::*;
381
382    #[test]
383    fn grapheme_ranges_preserve_common_clusters() {
384        let text = "a\u{301}πŸ‘©β€πŸš€πŸ‡¨πŸ‡³";
385        let clusters = graphemes(text).collect::<Vec<_>>();
386
387        assert_eq!(clusters.len(), 3);
388        assert_eq!(clusters[0].text, "a\u{301}");
389        assert_eq!(clusters[1].text, "πŸ‘©β€πŸš€");
390        assert_eq!(clusters[2].text, "πŸ‡¨πŸ‡³");
391    }
392
393    #[test]
394    fn line_breaks_preserve_words_and_cjk_punctuation() {
395        let latin = line_breaks("hello world").collect::<Vec<_>>();
396        let cjk = line_breaks("δ½ ε₯½οΌŒδΈ–η•Œ").collect::<Vec<_>>();
397
398        assert!(latin.contains(&LineBreak {
399            offset: 6,
400            kind: LineBreakKind::Allowed,
401        }));
402        assert!(!cjk
403            .iter()
404            .any(|line_break| line_break.offset == "δ½ ε₯½".len()));
405        assert_eq!(cjk.last().unwrap().kind, LineBreakKind::Mandatory);
406    }
407
408    #[test]
409    fn line_breaks_keep_bracket_edges_together() {
410        let text = "γ€Šζ–‡ε­—γ€‹γ€‰";
411        let offsets = line_breaks(text)
412            .filter(|line_break| line_break.kind == LineBreakKind::Allowed)
413            .map(|line_break| line_break.offset)
414            .collect::<Vec<_>>();
415
416        assert_eq!(offsets, ["γ€Šζ–‡".len()]);
417    }
418
419    #[cfg(feature = "unicode")]
420    #[test]
421    fn neutral_prefix_joins_the_first_strong_script() {
422        let text = "(Ψ§Ω„ΨΉΨ±Ψ¨ΩŠΨ©) Latin";
423        let runs = script_runs(text).collect::<Vec<_>>();
424
425        assert_eq!(runs.len(), 2);
426        assert_eq!(runs[0].script, Script::Arabic);
427        assert_eq!(&text[runs[0].text.clone()], "(Ψ§Ω„ΨΉΨ±Ψ¨ΩŠΨ©) ");
428        assert_eq!(runs[1].script, Script::Latin);
429    }
430}