kashida/
segmenter.rs

1//! Finding joined run boundaries.
2
3use crate::grapheme::{analyze_grapheme, is_joining_type};
4use icu_properties::props::JoiningType;
5use icu_segmenter::GraphemeClusterSegmenter;
6
7/// Splits text into runs of joined letters.
8///
9/// Returns boundary offsets, including 0 and the end of the string.
10/// An empty string has one boundary, at 0.
11///
12/// # Example
13///
14/// ```
15/// use kashida::JoinedRunSegmenter;
16///
17/// let text = "خط الزوال";
18/// let boundaries: Vec<usize> = JoinedRunSegmenter::new().segment_str(text).collect();
19/// let runs: Vec<&str> = boundaries.windows(2).map(|w| &text[w[0]..w[1]]).collect();
20/// assert_eq!(runs, ["خط", " ", "ا", "لز", "و", "ا", "ل"]);
21/// ```
22#[derive(Clone, Copy, Debug, Default)]
23pub struct JoinedRunSegmenter;
24
25impl JoinedRunSegmenter {
26    /// Creates a joined run segmenter.
27    pub const fn new() -> Self {
28        Self
29    }
30
31    /// Returns joined run boundaries as UTF-8 byte offsets.
32    pub fn segment_str(self, input: &str) -> impl Iterator<Item = usize> {
33        let grapheme_boundaries: Vec<usize> =
34            GraphemeClusterSegmenter::new().segment_str(input).collect();
35        let mut boundaries = vec![0];
36        let mut previous_type: Option<JoiningType> = None;
37        for pair in grapheme_boundaries.windows(2) {
38            let grapheme = analyze_grapheme(&input[pair[0]..pair[1]]);
39            // A transparent cluster stays in the segment before it.
40            if grapheme.joining_type == JoiningType::Transparent {
41                continue;
42            }
43            // Continue the run if the two clusters join each other.
44            let can_join = is_joining_type(grapheme.joining_type);
45            let continues_run = can_join
46                && previous_type.is_some_and(|previous| {
47                    previous != JoiningType::RightJoining
48                        && grapheme.joining_type != JoiningType::LeftJoining
49                });
50            if !continues_run && pair[0] > 0 {
51                boundaries.push(pair[0]);
52            }
53            previous_type = can_join.then_some(grapheme.joining_type);
54        }
55        if !input.is_empty() {
56            boundaries.push(input.len());
57        }
58        boundaries.into_iter()
59    }
60
61    /// Returns joined run boundaries as UTF-16 code unit offsets.
62    ///
63    /// Unpaired surrogates are treated as U+FFFD (the replacement character).
64    pub fn segment_utf16(self, input: &[u16]) -> impl Iterator<Item = usize> {
65        let text: String = char::decode_utf16(input.iter().copied())
66            .map(|c| c.unwrap_or(char::REPLACEMENT_CHARACTER))
67            .collect();
68        // Replacing an unpaired surrogate preserves its UTF-16 length.
69        let mut chars = text.chars();
70        let (mut byte_offset, mut code_unit_offset) = (0, 0);
71        let boundaries: Vec<usize> = self
72            .segment_str(&text)
73            .map(|boundary| {
74                while byte_offset < boundary {
75                    let c = chars.next().expect("boundaries fall between characters");
76                    byte_offset += c.len_utf8();
77                    code_unit_offset += c.len_utf16();
78                }
79                code_unit_offset
80            })
81            .collect();
82        boundaries.into_iter()
83    }
84}
kashida/segmenter.rs

kashida/
segmenter.rs
