1use core::iter::Peekable;
2use core::ops::Range;
3use core::str::CharIndices;
4
5use crate::properties::{is_close_punctuation, is_open_punctuation, is_wide};
6
7#[cfg(feature = "unicode")]
8#[derive(Clone, Copy, Debug, Eq, PartialEq)]
9pub enum Script {
10 Common,
11 Inherited,
12 Latin,
13 Greek,
14 Cyrillic,
15 Hebrew,
16 Arabic,
17 Devanagari,
18 Han,
19 Hiragana,
20 Katakana,
21 Hangul,
22 Unknown,
23}
24
25#[cfg(feature = "unicode")]
26impl Script {
27 pub fn of(character: char) -> Self {
28 script(character)
29 }
30
31 pub const fn iso15924_tag(self) -> [u8; 4] {
32 match self {
33 Self::Common => *b"Zyyy",
34 Self::Inherited => *b"Zinh",
35 Self::Latin => *b"Latn",
36 Self::Greek => *b"Grek",
37 Self::Cyrillic => *b"Cyrl",
38 Self::Hebrew => *b"Hebr",
39 Self::Arabic => *b"Arab",
40 Self::Devanagari => *b"Deva",
41 Self::Han => *b"Hani",
42 Self::Hiragana => *b"Hira",
43 Self::Katakana => *b"Kana",
44 Self::Hangul => *b"Hang",
45 Self::Unknown => *b"Zzzz",
46 }
47 }
48}
49
50#[derive(Clone, Debug, Eq, PartialEq)]
51pub struct Grapheme<'a> {
52 pub text: &'a str,
53 pub range: Range<usize>,
54}
55
56pub struct Graphemes<'a> {
57 text: &'a str,
58 cursor: usize,
59}
60
61pub fn graphemes(text: &str) -> Graphemes<'_> {
62 Graphemes { text, cursor: 0 }
63}
64
65impl<'a> Iterator for Graphemes<'a> {
66 type Item = Grapheme<'a>;
67
68 fn next(&mut self) -> Option<Self::Item> {
69 if self.cursor == self.text.len() {
70 return None;
71 }
72 let start = self.cursor;
73 let mut chars = self.text[start..].char_indices();
74 let (_, first) = chars.next()?;
75 let mut previous = grapheme_class(first);
76 let mut regional_count = usize::from(previous == GraphemeClass::Regional);
77 let mut end = start + first.len_utf8();
78
79 for (offset, character) in chars {
80 let next = grapheme_class(character);
81 if grapheme_break(previous, next, regional_count) {
82 break;
83 }
84 end = start + offset + character.len_utf8();
85 if next == GraphemeClass::Regional {
86 regional_count += 1;
87 } else if !matches!(next, GraphemeClass::Extend | GraphemeClass::Zwj) {
88 regional_count = 0;
89 }
90 previous = next;
91 }
92
93 self.cursor = end;
94 Some(Grapheme {
95 text: &self.text[start..end],
96 range: start..end,
97 })
98 }
99}
100
101#[derive(Clone, Copy, Debug, Eq, PartialEq)]
102pub enum LineBreakKind {
103 Allowed,
104 Mandatory,
105}
106
107#[derive(Clone, Copy, Debug, Eq, PartialEq)]
108pub struct LineBreak {
109 pub offset: usize,
110 pub kind: LineBreakKind,
111}
112
113pub struct LineBreaks<'a> {
114 text: &'a str,
115 chars: Peekable<CharIndices<'a>>,
116 emitted_end: bool,
117}
118
119pub fn line_breaks(text: &str) -> LineBreaks<'_> {
120 LineBreaks {
121 text,
122 chars: text.char_indices().peekable(),
123 emitted_end: false,
124 }
125}
126
127impl Iterator for LineBreaks<'_> {
128 type Item = LineBreak;
129
130 fn next(&mut self) -> Option<Self::Item> {
131 while let Some((start, character)) = self.chars.next() {
132 let offset = start + character.len_utf8();
133 let next = self.chars.peek().map(|(_, next)| *next);
134 let kind = if character == '\n' || character == '\r' && next != Some('\n') {
135 Some(LineBreakKind::Mandatory)
136 } else if is_space(character) {
137 (!next.is_some_and(is_space)).then_some(LineBreakKind::Allowed)
138 } else if matches!(character, '-' | '/' | '\u{2010}' | '\u{2013}')
139 || next.is_some_and(|next| allows_wide_break(character, next))
140 {
141 Some(LineBreakKind::Allowed)
142 } else {
143 None
144 };
145 if let Some(kind) = kind {
146 if offset == self.text.len() {
147 self.emitted_end = true;
148 return Some(LineBreak {
149 offset,
150 kind: LineBreakKind::Mandatory,
151 });
152 }
153 return Some(LineBreak { offset, kind });
154 }
155 }
156 if self.emitted_end {
157 None
158 } else {
159 self.emitted_end = true;
160 Some(LineBreak {
161 offset: self.text.len(),
162 kind: LineBreakKind::Mandatory,
163 })
164 }
165 }
166}
167
168#[cfg(feature = "unicode")]
169#[derive(Clone, Debug, Eq, PartialEq)]
170pub struct ScriptRun {
171 pub text: Range<usize>,
172 pub script: Script,
173}
174
175#[cfg(feature = "unicode")]
176pub fn script_runs(text: &str) -> ScriptRuns<'_> {
177 ScriptRuns {
178 text,
179 graphemes: graphemes(text),
180 pending: None,
181 }
182}
183
184#[cfg(feature = "unicode")]
185pub struct ScriptRuns<'a> {
186 text: &'a str,
187 graphemes: Graphemes<'a>,
188 pending: Option<Grapheme<'a>>,
189}
190
191#[cfg(feature = "unicode")]
192impl Iterator for ScriptRuns<'_> {
193 type Item = ScriptRun;
194
195 fn next(&mut self) -> Option<Self::Item> {
196 let first = self.pending.take().or_else(|| self.graphemes.next())?;
197 let start = first.range.start;
198 let mut run_script = strong_script(first.text);
199
200 for cluster in self.graphemes.by_ref() {
201 let next_script = strong_script(cluster.text);
202 if run_script.is_none() {
203 run_script = next_script;
204 } else if next_script.is_some() && next_script != run_script {
205 self.pending = Some(cluster);
206 break;
207 }
208 }
209
210 let end = self
211 .pending
212 .as_ref()
213 .map_or(self.text.len(), |cluster| cluster.range.start);
214 Some(ScriptRun {
215 text: start..end,
216 script: run_script.unwrap_or(Script::Common),
217 })
218 }
219}
220
221#[derive(Clone, Copy, Eq, PartialEq)]
222enum GraphemeClass {
223 Other,
224 Cr,
225 Lf,
226 Control,
227 Extend,
228 Zwj,
229 Regional,
230 Prepend,
231 SpacingMark,
232 L,
233 V,
234 T,
235 Lv,
236 Lvt,
237 ExtendedPictographic,
238}
239
240fn grapheme_break(previous: GraphemeClass, next: GraphemeClass, regional_count: usize) -> bool {
241 if previous == GraphemeClass::Cr && next == GraphemeClass::Lf {
242 return false;
243 }
244 if matches!(
245 previous,
246 GraphemeClass::Cr | GraphemeClass::Lf | GraphemeClass::Control
247 ) || matches!(
248 next,
249 GraphemeClass::Cr | GraphemeClass::Lf | GraphemeClass::Control
250 ) {
251 return true;
252 }
253 if matches!(
254 (previous, next),
255 (
256 GraphemeClass::L,
257 GraphemeClass::L | GraphemeClass::V | GraphemeClass::Lv | GraphemeClass::Lvt
258 ) | (
259 GraphemeClass::Lv | GraphemeClass::V,
260 GraphemeClass::V | GraphemeClass::T
261 ) | (GraphemeClass::Lvt | GraphemeClass::T, GraphemeClass::T)
262 ) {
263 return false;
264 }
265 if matches!(
266 next,
267 GraphemeClass::Extend | GraphemeClass::Zwj | GraphemeClass::SpacingMark
268 ) || previous == GraphemeClass::Prepend
269 || previous == GraphemeClass::Zwj && next == GraphemeClass::ExtendedPictographic
270 {
271 return false;
272 }
273 !(previous == GraphemeClass::Regional
274 && next == GraphemeClass::Regional
275 && regional_count % 2 == 1)
276}
277
278fn grapheme_class(character: char) -> GraphemeClass {
279 let value = character as u32;
280 match value {
281 0x000D => GraphemeClass::Cr,
282 0x000A => GraphemeClass::Lf,
283 0x0000..=0x001F | 0x007F..=0x009F => GraphemeClass::Control,
284 0x200D => GraphemeClass::Zwj,
285 0x1F1E6..=0x1F1FF => GraphemeClass::Regional,
286 0x0600..=0x0605 | 0x06DD | 0x070F | 0x08E2 => GraphemeClass::Prepend,
287 0x0903 | 0x093B..=0x0940 | 0x0949..=0x094C | 0x0982..=0x0983 => GraphemeClass::SpacingMark,
288 0x1100..=0x115F | 0xA960..=0xA97C => GraphemeClass::L,
289 0x1160..=0x11A7 | 0xD7B0..=0xD7C6 => GraphemeClass::V,
290 0x11A8..=0x11FF | 0xD7CB..=0xD7FB => GraphemeClass::T,
291 0xAC00..=0xD7A3 if (value - 0xAC00) % 28 == 0 => GraphemeClass::Lv,
292 0xAC00..=0xD7A3 => GraphemeClass::Lvt,
293 0x1F000..=0x1FAFF | 0x2600..=0x27BF => GraphemeClass::ExtendedPictographic,
294 _ if is_extend(value) => GraphemeClass::Extend,
295 _ => GraphemeClass::Other,
296 }
297}
298
299fn is_extend(value: u32) -> bool {
300 matches!(
301 value,
302 0x0300..=0x036F
303 | 0x0483..=0x0489
304 | 0x0591..=0x05BD
305 | 0x05BF
306 | 0x05C1..=0x05C2
307 | 0x0610..=0x061A
308 | 0x064B..=0x065F
309 | 0x0670
310 | 0x06D6..=0x06ED
311 | 0x0900..=0x0902
312 | 0x093A
313 | 0x093C
314 | 0x0941..=0x0948
315 | 0x094D
316 | 0x0951..=0x0957
317 | 0x1AB0..=0x1AFF
318 | 0x1DC0..=0x1DFF
319 | 0x20D0..=0x20FF
320 | 0xFE00..=0xFE0F
321 | 0xFE20..=0xFE2F
322 | 0x1F3FB..=0x1F3FF
323 | 0xE0100..=0xE01EF
324 )
325}
326
327#[cfg(feature = "unicode")]
328fn script(character: char) -> Script {
329 let value = character as u32;
330 match value {
331 _ if is_extend(value) => Script::Inherited,
332 0x0041..=0x005A | 0x0061..=0x007A | 0x00C0..=0x024F | 0x1E00..=0x1EFF => Script::Latin,
333 0x0370..=0x03FF | 0x1F00..=0x1FFF => Script::Greek,
334 0x0400..=0x052F | 0x2DE0..=0x2DFF | 0xA640..=0xA69F => Script::Cyrillic,
335 0x0590..=0x05FF => Script::Hebrew,
336 0x0600..=0x06FF | 0x0750..=0x077F | 0x08A0..=0x08FF | 0xFB50..=0xFDFF | 0xFE70..=0xFEFF => {
337 Script::Arabic
338 }
339 0x0900..=0x097F | 0xA8E0..=0xA8FF => Script::Devanagari,
340 0x3040..=0x309F => Script::Hiragana,
341 0x30A0..=0x30FF | 0x31F0..=0x31FF => Script::Katakana,
342 0x1100..=0x11FF | 0x3130..=0x318F | 0xAC00..=0xD7AF => Script::Hangul,
343 0x3400..=0x4DBF | 0x4E00..=0x9FFF | 0xF900..=0xFAFF | 0x20000..=0x323AF => Script::Han,
344 0x0000..=0x0040
345 | 0x005B..=0x0060
346 | 0x007B..=0x00BF
347 | 0x2000..=0x206F
348 | 0x20A0..=0x20CF
349 | 0x2100..=0x214F
350 | 0x2190..=0x2BFF
351 | 0x1F000..=0x1FAFF => Script::Common,
352 _ => Script::Unknown,
353 }
354}
355
356#[cfg(feature = "unicode")]
357fn strong_script(grapheme: &str) -> Option<Script> {
358 grapheme
359 .chars()
360 .map(script)
361 .find(|script| !matches!(script, Script::Common | Script::Inherited | Script::Unknown))
362}
363
364fn is_space(character: char) -> bool {
365 matches!(
366 character,
367 ' ' | '\t' | '\u{00A0}' | '\u{2000}'..='\u{200A}' | '\u{3000}'
368 )
369}
370
371fn allows_wide_break(previous: char, next: char) -> bool {
372 !is_open_punctuation(previous)
373 && !is_close_punctuation(next)
374 && (is_wide(previous) || is_wide(next))
375}
376
377#[cfg(test)]
378mod tests {
379 use super::*;
380 use std::prelude::v1::*;
381
382 #[test]
383 fn grapheme_ranges_preserve_common_clusters() {
384 let text = "a\u{301}π©βππ¨π³";
385 let clusters = graphemes(text).collect::<Vec<_>>();
386
387 assert_eq!(clusters.len(), 3);
388 assert_eq!(clusters[0].text, "a\u{301}");
389 assert_eq!(clusters[1].text, "π©βπ");
390 assert_eq!(clusters[2].text, "π¨π³");
391 }
392
393 #[test]
394 fn line_breaks_preserve_words_and_cjk_punctuation() {
395 let latin = line_breaks("hello world").collect::<Vec<_>>();
396 let cjk = line_breaks("δ½ ε₯½οΌδΈη").collect::<Vec<_>>();
397
398 assert!(latin.contains(&LineBreak {
399 offset: 6,
400 kind: LineBreakKind::Allowed,
401 }));
402 assert!(!cjk
403 .iter()
404 .any(|line_break| line_break.offset == "δ½ ε₯½".len()));
405 assert_eq!(cjk.last().unwrap().kind, LineBreakKind::Mandatory);
406 }
407
408 #[test]
409 fn line_breaks_keep_bracket_edges_together() {
410 let text = "γζεγγ";
411 let offsets = line_breaks(text)
412 .filter(|line_break| line_break.kind == LineBreakKind::Allowed)
413 .map(|line_break| line_break.offset)
414 .collect::<Vec<_>>();
415
416 assert_eq!(offsets, ["γζ".len()]);
417 }
418
419 #[cfg(feature = "unicode")]
420 #[test]
421 fn neutral_prefix_joins_the_first_strong_script() {
422 let text = "(Ψ§ΩΨΉΨ±Ψ¨ΩΨ©) Latin";
423 let runs = script_runs(text).collect::<Vec<_>>();
424
425 assert_eq!(runs.len(), 2);
426 assert_eq!(runs[0].script, Script::Arabic);
427 assert_eq!(&text[runs[0].text.clone()], "(Ψ§ΩΨΉΨ±Ψ¨ΩΨ©) ");
428 assert_eq!(runs[1].script, Script::Latin);
429 }
430}