1use core::iter::Peekable;
2use core::ops::Range;
3use core::str::CharIndices;
4
5use crate::properties::{is_close_punctuation, is_open_punctuation, is_wide};
6
7#[cfg(feature = "unicode")]
8#[derive(Clone, Copy, Debug, Eq, PartialEq)]
9pub enum Script {
10 Common,
11 Inherited,
12 Latin,
13 Greek,
14 Cyrillic,
15 Hebrew,
16 Arabic,
17 Thai,
18 Devanagari,
19 Han,
20 Hiragana,
21 Katakana,
22 Hangul,
23 Unknown,
24}
25
26#[cfg(feature = "unicode")]
27impl Script {
28 pub fn of(character: char) -> Self {
29 script(character)
30 }
31
32 pub const fn iso15924_tag(self) -> [u8; 4] {
33 match self {
34 Self::Common => *b"Zyyy",
35 Self::Inherited => *b"Zinh",
36 Self::Latin => *b"Latn",
37 Self::Greek => *b"Grek",
38 Self::Cyrillic => *b"Cyrl",
39 Self::Hebrew => *b"Hebr",
40 Self::Arabic => *b"Arab",
41 Self::Thai => *b"Thai",
42 Self::Devanagari => *b"Deva",
43 Self::Han => *b"Hani",
44 Self::Hiragana => *b"Hira",
45 Self::Katakana => *b"Kana",
46 Self::Hangul => *b"Hang",
47 Self::Unknown => *b"Zzzz",
48 }
49 }
50}
51
52#[derive(Clone, Debug, Eq, PartialEq)]
53pub struct Grapheme<'a> {
54 pub text: &'a str,
55 pub range: Range<usize>,
56}
57
58pub struct Graphemes<'a> {
59 text: &'a str,
60 cursor: usize,
61}
62
63pub fn graphemes(text: &str) -> Graphemes<'_> {
64 Graphemes { text, cursor: 0 }
65}
66
67impl<'a> Iterator for Graphemes<'a> {
68 type Item = Grapheme<'a>;
69
70 fn next(&mut self) -> Option<Self::Item> {
71 if self.cursor == self.text.len() {
72 return None;
73 }
74 let start = self.cursor;
75 let mut chars = self.text[start..].char_indices();
76 let (_, first) = chars.next()?;
77 let mut previous = grapheme_class(first);
78 let mut regional_count = usize::from(previous == GraphemeClass::Regional);
79 let mut indic_consonant = devanagari_consonant(first);
80 let mut indic_linked = false;
81 let mut end = start + first.len_utf8();
82
83 for (offset, character) in chars {
84 let next = grapheme_class(character);
85 let indic_conjunct = indic_consonant && indic_linked && devanagari_consonant(character);
86 if grapheme_break(previous, next, regional_count) && !indic_conjunct {
87 break;
88 }
89 end = start + offset + character.len_utf8();
90 if next == GraphemeClass::Regional {
91 regional_count += 1;
92 } else if !matches!(next, GraphemeClass::Extend | GraphemeClass::Zwj) {
93 regional_count = 0;
94 }
95 if character == '\u{094D}' && indic_consonant {
96 indic_linked = true;
97 } else if devanagari_consonant(character) {
98 indic_consonant = true;
99 indic_linked = false;
100 } else if !matches!(next, GraphemeClass::Extend | GraphemeClass::Zwj) {
101 indic_consonant = false;
102 indic_linked = false;
103 }
104 previous = next;
105 }
106
107 self.cursor = end;
108 Some(Grapheme {
109 text: &self.text[start..end],
110 range: start..end,
111 })
112 }
113}
114
115fn devanagari_consonant(character: char) -> bool {
116 matches!(
117 character as u32,
118 0x0915..=0x0939 | 0x0958..=0x095F | 0x0978..=0x097F
119 )
120}
121
122#[derive(Clone, Copy, Debug, Eq, PartialEq)]
123pub enum LineBreakKind {
124 Allowed,
125 Mandatory,
126}
127
128#[derive(Clone, Copy, Debug, Eq, PartialEq)]
129pub struct LineBreak {
130 pub offset: usize,
131 pub kind: LineBreakKind,
132}
133
134#[cfg(feature = "shaping")]
136pub trait LineBreakProvider {
137 fn break_at(&self, text: &str, offset: usize) -> Option<LineBreakKind>;
139}
140
141pub struct LineBreaks<'a> {
142 text: &'a str,
143 chars: Peekable<CharIndices<'a>>,
144 emitted_end: bool,
145}
146
147pub fn line_breaks(text: &str) -> LineBreaks<'_> {
148 LineBreaks {
149 text,
150 chars: text.char_indices().peekable(),
151 emitted_end: false,
152 }
153}
154
155pub(crate) fn hard_break_len(text: &str) -> usize {
156 match text.as_bytes() {
157 [b'\r', b'\n', ..] => 2,
158 [b'\r' | b'\n', ..] => 1,
159 _ => 0,
160 }
161}
162
163#[cfg(feature = "shaping")]
164pub(crate) fn is_hard_break(text: &str) -> bool {
165 let len = hard_break_len(text);
166 len != 0 && len == text.len()
167}
168
169impl Iterator for LineBreaks<'_> {
170 type Item = LineBreak;
171
172 fn next(&mut self) -> Option<Self::Item> {
173 while let Some((start, character)) = self.chars.next() {
174 let mut offset = start + character.len_utf8();
175 let next = self.chars.peek().map(|(_, next)| *next);
176 let separator = hard_break_len(&self.text[start..]);
177 let kind = if separator != 0 {
178 if separator == 2 {
179 self.chars.next();
180 offset = start + separator;
181 }
182 Some(LineBreakKind::Mandatory)
183 } else if character == '\u{200B}' {
184 Some(LineBreakKind::Allowed)
185 } else if is_space(character) {
186 (!next.is_some_and(is_space)).then_some(LineBreakKind::Allowed)
187 } else if matches!(character, '-' | '/' | '\u{2010}' | '\u{2013}')
188 || next.is_some_and(|next| allows_wide_break(character, next))
189 {
190 Some(LineBreakKind::Allowed)
191 } else {
192 None
193 };
194 if let Some(kind) = kind {
195 if offset == self.text.len() {
196 self.emitted_end = true;
197 return Some(LineBreak {
198 offset,
199 kind: LineBreakKind::Mandatory,
200 });
201 }
202 return Some(LineBreak { offset, kind });
203 }
204 }
205 if self.emitted_end {
206 None
207 } else {
208 self.emitted_end = true;
209 Some(LineBreak {
210 offset: self.text.len(),
211 kind: LineBreakKind::Mandatory,
212 })
213 }
214 }
215}
216
217#[cfg(feature = "unicode")]
218#[derive(Clone, Debug, Eq, PartialEq)]
219pub struct ScriptRun {
220 pub text: Range<usize>,
221 pub script: Script,
222}
223
224#[cfg(feature = "unicode")]
225pub fn script_runs(text: &str) -> ScriptRuns<'_> {
226 ScriptRuns {
227 text,
228 graphemes: graphemes(text),
229 pending: None,
230 }
231}
232
233#[cfg(feature = "unicode")]
234pub struct ScriptRuns<'a> {
235 text: &'a str,
236 graphemes: Graphemes<'a>,
237 pending: Option<Grapheme<'a>>,
238}
239
240#[cfg(feature = "unicode")]
241impl Iterator for ScriptRuns<'_> {
242 type Item = ScriptRun;
243
244 fn next(&mut self) -> Option<Self::Item> {
245 let first = self.pending.take().or_else(|| self.graphemes.next())?;
246 let start = first.range.start;
247 let mut run_script = strong_script(first.text);
248
249 for cluster in self.graphemes.by_ref() {
250 let next_script = strong_script(cluster.text);
251 if run_script.is_none() {
252 run_script = next_script;
253 } else if next_script.is_some() && next_script != run_script {
254 self.pending = Some(cluster);
255 break;
256 }
257 }
258
259 let end = self
260 .pending
261 .as_ref()
262 .map_or(self.text.len(), |cluster| cluster.range.start);
263 Some(ScriptRun {
264 text: start..end,
265 script: run_script.unwrap_or(Script::Common),
266 })
267 }
268}
269
270#[derive(Clone, Copy, Eq, PartialEq)]
271enum GraphemeClass {
272 Other,
273 Cr,
274 Lf,
275 Control,
276 Extend,
277 Zwj,
278 Regional,
279 Prepend,
280 SpacingMark,
281 L,
282 V,
283 T,
284 Lv,
285 Lvt,
286 ExtendedPictographic,
287}
288
289fn grapheme_break(previous: GraphemeClass, next: GraphemeClass, regional_count: usize) -> bool {
290 if previous == GraphemeClass::Cr && next == GraphemeClass::Lf {
291 return false;
292 }
293 if matches!(
294 previous,
295 GraphemeClass::Cr | GraphemeClass::Lf | GraphemeClass::Control
296 ) || matches!(
297 next,
298 GraphemeClass::Cr | GraphemeClass::Lf | GraphemeClass::Control
299 ) {
300 return true;
301 }
302 if matches!(
303 (previous, next),
304 (
305 GraphemeClass::L,
306 GraphemeClass::L | GraphemeClass::V | GraphemeClass::Lv | GraphemeClass::Lvt
307 ) | (
308 GraphemeClass::Lv | GraphemeClass::V,
309 GraphemeClass::V | GraphemeClass::T
310 ) | (GraphemeClass::Lvt | GraphemeClass::T, GraphemeClass::T)
311 ) {
312 return false;
313 }
314 if matches!(
315 next,
316 GraphemeClass::Extend | GraphemeClass::Zwj | GraphemeClass::SpacingMark
317 ) || previous == GraphemeClass::Prepend
318 || previous == GraphemeClass::Zwj && next == GraphemeClass::ExtendedPictographic
319 {
320 return false;
321 }
322 !(previous == GraphemeClass::Regional
323 && next == GraphemeClass::Regional
324 && regional_count % 2 == 1)
325}
326
327fn grapheme_class(character: char) -> GraphemeClass {
328 let value = character as u32;
329 match value {
330 0x000D => GraphemeClass::Cr,
331 0x000A => GraphemeClass::Lf,
332 0x0000..=0x001F | 0x007F..=0x009F => GraphemeClass::Control,
333 0x200D => GraphemeClass::Zwj,
334 0x1F1E6..=0x1F1FF => GraphemeClass::Regional,
335 0x0600..=0x0605 | 0x06DD | 0x070F | 0x08E2 => GraphemeClass::Prepend,
336 0x0903 | 0x093B..=0x0940 | 0x0949..=0x094C | 0x0982..=0x0983 => GraphemeClass::SpacingMark,
337 0x1100..=0x115F | 0xA960..=0xA97C => GraphemeClass::L,
338 0x1160..=0x11A7 | 0xD7B0..=0xD7C6 => GraphemeClass::V,
339 0x11A8..=0x11FF | 0xD7CB..=0xD7FB => GraphemeClass::T,
340 0xAC00..=0xD7A3 if (value - 0xAC00) % 28 == 0 => GraphemeClass::Lv,
341 0xAC00..=0xD7A3 => GraphemeClass::Lvt,
342 0x1F000..=0x1FAFF | 0x2600..=0x27BF => GraphemeClass::ExtendedPictographic,
343 _ if is_extend(value) => GraphemeClass::Extend,
344 _ => GraphemeClass::Other,
345 }
346}
347
348fn is_extend(value: u32) -> bool {
349 matches!(
350 value,
351 0x0300..=0x036F
352 | 0x0483..=0x0489
353 | 0x0591..=0x05BD
354 | 0x05BF
355 | 0x05C1..=0x05C2
356 | 0x0610..=0x061A
357 | 0x064B..=0x065F
358 | 0x0670
359 | 0x06D6..=0x06ED
360 | 0x0E31
361 | 0x0E34..=0x0E3A
362 | 0x0E47..=0x0E4E
363 | 0x0900..=0x0902
364 | 0x093A
365 | 0x093C
366 | 0x0941..=0x0948
367 | 0x094D
368 | 0x0951..=0x0957
369 | 0x1AB0..=0x1AFF
370 | 0x1DC0..=0x1DFF
371 | 0x20D0..=0x20FF
372 | 0xFE00..=0xFE0F
373 | 0xFE20..=0xFE2F
374 | 0x1F3FB..=0x1F3FF
375 | 0xE0100..=0xE01EF
376 )
377}
378
379#[cfg(feature = "unicode")]
380fn script(character: char) -> Script {
381 let value = character as u32;
382 match value {
383 _ if is_extend(value) => Script::Inherited,
384 0x0041..=0x005A | 0x0061..=0x007A | 0x00C0..=0x024F | 0x1E00..=0x1EFF => Script::Latin,
385 0x0370..=0x03FF | 0x1F00..=0x1FFF => Script::Greek,
386 0x0400..=0x052F | 0x2DE0..=0x2DFF | 0xA640..=0xA69F => Script::Cyrillic,
387 0x0590..=0x05FF => Script::Hebrew,
388 0x0600..=0x06FF | 0x0750..=0x077F | 0x08A0..=0x08FF | 0xFB50..=0xFDFF | 0xFE70..=0xFEFF => {
389 Script::Arabic
390 }
391 0x0E00..=0x0E7F => Script::Thai,
392 0x0900..=0x097F | 0xA8E0..=0xA8FF => Script::Devanagari,
393 0x3040..=0x309F => Script::Hiragana,
394 0x30A0..=0x30FF | 0x31F0..=0x31FF => Script::Katakana,
395 0x1100..=0x11FF | 0x3130..=0x318F | 0xAC00..=0xD7AF => Script::Hangul,
396 0x3400..=0x4DBF | 0x4E00..=0x9FFF | 0xF900..=0xFAFF | 0x20000..=0x323AF => Script::Han,
397 0x0000..=0x0040
398 | 0x005B..=0x0060
399 | 0x007B..=0x00BF
400 | 0x2000..=0x206F
401 | 0x20A0..=0x20CF
402 | 0x2100..=0x214F
403 | 0x2190..=0x2BFF
404 | 0x1F000..=0x1FAFF => Script::Common,
405 _ => Script::Unknown,
406 }
407}
408
409#[cfg(feature = "unicode")]
410fn strong_script(grapheme: &str) -> Option<Script> {
411 grapheme
412 .chars()
413 .map(script)
414 .find(|script| !matches!(script, Script::Common | Script::Inherited | Script::Unknown))
415}
416
417fn is_space(character: char) -> bool {
418 matches!(
419 character,
420 ' ' | '\t' | '\u{00A0}' | '\u{2000}'..='\u{200A}' | '\u{3000}'
421 )
422}
423
424fn allows_wide_break(previous: char, next: char) -> bool {
425 !is_open_punctuation(previous)
426 && !is_close_punctuation(next)
427 && (is_wide(previous) || is_wide(next))
428}
429
430#[cfg(test)]
431mod tests {
432 use super::*;
433 use std::prelude::v1::*;
434
435 #[test]
436 fn grapheme_ranges_preserve_common_clusters() {
437 let text = "a\u{301}👩🚀🇨🇳";
438 let clusters = graphemes(text).collect::<Vec<_>>();
439
440 assert_eq!(clusters.len(), 3);
441 assert_eq!(clusters[0].text, "a\u{301}");
442 assert_eq!(clusters[1].text, "👩🚀");
443 assert_eq!(clusters[2].text, "🇨🇳");
444 }
445
446 #[test]
447 fn thai_marks_stay_with_their_base() {
448 let text = "กิ่";
449 let clusters = graphemes(text).collect::<Vec<_>>();
450
451 assert_eq!(clusters.len(), 1);
452 assert_eq!(clusters[0].text, text);
453 }
454
455 #[test]
456 fn devanagari_linkers_keep_conjuncts_in_one_cluster() {
457 let text = "क्षत्रिय";
458 let clusters = graphemes(text).collect::<Vec<_>>();
459
460 assert_eq!(clusters.len(), 3);
461 assert_eq!(clusters[0].text, "क्ष");
462 assert_eq!(clusters[1].text, "त्रि");
463 assert_eq!(clusters[2].text, "य");
464 }
465
466 #[cfg(feature = "unicode")]
467 #[test]
468 fn thai_has_a_stable_script_identity() {
469 assert_eq!(Script::of('ก'), Script::Thai);
470 assert_eq!(Script::Thai.iso15924_tag(), *b"Thai");
471
472 let runs = script_runs("กิ่ ภาษา").collect::<Vec<_>>();
473 assert_eq!(runs.len(), 1);
474 assert_eq!(runs[0].script, Script::Thai);
475 }
476
477 #[test]
478 fn line_breaks_preserve_words_and_cjk_punctuation() {
479 let latin = line_breaks("hello world").collect::<Vec<_>>();
480 let cjk = line_breaks("你好,世界").collect::<Vec<_>>();
481
482 assert!(latin.contains(&LineBreak {
483 offset: 6,
484 kind: LineBreakKind::Allowed,
485 }));
486 assert!(!cjk
487 .iter()
488 .any(|line_break| line_break.offset == "你好".len()));
489 assert_eq!(cjk.last().unwrap().kind, LineBreakKind::Mandatory);
490 }
491
492 #[test]
493 fn zero_width_space_is_an_explicit_break() {
494 let text = "ภาษา\u{200B}ไทย";
495 let breaks = line_breaks(text).collect::<Vec<_>>();
496
497 assert!(breaks.contains(&LineBreak {
498 offset: "ภาษา\u{200B}".len(),
499 kind: LineBreakKind::Allowed,
500 }));
501 }
502
503 #[test]
504 fn line_breaks_keep_bracket_edges_together() {
505 let text = "《文字》〉";
506 let offsets = line_breaks(text)
507 .filter(|line_break| line_break.kind == LineBreakKind::Allowed)
508 .map(|line_break| line_break.offset)
509 .collect::<Vec<_>>();
510
511 assert_eq!(offsets, ["《文".len()]);
512 }
513
514 #[cfg(feature = "unicode")]
515 #[test]
516 fn neutral_prefix_joins_the_first_strong_script() {
517 let text = "(العربية) Latin";
518 let runs = script_runs(text).collect::<Vec<_>>();
519
520 assert_eq!(runs.len(), 2);
521 assert_eq!(runs[0].script, Script::Arabic);
522 assert_eq!(&text[runs[0].text.clone()], "(العربية) ");
523 assert_eq!(runs[1].script, Script::Latin);
524 }
525}