1use core::iter::Peekable;
2use core::ops::Range;
3use core::str::CharIndices;
4
5use crate::properties::{is_close_punctuation, is_open_punctuation, is_wide};
6
7#[cfg(feature = "unicode")]
8#[derive(Clone, Copy, Debug, Eq, PartialEq)]
9pub enum Script {
10 Common,
11 Inherited,
12 Latin,
13 Greek,
14 Cyrillic,
15 Hebrew,
16 Arabic,
17 Thai,
18 Devanagari,
19 Han,
20 Hiragana,
21 Katakana,
22 Hangul,
23 Unknown,
24}
25
26#[cfg(feature = "unicode")]
27impl Script {
28 pub fn of(character: char) -> Self {
29 script(character)
30 }
31
32 pub const fn iso15924_tag(self) -> [u8; 4] {
33 match self {
34 Self::Common => *b"Zyyy",
35 Self::Inherited => *b"Zinh",
36 Self::Latin => *b"Latn",
37 Self::Greek => *b"Grek",
38 Self::Cyrillic => *b"Cyrl",
39 Self::Hebrew => *b"Hebr",
40 Self::Arabic => *b"Arab",
41 Self::Thai => *b"Thai",
42 Self::Devanagari => *b"Deva",
43 Self::Han => *b"Hani",
44 Self::Hiragana => *b"Hira",
45 Self::Katakana => *b"Kana",
46 Self::Hangul => *b"Hang",
47 Self::Unknown => *b"Zzzz",
48 }
49 }
50}
51
52#[derive(Clone, Debug, Eq, PartialEq)]
53pub struct Grapheme<'a> {
54 pub text: &'a str,
55 pub range: Range<usize>,
56}
57
58pub struct Graphemes<'a> {
59 text: &'a str,
60 cursor: usize,
61}
62
63pub fn graphemes(text: &str) -> Graphemes<'_> {
64 Graphemes { text, cursor: 0 }
65}
66
67impl<'a> Iterator for Graphemes<'a> {
68 type Item = Grapheme<'a>;
69
70 fn next(&mut self) -> Option<Self::Item> {
71 if self.cursor == self.text.len() {
72 return None;
73 }
74 let start = self.cursor;
75 let mut chars = self.text[start..].char_indices();
76 let (_, first) = chars.next()?;
77 let mut previous = grapheme_class(first);
78 let mut regional_count = usize::from(previous == GraphemeClass::Regional);
79 let mut indic_consonant = devanagari_consonant(first);
80 let mut indic_linked = false;
81 let mut end = start + first.len_utf8();
82
83 for (offset, character) in chars {
84 let next = grapheme_class(character);
85 let indic_conjunct = indic_consonant && indic_linked && devanagari_consonant(character);
86 if grapheme_break(previous, next, regional_count) && !indic_conjunct {
87 break;
88 }
89 end = start + offset + character.len_utf8();
90 if next == GraphemeClass::Regional {
91 regional_count += 1;
92 } else if !matches!(next, GraphemeClass::Extend | GraphemeClass::Zwj) {
93 regional_count = 0;
94 }
95 if character == '\u{094D}' && indic_consonant {
96 indic_linked = true;
97 } else if devanagari_consonant(character) {
98 indic_consonant = true;
99 indic_linked = false;
100 } else if !matches!(next, GraphemeClass::Extend | GraphemeClass::Zwj) {
101 indic_consonant = false;
102 indic_linked = false;
103 }
104 previous = next;
105 }
106
107 self.cursor = end;
108 Some(Grapheme {
109 text: &self.text[start..end],
110 range: start..end,
111 })
112 }
113}
114
115fn devanagari_consonant(character: char) -> bool {
116 matches!(
117 character as u32,
118 0x0915..=0x0939 | 0x0958..=0x095F | 0x0978..=0x097F
119 )
120}
121
122#[derive(Clone, Copy, Debug, Eq, PartialEq)]
123pub enum LineBreakKind {
124 Allowed,
125 Mandatory,
126}
127
128#[derive(Clone, Copy, Debug, Eq, PartialEq)]
129pub struct LineBreak {
130 pub offset: usize,
131 pub kind: LineBreakKind,
132}
133
134#[cfg(feature = "shaping")]
136pub trait LineBreakProvider {
137 fn break_at(&self, text: &str, offset: usize) -> Option<LineBreakKind>;
139}
140
141pub struct LineBreaks<'a> {
142 text: &'a str,
143 chars: Peekable<CharIndices<'a>>,
144 emitted_end: bool,
145}
146
147pub fn line_breaks(text: &str) -> LineBreaks<'_> {
148 LineBreaks {
149 text,
150 chars: text.char_indices().peekable(),
151 emitted_end: false,
152 }
153}
154
155impl Iterator for LineBreaks<'_> {
156 type Item = LineBreak;
157
158 fn next(&mut self) -> Option<Self::Item> {
159 while let Some((start, character)) = self.chars.next() {
160 let offset = start + character.len_utf8();
161 let next = self.chars.peek().map(|(_, next)| *next);
162 let kind = if character == '\n' || character == '\r' && next != Some('\n') {
163 Some(LineBreakKind::Mandatory)
164 } else if character == '\u{200B}' {
165 Some(LineBreakKind::Allowed)
166 } else if is_space(character) {
167 (!next.is_some_and(is_space)).then_some(LineBreakKind::Allowed)
168 } else if matches!(character, '-' | '/' | '\u{2010}' | '\u{2013}')
169 || next.is_some_and(|next| allows_wide_break(character, next))
170 {
171 Some(LineBreakKind::Allowed)
172 } else {
173 None
174 };
175 if let Some(kind) = kind {
176 if offset == self.text.len() {
177 self.emitted_end = true;
178 return Some(LineBreak {
179 offset,
180 kind: LineBreakKind::Mandatory,
181 });
182 }
183 return Some(LineBreak { offset, kind });
184 }
185 }
186 if self.emitted_end {
187 None
188 } else {
189 self.emitted_end = true;
190 Some(LineBreak {
191 offset: self.text.len(),
192 kind: LineBreakKind::Mandatory,
193 })
194 }
195 }
196}
197
198#[cfg(feature = "unicode")]
199#[derive(Clone, Debug, Eq, PartialEq)]
200pub struct ScriptRun {
201 pub text: Range<usize>,
202 pub script: Script,
203}
204
205#[cfg(feature = "unicode")]
206pub fn script_runs(text: &str) -> ScriptRuns<'_> {
207 ScriptRuns {
208 text,
209 graphemes: graphemes(text),
210 pending: None,
211 }
212}
213
214#[cfg(feature = "unicode")]
215pub struct ScriptRuns<'a> {
216 text: &'a str,
217 graphemes: Graphemes<'a>,
218 pending: Option<Grapheme<'a>>,
219}
220
221#[cfg(feature = "unicode")]
222impl Iterator for ScriptRuns<'_> {
223 type Item = ScriptRun;
224
225 fn next(&mut self) -> Option<Self::Item> {
226 let first = self.pending.take().or_else(|| self.graphemes.next())?;
227 let start = first.range.start;
228 let mut run_script = strong_script(first.text);
229
230 for cluster in self.graphemes.by_ref() {
231 let next_script = strong_script(cluster.text);
232 if run_script.is_none() {
233 run_script = next_script;
234 } else if next_script.is_some() && next_script != run_script {
235 self.pending = Some(cluster);
236 break;
237 }
238 }
239
240 let end = self
241 .pending
242 .as_ref()
243 .map_or(self.text.len(), |cluster| cluster.range.start);
244 Some(ScriptRun {
245 text: start..end,
246 script: run_script.unwrap_or(Script::Common),
247 })
248 }
249}
250
251#[derive(Clone, Copy, Eq, PartialEq)]
252enum GraphemeClass {
253 Other,
254 Cr,
255 Lf,
256 Control,
257 Extend,
258 Zwj,
259 Regional,
260 Prepend,
261 SpacingMark,
262 L,
263 V,
264 T,
265 Lv,
266 Lvt,
267 ExtendedPictographic,
268}
269
270fn grapheme_break(previous: GraphemeClass, next: GraphemeClass, regional_count: usize) -> bool {
271 if previous == GraphemeClass::Cr && next == GraphemeClass::Lf {
272 return false;
273 }
274 if matches!(
275 previous,
276 GraphemeClass::Cr | GraphemeClass::Lf | GraphemeClass::Control
277 ) || matches!(
278 next,
279 GraphemeClass::Cr | GraphemeClass::Lf | GraphemeClass::Control
280 ) {
281 return true;
282 }
283 if matches!(
284 (previous, next),
285 (
286 GraphemeClass::L,
287 GraphemeClass::L | GraphemeClass::V | GraphemeClass::Lv | GraphemeClass::Lvt
288 ) | (
289 GraphemeClass::Lv | GraphemeClass::V,
290 GraphemeClass::V | GraphemeClass::T
291 ) | (GraphemeClass::Lvt | GraphemeClass::T, GraphemeClass::T)
292 ) {
293 return false;
294 }
295 if matches!(
296 next,
297 GraphemeClass::Extend | GraphemeClass::Zwj | GraphemeClass::SpacingMark
298 ) || previous == GraphemeClass::Prepend
299 || previous == GraphemeClass::Zwj && next == GraphemeClass::ExtendedPictographic
300 {
301 return false;
302 }
303 !(previous == GraphemeClass::Regional
304 && next == GraphemeClass::Regional
305 && regional_count % 2 == 1)
306}
307
308fn grapheme_class(character: char) -> GraphemeClass {
309 let value = character as u32;
310 match value {
311 0x000D => GraphemeClass::Cr,
312 0x000A => GraphemeClass::Lf,
313 0x0000..=0x001F | 0x007F..=0x009F => GraphemeClass::Control,
314 0x200D => GraphemeClass::Zwj,
315 0x1F1E6..=0x1F1FF => GraphemeClass::Regional,
316 0x0600..=0x0605 | 0x06DD | 0x070F | 0x08E2 => GraphemeClass::Prepend,
317 0x0903 | 0x093B..=0x0940 | 0x0949..=0x094C | 0x0982..=0x0983 => GraphemeClass::SpacingMark,
318 0x1100..=0x115F | 0xA960..=0xA97C => GraphemeClass::L,
319 0x1160..=0x11A7 | 0xD7B0..=0xD7C6 => GraphemeClass::V,
320 0x11A8..=0x11FF | 0xD7CB..=0xD7FB => GraphemeClass::T,
321 0xAC00..=0xD7A3 if (value - 0xAC00) % 28 == 0 => GraphemeClass::Lv,
322 0xAC00..=0xD7A3 => GraphemeClass::Lvt,
323 0x1F000..=0x1FAFF | 0x2600..=0x27BF => GraphemeClass::ExtendedPictographic,
324 _ if is_extend(value) => GraphemeClass::Extend,
325 _ => GraphemeClass::Other,
326 }
327}
328
329fn is_extend(value: u32) -> bool {
330 matches!(
331 value,
332 0x0300..=0x036F
333 | 0x0483..=0x0489
334 | 0x0591..=0x05BD
335 | 0x05BF
336 | 0x05C1..=0x05C2
337 | 0x0610..=0x061A
338 | 0x064B..=0x065F
339 | 0x0670
340 | 0x06D6..=0x06ED
341 | 0x0E31
342 | 0x0E34..=0x0E3A
343 | 0x0E47..=0x0E4E
344 | 0x0900..=0x0902
345 | 0x093A
346 | 0x093C
347 | 0x0941..=0x0948
348 | 0x094D
349 | 0x0951..=0x0957
350 | 0x1AB0..=0x1AFF
351 | 0x1DC0..=0x1DFF
352 | 0x20D0..=0x20FF
353 | 0xFE00..=0xFE0F
354 | 0xFE20..=0xFE2F
355 | 0x1F3FB..=0x1F3FF
356 | 0xE0100..=0xE01EF
357 )
358}
359
360#[cfg(feature = "unicode")]
361fn script(character: char) -> Script {
362 let value = character as u32;
363 match value {
364 _ if is_extend(value) => Script::Inherited,
365 0x0041..=0x005A | 0x0061..=0x007A | 0x00C0..=0x024F | 0x1E00..=0x1EFF => Script::Latin,
366 0x0370..=0x03FF | 0x1F00..=0x1FFF => Script::Greek,
367 0x0400..=0x052F | 0x2DE0..=0x2DFF | 0xA640..=0xA69F => Script::Cyrillic,
368 0x0590..=0x05FF => Script::Hebrew,
369 0x0600..=0x06FF | 0x0750..=0x077F | 0x08A0..=0x08FF | 0xFB50..=0xFDFF | 0xFE70..=0xFEFF => {
370 Script::Arabic
371 }
372 0x0E00..=0x0E7F => Script::Thai,
373 0x0900..=0x097F | 0xA8E0..=0xA8FF => Script::Devanagari,
374 0x3040..=0x309F => Script::Hiragana,
375 0x30A0..=0x30FF | 0x31F0..=0x31FF => Script::Katakana,
376 0x1100..=0x11FF | 0x3130..=0x318F | 0xAC00..=0xD7AF => Script::Hangul,
377 0x3400..=0x4DBF | 0x4E00..=0x9FFF | 0xF900..=0xFAFF | 0x20000..=0x323AF => Script::Han,
378 0x0000..=0x0040
379 | 0x005B..=0x0060
380 | 0x007B..=0x00BF
381 | 0x2000..=0x206F
382 | 0x20A0..=0x20CF
383 | 0x2100..=0x214F
384 | 0x2190..=0x2BFF
385 | 0x1F000..=0x1FAFF => Script::Common,
386 _ => Script::Unknown,
387 }
388}
389
390#[cfg(feature = "unicode")]
391fn strong_script(grapheme: &str) -> Option<Script> {
392 grapheme
393 .chars()
394 .map(script)
395 .find(|script| !matches!(script, Script::Common | Script::Inherited | Script::Unknown))
396}
397
398fn is_space(character: char) -> bool {
399 matches!(
400 character,
401 ' ' | '\t' | '\u{00A0}' | '\u{2000}'..='\u{200A}' | '\u{3000}'
402 )
403}
404
405fn allows_wide_break(previous: char, next: char) -> bool {
406 !is_open_punctuation(previous)
407 && !is_close_punctuation(next)
408 && (is_wide(previous) || is_wide(next))
409}
410
411#[cfg(test)]
412mod tests {
413 use super::*;
414 use std::prelude::v1::*;
415
416 #[test]
417 fn grapheme_ranges_preserve_common_clusters() {
418 let text = "a\u{301}👩🚀🇨🇳";
419 let clusters = graphemes(text).collect::<Vec<_>>();
420
421 assert_eq!(clusters.len(), 3);
422 assert_eq!(clusters[0].text, "a\u{301}");
423 assert_eq!(clusters[1].text, "👩🚀");
424 assert_eq!(clusters[2].text, "🇨🇳");
425 }
426
427 #[test]
428 fn thai_marks_stay_with_their_base() {
429 let text = "กิ่";
430 let clusters = graphemes(text).collect::<Vec<_>>();
431
432 assert_eq!(clusters.len(), 1);
433 assert_eq!(clusters[0].text, text);
434 }
435
436 #[test]
437 fn devanagari_linkers_keep_conjuncts_in_one_cluster() {
438 let text = "क्षत्रिय";
439 let clusters = graphemes(text).collect::<Vec<_>>();
440
441 assert_eq!(clusters.len(), 3);
442 assert_eq!(clusters[0].text, "क्ष");
443 assert_eq!(clusters[1].text, "त्रि");
444 assert_eq!(clusters[2].text, "य");
445 }
446
447 #[cfg(feature = "unicode")]
448 #[test]
449 fn thai_has_a_stable_script_identity() {
450 assert_eq!(Script::of('ก'), Script::Thai);
451 assert_eq!(Script::Thai.iso15924_tag(), *b"Thai");
452
453 let runs = script_runs("กิ่ ภาษา").collect::<Vec<_>>();
454 assert_eq!(runs.len(), 1);
455 assert_eq!(runs[0].script, Script::Thai);
456 }
457
458 #[test]
459 fn line_breaks_preserve_words_and_cjk_punctuation() {
460 let latin = line_breaks("hello world").collect::<Vec<_>>();
461 let cjk = line_breaks("你好,世界").collect::<Vec<_>>();
462
463 assert!(latin.contains(&LineBreak {
464 offset: 6,
465 kind: LineBreakKind::Allowed,
466 }));
467 assert!(!cjk
468 .iter()
469 .any(|line_break| line_break.offset == "你好".len()));
470 assert_eq!(cjk.last().unwrap().kind, LineBreakKind::Mandatory);
471 }
472
473 #[test]
474 fn zero_width_space_is_an_explicit_break() {
475 let text = "ภาษา\u{200B}ไทย";
476 let breaks = line_breaks(text).collect::<Vec<_>>();
477
478 assert!(breaks.contains(&LineBreak {
479 offset: "ภาษา\u{200B}".len(),
480 kind: LineBreakKind::Allowed,
481 }));
482 }
483
484 #[test]
485 fn line_breaks_keep_bracket_edges_together() {
486 let text = "《文字》〉";
487 let offsets = line_breaks(text)
488 .filter(|line_break| line_break.kind == LineBreakKind::Allowed)
489 .map(|line_break| line_break.offset)
490 .collect::<Vec<_>>();
491
492 assert_eq!(offsets, ["《文".len()]);
493 }
494
495 #[cfg(feature = "unicode")]
496 #[test]
497 fn neutral_prefix_joins_the_first_strong_script() {
498 let text = "(العربية) Latin";
499 let runs = script_runs(text).collect::<Vec<_>>();
500
501 assert_eq!(runs.len(), 2);
502 assert_eq!(runs[0].script, Script::Arabic);
503 assert_eq!(&text[runs[0].text.clone()], "(العربية) ");
504 assert_eq!(runs[1].script, Script::Latin);
505 }
506}