1use core::iter::Peekable;
2use core::ops::Range;
3use core::str::CharIndices;
4
5use crate::properties::{is_close_punctuation, is_open_punctuation, is_wide};
6
7#[cfg(feature = "unicode")]
8#[derive(Clone, Copy, Debug, Eq, PartialEq)]
9pub enum Script {
10 Common,
11 Inherited,
12 Latin,
13 Greek,
14 Cyrillic,
15 Hebrew,
16 Arabic,
17 Thai,
18 Devanagari,
19 Han,
20 Hiragana,
21 Katakana,
22 Hangul,
23 Unknown,
24}
25
26#[cfg(feature = "unicode")]
27impl Script {
28 pub fn of(character: char) -> Self {
29 script(character)
30 }
31
32 pub const fn iso15924_tag(self) -> [u8; 4] {
33 match self {
34 Self::Common => *b"Zyyy",
35 Self::Inherited => *b"Zinh",
36 Self::Latin => *b"Latn",
37 Self::Greek => *b"Grek",
38 Self::Cyrillic => *b"Cyrl",
39 Self::Hebrew => *b"Hebr",
40 Self::Arabic => *b"Arab",
41 Self::Thai => *b"Thai",
42 Self::Devanagari => *b"Deva",
43 Self::Han => *b"Hani",
44 Self::Hiragana => *b"Hira",
45 Self::Katakana => *b"Kana",
46 Self::Hangul => *b"Hang",
47 Self::Unknown => *b"Zzzz",
48 }
49 }
50}
51
52#[derive(Clone, Debug, Eq, PartialEq)]
53pub struct Grapheme<'a> {
54 pub text: &'a str,
55 pub range: Range<usize>,
56}
57
58pub struct Graphemes<'a> {
59 text: &'a str,
60 cursor: usize,
61}
62
63pub fn graphemes(text: &str) -> Graphemes<'_> {
64 Graphemes { text, cursor: 0 }
65}
66
67impl<'a> Iterator for Graphemes<'a> {
68 type Item = Grapheme<'a>;
69
70 fn next(&mut self) -> Option<Self::Item> {
71 if self.cursor == self.text.len() {
72 return None;
73 }
74 let start = self.cursor;
75 let mut chars = self.text[start..].char_indices();
76 let (_, first) = chars.next()?;
77 let mut previous = grapheme_class(first);
78 let mut regional_count = usize::from(previous == GraphemeClass::Regional);
79 let mut end = start + first.len_utf8();
80
81 for (offset, character) in chars {
82 let next = grapheme_class(character);
83 if grapheme_break(previous, next, regional_count) {
84 break;
85 }
86 end = start + offset + character.len_utf8();
87 if next == GraphemeClass::Regional {
88 regional_count += 1;
89 } else if !matches!(next, GraphemeClass::Extend | GraphemeClass::Zwj) {
90 regional_count = 0;
91 }
92 previous = next;
93 }
94
95 self.cursor = end;
96 Some(Grapheme {
97 text: &self.text[start..end],
98 range: start..end,
99 })
100 }
101}
102
103#[derive(Clone, Copy, Debug, Eq, PartialEq)]
104pub enum LineBreakKind {
105 Allowed,
106 Mandatory,
107}
108
109#[derive(Clone, Copy, Debug, Eq, PartialEq)]
110pub struct LineBreak {
111 pub offset: usize,
112 pub kind: LineBreakKind,
113}
114
115#[cfg(feature = "shaping")]
117pub trait LineBreakProvider {
118 fn break_at(&self, text: &str, offset: usize) -> Option<LineBreakKind>;
120}
121
122pub struct LineBreaks<'a> {
123 text: &'a str,
124 chars: Peekable<CharIndices<'a>>,
125 emitted_end: bool,
126}
127
128pub fn line_breaks(text: &str) -> LineBreaks<'_> {
129 LineBreaks {
130 text,
131 chars: text.char_indices().peekable(),
132 emitted_end: false,
133 }
134}
135
136impl Iterator for LineBreaks<'_> {
137 type Item = LineBreak;
138
139 fn next(&mut self) -> Option<Self::Item> {
140 while let Some((start, character)) = self.chars.next() {
141 let offset = start + character.len_utf8();
142 let next = self.chars.peek().map(|(_, next)| *next);
143 let kind = if character == '\n' || character == '\r' && next != Some('\n') {
144 Some(LineBreakKind::Mandatory)
145 } else if character == '\u{200B}' {
146 Some(LineBreakKind::Allowed)
147 } else if is_space(character) {
148 (!next.is_some_and(is_space)).then_some(LineBreakKind::Allowed)
149 } else if matches!(character, '-' | '/' | '\u{2010}' | '\u{2013}')
150 || next.is_some_and(|next| allows_wide_break(character, next))
151 {
152 Some(LineBreakKind::Allowed)
153 } else {
154 None
155 };
156 if let Some(kind) = kind {
157 if offset == self.text.len() {
158 self.emitted_end = true;
159 return Some(LineBreak {
160 offset,
161 kind: LineBreakKind::Mandatory,
162 });
163 }
164 return Some(LineBreak { offset, kind });
165 }
166 }
167 if self.emitted_end {
168 None
169 } else {
170 self.emitted_end = true;
171 Some(LineBreak {
172 offset: self.text.len(),
173 kind: LineBreakKind::Mandatory,
174 })
175 }
176 }
177}
178
179#[cfg(feature = "unicode")]
180#[derive(Clone, Debug, Eq, PartialEq)]
181pub struct ScriptRun {
182 pub text: Range<usize>,
183 pub script: Script,
184}
185
186#[cfg(feature = "unicode")]
187pub fn script_runs(text: &str) -> ScriptRuns<'_> {
188 ScriptRuns {
189 text,
190 graphemes: graphemes(text),
191 pending: None,
192 }
193}
194
195#[cfg(feature = "unicode")]
196pub struct ScriptRuns<'a> {
197 text: &'a str,
198 graphemes: Graphemes<'a>,
199 pending: Option<Grapheme<'a>>,
200}
201
202#[cfg(feature = "unicode")]
203impl Iterator for ScriptRuns<'_> {
204 type Item = ScriptRun;
205
206 fn next(&mut self) -> Option<Self::Item> {
207 let first = self.pending.take().or_else(|| self.graphemes.next())?;
208 let start = first.range.start;
209 let mut run_script = strong_script(first.text);
210
211 for cluster in self.graphemes.by_ref() {
212 let next_script = strong_script(cluster.text);
213 if run_script.is_none() {
214 run_script = next_script;
215 } else if next_script.is_some() && next_script != run_script {
216 self.pending = Some(cluster);
217 break;
218 }
219 }
220
221 let end = self
222 .pending
223 .as_ref()
224 .map_or(self.text.len(), |cluster| cluster.range.start);
225 Some(ScriptRun {
226 text: start..end,
227 script: run_script.unwrap_or(Script::Common),
228 })
229 }
230}
231
232#[derive(Clone, Copy, Eq, PartialEq)]
233enum GraphemeClass {
234 Other,
235 Cr,
236 Lf,
237 Control,
238 Extend,
239 Zwj,
240 Regional,
241 Prepend,
242 SpacingMark,
243 L,
244 V,
245 T,
246 Lv,
247 Lvt,
248 ExtendedPictographic,
249}
250
251fn grapheme_break(previous: GraphemeClass, next: GraphemeClass, regional_count: usize) -> bool {
252 if previous == GraphemeClass::Cr && next == GraphemeClass::Lf {
253 return false;
254 }
255 if matches!(
256 previous,
257 GraphemeClass::Cr | GraphemeClass::Lf | GraphemeClass::Control
258 ) || matches!(
259 next,
260 GraphemeClass::Cr | GraphemeClass::Lf | GraphemeClass::Control
261 ) {
262 return true;
263 }
264 if matches!(
265 (previous, next),
266 (
267 GraphemeClass::L,
268 GraphemeClass::L | GraphemeClass::V | GraphemeClass::Lv | GraphemeClass::Lvt
269 ) | (
270 GraphemeClass::Lv | GraphemeClass::V,
271 GraphemeClass::V | GraphemeClass::T
272 ) | (GraphemeClass::Lvt | GraphemeClass::T, GraphemeClass::T)
273 ) {
274 return false;
275 }
276 if matches!(
277 next,
278 GraphemeClass::Extend | GraphemeClass::Zwj | GraphemeClass::SpacingMark
279 ) || previous == GraphemeClass::Prepend
280 || previous == GraphemeClass::Zwj && next == GraphemeClass::ExtendedPictographic
281 {
282 return false;
283 }
284 !(previous == GraphemeClass::Regional
285 && next == GraphemeClass::Regional
286 && regional_count % 2 == 1)
287}
288
289fn grapheme_class(character: char) -> GraphemeClass {
290 let value = character as u32;
291 match value {
292 0x000D => GraphemeClass::Cr,
293 0x000A => GraphemeClass::Lf,
294 0x0000..=0x001F | 0x007F..=0x009F => GraphemeClass::Control,
295 0x200D => GraphemeClass::Zwj,
296 0x1F1E6..=0x1F1FF => GraphemeClass::Regional,
297 0x0600..=0x0605 | 0x06DD | 0x070F | 0x08E2 => GraphemeClass::Prepend,
298 0x0903 | 0x093B..=0x0940 | 0x0949..=0x094C | 0x0982..=0x0983 => GraphemeClass::SpacingMark,
299 0x1100..=0x115F | 0xA960..=0xA97C => GraphemeClass::L,
300 0x1160..=0x11A7 | 0xD7B0..=0xD7C6 => GraphemeClass::V,
301 0x11A8..=0x11FF | 0xD7CB..=0xD7FB => GraphemeClass::T,
302 0xAC00..=0xD7A3 if (value - 0xAC00) % 28 == 0 => GraphemeClass::Lv,
303 0xAC00..=0xD7A3 => GraphemeClass::Lvt,
304 0x1F000..=0x1FAFF | 0x2600..=0x27BF => GraphemeClass::ExtendedPictographic,
305 _ if is_extend(value) => GraphemeClass::Extend,
306 _ => GraphemeClass::Other,
307 }
308}
309
310fn is_extend(value: u32) -> bool {
311 matches!(
312 value,
313 0x0300..=0x036F
314 | 0x0483..=0x0489
315 | 0x0591..=0x05BD
316 | 0x05BF
317 | 0x05C1..=0x05C2
318 | 0x0610..=0x061A
319 | 0x064B..=0x065F
320 | 0x0670
321 | 0x06D6..=0x06ED
322 | 0x0E31
323 | 0x0E34..=0x0E3A
324 | 0x0E47..=0x0E4E
325 | 0x0900..=0x0902
326 | 0x093A
327 | 0x093C
328 | 0x0941..=0x0948
329 | 0x094D
330 | 0x0951..=0x0957
331 | 0x1AB0..=0x1AFF
332 | 0x1DC0..=0x1DFF
333 | 0x20D0..=0x20FF
334 | 0xFE00..=0xFE0F
335 | 0xFE20..=0xFE2F
336 | 0x1F3FB..=0x1F3FF
337 | 0xE0100..=0xE01EF
338 )
339}
340
341#[cfg(feature = "unicode")]
342fn script(character: char) -> Script {
343 let value = character as u32;
344 match value {
345 _ if is_extend(value) => Script::Inherited,
346 0x0041..=0x005A | 0x0061..=0x007A | 0x00C0..=0x024F | 0x1E00..=0x1EFF => Script::Latin,
347 0x0370..=0x03FF | 0x1F00..=0x1FFF => Script::Greek,
348 0x0400..=0x052F | 0x2DE0..=0x2DFF | 0xA640..=0xA69F => Script::Cyrillic,
349 0x0590..=0x05FF => Script::Hebrew,
350 0x0600..=0x06FF | 0x0750..=0x077F | 0x08A0..=0x08FF | 0xFB50..=0xFDFF | 0xFE70..=0xFEFF => {
351 Script::Arabic
352 }
353 0x0E00..=0x0E7F => Script::Thai,
354 0x0900..=0x097F | 0xA8E0..=0xA8FF => Script::Devanagari,
355 0x3040..=0x309F => Script::Hiragana,
356 0x30A0..=0x30FF | 0x31F0..=0x31FF => Script::Katakana,
357 0x1100..=0x11FF | 0x3130..=0x318F | 0xAC00..=0xD7AF => Script::Hangul,
358 0x3400..=0x4DBF | 0x4E00..=0x9FFF | 0xF900..=0xFAFF | 0x20000..=0x323AF => Script::Han,
359 0x0000..=0x0040
360 | 0x005B..=0x0060
361 | 0x007B..=0x00BF
362 | 0x2000..=0x206F
363 | 0x20A0..=0x20CF
364 | 0x2100..=0x214F
365 | 0x2190..=0x2BFF
366 | 0x1F000..=0x1FAFF => Script::Common,
367 _ => Script::Unknown,
368 }
369}
370
371#[cfg(feature = "unicode")]
372fn strong_script(grapheme: &str) -> Option<Script> {
373 grapheme
374 .chars()
375 .map(script)
376 .find(|script| !matches!(script, Script::Common | Script::Inherited | Script::Unknown))
377}
378
379fn is_space(character: char) -> bool {
380 matches!(
381 character,
382 ' ' | '\t' | '\u{00A0}' | '\u{2000}'..='\u{200A}' | '\u{3000}'
383 )
384}
385
386fn allows_wide_break(previous: char, next: char) -> bool {
387 !is_open_punctuation(previous)
388 && !is_close_punctuation(next)
389 && (is_wide(previous) || is_wide(next))
390}
391
392#[cfg(test)]
393mod tests {
394 use super::*;
395 use std::prelude::v1::*;
396
397 #[test]
398 fn grapheme_ranges_preserve_common_clusters() {
399 let text = "a\u{301}👩🚀🇨🇳";
400 let clusters = graphemes(text).collect::<Vec<_>>();
401
402 assert_eq!(clusters.len(), 3);
403 assert_eq!(clusters[0].text, "a\u{301}");
404 assert_eq!(clusters[1].text, "👩🚀");
405 assert_eq!(clusters[2].text, "🇨🇳");
406 }
407
408 #[test]
409 fn thai_marks_stay_with_their_base() {
410 let text = "กิ่";
411 let clusters = graphemes(text).collect::<Vec<_>>();
412
413 assert_eq!(clusters.len(), 1);
414 assert_eq!(clusters[0].text, text);
415 }
416
417 #[cfg(feature = "unicode")]
418 #[test]
419 fn thai_has_a_stable_script_identity() {
420 assert_eq!(Script::of('ก'), Script::Thai);
421 assert_eq!(Script::Thai.iso15924_tag(), *b"Thai");
422
423 let runs = script_runs("กิ่ ภาษา").collect::<Vec<_>>();
424 assert_eq!(runs.len(), 1);
425 assert_eq!(runs[0].script, Script::Thai);
426 }
427
428 #[test]
429 fn line_breaks_preserve_words_and_cjk_punctuation() {
430 let latin = line_breaks("hello world").collect::<Vec<_>>();
431 let cjk = line_breaks("你好,世界").collect::<Vec<_>>();
432
433 assert!(latin.contains(&LineBreak {
434 offset: 6,
435 kind: LineBreakKind::Allowed,
436 }));
437 assert!(!cjk
438 .iter()
439 .any(|line_break| line_break.offset == "你好".len()));
440 assert_eq!(cjk.last().unwrap().kind, LineBreakKind::Mandatory);
441 }
442
443 #[test]
444 fn zero_width_space_is_an_explicit_break() {
445 let text = "ภาษา\u{200B}ไทย";
446 let breaks = line_breaks(text).collect::<Vec<_>>();
447
448 assert!(breaks.contains(&LineBreak {
449 offset: "ภาษา\u{200B}".len(),
450 kind: LineBreakKind::Allowed,
451 }));
452 }
453
454 #[test]
455 fn line_breaks_keep_bracket_edges_together() {
456 let text = "《文字》〉";
457 let offsets = line_breaks(text)
458 .filter(|line_break| line_break.kind == LineBreakKind::Allowed)
459 .map(|line_break| line_break.offset)
460 .collect::<Vec<_>>();
461
462 assert_eq!(offsets, ["《文".len()]);
463 }
464
465 #[cfg(feature = "unicode")]
466 #[test]
467 fn neutral_prefix_joins_the_first_strong_script() {
468 let text = "(العربية) Latin";
469 let runs = script_runs(text).collect::<Vec<_>>();
470
471 assert_eq!(runs.len(), 2);
472 assert_eq!(runs[0].script, Script::Arabic);
473 assert_eq!(&text[runs[0].text.clone()], "(العربية) ");
474 assert_eq!(runs[1].script, Script::Latin);
475 }
476}