rich/cells.rs
1//! Terminal cell measurement.
2//!
3//! Port of upstream `rich/cells.py`. The width data is upstream's own, vendored
4//! into [`cell_widths`](crate::cell_widths) by `scripts/gen_cell_widths.py` —
5//! all 21 Unicode versions `rich._unicode_data` ships, selected at runtime by
6//! `UNICODE_VERSION` exactly as upstream's `load()` does.
7//!
8//! We used to delegate to the `unicode-width` crate on the premise that both
9//! implement the same East Asian Width rules. They disagree on 348 code points,
10//! and a `unicode-width` build can only ever be one Unicode version anyway.
11
12use std::sync::OnceLock;
13
14use crate::cell_widths::{NARROW_TO_WIDE, TABLES, VERSIONS};
15
16/// Parse a Unicode version string into `(major, minor, patch)`, padding missing
17/// components with zero and ignoring any beyond the third. Port of
18/// `rich._unicode_data._parse_version`; `None` stands in for its `ValueError`.
19fn parse_version(version: &str) -> Option<(i64, i64, i64)> {
20 let mut parts = [0i64; 3];
21 for (index, part) in version.split('.').enumerate() {
22 // `.trim()` because Python's `int()` accepts surrounding whitespace and
23 // this is a port of `map(int, version.split("."))`.
24 let value: i64 = part.trim().parse().ok()?;
25 if index < 3 {
26 parts[index] = value;
27 }
28 }
29 Some((parts[0], parts[1], parts[2]))
30}
31
32/// Index into [`VERSIONS`] of the table upstream would load for `requested`.
33/// Port of the version selection in `rich._unicode_data.load`.
34///
35/// Anything unparsable — including the literal `"latest"` — takes the newest
36/// table, and a version upstream does not ship falls back to the newest one
37/// *not newer* than it (`bisect_left` minus one, clamped at the oldest).
38fn resolve_version(requested: &str) -> usize {
39 let latest = VERSIONS.len() - 1;
40 let Some(wanted) = parse_version(requested) else {
41 return latest;
42 };
43 let shipped = || {
44 VERSIONS
45 .iter()
46 .map(|version| parse_version(version).expect("shipped versions parse"))
47 };
48 // An exact match wins: upstream checks the reformatted `major.minor.patch`
49 // against its version set before doing anything cleverer, so `"9.0"` finds
50 // `"9.0.0"` rather than bisecting to `"8.0.0"`.
51 if let Some(index) = shipped().position(|version| version == wanted) {
52 return index;
53 }
54 shipped()
55 .position(|version| version >= wanted)
56 .unwrap_or(VERSIONS.len())
57 .saturating_sub(1)
58}
59
60/// The width table this process measures with, chosen by `UNICODE_VERSION`.
61///
62/// Upstream's `_unicode_data.load("auto")` is `@cache`d, so the variable is read
63/// once per process however many strings get measured; the `OnceLock` matches
64/// that, and keeps [`char_cell_width`] free of a per-character env lookup.
65fn cell_table() -> &'static [(u32, u32, u8)] {
66 static TABLE: OnceLock<&'static [(u32, u32, u8)]> = OnceLock::new();
67 TABLE.get_or_init(|| {
68 // Unset behaves as `"latest"`, exactly as upstream's
69 // `os.environ.get("UNICODE_VERSION", "latest")` does.
70 let requested = std::env::var("UNICODE_VERSION").unwrap_or_else(|_| "latest".to_string());
71 table_for(&requested)
72 })
73}
74
75/// The table for `requested`, ignoring the environment.
76fn table_for(requested: &str) -> &'static [(u32, u32, u8)] {
77 TABLES[resolve_version(requested)]
78}
79
80/// The number of terminal cells `text` occupies. Port of `cell_len`.
81pub fn cell_len(text: &str) -> usize {
82 if let Some(width) = unjoined_cell_len(text) {
83 return width;
84 }
85
86 // Port of upstream `cells._cell_len`'s cluster pass. Two rules matter:
87 // a ZWJ consumes the character after it (so a family emoji measures as one
88 // emoji, not as its parts), and a variation selector promotes the preceding
89 // narrow character to two cells.
90 let chars: Vec<char> = text.chars().collect();
91 let mut total = 0usize;
92 let mut last_measured: Option<char> = None;
93 let mut index = 0usize;
94 while index < chars.len() {
95 let c = chars[index];
96 if c == ZERO_WIDTH_JOINER {
97 index += 1; // skip the joined character entirely
98 } else if c == VARIATION_SELECTOR_16 {
99 if let Some(previous) = last_measured.take() {
100 if NARROW_TO_WIDE.contains(&previous) {
101 total += 1;
102 }
103 }
104 } else {
105 let width = char_cell_width(c);
106 if width > 0 {
107 last_measured = Some(c);
108 total += width;
109 }
110 }
111 index += 1;
112 }
113 total
114}
115
116/// [`cell_len`] of text with no zero-width joiner or variation selector
117/// 16, which is the sum of its characters' widths; `None` when it has
118/// either, so only the cluster pass can measure it. Upstream's own fast
119/// path, taken in one pass: it measures as it looks for the two
120/// characters, rather than searching for each and then measuring
121/// (docs/PORTING.md, "Performance notes").
122///
123/// ASCII text is one cell for each printable character, none for the
124/// controls (NUL and DEL included), in every table. Upstream's shortcut is
125/// `_is_single_cell_widths`, `len(text)` for text of single-cell
126/// characters; this one also counts past controls without the table
127/// (pinned by `ascii_cell_len_matches_the_table`).
128#[inline]
129pub(crate) fn unjoined_cell_len(text: &str) -> Option<usize> {
130 if text.is_ascii() {
131 return Some(text.bytes().filter(|b| (b' '..=b'~').contains(b)).count());
132 }
133 let mut total = 0;
134 for c in text.chars() {
135 if c == ZERO_WIDTH_JOINER || c == VARIATION_SELECTOR_16 {
136 return None;
137 }
138 total += char_cell_width(c);
139 }
140 Some(total)
141}
142
143/// Zero-width joiner: binds emoji into a single cluster.
144const ZERO_WIDTH_JOINER: char = '\u{200d}';
145/// Variation selector 16: renders the preceding character as emoji (2 cells).
146const VARIATION_SELECTOR_16: char = '\u{fe0f}';
147
148/// Upstream's `_SINGLE_CELL_UNICODE_RANGES`: code points it is willing to assume
149/// occupy exactly one cell each, so a string built only from them can be sliced
150/// by code point without consulting the width table at all.
151const SINGLE_CELL_RANGES: [(u32, u32); 6] = [
152 (0x20, 0x7E), // Latin (excluding non-printable)
153 (0xA0, 0xAC), // NB: 0xAD (soft hyphen) is deliberately excluded
154 (0xAE, 0x2FF), //
155 (0x370, 0x482), // Greek / Cyrillic
156 (0x2500, 0x25FC), // Box drawing, box elements, geometric shapes
157 (0x2800, 0x28FF), // Braille
158];
159
160/// Whether every character of `text` is one of upstream's assumed-single-cell
161/// code points. Port of `cells._is_single_cell_widths`.
162///
163/// This is not merely an optimisation: [`chop_cells`] and [`set_cell_size`]
164/// take a genuinely different code path when it holds, slicing by code point
165/// rather than by grapheme, and upstream's output follows whichever path it
166/// took. Note that a tab is *not* single-cell (it is zero cells wide), so a
167/// tabbed string always takes the grapheme path.
168fn is_single_cell_widths(text: &str) -> bool {
169 text.chars().all(|c| {
170 let codepoint = c as u32;
171 SINGLE_CELL_RANGES
172 .iter()
173 .any(|(start, end)| (*start..=*end).contains(&codepoint))
174 })
175}
176
177/// Divide `text` into spans that each cover exactly one grapheme, and return the
178/// cell length of the whole string alongside. Port of `cells.split_graphemes`.
179///
180/// Each span is `(start_byte, end_byte, cell_length)`; upstream indexes by code
181/// point, we index by byte so the spans slice a `&str` directly. The spans cover
182/// every byte with no gaps, and a span's cell length may be zero (a lone joiner,
183/// a control code).
184///
185/// The two rules that make this more than a `char` iterator: a zero-width joiner
186/// swallows the character after it, so a family emoji is *one* grapheme; and
187/// U+FE0F promotes the preceding narrow character to two cells without starting
188/// a new grapheme. Zero-width characters attach to the grapheme before them.
189pub fn split_graphemes(text: &str) -> (Vec<(usize, usize, usize)>, usize) {
190 // One pass over the characters, each span's end taken from the character
191 // that ends it, rather than collecting the characters first
192 // (docs/PORTING.md, "Performance notes"). A test,
193 // `split_graphemes_matches_the_indexed_scan`, holds it to the first port.
194 let mut spans: Vec<(usize, usize, usize)> = Vec::new();
195 let mut total_width = 0usize;
196 let mut last_measured: Option<char> = None;
197 let mut chars = text.char_indices();
198
199 while let Some((start, character)) = chars.next() {
200 let end = start + character.len_utf8();
201 if character == ZERO_WIDTH_JOINER || character == VARIATION_SELECTOR_16 {
202 let Some(last) = spans.last_mut() else {
203 // A joiner or selector opening the string joins nothing. It is
204 // nonsense, but upstream handles it, so we must too.
205 spans.push((start, end, 0));
206 continue;
207 };
208 if character == ZERO_WIDTH_JOINER {
209 // Consume the joiner *and* whatever it joins — unless it is the
210 // last character, with nothing left to join.
211 last.1 = chars
212 .next()
213 .map_or(end, |(joined, c)| joined + c.len_utf8());
214 } else {
215 if last_measured.is_some_and(|previous| NARROW_TO_WIDE.contains(&previous)) {
216 last_measured = None;
217 last.2 += 1;
218 total_width += 1;
219 }
220 last.1 = end;
221 }
222 continue;
223 }
224
225 let width = char_cell_width(character);
226 if width > 0 {
227 last_measured = Some(character);
228 total_width += width;
229 spans.push((start, end, width));
230 } else if let Some(last) = spans.last_mut() {
231 // Zero-width characters belong to the grapheme before them.
232 last.1 = end;
233 } else {
234 spans.push((start, end, 0));
235 }
236 }
237
238 (spans, total_width)
239}
240
241/// Split `text` at `cell_position` cells. Port of `cells._split_text`.
242///
243/// A split that lands *inside* a double-width grapheme cannot be represented, so
244/// upstream replaces that grapheme with a space on each side of the cut — which
245/// is why cropping `"❤️❤️"` to one cell yields `" "`, not half a heart.
246fn split_text_inner(text: &str, cell_position: usize) -> (String, String) {
247 if cell_position == 0 {
248 return (String::new(), text.to_string());
249 }
250 let (spans, cell_length) = split_graphemes(text);
251 if cell_length == 0 || spans.is_empty() {
252 // Upstream divides by `cell_length` here and would raise; there is
253 // nothing measurable to cut, so the whole string stays on the left.
254 return (text.to_string(), String::new());
255 }
256
257 // Upstream's initial guess: assume the graphemes are evenly sized, then walk
258 // to the true boundary. `as usize` truncates, matching Python's `int()`.
259 let mut offset = ((cell_position as f64 / cell_length as f64) * spans.len() as f64) as usize;
260 offset = offset.min(spans.len());
261 let mut left_size: usize = spans[..offset].iter().map(|span| span.2).sum();
262
263 loop {
264 if left_size == cell_position {
265 let Some(&(split, _, _)) = spans.get(offset) else {
266 return (text.to_string(), String::new());
267 };
268 return (text[..split].to_string(), text[split..].to_string());
269 }
270 if left_size < cell_position {
271 let Some(&(start, end, cell_size)) = spans.get(offset) else {
272 return (text.to_string(), String::new());
273 };
274 if left_size + cell_size > cell_position {
275 return (format!("{} ", &text[..start]), format!(" {}", &text[end..]));
276 }
277 offset += 1;
278 left_size += cell_size;
279 } else {
280 let Some(&(start, end, cell_size)) =
281 offset.checked_sub(1).and_then(|index| spans.get(index))
282 else {
283 return (String::new(), text.to_string());
284 };
285 if left_size - cell_size < cell_position {
286 return (format!("{} ", &text[..start]), format!(" {}", &text[end..]));
287 }
288 offset -= 1;
289 left_size -= cell_size;
290 }
291 }
292}
293
294/// Split `text` at `cell_position` cells. Port of `cells.split_text`.
295pub fn split_text(text: &str, cell_position: usize) -> (String, String) {
296 if is_single_cell_widths(text) {
297 let split = char_boundary(text, cell_position);
298 return (text[..split].to_string(), text[split..].to_string());
299 }
300 split_text_inner(text, cell_position)
301}
302
303/// The byte offset of code point `index`, or the end of `text`.
304fn char_boundary(text: &str, index: usize) -> usize {
305 text.char_indices()
306 .nth(index)
307 .map_or(text.len(), |(offset, _)| offset)
308}
309
310/// The cell width of a single character (control chars count as 0).
311///
312/// Uses upstream's vendored table rather than the `unicode-width` crate: the two
313/// disagree on 348 code points (spacing marks, format characters, modifier
314/// symbols), and every disagreement misaligns a table, panel or wrap point.
315/// Port of `cells.get_character_cell_size`.
316#[inline]
317pub fn char_cell_width(c: char) -> usize {
318 if let Some(width) = common_width(c) {
319 return width;
320 }
321 width_in(cell_table(), c)
322}
323
324/// The width of the characters most text is made of, which every table
325/// upstream ships agrees on, without the table search: printable ASCII,
326/// Latin-1, Latin Extended, IPA, Greek and Cyrillic letters, box drawing,
327/// blocks and Braille (one cell), CJK ideographs and Hangul syllables
328/// (two). `None` for anything
329/// else, which [`width_in`] looks up. A fast path beside
330/// `get_character_cell_size`'s lookup (docs/PORTING.md, "Performance
331/// notes"); `common_widths_match_every_table` checks every character it
332/// answers against all 21 tables.
333#[inline]
334fn common_width(c: char) -> Option<usize> {
335 match c {
336 ' '..='~'
337 | '\u{a0}'..='\u{2ff}'
338 | '\u{370}'..='\u{482}'
339 | '\u{2500}'..='\u{259f}'
340 | '\u{2800}'..='\u{28ff}' => Some(1),
341 // Unicode 4.1 ends the ideographs at U+9FBB; later ones are wide
342 // only in the newer tables.
343 '\u{4e00}'..='\u{9fbb}' | '\u{ac00}'..='\u{d7a3}' => Some(2),
344 _ => None,
345 }
346}
347
348/// [`char_cell_width`] against an explicitly chosen table, so the version
349/// selection can be exercised without a process-wide environment variable.
350fn width_in(table: &[(u32, u32, u8)], c: char) -> usize {
351 let codepoint = c as u32;
352 if (codepoint > 0 && codepoint < 32) || (0x7F..0xA0).contains(&codepoint) {
353 return 0;
354 }
355 // Beyond the table's last range upstream assumes a single cell.
356 if codepoint > table[table.len() - 1].1 {
357 return 1;
358 }
359 let mut lower = 0usize;
360 let mut upper = table.len() - 1;
361 while lower <= upper {
362 let mid = (lower + upper) / 2;
363 let (start, end, width) = table[mid];
364 if codepoint > end {
365 lower = mid + 1;
366 } else if codepoint < start {
367 if mid == 0 {
368 break;
369 }
370 upper = mid - 1;
371 } else {
372 return width as usize;
373 }
374 }
375 1
376}
377
378/// Split `text` into chunks, each at most `width` cells wide. Port of
379/// `cells.chop_cells`. Used to fold over-long words during wrapping.
380///
381/// The fold is **grapheme**-aware, because a code-point fold cannot see that
382/// `"❤️"` is two cells: measured per code point the heart is one cell and the
383/// variation selector is zero, so twenty of them fit "within" thirty cells and
384/// the row silently renders forty cells wide, punching a hole through whatever
385/// border was drawn around it. Upstream splits on [`split_graphemes`]; so do we.
386///
387/// Two upstream quirks are preserved. A leading grapheme *wider* than `width`
388/// yields an empty leading chunk (`chop_cells("宽", 1) == ["", "宽"]`), and a
389/// trailing run that measures zero cells is dropped entirely
390/// (`chop_cells("\n", 4) == []`).
391pub fn chop_cells(text: &str, width: usize) -> Vec<String> {
392 if width == 0 {
393 // Upstream raises `ValueError` here (its slice step is `width`). Nothing
394 // in the port can usefully raise, and callers guard on zero width, so
395 // hand the text back whole rather than looping forever.
396 return vec![text.to_string()];
397 }
398 if is_single_cell_widths(text) {
399 // Upstream's fast path slices by code point, `width` at a time.
400 let chars: Vec<char> = text.chars().collect();
401 return chars
402 .chunks(width)
403 .map(|chunk| chunk.iter().collect())
404 .collect();
405 }
406
407 let (spans, _) = split_graphemes(text);
408 let mut lines: Vec<String> = Vec::new();
409 let mut line_size = 0usize;
410 let mut line_offset = 0usize;
411 for (start, _end, cell_size) in spans {
412 if line_size + cell_size > width {
413 lines.push(text[line_offset..start].to_string());
414 line_offset = start;
415 line_size = 0;
416 }
417 line_size += cell_size;
418 }
419 if line_size > 0 {
420 lines.push(text[line_offset..].to_string());
421 }
422 lines
423}
424
425/// Crop `text` to at most `width` cells, never padding. Unlike [`set_cell_size`]
426/// this leaves shorter text unchanged. Mirrors `Text.truncate` at the cell level.
427pub fn truncate(text: &str, width: usize) -> String {
428 if cell_len(text) <= width {
429 text.to_string()
430 } else {
431 set_cell_size(text, width)
432 }
433}
434
435/// Truncate or right-pad `text` (with spaces) so it occupies exactly `total`
436/// cells. Port of `set_cell_size`.
437///
438/// Cropping goes through [`split_text`]'s grapheme walk, so a cut landing inside
439/// a two-cell grapheme becomes a space rather than a half-rendered glyph — the
440/// difference between upstream's `" "` and a bare `"❤"` that the terminal still
441/// draws two cells wide.
442pub fn set_cell_size(text: &str, total: usize) -> String {
443 if is_single_cell_widths(text) {
444 let size = text.chars().count();
445 if size < total {
446 let mut padded = text.to_string();
447 padded.extend(std::iter::repeat_n(' ', total - size));
448 return padded;
449 }
450 return text[..char_boundary(text, total)].to_string();
451 }
452 if total == 0 {
453 return String::new();
454 }
455 let cell_size = cell_len(text);
456 if cell_size == total {
457 return text.to_string();
458 }
459 if cell_size < total {
460 let mut padded = text.to_string();
461 padded.extend(std::iter::repeat_n(' ', total - cell_size));
462 return padded;
463 }
464 split_text_inner(text, total).0
465}
466
467#[cfg(test)]
468mod tests {
469 use super::*;
470
471 /// The one-pass `split_graphemes` splits as the first port, which
472 /// indexed a collected list of characters, does.
473 #[test]
474 fn split_graphemes_matches_the_indexed_scan() {
475 fn reference(text: &str) -> (Vec<(usize, usize, usize)>, usize) {
476 let chars: Vec<(usize, char)> = text.char_indices().collect();
477 let count = chars.len();
478 // The byte offset of code point `index`, with `count` meaning "the end".
479 let byte_at = |index: usize| chars.get(index).map_or(text.len(), |(offset, _)| *offset);
480
481 let mut spans: Vec<(usize, usize, usize)> = Vec::new();
482 let mut total_width = 0usize;
483 let mut last_measured: Option<char> = None;
484 let mut index = 0usize;
485
486 while index < count {
487 let character = chars[index].1;
488 if character == ZERO_WIDTH_JOINER || character == VARIATION_SELECTOR_16 {
489 let Some(last) = spans.last_mut() else {
490 // A joiner or selector opening the string joins nothing. It is
491 // nonsense, but upstream handles it, so we must too.
492 let start = byte_at(index);
493 index += 1;
494 spans.push((start, byte_at(index), 0));
495 continue;
496 };
497 if character == ZERO_WIDTH_JOINER {
498 // Consume the joiner *and* whatever it joins — unless it is the
499 // last character, with nothing left to join.
500 index += if index < count - 1 { 2 } else { 1 };
501 last.1 = byte_at(index);
502 } else {
503 index += 1;
504 if last_measured.is_some_and(|previous| NARROW_TO_WIDE.contains(&previous))
505 {
506 last_measured = None;
507 last.2 += 1;
508 total_width += 1;
509 }
510 last.1 = byte_at(index);
511 }
512 continue;
513 }
514
515 let start = byte_at(index);
516 let width = char_cell_width(character);
517 index += 1;
518 if width > 0 {
519 last_measured = Some(character);
520 total_width += width;
521 spans.push((start, byte_at(index), width));
522 } else if let Some(last) = spans.last_mut() {
523 // Zero-width characters belong to the grapheme before them.
524 last.1 = byte_at(index);
525 } else {
526 spans.push((start, byte_at(index), 0));
527 }
528 }
529
530 (spans, total_width)
531 }
532 const ALPHABET: [char; 10] = [
533 'a',
534 '\u{200d}',
535 '\u{fe0f}',
536 '\u{301}',
537 '界',
538 '\u{2764}',
539 '\u{1f468}',
540 ' ',
541 '\t',
542 'é',
543 ];
544 let mut texts = vec![String::new()];
545 for _ in 0..5 {
546 texts = texts
547 .iter()
548 .flat_map(|text| ALPHABET.iter().map(move |c| format!("{text}{c}")))
549 .collect();
550 for text in &texts {
551 assert_eq!(split_graphemes(text), reference(text), "{text:?}");
552 }
553 }
554 }
555
556 /// The fast path answers every character it covers as every shipped
557 /// table does, so it holds whatever `UNICODE_VERSION` picks.
558 #[test]
559 fn common_widths_match_every_table() {
560 let mut covered = 0;
561 for c in (0..=0x10ffffu32).filter_map(char::from_u32) {
562 if let Some(width) = common_width(c) {
563 covered += 1;
564 for (table, version) in TABLES.iter().zip(VERSIONS) {
565 assert_eq!(width, width_in(table, c), "{c:?} in Unicode {version}");
566 }
567 }
568 }
569 assert!(covered > 30_000, "{covered}");
570 }
571
572 /// ASCII text measures as the sum of its characters' table widths,
573 /// controls included, in every table.
574 #[test]
575 fn ascii_cell_len_matches_the_table() {
576 for table in TABLES {
577 for byte in 0..=0x7fu8 {
578 let c = char::from(byte);
579 let expected = width_in(table, c);
580 assert_eq!(cell_len(&c.to_string()), expected, "{c:?}");
581 }
582 }
583 let text = "a\tb\u{0}c\u{7f} d\ne\u{1b}[1m";
584 let expected: usize = text.chars().map(|c| width_in(cell_table(), c)).sum();
585 assert_eq!(cell_len(text), expected);
586 }
587
588 #[test]
589 fn ascii_len() {
590 assert_eq!(cell_len("hello"), 5);
591 }
592
593 #[test]
594 fn wide_chars_count_double() {
595 assert_eq!(cell_len("宽"), 2);
596 }
597
598 #[test]
599 fn set_size_pads_and_truncates() {
600 assert_eq!(set_cell_size("hi", 5), "hi ");
601 assert_eq!(set_cell_size("hello", 3), "hel");
602 }
603
604 #[test]
605 fn chop_ascii_and_wide() {
606 // Matches real rich 15.0.0 `chop_cells`.
607 assert_eq!(chop_cells("abcdefghij", 4), vec!["abcd", "efgh", "ij"]);
608 // Each wide char (2 cells) gets its own chunk at width 3.
609 assert_eq!(chop_cells("宽宽宽宽", 3), vec!["宽", "宽", "宽", "宽"]);
610 }
611
612 #[test]
613 fn chop_char_wider_than_width_emits_empty_leading_chunk() {
614 // Upstream quirk: folding a 2-cell CJK char to width 1 yields an empty
615 // leading chunk. Captured from real rich 15.0.0 `chop_cells`:
616 // chop_cells("宽宽", 1) == ["", "宽", "宽"]
617 // chop_cells("宽", 1) == ["", "宽"]
618 // chop_cells("a宽b", 2) == ["a", "宽", "b"]
619 assert_eq!(chop_cells("宽宽", 1), vec!["", "宽", "宽"]);
620 assert_eq!(chop_cells("宽", 1), vec!["", "宽"]);
621 assert_eq!(chop_cells("a宽b", 2), vec!["a", "宽", "b"]);
622 }
623
624 /// The fold is per **grapheme**, not per code point. Measured per code point
625 /// a VS16 heart looks like one cell (heart 1 + selector 0), so twenty of them
626 /// "fit" in thirty cells and the row renders forty cells wide — straight
627 /// through whatever panel or table border was drawn around it.
628 ///
629 /// Captured from real rich 15.0.0:
630 ///
631 /// ```text
632 /// [cell_len(c) for c in chop_cells("❤️" * 20, 30)] == [30, 10]
633 /// [cell_len(c) for c in chop_cells(FAMILY * 4, 5)] == [4, 4]
634 /// ```
635 #[test]
636 fn chop_cells_folds_by_grapheme_not_by_code_point() {
637 let hearts = "\u{2764}\u{fe0f}".repeat(20);
638 let chunks = chop_cells(&hearts, 30);
639 assert_eq!(
640 chunks.iter().map(|c| cell_len(c)).collect::<Vec<_>>(),
641 vec![30, 10],
642 "a VS16 run overflowed its width"
643 );
644 let family = "\u{1f468}\u{200d}\u{1f469}\u{200d}\u{1f467}\u{200d}\u{1f466}".repeat(4);
645 assert_eq!(
646 chop_cells(&family, 5)
647 .iter()
648 .map(|c| cell_len(c))
649 .collect::<Vec<_>>(),
650 vec![4, 4],
651 "a ZWJ cluster was split"
652 );
653 }
654
655 /// Upstream appends the final chunk only `if line_size:` — a tail that
656 /// measures zero cells is dropped, not emitted. Real rich 15.0.0:
657 /// `chop_cells("\n", 4) == []`.
658 #[test]
659 fn chop_cells_drops_a_trailing_zero_cell_run() {
660 assert_eq!(chop_cells("\n", 4), Vec::<String>::new());
661 }
662
663 /// A cut landing inside a two-cell grapheme cannot be represented, so
664 /// upstream's `_split_text` swaps that grapheme for a space. Truncating per
665 /// code point instead kept the bare `❤` — still two cells on the terminal,
666 /// so the crop did not crop. Real rich 15.0.0:
667 ///
668 /// ```text
669 /// set_cell_size("❤️❤️", 1) == " "
670 /// set_cell_size("❤️❤️", 3) == "❤️ "
671 /// set_cell_size("宽宽", 3) == "宽 "
672 /// ```
673 #[test]
674 fn set_cell_size_swaps_a_straddled_grapheme_for_a_space() {
675 let hearts = "\u{2764}\u{fe0f}\u{2764}\u{fe0f}";
676 assert_eq!(set_cell_size(hearts, 1), " ");
677 assert_eq!(set_cell_size(hearts, 3), "\u{2764}\u{fe0f} ");
678 assert_eq!(set_cell_size("宽宽", 3), "宽 ");
679 // Unchanged where the cut is clean.
680 assert_eq!(set_cell_size(hearts, 2), "\u{2764}\u{fe0f}");
681 assert_eq!(set_cell_size(hearts, 4), hearts);
682 }
683
684 /// Spans cover every byte, a ZWJ swallows what follows it, and a variation
685 /// selector widens the grapheme before it without starting a new one.
686 /// Upstream indexes by code point where we index by byte, so the *shape* is
687 /// captured from real rich 15.0.0 and the offsets converted:
688 ///
689 /// ```text
690 /// split_graphemes("a" + FAMILY + "b") == ([(0,1,1), (1,8,2), (8,9,1)], 4)
691 /// split_graphemes("❤️") == ([(0,2,2)], 2)
692 /// ```
693 #[test]
694 fn split_graphemes_clusters_joiners_and_selectors() {
695 let family = "\u{1f468}\u{200d}\u{1f469}\u{200d}\u{1f467}\u{200d}\u{1f466}";
696 let text = format!("a{family}b");
697 // The family is 25 bytes: four 4-byte emoji and three 3-byte joiners.
698 assert_eq!(
699 split_graphemes(&text),
700 (vec![(0, 1, 1), (1, 26, 2), (26, 27, 1)], 4)
701 );
702 assert_eq!(
703 split_graphemes("\u{2764}\u{fe0f}"),
704 (vec![(0, 6, 2)], 2),
705 "heart (3 bytes) + VS16 (3 bytes) is one 2-cell grapheme"
706 );
707 }
708
709 #[test]
710 fn chop_keeps_combining_marks_attached() {
711 // base char + U+0301 (combining acute): each grapheme is one cell and
712 // the combining mark is 0-width, so `split_graphemes` attaches it to the
713 // character before it and the pair never straddles a fold.
714 let decomposed: String = "abcdef".chars().flat_map(|c| [c, '\u{301}']).collect();
715 let chunks = chop_cells(&decomposed, 3);
716 assert_eq!(chunks.len(), 2);
717 assert_eq!(
718 chunks.iter().map(|c| cell_len(c)).collect::<Vec<_>>(),
719 vec![3, 3]
720 );
721 // Three base chars + three combining marks per chunk.
722 assert_eq!(chunks[0].chars().count(), 6);
723 }
724
725 /// Upstream vendors its own width table and measures emoji *clusters*: a ZWJ
726 /// consumes the character after it and U+FE0F promotes a narrow character to
727 /// two cells. Summing per code point from the `unicode-width` crate gave 8
728 /// cells for a family emoji and 1 for a heart, misaligning every table and
729 /// panel that contained one.
730 #[test]
731 fn emoji_clusters_measure_as_one_glyph() {
732 for (text, expected, what) in [
733 ("\u{2764}\u{fe0f}", 2, "heart + VS16"),
734 ("\u{26a0}\u{fe0f}", 2, "warning + VS16"),
735 (
736 "\u{1f468}\u{200d}\u{1f469}\u{200d}\u{1f467}\u{200d}\u{1f466}",
737 2,
738 "ZWJ family",
739 ),
740 ("\u{1f44d}\u{1f3fb}", 2, "thumbs up + skin tone"),
741 (
742 "\u{1f3f3}\u{fe0f}\u{200d}\u{1f308}",
743 2,
744 "rainbow flag (ZWJ)",
745 ),
746 ("1\u{fe0f}\u{20e3}", 2, "keycap"),
747 ] {
748 assert_eq!(cell_len(text), expected, "{what} measured wrongly");
749 }
750 }
751
752 /// Upstream ships 21 width tables and picks between them with
753 /// `UNICODE_VERSION` (`rich._unicode_data.load`); we only ever measured with
754 /// the newest, so a terminal pinned to an older Unicode was mismeasured.
755 ///
756 /// Every expectation captured from real rich 15.0.0 by setting the variable
757 /// and reading `load("auto").unicode_version`:
758 ///
759 /// ```text
760 /// '9' -> 9.0.0 '9.0' -> 9.0.0 '9.0.0.7' -> 9.0.0 ' 9 ' -> 9.0.0
761 /// '13.1' -> 13.0.0 '12.1' -> 12.1.0 '18.0.0' -> 17.0.0 '99' -> 17.0.0
762 /// '0' -> 4.1.0 '-1' -> 4.1.0 '1.0.0' -> 4.1.0
763 /// 'latest' -> 17.0.0 'auto' -> 17.0.0 'banana' -> 17.0.0 '' -> 17.0.0
764 /// ```
765 ///
766 /// Three rules to keep straight: an exact match wins outright (so `"12.1"`
767 /// finds `12.1.0` rather than bisecting past it), an unknown version falls
768 /// back to the newest table *not newer* than it, and anything unparsable
769 /// takes the latest.
770 #[test]
771 fn unicode_version_selects_upstreams_table() {
772 for (requested, expected) in [
773 ("9", "9.0.0"),
774 ("9.0", "9.0.0"),
775 ("9.0.0", "9.0.0"),
776 ("9.0.0.7", "9.0.0"),
777 (" 9 ", "9.0.0"),
778 ("13.1", "13.0.0"),
779 ("12.1", "12.1.0"),
780 ("12.1.0", "12.1.0"),
781 ("0", "4.1.0"),
782 ("-1", "4.1.0"),
783 ("1.0.0", "4.1.0"),
784 ("4.1.0", "4.1.0"),
785 ("17.0.0", "17.0.0"),
786 ("18.0.0", "17.0.0"),
787 ("99", "17.0.0"),
788 ("latest", "17.0.0"),
789 ("auto", "17.0.0"),
790 ("banana", "17.0.0"),
791 ("", "17.0.0"),
792 ] {
793 assert_eq!(
794 VERSIONS[resolve_version(requested)],
795 expected,
796 "UNICODE_VERSION={requested:?} chose the wrong table"
797 );
798 }
799 }
800
801 /// The tables are not interchangeable, which is the whole point of honouring
802 /// the variable. Widths from real rich 15.0.0's
803 /// `get_character_cell_size(chr(cp), version)`:
804 ///
805 /// ```text
806 /// 4.1.0 8.0.0 9.0.0 12.0.0 17.0.0
807 /// U+1F600 1 1 2 2 2 grinning face
808 /// U+231A 1 1 2 2 2 watch
809 /// U+1F9E0 1 1 1 2 2 brain
810 /// U+1FAF0 1 1 1 1 2 hand with index finger
811 /// ```
812 #[test]
813 fn an_older_table_measures_emoji_narrower() {
814 for (c, widths) in [
815 ('\u{1F600}', [1, 1, 2, 2, 2]),
816 ('\u{231A}', [1, 1, 2, 2, 2]),
817 ('\u{1F9E0}', [1, 1, 1, 2, 2]),
818 ('\u{1FAF0}', [1, 1, 1, 1, 2]),
819 ] {
820 let measured: Vec<usize> = ["4.1.0", "8.0.0", "9.0.0", "12.0.0", "17.0.0"]
821 .iter()
822 .map(|version| width_in(table_for(version), c))
823 .collect();
824 assert_eq!(measured, widths.to_vec(), "{c:?} measured wrongly");
825 }
826 }
827
828 /// Spacing marks, format characters and modifier symbols are where the
829 /// `unicode-width` crate and upstream disagreed most (312 of 348 mismatches
830 /// were Mc spacing marks).
831 #[test]
832 fn combining_and_modifier_characters_take_no_cells() {
833 assert_eq!(
834 cell_len("\u{915}\u{93f}"),
835 1,
836 "Devanagari vowel sign should be zero-width"
837 );
838 assert_eq!(
839 cell_len("\u{1f3fb}"),
840 0,
841 "a lone skin-tone modifier is zero-width"
842 );
843 assert_eq!(cell_len("\u{4f60}\u{597d}"), 4, "CJK stayed wide");
844 assert_eq!(cell_len("ascii"), 5, "ASCII unaffected");
845 }
846}