ctl_text/
unsafe_chars.rs

1//! Which characters are unsafe to print to a terminal as they are, and the
2//! function that removes them.
3//!
4//! Text from outside the program can carry characters that act on the
5//! terminal or on the layout instead of showing. A title, a commit subject,
6//! or a file's contents can all hold them.
7//!
8//! A CLI that prints such text drops what [`is_unsafe`] refuses, with
9//! [`strip_unsafe`].
10//! [`Span::new`](crate::Span::new) and [`Text`](crate::Text) apply it too,
11//! but keep a line break as a space or a newline. The words around it then
12//! stay apart.
13//!
14//! The set is three whole sets, each taken by its definition:
15//!
16//! 1. General category Cc, the C0 controls, DEL, and the C1 controls, 65
17//!    characters, through [`char::is_control`]. A control can start an escape
18//!    sequence that moves the cursor, rewrites the screen, or sets the
19//!    clipboard.
20//! 2. The 12 directional formatting characters of UAX #9, from
21//!    [`unicode_bidi::format_chars`]: ALM, LRM, RLM, LRE, RLE, PDF, LRO, RLO,
22//!    LRI, RLI, FSI, and PDI. They reorder what the reader sees, so the text
23//!    shown differs from the text held.
24//! 3. General categories Zl and Zp, one character each: [`LINE_SEPARATOR`] and
25//!    [`PARAGRAPH_SEPARATOR`]. A terminal or a tool may break a line there that
26//!    the layout counted as one.
27//!
28//! The set follows Unicode 16.0.0 and UAX #9 revision 50, which unicode-bidi
29//! 0.3.18 implements. Cc, Zl, and Zp have had the same members since
30//! Unicode 1.1.
31//!
32//! The other format characters (general category Cf) stay. They neither act on
33//! the terminal nor reorder text, and text needs them.
34//!
35//! - ZWJ joins emoji.
36//! - ZWNJ shapes Persian and Indic scripts.
37//! - The tag characters spell subdivision flags.
38//! - The soft hyphen marks a break.
39//! - The Arabic number signs render as visible marks.
40
41use unicode_bidi::format_chars::{ALM, FSI, LRE, LRI, LRM, LRO, PDF, PDI, RLE, RLI, RLM, RLO};
42
43/// U+2028 LINE SEPARATOR, the only character in general category Zl.
44/// [`is_unsafe`] refuses it.
45pub const LINE_SEPARATOR: char = '\u{2028}';
46
47/// U+2029 PARAGRAPH SEPARATOR, the only character in general category Zp.
48/// [`is_unsafe`] refuses it.
49pub const PARAGRAPH_SEPARATOR: char = '\u{2029}';
50
51/// Whether `character` is a Cc control, a directional formatting character,
52/// or U+2028 or U+2029: the set the [module](self) defines.
53///
54/// Newline and tab are controls, so they are refused too. A caller that
55/// keeps them checks for them first, or passes [`Keep`] to [`strip_unsafe`].
56#[must_use]
57pub fn is_unsafe(character: char) -> bool {
58    character.is_control()
59        || matches!(
60            character,
61            ALM | LRM
62                | RLM
63                | LRE
64                | RLE
65                | PDF
66                | LRO
67                | RLO
68                | LRI
69                | RLI
70                | FSI
71                | PDI
72                | LINE_SEPARATOR
73                | PARAGRAPH_SEPARATOR
74        )
75}
76
77/// The controls [`strip_unsafe`] leaves in place.
78#[derive(Clone, Copy, Debug, Eq, Hash, PartialEq)]
79#[non_exhaustive]
80pub enum Keep {
81    /// Nothing: every unsafe character goes, newline and tab included.
82    Nothing,
83    /// Newline and tab, the two controls that preformatted text needs.
84    NewlineAndTab,
85}
86
87/// `text` without the characters [`is_unsafe`] refuses, except those `keep`
88/// names.
89///
90/// A removed character leaves no space behind, so words on either side of a
91/// removed line separator join. [`Text`](crate::Text) turns a line break into
92/// a newline instead.
93#[must_use]
94pub fn strip_unsafe(text: &str, keep: Keep) -> String {
95    text.chars()
96        .filter(|character| match keep {
97            Keep::Nothing => !is_unsafe(*character),
98            Keep::NewlineAndTab => matches!(character, '\n' | '\t') || !is_unsafe(*character),
99        })
100        .collect()
101}
102
103#[cfg(test)]
104mod tests {
105    use unicode_bidi::format_chars::{ALM, FSI, LRE, LRI, LRM, LRO, PDF, PDI, RLE, RLI, RLM, RLO};
106
107    use super::{Keep, LINE_SEPARATOR, PARAGRAPH_SEPARATOR, is_unsafe, strip_unsafe};
108
109    const BIDI: [char; 12] = [ALM, LRM, RLM, LRE, RLE, PDF, LRO, RLO, LRI, RLI, FSI, PDI];
110
111    /// C0, DEL, and C1, as UnicodeData.txt lists general category Cc.
112    fn controls() -> impl Iterator<Item = char> {
113        ('\u{0}'..='\u{1f}').chain('\u{7f}'..='\u{9f}')
114    }
115
116    #[test]
117    fn refuses_every_control() {
118        assert_eq!(controls().count(), 65);
119        for character in controls() {
120            assert!(is_unsafe(character), "{character:?} passed");
121        }
122    }
123
124    #[test]
125    fn refuses_every_bidi_formatting_character() {
126        for character in BIDI {
127            assert!(is_unsafe(character), "{character:?} passed");
128        }
129    }
130
131    #[test]
132    fn refuses_both_separators() {
133        assert!(is_unsafe(LINE_SEPARATOR));
134        assert!(is_unsafe(PARAGRAPH_SEPARATOR));
135    }
136
137    #[test]
138    fn keeps_the_neighbours_of_each_set() {
139        let neighbours = [
140            ' ', '~', '\u{a0}', '\u{61b}', '\u{61d}', '\u{200d}', '\u{2010}', '\u{2027}',
141            '\u{202f}', '\u{2065}', '\u{206a}',
142        ];
143        for character in neighbours {
144            assert!(!is_unsafe(character), "{character:?} refused");
145        }
146    }
147
148    #[test]
149    fn keeps_the_other_format_characters() {
150        let kept = [
151            '\u{ad}',
152            '\u{600}',
153            '\u{200b}',
154            '\u{200c}',
155            '\u{200d}',
156            '\u{2060}',
157            '\u{feff}',
158            '\u{e0001}',
159            '\u{e0067}',
160            '\u{e007f}',
161        ];
162        for character in kept {
163            assert!(!is_unsafe(character), "{character:?} refused");
164        }
165    }
166
167    #[test]
168    fn refuses_nothing_outside_the_three_sets() {
169        let refused = (0..=u32::from(char::MAX))
170            .filter_map(char::from_u32)
171            .filter(|character| is_unsafe(*character))
172            .count();
173        assert_eq!(refused, 65 + BIDI.len() + 2);
174    }
175
176    #[test]
177    fn strips_every_unsafe_character_and_keeps_the_rest() {
178        let text = "a\u{1b}[2J b\u{202e}c\u{2028}d\u{7f}\tx\ny";
179        assert_eq!(strip_unsafe(text, Keep::Nothing), "a[2J bcdxy");
180    }
181
182    #[test]
183    fn keeps_newline_and_tab_when_asked() {
184        let text = "a\u{1b}b\tc\nd\u{2029}e";
185        assert_eq!(strip_unsafe(text, Keep::NewlineAndTab), "ab\tc\nde");
186    }
187
188    #[test]
189    fn strips_the_whole_unsafe_set_and_nothing_else() {
190        let everything: String = (0..=u32::from(char::MAX))
191            .filter_map(char::from_u32)
192            .collect();
193        let stripped = strip_unsafe(&everything, Keep::Nothing);
194        assert_eq!(
195            everything.chars().count() - stripped.chars().count(),
196            65 + BIDI.len() + 2
197        );
198        assert!(stripped.chars().all(|character| !is_unsafe(character)));
199    }
200
201    #[test]
202    fn unicode_bidi_still_targets_the_documented_version() {
203        assert_eq!(unicode_bidi::UNICODE_VERSION, (16, 0, 0));
204    }
205}
ctl_text/unsafe_chars.rs

ctl_text/
unsafe_chars.rs
