one_brackets/
text.rs

1//! Text of strings and atoms: Unicode that may hold unpaired surrogates.
2
3use std::borrow::Cow;
4use std::fmt;
5
6/// Text of a string or an atom.
7///
8/// The platform keeps text as UTF-16 code units and writes each surrogate of a quoted string
9/// as an escape `"\xxxx`, so a string may hold an unpaired surrogate. `Text` keeps such
10/// strings without losses; [`as_str`](Text::as_str) gives `None` for them and
11/// [`to_string_lossy`](Text::to_string_lossy) replaces unpaired surrogates with U+FFFD.
12///
13/// Stored as [WTF-8](https://simonsapin.github.io/wtf-8/): UTF-8 that also encodes unpaired
14/// surrogates. Text without them is plain UTF-8.
15#[derive(Clone, Default, PartialEq, Eq, Hash, PartialOrd, Ord)]
16pub struct Text {
17    bytes: Vec<u8>,
18}
19
20impl Text {
21    pub fn new() -> Self {
22        Self::default()
23    }
24
25    /// Text from UTF-16 code units; unpaired surrogates are kept.
26    pub fn from_utf16(units: &[u16]) -> Self {
27        let mut bytes = Vec::with_capacity(units.len());
28        for &unit in units {
29            push_code_point(&mut bytes, u32::from(unit));
30        }
31        Self { bytes }
32    }
33
34    /// WTF-8 bytes produced by this module or validated as UTF-8.
35    pub(crate) fn from_wtf8(bytes: Vec<u8>) -> Self {
36        Self { bytes }
37    }
38
39    /// The text, unless it holds an unpaired surrogate.
40    pub fn as_str(&self) -> Option<&str> {
41        std::str::from_utf8(&self.bytes).ok()
42    }
43
44    /// The text with unpaired surrogates replaced by U+FFFD.
45    pub fn to_string_lossy(&self) -> Cow<'_, str> {
46        match self.as_str() {
47            Some(s) => Cow::Borrowed(s),
48            None => Cow::Owned(
49                self.code_points()
50                    .map(|c| char::from_u32(c).unwrap_or(char::REPLACEMENT_CHARACTER))
51                    .collect(),
52            ),
53        }
54    }
55
56    /// The text as a `String`; `Err` gives it back if it holds an unpaired surrogate.
57    pub fn into_string(self) -> Result<String, Text> {
58        String::from_utf8(self.bytes).map_err(|e| Text {
59            bytes: e.into_bytes(),
60        })
61    }
62
63    /// UTF-16 code units, as the platform keeps the text.
64    pub fn encode_utf16(&self) -> impl Iterator<Item = u16> + '_ {
65        self.code_points().flat_map(|c| {
66            let (first, second) = match c {
67                0x10000.. => {
68                    let c = c - 0x10000;
69                    (0xD800 | (c >> 10) as u16, Some(0xDC00 | (c & 0x3FF) as u16))
70                }
71                _ => (c as u16, None),
72            };
73            std::iter::once(first).chain(second)
74        })
75    }
76
77    /// Unicode code points; unpaired surrogates come as themselves (`0xD800..=0xDFFF`).
78    pub fn code_points(&self) -> impl Iterator<Item = u32> + '_ {
79        let bytes = &self.bytes;
80        let mut i = 0;
81        std::iter::from_fn(move || {
82            let first = *bytes.get(i)?;
83            let (len, mut c) = match first {
84                0..=0x7F => (1, u32::from(first)),
85                0xC0..=0xDF => (2, u32::from(first & 0x1F)),
86                0xE0..=0xEF => (3, u32::from(first & 0x0F)),
87                _ => (4, u32::from(first & 0x07)),
88            };
89            for &b in &bytes[i + 1..i + len] {
90                c = c << 6 | u32::from(b & 0x3F);
91            }
92            i += len;
93            Some(c)
94        })
95    }
96
97    /// WTF-8 bytes: UTF-8 if the text has no unpaired surrogates.
98    pub fn as_wtf8(&self) -> &[u8] {
99        &self.bytes
100    }
101
102    pub fn is_empty(&self) -> bool {
103        self.bytes.is_empty()
104    }
105
106    /// Length in WTF-8 bytes.
107    pub fn len(&self) -> usize {
108        self.bytes.len()
109    }
110}
111
112/// Appends a code point, joining a low surrogate with a high one before it.
113pub(crate) fn push_code_point(bytes: &mut Vec<u8>, c: u32) {
114    let mut c = c;
115    if (0xDC00..=0xDFFF).contains(&c) {
116        if let Some(high) = trailing_high_surrogate(bytes) {
117            bytes.truncate(bytes.len() - 3);
118            c = 0x10000 + ((high - 0xD800) << 10) + (c - 0xDC00);
119        }
120    }
121    match char::from_u32(c) {
122        Some(ch) => bytes.extend_from_slice(ch.encode_utf8(&mut [0; 4]).as_bytes()),
123        // A surrogate: the 3-byte form UTF-8 would have for it.
124        None => bytes.extend_from_slice(&[
125            0xE0 | (c >> 12) as u8,
126            0x80 | (c >> 6 & 0x3F) as u8,
127            0x80 | (c & 0x3F) as u8,
128        ]),
129    }
130}
131
132fn trailing_high_surrogate(bytes: &[u8]) -> Option<u32> {
133    match bytes {
134        [.., 0xED, b1 @ 0xA0..=0xAF, b2] => Some(surrogate(*b1, *b2)),
135        _ => None,
136    }
137}
138
139/// The surrogate encoded as `ED b1 b2`.
140fn surrogate(b1: u8, b2: u8) -> u32 {
141    0xD000 | u32::from(b1 & 0x3F) << 6 | u32::from(b2 & 0x3F)
142}
143
144impl From<String> for Text {
145    fn from(s: String) -> Self {
146        Self {
147            bytes: s.into_bytes(),
148        }
149    }
150}
151
152impl From<&str> for Text {
153    fn from(s: &str) -> Self {
154        Self::from(s.to_owned())
155    }
156}
157
158impl From<char> for Text {
159    fn from(c: char) -> Self {
160        Self::from(c.to_string())
161    }
162}
163
164impl From<&String> for Text {
165    fn from(s: &String) -> Self {
166        Self::from(s.as_str())
167    }
168}
169
170impl From<Cow<'_, str>> for Text {
171    fn from(s: Cow<'_, str>) -> Self {
172        Self::from(s.into_owned())
173    }
174}
175
176impl From<&Text> for Text {
177    fn from(text: &Text) -> Self {
178        text.clone()
179    }
180}
181
182impl PartialEq<str> for Text {
183    fn eq(&self, other: &str) -> bool {
184        self.bytes == other.as_bytes()
185    }
186}
187
188impl PartialEq<&str> for Text {
189    fn eq(&self, other: &&str) -> bool {
190        self.bytes == other.as_bytes()
191    }
192}
193
194impl PartialEq<String> for Text {
195    fn eq(&self, other: &String) -> bool {
196        self.bytes == other.as_bytes()
197    }
198}
199
200impl fmt::Debug for Text {
201    /// Like `str`; unpaired surrogates are shown as `\u{d800}`.
202    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
203        if let Some(s) = self.as_str() {
204            return fmt::Debug::fmt(s, f);
205        }
206        f.write_str("\"")?;
207        for c in self.code_points() {
208            match char::from_u32(c) {
209                Some('\'') => f.write_str("'")?,
210                Some(ch) => write!(f, "{}", ch.escape_debug())?,
211                None => write!(f, "\\u{{{c:x}}}")?,
212            }
213        }
214        f.write_str("\"")
215    }
216}
217
218impl fmt::Display for Text {
219    /// Unpaired surrogates are shown as U+FFFD.
220    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
221        f.write_str(&self.to_string_lossy())
222    }
223}
224
225#[cfg(test)]
226mod tests {
227    use super::*;
228
229    #[test]
230    fn keeps_unpaired_surrogates() {
231        let text = Text::from_utf16(&[0x61, 0xD83D, 0xDE00, 0xD800, 0x62, 0xDC00]);
232        assert_eq!(text.as_str(), None);
233        assert_eq!(text.to_string_lossy(), "a😀\u{fffd}b\u{fffd}");
234        assert_eq!(
235            text.encode_utf16().collect::<Vec<_>>(),
236            [0x61, 0xD83D, 0xDE00, 0xD800, 0x62, 0xDC00]
237        );
238        assert_eq!(format!("{text:?}"), r#""a😀\u{d800}b\u{dc00}""#);
239        assert_eq!(text.to_string(), "a😀\u{fffd}b\u{fffd}");
240        assert!(text.clone().into_string().is_err());
241    }
242
243    #[test]
244    fn pairs_become_characters() {
245        let text = Text::from_utf16(&[0xD83D, 0xDE00]);
246        assert_eq!(text.as_str(), Some("😀"));
247        assert_eq!(text, "😀");
248
249        let mut bytes = Vec::new();
250        push_code_point(&mut bytes, 0xD83D);
251        push_code_point(&mut bytes, 0x78);
252        assert_eq!(Text::from_wtf8(bytes).to_string_lossy(), "\u{fffd}x");
253    }
254
255    #[test]
256    fn debug_matches_str() {
257        assert_eq!(
258            format!("{:?}", Text::from("a \"b\" 'c'\n")),
259            r#""a \"b\" 'c'\n""#
260        );
261    }
262}
one_brackets/text.rs

one_brackets/
text.rs
