squawk_syntax/
unescape.rs

1use std::fmt;
2use std::ops::{Range, RangeInclusive};
3
4use rowan::TextSize;
5
6use crate::decoded_text::DecodedText;
7
8pub enum UnicodeEscapeKind {
9    Extended,
10    Short,
11}
12
13impl UnicodeEscapeKind {
14    fn count(&self) -> u32 {
15        match self {
16            UnicodeEscapeKind::Extended => 6,
17            UnicodeEscapeKind::Short => 4,
18        }
19    }
20}
21
22pub enum UnicodeEscError {
23    InvalidEscape,
24    InvalidSurrogatePair,
25    OutOfRange,
26    RequiresHexDigits {
27        kind: UnicodeEscapeKind,
28        escape_char: char,
29    },
30}
31
32impl fmt::Display for UnicodeEscError {
33    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
34        match self {
35            Self::InvalidEscape => f.write_str("Invalid Unicode escape sequence"),
36            Self::InvalidSurrogatePair => f.write_str("Invalid Unicode surrogate pair"),
37            Self::OutOfRange => f.write_str("Unicode escape value out of range"),
38            Self::RequiresHexDigits { kind, escape_char } => {
39                let required = kind.count();
40                let plus = match kind {
41                    UnicodeEscapeKind::Extended => "+",
42                    UnicodeEscapeKind::Short => "",
43                };
44                let xs = "X".repeat(required as usize);
45                write!(
46                    f,
47                    "Unicode escape requires {required} hex digits: {escape_char}{plus}{xs}"
48                )
49            }
50        }
51    }
52}
53
54pub fn escape_unicode_esc_str<F>(text: &str, escape_char: char, mut callback: F)
55where
56    F: FnMut(Range<usize>, Result<char, UnicodeEscError>),
57{
58    const HIGH_SURROGATE: RangeInclusive<u32> = 0xD800..=0xDBFF;
59    const LOW_SURROGATE: RangeInclusive<u32> = 0xDC00..=0xDFFF;
60    const MAX_CODEPOINT: u32 = 0x10FFFF;
61
62    let mut chars = text.char_indices().peekable();
63    let mut high_surrogate: Option<(Range<usize>, u32)> = None;
64
65    while let Some((escape_start, c)) = chars.next() {
66        if c != escape_char {
67            if let Some((hi_range, _)) = high_surrogate.take() {
68                callback(hi_range, Err(UnicodeEscError::InvalidSurrogatePair));
69            }
70            callback(escape_start..escape_start + c.len_utf8(), Ok(c));
71            continue;
72        }
73        let kind = match chars.peek() {
74            Some(&(_, c)) if c == escape_char => {
75                chars.next();
76                if let Some((hi_range, _)) = high_surrogate.take() {
77                    callback(hi_range, Err(UnicodeEscError::InvalidSurrogatePair));
78                }
79                let end = escape_start + escape_char.len_utf8() * 2;
80                callback(escape_start..end, Ok(escape_char));
81                continue;
82            }
83            Some(&(_, '+')) => {
84                chars.next();
85                UnicodeEscapeKind::Extended
86            }
87            Some(&(_, c)) if c.is_ascii_hexdigit() => UnicodeEscapeKind::Short,
88            _ => {
89                let end = chars
90                    .next()
91                    .map(|(i, c)| i + c.len_utf8())
92                    .unwrap_or(text.len());
93                if let Some((hi_range, _)) = high_surrogate.take() {
94                    callback(hi_range, Err(UnicodeEscError::InvalidSurrogatePair));
95                }
96                callback(escape_start..end, Err(UnicodeEscError::InvalidEscape));
97                continue;
98            }
99        };
100        let mut codepoint: u32 = 0;
101        let mut got_all = true;
102        let mut last_end = chars.peek().map(|&(i, _)| i).unwrap_or(text.len());
103        for _ in 0..kind.count() {
104            let radix = 16;
105            let Some(&(i, ch)) = chars.peek() else {
106                got_all = false;
107                break;
108            };
109            let Some(d) = ch.to_digit(radix) else {
110                got_all = false;
111                break;
112            };
113            chars.next();
114            codepoint = codepoint * radix + d;
115            last_end = i + ch.len_utf8();
116        }
117        if !got_all {
118            if let Some((hi_range, _)) = high_surrogate.take() {
119                callback(hi_range, Err(UnicodeEscError::InvalidSurrogatePair));
120            }
121            callback(
122                escape_start..last_end,
123                Err(UnicodeEscError::RequiresHexDigits { kind, escape_char }),
124            );
125            continue;
126        }
127        if let Some((hi_range, hi_cp)) = high_surrogate.take() {
128            if LOW_SURROGATE.contains(&codepoint) {
129                let combined = 0x10000 + ((hi_cp - 0xD800) << 10) + (codepoint - 0xDC00);
130                let ch = char::from_u32(combined).unwrap();
131                callback(hi_range.start..last_end, Ok(ch));
132                continue;
133            }
134            callback(
135                hi_range.start..last_end,
136                Err(UnicodeEscError::InvalidSurrogatePair),
137            );
138            continue;
139        }
140        if codepoint > MAX_CODEPOINT {
141            callback(escape_start..last_end, Err(UnicodeEscError::OutOfRange));
142        } else if HIGH_SURROGATE.contains(&codepoint) {
143            high_surrogate = Some((escape_start..last_end, codepoint));
144        } else if LOW_SURROGATE.contains(&codepoint) {
145            callback(
146                escape_start..last_end,
147                Err(UnicodeEscError::InvalidSurrogatePair),
148            );
149        } else {
150            let ch = char::from_u32(codepoint).unwrap();
151            callback(escape_start..last_end, Ok(ch));
152        }
153    }
154    if let Some((range, _)) = high_surrogate {
155        callback(range, Err(UnicodeEscError::InvalidSurrogatePair));
156    }
157}
158
159// https://github.com/postgres/postgres/blob/228a1f9542792c6533ef74c2e7aefad0da1d9a7a/src/backend/parser/parser.c#L350
160const fn is_valid_uescape_char(byte: u8) -> bool {
161    !byte.is_ascii_hexdigit()
162        && byte != b'+'
163        && byte != b'\''
164        && byte != b'"'
165        && !matches!(
166            byte,
167            b' ' | b'\t' | b'\n' | b'\r' | /* b'\v' */ 0x0B | /* b'\f' */ 0x0C
168        )
169}
170
171pub fn uescape_char(text: &str) -> Option<char> {
172    let inner = text.strip_prefix('\'')?.strip_suffix('\'')?;
173    let &[byte] = inner.as_bytes() else {
174        return None;
175    };
176    is_valid_uescape_char(byte).then(|| char::from(byte))
177}
178
179pub fn decode_plain_string(inner: &str, start_pos: TextSize, out: &mut DecodedText) {
180    let mut chars = inner.char_indices().peekable();
181    while let Some((i, c)) = chars.next() {
182        let pos = start_pos + TextSize::new(i as u32);
183        if c == '\'' && chars.peek().is_some_and(|&(_, next)| next == '\'') {
184            chars.next();
185        }
186        out.push_char(c, pos);
187    }
188}
189
190struct EscBuffer {
191    bytes: Vec<u8>,
192    pos: TextSize,
193}
194
195impl EscBuffer {
196    fn new(pos: TextSize) -> Self {
197        Self { bytes: vec![], pos }
198    }
199
200    fn push(&mut self, byte: u8, pos: TextSize) {
201        if self.bytes.is_empty() {
202            self.pos = pos;
203        }
204        self.bytes.push(byte);
205    }
206
207    fn push_char(&mut self, c: char, pos: TextSize) {
208        if self.bytes.is_empty() {
209            self.pos = pos;
210        }
211        let mut buf = [0; 4];
212        self.bytes
213            .extend_from_slice(c.encode_utf8(&mut buf).as_bytes());
214    }
215
216    fn drain(&mut self, out: &mut DecodedText) {
217        if self.bytes.is_empty() {
218            return;
219        }
220        match std::str::from_utf8(&self.bytes) {
221            Ok(text) => out.push_str(text, self.pos),
222            Err(err) if err.error_len().is_some() => {
223                out.push_str(&String::from_utf8_lossy(&self.bytes), self.pos);
224            }
225            Err(_) => return,
226        }
227        self.bytes.clear();
228    }
229
230    fn flush(&mut self, out: &mut DecodedText) {
231        if self.bytes.is_empty() {
232            return;
233        }
234        out.push_str(&String::from_utf8_lossy(&self.bytes), self.pos);
235        self.bytes.clear();
236    }
237}
238
239pub fn decode_esc_string(inner: &str, start_pos: TextSize, out: &mut DecodedText) {
240    let mut chars = inner.char_indices().peekable();
241    let mut esc = EscBuffer::new(start_pos);
242
243    while let Some((i, c)) = chars.next() {
244        let pos = start_pos + TextSize::new(i as u32);
245
246        if c == '\'' && chars.peek().is_some_and(|&(_, next)| next == '\'') {
247            chars.next();
248            esc.flush(out);
249            out.push_char('\'', pos);
250            continue;
251        }
252        if c != '\\' {
253            esc.flush(out);
254            out.push_char(c, pos);
255            continue;
256        }
257        let Some(&(_, next)) = chars.peek() else {
258            esc.flush(out);
259            out.push_char('\\', pos);
260            break;
261        };
262        match next {
263            'b' => {
264                chars.next();
265                esc.push(b'\x08', pos);
266            }
267            'f' => {
268                chars.next();
269                esc.push(b'\x0C', pos);
270            }
271            'n' => {
272                chars.next();
273                esc.push(b'\n', pos);
274            }
275            'r' => {
276                chars.next();
277                esc.push(b'\r', pos);
278            }
279            't' => {
280                chars.next();
281                esc.push(b'\t', pos);
282            }
283            '0'..='7' => {
284                let mut value: u32 = 0;
285                for _ in 0..3 {
286                    match chars.peek() {
287                        Some(&(_, d)) if ('0'..='7').contains(&d) => {
288                            chars.next();
289                            value = value * 8 + d.to_digit(8).unwrap();
290                        }
291                        _ => break,
292                    }
293                }
294                if value != 0 {
295                    esc.push(value as u8, pos);
296                }
297            }
298            'x' => {
299                chars.next();
300                let mut value: u8 = 0;
301                let mut got_any = false;
302                for _ in 0..2 {
303                    match chars.peek() {
304                        Some(&(_, d)) if d.is_ascii_hexdigit() => {
305                            chars.next();
306                            value = value * 16 + d.to_digit(16).unwrap() as u8;
307                            got_any = true;
308                        }
309                        _ => break,
310                    }
311                }
312                if got_any {
313                    if value != 0 {
314                        esc.push(value, pos);
315                    }
316                } else {
317                    esc.push(b'x', pos);
318                }
319            }
320            'u' | 'U' => {
321                chars.next();
322                let required = if next == 'u' { 4 } else { 8 };
323                let mut value: u32 = 0;
324                let mut got_all = true;
325                for _ in 0..required {
326                    match chars.peek() {
327                        Some(&(_, d)) if d.is_ascii_hexdigit() => {
328                            chars.next();
329                            value = value * 16 + d.to_digit(16).unwrap();
330                        }
331                        _ => {
332                            got_all = false;
333                            break;
334                        }
335                    }
336                }
337                if got_all
338                    && let Some(ch) = char::from_u32(value)
339                    && ch != '\0'
340                {
341                    esc.push_char(ch, pos);
342                }
343            }
344            _ => {
345                chars.next();
346                esc.push_char(next, pos);
347            }
348        }
349        esc.drain(out);
350    }
351
352    esc.flush(out);
353}
354
355pub fn decode_unicode_esc_string(
356    inner: &str,
357    start_pos: TextSize,
358    escape_char: char,
359    out: &mut DecodedText,
360) {
361    let mut dequoted = DecodedText::new(start_pos);
362    decode_plain_string(inner, start_pos, &mut dequoted);
363
364    escape_unicode_esc_str(dequoted.text(), escape_char, |range, result| {
365        if let Ok(ch) = result {
366            out.push_char(ch, dequoted.source_pos(TextSize::new(range.start as u32)));
367        }
368    });
369}
370
371#[cfg(test)]
372mod tests {
373    use insta::assert_snapshot;
374
375    use super::*;
376
377    fn unicode_escape_events(text: &str, escape_char: char) -> String {
378        let mut events = vec![];
379
380        escape_unicode_esc_str(text, escape_char, |range, result| {
381            let entry = match result {
382                Ok(ch) => format!("{}..{} ok {ch:?}", range.start, range.end),
383                Err(err) => format!("{}..{} err {err}", range.start, range.end),
384            };
385            events.push(entry);
386        });
387
388        events.join("\n")
389    }
390
391    fn decode_escape_string(inner: &str) -> String {
392        let mut out = DecodedText::new(TextSize::new(0));
393        decode_esc_string(inner, TextSize::new(0), &mut out);
394        out.into_text()
395    }
396
397    fn decode_unicode_escape_string(inner: &str, escape_char: char) -> String {
398        let mut out = DecodedText::new(TextSize::new(0));
399        decode_unicode_esc_string(inner, TextSize::new(0), escape_char, &mut out);
400        out.into_text()
401    }
402
403    #[test]
404    fn ok() {
405        assert_snapshot!(unicode_escape_events(r"hello world", '\\'), @"
406        0..1 ok 'h'
407        1..2 ok 'e'
408        2..3 ok 'l'
409        3..4 ok 'l'
410        4..5 ok 'o'
411        5..6 ok ' '
412        6..7 ok 'w'
413        7..8 ok 'o'
414        8..9 ok 'r'
415        9..10 ok 'l'
416        10..11 ok 'd'
417        ");
418    }
419
420    #[test]
421    fn incomplete_unicode_escape_breaks_surrogate_pairing() {
422        assert_snapshot!(unicode_escape_events(r"\D800\006\DC00", '\\'), @r"
423        0..5 err Invalid Unicode surrogate pair
424        5..9 err Unicode escape requires 4 hex digits: \XXXX
425        9..14 err Invalid Unicode surrogate pair
426        ");
427    }
428
429    #[test]
430    fn invalid_unicode_escape_breaks_surrogate_pairing() {
431        assert_snapshot!(unicode_escape_events(r"\D800\Q\DC00", '\\'), @r"
432        0..5 err Invalid Unicode surrogate pair
433        5..7 err Invalid Unicode escape sequence
434        7..12 err Invalid Unicode surrogate pair
435        ");
436    }
437
438    #[test]
439    fn invalid_unicode_escape_does_not_emit_literal_char() {
440        assert_snapshot!(unicode_escape_events(r"\0061\Q\0062", '\\'), @r"
441        0..5 ok 'a'
442        5..7 err Invalid Unicode escape sequence
443        7..12 ok 'b'
444        ");
445    }
446
447    #[test]
448    fn invalid_unicode_escape_works_with_custom_escape_char() {
449        assert_snapshot!(unicode_escape_events("!0061!Q!0062", '!'), @r"
450        0..5 ok 'a'
451        5..7 err Invalid Unicode escape sequence
452        7..12 ok 'b'
453        ");
454    }
455
456    #[test]
457    fn valid_unicode_escape_after_high_surrogate_only_emits_error() {
458        assert_snapshot!(unicode_escape_events(r"\D800\0061", '\\'), @r"
459        0..10 err Invalid Unicode surrogate pair
460        ");
461    }
462
463    #[test]
464    fn decode_escape_string_hex_bytes_as_utf8() {
465        assert_snapshot!(decode_escape_string(r"\xC3\xA9"), @"é");
466    }
467
468    #[test]
469    fn decode_escape_string_skips_nul_byte() {
470        assert_snapshot!(decode_escape_string(r"a\000b"), @"ab");
471    }
472
473    #[test]
474    fn escape_string_incomplete_byte_escape() {
475        assert_snapshot!(decode_escape_string(r"\xc3a"), @"�a");
476        assert_snapshot!(decode_escape_string(r"\xc3"), @"�");
477    }
478
479    #[test]
480    fn escape_string_trailing_backslash() {
481        assert_snapshot!(decode_escape_string(r"a\"), @r"a\");
482        assert_snapshot!(decode_escape_string(r"\xC3\"), @r"�\");
483    }
484
485    #[test]
486    fn decode_unicode_string_collapses_doubled_quotes() {
487        assert_snapshot!(decode_unicode_escape_string("a''b", '\\'), @"a'b");
488    }
489}
squawk_syntax/unescape.rs

squawk_syntax/
unescape.rs
