Skip to main content

fatou_parser/parser/
unescape.rs

1//! Decoding a string literal's source text into the value it denotes.
2//!
3//! The CST is lossless, so a `STRING_CONTENT` token carries the *source* bytes
4//! between the quotes: `"sub\\a.jl"` holds the ten characters `sub\\a.jl`, not
5//! the nine the literal denotes. Every consumer that reads a literal as **data**
6//! — the s-expression projector, `include` path resolution — has to decode
7//! first, and they all decode the same way, so the decoding lives here once.
8//!
9//! Julia's rules, as Base's own lexer reads a double-quoted string: the named
10//! escapes, `\xNN` and octal escapes contributing a single *byte*, `\u`/`\U`
11//! contributing a codepoint, and a backslash-newline line continuation that
12//! swallows the newline and the indentation after it. Byte escapes are why the
13//! decoder accumulates bytes rather than chars: `"\xce\xb1"` is one `α`.
14
15/// Why a `STRING_CONTENT` token's source could not be reduced to a value. The
16/// two failures matter separately to the s-expression projector, which shows
17/// them differently; a consumer that just wants the value treats both as "this
18/// literal does not denote a string".
19#[derive(Debug, Clone, Copy, PartialEq, Eq)]
20pub enum StringDecodeError {
21    /// A malformed backslash escape (`\xq`, `\q`, `\400`).
22    BadEscape,
23    /// Well-formed bytes that don't decode as UTF-8 (`\xff`). Julia's `String`
24    /// holds them; Rust's does not.
25    BadUtf8,
26}
27
28/// The value a `STRING_CONTENT` token's source text denotes: escapes decoded and
29/// line continuations dropped.
30pub fn string_value(text: &str) -> Result<String, StringDecodeError> {
31    Ok(string_chunks(text)?.concat())
32}
33
34/// Decode one `STRING_CONTENT` token's source into its literal value, split into
35/// chunks at each `\`-newline line continuation (JuliaSyntax keeps one `String`
36/// per chunk; [`string_value`] just concatenates them). A continuation drops the
37/// backslash, the newline (`\n`/`\r`/`\r\n`), and the run of spaces/tabs that
38/// follow.
39pub(crate) fn string_chunks(text: &str) -> Result<Vec<String>, StringDecodeError> {
40    let mut chunks: Vec<Vec<u8>> = vec![Vec::new()];
41    let mut buf = [0u8; 4];
42    let mut chars = text.chars().peekable();
43    while let Some(c) = chars.next() {
44        if c != '\\' {
45            let last = chunks.last_mut().unwrap();
46            last.extend_from_slice(c.encode_utf8(&mut buf).as_bytes());
47            continue;
48        }
49        match chars.peek() {
50            Some('\n') | Some('\r') => {
51                let nl = chars.next().unwrap();
52                if nl == '\r' && chars.peek() == Some(&'\n') {
53                    chars.next();
54                }
55                while matches!(chars.peek(), Some(' ') | Some('\t')) {
56                    chars.next();
57                }
58                chunks.push(Vec::new());
59            }
60            _ => decode_escape_into(&mut chars, chunks.last_mut().unwrap())
61                .ok_or(StringDecodeError::BadEscape)?,
62        }
63    }
64    chunks
65        .into_iter()
66        .map(|c| String::from_utf8(c).map_err(|_| StringDecodeError::BadUtf8))
67        .collect()
68}
69
70/// Decode a single backslash escape (the backslash already consumed) into `bytes`,
71/// the way Julia reads a `Char`/`String` literal: byte escapes (`\xNN`, octal) push
72/// one byte; `\u`/`\U` push a codepoint's UTF-8 bytes; named escapes push their
73/// control byte. Returns `None` on a malformed or unknown escape.
74pub(crate) fn decode_escape_into(
75    chars: &mut std::iter::Peekable<std::str::Chars>,
76    bytes: &mut Vec<u8>,
77) -> Option<()> {
78    let mut buf = [0u8; 4];
79    match chars.next()? {
80        'n' => bytes.push(b'\n'),
81        't' => bytes.push(b'\t'),
82        'r' => bytes.push(b'\r'),
83        'a' => bytes.push(0x07),
84        'b' => bytes.push(0x08),
85        'f' => bytes.push(0x0c),
86        'v' => bytes.push(0x0b),
87        'e' => bytes.push(0x1b),
88        '\\' => bytes.push(b'\\'),
89        '\'' => bytes.push(b'\''),
90        '"' => bytes.push(b'"'),
91        '$' => bytes.push(b'$'),
92        'x' => bytes.push(take_hex(chars, 2)? as u8),
93        'u' => {
94            let cp = char::from_u32(take_hex(chars, 4)?)?;
95            bytes.extend_from_slice(cp.encode_utf8(&mut buf).as_bytes());
96        }
97        'U' => {
98            let cp = char::from_u32(take_hex(chars, 8)?)?;
99            bytes.extend_from_slice(cp.encode_utf8(&mut buf).as_bytes());
100        }
101        d @ '0'..='7' => {
102            let mut val = d.to_digit(8)?;
103            for _ in 0..2 {
104                match chars.peek().and_then(|c| c.to_digit(8)) {
105                    Some(o) => {
106                        val = val * 8 + o;
107                        chars.next();
108                    }
109                    None => break,
110                }
111            }
112            // Julia caps an octal escape at one byte; `\400` and up overflow.
113            bytes.push(u8::try_from(val).ok()?);
114        }
115        _ => return None,
116    }
117    Some(())
118}
119
120/// Consume up to `max` hex digits from `chars` and return their value; `None`
121/// if there is not at least one digit.
122pub(crate) fn take_hex(
123    chars: &mut std::iter::Peekable<std::str::Chars>,
124    max: usize,
125) -> Option<u32> {
126    let mut val = 0u32;
127    let mut n = 0;
128    while n < max {
129        match chars.peek().and_then(|c| c.to_digit(16)) {
130            Some(d) => {
131                val = val * 16 + d;
132                chars.next();
133                n += 1;
134            }
135            None => break,
136        }
137    }
138    (n > 0).then_some(val)
139}
140
141#[cfg(test)]
142mod tests {
143    use super::*;
144
145    #[test]
146    fn escape_free_content_is_its_own_value() {
147        assert_eq!(string_value("sub/a.jl").as_deref(), Ok("sub/a.jl"));
148        assert_eq!(string_value("").as_deref(), Ok(""));
149    }
150
151    #[test]
152    fn a_doubled_backslash_denotes_one() {
153        assert_eq!(string_value("sub\\\\a.jl").as_deref(), Ok("sub\\a.jl"));
154    }
155
156    #[test]
157    fn named_and_numeric_escapes_decode() {
158        assert_eq!(string_value("a\\tb\\n").as_deref(), Ok("a\tb\n"));
159        assert_eq!(string_value("\\x41\\101").as_deref(), Ok("AA"));
160        assert_eq!(string_value("\\u03b1\\U0001f600").as_deref(), Ok("α😀"));
161    }
162
163    #[test]
164    fn byte_escapes_join_into_one_codepoint() {
165        assert_eq!(string_value("\\xce\\xb1").as_deref(), Ok("α"));
166    }
167
168    #[test]
169    fn quote_dollar_and_backslash_escapes_decode() {
170        assert_eq!(string_value("\\\"\\$\\\\").as_deref(), Ok("\"$\\"));
171    }
172
173    #[test]
174    fn a_line_continuation_swallows_the_newline_and_indent() {
175        assert_eq!(string_value("a\\\n    b").as_deref(), Ok("ab"));
176        assert_eq!(string_value("a\\\r\n\tb").as_deref(), Ok("ab"));
177    }
178
179    #[test]
180    fn a_malformed_escape_is_bad_escape() {
181        assert_eq!(string_value("a\\q"), Err(StringDecodeError::BadEscape));
182        assert_eq!(string_value("\\xq"), Err(StringDecodeError::BadEscape));
183        assert_eq!(string_value("\\400"), Err(StringDecodeError::BadEscape));
184        assert_eq!(string_value("a\\"), Err(StringDecodeError::BadEscape));
185    }
186
187    #[test]
188    fn a_lone_non_utf8_byte_is_bad_utf8() {
189        assert_eq!(string_value("\\xff"), Err(StringDecodeError::BadUtf8));
190    }
191
192    #[test]
193    fn chunks_split_at_each_line_continuation() {
194        assert_eq!(
195            string_chunks("a\\\nb\\\nc").unwrap(),
196            ["a".to_string(), "b".to_string(), "c".to_string()]
197        );
198    }
199}