Skip to main content

buffa/text/
string.rs

1//! Textproto string-literal escaping and unescaping.
2//!
3//! The textproto string grammar is byte-oriented: `\xNN` and `\NNN` octal
4//! escapes produce raw bytes, so a string literal may decode to arbitrary
5//! non-UTF-8 data. This is intentional — proto `bytes` fields use the same
6//! literal syntax. [`unescape`] therefore returns `Vec<u8>`, and the
7//! UTF-8-validating wrapper [`unescape_str`] sits on top.
8//!
9//! Escape sequences (reference: protobuf-go `decode_string.go` and the
10//! [C++ tokenizer](https://github.com/protocolbuffers/protobuf/blob/v33.5/src/google/protobuf/io/tokenizer.cc)):
11//!
12//! | escape      | result                                                  |
13//! |-------------|---------------------------------------------------------|
14//! | `\n \r \t`  | newline, carriage return, tab                           |
15//! | `\" \' \\`  | literal quote, apostrophe, backslash                    |
16//! | `\?`        | literal `?` (C legacy)                                  |
17//! | `\a \b`     | BEL (0x07), BS (0x08)                                   |
18//! | `\f \v`     | FF (0x0C), VT (0x0B)                                    |
19//! | `\NNN`      | 1–3 octal digits → one byte (value must be ≤ 255)       |
20//! | `\xNN \XNN` | 1–2 hex digits → one byte                               |
21//! | `\uNNNN`    | 4 hex digits → UTF-8 encoding of that code point        |
22//! | `\UNNNNNNNN`| 8 hex digits → UTF-8 encoding of that code point        |
23//!
24//! `\u` is BMP-only — surrogates (U+D800..U+DFFF) are rejected, even as a
25//! well-formed pair. For non-BMP code points use `\U` (e.g. `\U0001F600`).
26
27use alloc::borrow::Cow;
28use alloc::vec::Vec;
29use core::fmt::Write;
30
31/// Error returned by [`unescape`] and [`unescape_str`].
32///
33/// Its `Display` text is the [`BadEscape`](Self::BadEscape) string, or
34/// `unescaped bytes are not valid UTF-8`.
35#[derive(Debug, Clone, Copy, PartialEq, Eq, thiserror::Error)]
36pub enum UnescapeError {
37    /// The unescaped bytes are not valid UTF-8. Only produced by
38    /// [`unescape_str`]; [`unescape`] is byte-level and never returns this.
39    #[error("unescaped bytes are not valid UTF-8")]
40    InvalidUtf8,
41    /// A malformed escape sequence or structural problem in the literal.
42    /// The string describes the specific failure.
43    #[error("{0}")]
44    BadEscape(&'static str),
45}
46
47/// Unescape a textproto string token — one or more adjacent quoted literals —
48/// into a byte vector.
49///
50/// Adjacent literals (`"foo" 'bar'`) are concatenated: whitespace and `#`
51/// comments between them are consumed, the enclosing quotes are stripped, and
52/// escapes are resolved. This matches the textproto grammar's treatment of
53/// `"a" "b"` as a single scalar value `"ab"`.
54///
55/// # Errors
56///
57/// Returns [`UnescapeError::BadEscape`] when `raw` does not begin with `"` or
58/// `'`, when anything other than whitespace and `#` comments follows the last
59/// literal, and on the first malformed escape or unterminated literal.
60pub fn unescape(raw: &str) -> Result<Vec<u8>, UnescapeError> {
61    let mut out = Vec::new();
62    let mut s = raw.as_bytes();
63    loop {
64        // Each iteration consumes one quoted literal. Every iteration after
65        // the first starts at a quote (checked at the bottom of the loop), so
66        // the two error arms report on the start of `raw`.
67        let quote = match s.first() {
68            Some(&quote @ (b'"' | b'\'')) => quote,
69            Some(_) => {
70                return Err(UnescapeError::BadEscape(
71                    "string literal must start with a quote",
72                ));
73            }
74            None => return Err(UnescapeError::BadEscape("unterminated string")),
75        };
76        s = &s[1..];
77        loop {
78            match s.first() {
79                None => return Err(UnescapeError::BadEscape("unterminated string")),
80                Some(&c) if c == quote => {
81                    s = &s[1..];
82                    break;
83                }
84                Some(b'\n') | Some(0) => {
85                    return Err(UnescapeError::BadEscape(
86                        "raw newline/NUL in string literal",
87                    ));
88                }
89                Some(b'\\') => {
90                    s = &s[1..];
91                    let Some(&esc) = s.first() else {
92                        return Err(UnescapeError::BadEscape("unterminated escape"));
93                    };
94                    s = &s[1..];
95                    match esc {
96                        b'"' | b'\'' | b'\\' | b'?' => out.push(esc),
97                        b'n' => out.push(b'\n'),
98                        b'r' => out.push(b'\r'),
99                        b't' => out.push(b'\t'),
100                        b'a' => out.push(0x07),
101                        b'b' => out.push(0x08),
102                        b'f' => out.push(0x0C),
103                        b'v' => out.push(0x0B),
104                        b'0'..=b'7' => {
105                            // 1–3 octal digits, first already consumed.
106                            let mut v = (esc - b'0') as u32;
107                            let mut n = 0;
108                            while n < 2 {
109                                match s.first() {
110                                    Some(&d @ b'0'..=b'7') => {
111                                        v = v * 8 + (d - b'0') as u32;
112                                        s = &s[1..];
113                                        n += 1;
114                                    }
115                                    _ => break,
116                                }
117                            }
118                            if v > 0xFF {
119                                return Err(UnescapeError::BadEscape("octal escape out of range"));
120                            }
121                            out.push(v as u8);
122                        }
123                        b'x' | b'X' => {
124                            // 1–2 hex digits.
125                            let (v, consumed) = take_hex(s, 2);
126                            if consumed == 0 {
127                                return Err(UnescapeError::BadEscape("invalid \\x escape"));
128                            }
129                            s = &s[consumed..];
130                            out.push(v as u8);
131                        }
132                        b'u' => {
133                            let (cp, n) = take_hex(s, 4);
134                            if n != 4 {
135                                return Err(UnescapeError::BadEscape("invalid \\u escape"));
136                            }
137                            s = &s[4..];
138                            // Textproto's \u is BMP-only — surrogate codepoints are
139                            // rejected outright (no UTF-16 pair recombination). For
140                            // non-BMP, use \U00010437 etc.
141                            if (0xD800..0xE000).contains(&cp) {
142                                return Err(UnescapeError::BadEscape(
143                                    "\\u escape is surrogate; use \\U for non-BMP",
144                                ));
145                            }
146                            push_utf8(&mut out, cp)?;
147                        }
148                        b'U' => {
149                            let (cp, n) = take_hex(s, 8);
150                            if n != 8 {
151                                return Err(UnescapeError::BadEscape("invalid \\U escape"));
152                            }
153                            s = &s[8..];
154                            push_utf8(&mut out, cp)?;
155                        }
156                        _ => return Err(UnescapeError::BadEscape("unrecognised escape sequence")),
157                    }
158                }
159                Some(&c) => {
160                    out.push(c);
161                    s = &s[1..];
162                }
163            }
164        }
165        // Skip whitespace and comments. Another quote continues the
166        // concatenation; the end of input completes it.
167        s = super::token::consume_ws(s);
168        match s.first() {
169            Some(b'"' | b'\'') => {}
170            Some(_) => {
171                return Err(UnescapeError::BadEscape(
172                    "unexpected content after string literal",
173                ));
174            }
175            None => return Ok(out),
176        }
177    }
178}
179
180/// Unescape a textproto string token and validate as UTF-8.
181///
182/// Borrows the input when the token is a single literal with no escapes and
183/// no adjacent concatenation — the common case for identifiers and short
184/// strings. Otherwise allocates.
185///
186/// # Errors
187///
188/// As [`unescape`], plus [`UnescapeError::InvalidUtf8`] if the unescaped
189/// bytes are not a valid UTF-8 sequence.
190pub fn unescape_str(raw: &str) -> Result<Cow<'_, str>, UnescapeError> {
191    // Fast path: single literal, ASCII quote, no escapes, no adjacent concat.
192    // Scan once for anything that would force owned output.
193    let bytes = raw.as_bytes();
194    if let Some(&quote) = bytes.first() {
195        if (quote == b'"' || quote == b'\'') && bytes.len() >= 2 {
196            let inner = &bytes[1..];
197            let mut i = 0;
198            while i < inner.len() {
199                let b = inner[i];
200                if b == quote {
201                    // Found closing quote. If nothing follows, we can borrow.
202                    // Trailing whitespace and comments do not change the literal.
203                    let tail = &inner[i + 1..];
204                    if super::token::consume_ws(tail).is_empty() {
205                        // The original `raw` is &str, so this slice is valid
206                        // UTF-8 by construction (no escapes present means no
207                        // byte-level rewriting happened).
208                        return Ok(Cow::Borrowed(&raw[1..1 + i]));
209                    }
210                    break;
211                }
212                if b == b'\\' || b == b'\n' || b == 0 {
213                    break;
214                }
215                i += 1;
216            }
217        }
218    }
219    // Slow path.
220    let owned = unescape(raw)?;
221    alloc::string::String::from_utf8(owned)
222        .map(Cow::Owned)
223        .map_err(|_| UnescapeError::InvalidUtf8)
224}
225
226/// Consume up to `max` hex digits from the front of `s`, returning the
227/// accumulated value and the count of digits consumed.
228#[inline]
229fn take_hex(s: &[u8], max: usize) -> (u32, usize) {
230    let mut v: u32 = 0;
231    let mut n = 0;
232    while n < max && n < s.len() {
233        let d = match s[n] {
234            c @ b'0'..=b'9' => c - b'0',
235            c @ b'a'..=b'f' => c - b'a' + 10,
236            c @ b'A'..=b'F' => c - b'A' + 10,
237            _ => break,
238        };
239        v = (v << 4) | d as u32;
240        n += 1;
241    }
242    (v, n)
243}
244
245/// Push the UTF-8 encoding of `cp` onto `out`.
246#[inline]
247fn push_utf8(out: &mut Vec<u8>, cp: u32) -> Result<(), UnescapeError> {
248    let c = char::from_u32(cp).ok_or(UnescapeError::BadEscape("invalid unicode code point"))?;
249    let mut buf = [0u8; 4];
250    out.extend_from_slice(c.encode_utf8(&mut buf).as_bytes());
251    Ok(())
252}
253
254/// Write `bytes` as a double-quoted textproto string literal to `w`.
255///
256/// Printable ASCII passes through unchanged except for `"` and `\`, which are
257/// escaped. All other bytes (control characters, 0x7F, high-bit bytes) are
258/// emitted as `\NNN` three-digit octal escapes. This produces pure-ASCII
259/// output regardless of the input bytes.
260///
261/// # Errors
262///
263/// Propagates [`core::fmt::Error`] from the underlying writer.
264pub fn escape_bytes<W: Write + ?Sized>(bytes: &[u8], w: &mut W) -> core::fmt::Result {
265    w.write_char('"')?;
266    for &b in bytes {
267        match b {
268            b'"' => w.write_str("\\\"")?,
269            b'\\' => w.write_str("\\\\")?,
270            b'\n' => w.write_str("\\n")?,
271            b'\r' => w.write_str("\\r")?,
272            b'\t' => w.write_str("\\t")?,
273            0x20..=0x7E => w.write_char(b as char)?,
274            _ => {
275                // Three-digit octal. Always three digits so the next char
276                // can't be misread as continuing the escape.
277                w.write_char('\\')?;
278                w.write_char((b'0' + (b >> 6)) as char)?;
279                w.write_char((b'0' + ((b >> 3) & 7)) as char)?;
280                w.write_char((b'0' + (b & 7)) as char)?;
281            }
282        }
283    }
284    w.write_char('"')
285}
286
287/// Write `s` as a double-quoted textproto string literal to `w`, preserving
288/// multi-byte UTF-8 codepoints as-is.
289///
290/// Control characters (`< 0x20`), `"`, `\`, and DEL (0x7F) are escaped.
291/// Everything else — including non-ASCII codepoints — passes through
292/// unmodified. Output is therefore UTF-8, not pure ASCII.
293///
294/// # Errors
295///
296/// Propagates [`core::fmt::Error`] from the underlying writer.
297pub fn escape_str<W: Write + ?Sized>(s: &str, w: &mut W) -> core::fmt::Result {
298    w.write_char('"')?;
299    for c in s.chars() {
300        match c {
301            '"' => w.write_str("\\\"")?,
302            '\\' => w.write_str("\\\\")?,
303            '\n' => w.write_str("\\n")?,
304            '\r' => w.write_str("\\r")?,
305            '\t' => w.write_str("\\t")?,
306            '\x00'..='\x1F' | '\x7F' => {
307                // Octal escape for control chars.
308                let b = c as u8;
309                w.write_char('\\')?;
310                w.write_char((b'0' + (b >> 6)) as char)?;
311                w.write_char((b'0' + ((b >> 3) & 7)) as char)?;
312                w.write_char((b'0' + (b & 7)) as char)?;
313            }
314            _ => w.write_char(c)?,
315        }
316    }
317    w.write_char('"')
318}
319
320#[cfg(test)]
321mod tests {
322    use super::*;
323    use alloc::string::String;
324
325    // ── unescape ────────────────────────────────────────────────────────────
326
327    #[cfg(feature = "std")]
328    #[test]
329    fn unescape_error_is_a_std_error() {
330        fn assert_error<E: std::error::Error>() {}
331        assert_error::<UnescapeError>();
332    }
333
334    #[test]
335    fn unescape_error_display() {
336        assert_eq!(
337            alloc::format!("{}", unescape_str(r#""\xff""#).unwrap_err()),
338            "unescaped bytes are not valid UTF-8"
339        );
340        assert_eq!(
341            alloc::format!("{}", unescape(r#""\xzz""#).unwrap_err()),
342            r"invalid \x escape"
343        );
344    }
345
346    #[test]
347    fn unescape_table() {
348        // Some(bytes) = success, None = error.
349        #[rustfmt::skip]
350        let cases: &[(&str, Option<&[u8]>)] = &[
351            (r#""hello""#,             Some(b"hello")),
352            (r#"'hello'"#,             Some(b"hello")),
353            (r#""""#,                  Some(b"")),
354            (r#""\n""#,                Some(b"\n")),
355            (r#""\r\t""#,              Some(b"\r\t")),
356            (r#""\"\\\'""#,            Some(b"\"\\'")),
357            (r#""\?""#,                Some(b"?")),
358            (r#""\a\b\f\v""#,          Some(&[0x07, 0x08, 0x0C, 0x0B])),
359            (r#""\0""#,                Some(&[0x00])),
360            (r#""\7""#,                Some(&[0x07])),
361            (r#""\77""#,               Some(&[0o77])),       // 63
362            (r#""\377""#,              Some(&[0xFF])),       // 255
363            (r#""\1234""#,             Some(&[0o123, b'4'])),// 3 digits max, then literal
364            (r#""\x41""#,              Some(b"A")),
365            (r#""\xa""#,               Some(&[0x0A])),       // 1 hex digit ok
366            (r#""\xFF""#,              Some(&[0xFF])),
367            (r#""\u0041""#,            Some(b"A")),
368            (r#""\u00e9""#,            Some("é".as_bytes())),
369            (r#""\U0001F600""#,        Some("😀".as_bytes())),
370            (r#""foo" "bar""#,         Some(b"foobar")),        // adjacent concat
371            (r#""foo"'bar'"baz""#,     Some(b"foobarbaz")),     // no ws required
372            (r#""a"  "b""#,            Some(b"ab")),            // ws between is ok
373            ("\"a\" # between\n \"b\"", Some(b"ab")),          // comment between is ok
374            ("\"a\" # \"x\"\n \"b\"",   Some(b"ab")),          // a quote in a comment is not a literal
375            ("\"a\" # trailing",       Some(b"a")),           // comment running to EOF
376            (r##""a#" "#b""##,         Some(b"a##b")),          // `#` inside a literal is data
377            // errors:
378            (r#""unterminated"#,       None),
379            (r#""\"#,                  None),  // lone backslash
380            (r#""\400""#,              None),  // octal > 255
381            (r#""\x""#,                None),  // \x with no digits
382            (r#""\u00""#,              None),  // \u needs 4
383            (r#""\U0000""#,            None),  // \U needs 8
384            (r#""\uD800""#,            None),  // surrogate (BMP-only: use \U)
385            (r#""\uDC00""#,            None),  // surrogate
386            (r#""\uD83D\uDE00""#,      None),  // even a well-formed pair — no JSON-style recombination
387            (r#""\z""#,                None),  // unknown escape
388            ("\"line\nbreak\"",        None),  // raw newline
389        ];
390        for &(input, expected) in cases {
391            let got = unescape(input).ok();
392            assert_eq!(got.as_deref(), expected, "input: {input:?}");
393        }
394    }
395
396    #[test]
397    fn unescape_rejects_content_outside_quotes() {
398        for (input, reason) in [
399            ("", "unterminated string"),
400            ("hello", "string literal must start with a quote"),
401            (" hello", "string literal must start with a quote"),
402            (" \"hello\"", "string literal must start with a quote"),
403            ("\"a\" b", "unexpected content after string literal"),
404            ("\"\" hello", "unexpected content after string literal"),
405            ("\"a\" # c\n b", "unexpected content after string literal"),
406        ] {
407            let want = UnescapeError::BadEscape(reason);
408            assert_eq!(unescape(input), Err(want), "input: {input:?}");
409            assert_eq!(unescape_str(input), Err(want), "input: {input:?}");
410        }
411    }
412
413    #[test]
414    fn unescape_str_borrows_when_trivial() {
415        let got = unescape_str(r#""hello""#).unwrap();
416        assert!(matches!(got, Cow::Borrowed("hello")));
417    }
418
419    #[test]
420    fn unescape_str_owns_when_escaped() {
421        let got = unescape_str(r#""hel\nlo""#).unwrap();
422        assert!(matches!(got, Cow::Owned(_)));
423        assert_eq!(got, "hel\nlo");
424    }
425
426    #[test]
427    fn unescape_str_owns_when_concatenated() {
428        let got = unescape_str(r#""foo" "bar""#).unwrap();
429        assert!(matches!(got, Cow::Owned(_)));
430        assert_eq!(got, "foobar");
431    }
432
433    #[test]
434    fn unescape_str_borrows_with_trailing_ws() {
435        // Tokenizer may hand over a raw span with trailing whitespace.
436        let got = unescape_str("\"hello\"  ").unwrap();
437        assert!(matches!(got, Cow::Borrowed("hello")));
438    }
439
440    #[test]
441    fn unescape_str_borrows_with_trailing_comments() {
442        for (input, expected) in [
443            ("\"hello\"# trailing", "hello"),
444            ("'hello' \t# trailing\n \r\x0B\x0C", "hello"),
445            ("\"café 😀\" # café 😀\n# another comment\n", "café 😀"),
446            ("\"hello\" # \"another literal\" \\z", "hello"),
447            ("\"\" # trailing", ""),
448            ("\"a#b\" # trailing", "a#b"),
449        ] {
450            let got = unescape_str(input).unwrap();
451            assert_eq!(got, expected, "input: {input:?}");
452            assert!(matches!(got, Cow::Borrowed(_)), "input: {input:?}");
453        }
454    }
455
456    #[test]
457    fn unescape_str_owns_when_comments_separate_literals() {
458        for input in ["\"foo\" # between\n \"bar\"", "'foo' # between\n 'bar'"] {
459            let got = unescape_str(input).unwrap();
460            assert_eq!(got, "foobar", "input: {input:?}");
461            assert!(matches!(got, Cow::Owned(_)), "input: {input:?}");
462        }
463    }
464
465    #[test]
466    fn unescape_str_owns_when_escaped_with_trailing_comment() {
467        let got = unescape_str("\"hel\\nlo\" # trailing").unwrap();
468        assert!(matches!(got, Cow::Owned(_)));
469        assert_eq!(got, "hel\nlo");
470    }
471
472    #[test]
473    fn unescape_str_rejects_bad_utf8() {
474        // \xFF is a valid textproto byte escape but not a valid UTF-8 byte.
475        assert!(unescape(r#""\xFF""#).is_ok());
476        assert!(unescape_str(r#""\xFF""#).is_err());
477    }
478
479    #[test]
480    fn unescape_uppercase_hex_escapes() {
481        let cases: &[(&str, &[u8])] = &[
482            (r#""\X41""#, b"A"),
483            (r#"'\X4'"#, &[0x04]),
484            (r#""\Xff\XaB\X0""#, &[0xFF, 0xAB, 0x00]),
485            (r#""\X414""#, b"A4"),
486            (r#""\X4G""#, &[0x04, b'G']),
487            (r#""\x41" '\X42'"#, b"AB"),
488            ("\"\\X41\" # between\n '\\X42'", b"AB"),
489            (r#""\XC3\xA9""#, "é".as_bytes()),
490        ];
491        for &(input, expected) in cases {
492            assert_eq!(unescape(input).unwrap(), expected, "input: {input:?}");
493        }
494    }
495
496    #[test]
497    fn unescape_str_uppercase_hex_escapes() {
498        for (input, expected) in [(r#""\X41""#, "A"), (r#""\XC3\XA9""#, "é")] {
499            let decoded = unescape_str(input).unwrap();
500            assert!(matches!(decoded, Cow::Owned(_)));
501            assert_eq!(decoded, expected, "input: {input:?}");
502        }
503        assert_eq!(unescape_str(r#""\Xff""#), Err(UnescapeError::InvalidUtf8));
504    }
505
506    #[test]
507    fn unescape_uppercase_hex_requires_a_digit() {
508        for input in [r#""\X""#, r#"'\Xg'"#, r#""\X-1""#] {
509            let expected = UnescapeError::BadEscape("invalid \\x escape");
510            assert_eq!(unescape(input), Err(expected), "input: {input:?}");
511            assert_eq!(unescape_str(input), Err(expected), "input: {input:?}");
512        }
513    }
514
515    // ── escape ──────────────────────────────────────────────────────────────
516
517    #[test]
518    fn escape_bytes_table() {
519        #[rustfmt::skip]
520        let cases: &[(&[u8], &str)] = &[
521            (b"hello",        r#""hello""#),
522            (b"",             r#""""#),
523            (b"\n\r\t",       r#""\n\r\t""#),
524            (b"\"\\",         r#""\"\\""#),
525            (&[0x00],         r#""\000""#),
526            (&[0x01],         r#""\001""#),
527            (&[0x7F],         r#""\177""#),
528            (&[0xFF],         r#""\377""#),
529            (&[0x80, 0x81],   r#""\200\201""#),   // high bytes → octal
530            (b"a'b",          r#""a'b""#),        // ' not escaped under "
531        ];
532        for &(input, want) in cases {
533            let mut out = String::new();
534            escape_bytes(input, &mut out).unwrap();
535            assert_eq!(out, want, "input: {input:?}");
536        }
537    }
538
539    #[test]
540    fn escape_str_preserves_unicode() {
541        let mut out = String::new();
542        escape_str("café 😀", &mut out).unwrap();
543        assert_eq!(out, r#""café 😀""#);
544    }
545
546    #[test]
547    fn escape_str_escapes_controls() {
548        let mut out = String::new();
549        escape_str("\x01\x7F", &mut out).unwrap();
550        assert_eq!(out, r#""\001\177""#);
551    }
552
553    #[test]
554    fn escape_bytes_vs_str_non_ascii() {
555        // escape_bytes treats each byte of "é" (0xC3 0xA9) as a raw byte → octal.
556        // escape_str treats it as one codepoint → passthrough.
557        let mut a = String::new();
558        let mut b = String::new();
559        escape_bytes("é".as_bytes(), &mut a).unwrap();
560        escape_str("é", &mut b).unwrap();
561        assert_eq!(a, r#""\303\251""#);
562        assert_eq!(b, r#""é""#);
563    }
564
565    #[test]
566    fn escape_unescape_roundtrip() {
567        // escape_bytes output is always valid unescape input.
568        #[rustfmt::skip]
569        let cases: &[&[u8]] = &[
570            b"",
571            b"plain",
572            b"\x00\x01\x02\xFD\xFE\xFF",
573            b"mix\nall\"the\\things\x7F",
574            &[0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13],
575        ];
576        for &input in cases {
577            let mut escaped = String::new();
578            escape_bytes(input, &mut escaped).unwrap();
579            let back = unescape(&escaped).unwrap();
580            assert_eq!(back, input, "escaped form: {escaped}");
581        }
582    }
583}