jsonc_parser/
string.rs

1use std::borrow::Cow;
2
3pub struct ParseStringError {
4  pub byte_index: usize,
5  pub kind: ParseStringErrorKind,
6}
7
8#[derive(Debug, Clone, PartialEq, Eq, Hash)]
9pub enum ParseStringErrorKind {
10  InvalidEscapeInSingleQuoteString,
11  InvalidEscapeInDoubleQuoteString,
12  ExpectedFourHexDigits,
13  ExpectedTwoHexDigits,
14  InvalidUnicodeEscapeSequence(String),
15  InvalidEscape,
16  UnterminatedStringLiteral,
17}
18
19impl std::error::Error for ParseStringErrorKind {}
20
21impl std::fmt::Display for ParseStringErrorKind {
22  fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
23    match self {
24      ParseStringErrorKind::InvalidEscapeInSingleQuoteString => {
25        write!(f, "Invalid escape in single quote string")
26      }
27      ParseStringErrorKind::InvalidEscapeInDoubleQuoteString => {
28        write!(f, "Invalid escape in double quote string")
29      }
30      ParseStringErrorKind::ExpectedFourHexDigits => {
31        write!(f, "Expected four hex digits")
32      }
33      ParseStringErrorKind::ExpectedTwoHexDigits => {
34        write!(f, "Expected two hex digits")
35      }
36      ParseStringErrorKind::InvalidUnicodeEscapeSequence(value) => {
37        write!(
38          f,
39          "Invalid unicode escape sequence. '{}' is not a valid UTF8 character",
40          value
41        )
42      }
43      ParseStringErrorKind::InvalidEscape => {
44        write!(f, "Invalid escape")
45      }
46      ParseStringErrorKind::UnterminatedStringLiteral => {
47        write!(f, "Unterminated string literal")
48      }
49    }
50  }
51}
52
53pub trait CharProvider<'a> {
54  fn current_char(&mut self) -> Option<char>;
55  fn byte_index(&self) -> usize;
56  fn move_next_char(&mut self) -> Option<char>;
57  fn text(&self) -> &'a str;
58}
59
60#[cfg(feature = "cst")]
61pub fn parse_string(text: &str) -> Result<Cow<'_, str>, ParseStringError> {
62  struct StringCharProvider<'a> {
63    text: &'a str,
64    byte_index: usize,
65    current_char: Option<char>,
66    chars: std::str::Chars<'a>,
67  }
68
69  impl<'a> CharProvider<'a> for StringCharProvider<'a> {
70    fn current_char(&mut self) -> Option<char> {
71      self.current_char
72    }
73
74    fn byte_index(&self) -> usize {
75      self.byte_index
76    }
77
78    fn move_next_char(&mut self) -> Option<char> {
79      if let Some(current_char) = self.current_char {
80        self.byte_index += current_char.len_utf8();
81      }
82      self.current_char = self.chars.next();
83      self.current_char
84    }
85
86    fn text(&self) -> &'a str {
87      self.text
88    }
89  }
90
91  let mut chars = text.chars();
92  let mut provider = StringCharProvider {
93    text,
94    byte_index: 0,
95    current_char: chars.next(),
96    chars,
97  };
98
99  // the scanner already validated this text, or the caller set it raw, so decode it leniently
100  parse_string_with_char_provider(&mut provider, true)
101}
102
103pub fn parse_string_with_char_provider<'a, T: CharProvider<'a>>(
104  chars: &mut T,
105  allow_extended_escapes: bool,
106) -> Result<Cow<'a, str>, ParseStringError> {
107  debug_assert!(
108    chars.current_char() == Some('\'') || chars.current_char() == Some('"'),
109    "Expected \", was {:?}",
110    chars.current_char()
111  );
112  let is_double_quote = chars.current_char() == Some('"');
113  let mut last_start_byte_index = chars.byte_index() + 1;
114  let mut text: Option<String> = None;
115  let mut last_was_backslash = false;
116  let mut found_end_string = false;
117  let token_start = chars.byte_index();
118
119  while let Some(current_char) = chars.move_next_char() {
120    if last_was_backslash {
121      let escape_start = chars.byte_index() - 1; // -1 for backslash
122      if allow_extended_escapes && let Some(decoded) = parse_extended_escape(chars, current_char, escape_start)? {
123        let previous_text = &chars.text()[last_start_byte_index..escape_start];
124        let text = text.get_or_insert_with(String::new);
125        text.push_str(previous_text);
126        if let Some(decoded) = decoded {
127          text.push(decoded);
128        }
129        last_start_byte_index = chars.byte_index() + chars.current_char().map(|c| c.len_utf8()).unwrap_or(0);
130        last_was_backslash = false;
131        continue;
132      }
133      match current_char {
134        '"' | '\'' | '\\' | '/' | 'b' | 'f' | 'u' | 'r' | 'n' | 't' => {
135          if current_char == '"' {
136            if !is_double_quote {
137              return Err(ParseStringError {
138                byte_index: escape_start,
139                kind: ParseStringErrorKind::InvalidEscapeInSingleQuoteString,
140              });
141            }
142          } else if current_char == '\'' && is_double_quote {
143            return Err(ParseStringError {
144              byte_index: escape_start,
145              kind: ParseStringErrorKind::InvalidEscapeInDoubleQuoteString,
146            });
147          }
148
149          let previous_text = &chars.text()[last_start_byte_index..escape_start];
150          if text.is_none() {
151            text = Some(String::new());
152          }
153          let text = text.as_mut().unwrap();
154          text.push_str(previous_text);
155          if current_char == 'u' {
156            let hex_char = parse_hex_char(chars).map_err(|kind| ParseStringError {
157              byte_index: escape_start,
158              kind,
159            })?;
160            text.push(hex_char);
161            last_start_byte_index = chars.byte_index() + chars.current_char().map(|c| c.len_utf8()).unwrap_or(0);
162          } else {
163            text.push(match current_char {
164              'b' => '\u{08}',
165              'f' => '\u{0C}',
166              't' => '\t',
167              'r' => '\r',
168              'n' => '\n',
169              _ => current_char,
170            });
171            last_start_byte_index = chars.byte_index() + current_char.len_utf8();
172          }
173        }
174        _ => {
175          return Err(ParseStringError {
176            byte_index: escape_start,
177            kind: ParseStringErrorKind::InvalidEscape,
178          });
179        }
180      }
181      last_was_backslash = false;
182    } else if is_double_quote && current_char == '"' || !is_double_quote && current_char == '\'' {
183      found_end_string = true;
184      break;
185    } else {
186      last_was_backslash = current_char == '\\';
187    }
188  }
189
190  if found_end_string {
191    chars.move_next_char();
192    let final_segment = &chars.text()[last_start_byte_index..chars.byte_index() - 1];
193    Ok(match text {
194      Some(mut text) => {
195        text.push_str(final_segment);
196        Cow::Owned(text)
197      }
198      None => Cow::Borrowed(final_segment),
199    })
200  } else {
201    Err(ParseStringError {
202      byte_index: token_start,
203      kind: ParseStringErrorKind::UnterminatedStringLiteral,
204    })
205  }
206}
207
208// `Ok(None)` leaves the escape to the JSON path; `Ok(Some(None))` is a line continuation, which decodes to nothing
209fn parse_extended_escape<'a, T: CharProvider<'a>>(
210  chars: &mut T,
211  current_char: char,
212  escape_start: usize,
213) -> Result<Option<Option<char>>, ParseStringError> {
214  let invalid_escape = || ParseStringError {
215    byte_index: escape_start,
216    kind: ParseStringErrorKind::InvalidEscape,
217  };
218  let decoded = match current_char {
219    'v' => Some('\u{0B}'),
220    '0' => {
221      // `\0` must not be followed by a digit, which would make it an octal escape
222      let next_index = chars.byte_index() + 1;
223      if chars.text()[next_index..].starts_with(|c: char| c.is_ascii_digit()) {
224        return Err(invalid_escape());
225      }
226      Some('\0')
227    }
228    'x' => {
229      let mut value = 0;
230      for _ in 0..2 {
231        match chars.move_next_char().and_then(|c| c.to_digit(16)) {
232          Some(digit) => value = value * 16 + digit,
233          None => {
234            return Err(ParseStringError {
235              byte_index: escape_start,
236              kind: ParseStringErrorKind::ExpectedTwoHexDigits,
237            });
238          }
239        }
240      }
241      // two hex digits are at most 0xFF, which is always a valid char
242      Some(char::from_u32(value).unwrap())
243    }
244    '\n' | '\u{2028}' | '\u{2029}' => None,
245    '\r' => {
246      if chars.text()[chars.byte_index() + 1..].starts_with('\n') {
247        chars.move_next_char();
248      }
249      None
250    }
251    '1'..='9' => return Err(invalid_escape()),
252    '\\' | '/' | 'b' | 'f' | 'u' | 'r' | 'n' | 't' => return Ok(None),
253    _ => Some(current_char),
254  };
255  Ok(Some(decoded))
256}
257
258fn read_four_hex_digits<'a, T: CharProvider<'a>>(
259  chars: &mut T,
260  buf: &mut [u8; 4],
261) -> Result<u32, ParseStringErrorKind> {
262  for slot in buf.iter_mut() {
263    match chars.move_next_char() {
264      Some(c) if c.is_ascii_hexdigit() => *slot = c as u8,
265      _ => return Err(ParseStringErrorKind::ExpectedFourHexDigits),
266    }
267  }
268  // safety: buf contains only ASCII hex digits
269  let hex_str = std::str::from_utf8(buf).unwrap();
270  u32::from_str_radix(hex_str, 16).map_err(|_| ParseStringErrorKind::InvalidUnicodeEscapeSequence(hex_str.to_string()))
271}
272
273fn hex_buf_to_str(buf: &[u8; 4]) -> String {
274  std::str::from_utf8(buf).unwrap().to_string()
275}
276
277fn parse_hex_char<'a, T: CharProvider<'a>>(chars: &mut T) -> Result<char, ParseStringErrorKind> {
278  let mut buf1 = [0u8; 4];
279  let hex_value = read_four_hex_digits(chars, &mut buf1)?;
280
281  // check if this is a high surrogate (0xD800-0xDBFF)
282  let hex_char = if (0xD800..=0xDBFF).contains(&hex_value) {
283    // high surrogate - must be followed by low surrogate (\uXXXX)
284    if chars.move_next_char() != Some('\\') {
285      return Err(ParseStringErrorKind::InvalidUnicodeEscapeSequence(format!(
286        "{} (unpaired high surrogate)",
287        hex_buf_to_str(&buf1)
288      )));
289    }
290
291    if chars.move_next_char() != Some('u') {
292      return Err(ParseStringErrorKind::InvalidUnicodeEscapeSequence(format!(
293        "{} (unpaired high surrogate)",
294        hex_buf_to_str(&buf1)
295      )));
296    }
297
298    // parse the second \uXXXX
299    let mut buf2 = [0u8; 4];
300    let hex_value2 = read_four_hex_digits(chars, &mut buf2)?;
301
302    // verify it's a low surrogate (0xDC00-0xDFFF)
303    if !(0xDC00..=0xDFFF).contains(&hex_value2) {
304      return Err(ParseStringErrorKind::InvalidUnicodeEscapeSequence(format!(
305        "{} (high surrogate not followed by low surrogate)",
306        hex_buf_to_str(&buf1)
307      )));
308    }
309
310    // combine surrogate pair using RFC 8259 formula
311    let code_point = ((hex_value - 0xD800) * 0x400) + (hex_value2 - 0xDC00) + 0x10000;
312
313    match std::char::from_u32(code_point) {
314      Some(c) => c,
315      None => {
316        return Err(ParseStringErrorKind::InvalidUnicodeEscapeSequence(format!(
317          "{}\\u{} (invalid surrogate pair)",
318          hex_buf_to_str(&buf1),
319          hex_buf_to_str(&buf2)
320        )));
321      }
322    }
323  } else if (0xDC00..=0xDFFF).contains(&hex_value) {
324    // low surrogate without high surrogate
325    return Err(ParseStringErrorKind::InvalidUnicodeEscapeSequence(format!(
326      "{} (unpaired low surrogate)",
327      hex_buf_to_str(&buf1)
328    )));
329  } else {
330    // normal unicode escape
331    match std::char::from_u32(hex_value) {
332      Some(hex_char) => hex_char,
333      None => {
334        return Err(ParseStringErrorKind::InvalidUnicodeEscapeSequence(hex_buf_to_str(
335          &buf1,
336        )));
337      }
338    }
339  };
340  Ok(hex_char)
341}
jsonc_parser/string.rs

jsonc_parser/
string.rs
