rupg_types/
json.rs

1//! `json` in the text format.
2//!
3//! A `json` value is its text, so the input function only checks the syntax. This is a port of `json_in` in `src/backend/utils/adt/json.c` and of the lexer and the parser in `src/common/jsonapi.c`, with the same detail text for each error. The parser of PostgreSQL is recursive descent. This one keeps a stack of the open arrays and objects instead, so it takes the tokens in the same order and finds the same first error.
4//!
5//! `json_in` does not decode the escapes, so it takes `\u0000` and a lone surrogate such as `\ud800`. Only `jsonb` refuses them: its lexer decodes each string as it reads it, which is `need_escapes` in `jsonapi.c`, and gives the events of the parse to [`Sink`].
6//!
7//! Lifted from `crates/rudb-pgtypes/src/json.rs` of tamnd/rudb at f5f7065a (spec/04 section 4.9).
8
9use rupg_common::SqlState;
10
11use crate::error::TypeError;
12
13#[derive(Debug, Clone, Copy, PartialEq, Eq)]
14pub(crate) enum Kind {
15    ObjectStart,
16    ObjectEnd,
17    ArrayStart,
18    ArrayEnd,
19    Comma,
20    Colon,
21    String,
22    Number,
23    True,
24    False,
25    Null,
26    End,
27}
28
29/// A token and its bytes in the input. The bytes go in the detail of an error.
30#[derive(Debug, Clone, Copy)]
31struct Token {
32    kind: Kind,
33    start: usize,
34    end: usize,
35}
36
37/// The errors of `JsonParseErrorType` that `json_in` can give.
38#[derive(Debug, Clone, Copy, PartialEq, Eq)]
39enum Error {
40    InvalidToken,
41    EscapingInvalid,
42    EscapingRequired,
43    UnicodeEscapeFormat,
44    ExpectedEnd,
45    ExpectedArrayNext,
46    ExpectedColon,
47    ExpectedJson,
48    ExpectedMore,
49    ExpectedObjectFirst,
50    ExpectedObjectNext,
51    ExpectedString,
52    CodePointZero,
53    HighSurrogate,
54    LowSurrogate,
55}
56
57/// An error and the bytes of the token that it is about.
58#[derive(Debug, Clone, Copy)]
59pub(crate) struct Fail {
60    error: Error,
61    start: usize,
62    end: usize,
63}
64
65/// `JSON_ALPHANUMERIC_CHAR`: a byte that the lexer keeps in a bad word, so that the error shows the whole word. Each byte of a character that is not ASCII is in it.
66fn alphanumeric(b: u8) -> bool {
67    b.is_ascii_alphanumeric() || b == b'_' || b >= 0x80
68}
69
70struct Lexer<'a> {
71    input: &'a [u8],
72    at: usize,
73    /// Whether the lexer decodes the strings into `text`.
74    escapes: bool,
75    /// The last string, with its escapes decoded, when `escapes` is set.
76    text: String,
77}
78
79impl Lexer<'_> {
80    /// `json_lex`.
81    fn next(&mut self) -> Result<Token, Fail> {
82        let b = self.input;
83        while self.at < b.len() && matches!(b[self.at], b' ' | b'\t' | b'\n' | b'\r') {
84            self.at += 1;
85        }
86        let start = self.at;
87        let token = |kind, end| Token { kind, start, end };
88        let Some(&first) = b.get(start) else {
89            return Ok(token(Kind::End, start));
90        };
91        let kind = match first {
92            b'{' => Kind::ObjectStart,
93            b'}' => Kind::ObjectEnd,
94            b'[' => Kind::ArrayStart,
95            b']' => Kind::ArrayEnd,
96            b',' => Kind::Comma,
97            b':' => Kind::Colon,
98            b'"' => {
99                self.at = self.string(start)?;
100                return Ok(token(Kind::String, self.at));
101            }
102            b'-' => {
103                self.at = self.number(start, start + 1)?;
104                return Ok(token(Kind::Number, self.at));
105            }
106            b'0'..=b'9' => {
107                self.at = self.number(start, start)?;
108                return Ok(token(Kind::Number, self.at));
109            }
110            _ => {
111                let end = start + b[start..].iter().take_while(|&&c| alphanumeric(c)).count();
112                if end == start {
113                    return Err(Fail { error: Error::InvalidToken, start, end: start + 1 });
114                }
115                self.at = end;
116                return match &b[start..end] {
117                    b"true" => Ok(token(Kind::True, end)),
118                    b"false" => Ok(token(Kind::False, end)),
119                    b"null" => Ok(token(Kind::Null, end)),
120                    _ => Err(Fail { error: Error::InvalidToken, start, end }),
121                };
122            }
123        };
124        self.at = start + 1;
125        Ok(token(kind, self.at))
126    }
127
128    /// The end of the character that starts at `at`, for an error that shows that character.
129    fn char_end(&self, at: usize) -> usize {
130        let len = match self.input[at] {
131            b if b < 0x80 => 1,
132            b if b & 0xe0 == 0xc0 => 2,
133            b if b & 0xf0 == 0xe0 => 3,
134            _ => 4,
135        };
136        (at + len).min(self.input.len())
137    }
138
139    /// `json_lex_string`: the string that starts with the quote at `start`. Gives the end of the string after the closing quote. With `escapes` set, the string goes into `text` with its escapes decoded, and a surrogate pair is one character.
140    fn string(&mut self, start: usize) -> Result<usize, Fail> {
141        let b = self.input;
142        let unterminated = Fail { error: Error::InvalidToken, start, end: b.len() };
143        let escapes = self.escapes;
144        self.text.clear();
145        let mut high: Option<u32> = None;
146        let mut s = start + 1;
147        loop {
148            // The bytes that need no work. A byte below 32 must be an escape.
149            let run = b[s..].iter().position(|&c| c == b'"' || c == b'\\' || c < 32);
150            let Some(run) = run else {
151                return Err(unterminated);
152            };
153            if escapes && run > 0 {
154                if high.is_some() {
155                    let end = self.char_end(s);
156                    return Err(Fail { error: Error::LowSurrogate, start, end });
157                }
158                self.text.push_str(std::str::from_utf8(&b[s..s + run]).unwrap_or_default());
159            }
160            s += run;
161            match b[s] {
162                b'"' => {
163                    if high.is_some() {
164                        return Err(Fail { error: Error::LowSurrogate, start, end: s + 1 });
165                    }
166                    return Ok(s + 1);
167                }
168                b'\\' => {
169                    s += 1;
170                    match b.get(s) {
171                        None => return Err(unterminated),
172                        Some(b'u') => {
173                            let mut ch = 0u32;
174                            for _ in 0..4 {
175                                s += 1;
176                                match b.get(s) {
177                                    None => return Err(unterminated),
178                                    Some(c) if c.is_ascii_hexdigit() => {
179                                        ch = ch * 16 + char::from(*c).to_digit(16).unwrap_or(0);
180                                    }
181                                    Some(_) => {
182                                        let end = self.char_end(s);
183                                        return Err(Fail {
184                                            error: Error::UnicodeEscapeFormat,
185                                            start,
186                                            end,
187                                        });
188                                    }
189                                }
190                            }
191                            if escapes {
192                                let fail = |error| Fail { error, start, end: s + 1 };
193                                if (0xd800..=0xdbff).contains(&ch) {
194                                    if high.is_some() {
195                                        return Err(fail(Error::HighSurrogate));
196                                    }
197                                    high = Some(ch);
198                                    s += 1;
199                                    continue;
200                                }
201                                if (0xdc00..=0xdfff).contains(&ch) {
202                                    let Some(first) = high.take() else {
203                                        return Err(fail(Error::LowSurrogate));
204                                    };
205                                    ch = 0x10000 + ((first & 0x3ff) << 10) + (ch & 0x3ff);
206                                }
207                                if high.is_some() {
208                                    return Err(fail(Error::LowSurrogate));
209                                }
210                                if ch == 0 {
211                                    return Err(fail(Error::CodePointZero));
212                                }
213                                self.text.push(char::from_u32(ch).unwrap_or('\u{fffd}'));
214                            }
215                        }
216                        Some(&c @ (b'"' | b'\\' | b'/' | b'b' | b'f' | b'n' | b'r' | b't')) => {
217                            if escapes {
218                                if high.is_some() {
219                                    return Err(Fail {
220                                        error: Error::LowSurrogate,
221                                        start,
222                                        end: s + 1,
223                                    });
224                                }
225                                self.text.push(match c {
226                                    b'b' => '\u{8}',
227                                    b'f' => '\u{c}',
228                                    b'n' => '\n',
229                                    b'r' => '\r',
230                                    b't' => '\t',
231                                    other => char::from(other),
232                                });
233                            }
234                        }
235                        Some(_) => {
236                            if escapes && high.is_some() {
237                                let end = self.char_end(s);
238                                return Err(Fail { error: Error::LowSurrogate, start, end });
239                            }
240                            // The error shows only the escape and not the whole string.
241                            let end = self.char_end(s);
242                            return Err(Fail { error: Error::EscapingInvalid, start: s, end });
243                        }
244                    }
245                    s += 1;
246                }
247                _ => return Err(Fail { error: Error::EscapingRequired, start, end: s }),
248            }
249        }
250    }
251
252    /// `json_lex_number`: an optional minus sign, then `0` or digits that do not start with `0`, then an optional fraction and an optional exponent. Letters and digits after the number are part of the bad token. `s` is the byte after the sign.
253    fn number(&self, start: usize, mut s: usize) -> Result<usize, Fail> {
254        let b = self.input;
255        let digit = |s: usize| b.get(s).is_some_and(u8::is_ascii_digit);
256        let digits = |mut s: usize| {
257            while digit(s) {
258                s += 1;
259            }
260            s
261        };
262        let mut error = false;
263        match b.get(s) {
264            Some(b'0') => s += 1,
265            Some(b'1'..=b'9') => s = digits(s),
266            _ => error = true,
267        }
268        if b.get(s) == Some(&b'.') {
269            s += 1;
270            if digit(s) { s = digits(s) } else { error = true }
271        }
272        if matches!(b.get(s), Some(b'e' | b'E')) {
273            s += 1;
274            if matches!(b.get(s), Some(b'+' | b'-')) {
275                s += 1;
276            }
277            if digit(s) { s = digits(s) } else { error = true }
278        }
279        while b.get(s).is_some_and(|&c| alphanumeric(c)) {
280            s += 1;
281            error = true;
282        }
283        if error {
284            return Err(Fail { error: Error::InvalidToken, start, end: s });
285        }
286        Ok(s)
287    }
288}
289
290/// An array or an object that the parser is in.
291#[derive(Debug, Clone, Copy, PartialEq, Eq)]
292pub(crate) enum Open {
293    Array,
294    Object,
295}
296
297/// What the parser reads next.
298enum Step {
299    /// A value: `parse_object`, `parse_array` or `parse_scalar`.
300    Value,
301    /// A field of an object: `parse_object_field`.
302    Field,
303    /// After a value: a comma, the end of the array or the object around it, or the end of the input.
304    After,
305}
306
307/// `report_parse_error`: the error when the token is not one that the context can take.
308fn unexpected(token: Token, error: Error) -> Fail {
309    let error = if token.kind == Kind::End { Error::ExpectedMore } else { error };
310    Fail { error, start: token.start, end: token.end }
311}
312
313/// The semantic actions of a parse, in the order of `pg_parse_json`. An action can stop the parse with an error. The order decides which error comes first:
314///
315/// - The start of an array or an object comes before the lexer reads the next token.
316/// - The end of an array or an object comes after the lexer reads the token after it, as in `parse_array` and `parse_object`.
317/// - The key of a field and the start of an element of an array come after the lexer reads the first token of the value.
318/// - A scalar comes after the lexer reads the next token, as in `parse_scalar`.
319pub(crate) trait Sink {
320    /// The start of an array or an object.
321    ///
322    /// # Errors
323    ///
324    /// The error of the action, which stops the parse.
325    fn open(&mut self, open: Open) -> Result<(), TypeError>;
326    /// The end of the array or the object that was opened last.
327    ///
328    /// # Errors
329    ///
330    /// The error of the action, which stops the parse.
331    fn close(&mut self, open: Open) -> Result<(), TypeError>;
332    /// The key of the next field of an object, decoded.
333    ///
334    /// # Errors
335    ///
336    /// The error of the action, which stops the parse.
337    fn key(&mut self, key: &str) -> Result<(), TypeError>;
338    /// The start of an element of an array, with true when the element is `null`.
339    ///
340    /// # Errors
341    ///
342    /// The error of the action, which stops the parse.
343    fn element(&mut self, null: bool) -> Result<(), TypeError> {
344        let _ = null;
345        Ok(())
346    }
347    /// A scalar: its kind, its bytes in the input and, for a string, its decoded text.
348    ///
349    /// # Errors
350    ///
351    /// The error of the action, which stops the parse.
352    fn scalar(&mut self, kind: Kind, token: &str, text: &str) -> Result<(), TypeError>;
353    /// [`Sink::open`] with the place of the bracket in the input.
354    ///
355    /// # Errors
356    ///
357    /// The error of the action, which stops the parse.
358    fn open_at(&mut self, open: Open, start: usize) -> Result<(), TypeError> {
359        let _ = start;
360        self.open(open)
361    }
362    /// [`Sink::close`] with the place just after the bracket in the input.
363    ///
364    /// # Errors
365    ///
366    /// The error of the action, which stops the parse.
367    fn close_at(&mut self, open: Open, end: usize) -> Result<(), TypeError> {
368        let _ = end;
369        self.close(open)
370    }
371    /// [`Sink::scalar`] with the place of the token in the input.
372    ///
373    /// # Errors
374    ///
375    /// The error of the action, which stops the parse.
376    fn scalar_at(
377        &mut self,
378        kind: Kind,
379        start: usize,
380        token: &str,
381        text: &str,
382    ) -> Result<(), TypeError> {
383        let _ = start;
384        self.scalar(kind, token, text)
385    }
386}
387
388/// Why a parse stopped.
389pub(crate) enum Stop {
390    Syntax(Fail),
391    Action(TypeError),
392}
393
394impl From<Fail> for Stop {
395    fn from(fail: Fail) -> Stop {
396        Stop::Syntax(fail)
397    }
398}
399
400/// A sink with no actions, for `json_in`.
401struct Check;
402
403impl Sink for Check {
404    fn open(&mut self, _: Open) -> Result<(), TypeError> {
405        Ok(())
406    }
407    fn close(&mut self, _: Open) -> Result<(), TypeError> {
408        Ok(())
409    }
410    fn key(&mut self, _: &str) -> Result<(), TypeError> {
411        Ok(())
412    }
413    fn scalar(&mut self, _: Kind, _: &str, _: &str) -> Result<(), TypeError> {
414        Ok(())
415    }
416}
417
418/// `pg_parse_json`, which gives each event to `sink`. With `escapes` set, the lexer decodes the strings and refuses the escapes that do not make text.
419pub(crate) fn parse(input: &str, escapes: bool, sink: &mut impl Sink) -> Result<(), Stop> {
420    let text = |token: Token| &input[token.start..token.end];
421    let input = input.as_bytes();
422    let mut lexer = Lexer { input, at: 0, escapes, text: String::new() };
423    let mut stack = Vec::new();
424    // The decoded text of the last scalar. The lexer reads the next token before the action takes the scalar, so the two texts swap.
425    let mut decoded = String::new();
426    // The key of the field that the parser is in.
427    let mut name = String::new();
428    let mut token = lexer.next()?;
429    let mut step = Step::Value;
430    loop {
431        step = match step {
432            Step::Value => {
433                if stack.last() == Some(&Open::Array) {
434                    sink.element(token.kind == Kind::Null).map_err(Stop::Action)?;
435                }
436                match token.kind {
437                    Kind::ObjectStart => {
438                        stack.push(Open::Object);
439                        sink.open_at(Open::Object, token.start).map_err(Stop::Action)?;
440                        token = lexer.next()?;
441                        match token.kind {
442                            Kind::String => Step::Field,
443                            Kind::ObjectEnd => {
444                                stack.pop();
445                                let end = token.end;
446                                token = lexer.next()?;
447                                sink.close_at(Open::Object, end).map_err(Stop::Action)?;
448                                Step::After
449                            }
450                            _ => return Err(unexpected(token, Error::ExpectedObjectFirst).into()),
451                        }
452                    }
453                    Kind::ArrayStart => {
454                        sink.open_at(Open::Array, token.start).map_err(Stop::Action)?;
455                        token = lexer.next()?;
456                        if token.kind == Kind::ArrayEnd {
457                            let end = token.end;
458                            token = lexer.next()?;
459                            sink.close_at(Open::Array, end).map_err(Stop::Action)?;
460                            Step::After
461                        } else {
462                            stack.push(Open::Array);
463                            Step::Value
464                        }
465                    }
466                    Kind::String | Kind::Number | Kind::True | Kind::False | Kind::Null => {
467                        // `parse_scalar` reads the next token before it calls the action.
468                        let scalar = token;
469                        std::mem::swap(&mut lexer.text, &mut decoded);
470                        token = lexer.next()?;
471                        sink.scalar_at(scalar.kind, scalar.start, text(scalar), &decoded)
472                            .map_err(Stop::Action)?;
473                        Step::After
474                    }
475                    _ => return Err(unexpected(token, Error::ExpectedJson).into()),
476                }
477            }
478            Step::Field => {
479                if token.kind != Kind::String {
480                    return Err(unexpected(token, Error::ExpectedString).into());
481                }
482                name.clear();
483                name.push_str(&lexer.text);
484                token = lexer.next()?;
485                if token.kind != Kind::Colon {
486                    return Err(unexpected(token, Error::ExpectedColon).into());
487                }
488                token = lexer.next()?;
489                sink.key(&name).map_err(Stop::Action)?;
490                Step::Value
491            }
492            Step::After => {
493                let Some(&open) = stack.last() else {
494                    if token.kind != Kind::End {
495                        return Err(unexpected(token, Error::ExpectedEnd).into());
496                    }
497                    return Ok(());
498                };
499                let (close, error, next) = match open {
500                    Open::Array => (Kind::ArrayEnd, Error::ExpectedArrayNext, Step::Value),
501                    Open::Object => (Kind::ObjectEnd, Error::ExpectedObjectNext, Step::Field),
502                };
503                if token.kind == Kind::Comma {
504                    token = lexer.next()?;
505                    next
506                } else if token.kind == close {
507                    stack.pop();
508                    let end = token.end;
509                    token = lexer.next()?;
510                    sink.close_at(open, end).map_err(Stop::Action)?;
511                    Step::After
512                } else {
513                    return Err(unexpected(token, error).into());
514                }
515            }
516        };
517    }
518}
519
520/// The text input of `json`: the string, after a check of the syntax.
521pub fn json_in(s: &str) -> Result<&str, TypeError> {
522    match parse(s, false, &mut Check) {
523        Ok(()) => Ok(s),
524        Err(Stop::Syntax(fail)) => Err(syntax(s, fail)),
525        Err(Stop::Action(error)) => Err(error),
526    }
527}
528
529/// The error of PostgreSQL for a parse that failed, which says `json` for `jsonb` too.
530pub(crate) fn syntax(s: &str, fail: Fail) -> TypeError {
531    let token = &s[fail.start..fail.end];
532    let detail = match fail.error {
533        Error::InvalidToken => format!("Token \"{token}\" is invalid."),
534        Error::EscapingInvalid => format!("Escape sequence \"\\{token}\" is invalid."),
535        Error::EscapingRequired => {
536            format!("Character with value 0x{:02x} must be escaped.", s.as_bytes()[fail.end])
537        }
538        Error::UnicodeEscapeFormat => {
539            "\"\\u\" must be followed by four hexadecimal digits.".to_string()
540        }
541        Error::ExpectedEnd => format!("Expected end of input, but found \"{token}\"."),
542        Error::ExpectedArrayNext => format!("Expected \",\" or \"]\", but found \"{token}\"."),
543        Error::ExpectedColon => format!("Expected \":\", but found \"{token}\"."),
544        Error::ExpectedJson => format!("Expected JSON value, but found \"{token}\"."),
545        Error::ExpectedMore => "The input string ended unexpectedly.".to_string(),
546        Error::ExpectedObjectFirst => format!("Expected string or \"}}\", but found \"{token}\"."),
547        Error::ExpectedObjectNext => format!("Expected \",\" or \"}}\", but found \"{token}\"."),
548        Error::ExpectedString => format!("Expected string, but found \"{token}\"."),
549        Error::CodePointZero => "\\u0000 cannot be converted to text.".to_string(),
550        Error::HighSurrogate => {
551            "Unicode high surrogate must not follow a high surrogate.".to_string()
552        }
553        Error::LowSurrogate => "Unicode low surrogate must follow a high surrogate.".to_string(),
554    };
555    let mut error = match fail.error {
556        Error::CodePointZero => TypeError::new(
557            SqlState::UNTRANSLATABLE_CHARACTER,
558            "unsupported Unicode escape sequence".to_string(),
559        ),
560        _ => TypeError::new(
561            SqlState::INVALID_TEXT_REPRESENTATION,
562            "invalid input syntax for type json".to_string(),
563        ),
564    };
565    error.detail = Some(detail);
566    error.context = Some(context(s, fail));
567    error
568}
569
570/// `report_json_context`: the line of the input up to the end of the bad token, cut to about 50 bytes before that end. `...` marks the text that is not shown.
571fn context(s: &str, fail: Fail) -> String {
572    let b = s.as_bytes();
573    let end = fail.end;
574    let line_start =
575        b[..fail.start.min(end)].iter().rposition(|&c| c == b'\n').map_or(0, |at| at + 1);
576    let line = 1 + b[..line_start].iter().filter(|&&c| c == b'\n').count();
577    let mut start = line_start;
578    while end - start >= 50 {
579        start += s[start..].chars().next().map_or(1, char::len_utf8);
580    }
581    if start - line_start <= 3 {
582        start = line_start;
583    }
584    let prefix = if start > line_start { "..." } else { "" };
585    let more = fail.error != Error::ExpectedMore && end < b.len();
586    let suffix = if more && !matches!(b[end], b'\n' | b'\r') { "..." } else { "" };
587    format!("JSON data, line {line}: {prefix}{}{suffix}", &s[start..end])
588}
589
590#[cfg(test)]
591mod tests {
592    use super::*;
593
594    fn detail(s: &str) -> String {
595        json_in(s).unwrap_err().detail.unwrap()
596    }
597
598    #[test]
599    fn the_first_error_is_the_error_of_postgres() {
600        assert!(json_in(r#" {"a": [1, -2.5e+3, true, null, "\u00e9\n", {}, []]} "#).is_ok());
601        assert_eq!(detail("[1 2]"), r#"Expected "," or "]", but found "2"."#);
602        assert_eq!(detail("[1, 2x]"), r#"Token "2x" is invalid."#);
603        assert_eq!(detail(r#"{"a" 1}"#), r#"Expected ":", but found "1"."#);
604        assert_eq!(detail(r#"{"a": 1,}"#), r#"Expected string, but found "}"."#);
605        assert_eq!(detail("[1,"), "The input string ended unexpectedly.");
606        assert_eq!(detail(r#""\é""#), r#"Escape sequence "\é" is invalid."#);
607        assert_eq!(detail("\"a\tb\""), "Character with value 0x09 must be escaped.");
608        assert_eq!(detail(r#""abc"#), r#"Token ""abc" is invalid."#);
609        let context = |s: &str| json_in(s).unwrap_err().context.unwrap();
610        assert_eq!(context("[1 2]"), "JSON data, line 1: [1 2...");
611        assert_eq!(context("[1,"), "JSON data, line 1: [1,");
612        assert_eq!(context("[1,\n 2,\n x]"), "JSON data, line 3:  x...");
613        let long = format!("[{}x]", "1, ".repeat(30));
614        assert_eq!(context(&long), format!("JSON data, line 1: ...{}...", &long[43..92]));
615    }
616
617    #[test]
618    fn deep_nesting_needs_no_recursion() {
619        let deep = format!("{}{}", "[".repeat(100_000), "]".repeat(100_000));
620        assert!(json_in(&deep).is_ok());
621    }
622}
rupg_types/json.rs

rupg_types/
json.rs
