json_streaming/nonblocking/
read.rs

1use crate::nonblocking::io::NonBlockingRead;
2use crate::shared::*;
3use core::str::FromStr;
4
5/// A [JsonReader] wraps a sequence of bytes, aggregating them into a sequence of JSON tokens. It
6///  is in essentially a tokenizer, adding some rudimentary convenience for JSON's grammar.
7///
8/// It does *not* try to be a full-blown JSON parser. The reason is that if an application consumes
9///  an actual JSON document, it assumes structure beyond the document being a valid JSON document
10///  ('name is a mandatory string, age is a u32, but it is ok if it is null or not present in the
11///  first place'). [JsonReader] makes it easy for application code to consume data based on such
12///  assumptions - and while it does, application code checks the document's conformity to the JSON
13///  grammar automatically.
14///
15/// A [JsonReader] wraps and consumes a [NonBlockingRead] which is a platform-independent
16///  abstraction for async reading. There is a blanket implementation for [tokio::io::AsyncRead]
17///  which is included with the `tokio` feature flag.
18///
19/// The [JsonReader] holds a mutable reference to the reader rather than taking ownership of it.
20///  That means it needs to have a lifetime parameter, allowing the compiler to ensure that the
21///  reader lives at least as long as the wrapping [JsonReader].
22///
23/// The [JsonReader] also holds a fixed read buffer which it uses to assemble tokens. [JsonReader]
24///  can either work with a buffer passed to it on construction, or it can allocate the buffer as a
25///  convenience, and it does not do *any* memory allocation on the heap beyond that.
26///
27/// The buffer is fixed in length, and if some token (e.g. a string value) does not fit into the
28///  buffer, that causes a failure. While this may look like an inconvenient restriction, it is
29///  actually an important security safeguard: JSON documents often come from external sources and
30///  have limited trustworthiness at best. Without an upper bound on token size, a maliciously or
31///  negligently created document could contain a string literal with a size of a gigabyte,
32///  working as a denial-of-service attack. JSON parsers that work on materialized documents often
33///  address this by placing a restriction on the maximum size of the whole document.
34///
35/// The following code snippet shows a small working example of reading a JSON document with a
36///  single object:
37///
38/// ```
39/// use json_streaming::nonblocking::*;
40/// use json_streaming::shared::*;
41///
42/// async fn read_something<R: NonBlockingRead>(r: &mut R) -> JsonParseResult<(), R::Error> {
43///     let mut json_reader = JsonReader::new(1024, r);
44///
45///     json_reader.expect_next_start_object().await?;
46///     loop {
47///         match json_reader.expect_next_key().await? {
48///             Some("a") => println!("a: {}", json_reader.expect_next_string().await?),
49///             Some("b") => println!("b: {}", json_reader.expect_next_string().await?),
50///             Some(_other) => {
51///                 return Err(JsonParseError::Parse("unexpected key parsing 'person'", json_reader.location()));
52///             },
53///             None => break,
54///         }
55///     }
56///     Ok(())
57/// }
58/// ```
59pub struct JsonReader<'a, B: AsMut<[u8]>, R: NonBlockingRead> {
60    inner: ReaderInner<B, R::Error>,
61    reader: &'a mut R,
62}
63
64impl<'a, R: NonBlockingRead> JsonReader<'a, Vec<u8>, R> {
65    /// Create a [JsonReader], allocating a read buffer of given size on the heap.
66    pub fn new(buf_size: usize, reader: &'a mut R) -> Self {
67        let buf = vec![0u8; buf_size];
68        Self::new_with_provided_buffer(buf, reader, false)
69    }
70
71    /// Create a [JsonReader] without requiring commas between objects. This is intended for
72    ///  reading [https://jsonlines.org] documents - see the `jsonlines.rs` example for details.
73    pub fn new_with_lenient_comma_handling(buf_size: usize, reader: &'a mut R) -> Self {
74        let buf = vec![0u8; buf_size];
75        Self::new_with_provided_buffer(buf, reader, true)
76    }
77}
78
79//TODO Rust Doc
80
81
82impl<'a, B: AsMut<[u8]>, R: NonBlockingRead> JsonReader<'a, B, R> {
83    /// Create a [JsonReader] that uses an externally provided buffer as its read buffer.
84    pub fn new_with_provided_buffer(buf: B, reader: &'a mut R, lenient_comma_handling: bool) -> Self {
85        Self {
86            inner: ReaderInner::new(buf, lenient_comma_handling),
87            reader,
88        }
89    }
90
91    /// Return the next JSON token. This is the work horse of [JsonReader] and the foundation for
92    ///  other convenience abstraction.
93    ///
94    /// The function does only limited checks of JSON grammar and basically returns whatever tokens
95    ///  it finds.
96    pub async fn next(&mut self) -> JsonParseResult<JsonReadToken<'_>, R::Error> {
97        self.consume_whitespace().await?;
98
99        match self.read_next_byte().await? {
100            None => {
101                Ok(JsonReadToken::EndOfStream)
102            },
103            Some(b',') => {
104                self.inner.on_comma()?;
105                Box::pin(self.next()).await
106            }
107            Some(b'{') => {
108                self.inner.ensure_accept_value()?;
109                self.inner.state = ReaderState::Initial;
110                Ok(JsonReadToken::StartObject)
111            },
112            Some(b'}') => {
113                self.inner.ensure_accept_end_nested()?;
114                self.inner.state = ReaderState::AfterValue;
115                Ok(JsonReadToken::EndObject)
116            },
117            Some(b'[') => {
118                self.inner.ensure_accept_value()?;
119                self.inner.state = ReaderState::Initial;
120                Ok(JsonReadToken::StartArray)
121            },
122            Some(b']') => {
123                self.inner.ensure_accept_end_nested()?;
124                self.inner.state = ReaderState::AfterValue;
125                Ok(JsonReadToken::EndArray)
126            },
127
128            Some(b'n') => {
129                self.inner.state_change_for_value()?;
130                self.consume_null_literal().await
131            },
132            Some(b't') => {
133                self.inner.state_change_for_value()?;
134                self.consume_true_literal().await
135            },
136            Some(b'f') => {
137                self.inner.state_change_for_value()?;
138                self.consume_false_literal().await
139            },
140
141            Some(b'"') => self.parse_after_quote().await, // key or string value based on following ':'
142            Some(b) => {
143                self.inner.state_change_for_value()?;
144                match b {
145                    b'-' | b'0'..=b'9' => self.parse_number_literal(b).await,
146                    _ => self.inner.parse_err("invalid JSON literal")
147                }
148            },
149        }
150    }
151
152    /// This is the function for the loop to read the members of a JSON object: It returns either
153    ///  a JSON key or `None` if it encounters the `}` that ends the object. All other tokens are
154    ///  invalid and cause the function to fail.
155    pub async fn expect_next_key(&mut self) -> JsonParseResult<Option<&str>, R::Error> {
156        let location = self.location();
157        let next = self.next().await?;
158        match next {
159            JsonReadToken::Key(key) => Ok(Some(key)),
160            JsonReadToken::EndObject => Ok(None),
161            other => Err(JsonParseError::UnexpectedToken(other.kind(), location)),
162        }
163    }
164
165    /// Returns a JSON number as a [JsonNumber], and fails if the next token is anything other
166    ///  than a number. That includes the case that the next token is `null` - if the number
167    ///  is optional and `null` a valid value, use [crate::blocking::JsonReader::expect_next_opt_raw_number] instead.
168    pub async fn expect_next_raw_number(&mut self) -> JsonParseResult<JsonNumber<'_>, R::Error> {
169        let location = self.location();
170        let next = self.next().await?;
171        match next {
172            JsonReadToken::NumberLiteral(n) => Ok(n),
173            other => Err(JsonParseError::UnexpectedToken(other.kind(), location)),
174        }
175    }
176
177    /// Returns a [JsonNumber] if the next token is a JSON number, or `None` if the next token
178    ///  is `null`. All other tokens cause the function to fail.
179    pub async fn expect_next_opt_raw_number(&mut self) -> JsonParseResult<Option<JsonNumber<'_>>, R::Error> {
180        let location = self.location();
181        let next = self.next().await?;
182        match next {
183            JsonReadToken::NullLiteral => Ok(None),
184            JsonReadToken::NumberLiteral(n) => Ok(Some(n)),
185            other => Err(JsonParseError::UnexpectedToken(other.kind(), location)),
186        }
187    }
188
189    /// Returns a number parsed to an application-provided type, failing if the next token is not
190    ///  a JSON number or the number is not parseable to the provided type (e.g. trying to
191    ///  get retrieve a floating point number as a u32).
192    ///
193    /// The expected numeric type can be provided explicitly, like this:
194    /// ```
195    /// # use json_streaming::nonblocking::*;
196    /// # use json_streaming::shared::*;
197    /// # async fn get_num<R: NonBlockingRead>(r: &mut R) -> JsonParseResult<(), R::Error> {
198    /// # let mut json_reader = JsonReader::new(128, r);
199    /// let n = json_reader.expect_next_number::<u32>().await?;
200    /// # Ok(()) }
201    /// ```
202    /// or inferred by the compiler:
203    /// ```
204    /// # use json_streaming::nonblocking::*;
205    /// # use json_streaming::shared::*;
206    /// # async fn get_num<R: NonBlockingRead>(r: &mut R) -> JsonParseResult<(), R::Error> {
207    /// # let mut json_reader = JsonReader::new(128, r);
208    /// let n:u32 = json_reader.expect_next_number().await?;
209    /// # Ok(()) }
210    /// ```
211    pub async fn expect_next_number<T: FromStr>(&mut self) -> JsonParseResult<T, R::Error> {
212        match self.expect_next_raw_number().await?.parse::<T>() {
213            Ok(n) => Ok(n),
214            Err(_) => self.inner.parse_err("invalid number"),
215        }
216    }
217
218    /// The same as [JsonReader::expect_next_number], but accepting a `null` literal which it
219    ///  returns as `None`.
220    pub async fn expect_next_opt_number<T: FromStr>(&mut self) -> JsonParseResult<Option<T>, R::Error> {
221        match self.expect_next_opt_raw_number().await {
222            Ok(None) => Ok(None),
223            Ok(Some(n)) => {
224                match n.parse::<T>() {
225                    Ok(n) => Ok(Some(n)),
226                    Err(_) => self.inner.parse_err("invalid number"),
227                }
228            },
229            Err(err) => Err(err),
230        }
231    }
232
233    /// If the next token is a string literal, return that, and fail for any other token.
234    pub async fn expect_next_string(&mut self) -> JsonParseResult<&str, R::Error> {
235        let location = self.location();
236        let next = self.next().await?;
237        match next {
238            JsonReadToken::StringLiteral(s) => Ok(s),
239            other => Err(JsonParseError::UnexpectedToken(other.kind(), location)),
240        }
241    }
242
243    /// The same as [JsonReader::expect_next_string], but accepting a `null` literal which is
244    ///  returned as `None`.
245    pub async fn expect_next_opt_string(&mut self) -> JsonParseResult<Option<&str>, R::Error> {
246        let location = self.location();
247        let next = self.next().await?;
248        match next {
249            JsonReadToken::NullLiteral => Ok(None),
250            JsonReadToken::StringLiteral(s) => Ok(Some(s)),
251            other => Err(JsonParseError::UnexpectedToken(other.kind(), location)),
252        }
253    }
254
255    /// `true` and `false` literals are returned as a boolean token, all other tokens cause the
256    ///  function to fail.
257    pub async fn expect_next_bool(&mut self) -> JsonParseResult<bool, R::Error> {
258        let location = self.location();
259        let next = self.next().await?;
260        match next {
261            JsonReadToken::BooleanLiteral(b) => Ok(b),
262            other => Err(JsonParseError::UnexpectedToken(other.kind(), location)),
263        }
264    }
265
266    /// The same as [JsonReader::expect_next_bool], but accepting a `null` literal which is
267    ///  returned as `None`.
268    pub async fn expect_next_opt_bool(&mut self) -> JsonParseResult<Option<bool>, R::Error> {
269        let location = self.location();
270        let next = self.next().await?;
271        match next {
272            JsonReadToken::NullLiteral => Ok(None),
273            JsonReadToken::BooleanLiteral(b) => Ok(Some(b)),
274            other => Err(JsonParseError::UnexpectedToken(other.kind(), location)),
275        }
276    }
277
278    /// Fails for any token except the `{` that starts an object.
279    pub async fn expect_next_start_object(&mut self) -> JsonParseResult<(), R::Error> {
280        let location = self.location();
281        let next = self.next().await?;
282        match next {
283            JsonReadToken::StartObject => Ok(()),
284            other => Err(JsonParseError::UnexpectedToken(other.kind(), location)),
285        }
286    }
287
288    /// The same as [JsonReader::expect_next_start_object], but accepting a `null` literal which is
289    ///  returned as `None`.
290    pub async fn expect_next_opt_start_object(&mut self) -> JsonParseResult<Option<()>, R::Error> {
291        let location = self.location();
292        let next = self.next().await?;
293        match next {
294            JsonReadToken::NullLiteral => Ok(None),
295            JsonReadToken::StartObject => Ok(Some(())),
296            other => Err(JsonParseError::UnexpectedToken(other.kind(), location)),
297        }
298    }
299
300    /// Fails for any token except the `[` that starts an array.
301    pub async fn expect_next_start_array(&mut self) -> JsonParseResult<(), R::Error> {
302        let location = self.location();
303        let next = self.next().await?;
304        match next {
305            JsonReadToken::StartArray => Ok(()),
306            other => Err(JsonParseError::UnexpectedToken(other.kind(), location)),
307        }
308    }
309
310    /// The same as [JsonReader::expect_next_start_array], but accepting a `null` literal which is
311    ///  returned as `None`.
312    pub async fn expect_next_opt_start_array(&mut self) -> JsonParseResult<Option<()>, R::Error> {
313        let location = self.location();
314        let next = self.next().await?;
315        match next {
316            JsonReadToken::NullLiteral => Ok(None),
317            JsonReadToken::StartArray => Ok(Some(())),
318            other => Err(JsonParseError::UnexpectedToken(other.kind(), location)),
319        }
320    }
321
322    /// This function assumes that it is called inside an object or array, and silently consumes
323    ///  all tokens until and including the closing `}` or `]`.
324    ///
325    /// This function is useful for gracefully ignoring array elements with an unsupported type. See
326    ///  the `skipping.rs` example for details.
327    pub async fn skip_to_end_of_current_scope(&mut self) -> JsonParseResult<(), R::Error> {
328        let mut nesting_level = 1;
329        loop {
330            match self.next().await? {
331                JsonReadToken::StartObject | JsonReadToken::StartArray => {
332                    nesting_level += 1;
333                }
334                JsonReadToken::EndObject | JsonReadToken::EndArray=> {
335                    nesting_level -= 1;
336                    if nesting_level == 0 {
337                        break;
338                    }
339                }
340                JsonReadToken::EndOfStream => {
341                    return Err(JsonParseError::UnexpectedToken(JsonReadToken::EndOfStream.kind(), self.location()));
342                }
343                _ => {
344                    continue;
345                }
346            }
347        }
348        Ok(())
349    }
350
351    /// This function skips the value starting with the next token. This can be a single-token value,
352    ///  but it can also be an object or array of unknown structure and nesting depth, in which
353    ///  case this function silently consumes tokens until it reaches the matching closing `}` or
354    ///  `]`.
355    ///
356    /// This function is useful for gracefully ignoring object members with an unknown key - see
357    ///  the `skipping.rs` example for details.
358    pub async fn skip_value(&mut self) -> JsonParseResult<(), R::Error> {
359        match self.next().await? {
360            JsonReadToken::Key(_) |
361            JsonReadToken::EndObject |
362            JsonReadToken::EndArray |
363            JsonReadToken::EndOfStream => {
364                Err(JsonParseError::UnexpectedToken(JsonReadToken::EndOfStream.kind(), self.location()))
365            }
366            JsonReadToken::StartObject |
367            JsonReadToken::StartArray => {
368                self.skip_to_end_of_current_scope().await
369            }
370            JsonReadToken::StringLiteral(_) |
371            JsonReadToken::NumberLiteral(_) |
372            JsonReadToken::BooleanLiteral(_) |
373            JsonReadToken::NullLiteral => {
374                Ok(())
375            }
376        }
377    }
378
379    async fn consume_whitespace(&mut self) -> JsonParseResult<(), R::Error> {
380        while let Some(next) = self.read_next_byte().await? {
381            match next {
382                b' ' | b'\t' | b'\n' | b'\r' => {
383                }
384                next => {
385                    self.inner.parked_next = Some(next);
386                    break;
387                }
388            }
389        }
390        Ok(())
391    }
392
393    async fn read_next_byte(&mut self) -> JsonParseResult<Option<u8>, R::Error> {
394        // Parsing JSON requires a lookahead of a single byte, which is stored in 'parked_next'
395        if let Some(parked) = self.inner.parked_next.take() {
396            return Ok(Some(parked));
397        }
398
399        if let Some(byte) =self.reader.read().await? {
400            self.inner.cur_location.after_byte(byte);
401            Ok(Some(byte))
402        }
403        else {
404            Ok(None)
405        }
406    }
407
408    async fn consume_null_literal(&mut self) -> JsonParseResult<JsonReadToken<'_>, R::Error> {
409        if self.read_next_byte().await? != Some(b'u') {
410            return self.inner.parse_err("incomplete null literal");
411        }
412        if self.read_next_byte().await? != Some(b'l') {
413            return self.inner.parse_err("incomplete null literal");
414        }
415        if self.read_next_byte().await? != Some(b'l') {
416            return self.inner.parse_err("incomplete null literal");
417        }
418        Ok(JsonReadToken::NullLiteral)
419    }
420
421    async fn consume_true_literal(&mut self) -> JsonParseResult<JsonReadToken<'_>, R::Error> {
422        if self.read_next_byte().await? != Some(b'r') {
423            return self.inner.parse_err("incomplete true literal");
424        }
425        if self.read_next_byte().await? != Some(b'u') {
426            return self.inner.parse_err("incomplete true literal");
427        }
428        if self.read_next_byte().await? != Some(b'e') {
429            return self.inner.parse_err("incomplete true literal");
430        }
431        Ok(JsonReadToken::BooleanLiteral(true))
432    }
433
434    async fn consume_false_literal(&mut self) -> JsonParseResult<JsonReadToken<'_>, R::Error> {
435        if self.read_next_byte().await? != Some(b'a') {
436            return self.inner.parse_err("incomplete false literal");
437        }
438        if self.read_next_byte().await? != Some(b'l') {
439            return self.inner.parse_err("incomplete false literal");
440        }
441        if self.read_next_byte().await? != Some(b's') {
442            return self.inner.parse_err("incomplete false literal");
443        }
444        if self.read_next_byte().await? != Some(b'e') {
445            return self.inner.parse_err("incomplete false literal");
446        }
447        Ok(JsonReadToken::BooleanLiteral(false))
448    }
449
450    async fn parse_after_quote(&mut self) -> JsonParseResult<JsonReadToken<'_>, R::Error> {
451        self.inner.ind_end_buf = 0;
452
453        loop {
454            if let Some(next) = self.read_next_byte().await? {
455                match next {
456                    b'"' => break,
457                    b'\\' => {
458                        match self.read_next_byte().await? {
459                            Some(b'"') => self.inner.append_to_buf(b'"')?,
460                            Some(b'\\') => self.inner.append_to_buf(b'\\')?,
461                            Some(b'/') => self.inner.append_to_buf(b'/')?,
462                            Some(b'b') => self.inner.append_to_buf(0x08)?,
463                            Some(b'f') => self.inner.append_to_buf(0x0c)?,
464                            Some(b'n') => self.inner.append_to_buf(b'\n')?,
465                            Some(b'r') => self.inner.append_to_buf(b'\r')?,
466                            Some(b't') => self.inner.append_to_buf(b'\t')?,
467                            Some(b'u') => {
468                                let cp = self.parse_unicode_codepoint().await?;
469                                self.inner.append_code_point(cp)?;
470                            },
471                            _ => return self.inner.parse_err("invalid escape in string literal"),
472                        }
473                    },
474                    ch => {
475                        self.inner.append_to_buf(ch)?;
476                    }
477                }
478            }
479            else {
480                return self.inner.parse_err("unterminated string literal");
481            }
482        }
483
484        // the buffer contains the string's contents - the next character determines whether this
485        //  is key or a string value. Recall that we don't check for valid JSON.
486
487        self.consume_whitespace().await?;
488        match self.read_next_byte().await? {
489            Some(b':') => {
490                match self.inner.state {
491                    ReaderState::Initial |
492                    ReaderState::BeforeEntry => {
493                        self.inner.state = ReaderState::AfterKey;
494                    }
495                    ReaderState::AfterKey => {
496                        return self.inner.parse_err("two keys without value");
497                    }
498                    ReaderState::AfterValue => {
499                        return self.inner.parse_err("missing comma");
500                    }
501                }
502                Ok(JsonReadToken::Key(self.inner.buf_as_str()?))
503            },
504            other => {
505                self.inner.state_change_for_value()?;
506                self.inner.parked_next = other;
507                Ok(JsonReadToken::StringLiteral(self.inner.buf_as_str()?))
508            }
509        }
510    }
511
512    async fn parse_unicode_codepoint(&mut self) -> JsonParseResult<u16, R::Error> {
513        // exactly four hex digits specifying a code point
514        let mut cp: u16 = 0;
515        for _ in 0..4 {
516            if let Some(b) = self.read_next_byte().await? {
517                cp = cp << 4;
518                match b {
519                    b'0'..=b'9' => cp += (b - b'0') as u16,
520                    b'a'..=b'f' => cp += (b - b'a' + 10) as u16,
521                    b'A'..=b'Z' => cp += (b - b'A' + 10) as u16,
522                    _ => {
523                        return self.inner.parse_err("not a four-digit hex number after \\u");
524                    }
525                }
526            }
527            else {
528                return self.inner.parse_err("incomplete UTF codepoint in string literal");
529            }
530        }
531        Ok(cp)
532    }
533
534    async fn parse_number_literal(&mut self, b: u8) -> JsonParseResult<JsonReadToken<'_>, R::Error> {
535        self.inner.buf.as_mut()[0] = b;
536        self.inner.ind_end_buf = 1;
537
538        while let Some(next) = self.read_next_byte().await? {
539            match next {
540                b'0'..=b'9' |
541                b'+' | b'-' | b'e' | b'E' |
542                b'.' => {
543                    self.inner.append_to_buf(next)?;
544                }
545                other => {
546                    self.inner.parked_next = Some(other);
547                    break;
548                }
549            }
550        }
551        Ok(JsonReadToken::NumberLiteral(JsonNumber(self.inner.buf_as_str()?)))
552    }
553
554    /// Returns the current parse location in the underlying reader - offset, row and column.
555    #[inline]
556    pub fn location(&self) -> Location {
557        self.inner.cur_location
558    }
559}
560
561#[cfg(test)]
562mod tests {
563    use super::*;
564    use rstest::*;
565    use std::io;
566    use std::io::Cursor;
567
568    fn assert_is_similar_error(actual: &JsonParseError<std::io::Error>, expected: &JsonParseError<std::io::Error>) {
569        match actual {
570            JsonParseError::Io(self_e) => {
571                if let JsonParseError::Io(other_e) = expected {
572                    assert_eq!(self_e.kind(), other_e.kind());
573                    return;
574                }
575            }
576            JsonParseError::Utf8(_) => {
577                if let JsonParseError::Utf8(_) = expected {
578                    return;
579                }
580            }
581            JsonParseError::Parse(msg, _) => {
582                if let JsonParseError::Parse(other_msg, _) = expected {
583                    assert_eq!(msg, other_msg);
584                    return;
585                }
586            }
587            JsonParseError::UnexpectedToken(_, _) => {
588                if let JsonParseError::UnexpectedToken(_, _) = expected {
589                    return;
590                }
591            }
592            JsonParseError::BufferOverflow(_) => {
593                if let JsonParseError::BufferOverflow(_) = expected {
594                    return;
595                }
596            }
597        }
598
599        panic!("{:?} != {:?}", actual, expected);
600    }
601
602    #[rstest]
603    #[case::empty("", vec![], None)]
604    #[case::empty_repeated_end_of_stream("", vec![JsonReadToken::EndOfStream, JsonReadToken::EndOfStream, JsonReadToken::EndOfStream, ], None)]
605
606    #[case::null_literal("null", vec![JsonReadToken::NullLiteral], None)]
607    #[case::true_literal("true", vec![JsonReadToken::BooleanLiteral(true)], None)]
608    #[case::false_literal("false", vec![JsonReadToken::BooleanLiteral(false)], None)]
609    #[case::start_object("{", vec![JsonReadToken::StartObject], None)]
610    #[case::end_object("{}", vec![JsonReadToken::StartObject, JsonReadToken::EndObject], None)]
611    #[case::start_array("[", vec![JsonReadToken::StartArray], None)]
612    #[case::end_array("[]", vec![JsonReadToken::StartArray, JsonReadToken::EndArray], None)]
613
614    #[case::key("\"xyz\":", vec![JsonReadToken::Key("xyz")], None)]
615    #[case::key_with_escapes("\"x\\ry\\nz\":", vec![JsonReadToken::Key("x\ry\nz")], None)]
616    #[case::key_ws("\"xyz\" \n:", vec![JsonReadToken::Key("xyz")], None)]
617    #[case::key_value("\"xyz\" \n:\r\tfalse", vec![JsonReadToken::Key("xyz"), JsonReadToken::BooleanLiteral(false)], None)]
618
619    #[case::string_literal(r#""abc""#, vec![JsonReadToken::StringLiteral("abc")], None)]
620    #[case::string_literal_empty(r#""""#, vec![JsonReadToken::StringLiteral("")], None)]
621    #[case::string_literal_quot(r#""\"""#, vec![JsonReadToken::StringLiteral("\"")], None)]
622    #[case::string_literal_backslash(r#""\\""#, vec![JsonReadToken::StringLiteral("\\")], None)]
623    #[case::string_literal_slash(r#""\/""#, vec![JsonReadToken::StringLiteral("/")], None)]
624    #[case::string_literal_backslash(r#""\b""#, vec![JsonReadToken::StringLiteral("\x08")], None)]
625    #[case::string_literal_formfeed(r#""\f""#, vec![JsonReadToken::StringLiteral("\x0c")], None)]
626    #[case::string_literal_linefeed(r#""\n""#, vec![JsonReadToken::StringLiteral("\n")], None)]
627    #[case::string_literal_carriage_return(r#""\r""#, vec![JsonReadToken::StringLiteral("\r")], None)]
628    #[case::string_literal_tab(r#""\t""#, vec![JsonReadToken::StringLiteral("\t")], None)]
629    #[case::string_literal_unicode_y(r#""\u0079""#, vec![JsonReadToken::StringLiteral("y")], None)]
630    #[case::string_literal_unicode_umlaut_two_bytes(r#""\u00e4""#, vec![JsonReadToken::StringLiteral("ä")], None)]
631    #[case::string_literal_unicode_omega_two_bytes(r#""\u03a9""#, vec![JsonReadToken::StringLiteral("Ω")], None)]
632    #[case::string_literal_unicode_euro_three_bytes(r#""\u20ac""#, vec![JsonReadToken::StringLiteral("€")], None)]
633    #[case::string_literal_combined(r#""a\n b\t \u00e4öü \u03a9 12.2\u20ac""#, vec![JsonReadToken::StringLiteral("a\n b\t äöü Ω 12.2€")], None)]
634
635    #[case::number_literal("123", vec![JsonReadToken::NumberLiteral(JsonNumber("123"))], None)]
636    #[case::number_literal_negative("-456", vec![JsonReadToken::NumberLiteral(JsonNumber("-456"))], None)]
637    #[case::number_literal_zero("0", vec![JsonReadToken::NumberLiteral(JsonNumber("0"))], None)]
638    #[case::number_literal_fraction("0.92", vec![JsonReadToken::NumberLiteral(JsonNumber("0.92"))], None)]
639    #[case::number_literal_fraction_small("0.0000000000000092", vec![JsonReadToken::NumberLiteral(JsonNumber("0.0000000000000092"))], None)]
640    #[case::number_literal_fraction_neg("-0.0000000000000092", vec![JsonReadToken::NumberLiteral(JsonNumber("-0.0000000000000092"))], None)]
641    #[case::number_literal_exp_lower("0.92e4", vec![JsonReadToken::NumberLiteral(JsonNumber("0.92e4"))], None)]
642    #[case::number_literal_exp_upper("0.92E6", vec![JsonReadToken::NumberLiteral(JsonNumber("0.92E6"))], None)]
643    #[case::number_literal_pos_exp_lower("0.92e+4", vec![JsonReadToken::NumberLiteral(JsonNumber("0.92e+4"))], None)]
644    #[case::number_literal_pos_exp_upper("0.92E+6", vec![JsonReadToken::NumberLiteral(JsonNumber("0.92E+6"))], None)]
645    #[case::number_literal_neg_exp_lower("0.92e-4", vec![JsonReadToken::NumberLiteral(JsonNumber("0.92e-4"))], None)]
646    #[case::number_literal_neg_exp_upper("0.92E-6", vec![JsonReadToken::NumberLiteral(JsonNumber("0.92E-6"))], None)]
647
648    #[case::number_literal_no_leading_zero(".1", vec![], Some(JsonParseError::Parse("invalid JSON literal", Location::start())))]
649    #[case::no_matching_literal("x", vec![], Some(JsonParseError::Parse("invalid JSON literal", Location::start())))]
650    #[case::invalid_number_continuation("1x", vec![JsonReadToken::NumberLiteral(JsonNumber("1"))], Some(JsonParseError::Parse("missing comma", Location::start())))]
651    #[case::invalid_number_continuation_quote("x\"", vec![], Some(JsonParseError::Parse("invalid JSON literal", Location::start())))]
652
653    #[case::string_literal_unterminated_short(r#""abc "#, vec![], Some(JsonParseError::Parse("unterminated string literal", Location::start())))]
654    #[case::string_literal_unterminated_long(r#""abc                                                                         "#, vec![], Some(JsonParseError::BufferOverflow(Location::start())))]
655    #[case::string_literal_invalid_escape(r#""\q""#, vec![], Some(JsonParseError::Parse("invalid escape in string literal", Location::start())))]
656    #[case::string_literal_unicode_string_ends(r#""\u004""#, vec![], Some(JsonParseError::Parse("not a four-digit hex number after \\u", Location::start())))]
657    #[case::string_literal_unicode_invalid_character_1(r#""\ux041""#, vec![], Some(JsonParseError::Parse("not a four-digit hex number after \\u", Location::start())))]
658    #[case::string_literal_unicode_invalid_character_2(r#""\u0x41""#, vec![], Some(JsonParseError::Parse("not a four-digit hex number after \\u", Location::start())))]
659    #[case::string_literal_unicode_invalid_character_3(r#""\u00x1""#, vec![], Some(JsonParseError::Parse("not a four-digit hex number after \\u", Location::start())))]
660    #[case::string_literal_unicode_invalid_character_4(r#""\u004x""#, vec![], Some(JsonParseError::Parse("not a four-digit hex number after \\u", Location::start())))]
661    #[case::string_literal_unicode_uppercase_u(r#""\U0041""#, vec![], Some(JsonParseError::Parse("invalid escape in string literal", Location::start())))]
662    #[case::string_literal_unicode_uppercase(r#""\uABCD""#, vec![JsonReadToken::StringLiteral("\u{abcd}")], None)]
663    #[case::string_literal_unicode_mixed_case_1(r#""\uaBcD""#, vec![JsonReadToken::StringLiteral("\u{abcd}")], None)]
664    #[case::string_literal_unicode_mixed_case_2(r#""\uAbCd""#, vec![JsonReadToken::StringLiteral("\u{abcd}")], None)]
665
666    #[case::null_wrong_continuation_1("nul", vec![], Some(JsonParseError::Parse("incomplete null literal", Location::start())))]
667    #[case::null_wrong_continuation_2("nxll", vec![], Some(JsonParseError::Parse("incomplete null literal", Location::start())))]
668    #[case::null_wrong_continuation_3("nUll", vec![], Some(JsonParseError::Parse("incomplete null literal", Location::start())))]
669    #[case::null_wrong_continuation_4("nuxl", vec![], Some(JsonParseError::Parse("incomplete null literal", Location::start())))]
670    #[case::null_wrong_continuation_5("nuLl", vec![], Some(JsonParseError::Parse("incomplete null literal", Location::start())))]
671    #[case::null_wrong_continuation_6("nulx", vec![], Some(JsonParseError::Parse("incomplete null literal", Location::start())))]
672    #[case::null_wrong_continuation_7("nulL", vec![], Some(JsonParseError::Parse("incomplete null literal", Location::start())))]
673    #[case::null_uppercase("Null", vec![], Some(JsonParseError::Parse("invalid JSON literal", Location::start())))]
674    #[case::null_uppercase_2("NULL", vec![], Some(JsonParseError::Parse("invalid JSON literal", Location::start())))]
675
676    #[case::true_wrong_continuation_1("tru", vec![], Some(JsonParseError::Parse("incomplete true literal", Location::start())))]
677    #[case::true_wrong_continuation_2("txue", vec![], Some(JsonParseError::Parse("incomplete true literal", Location::start())))]
678    #[case::true_wrong_continuation_3("tRue", vec![], Some(JsonParseError::Parse("incomplete true literal", Location::start())))]
679    #[case::true_wrong_continuation_4("trxe", vec![], Some(JsonParseError::Parse("incomplete true literal", Location::start())))]
680    #[case::true_wrong_continuation_5("trUe", vec![], Some(JsonParseError::Parse("incomplete true literal", Location::start())))]
681    #[case::true_wrong_continuation_6("trux", vec![], Some(JsonParseError::Parse("incomplete true literal", Location::start())))]
682    #[case::true_wrong_continuation_7("truE", vec![], Some(JsonParseError::Parse("incomplete true literal", Location::start())))]
683    #[case::true_uppercase_1("True", vec![], Some(JsonParseError::Parse("invalid JSON literal", Location::start())))]
684    #[case::true_uppercase_2("TRUE", vec![], Some(JsonParseError::Parse("invalid JSON literal", Location::start())))]
685
686    #[case::false_wrong_continuation_1("fals", vec![], Some(JsonParseError::Parse("incomplete false literal", Location::start())))]
687    #[case::false_wrong_continuation_2("fxlse", vec![], Some(JsonParseError::Parse("incomplete false literal", Location::start())))]
688    #[case::false_wrong_continuation_3("fAlse", vec![], Some(JsonParseError::Parse("incomplete false literal", Location::start())))]
689    #[case::false_wrong_continuation_4("faxse", vec![], Some(JsonParseError::Parse("incomplete false literal", Location::start())))]
690    #[case::false_wrong_continuation_5("faLse", vec![], Some(JsonParseError::Parse("incomplete false literal", Location::start())))]
691    #[case::false_wrong_continuation_6("falxe", vec![], Some(JsonParseError::Parse("incomplete false literal", Location::start())))]
692    #[case::false_wrong_continuation_7("falSe", vec![], Some(JsonParseError::Parse("incomplete false literal", Location::start())))]
693    #[case::false_wrong_continuation_8("falsx", vec![], Some(JsonParseError::Parse("incomplete false literal", Location::start())))]
694    #[case::false_wrong_continuation_9("falsE", vec![], Some(JsonParseError::Parse("incomplete false literal", Location::start())))]
695    #[case::false_uppercase_1("False", vec![], Some(JsonParseError::Parse("invalid JSON literal", Location::start())))]
696    #[case::false_uppercase_2("FALSE", vec![], Some(JsonParseError::Parse("invalid JSON literal", Location::start())))]
697
698    #[case::object_end_just_comma(r#"{, }"#, vec![JsonReadToken::StartObject], Some(JsonParseError::Parse("unexpected comma", Location::start())))]
699    #[case::object_end_trailing_comma(r#"{"a": null, }"#, vec![JsonReadToken::StartObject, JsonReadToken::Key("a"), JsonReadToken::NullLiteral], Some(JsonParseError::Parse("trailing comma", Location::start())))]
700    #[case::object_end_after_key(r#"{"a": }"#, vec![JsonReadToken::StartObject, JsonReadToken::Key("a")], Some(JsonParseError::Parse("key without a value", Location::start())))]
701    #[case::array_end_just_comma(r#"[, ]"#, vec![JsonReadToken::StartArray], Some(JsonParseError::Parse("unexpected comma", Location::start())))]
702    #[case::array_end_trailing_comma(r#"[null, ]"#, vec![JsonReadToken::StartArray, JsonReadToken::NullLiteral], Some(JsonParseError::Parse("trailing comma", Location::start())))]
703    #[case::array_end_after_key(r#"["a": ]"#, vec![JsonReadToken::StartArray, JsonReadToken::Key("a")], Some(JsonParseError::Parse("key without a value", Location::start())))]
704
705    #[case::missing_comma_null(r#"[null null]"#, vec![JsonReadToken::StartArray, JsonReadToken::NullLiteral], Some(JsonParseError::Parse("missing comma", Location::start())))]
706    #[case::missing_comma_true(r#"[null true]"#, vec![JsonReadToken::StartArray, JsonReadToken::NullLiteral], Some(JsonParseError::Parse("missing comma", Location::start())))]
707    #[case::missing_comma_false(r#"[null false]"#, vec![JsonReadToken::StartArray, JsonReadToken::NullLiteral], Some(JsonParseError::Parse("missing comma", Location::start())))]
708    #[case::missing_comma_number(r#"[null 123]"#, vec![JsonReadToken::StartArray, JsonReadToken::NullLiteral], Some(JsonParseError::Parse("missing comma", Location::start())))]
709    #[case::missing_comma_string(r#"[null "abc"]"#, vec![JsonReadToken::StartArray, JsonReadToken::NullLiteral], Some(JsonParseError::Parse("missing comma", Location::start())))]
710    #[case::missing_comma_object(r#"[null {}]"#, vec![JsonReadToken::StartArray, JsonReadToken::NullLiteral], Some(JsonParseError::Parse("missing comma", Location::start())))]
711    #[case::missing_comma_array(r#"[null []]"#, vec![JsonReadToken::StartArray, JsonReadToken::NullLiteral], Some(JsonParseError::Parse("missing comma", Location::start())))]
712    #[case::missing_comma_key(r#"{"a": null "b": 1}"#, vec![JsonReadToken::StartObject, JsonReadToken::Key("a"), JsonReadToken::NullLiteral], Some(JsonParseError::Parse("missing comma", Location::start())))]
713    #[case::key_after_key(r#"{"a": "b": 1}"#, vec![JsonReadToken::StartObject, JsonReadToken::Key("a")], Some(JsonParseError::Parse("two keys without value", Location::start())))]
714    #[case::comma_after_key(r#"{"a": , "b": 1}"#, vec![JsonReadToken::StartObject, JsonReadToken::Key("a")], Some(JsonParseError::Parse("unexpected comma", Location::start())))]
715
716    #[case::object_comma_after_comma(r#"{"a": null, ,}"#, vec![JsonReadToken::StartObject, JsonReadToken::Key("a"), JsonReadToken::NullLiteral], Some(JsonParseError::Parse("unexpected comma", Location::start())))]
717    #[case::array_comma_after_comma(r#"[ null, ,]"#, vec![JsonReadToken::StartArray, JsonReadToken::NullLiteral], Some(JsonParseError::Parse("unexpected comma", Location::start())))]
718
719    #[case::object(r#"{ "a": 1, "b": true, "c": "xyz" }"#, vec![
720        JsonReadToken::StartObject,
721        JsonReadToken::Key("a"),
722        JsonReadToken::NumberLiteral(JsonNumber("1")),
723        JsonReadToken::Key("b"),
724        JsonReadToken::BooleanLiteral(true),
725        JsonReadToken::Key("c"),
726        JsonReadToken::StringLiteral("xyz"),
727        JsonReadToken::EndObject,
728    ], None)]
729    #[case::array(r#"[ 6, "xy", true, null ]"#, vec![
730        JsonReadToken::StartArray,
731        JsonReadToken::NumberLiteral(JsonNumber("6")),
732        JsonReadToken::StringLiteral("xy"),
733        JsonReadToken::BooleanLiteral(true),
734        JsonReadToken::NullLiteral,
735        JsonReadToken::EndArray,
736    ], None)]
737    #[case::complex(r#"{"abc":"yo","xyz":"yo","aaaa":["111","11",{},[],null,true,false,-23987,23987,23.235,null,null,23.235e-1,null,null],"ooo":{"lll":"whatever","ar":[]}}"#, vec![
738        JsonReadToken::StartObject,
739        JsonReadToken::Key("abc"),
740        JsonReadToken::StringLiteral("yo"),
741        JsonReadToken::Key("xyz"),
742        JsonReadToken::StringLiteral("yo"),
743        JsonReadToken::Key("aaaa"),
744        JsonReadToken::StartArray,
745        JsonReadToken::StringLiteral("111"),
746        JsonReadToken::StringLiteral("11"),
747        JsonReadToken::StartObject,
748        JsonReadToken::EndObject,
749        JsonReadToken::StartArray,
750        JsonReadToken::EndArray,
751        JsonReadToken::NullLiteral,
752        JsonReadToken::BooleanLiteral(true),
753        JsonReadToken::BooleanLiteral(false),
754        JsonReadToken::NumberLiteral(JsonNumber("-23987")),
755        JsonReadToken::NumberLiteral(JsonNumber("23987")),
756        JsonReadToken::NumberLiteral(JsonNumber("23.235")),
757        JsonReadToken::NullLiteral,
758        JsonReadToken::NullLiteral,
759        JsonReadToken::NumberLiteral(JsonNumber("23.235e-1")),
760        JsonReadToken::NullLiteral,
761        JsonReadToken::NullLiteral,
762        JsonReadToken::EndArray,
763        JsonReadToken::Key("ooo"),
764        JsonReadToken::StartObject,
765        JsonReadToken::Key("lll"),
766        JsonReadToken::StringLiteral("whatever"),
767        JsonReadToken::Key("ar"),
768        JsonReadToken::StartArray,
769        JsonReadToken::EndArray,
770        JsonReadToken::EndObject,
771        JsonReadToken::EndObject,
772    ], None)]
773
774    #[tokio::test]
775    async fn test_parse(#[case] input: &str, #[case] expected: Vec<JsonReadToken<'static>>, #[case] expected_error: Option<JsonParseError<std::io::Error>>) {
776        let input_with_whitespace = format!(" \r\n\t{} \r\n\t", input);
777
778        {
779            let mut r = Cursor::new(input.as_bytes().to_vec());
780            let mut json_reader = JsonReader::new(64, &mut r);
781            for evt in &expected {
782                let parsed_evt = json_reader.next().await;
783                assert_eq!(&parsed_evt.unwrap(), evt);
784            }
785            if let Some(expected_error) = &expected_error {
786                match json_reader.next().await {
787                    Ok(_) => panic!("expected error but was ok: {}", expected_error),
788                    Err(e) => assert_is_similar_error(&e, expected_error),
789                }
790            }
791            else {
792                assert_eq!(json_reader.next().await.unwrap(), JsonReadToken::EndOfStream);
793            }
794        }
795        {
796            let mut r = Cursor::new(input_with_whitespace.as_bytes().to_vec());
797            let mut json_reader = JsonReader::new(64, &mut r);
798            for evt in &expected {
799                assert_eq!(&json_reader.next().await.unwrap(), evt);
800            }
801            if let Some(expected_error) = &expected_error {
802                match json_reader.next().await {
803                    Ok(_) => panic!("expected error but was ok: {}", expected_error),
804                    Err(e) => assert_is_similar_error(&e, expected_error),
805                }
806            }
807            else {
808                assert_eq!(json_reader.next().await.unwrap(), JsonReadToken::EndOfStream);
809            }
810        }
811    }
812
813    #[tokio::test]
814    async fn test_provided_buffer_fits() -> Result<(), JsonParseError<io::Error>> {
815        let buf = [0u8;8];
816        let mut r = Cursor::new(b"123".to_vec());
817        let mut reader = JsonReader::new_with_provided_buffer(buf, &mut r, false);
818        assert_eq!(reader.next().await?, JsonReadToken::NumberLiteral(JsonNumber("123")));
819        assert_eq!(reader.next().await?, JsonReadToken::EndOfStream);
820        Ok(())
821    }
822
823    #[tokio::test]
824    async fn test_provided_buffer_overflow() -> Result<(), JsonParseError<io::Error>> {
825        let buf = [0u8;8];
826        let mut r = Cursor::new(b"\"123 123 x\"".to_vec());
827        let mut reader = JsonReader::new_with_provided_buffer(buf, &mut r, false);
828        match reader.next().await {
829            Ok(_) => panic!("expected an error"),
830            Err(e) => assert_is_similar_error(&e, &JsonParseError::BufferOverflow(Location::start())),
831        }
832        Ok(())
833    }
834
835    #[rstest]
836    #[case::simple("1", Some(1), Some(1), 1.0, 1.0)]
837    #[case::big("1345678345", Some(1345678345), Some(1345678345), 1345678345.0, 1345678345.0)]
838    #[case::bigger("3345678345", Some(3345678345), None, 3345678345.0, 3345678345.0)]
839    #[case::too_big("13456783459", None, None, 13456783459.0, 13456783459.0)]
840    #[case::negative("-1", None, Some(-1), -1.0, -1.0)]
841    #[case::fract("1.0", None, None, 1.0, 1.0)]
842    #[case::exp("1e3", None, None, 1e3, 1e3)]
843    #[case::neg_exp("1e-3", None, None, 1e-3, 1e-3)]
844    #[case::pos_exp("1e+3", None, None, 1e3, 1e3)]
845    #[case::fract_exp("1.23e3", None, None, 1230.0, 1230.0)]
846    #[case::fract_neg_exp("1.23e-3", None, None, 1.23e-3, 1.23e-3)]
847    #[case::fract_pos_exp("1.23e+3", None, None, 1.23e3, 1.23e3)]
848    fn test_json_number_parse(#[case] s: &str, #[case] expected_u32: Option<u32>, #[case] expected_i32: Option<i32>, #[case] expected_f64: f64, #[case] expected_f32: f32) {
849        let n = JsonNumber(s);
850
851        {
852            let parsed = n.parse::<u32>();
853            match expected_u32 {
854                Some(e) => assert_eq!(e, parsed.unwrap()),
855                None => assert!(parsed.is_err()),
856            }
857        }
858        {
859            let parsed = n.parse::<i32>();
860            match expected_i32 {
861                Some(e) => assert_eq!(e, parsed.unwrap()),
862                None => assert!(parsed.is_err()),
863            }
864        }
865        {
866            let parsed = n.parse::<f64>();
867            assert_eq!(expected_f64, parsed.unwrap());
868        }
869        {
870            let parsed = n.parse::<f32>();
871            assert_eq!(expected_f32, parsed.unwrap());
872        }
873    }
874
875
876    #[rstest]
877    #[case::simple(Location::start(), vec![b'a'], Location { offset: 1, line: 1, column: 2,})]
878    #[case::cr(Location::start(), vec![b'\r'], Location { offset: 1, line: 1, column: 2,})]
879    #[case::tab(Location::start(), vec![b'\t'], Location { offset: 1, line: 1, column: 2,})]
880    #[case::nl(Location::start(), vec![b'\n'], Location { offset: 1, line: 2, column: 1,})]
881    #[case::in_line(Location::start(), vec![b'\r', b'\n', b'\n', b'x', b'y'], Location { offset: 5, line: 3, column: 3,})]
882    #[case::sequence(Location::start(), vec![b'a', b'b', b'\n', b'x', b'\n'], Location { offset: 5, line: 3, column: 1,})]
883    fn test_location_after_byte(#[case] mut initial: Location, #[case] bytes: Vec<u8>, #[case] expected: Location) {
884        for byte in bytes {
885            initial.after_byte(byte);
886        }
887        assert_eq!(initial, expected);
888    }
889
890
891    #[rstest]
892    #[case::key(r#""abc": null"#, Some(Some("abc")))]
893    #[case::other_key(r#""xyz": null"#, Some(Some("xyz")))]
894    #[case::end_object(r#"}"#, Some(None))]
895    #[case::null(r#"null"#, None)]
896    #[case::bool(r#"true"#, None)]
897    #[case::number(r#"1"#, None)]
898    #[case::string(r#""a""#, None)]
899    #[case::start_object(r#"{"#, None)]
900    #[case::start_array(r#"["#, None)]
901    #[case::end_array(r#"]"#, None)]
902    #[tokio::test]
903    async fn test_expect_next_key(#[case] json: &str, #[case] expected: Option<Option<&str>>) {
904        let mut r = Cursor::new(json.as_bytes().to_vec());
905        let mut json_reader = JsonReader::new(64, &mut r);
906        match json_reader.expect_next_key().await {
907            Ok(actual) => assert_eq!(actual, expected.unwrap()),
908            Err(JsonParseError::UnexpectedToken(_,_)) => assert!(expected.is_none()),
909            Err(e) => panic!("unexpected error: {}", e)
910        }
911    }
912
913    #[rstest]
914    #[case::simple("1", Ok(1))]
915    #[case::other_number("500", Err(JsonParseError::Parse("invalid number", Location::start())))]
916    #[case::null("null", Err(JsonParseError::UnexpectedToken("", Location::start())))]
917    #[case::string("\"abc\"", Err(JsonParseError::UnexpectedToken("", Location::start())))]
918    #[case::key("\"abc\": ", Err(JsonParseError::UnexpectedToken("", Location::start())))]
919    #[case::bool("true", Err(JsonParseError::UnexpectedToken("", Location::start())))]
920    #[case::start_object("{", Err(JsonParseError::UnexpectedToken("", Location::start())))]
921    #[case::end_object("}", Err(JsonParseError::UnexpectedToken("", Location::start())))]
922    #[case::start_array("[", Err(JsonParseError::UnexpectedToken("", Location::start())))]
923    #[case::end_array("]", Err(JsonParseError::UnexpectedToken("", Location::start())))]
924    #[tokio::test]
925    async fn test_expect_next_number(#[case] json: &str, #[case] expected_num: JsonParseResult<u8, io::Error>) {
926        let mut r = Cursor::new(json.as_bytes().to_vec());
927        let mut json_reader = JsonReader::new(64, &mut r);
928        match json_reader.expect_next_number::<u8>().await {
929            Ok(n) => assert_eq!(n, expected_num.unwrap()),
930            Err(act_e) => match expected_num {
931                Ok(_) => panic!("unexpected error: {}", act_e),
932                Err(exp_e) => assert_is_similar_error(&act_e, &exp_e),
933            }
934        }
935    }
936
937    #[rstest]
938    #[case::simple("1", Ok(Some(1)))]
939    #[case::other_number("500", Err(JsonParseError::Parse("invalid number", Location::start())))]
940    #[case::null("null", Ok(None))]
941    #[case::string("\"abc\"", Err(JsonParseError::UnexpectedToken("", Location::start())))]
942    #[case::key("\"abc\": ", Err(JsonParseError::UnexpectedToken("", Location::start())))]
943    #[case::bool("true", Err(JsonParseError::UnexpectedToken("", Location::start())))]
944    #[case::start_object("{", Err(JsonParseError::UnexpectedToken("", Location::start())))]
945    #[case::end_object("}", Err(JsonParseError::UnexpectedToken("", Location::start())))]
946    #[case::start_array("[", Err(JsonParseError::UnexpectedToken("", Location::start())))]
947    #[case::end_array("]", Err(JsonParseError::UnexpectedToken("", Location::start())))]
948    #[tokio::test]
949    async fn test_expect_next_opt_number(#[case] json: &str, #[case] expected_num: JsonParseResult<Option<u8>, io::Error>) {
950        let mut r = Cursor::new(json.as_bytes().to_vec());
951        let mut json_reader = JsonReader::new(64, &mut r);
952        match json_reader.expect_next_opt_number::<u8>().await {
953            Ok(n) => assert_eq!(n, expected_num.unwrap()),
954            Err(act_e) => match expected_num {
955                Ok(_) => panic!("unexpected error: {}", act_e),
956                Err(exp_e) => assert_is_similar_error(&act_e, &exp_e),
957            }
958        }
959    }
960
961    #[tokio::test]
962    async fn test_expect_next_raw_number() {
963        let json = " 123.45  ";
964        let mut r = Cursor::new(json.as_bytes().to_vec());
965        let mut json_reader = JsonReader::new(64, &mut r);
966        let actual = json_reader.expect_next_raw_number().await.unwrap();
967        assert_eq!(actual, JsonNumber("123.45"));
968    }
969
970    #[rstest]
971    #[case::number(" 5 ", Ok(Some(JsonNumber("5"))))]
972    #[case::null(" null ", Ok(None))]
973    #[case::boolean(" true ", Err(JsonParseError::UnexpectedToken("", Location::start())))]
974    #[tokio::test]
975    async fn test_expect_next_opt_raw_number(#[case] json: &str, #[case] expected_num: JsonParseResult<Option<JsonNumber<'_>>, io::Error>) {
976        let mut r = Cursor::new(json.as_bytes().to_vec());
977        let mut json_reader = JsonReader::new(64, &mut r);
978        match json_reader.expect_next_opt_raw_number().await {
979            Ok(o) => assert_eq!(o, expected_num.unwrap()),
980            Err(act_e) => match expected_num {
981                Ok(_) => panic!("unexpected error: {}", act_e),
982                Err(exp_e) => assert_is_similar_error(&act_e, &exp_e),
983            }
984
985        }
986    }
987
988    #[rstest]
989    #[case::simple("\"qrs\"", Ok("qrs"))]
990    #[case::null("null", Err(JsonParseError::UnexpectedToken("", Location::start())))]
991    #[case::number("12", Err(JsonParseError::UnexpectedToken("", Location::start())))]
992    #[case::key("\"abc\": ", Err(JsonParseError::UnexpectedToken("", Location::start())))]
993    #[case::bool("true", Err(JsonParseError::UnexpectedToken("", Location::start())))]
994    #[case::start_object("{", Err(JsonParseError::UnexpectedToken("", Location::start())))]
995    #[case::end_object("}", Err(JsonParseError::UnexpectedToken("", Location::start())))]
996    #[case::start_array("[", Err(JsonParseError::UnexpectedToken("", Location::start())))]
997    #[case::end_array("]", Err(JsonParseError::UnexpectedToken("", Location::start())))]
998    #[tokio::test]
999    async fn test_expect_next_string(#[case] json: &str, #[case] expected: JsonParseResult<&str, io::Error>) {
1000        let mut r = Cursor::new(json.as_bytes().to_vec());
1001        let mut json_reader = JsonReader::new(64, &mut r);
1002        match json_reader.expect_next_string().await {
1003            Ok(n) => assert_eq!(n, expected.unwrap()),
1004            Err(act_e) => match expected {
1005                Ok(_) => panic!("unexpected error: {}", act_e),
1006                Err(exp_e) => assert_is_similar_error(&act_e, &exp_e),
1007            }
1008        }
1009    }
1010
1011    #[rstest]
1012    #[case::simple("\"rst\"", Ok(Some("rst")))]
1013    #[case::null("null", Ok(None))]
1014    #[case::number("12", Err(JsonParseError::UnexpectedToken("", Location::start())))]
1015    #[case::key("\"abc\": ", Err(JsonParseError::UnexpectedToken("", Location::start())))]
1016    #[case::bool("true", Err(JsonParseError::UnexpectedToken("", Location::start())))]
1017    #[case::start_object("{", Err(JsonParseError::UnexpectedToken("", Location::start())))]
1018    #[case::end_object("}", Err(JsonParseError::UnexpectedToken("", Location::start())))]
1019    #[case::start_array("[", Err(JsonParseError::UnexpectedToken("", Location::start())))]
1020    #[case::end_array("]", Err(JsonParseError::UnexpectedToken("", Location::start())))]
1021    #[tokio::test]
1022    async fn test_expect_next_opt_string(#[case] json: &str, #[case] expected: JsonParseResult<Option<&str>, io::Error>) {
1023        let mut r = Cursor::new(json.as_bytes().to_vec());
1024        let mut json_reader = JsonReader::new(64, &mut r);
1025        match json_reader.expect_next_opt_string().await {
1026            Ok(n) => assert_eq!(n, expected.unwrap()),
1027            Err(act_e) => match expected {
1028                Ok(_) => panic!("unexpected error: {}", act_e),
1029                Err(exp_e) => assert_is_similar_error(&act_e, &exp_e),
1030            }
1031        }
1032    }
1033    #[rstest]
1034    #[case::bool_true("true", Ok(true))]
1035    #[case::bool_false("false", Ok(false))]
1036    #[case::null("null", Err(JsonParseError::UnexpectedToken("", Location::start())))]
1037    #[case::string("\"a\"", Err(JsonParseError::UnexpectedToken("", Location::start())))]
1038    #[case::number("12", Err(JsonParseError::UnexpectedToken("", Location::start())))]
1039    #[case::key("\"abc\": ", Err(JsonParseError::UnexpectedToken("", Location::start())))]
1040    #[case::start_object("{", Err(JsonParseError::UnexpectedToken("", Location::start())))]
1041    #[case::end_object("}", Err(JsonParseError::UnexpectedToken("", Location::start())))]
1042    #[case::start_array("[", Err(JsonParseError::UnexpectedToken("", Location::start())))]
1043    #[case::end_array("]", Err(JsonParseError::UnexpectedToken("", Location::start())))]
1044    #[tokio::test]
1045    async fn test_expect_next_bool(#[case] json: &str, #[case] expected: JsonParseResult<bool, io::Error>) {
1046        let mut r = Cursor::new(json.as_bytes().to_vec());
1047        let mut json_reader = JsonReader::new(64, &mut r);
1048        match json_reader.expect_next_bool().await {
1049            Ok(n) => assert_eq!(n, expected.unwrap()),
1050            Err(act_e) => match expected {
1051                Ok(_) => panic!("unexpected error: {}", act_e),
1052                Err(exp_e) => assert_is_similar_error(&act_e, &exp_e),
1053            }
1054        }
1055    }
1056
1057    #[rstest]
1058    #[case::bool_true("true", Ok(Some(true)))]
1059    #[case::bool_false("false", Ok(Some(false)))]
1060    #[case::null("null", Ok(None))]
1061    #[case::string("\"x\"", Err(JsonParseError::UnexpectedToken("", Location::start())))]
1062    #[case::number("12", Err(JsonParseError::UnexpectedToken("", Location::start())))]
1063    #[case::key("\"abc\": ", Err(JsonParseError::UnexpectedToken("", Location::start())))]
1064    #[case::start_object("{", Err(JsonParseError::UnexpectedToken("", Location::start())))]
1065    #[case::end_object("}", Err(JsonParseError::UnexpectedToken("", Location::start())))]
1066    #[case::start_array("[", Err(JsonParseError::UnexpectedToken("", Location::start())))]
1067    #[case::end_array("]", Err(JsonParseError::UnexpectedToken("", Location::start())))]
1068    #[tokio::test]
1069    async fn test_expect_next_opt_bool(#[case] json: &str, #[case] expected: JsonParseResult<Option<bool>, io::Error>) {
1070        let mut r = Cursor::new(json.as_bytes().to_vec());
1071        let mut json_reader = JsonReader::new(64, &mut r);
1072        match json_reader.expect_next_opt_bool().await {
1073            Ok(n) => assert_eq!(n, expected.unwrap()),
1074            Err(act_e) => match expected {
1075                Ok(_) => panic!("unexpected error: {}", act_e),
1076                Err(exp_e) => assert_is_similar_error(&act_e, &exp_e),
1077            }
1078        }
1079    }
1080
1081    #[rstest]
1082    #[case::null("null", Err(JsonParseError::UnexpectedToken("", Location::start())))]
1083    #[case::bool("true", Err(JsonParseError::UnexpectedToken("", Location::start())))]
1084    #[case::string("\"a\"", Err(JsonParseError::UnexpectedToken("", Location::start())))]
1085    #[case::number("12", Err(JsonParseError::UnexpectedToken("", Location::start())))]
1086    #[case::key("\"abc\": ", Err(JsonParseError::UnexpectedToken("", Location::start())))]
1087    #[case::start_object("{", Ok(()))]
1088    #[case::end_object("}", Err(JsonParseError::UnexpectedToken("", Location::start())))]
1089    #[case::start_array("[", Err(JsonParseError::UnexpectedToken("", Location::start())))]
1090    #[case::end_array("]", Err(JsonParseError::UnexpectedToken("", Location::start())))]
1091    #[tokio::test]
1092    async fn test_expect_next_start_object(#[case] json: &str, #[case] expected: JsonParseResult<(), io::Error>) {
1093        let mut r = Cursor::new(json.as_bytes().to_vec());
1094        let mut json_reader = JsonReader::new(64, &mut r);
1095        match json_reader.expect_next_start_object().await {
1096            Ok(n) => assert_eq!(n, expected.unwrap()),
1097            Err(act_e) => match expected {
1098                Ok(_) => panic!("unexpected error: {}", act_e),
1099                Err(exp_e) => assert_is_similar_error(&act_e, &exp_e),
1100            }
1101        }
1102    }
1103
1104    #[rstest]
1105    #[case::bool("false", Err(JsonParseError::UnexpectedToken("", Location::start())))]
1106    #[case::null("null", Ok(None))]
1107    #[case::string("\"x\"", Err(JsonParseError::UnexpectedToken("", Location::start())))]
1108    #[case::number("12", Err(JsonParseError::UnexpectedToken("", Location::start())))]
1109    #[case::key("\"abc\": ", Err(JsonParseError::UnexpectedToken("", Location::start())))]
1110    #[case::start_object("{", Ok(Some(())))]
1111    #[case::end_object("}", Err(JsonParseError::UnexpectedToken("", Location::start())))]
1112    #[case::start_array("[", Err(JsonParseError::UnexpectedToken("", Location::start())))]
1113    #[case::end_array("]", Err(JsonParseError::UnexpectedToken("", Location::start())))]
1114    #[tokio::test]
1115    async fn test_expect_next_opt_start_object(#[case] json: &str, #[case] expected: JsonParseResult<Option<()>, io::Error>) {
1116        let mut r = Cursor::new(json.as_bytes().to_vec());
1117        let mut json_reader = JsonReader::new(64, &mut r);
1118        match json_reader.expect_next_opt_start_object().await {
1119            Ok(n) => assert_eq!(n, expected.unwrap()),
1120            Err(act_e) => match expected {
1121                Ok(_) => panic!("unexpected error: {}", act_e),
1122                Err(exp_e) => assert_is_similar_error(&act_e, &exp_e),
1123            }
1124        }
1125    }
1126
1127    #[rstest]
1128    #[case::null("null", Err(JsonParseError::UnexpectedToken("", Location::start())))]
1129    #[case::bool("true", Err(JsonParseError::UnexpectedToken("", Location::start())))]
1130    #[case::string("\"a\"", Err(JsonParseError::UnexpectedToken("", Location::start())))]
1131    #[case::number("12", Err(JsonParseError::UnexpectedToken("", Location::start())))]
1132    #[case::key("\"abc\": ", Err(JsonParseError::UnexpectedToken("", Location::start())))]
1133    #[case::start_object("{", Err(JsonParseError::UnexpectedToken("", Location::start())))]
1134    #[case::end_object("}", Err(JsonParseError::UnexpectedToken("", Location::start())))]
1135    #[case::start_array("[", Ok(()))]
1136    #[case::end_array("]", Err(JsonParseError::UnexpectedToken("", Location::start())))]
1137    #[tokio::test]
1138    async fn test_expect_next_start_array(#[case] json: &str, #[case] expected: JsonParseResult<(), io::Error>) {
1139        let mut r = Cursor::new(json.as_bytes().to_vec());
1140        let mut json_reader = JsonReader::new(64, &mut r);
1141        match json_reader.expect_next_start_array().await {
1142            Ok(n) => assert_eq!(n, expected.unwrap()),
1143            Err(act_e) => match expected {
1144                Ok(_) => panic!("unexpected error: {}", act_e),
1145                Err(exp_e) => assert_is_similar_error(&act_e, &exp_e),
1146            }
1147        }
1148    }
1149
1150    #[rstest]
1151    #[case::bool("false", Err(JsonParseError::UnexpectedToken("", Location::start())))]
1152    #[case::null("null", Ok(None))]
1153    #[case::string("\"x\"", Err(JsonParseError::UnexpectedToken("", Location::start())))]
1154    #[case::number("12", Err(JsonParseError::UnexpectedToken("", Location::start())))]
1155    #[case::key("\"abc\": ", Err(JsonParseError::UnexpectedToken("", Location::start())))]
1156    #[case::start_object("{", Err(JsonParseError::UnexpectedToken("", Location::start())))]
1157    #[case::end_object("}", Err(JsonParseError::UnexpectedToken("", Location::start())))]
1158    #[case::start_array("[", Ok(Some(())))]
1159    #[case::end_array("]", Err(JsonParseError::UnexpectedToken("", Location::start())))]
1160    #[tokio::test]
1161    async fn test_expect_next_opt_start_array(#[case] json: &str, #[case] expected: JsonParseResult<Option<()>, io::Error>) {
1162        let mut r = Cursor::new(json.as_bytes().to_vec());
1163        let mut json_reader = JsonReader::new(64, &mut r);
1164        match json_reader.expect_next_opt_start_array().await {
1165            Ok(n) => assert_eq!(n, expected.unwrap()),
1166            Err(act_e) => match expected {
1167                Ok(_) => panic!("unexpected error: {}", act_e),
1168                Err(exp_e) => assert_is_similar_error(&act_e, &exp_e),
1169            }
1170        }
1171    }
1172
1173    #[tokio::test]
1174    async fn test_with_lenient_comma() {
1175        let json = r#"
1176        { "a": 1}
1177        { "a": 2}
1178        "#;
1179
1180        {
1181            let mut r = Cursor::new(json.as_bytes().to_vec());
1182            let result = try_read_json_lines(&mut JsonReader::new(64, &mut r)).await;
1183            assert!(result.is_err());
1184        }
1185
1186        {
1187            let mut r = Cursor::new(json.as_bytes().to_vec());
1188            let result = try_read_json_lines(&mut JsonReader::new_with_lenient_comma_handling(64, &mut r)).await;
1189            assert_eq!(result.unwrap(), vec![1, 2]);
1190        }
1191    }
1192
1193    async fn try_read_json_lines(json_reader: &mut JsonReader<'_, Vec<u8>, Cursor<Vec<u8>>>) -> JsonParseResult<Vec<u32>, io::Error> {
1194        let mut result = Vec::new();
1195        while json_reader.next().await? == JsonReadToken::StartObject {
1196            loop {
1197                match json_reader.expect_next_key().await? {
1198                    Some("a") => result.push( json_reader.expect_next_number().await?),
1199                    Some(_) => json_reader.skip_value().await?,
1200                    None => break,
1201                }
1202            }
1203        }
1204        Ok(result)
1205    }
1206
1207    #[rstest]
1208    #[case::end_object_empty("}, 77", false)]
1209    #[case::end_object_simple(r#" "a": 10, "b": null }, 77"#, false)]
1210    #[case::end_object_nested(r#" "a": 10, "x": { "q": 99, "r": [1, 2, 3] }, "b": null }, 77"#, false)]
1211    #[case::end_array_empty("], 77", false)]
1212    #[case::end_array_simple(r#"99, "abc", null, true], 77"#, false)]
1213    #[case::end_array_nested(r#"99, [1, [], true, {"abc": { "xyz": [1, 2, 3]}}], "abc", null, true], 77"#, false)]
1214    #[case::number("1, 77", true)]
1215    #[case::boolean("true, 77", true)]
1216    #[case::null("null, 77", true)]
1217    #[tokio::test]
1218    async fn test_skip_to_end_of_current_scope(#[case] json: &str, #[case] should_fail: bool) -> JsonParseResult<(), io::Error> {
1219        let mut r = Cursor::new(json.as_bytes().to_vec());
1220        let mut json_reader = JsonReader::new(64, &mut r);
1221        match json_reader.skip_to_end_of_current_scope().await {
1222            Ok(_) => {
1223                assert!(!should_fail);
1224                assert_eq!(77, json_reader.expect_next_number::<u32>().await?);
1225            }
1226            Err(_) => {
1227                assert!(should_fail);
1228            }
1229        }
1230        Ok(())
1231    }
1232
1233    #[rstest]
1234    #[case::number(r#"123, 77"#, false)]
1235    #[case::string(r#""abc", 77"#, false)]
1236    #[case::boolean(r#"true, 77"#, false)]
1237    #[case::null(r#"null, 77"#, false)]
1238    #[case::object_empty(r#"{}, 77"#, false)]
1239    #[case::object_simple(r#"{ "a": 10, "b": null }, 77"#, false)]
1240    #[case::object_nested(r#"{ "a": 10, "x": { "q": 99, "r": [1, 2, 3] }, "b": null }, 77"#, false)]
1241    #[case::array_empty("[], 77", false)]
1242    #[case::array_simple(r#"[99, "abc", null, true], 77"#, false)]
1243    #[case::array_nested(r#"[99, [1, [], true, {"abc": { "xyz": [1, 2, 3]}}], "abc", null, true], 77"#, false)]
1244    #[tokio::test]
1245    async fn test_skip_value(#[case] json: &str, #[case] should_fail: bool) -> JsonParseResult<(), io::Error> {
1246        let mut r = Cursor::new(json.as_bytes().to_vec());
1247        let mut json_reader = JsonReader::new(64, &mut r);
1248        match json_reader.skip_value().await {
1249            Ok(_) => {
1250                assert!(!should_fail);
1251                assert_eq!(77, json_reader.expect_next_number::<u32>().await?);
1252            }
1253            Err(_) => {
1254                assert!(should_fail);
1255            }
1256        }
1257        Ok(())
1258    }
1259}
json_streaming/nonblocking/read.rs

json_streaming/nonblocking/
read.rs
