jsonx_lib/stream.rs
1//! The incremental reader: a document fed in chunks as they arrive.
2
3use alloc::string::String;
4
5use crate::{
6 DecodeError, DuplicateKeys, FromJson, Kind, Limits, Parser, Value,
7 error::{Error, ErrorKind},
8 machine::Machine,
9 reader::Event,
10};
11
12/// The answer of a [`StreamReader`] call: ready, or waiting for more input.
13///
14/// `NeedInput` is not an error: the input so far is a valid beginning, and
15/// the call can be made again after [`feed`](StreamReader::feed) or
16/// [`end_input`](StreamReader::end_input).
17///
18/// # Examples
19///
20/// ```
21/// use jsonx_lib::{Event, Step, StreamReader};
22///
23/// let mut r = StreamReader::new();
24/// assert_eq!(r.next_event()?, Step::NeedInput);
25/// r.feed(b"[tr")?;
26/// assert_eq!(r.next_event()?, Step::Ready(Some(Event::StartArray)));
27/// assert_eq!(r.next_event()?, Step::NeedInput); // `tr` is not a value yet
28/// r.feed(b"ue]")?;
29/// assert_eq!(r.next_event()?.ready(), Some(Some(Event::Bool(true))));
30/// # Ok::<(), jsonx_lib::Error>(())
31/// ```
32#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash)]
33#[must_use]
34pub enum Step<T> {
35 /// The call completed with this result.
36 Ready(T),
37 /// The input so far ends inside what the call needs: feed more, or end
38 /// the input, and call again.
39 NeedInput,
40}
41
42impl<T> Step<T> {
43 /// The result, if ready.
44 ///
45 /// # Examples
46 ///
47 /// ```
48 /// use jsonx_lib::Step;
49 ///
50 /// assert_eq!(Step::Ready(3).ready(), Some(3));
51 /// assert_eq!(Step::<u8>::NeedInput.ready(), None);
52 /// ```
53 #[inline]
54 pub fn ready(self) -> Option<T> {
55 match self {
56 Self::Ready(value) => Some(value),
57 Self::NeedInput => None,
58 }
59 }
60
61 /// Whether the call completed.
62 ///
63 /// # Examples
64 ///
65 /// ```
66 /// assert!(jsonx_lib::Step::Ready(()).is_ready());
67 /// ```
68 #[inline]
69 #[must_use]
70 pub const fn is_ready(&self) -> bool {
71 matches!(self, Self::Ready(_))
72 }
73
74 /// Applies `f` to a ready result.
75 ///
76 /// # Examples
77 ///
78 /// ```
79 /// use jsonx_lib::Step;
80 ///
81 /// assert_eq!(Step::Ready(2).map(|n| n * 10), Step::Ready(20));
82 /// ```
83 #[inline]
84 pub fn map<U>(self, f: impl FnOnce(T) -> U) -> Step<U> {
85 match self {
86 Self::Ready(value) => Step::Ready(f(value)),
87 Self::NeedInput => Step::NeedInput,
88 }
89 }
90}
91
92/// A raw value being skipped across calls.
93#[derive(Clone, Copy, Debug)]
94struct RawProgress {
95 /// The depth the value started at; it is complete when the reader is back.
96 depth: usize,
97 /// The value's first byte (absolute), once its first event was read.
98 start: Option<usize>,
99}
100
101/// How much consumed text the buffer may keep before it is compacted.
102const COMPACT_AT: usize = 4 * 1024;
103
104/// A pull reader over input that arrives in pieces: bytes go in with
105/// [`feed`](Self::feed) as they arrive, events come out as soon as they are
106/// complete.
107///
108/// It is the same reader as [`Reader`](crate::Reader), over a buffer: the
109/// same grammar, the same budgets, and the same errors at the same offsets
110/// (counted from the first byte ever fed), whatever the input's chunking.
111/// When a call needs bytes that have not arrived it returns
112/// [`Step::NeedInput`] and changes nothing a later call would see
113/// differently; after [`end_input`](Self::end_input), running out is the
114/// [`ErrorKind::UnexpectedEnd`] that [`Reader`](crate::Reader) reports.
115///
116/// Events borrow from the reader's buffer, so each one must be dropped
117/// before the next call. In particular, feed more input after the borrow
118/// ends: in an `else` after `if let Step::Ready(..) = reader.next_event()?`
119/// (Rust 2024 drops the `if let` temporaries before the `else`), or after
120/// the loop that drains the ready events, as below; a `match` keeps the
121/// event borrowed in every arm. Consumed input is dropped from the buffer as more
122/// is fed, so a long document read with [`next_event`](Self::next_event) as
123/// it arrives needs memory for one token and one chunk, not for the
124/// document. The whole-value calls ([`read_raw`](Self::read_raw),
125/// [`read_value`](Self::read_value), [`decode`](Self::decode)) are
126/// different: each keeps its value's text in the buffer until the value
127/// ends, bounded by [`Limits::max_value_bytes`]. A token cut by a chunk
128/// boundary is continued where it stopped, never scanned again from its
129/// start: a string fed one byte at a time costs linear time.
130///
131/// The input must be UTF-8, checked as it arrives (a character may be split
132/// across chunks). The first invalid byte is reported as
133/// [`ErrorKind::InvalidUtf8`] when the reader reaches it: a syntax error
134/// earlier in the document is reported first. (A complete document given to
135/// [`parse_bytes`](crate::parse_bytes) checks the encoding before the
136/// syntax, so the two can name different errors for a document that has
137/// both.)
138///
139/// [`Limits::max_input_bytes`] bounds the total fed (in
140/// [`documents`](Self::documents) mode, what is fed for each document), and
141/// [`Limits::max_value_bytes`] what a whole-value call may hold; the other
142/// budgets apply exactly as for a complete document. After an error the
143/// reader is finished: every later call returns the same error.
144///
145/// # Examples
146///
147/// ```
148/// use jsonx_lib::{Event, Step, StreamReader};
149///
150/// let mut reader = StreamReader::new();
151/// let mut keys = Vec::new();
152/// for chunk in [&b"{\"jsonr"[..], b"pc\": \"2.0\", \"i", b"d\": 12", b"}"] {
153/// reader.feed(chunk)?;
154/// while let Step::Ready(Some(event)) = reader.next_event()? {
155/// if let Event::Key(key) = event {
156/// keys.push(key.into_owned());
157/// }
158/// }
159/// }
160/// reader.end_input();
161/// assert_eq!(reader.next_event()?, Step::Ready(None)); // the document is complete
162/// assert_eq!(keys, ["jsonrpc", "id"]);
163/// # Ok::<(), jsonx_lib::Error>(())
164/// ```
165#[derive(Clone, Debug)]
166pub struct StreamReader {
167 /// The input not yet dropped: valid UTF-8, starting at absolute offset
168 /// `base`.
169 buf: String,
170 base: usize,
171 /// An incomplete UTF-8 sequence at the end of the input so far.
172 tail: [u8; 4],
173 tail_len: usize,
174 /// The absolute offset of the first byte that is not UTF-8. Nothing
175 /// from it on is kept.
176 utf8_error: Option<usize>,
177 /// Bytes fed in total.
178 fed: usize,
179 ended: bool,
180 machine: Machine,
181 failed: Option<Error>,
182 /// The absolute offset where the most recent event's token began.
183 last: usize,
184 raw: Option<RawProgress>,
185}
186
187impl Default for StreamReader {
188 fn default() -> Self {
189 Self::new()
190 }
191}
192
193impl StreamReader {
194 /// A reader with the default [`Limits`], refusing duplicate keys in what
195 /// [`read_value`](Self::read_value) builds. For others, use
196 /// [`Parser::stream_reader`].
197 ///
198 /// # Examples
199 ///
200 /// ```
201 /// let r = jsonx_lib::StreamReader::new();
202 /// assert_eq!(r.offset(), 0);
203 /// ```
204 #[must_use]
205 pub fn new() -> Self {
206 Self::with_config(Limits::new(), DuplicateKeys::Error)
207 }
208
209 /// A reader with the given budgets and duplicate-key policy.
210 pub(crate) fn with_config(limits: Limits, duplicates: DuplicateKeys) -> Self {
211 Self {
212 buf: String::new(),
213 base: 0,
214 tail: [0; 4],
215 tail_len: 0,
216 utf8_error: None,
217 fed: 0,
218 ended: false,
219 machine: {
220 let mut machine = Machine::new(limits, duplicates, false);
221 // Keys outlive the front of the buffer, which is dropped.
222 machine.copy_keys = true;
223 machine
224 },
225 failed: None,
226 last: 0,
227 raw: None,
228 }
229 }
230
231 /// Reads a sequence of documents instead of one, as
232 /// [`Reader::documents`](crate::Reader::documents) does: NDJSON, JSON
233 /// Lines, or concatenated JSON, fed as it arrives. Whole-value reads
234 /// return one document each, and `Ready(None)` once the input has ended
235 /// after the last.
236 ///
237 /// [`Limits::max_input_bytes`] then applies to each document, not to the
238 /// stream: input is counted from where the current document began (or,
239 /// between documents, from the end of the last one), so a connection
240 /// that carries documents for as long as it stays open is never refused
241 /// for its total. A [`Reader`](crate::Reader), which has its whole input
242 /// already, applies the budget to all of it.
243 ///
244 /// # Examples
245 ///
246 /// ```
247 /// use jsonx_lib::{Step, StreamReader};
248 ///
249 /// let mut r = StreamReader::new().documents();
250 /// let mut seen = Vec::new();
251 /// for chunk in [&b"{\"id\": 1}\n{\"i"[..], b"d\": 2}\n"] {
252 /// r.feed(chunk)?;
253 /// while let Step::Ready(Some(doc)) = r.read_value()? {
254 /// seen.push(doc["id"].as_i64());
255 /// }
256 /// }
257 /// r.end_input();
258 /// assert_eq!(r.read_value()?, Step::Ready(None));
259 /// assert_eq!(seen, [Some(1), Some(2)]);
260 /// # Ok::<(), jsonx_lib::Error>(())
261 /// ```
262 #[must_use]
263 pub fn documents(mut self) -> Self {
264 self.machine.documents = true;
265 self
266 }
267
268 /// Adds the next piece of input. A chunk may end anywhere, even inside a
269 /// token or a UTF-8 character.
270 ///
271 /// # Errors
272 ///
273 /// [`ErrorKind::InputTooLarge`] when the input in total (in documents
274 /// mode, of the current document) grows past [`Limits::max_input_bytes`],
275 /// or when the value a pending whole-value call is reading grows past
276 /// [`Limits::max_value_bytes`], at the offset of the first byte over the
277 /// budget (the reader is then finished);
278 /// [`ErrorKind::InputAfterEnd`] after [`end_input`](Self::end_input);
279 /// any error the reader already stopped at.
280 ///
281 /// # Examples
282 ///
283 /// ```
284 /// use jsonx_lib::{ErrorKind, Limits, Parser};
285 ///
286 /// let mut r = Parser::new().limits(Limits::new().with_max_input_bytes(4)).stream_reader();
287 /// assert!(r.feed(b"[1,").is_ok());
288 /// assert_eq!(r.feed(b"22]").unwrap_err().kind(), ErrorKind::InputTooLarge);
289 /// ```
290 pub fn feed(&mut self, bytes: &[u8]) -> Result<(), Error> {
291 if let Some(e) = self.failed {
292 return Err(e);
293 }
294 if self.ended {
295 return Err(Error::new(ErrorKind::InputAfterEnd, self.fed));
296 }
297 // The input budget, from where the current document began (the
298 // start of the input outside documents mode).
299 let start = self.machine.document_start();
300 let max = self.machine.limits.max_input_bytes;
301 if bytes.len() > max.saturating_sub(self.fed.saturating_sub(start)) {
302 let e = Error::new(ErrorKind::InputTooLarge, start.saturating_add(max));
303 return Err(*self.failed.insert(e));
304 }
305 // The value a whole-value call is waiting on stays in the buffer.
306 if let Some(progress) = self.raw {
307 let start = progress
308 .start
309 .unwrap_or(self.base + self.machine.needed_from());
310 let max = self.machine.limits.max_value_bytes;
311 if bytes.len() > max.saturating_sub(self.fed.saturating_sub(start)) {
312 let e = Error::new(ErrorKind::InputTooLarge, start.saturating_add(max));
313 return Err(*self.failed.insert(e));
314 }
315 }
316 self.fed += bytes.len();
317 self.compact();
318 if self.utf8_error.is_none() {
319 self.append(bytes);
320 }
321 Ok(())
322 }
323
324 /// Says that the input has ended: nothing more will be fed. From now on,
325 /// running out of input is an error, as for a complete document, and a
326 /// number or whitespace at the very end is final.
327 ///
328 /// # Examples
329 ///
330 /// ```
331 /// use jsonx_lib::{Event, Step, StreamReader};
332 ///
333 /// let mut r = StreamReader::new();
334 /// r.feed(b"12")?;
335 /// assert_eq!(r.next_event()?, Step::NeedInput); // `123` may follow
336 /// r.end_input();
337 /// assert_eq!(r.next_event()?, Step::Ready(Some(Event::Number(12.into()))));
338 /// assert_eq!(r.next_event()?, Step::Ready(None));
339 /// # Ok::<(), jsonx_lib::Error>(())
340 /// ```
341 pub fn end_input(&mut self) {
342 if self.ended {
343 return;
344 }
345 self.ended = true;
346 if self.tail_len > 0 && self.utf8_error.is_none() {
347 // A character cut off by the end of the input.
348 self.utf8_error = Some(self.base + self.buf.len());
349 }
350 self.machine.complete = true;
351 }
352
353 /// Reads up to 8 KiB from `source` and feeds it, or calls
354 /// [`end_input`](Self::end_input) when `source` is at its end. Returns
355 /// the number of bytes read (`0` at the end). Interrupted reads are
356 /// retried.
357 ///
358 /// A budget error from [`feed`](Self::feed) is not an I/O error: it is
359 /// kept, and returned by the next reading call.
360 ///
361 /// # Errors
362 ///
363 /// Any error from `source` other than [`std::io::ErrorKind::Interrupted`].
364 ///
365 /// # Examples
366 ///
367 /// ```
368 /// use jsonx_lib::{Step, StreamReader};
369 ///
370 /// let mut source: &[u8] = br#"{"big": [1, 2, 3]}"#;
371 /// let mut r = StreamReader::new();
372 /// let mut events = 0;
373 /// loop {
374 /// if let Step::Ready(event) = r.next_event()? {
375 /// match event {
376 /// Some(_) => events += 1,
377 /// None => break,
378 /// }
379 /// } else {
380 /// r.read_from(&mut source)?;
381 /// }
382 /// }
383 /// assert_eq!(events, 8); // {, "big", [, 1, 2, 3, ], }
384 /// # Ok::<(), Box<dyn std::error::Error>>(())
385 /// ```
386 #[cfg(feature = "std")]
387 #[cfg_attr(docsrs, doc(cfg(feature = "std")))]
388 pub fn read_from(&mut self, source: &mut dyn std::io::Read) -> std::io::Result<usize> {
389 let mut chunk = [0u8; 8 * 1024];
390 loop {
391 match source.read(&mut chunk) {
392 Ok(0) => {
393 self.end_input();
394 return Ok(0);
395 }
396 Ok(n) => {
397 // A refused feed is kept in `failed` and reported by the
398 // next reading call.
399 let _kept = self.feed(chunk.get(..n).unwrap_or_default());
400 return Ok(n);
401 }
402 Err(e) if e.kind() == std::io::ErrorKind::Interrupted => {}
403 Err(e) => return Err(e),
404 }
405 }
406 }
407
408 /// Reads the next event: `Ready(Some(event))`, `Ready(None)` once the
409 /// document is complete and the input has ended with nothing but
410 /// whitespace after it, or `NeedInput`.
411 ///
412 /// # Errors
413 ///
414 /// The first syntax error, exceeded budget, or invalid UTF-8, at its
415 /// byte offset. Once an error is returned, every later call returns it.
416 ///
417 /// # Examples
418 ///
419 /// ```
420 /// use jsonx_lib::{ErrorKind, Event, Step, StreamReader};
421 ///
422 /// let mut r = StreamReader::new();
423 /// r.feed(b"[1,")?;
424 /// assert_eq!(r.next_event()?, Step::Ready(Some(Event::StartArray)));
425 /// assert!(r.next_event()?.is_ready()); // 1
426 /// r.feed(b"]")?;
427 /// let err = r.next_event().unwrap_err();
428 /// assert_eq!((err.kind(), err.offset()), (ErrorKind::TrailingComma, 3));
429 /// # Ok::<(), jsonx_lib::Error>(())
430 /// ```
431 pub fn next_event(&mut self) -> Result<Step<Option<Event<'_>>>, Error> {
432 if let Some(e) = self.failed {
433 return Err(e);
434 }
435 let Self {
436 buf,
437 machine,
438 base,
439 last,
440 failed,
441 utf8_error,
442 ..
443 } = self;
444 match machine.step(buf) {
445 Ok(Some(event)) => {
446 *last = *base + machine.last;
447 Ok(Step::Ready(Some(event)))
448 }
449 Ok(None) => match utf8_error {
450 // The valid text holds a whole document, then bad bytes.
451 Some(at) => Err(*failed.insert(Error::new(ErrorKind::InvalidUtf8, *at))),
452 None => Ok(Step::Ready(None)),
453 },
454 Err(e) => halt(failed, machine, buf.len(), *base, *utf8_error, e),
455 }
456 }
457
458 /// Reads the next value whole and returns its exact source text, without
459 /// decoding or allocating anything, as [`Reader::read_raw`](crate::Reader::read_raw)
460 /// does, with the same positioning rules: `Ready(None)` when no value
461 /// comes next.
462 ///
463 /// The value must be in the buffer whole before it can be returned, so
464 /// a large value costs memory for all of it, whitespace included, up to
465 /// [`Limits::max_value_bytes`] (past it, [`feed`](Self::feed) refuses
466 /// more). `NeedInput` keeps the progress: the next call continues the
467 /// same value.
468 ///
469 /// After a single document's value, a call answers as
470 /// [`next_event`](Self::next_event) does there: `NeedInput` while only
471 /// whitespace has followed and the input has not ended, `Ready(None)`
472 /// once it has, and [`ErrorKind::TrailingCharacters`] for anything but
473 /// whitespace. [`finish`](Self::finish) makes the same check in one call.
474 ///
475 /// # Errors
476 ///
477 /// As [`next_event`](Self::next_event), for any error inside the value,
478 /// and [`ErrorKind::TrailingCharacters`] after the document.
479 ///
480 /// # Examples
481 ///
482 /// ```
483 /// use jsonx_lib::{Event, Step, StreamReader};
484 ///
485 /// let mut r = StreamReader::new();
486 /// r.feed(br#"{"id": 1, "params": {"line""#)?;
487 /// assert!(r.next_event()?.is_ready()); // {
488 /// assert!(r.next_event()?.is_ready()); // "id"
489 /// assert_eq!(r.read_raw()?, Step::Ready(Some("1")));
490 /// assert!(r.next_event()?.is_ready()); // "params"
491 /// assert_eq!(r.read_raw()?, Step::NeedInput);
492 /// r.feed(br#": 3}}"#)?;
493 /// assert_eq!(r.read_raw()?, Step::Ready(Some(r#"{"line": 3}"#)));
494 /// # Ok::<(), jsonx_lib::Error>(())
495 /// ```
496 pub fn read_raw(&mut self) -> Result<Step<Option<&str>>, Error> {
497 Ok(self.raw_span()?.map(|span| {
498 span.and_then(|(start, end)| self.buf.get(start - self.base..end - self.base))
499 }))
500 }
501
502 /// Reads the next value whole and builds it as a [`Value`], applying the
503 /// reader's [`DuplicateKeys`] policy; positioning as for
504 /// [`read_raw`](Self::read_raw). The value borrows from the buffer:
505 /// call [`Value::into_owned`] to keep it.
506 ///
507 /// # Errors
508 ///
509 /// As [`read_raw`](Self::read_raw), plus [`ErrorKind::DuplicateKey`]
510 /// under [`DuplicateKeys::Error`].
511 ///
512 /// # Examples
513 ///
514 /// ```
515 /// use jsonx_lib::{Step, StreamReader};
516 ///
517 /// let mut r = StreamReader::new();
518 /// r.feed(br#"{"a": [1, "#)?;
519 /// assert_eq!(r.read_value()?, Step::NeedInput);
520 /// r.feed(br#"2]}"#)?;
521 /// let value = r.read_value()?.ready().flatten().unwrap_or_default();
522 /// assert_eq!(value["a"][1], 2);
523 /// # Ok::<(), jsonx_lib::Error>(())
524 /// ```
525 pub fn read_value(&mut self) -> Result<Step<Option<Value<'_>>>, Error> {
526 if let Some(e) = self.failed {
527 return Err(e);
528 }
529 let Step::Ready(span) = self.raw_span()? else {
530 return Ok(Step::NeedInput);
531 };
532 let Some((start, end)) = span else {
533 return Ok(Step::Ready(None));
534 };
535 let text = self
536 .buf
537 .get(start - self.base..end - self.base)
538 .unwrap_or("");
539 // The budgets, and duplicates under `DuplicateKeys::Error`, were
540 // checked while the value was skipped; the memory budget, and
541 // `First` and `Last`, remain to apply to the tree.
542 let limits = Limits::unlimited().with_max_memory(self.machine.limits.max_memory);
543 let parser = Parser::new()
544 .limits(limits)
545 .duplicate_keys(self.machine.duplicates);
546 match parser.parse(text) {
547 Ok(value) => Ok(Step::Ready(Some(value))),
548 Err(e) => {
549 let e = Error::new(e.kind(), e.offset() + start);
550 self.failed = Some(e);
551 Err(e)
552 }
553 }
554 }
555
556 /// Reads the next value whole as a `T` ([`FromJson`]); positioning as
557 /// for [`read_raw`](Self::read_raw). The value borrows from the buffer,
558 /// as `T` allows.
559 ///
560 /// The value is first skipped (validated, with every budget) as it
561 /// arrives, then read as `T` once it is all there. Byte offsets in an
562 /// error count from the first byte fed.
563 ///
564 /// # Errors
565 ///
566 /// The [`DecodeError`] of `T`, or the reader's error.
567 ///
568 /// # Examples
569 ///
570 /// ```
571 /// use jsonx_lib::{Step, StreamReader};
572 ///
573 /// let mut r = StreamReader::new().documents();
574 /// r.feed(b"[1, 2]\n[3")?;
575 /// assert_eq!(r.decode::<Vec<u8>>()?, Step::Ready(Some(vec![1, 2])));
576 /// assert_eq!(r.decode::<Vec<u8>>()?, Step::NeedInput);
577 /// r.feed(b"]\n")?;
578 /// r.end_input();
579 /// assert_eq!(r.decode::<Vec<u8>>()?, Step::Ready(Some(vec![3])));
580 /// assert_eq!(r.decode::<Vec<u8>>()?, Step::Ready(None));
581 /// # Ok::<(), jsonx_lib::DecodeError>(())
582 /// ```
583 pub fn decode<'s, T: FromJson<'s>>(&'s mut self) -> Result<Step<Option<T>>, DecodeError> {
584 let Step::Ready(span) = self.raw_span()? else {
585 return Ok(Step::NeedInput);
586 };
587 let Some((start, end)) = span else {
588 return Ok(Step::Ready(None));
589 };
590 let text = self
591 .buf
592 .get(start - self.base..end - self.base)
593 .unwrap_or("");
594 // The budgets, and duplicates under `DuplicateKeys::Error`, were
595 // checked while the value was skipped. The depth budget stays, to
596 // bound a recursive `T`, and the memory budget, for what `T` holds.
597 let limits = Limits::unlimited()
598 .with_max_depth(self.machine.limits.max_depth)
599 .with_max_memory(self.machine.limits.max_memory);
600 let parser = Parser::new()
601 .limits(limits)
602 .duplicate_keys(self.machine.duplicates)
603 .numbers(self.machine.numbers);
604 parser
605 .decode(text)
606 .map(|value| Step::Ready(Some(value)))
607 .map_err(|e| e.shifted(start))
608 }
609
610 /// Reads to the end of the input, checking that the rest of the
611 /// document (in documents mode, of every document) is valid and that
612 /// nothing but whitespace follows: `Ready(())` once the input has ended
613 /// and all of it was read, `NeedInput` before that. As
614 /// [`Reader::finish`](crate::Reader::finish), for input that arrives in
615 /// pieces.
616 ///
617 /// The whole-value calls stop at the end of the value they read, and a
618 /// value nested in the document says nothing of the rest of it. Call
619 /// this after the last of them to know that the input held exactly one
620 /// valid document (or a sequence of valid ones).
621 ///
622 /// # Errors
623 ///
624 /// The first error in the rest of the input, as
625 /// [`next_event`](Self::next_event) reports it: a syntax error, an
626 /// exceeded budget, invalid UTF-8, or [`ErrorKind::TrailingCharacters`].
627 ///
628 /// # Examples
629 ///
630 /// ```
631 /// use jsonx_lib::{ErrorKind, Step, StreamReader};
632 ///
633 /// let mut r = StreamReader::new();
634 /// r.feed(br#"{"id": 1} "#)?;
635 /// assert!(r.read_value()?.is_ready());
636 /// assert_eq!(r.finish()?, Step::NeedInput); // more may follow
637 /// r.end_input();
638 /// assert_eq!(r.finish()?, Step::Ready(()));
639 ///
640 /// let mut r = StreamReader::new();
641 /// r.feed(br#"{"id": 1}}"#)?;
642 /// r.end_input();
643 /// assert!(r.read_value()?.is_ready());
644 /// let err = r.finish().unwrap_err();
645 /// assert_eq!((err.kind(), err.offset()), (ErrorKind::TrailingCharacters, 9));
646 /// # Ok::<(), jsonx_lib::Error>(())
647 /// ```
648 pub fn finish(&mut self) -> Result<Step<()>, Error> {
649 // A whole-value call left waiting is given up: its value is read
650 // as events, like the rest.
651 self.raw = None;
652 loop {
653 match self.next_event()? {
654 Step::Ready(Some(_)) => {}
655 Step::Ready(None) => return Ok(Step::Ready(())),
656 Step::NeedInput => return Ok(Step::NeedInput),
657 }
658 }
659 }
660
661 /// The kind of the next value, without consuming anything: `Ready(None)`
662 /// when no value comes next, or when the next byte cannot start one (the
663 /// next [`next_event`](Self::next_event) reports why).
664 ///
665 /// # Errors
666 ///
667 /// Any error the reader already stopped at.
668 ///
669 /// # Examples
670 ///
671 /// ```
672 /// use jsonx_lib::{Kind, Step, StreamReader};
673 ///
674 /// let mut r = StreamReader::new();
675 /// r.feed(b"[ ")?;
676 /// assert!(r.next_event()?.is_ready());
677 /// assert_eq!(r.peek_kind()?, Step::NeedInput);
678 /// r.feed(b"\"x\"]")?;
679 /// assert_eq!(r.peek_kind()?, Step::Ready(Some(Kind::String)));
680 /// # Ok::<(), jsonx_lib::Error>(())
681 /// ```
682 pub fn peek_kind(&mut self) -> Result<Step<Option<Kind>>, Error> {
683 if let Some(e) = self.failed {
684 return Err(e);
685 }
686 match self.machine.at_value(&self.buf) {
687 Ok(false) => Ok(Step::Ready(None)),
688 Ok(true) => match self.machine.peek_kind(&self.buf) {
689 Some(kind) => Ok(Step::Ready(Some(kind))),
690 // The value's first byte has not arrived.
691 None if self.machine.needed_from() == self.buf.len() && !self.machine.complete => {
692 self.halt(Error::new(ErrorKind::UnexpectedEnd, self.buf.len()))
693 }
694 None => Ok(Step::Ready(None)),
695 },
696 Err(e) => self.halt(e),
697 }
698 }
699
700 /// The number of arrays and objects open.
701 ///
702 /// # Examples
703 ///
704 /// ```
705 /// let mut r = jsonx_lib::StreamReader::new();
706 /// r.feed(b"[[")?;
707 /// let _ = r.next_event()?;
708 /// let _ = r.next_event()?;
709 /// assert_eq!(r.depth(), 2);
710 /// # Ok::<(), jsonx_lib::Error>(())
711 /// ```
712 #[inline]
713 #[must_use]
714 pub fn depth(&self) -> usize {
715 self.machine.depth()
716 }
717
718 /// The byte offset, from the first byte fed, just past the input
719 /// consumed so far.
720 ///
721 /// # Examples
722 ///
723 /// ```
724 /// let mut r = jsonx_lib::StreamReader::new();
725 /// r.feed(b"[10, 20]")?;
726 /// let _ = r.next_event()?;
727 /// let _ = r.next_event()?;
728 /// assert_eq!(r.offset(), 3);
729 /// # Ok::<(), jsonx_lib::Error>(())
730 /// ```
731 #[inline]
732 #[must_use]
733 pub fn offset(&self) -> usize {
734 self.base + self.machine.pos
735 }
736
737 /// The byte offset, from the first byte fed, where the most recent
738 /// event's token began (see [`Reader::event_offset`](crate::Reader::event_offset)).
739 ///
740 /// # Examples
741 ///
742 /// ```
743 /// let mut r = jsonx_lib::StreamReader::new();
744 /// r.feed(b"[ 7]")?;
745 /// let _ = r.next_event()?;
746 /// let _ = r.next_event()?;
747 /// assert_eq!(r.event_offset(), 2);
748 /// # Ok::<(), jsonx_lib::Error>(())
749 /// ```
750 #[inline]
751 #[must_use]
752 pub fn event_offset(&self) -> usize {
753 self.last
754 }
755
756 /// The bytes of input the reader holds: what has been fed and not yet
757 /// dropped. Consumed input is dropped as more is fed.
758 ///
759 /// # Examples
760 ///
761 /// ```
762 /// let mut r = jsonx_lib::StreamReader::new();
763 /// r.feed(b"[1, 2")?;
764 /// assert_eq!(r.buffered(), 5);
765 /// # Ok::<(), jsonx_lib::Error>(())
766 /// ```
767 #[inline]
768 #[must_use]
769 pub fn buffered(&self) -> usize {
770 self.buf.len() + self.tail_len
771 }
772
773 // ---- internals -----------------------------------------------------------
774
775 /// [`halt`] for `self`.
776 fn halt<T>(&mut self, e: Error) -> Result<Step<T>, Error> {
777 halt(
778 &mut self.failed,
779 &self.machine,
780 self.buf.len(),
781 self.base,
782 self.utf8_error,
783 e,
784 )
785 }
786
787 /// Skips the next value (continuing one already started) and returns
788 /// its absolute span, or `None` when no value comes next.
789 fn raw_span(&mut self) -> Result<Step<Option<(usize, usize)>>, Error> {
790 if let Some(e) = self.failed {
791 return Err(e);
792 }
793 let mut progress = match self.raw {
794 Some(progress) => progress,
795 None => match self.machine.at_value(&self.buf) {
796 Ok(false) => {
797 return match self.machine.check_trailing(&self.buf) {
798 Ok(()) => Ok(Step::Ready(None)),
799 Err(e) => self.halt(e),
800 };
801 }
802 Ok(true) => RawProgress {
803 depth: self.machine.depth(),
804 start: None,
805 },
806 Err(e) => return self.halt(e),
807 },
808 };
809 self.machine.skipping = true;
810 let outcome = loop {
811 match self.machine.step(&self.buf) {
812 Ok(Some(_)) => {
813 self.last = self.base + self.machine.last;
814 let start = *progress.start.get_or_insert(self.last);
815 if self.machine.depth() == progress.depth {
816 break Ok(Some((start, self.base + self.machine.pos)));
817 }
818 }
819 // `at_value` saw a value, so the document cannot end here.
820 Ok(None) => break Err(Error::new(ErrorKind::UnexpectedEnd, self.buf.len())),
821 Err(e) => break Err(e),
822 }
823 };
824 self.machine.skipping = false;
825 match outcome {
826 Ok(span) => {
827 self.raw = None;
828 Ok(Step::Ready(span))
829 }
830 Err(e) => {
831 self.raw = Some(progress);
832 self.halt(e)
833 }
834 }
835 }
836
837 /// Drops consumed input from the front of the buffer once there is
838 /// enough of it to be worth the copy (amortized constant per byte).
839 fn compact(&mut self) {
840 let mut keep_from = self.machine.needed_from();
841 if let Some(RawProgress {
842 start: Some(start), ..
843 }) = self.raw
844 {
845 keep_from = keep_from.min(start - self.base);
846 }
847 if keep_from < COMPACT_AT
848 || keep_from * 2 < self.buf.len()
849 || !self.buf.is_char_boundary(keep_from)
850 {
851 return;
852 }
853 let _dropped = self.buf.drain(..keep_from);
854 self.base += keep_from;
855 self.machine.rebase(keep_from);
856 }
857
858 /// Records that the input stops being UTF-8 at the end of the buffer.
859 /// No valid byte can follow, so the text so far is all there will be.
860 fn bad_byte(&mut self) {
861 self.utf8_error = Some(self.base + self.buf.len());
862 self.machine.complete = true;
863 }
864
865 /// Appends `bytes` to the buffer, keeping only valid UTF-8 and holding
866 /// back a character cut off at the end.
867 fn append(&mut self, bytes: &[u8]) {
868 let mut rest = bytes;
869 if self.tail_len > 0 {
870 // Complete the character the last chunk cut off.
871 let take = (4 - self.tail_len).min(rest.len());
872 let (head, after) = rest.split_at(take);
873 if let Some(slot) = self.tail.get_mut(self.tail_len..self.tail_len + take) {
874 slot.copy_from_slice(head);
875 }
876 let have = self.tail_len + take;
877 let pending = self.tail.get(..have).unwrap_or_default();
878 match core::str::from_utf8(pending) {
879 Ok(text) => {
880 self.buf.push_str(text);
881 self.tail_len = 0;
882 rest = after;
883 }
884 Err(e) => {
885 let used = e.valid_up_to();
886 if used > 0 {
887 // The cut character completed; the bytes after it
888 // in `pending` are new input to read normally.
889 let done = pending.get(..used).unwrap_or_default();
890 self.buf.push_str(core::str::from_utf8(done).unwrap_or(""));
891 let extra = used - self.tail_len;
892 self.tail_len = 0;
893 rest = bytes.get(extra..).unwrap_or_default();
894 } else if e.error_len().is_none() && after.is_empty() {
895 // Still incomplete, and this chunk is used up.
896 self.tail_len = have;
897 return;
898 } else {
899 self.bad_byte();
900 return;
901 }
902 }
903 }
904 }
905 match core::str::from_utf8(rest) {
906 Ok(text) => self.buf.push_str(text),
907 Err(e) => {
908 let (valid, invalid) = rest.split_at(e.valid_up_to());
909 self.buf.push_str(core::str::from_utf8(valid).unwrap_or(""));
910 if e.error_len().is_none() {
911 // A character cut off by the end of this chunk.
912 if let Some(slot) = self.tail.get_mut(..invalid.len()) {
913 slot.copy_from_slice(invalid);
914 }
915 self.tail_len = invalid.len();
916 } else {
917 self.bad_byte();
918 }
919 }
920 }
921 }
922}
923
924/// Turns a machine error into a call's answer: `NeedInput` when the buffer
925/// (of `len` bytes, starting at absolute offset `base`) merely ran out,
926/// otherwise the error, made absolute and kept in `failed`.
927fn halt<T>(
928 failed: &mut Option<Error>,
929 machine: &Machine,
930 len: usize,
931 base: usize,
932 utf8_error: Option<usize>,
933 e: Error,
934) -> Result<Step<T>, Error> {
935 let ran_out = e.kind() == ErrorKind::UnexpectedEnd && e.offset() == len;
936 let e = match (ran_out, utf8_error) {
937 (true, _) if !machine.complete => return Ok(Step::NeedInput),
938 // The valid text ends where the input stops being UTF-8.
939 (true, Some(at)) => Error::new(ErrorKind::InvalidUtf8, at),
940 // Wrapping: a repeated key's quote can lie in the dropped front of
941 // the buffer, where its text offset wrapped below zero (see
942 // `Machine::check_object`); adding `base` back gives its exact
943 // offset in the input.
944 _ => Error::new(e.kind(), e.offset().wrapping_add(base)),
945 };
946 *failed = Some(e);
947 Err(e)
948}
949
950impl Parser {
951 /// A [`StreamReader`] with this parser's budgets and duplicate-key
952 /// policy (which [`StreamReader::read_value`] applies).
953 ///
954 /// # Examples
955 ///
956 /// ```
957 /// use jsonx_lib::{ErrorKind, Limits, Parser};
958 ///
959 /// let mut r = Parser::new().limits(Limits::new().with_max_depth(1)).stream_reader();
960 /// r.feed(b"[[]]")?;
961 /// assert!(r.next_event().is_ok());
962 /// assert_eq!(r.next_event().unwrap_err().kind(), ErrorKind::DepthLimitExceeded);
963 /// # Ok::<(), jsonx_lib::Error>(())
964 /// ```
965 #[must_use]
966 pub fn stream_reader(&self) -> StreamReader {
967 let mut reader = StreamReader::with_config(self.get_limits(), self.get_duplicate_keys());
968 reader.machine.numbers = self.get_numbers();
969 reader
970 }
971}
972
973#[cfg(test)]
974mod tests {
975 use super::*;
976
977 #[test]
978 fn test_utf8_split_at_every_byte_of_a_character() {
979 let text = "[\"a\u{e9}\u{20ac}\u{1f600}\"]";
980 for cut in 0..=text.len() {
981 let (a, b) = text.as_bytes().split_at(cut);
982 let mut r = StreamReader::new();
983 assert!(r.feed(a).is_ok() && r.feed(b).is_ok());
984 r.end_input();
985 let _start = r.next_event();
986 let got = r.next_event().ok().and_then(Step::ready).flatten();
987 assert_eq!(
988 got,
989 Some(Event::String("a\u{e9}\u{20ac}\u{1f600}".into())),
990 "cut {cut}"
991 );
992 }
993 // One byte at a time.
994 let mut r = StreamReader::new();
995 for &byte in text.as_bytes() {
996 assert!(r.feed(&[byte]).is_ok());
997 }
998 assert_eq!(r.buffered(), text.len());
999 }
1000
1001 #[test]
1002 fn test_invalid_utf8_is_reported_where_the_reader_reaches_it() {
1003 let mut r = StreamReader::new();
1004 assert!(r.feed(b"[1, \"a\xC3").is_ok());
1005 assert!(r.feed(b"(\"]").is_ok()); // 0xC3 0x28 is not UTF-8
1006 let _start = r.next_event();
1007 let _one = r.next_event();
1008 let err = r.next_event().err();
1009 assert_eq!(
1010 err.map(|e| (e.kind(), e.offset())),
1011 Some((ErrorKind::InvalidUtf8, 6))
1012 );
1013 // A character cut off by the end of the input.
1014 let mut r = StreamReader::new();
1015 assert!(r.feed(b"\"\xE2\x82").is_ok());
1016 assert_eq!(r.next_event().ok(), Some(Step::NeedInput));
1017 r.end_input();
1018 let err = r.next_event().err();
1019 assert_eq!(
1020 err.map(|e| (e.kind(), e.offset())),
1021 Some((ErrorKind::InvalidUtf8, 1))
1022 );
1023 // A syntax error before the bad byte comes first.
1024 let mut r = StreamReader::new();
1025 assert!(r.feed(b"[1 2, \xFF]").is_ok());
1026 let _start = r.next_event();
1027 let _one = r.next_event();
1028 let err = r.next_event().err();
1029 assert_eq!(
1030 err.map(|e| (e.kind(), e.offset())),
1031 Some((ErrorKind::ExpectedCommaOrBracket, 3))
1032 );
1033 // An invalid continuation completing a cut character.
1034 let mut r = StreamReader::new();
1035 assert!(r.feed(b"\"\xE2").is_ok());
1036 assert!(r.feed(b"\x41\"").is_ok());
1037 let err = r.next_event().err();
1038 assert_eq!(
1039 err.map(|e| (e.kind(), e.offset())),
1040 Some((ErrorKind::InvalidUtf8, 1))
1041 );
1042 // A cut character completed, with more input after it in the chunk.
1043 let mut r = StreamReader::new();
1044 assert!(r.feed(b"\"\xE2\x82").is_ok());
1045 assert!(r.feed(b"\xACabc\"").is_ok());
1046 assert_eq!(
1047 r.next_event().ok().and_then(Step::ready).flatten(),
1048 Some(Event::String("\u{20ac}abc".into()))
1049 );
1050 }
1051
1052 #[test]
1053 fn test_feed_after_end_and_errors_latch() {
1054 let mut r = StreamReader::new();
1055 assert!(r.feed(b"[").is_ok());
1056 r.end_input();
1057 r.end_input();
1058 assert_eq!(
1059 r.feed(b"]").err().map(|e| e.kind()),
1060 Some(ErrorKind::InputAfterEnd)
1061 );
1062 let _start = r.next_event();
1063 let first = r.next_event().err();
1064 assert_eq!(first.map(|e| e.kind()), Some(ErrorKind::UnexpectedEnd));
1065 assert_eq!(r.next_event().err(), first);
1066 assert_eq!(r.read_raw().err(), first);
1067 assert_eq!(r.read_value().err(), first);
1068 assert_eq!(r.peek_kind().err(), first);
1069 assert_eq!(r.feed(b"x").err(), first);
1070 }
1071
1072 #[test]
1073 fn test_compaction_keeps_memory_to_a_chunk_and_offsets_absolute() {
1074 let mut r = StreamReader::new();
1075 let item = b"12345, ";
1076 let mut total = 1;
1077 assert!(r.feed(b"[").is_ok());
1078 let mut max_buffered = 0;
1079 for _ in 0..20_000 {
1080 assert!(r.feed(item).is_ok());
1081 total += item.len();
1082 while let Ok(Step::Ready(Some(_))) = r.next_event() {}
1083 max_buffered = max_buffered.max(r.buffered());
1084 }
1085 assert!(max_buffered < 3 * COMPACT_AT, "{max_buffered}");
1086 assert!(r.feed(b"]").is_ok());
1087 let err = r.next_event().err();
1088 assert_eq!(
1089 err.map(|e| (e.kind(), e.offset())),
1090 Some((ErrorKind::TrailingComma, total))
1091 );
1092 }
1093
1094 #[test]
1095 fn test_raw_value_spanning_many_chunks_survives_compaction() {
1096 let mut r = StreamReader::new();
1097 assert!(r.feed(b"[0, ").is_ok());
1098 let _start = r.next_event();
1099 assert_eq!(
1100 r.read_raw().ok().map(|s| s.map(|o| o.map(str::len))),
1101 Some(Step::Ready(Some(1)))
1102 );
1103 assert!(r.feed(b"{\"k\": [").is_ok());
1104 assert_eq!(r.read_raw().ok(), Some(Step::NeedInput));
1105 let mut expected = String::from("{\"k\": [");
1106 for i in 0..5_000 {
1107 let piece = alloc::format!("{i}, ");
1108 expected.push_str(&piece);
1109 assert!(r.feed(piece.as_bytes()).is_ok());
1110 assert_eq!(r.read_raw().ok(), Some(Step::NeedInput));
1111 }
1112 assert!(r.feed(b"0]}]").is_ok());
1113 expected.push_str("0]}");
1114 let raw = r
1115 .read_raw()
1116 .ok()
1117 .and_then(Step::ready)
1118 .flatten()
1119 .map(String::from);
1120 assert_eq!(raw.as_deref(), Some(expected.as_str()));
1121 assert_eq!(r.event_offset(), 4 + expected.len() - 1);
1122 }
1123}