Skip to main content

jsonx_lib/
stream.rs

1//! The incremental reader: a document fed in chunks as they arrive.
2
3use alloc::string::String;
4
5use crate::{
6    DecodeError, DuplicateKeys, FromJson, Kind, Limits, Parser, Value,
7    error::{Error, ErrorKind},
8    machine::Machine,
9    reader::Event,
10};
11
12/// The answer of a [`StreamReader`] call: ready, or waiting for more input.
13///
14/// `NeedInput` is not an error: the input so far is a valid beginning, and
15/// the call can be made again after [`feed`](StreamReader::feed) or
16/// [`end_input`](StreamReader::end_input).
17///
18/// # Examples
19///
20/// ```
21/// use jsonx_lib::{Event, Step, StreamReader};
22///
23/// let mut r = StreamReader::new();
24/// assert_eq!(r.next_event()?, Step::NeedInput);
25/// r.feed(b"[tr")?;
26/// assert_eq!(r.next_event()?, Step::Ready(Some(Event::StartArray)));
27/// assert_eq!(r.next_event()?, Step::NeedInput); // `tr` is not a value yet
28/// r.feed(b"ue]")?;
29/// assert_eq!(r.next_event()?.ready(), Some(Some(Event::Bool(true))));
30/// # Ok::<(), jsonx_lib::Error>(())
31/// ```
32#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash)]
33#[must_use]
34pub enum Step<T> {
35    /// The call completed with this result.
36    Ready(T),
37    /// The input so far ends inside what the call needs: feed more, or end
38    /// the input, and call again.
39    NeedInput,
40}
41
42impl<T> Step<T> {
43    /// The result, if ready.
44    ///
45    /// # Examples
46    ///
47    /// ```
48    /// use jsonx_lib::Step;
49    ///
50    /// assert_eq!(Step::Ready(3).ready(), Some(3));
51    /// assert_eq!(Step::<u8>::NeedInput.ready(), None);
52    /// ```
53    #[inline]
54    pub fn ready(self) -> Option<T> {
55        match self {
56            Self::Ready(value) => Some(value),
57            Self::NeedInput => None,
58        }
59    }
60
61    /// Whether the call completed.
62    ///
63    /// # Examples
64    ///
65    /// ```
66    /// assert!(jsonx_lib::Step::Ready(()).is_ready());
67    /// ```
68    #[inline]
69    #[must_use]
70    pub const fn is_ready(&self) -> bool {
71        matches!(self, Self::Ready(_))
72    }
73
74    /// Applies `f` to a ready result.
75    ///
76    /// # Examples
77    ///
78    /// ```
79    /// use jsonx_lib::Step;
80    ///
81    /// assert_eq!(Step::Ready(2).map(|n| n * 10), Step::Ready(20));
82    /// ```
83    #[inline]
84    pub fn map<U>(self, f: impl FnOnce(T) -> U) -> Step<U> {
85        match self {
86            Self::Ready(value) => Step::Ready(f(value)),
87            Self::NeedInput => Step::NeedInput,
88        }
89    }
90}
91
92/// A raw value being skipped across calls.
93#[derive(Clone, Copy, Debug)]
94struct RawProgress {
95    /// The depth the value started at; it is complete when the reader is back.
96    depth: usize,
97    /// The value's first byte (absolute), once its first event was read.
98    start: Option<usize>,
99}
100
101/// How much consumed text the buffer may keep before it is compacted.
102const COMPACT_AT: usize = 4 * 1024;
103
104/// A pull reader over input that arrives in pieces: bytes go in with
105/// [`feed`](Self::feed) as they arrive, events come out as soon as they are
106/// complete.
107///
108/// It is the same reader as [`Reader`](crate::Reader), over a buffer: the
109/// same grammar, the same budgets, and the same errors at the same offsets
110/// (counted from the first byte ever fed), whatever the input's chunking.
111/// When a call needs bytes that have not arrived it returns
112/// [`Step::NeedInput`] and changes nothing a later call would see
113/// differently; after [`end_input`](Self::end_input), running out is the
114/// [`ErrorKind::UnexpectedEnd`] that [`Reader`](crate::Reader) reports.
115///
116/// Events borrow from the reader's buffer, so each one must be dropped
117/// before the next call. In particular, feed more input after the borrow
118/// ends: in an `else` after `if let Step::Ready(..) = reader.next_event()?`
119/// (Rust 2024 drops the `if let` temporaries before the `else`), or after
120/// the loop that drains the ready events, as below; a `match` keeps the
121/// event borrowed in every arm. Consumed input is dropped from the buffer as more
122/// is fed, so a long document read with [`next_event`](Self::next_event) as
123/// it arrives needs memory for one token and one chunk, not for the
124/// document. The whole-value calls ([`read_raw`](Self::read_raw),
125/// [`read_value`](Self::read_value), [`decode`](Self::decode)) are
126/// different: each keeps its value's text in the buffer until the value
127/// ends, bounded by [`Limits::max_value_bytes`]. A token cut by a chunk
128/// boundary is continued where it stopped, never scanned again from its
129/// start: a string fed one byte at a time costs linear time.
130///
131/// The input must be UTF-8, checked as it arrives (a character may be split
132/// across chunks). The first invalid byte is reported as
133/// [`ErrorKind::InvalidUtf8`] when the reader reaches it: a syntax error
134/// earlier in the document is reported first. (A complete document given to
135/// [`parse_bytes`](crate::parse_bytes) checks the encoding before the
136/// syntax, so the two can name different errors for a document that has
137/// both.)
138///
139/// [`Limits::max_input_bytes`] bounds the total fed (in
140/// [`documents`](Self::documents) mode, what is fed for each document), and
141/// [`Limits::max_value_bytes`] what a whole-value call may hold; the other
142/// budgets apply exactly as for a complete document. After an error the
143/// reader is finished: every later call returns the same error.
144///
145/// # Examples
146///
147/// ```
148/// use jsonx_lib::{Event, Step, StreamReader};
149///
150/// let mut reader = StreamReader::new();
151/// let mut keys = Vec::new();
152/// for chunk in [&b"{\"jsonr"[..], b"pc\": \"2.0\", \"i", b"d\": 12", b"}"] {
153///     reader.feed(chunk)?;
154///     while let Step::Ready(Some(event)) = reader.next_event()? {
155///         if let Event::Key(key) = event {
156///             keys.push(key.into_owned());
157///         }
158///     }
159/// }
160/// reader.end_input();
161/// assert_eq!(reader.next_event()?, Step::Ready(None)); // the document is complete
162/// assert_eq!(keys, ["jsonrpc", "id"]);
163/// # Ok::<(), jsonx_lib::Error>(())
164/// ```
165#[derive(Clone, Debug)]
166pub struct StreamReader {
167    /// The input not yet dropped: valid UTF-8, starting at absolute offset
168    /// `base`.
169    buf: String,
170    base: usize,
171    /// An incomplete UTF-8 sequence at the end of the input so far.
172    tail: [u8; 4],
173    tail_len: usize,
174    /// The absolute offset of the first byte that is not UTF-8. Nothing
175    /// from it on is kept.
176    utf8_error: Option<usize>,
177    /// Bytes fed in total.
178    fed: usize,
179    ended: bool,
180    machine: Machine,
181    failed: Option<Error>,
182    /// The absolute offset where the most recent event's token began.
183    last: usize,
184    raw: Option<RawProgress>,
185}
186
187impl Default for StreamReader {
188    fn default() -> Self {
189        Self::new()
190    }
191}
192
193impl StreamReader {
194    /// A reader with the default [`Limits`], refusing duplicate keys in what
195    /// [`read_value`](Self::read_value) builds. For others, use
196    /// [`Parser::stream_reader`].
197    ///
198    /// # Examples
199    ///
200    /// ```
201    /// let r = jsonx_lib::StreamReader::new();
202    /// assert_eq!(r.offset(), 0);
203    /// ```
204    #[must_use]
205    pub fn new() -> Self {
206        Self::with_config(Limits::new(), DuplicateKeys::Error)
207    }
208
209    /// A reader with the given budgets and duplicate-key policy.
210    pub(crate) fn with_config(limits: Limits, duplicates: DuplicateKeys) -> Self {
211        Self {
212            buf: String::new(),
213            base: 0,
214            tail: [0; 4],
215            tail_len: 0,
216            utf8_error: None,
217            fed: 0,
218            ended: false,
219            machine: {
220                let mut machine = Machine::new(limits, duplicates, false);
221                // Keys outlive the front of the buffer, which is dropped.
222                machine.copy_keys = true;
223                machine
224            },
225            failed: None,
226            last: 0,
227            raw: None,
228        }
229    }
230
231    /// Reads a sequence of documents instead of one, as
232    /// [`Reader::documents`](crate::Reader::documents) does: NDJSON, JSON
233    /// Lines, or concatenated JSON, fed as it arrives. Whole-value reads
234    /// return one document each, and `Ready(None)` once the input has ended
235    /// after the last.
236    ///
237    /// [`Limits::max_input_bytes`] then applies to each document, not to the
238    /// stream: input is counted from where the current document began (or,
239    /// between documents, from the end of the last one), so a connection
240    /// that carries documents for as long as it stays open is never refused
241    /// for its total. A [`Reader`](crate::Reader), which has its whole input
242    /// already, applies the budget to all of it.
243    ///
244    /// # Examples
245    ///
246    /// ```
247    /// use jsonx_lib::{Step, StreamReader};
248    ///
249    /// let mut r = StreamReader::new().documents();
250    /// let mut seen = Vec::new();
251    /// for chunk in [&b"{\"id\": 1}\n{\"i"[..], b"d\": 2}\n"] {
252    ///     r.feed(chunk)?;
253    ///     while let Step::Ready(Some(doc)) = r.read_value()? {
254    ///         seen.push(doc["id"].as_i64());
255    ///     }
256    /// }
257    /// r.end_input();
258    /// assert_eq!(r.read_value()?, Step::Ready(None));
259    /// assert_eq!(seen, [Some(1), Some(2)]);
260    /// # Ok::<(), jsonx_lib::Error>(())
261    /// ```
262    #[must_use]
263    pub fn documents(mut self) -> Self {
264        self.machine.documents = true;
265        self
266    }
267
268    /// Adds the next piece of input. A chunk may end anywhere, even inside a
269    /// token or a UTF-8 character.
270    ///
271    /// # Errors
272    ///
273    /// [`ErrorKind::InputTooLarge`] when the input in total (in documents
274    /// mode, of the current document) grows past [`Limits::max_input_bytes`],
275    /// or when the value a pending whole-value call is reading grows past
276    /// [`Limits::max_value_bytes`], at the offset of the first byte over the
277    /// budget (the reader is then finished);
278    /// [`ErrorKind::InputAfterEnd`] after [`end_input`](Self::end_input);
279    /// any error the reader already stopped at.
280    ///
281    /// # Examples
282    ///
283    /// ```
284    /// use jsonx_lib::{ErrorKind, Limits, Parser};
285    ///
286    /// let mut r = Parser::new().limits(Limits::new().with_max_input_bytes(4)).stream_reader();
287    /// assert!(r.feed(b"[1,").is_ok());
288    /// assert_eq!(r.feed(b"22]").unwrap_err().kind(), ErrorKind::InputTooLarge);
289    /// ```
290    pub fn feed(&mut self, bytes: &[u8]) -> Result<(), Error> {
291        if let Some(e) = self.failed {
292            return Err(e);
293        }
294        if self.ended {
295            return Err(Error::new(ErrorKind::InputAfterEnd, self.fed));
296        }
297        // The input budget, from where the current document began (the
298        // start of the input outside documents mode).
299        let start = self.machine.document_start();
300        let max = self.machine.limits.max_input_bytes;
301        if bytes.len() > max.saturating_sub(self.fed.saturating_sub(start)) {
302            let e = Error::new(ErrorKind::InputTooLarge, start.saturating_add(max));
303            return Err(*self.failed.insert(e));
304        }
305        // The value a whole-value call is waiting on stays in the buffer.
306        if let Some(progress) = self.raw {
307            let start = progress
308                .start
309                .unwrap_or(self.base + self.machine.needed_from());
310            let max = self.machine.limits.max_value_bytes;
311            if bytes.len() > max.saturating_sub(self.fed.saturating_sub(start)) {
312                let e = Error::new(ErrorKind::InputTooLarge, start.saturating_add(max));
313                return Err(*self.failed.insert(e));
314            }
315        }
316        self.fed += bytes.len();
317        self.compact();
318        if self.utf8_error.is_none() {
319            self.append(bytes);
320        }
321        Ok(())
322    }
323
324    /// Says that the input has ended: nothing more will be fed. From now on,
325    /// running out of input is an error, as for a complete document, and a
326    /// number or whitespace at the very end is final.
327    ///
328    /// # Examples
329    ///
330    /// ```
331    /// use jsonx_lib::{Event, Step, StreamReader};
332    ///
333    /// let mut r = StreamReader::new();
334    /// r.feed(b"12")?;
335    /// assert_eq!(r.next_event()?, Step::NeedInput); // `123` may follow
336    /// r.end_input();
337    /// assert_eq!(r.next_event()?, Step::Ready(Some(Event::Number(12.into()))));
338    /// assert_eq!(r.next_event()?, Step::Ready(None));
339    /// # Ok::<(), jsonx_lib::Error>(())
340    /// ```
341    pub fn end_input(&mut self) {
342        if self.ended {
343            return;
344        }
345        self.ended = true;
346        if self.tail_len > 0 && self.utf8_error.is_none() {
347            // A character cut off by the end of the input.
348            self.utf8_error = Some(self.base + self.buf.len());
349        }
350        self.machine.complete = true;
351    }
352
353    /// Reads up to 8 KiB from `source` and feeds it, or calls
354    /// [`end_input`](Self::end_input) when `source` is at its end. Returns
355    /// the number of bytes read (`0` at the end). Interrupted reads are
356    /// retried.
357    ///
358    /// A budget error from [`feed`](Self::feed) is not an I/O error: it is
359    /// kept, and returned by the next reading call.
360    ///
361    /// # Errors
362    ///
363    /// Any error from `source` other than [`std::io::ErrorKind::Interrupted`].
364    ///
365    /// # Examples
366    ///
367    /// ```
368    /// use jsonx_lib::{Step, StreamReader};
369    ///
370    /// let mut source: &[u8] = br#"{"big": [1, 2, 3]}"#;
371    /// let mut r = StreamReader::new();
372    /// let mut events = 0;
373    /// loop {
374    ///     if let Step::Ready(event) = r.next_event()? {
375    ///         match event {
376    ///             Some(_) => events += 1,
377    ///             None => break,
378    ///         }
379    ///     } else {
380    ///         r.read_from(&mut source)?;
381    ///     }
382    /// }
383    /// assert_eq!(events, 8); // {, "big", [, 1, 2, 3, ], }
384    /// # Ok::<(), Box<dyn std::error::Error>>(())
385    /// ```
386    #[cfg(feature = "std")]
387    #[cfg_attr(docsrs, doc(cfg(feature = "std")))]
388    pub fn read_from(&mut self, source: &mut dyn std::io::Read) -> std::io::Result<usize> {
389        let mut chunk = [0u8; 8 * 1024];
390        loop {
391            match source.read(&mut chunk) {
392                Ok(0) => {
393                    self.end_input();
394                    return Ok(0);
395                }
396                Ok(n) => {
397                    // A refused feed is kept in `failed` and reported by the
398                    // next reading call.
399                    let _kept = self.feed(chunk.get(..n).unwrap_or_default());
400                    return Ok(n);
401                }
402                Err(e) if e.kind() == std::io::ErrorKind::Interrupted => {}
403                Err(e) => return Err(e),
404            }
405        }
406    }
407
408    /// Reads the next event: `Ready(Some(event))`, `Ready(None)` once the
409    /// document is complete and the input has ended with nothing but
410    /// whitespace after it, or `NeedInput`.
411    ///
412    /// # Errors
413    ///
414    /// The first syntax error, exceeded budget, or invalid UTF-8, at its
415    /// byte offset. Once an error is returned, every later call returns it.
416    ///
417    /// # Examples
418    ///
419    /// ```
420    /// use jsonx_lib::{ErrorKind, Event, Step, StreamReader};
421    ///
422    /// let mut r = StreamReader::new();
423    /// r.feed(b"[1,")?;
424    /// assert_eq!(r.next_event()?, Step::Ready(Some(Event::StartArray)));
425    /// assert!(r.next_event()?.is_ready()); // 1
426    /// r.feed(b"]")?;
427    /// let err = r.next_event().unwrap_err();
428    /// assert_eq!((err.kind(), err.offset()), (ErrorKind::TrailingComma, 3));
429    /// # Ok::<(), jsonx_lib::Error>(())
430    /// ```
431    pub fn next_event(&mut self) -> Result<Step<Option<Event<'_>>>, Error> {
432        if let Some(e) = self.failed {
433            return Err(e);
434        }
435        let Self {
436            buf,
437            machine,
438            base,
439            last,
440            failed,
441            utf8_error,
442            ..
443        } = self;
444        match machine.step(buf) {
445            Ok(Some(event)) => {
446                *last = *base + machine.last;
447                Ok(Step::Ready(Some(event)))
448            }
449            Ok(None) => match utf8_error {
450                // The valid text holds a whole document, then bad bytes.
451                Some(at) => Err(*failed.insert(Error::new(ErrorKind::InvalidUtf8, *at))),
452                None => Ok(Step::Ready(None)),
453            },
454            Err(e) => halt(failed, machine, buf.len(), *base, *utf8_error, e),
455        }
456    }
457
458    /// Reads the next value whole and returns its exact source text, without
459    /// decoding or allocating anything, as [`Reader::read_raw`](crate::Reader::read_raw)
460    /// does, with the same positioning rules: `Ready(None)` when no value
461    /// comes next.
462    ///
463    /// The value must be in the buffer whole before it can be returned, so
464    /// a large value costs memory for all of it, whitespace included, up to
465    /// [`Limits::max_value_bytes`] (past it, [`feed`](Self::feed) refuses
466    /// more). `NeedInput` keeps the progress: the next call continues the
467    /// same value.
468    ///
469    /// After a single document's value, a call answers as
470    /// [`next_event`](Self::next_event) does there: `NeedInput` while only
471    /// whitespace has followed and the input has not ended, `Ready(None)`
472    /// once it has, and [`ErrorKind::TrailingCharacters`] for anything but
473    /// whitespace. [`finish`](Self::finish) makes the same check in one call.
474    ///
475    /// # Errors
476    ///
477    /// As [`next_event`](Self::next_event), for any error inside the value,
478    /// and [`ErrorKind::TrailingCharacters`] after the document.
479    ///
480    /// # Examples
481    ///
482    /// ```
483    /// use jsonx_lib::{Event, Step, StreamReader};
484    ///
485    /// let mut r = StreamReader::new();
486    /// r.feed(br#"{"id": 1, "params": {"line""#)?;
487    /// assert!(r.next_event()?.is_ready()); // {
488    /// assert!(r.next_event()?.is_ready()); // "id"
489    /// assert_eq!(r.read_raw()?, Step::Ready(Some("1")));
490    /// assert!(r.next_event()?.is_ready()); // "params"
491    /// assert_eq!(r.read_raw()?, Step::NeedInput);
492    /// r.feed(br#": 3}}"#)?;
493    /// assert_eq!(r.read_raw()?, Step::Ready(Some(r#"{"line": 3}"#)));
494    /// # Ok::<(), jsonx_lib::Error>(())
495    /// ```
496    pub fn read_raw(&mut self) -> Result<Step<Option<&str>>, Error> {
497        Ok(self.raw_span()?.map(|span| {
498            span.and_then(|(start, end)| self.buf.get(start - self.base..end - self.base))
499        }))
500    }
501
502    /// Reads the next value whole and builds it as a [`Value`], applying the
503    /// reader's [`DuplicateKeys`] policy; positioning as for
504    /// [`read_raw`](Self::read_raw). The value borrows from the buffer:
505    /// call [`Value::into_owned`] to keep it.
506    ///
507    /// # Errors
508    ///
509    /// As [`read_raw`](Self::read_raw), plus [`ErrorKind::DuplicateKey`]
510    /// under [`DuplicateKeys::Error`].
511    ///
512    /// # Examples
513    ///
514    /// ```
515    /// use jsonx_lib::{Step, StreamReader};
516    ///
517    /// let mut r = StreamReader::new();
518    /// r.feed(br#"{"a": [1, "#)?;
519    /// assert_eq!(r.read_value()?, Step::NeedInput);
520    /// r.feed(br#"2]}"#)?;
521    /// let value = r.read_value()?.ready().flatten().unwrap_or_default();
522    /// assert_eq!(value["a"][1], 2);
523    /// # Ok::<(), jsonx_lib::Error>(())
524    /// ```
525    pub fn read_value(&mut self) -> Result<Step<Option<Value<'_>>>, Error> {
526        if let Some(e) = self.failed {
527            return Err(e);
528        }
529        let Step::Ready(span) = self.raw_span()? else {
530            return Ok(Step::NeedInput);
531        };
532        let Some((start, end)) = span else {
533            return Ok(Step::Ready(None));
534        };
535        let text = self
536            .buf
537            .get(start - self.base..end - self.base)
538            .unwrap_or("");
539        // The budgets, and duplicates under `DuplicateKeys::Error`, were
540        // checked while the value was skipped; the memory budget, and
541        // `First` and `Last`, remain to apply to the tree.
542        let limits = Limits::unlimited().with_max_memory(self.machine.limits.max_memory);
543        let parser = Parser::new()
544            .limits(limits)
545            .duplicate_keys(self.machine.duplicates);
546        match parser.parse(text) {
547            Ok(value) => Ok(Step::Ready(Some(value))),
548            Err(e) => {
549                let e = Error::new(e.kind(), e.offset() + start);
550                self.failed = Some(e);
551                Err(e)
552            }
553        }
554    }
555
556    /// Reads the next value whole as a `T` ([`FromJson`]); positioning as
557    /// for [`read_raw`](Self::read_raw). The value borrows from the buffer,
558    /// as `T` allows.
559    ///
560    /// The value is first skipped (validated, with every budget) as it
561    /// arrives, then read as `T` once it is all there. Byte offsets in an
562    /// error count from the first byte fed.
563    ///
564    /// # Errors
565    ///
566    /// The [`DecodeError`] of `T`, or the reader's error.
567    ///
568    /// # Examples
569    ///
570    /// ```
571    /// use jsonx_lib::{Step, StreamReader};
572    ///
573    /// let mut r = StreamReader::new().documents();
574    /// r.feed(b"[1, 2]\n[3")?;
575    /// assert_eq!(r.decode::<Vec<u8>>()?, Step::Ready(Some(vec![1, 2])));
576    /// assert_eq!(r.decode::<Vec<u8>>()?, Step::NeedInput);
577    /// r.feed(b"]\n")?;
578    /// r.end_input();
579    /// assert_eq!(r.decode::<Vec<u8>>()?, Step::Ready(Some(vec![3])));
580    /// assert_eq!(r.decode::<Vec<u8>>()?, Step::Ready(None));
581    /// # Ok::<(), jsonx_lib::DecodeError>(())
582    /// ```
583    pub fn decode<'s, T: FromJson<'s>>(&'s mut self) -> Result<Step<Option<T>>, DecodeError> {
584        let Step::Ready(span) = self.raw_span()? else {
585            return Ok(Step::NeedInput);
586        };
587        let Some((start, end)) = span else {
588            return Ok(Step::Ready(None));
589        };
590        let text = self
591            .buf
592            .get(start - self.base..end - self.base)
593            .unwrap_or("");
594        // The budgets, and duplicates under `DuplicateKeys::Error`, were
595        // checked while the value was skipped. The depth budget stays, to
596        // bound a recursive `T`, and the memory budget, for what `T` holds.
597        let limits = Limits::unlimited()
598            .with_max_depth(self.machine.limits.max_depth)
599            .with_max_memory(self.machine.limits.max_memory);
600        let parser = Parser::new()
601            .limits(limits)
602            .duplicate_keys(self.machine.duplicates)
603            .numbers(self.machine.numbers);
604        parser
605            .decode(text)
606            .map(|value| Step::Ready(Some(value)))
607            .map_err(|e| e.shifted(start))
608    }
609
610    /// Reads to the end of the input, checking that the rest of the
611    /// document (in documents mode, of every document) is valid and that
612    /// nothing but whitespace follows: `Ready(())` once the input has ended
613    /// and all of it was read, `NeedInput` before that. As
614    /// [`Reader::finish`](crate::Reader::finish), for input that arrives in
615    /// pieces.
616    ///
617    /// The whole-value calls stop at the end of the value they read, and a
618    /// value nested in the document says nothing of the rest of it. Call
619    /// this after the last of them to know that the input held exactly one
620    /// valid document (or a sequence of valid ones).
621    ///
622    /// # Errors
623    ///
624    /// The first error in the rest of the input, as
625    /// [`next_event`](Self::next_event) reports it: a syntax error, an
626    /// exceeded budget, invalid UTF-8, or [`ErrorKind::TrailingCharacters`].
627    ///
628    /// # Examples
629    ///
630    /// ```
631    /// use jsonx_lib::{ErrorKind, Step, StreamReader};
632    ///
633    /// let mut r = StreamReader::new();
634    /// r.feed(br#"{"id": 1} "#)?;
635    /// assert!(r.read_value()?.is_ready());
636    /// assert_eq!(r.finish()?, Step::NeedInput); // more may follow
637    /// r.end_input();
638    /// assert_eq!(r.finish()?, Step::Ready(()));
639    ///
640    /// let mut r = StreamReader::new();
641    /// r.feed(br#"{"id": 1}}"#)?;
642    /// r.end_input();
643    /// assert!(r.read_value()?.is_ready());
644    /// let err = r.finish().unwrap_err();
645    /// assert_eq!((err.kind(), err.offset()), (ErrorKind::TrailingCharacters, 9));
646    /// # Ok::<(), jsonx_lib::Error>(())
647    /// ```
648    pub fn finish(&mut self) -> Result<Step<()>, Error> {
649        // A whole-value call left waiting is given up: its value is read
650        // as events, like the rest.
651        self.raw = None;
652        loop {
653            match self.next_event()? {
654                Step::Ready(Some(_)) => {}
655                Step::Ready(None) => return Ok(Step::Ready(())),
656                Step::NeedInput => return Ok(Step::NeedInput),
657            }
658        }
659    }
660
661    /// The kind of the next value, without consuming anything: `Ready(None)`
662    /// when no value comes next, or when the next byte cannot start one (the
663    /// next [`next_event`](Self::next_event) reports why).
664    ///
665    /// # Errors
666    ///
667    /// Any error the reader already stopped at.
668    ///
669    /// # Examples
670    ///
671    /// ```
672    /// use jsonx_lib::{Kind, Step, StreamReader};
673    ///
674    /// let mut r = StreamReader::new();
675    /// r.feed(b"[ ")?;
676    /// assert!(r.next_event()?.is_ready());
677    /// assert_eq!(r.peek_kind()?, Step::NeedInput);
678    /// r.feed(b"\"x\"]")?;
679    /// assert_eq!(r.peek_kind()?, Step::Ready(Some(Kind::String)));
680    /// # Ok::<(), jsonx_lib::Error>(())
681    /// ```
682    pub fn peek_kind(&mut self) -> Result<Step<Option<Kind>>, Error> {
683        if let Some(e) = self.failed {
684            return Err(e);
685        }
686        match self.machine.at_value(&self.buf) {
687            Ok(false) => Ok(Step::Ready(None)),
688            Ok(true) => match self.machine.peek_kind(&self.buf) {
689                Some(kind) => Ok(Step::Ready(Some(kind))),
690                // The value's first byte has not arrived.
691                None if self.machine.needed_from() == self.buf.len() && !self.machine.complete => {
692                    self.halt(Error::new(ErrorKind::UnexpectedEnd, self.buf.len()))
693                }
694                None => Ok(Step::Ready(None)),
695            },
696            Err(e) => self.halt(e),
697        }
698    }
699
700    /// The number of arrays and objects open.
701    ///
702    /// # Examples
703    ///
704    /// ```
705    /// let mut r = jsonx_lib::StreamReader::new();
706    /// r.feed(b"[[")?;
707    /// let _ = r.next_event()?;
708    /// let _ = r.next_event()?;
709    /// assert_eq!(r.depth(), 2);
710    /// # Ok::<(), jsonx_lib::Error>(())
711    /// ```
712    #[inline]
713    #[must_use]
714    pub fn depth(&self) -> usize {
715        self.machine.depth()
716    }
717
718    /// The byte offset, from the first byte fed, just past the input
719    /// consumed so far.
720    ///
721    /// # Examples
722    ///
723    /// ```
724    /// let mut r = jsonx_lib::StreamReader::new();
725    /// r.feed(b"[10, 20]")?;
726    /// let _ = r.next_event()?;
727    /// let _ = r.next_event()?;
728    /// assert_eq!(r.offset(), 3);
729    /// # Ok::<(), jsonx_lib::Error>(())
730    /// ```
731    #[inline]
732    #[must_use]
733    pub fn offset(&self) -> usize {
734        self.base + self.machine.pos
735    }
736
737    /// The byte offset, from the first byte fed, where the most recent
738    /// event's token began (see [`Reader::event_offset`](crate::Reader::event_offset)).
739    ///
740    /// # Examples
741    ///
742    /// ```
743    /// let mut r = jsonx_lib::StreamReader::new();
744    /// r.feed(b"[ 7]")?;
745    /// let _ = r.next_event()?;
746    /// let _ = r.next_event()?;
747    /// assert_eq!(r.event_offset(), 2);
748    /// # Ok::<(), jsonx_lib::Error>(())
749    /// ```
750    #[inline]
751    #[must_use]
752    pub fn event_offset(&self) -> usize {
753        self.last
754    }
755
756    /// The bytes of input the reader holds: what has been fed and not yet
757    /// dropped. Consumed input is dropped as more is fed.
758    ///
759    /// # Examples
760    ///
761    /// ```
762    /// let mut r = jsonx_lib::StreamReader::new();
763    /// r.feed(b"[1, 2")?;
764    /// assert_eq!(r.buffered(), 5);
765    /// # Ok::<(), jsonx_lib::Error>(())
766    /// ```
767    #[inline]
768    #[must_use]
769    pub fn buffered(&self) -> usize {
770        self.buf.len() + self.tail_len
771    }
772
773    // ---- internals -----------------------------------------------------------
774
775    /// [`halt`] for `self`.
776    fn halt<T>(&mut self, e: Error) -> Result<Step<T>, Error> {
777        halt(
778            &mut self.failed,
779            &self.machine,
780            self.buf.len(),
781            self.base,
782            self.utf8_error,
783            e,
784        )
785    }
786
787    /// Skips the next value (continuing one already started) and returns
788    /// its absolute span, or `None` when no value comes next.
789    fn raw_span(&mut self) -> Result<Step<Option<(usize, usize)>>, Error> {
790        if let Some(e) = self.failed {
791            return Err(e);
792        }
793        let mut progress = match self.raw {
794            Some(progress) => progress,
795            None => match self.machine.at_value(&self.buf) {
796                Ok(false) => {
797                    return match self.machine.check_trailing(&self.buf) {
798                        Ok(()) => Ok(Step::Ready(None)),
799                        Err(e) => self.halt(e),
800                    };
801                }
802                Ok(true) => RawProgress {
803                    depth: self.machine.depth(),
804                    start: None,
805                },
806                Err(e) => return self.halt(e),
807            },
808        };
809        self.machine.skipping = true;
810        let outcome = loop {
811            match self.machine.step(&self.buf) {
812                Ok(Some(_)) => {
813                    self.last = self.base + self.machine.last;
814                    let start = *progress.start.get_or_insert(self.last);
815                    if self.machine.depth() == progress.depth {
816                        break Ok(Some((start, self.base + self.machine.pos)));
817                    }
818                }
819                // `at_value` saw a value, so the document cannot end here.
820                Ok(None) => break Err(Error::new(ErrorKind::UnexpectedEnd, self.buf.len())),
821                Err(e) => break Err(e),
822            }
823        };
824        self.machine.skipping = false;
825        match outcome {
826            Ok(span) => {
827                self.raw = None;
828                Ok(Step::Ready(span))
829            }
830            Err(e) => {
831                self.raw = Some(progress);
832                self.halt(e)
833            }
834        }
835    }
836
837    /// Drops consumed input from the front of the buffer once there is
838    /// enough of it to be worth the copy (amortized constant per byte).
839    fn compact(&mut self) {
840        let mut keep_from = self.machine.needed_from();
841        if let Some(RawProgress {
842            start: Some(start), ..
843        }) = self.raw
844        {
845            keep_from = keep_from.min(start - self.base);
846        }
847        if keep_from < COMPACT_AT
848            || keep_from * 2 < self.buf.len()
849            || !self.buf.is_char_boundary(keep_from)
850        {
851            return;
852        }
853        let _dropped = self.buf.drain(..keep_from);
854        self.base += keep_from;
855        self.machine.rebase(keep_from);
856    }
857
858    /// Records that the input stops being UTF-8 at the end of the buffer.
859    /// No valid byte can follow, so the text so far is all there will be.
860    fn bad_byte(&mut self) {
861        self.utf8_error = Some(self.base + self.buf.len());
862        self.machine.complete = true;
863    }
864
865    /// Appends `bytes` to the buffer, keeping only valid UTF-8 and holding
866    /// back a character cut off at the end.
867    fn append(&mut self, bytes: &[u8]) {
868        let mut rest = bytes;
869        if self.tail_len > 0 {
870            // Complete the character the last chunk cut off.
871            let take = (4 - self.tail_len).min(rest.len());
872            let (head, after) = rest.split_at(take);
873            if let Some(slot) = self.tail.get_mut(self.tail_len..self.tail_len + take) {
874                slot.copy_from_slice(head);
875            }
876            let have = self.tail_len + take;
877            let pending = self.tail.get(..have).unwrap_or_default();
878            match core::str::from_utf8(pending) {
879                Ok(text) => {
880                    self.buf.push_str(text);
881                    self.tail_len = 0;
882                    rest = after;
883                }
884                Err(e) => {
885                    let used = e.valid_up_to();
886                    if used > 0 {
887                        // The cut character completed; the bytes after it
888                        // in `pending` are new input to read normally.
889                        let done = pending.get(..used).unwrap_or_default();
890                        self.buf.push_str(core::str::from_utf8(done).unwrap_or(""));
891                        let extra = used - self.tail_len;
892                        self.tail_len = 0;
893                        rest = bytes.get(extra..).unwrap_or_default();
894                    } else if e.error_len().is_none() && after.is_empty() {
895                        // Still incomplete, and this chunk is used up.
896                        self.tail_len = have;
897                        return;
898                    } else {
899                        self.bad_byte();
900                        return;
901                    }
902                }
903            }
904        }
905        match core::str::from_utf8(rest) {
906            Ok(text) => self.buf.push_str(text),
907            Err(e) => {
908                let (valid, invalid) = rest.split_at(e.valid_up_to());
909                self.buf.push_str(core::str::from_utf8(valid).unwrap_or(""));
910                if e.error_len().is_none() {
911                    // A character cut off by the end of this chunk.
912                    if let Some(slot) = self.tail.get_mut(..invalid.len()) {
913                        slot.copy_from_slice(invalid);
914                    }
915                    self.tail_len = invalid.len();
916                } else {
917                    self.bad_byte();
918                }
919            }
920        }
921    }
922}
923
924/// Turns a machine error into a call's answer: `NeedInput` when the buffer
925/// (of `len` bytes, starting at absolute offset `base`) merely ran out,
926/// otherwise the error, made absolute and kept in `failed`.
927fn halt<T>(
928    failed: &mut Option<Error>,
929    machine: &Machine,
930    len: usize,
931    base: usize,
932    utf8_error: Option<usize>,
933    e: Error,
934) -> Result<Step<T>, Error> {
935    let ran_out = e.kind() == ErrorKind::UnexpectedEnd && e.offset() == len;
936    let e = match (ran_out, utf8_error) {
937        (true, _) if !machine.complete => return Ok(Step::NeedInput),
938        // The valid text ends where the input stops being UTF-8.
939        (true, Some(at)) => Error::new(ErrorKind::InvalidUtf8, at),
940        // Wrapping: a repeated key's quote can lie in the dropped front of
941        // the buffer, where its text offset wrapped below zero (see
942        // `Machine::check_object`); adding `base` back gives its exact
943        // offset in the input.
944        _ => Error::new(e.kind(), e.offset().wrapping_add(base)),
945    };
946    *failed = Some(e);
947    Err(e)
948}
949
950impl Parser {
951    /// A [`StreamReader`] with this parser's budgets and duplicate-key
952    /// policy (which [`StreamReader::read_value`] applies).
953    ///
954    /// # Examples
955    ///
956    /// ```
957    /// use jsonx_lib::{ErrorKind, Limits, Parser};
958    ///
959    /// let mut r = Parser::new().limits(Limits::new().with_max_depth(1)).stream_reader();
960    /// r.feed(b"[[]]")?;
961    /// assert!(r.next_event().is_ok());
962    /// assert_eq!(r.next_event().unwrap_err().kind(), ErrorKind::DepthLimitExceeded);
963    /// # Ok::<(), jsonx_lib::Error>(())
964    /// ```
965    #[must_use]
966    pub fn stream_reader(&self) -> StreamReader {
967        let mut reader = StreamReader::with_config(self.get_limits(), self.get_duplicate_keys());
968        reader.machine.numbers = self.get_numbers();
969        reader
970    }
971}
972
973#[cfg(test)]
974mod tests {
975    use super::*;
976
977    #[test]
978    fn test_utf8_split_at_every_byte_of_a_character() {
979        let text = "[\"a\u{e9}\u{20ac}\u{1f600}\"]";
980        for cut in 0..=text.len() {
981            let (a, b) = text.as_bytes().split_at(cut);
982            let mut r = StreamReader::new();
983            assert!(r.feed(a).is_ok() && r.feed(b).is_ok());
984            r.end_input();
985            let _start = r.next_event();
986            let got = r.next_event().ok().and_then(Step::ready).flatten();
987            assert_eq!(
988                got,
989                Some(Event::String("a\u{e9}\u{20ac}\u{1f600}".into())),
990                "cut {cut}"
991            );
992        }
993        // One byte at a time.
994        let mut r = StreamReader::new();
995        for &byte in text.as_bytes() {
996            assert!(r.feed(&[byte]).is_ok());
997        }
998        assert_eq!(r.buffered(), text.len());
999    }
1000
1001    #[test]
1002    fn test_invalid_utf8_is_reported_where_the_reader_reaches_it() {
1003        let mut r = StreamReader::new();
1004        assert!(r.feed(b"[1, \"a\xC3").is_ok());
1005        assert!(r.feed(b"(\"]").is_ok()); // 0xC3 0x28 is not UTF-8
1006        let _start = r.next_event();
1007        let _one = r.next_event();
1008        let err = r.next_event().err();
1009        assert_eq!(
1010            err.map(|e| (e.kind(), e.offset())),
1011            Some((ErrorKind::InvalidUtf8, 6))
1012        );
1013        // A character cut off by the end of the input.
1014        let mut r = StreamReader::new();
1015        assert!(r.feed(b"\"\xE2\x82").is_ok());
1016        assert_eq!(r.next_event().ok(), Some(Step::NeedInput));
1017        r.end_input();
1018        let err = r.next_event().err();
1019        assert_eq!(
1020            err.map(|e| (e.kind(), e.offset())),
1021            Some((ErrorKind::InvalidUtf8, 1))
1022        );
1023        // A syntax error before the bad byte comes first.
1024        let mut r = StreamReader::new();
1025        assert!(r.feed(b"[1 2, \xFF]").is_ok());
1026        let _start = r.next_event();
1027        let _one = r.next_event();
1028        let err = r.next_event().err();
1029        assert_eq!(
1030            err.map(|e| (e.kind(), e.offset())),
1031            Some((ErrorKind::ExpectedCommaOrBracket, 3))
1032        );
1033        // An invalid continuation completing a cut character.
1034        let mut r = StreamReader::new();
1035        assert!(r.feed(b"\"\xE2").is_ok());
1036        assert!(r.feed(b"\x41\"").is_ok());
1037        let err = r.next_event().err();
1038        assert_eq!(
1039            err.map(|e| (e.kind(), e.offset())),
1040            Some((ErrorKind::InvalidUtf8, 1))
1041        );
1042        // A cut character completed, with more input after it in the chunk.
1043        let mut r = StreamReader::new();
1044        assert!(r.feed(b"\"\xE2\x82").is_ok());
1045        assert!(r.feed(b"\xACabc\"").is_ok());
1046        assert_eq!(
1047            r.next_event().ok().and_then(Step::ready).flatten(),
1048            Some(Event::String("\u{20ac}abc".into()))
1049        );
1050    }
1051
1052    #[test]
1053    fn test_feed_after_end_and_errors_latch() {
1054        let mut r = StreamReader::new();
1055        assert!(r.feed(b"[").is_ok());
1056        r.end_input();
1057        r.end_input();
1058        assert_eq!(
1059            r.feed(b"]").err().map(|e| e.kind()),
1060            Some(ErrorKind::InputAfterEnd)
1061        );
1062        let _start = r.next_event();
1063        let first = r.next_event().err();
1064        assert_eq!(first.map(|e| e.kind()), Some(ErrorKind::UnexpectedEnd));
1065        assert_eq!(r.next_event().err(), first);
1066        assert_eq!(r.read_raw().err(), first);
1067        assert_eq!(r.read_value().err(), first);
1068        assert_eq!(r.peek_kind().err(), first);
1069        assert_eq!(r.feed(b"x").err(), first);
1070    }
1071
1072    #[test]
1073    fn test_compaction_keeps_memory_to_a_chunk_and_offsets_absolute() {
1074        let mut r = StreamReader::new();
1075        let item = b"12345, ";
1076        let mut total = 1;
1077        assert!(r.feed(b"[").is_ok());
1078        let mut max_buffered = 0;
1079        for _ in 0..20_000 {
1080            assert!(r.feed(item).is_ok());
1081            total += item.len();
1082            while let Ok(Step::Ready(Some(_))) = r.next_event() {}
1083            max_buffered = max_buffered.max(r.buffered());
1084        }
1085        assert!(max_buffered < 3 * COMPACT_AT, "{max_buffered}");
1086        assert!(r.feed(b"]").is_ok());
1087        let err = r.next_event().err();
1088        assert_eq!(
1089            err.map(|e| (e.kind(), e.offset())),
1090            Some((ErrorKind::TrailingComma, total))
1091        );
1092    }
1093
1094    #[test]
1095    fn test_raw_value_spanning_many_chunks_survives_compaction() {
1096        let mut r = StreamReader::new();
1097        assert!(r.feed(b"[0, ").is_ok());
1098        let _start = r.next_event();
1099        assert_eq!(
1100            r.read_raw().ok().map(|s| s.map(|o| o.map(str::len))),
1101            Some(Step::Ready(Some(1)))
1102        );
1103        assert!(r.feed(b"{\"k\": [").is_ok());
1104        assert_eq!(r.read_raw().ok(), Some(Step::NeedInput));
1105        let mut expected = String::from("{\"k\": [");
1106        for i in 0..5_000 {
1107            let piece = alloc::format!("{i}, ");
1108            expected.push_str(&piece);
1109            assert!(r.feed(piece.as_bytes()).is_ok());
1110            assert_eq!(r.read_raw().ok(), Some(Step::NeedInput));
1111        }
1112        assert!(r.feed(b"0]}]").is_ok());
1113        expected.push_str("0]}");
1114        let raw = r
1115            .read_raw()
1116            .ok()
1117            .and_then(Step::ready)
1118            .flatten()
1119            .map(String::from);
1120        assert_eq!(raw.as_deref(), Some(expected.as_str()));
1121        assert_eq!(r.event_offset(), 4 + expected.len() - 1);
1122    }
1123}