Skip to main content

jsonx_lib/
stream.rs

1//! The incremental reader: a document fed in chunks as they arrive.
2
3use alloc::string::String;
4
5use crate::{
6    DecodeError, DuplicateKeys, FromJson, Kind, Limits, Parser, Value,
7    error::{Error, ErrorKind},
8    machine::Machine,
9    reader::Event,
10};
11
12/// The answer of a [`StreamReader`] call: ready, or waiting for more input.
13///
14/// `NeedInput` is not an error: the input so far is a valid beginning, and
15/// the call can be made again after [`feed`](StreamReader::feed) or
16/// [`end_input`](StreamReader::end_input).
17///
18/// # Examples
19///
20/// ```
21/// use jsonx_lib::{Event, Step, StreamReader};
22///
23/// let mut r = StreamReader::new();
24/// assert_eq!(r.next_event()?, Step::NeedInput);
25/// r.feed(b"[tr")?;
26/// assert_eq!(r.next_event()?, Step::Ready(Some(Event::StartArray)));
27/// assert_eq!(r.next_event()?, Step::NeedInput); // `tr` is not a value yet
28/// r.feed(b"ue]")?;
29/// assert_eq!(r.next_event()?.ready(), Some(Some(Event::Bool(true))));
30/// # Ok::<(), jsonx_lib::Error>(())
31/// ```
32#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash)]
33#[must_use]
34pub enum Step<T> {
35    /// The call completed with this result.
36    Ready(T),
37    /// The input so far ends inside what the call needs: feed more, or end
38    /// the input, and call again.
39    NeedInput,
40}
41
42impl<T> Step<T> {
43    /// The result, if ready.
44    ///
45    /// # Examples
46    ///
47    /// ```
48    /// use jsonx_lib::Step;
49    ///
50    /// assert_eq!(Step::Ready(3).ready(), Some(3));
51    /// assert_eq!(Step::<u8>::NeedInput.ready(), None);
52    /// ```
53    #[inline]
54    pub fn ready(self) -> Option<T> {
55        match self {
56            Self::Ready(value) => Some(value),
57            Self::NeedInput => None,
58        }
59    }
60
61    /// Whether the call completed.
62    ///
63    /// # Examples
64    ///
65    /// ```
66    /// assert!(jsonx_lib::Step::Ready(()).is_ready());
67    /// ```
68    #[inline]
69    #[must_use]
70    pub const fn is_ready(&self) -> bool {
71        matches!(self, Self::Ready(_))
72    }
73
74    /// Applies `f` to a ready result.
75    ///
76    /// # Examples
77    ///
78    /// ```
79    /// use jsonx_lib::Step;
80    ///
81    /// assert_eq!(Step::Ready(2).map(|n| n * 10), Step::Ready(20));
82    /// ```
83    #[inline]
84    pub fn map<U>(self, f: impl FnOnce(T) -> U) -> Step<U> {
85        match self {
86            Self::Ready(value) => Step::Ready(f(value)),
87            Self::NeedInput => Step::NeedInput,
88        }
89    }
90}
91
92/// A raw value being skipped across calls.
93#[derive(Clone, Copy, Debug)]
94struct RawProgress {
95    /// The depth the value started at; it is complete when the reader is back.
96    depth: usize,
97    /// The value's first byte (absolute), once its first event was read.
98    start: Option<usize>,
99}
100
101/// How much consumed text the buffer may keep before it is compacted.
102const COMPACT_AT: usize = 4 * 1024;
103
104/// A pull reader over input that arrives in pieces: bytes go in with
105/// [`feed`](Self::feed) as they arrive, events come out as soon as they are
106/// complete.
107///
108/// It is the same reader as [`Reader`](crate::Reader), over a buffer: the
109/// same grammar, the same budgets, and the same errors at the same offsets
110/// (counted from the first byte ever fed), whatever the input's chunking.
111/// When a call needs bytes that have not arrived it returns
112/// [`Step::NeedInput`] and changes nothing a later call would see
113/// differently; after [`end_input`](Self::end_input), running out is the
114/// [`ErrorKind::UnexpectedEnd`] that [`Reader`](crate::Reader) reports.
115///
116/// Events borrow from the reader's buffer, so each one must be dropped
117/// before the next call. In particular, feed more input after the borrow
118/// ends: in an `else` after `if let Step::Ready(..) = reader.next_event()?`
119/// (Rust 2024 drops the `if let` temporaries before the `else`), or after
120/// the loop that drains the ready events, as below; a `match` keeps the
121/// event borrowed in every arm. Consumed input is dropped from the buffer as more
122/// is fed, so a long document read as it arrives needs memory for one token
123/// and one chunk, not for the document. A token cut by a chunk boundary is
124/// continued where it stopped, never scanned again from its start: a string
125/// fed one byte at a time costs linear time.
126///
127/// The input must be UTF-8, checked as it arrives (a character may be split
128/// across chunks). The first invalid byte is reported as
129/// [`ErrorKind::InvalidUtf8`] when the reader reaches it: a syntax error
130/// earlier in the document is reported first. (A complete document given to
131/// [`parse_bytes`](crate::parse_bytes) checks the encoding before the
132/// syntax, so the two can name different errors for a document that has
133/// both.)
134///
135/// [`Limits::max_input_bytes`] bounds the total fed; the other budgets apply
136/// exactly as for a complete document. After an error the reader is
137/// finished: every later call returns the same error.
138///
139/// # Examples
140///
141/// ```
142/// use jsonx_lib::{Event, Step, StreamReader};
143///
144/// let mut reader = StreamReader::new();
145/// let mut keys = Vec::new();
146/// for chunk in [&b"{\"jsonr"[..], b"pc\": \"2.0\", \"i", b"d\": 12", b"}"] {
147///     reader.feed(chunk)?;
148///     while let Step::Ready(Some(event)) = reader.next_event()? {
149///         if let Event::Key(key) = event {
150///             keys.push(key.into_owned());
151///         }
152///     }
153/// }
154/// reader.end_input();
155/// assert_eq!(reader.next_event()?, Step::Ready(None)); // the document is complete
156/// assert_eq!(keys, ["jsonrpc", "id"]);
157/// # Ok::<(), jsonx_lib::Error>(())
158/// ```
159#[derive(Clone, Debug)]
160pub struct StreamReader {
161    /// The input not yet dropped: valid UTF-8, starting at absolute offset
162    /// `base`.
163    buf: String,
164    base: usize,
165    /// An incomplete UTF-8 sequence at the end of the input so far.
166    tail: [u8; 4],
167    tail_len: usize,
168    /// The absolute offset of the first byte that is not UTF-8. Nothing
169    /// from it on is kept.
170    utf8_error: Option<usize>,
171    /// Bytes fed in total.
172    fed: usize,
173    ended: bool,
174    machine: Machine,
175    failed: Option<Error>,
176    /// The absolute offset where the most recent event's token began.
177    last: usize,
178    raw: Option<RawProgress>,
179}
180
181impl Default for StreamReader {
182    fn default() -> Self {
183        Self::new()
184    }
185}
186
187impl StreamReader {
188    /// A reader with the default [`Limits`], refusing duplicate keys in what
189    /// [`read_value`](Self::read_value) builds. For others, use
190    /// [`Parser::stream_reader`].
191    ///
192    /// # Examples
193    ///
194    /// ```
195    /// let r = jsonx_lib::StreamReader::new();
196    /// assert_eq!(r.offset(), 0);
197    /// ```
198    #[must_use]
199    pub fn new() -> Self {
200        Self::with_config(Limits::new(), DuplicateKeys::Error)
201    }
202
203    /// A reader with the given budgets and duplicate-key policy.
204    pub(crate) fn with_config(limits: Limits, duplicates: DuplicateKeys) -> Self {
205        Self {
206            buf: String::new(),
207            base: 0,
208            tail: [0; 4],
209            tail_len: 0,
210            utf8_error: None,
211            fed: 0,
212            ended: false,
213            machine: {
214                let mut machine = Machine::new(limits, duplicates, false);
215                // Keys outlive the front of the buffer, which is dropped.
216                machine.copy_keys = true;
217                machine
218            },
219            failed: None,
220            last: 0,
221            raw: None,
222        }
223    }
224
225    /// Reads a sequence of documents instead of one, as
226    /// [`Reader::documents`](crate::Reader::documents) does: NDJSON, JSON
227    /// Lines, or concatenated JSON, fed as it arrives. Whole-value reads
228    /// return one document each, and `Ready(None)` once the input has ended
229    /// after the last.
230    ///
231    /// # Examples
232    ///
233    /// ```
234    /// use jsonx_lib::{Step, StreamReader};
235    ///
236    /// let mut r = StreamReader::new().documents();
237    /// let mut seen = Vec::new();
238    /// for chunk in [&b"{\"id\": 1}\n{\"i"[..], b"d\": 2}\n"] {
239    ///     r.feed(chunk)?;
240    ///     while let Step::Ready(Some(doc)) = r.read_value()? {
241    ///         seen.push(doc["id"].as_i64());
242    ///     }
243    /// }
244    /// r.end_input();
245    /// assert_eq!(r.read_value()?, Step::Ready(None));
246    /// assert_eq!(seen, [Some(1), Some(2)]);
247    /// # Ok::<(), jsonx_lib::Error>(())
248    /// ```
249    #[must_use]
250    pub fn documents(mut self) -> Self {
251        self.machine.documents = true;
252        self
253    }
254
255    /// Adds the next piece of input. A chunk may end anywhere, even inside a
256    /// token or a UTF-8 character.
257    ///
258    /// # Errors
259    ///
260    /// [`ErrorKind::InputTooLarge`] when the input in total grows past
261    /// [`Limits::max_input_bytes`] (the reader is then finished);
262    /// [`ErrorKind::InputAfterEnd`] after [`end_input`](Self::end_input);
263    /// any error the reader already stopped at.
264    ///
265    /// # Examples
266    ///
267    /// ```
268    /// use jsonx_lib::{ErrorKind, Limits, Parser};
269    ///
270    /// let mut r = Parser::new().limits(Limits::new().with_max_input_bytes(4)).stream_reader();
271    /// assert!(r.feed(b"[1,").is_ok());
272    /// assert_eq!(r.feed(b"22]").unwrap_err().kind(), ErrorKind::InputTooLarge);
273    /// ```
274    pub fn feed(&mut self, bytes: &[u8]) -> Result<(), Error> {
275        if let Some(e) = self.failed {
276            return Err(e);
277        }
278        if self.ended {
279            return Err(Error::new(ErrorKind::InputAfterEnd, self.fed));
280        }
281        let max = self.machine.limits.max_input_bytes;
282        if bytes.len() > max - self.fed {
283            let e = Error::new(ErrorKind::InputTooLarge, max);
284            self.failed = Some(e);
285            return Err(e);
286        }
287        self.fed += bytes.len();
288        self.compact();
289        if self.utf8_error.is_none() {
290            self.append(bytes);
291        }
292        Ok(())
293    }
294
295    /// Says that the input has ended: nothing more will be fed. From now on,
296    /// running out of input is an error, as for a complete document, and a
297    /// number or whitespace at the very end is final.
298    ///
299    /// # Examples
300    ///
301    /// ```
302    /// use jsonx_lib::{Event, Step, StreamReader};
303    ///
304    /// let mut r = StreamReader::new();
305    /// r.feed(b"12")?;
306    /// assert_eq!(r.next_event()?, Step::NeedInput); // `123` may follow
307    /// r.end_input();
308    /// assert_eq!(r.next_event()?, Step::Ready(Some(Event::Number(12.into()))));
309    /// assert_eq!(r.next_event()?, Step::Ready(None));
310    /// # Ok::<(), jsonx_lib::Error>(())
311    /// ```
312    pub fn end_input(&mut self) {
313        if self.ended {
314            return;
315        }
316        self.ended = true;
317        if self.tail_len > 0 && self.utf8_error.is_none() {
318            // A character cut off by the end of the input.
319            self.utf8_error = Some(self.base + self.buf.len());
320        }
321        self.machine.complete = true;
322    }
323
324    /// Reads up to 8 KiB from `source` and feeds it, or calls
325    /// [`end_input`](Self::end_input) when `source` is at its end. Returns
326    /// the number of bytes read (`0` at the end). Interrupted reads are
327    /// retried.
328    ///
329    /// A budget error from [`feed`](Self::feed) is not an I/O error: it is
330    /// kept, and returned by the next reading call.
331    ///
332    /// # Errors
333    ///
334    /// Any error from `source` other than [`std::io::ErrorKind::Interrupted`].
335    ///
336    /// # Examples
337    ///
338    /// ```
339    /// use jsonx_lib::{Step, StreamReader};
340    ///
341    /// let mut source: &[u8] = br#"{"big": [1, 2, 3]}"#;
342    /// let mut r = StreamReader::new();
343    /// let mut events = 0;
344    /// loop {
345    ///     if let Step::Ready(event) = r.next_event()? {
346    ///         match event {
347    ///             Some(_) => events += 1,
348    ///             None => break,
349    ///         }
350    ///     } else {
351    ///         r.read_from(&mut source)?;
352    ///     }
353    /// }
354    /// assert_eq!(events, 8); // {, "big", [, 1, 2, 3, ], }
355    /// # Ok::<(), Box<dyn std::error::Error>>(())
356    /// ```
357    #[cfg(feature = "std")]
358    #[cfg_attr(docsrs, doc(cfg(feature = "std")))]
359    pub fn read_from(&mut self, source: &mut dyn std::io::Read) -> std::io::Result<usize> {
360        let mut chunk = [0u8; 8 * 1024];
361        loop {
362            match source.read(&mut chunk) {
363                Ok(0) => {
364                    self.end_input();
365                    return Ok(0);
366                }
367                Ok(n) => {
368                    // A refused feed is kept in `failed` and reported by the
369                    // next reading call.
370                    let _kept = self.feed(chunk.get(..n).unwrap_or_default());
371                    return Ok(n);
372                }
373                Err(e) if e.kind() == std::io::ErrorKind::Interrupted => {}
374                Err(e) => return Err(e),
375            }
376        }
377    }
378
379    /// Reads the next event: `Ready(Some(event))`, `Ready(None)` once the
380    /// document is complete and the input has ended with nothing but
381    /// whitespace after it, or `NeedInput`.
382    ///
383    /// # Errors
384    ///
385    /// The first syntax error, exceeded budget, or invalid UTF-8, at its
386    /// byte offset. Once an error is returned, every later call returns it.
387    ///
388    /// # Examples
389    ///
390    /// ```
391    /// use jsonx_lib::{ErrorKind, Event, Step, StreamReader};
392    ///
393    /// let mut r = StreamReader::new();
394    /// r.feed(b"[1,")?;
395    /// assert_eq!(r.next_event()?, Step::Ready(Some(Event::StartArray)));
396    /// assert!(r.next_event()?.is_ready()); // 1
397    /// r.feed(b"]")?;
398    /// let err = r.next_event().unwrap_err();
399    /// assert_eq!((err.kind(), err.offset()), (ErrorKind::TrailingComma, 3));
400    /// # Ok::<(), jsonx_lib::Error>(())
401    /// ```
402    pub fn next_event(&mut self) -> Result<Step<Option<Event<'_>>>, Error> {
403        if let Some(e) = self.failed {
404            return Err(e);
405        }
406        let Self {
407            buf,
408            machine,
409            base,
410            last,
411            failed,
412            utf8_error,
413            ..
414        } = self;
415        match machine.step(buf) {
416            Ok(Some(event)) => {
417                *last = *base + machine.last;
418                Ok(Step::Ready(Some(event)))
419            }
420            Ok(None) => match utf8_error {
421                // The valid text holds a whole document, then bad bytes.
422                Some(at) => Err(*failed.insert(Error::new(ErrorKind::InvalidUtf8, *at))),
423                None => Ok(Step::Ready(None)),
424            },
425            Err(e) => halt(failed, machine, buf.len(), *base, *utf8_error, e),
426        }
427    }
428
429    /// Reads the next value whole and returns its exact source text, without
430    /// decoding or allocating anything, as [`Reader::read_raw`](crate::Reader::read_raw)
431    /// does, with the same positioning rules: `Ready(None)` when no value
432    /// comes next.
433    ///
434    /// The value must be in the buffer whole before it can be returned, so
435    /// a large value costs memory for all of it. `NeedInput` keeps the
436    /// progress: the next call continues the same value.
437    ///
438    /// # Errors
439    ///
440    /// As [`next_event`](Self::next_event), for any error inside the value.
441    ///
442    /// # Examples
443    ///
444    /// ```
445    /// use jsonx_lib::{Event, Step, StreamReader};
446    ///
447    /// let mut r = StreamReader::new();
448    /// r.feed(br#"{"id": 1, "params": {"line""#)?;
449    /// assert!(r.next_event()?.is_ready()); // {
450    /// assert!(r.next_event()?.is_ready()); // "id"
451    /// assert_eq!(r.read_raw()?, Step::Ready(Some("1")));
452    /// assert!(r.next_event()?.is_ready()); // "params"
453    /// assert_eq!(r.read_raw()?, Step::NeedInput);
454    /// r.feed(br#": 3}}"#)?;
455    /// assert_eq!(r.read_raw()?, Step::Ready(Some(r#"{"line": 3}"#)));
456    /// # Ok::<(), jsonx_lib::Error>(())
457    /// ```
458    pub fn read_raw(&mut self) -> Result<Step<Option<&str>>, Error> {
459        Ok(self.raw_span()?.map(|span| {
460            span.and_then(|(start, end)| self.buf.get(start - self.base..end - self.base))
461        }))
462    }
463
464    /// Reads the next value whole and builds it as a [`Value`], applying the
465    /// reader's [`DuplicateKeys`] policy; positioning as for
466    /// [`read_raw`](Self::read_raw). The value borrows from the buffer:
467    /// call [`Value::into_owned`] to keep it.
468    ///
469    /// # Errors
470    ///
471    /// As [`read_raw`](Self::read_raw), plus [`ErrorKind::DuplicateKey`]
472    /// under [`DuplicateKeys::Error`].
473    ///
474    /// # Examples
475    ///
476    /// ```
477    /// use jsonx_lib::{Step, StreamReader};
478    ///
479    /// let mut r = StreamReader::new();
480    /// r.feed(br#"{"a": [1, "#)?;
481    /// assert_eq!(r.read_value()?, Step::NeedInput);
482    /// r.feed(br#"2]}"#)?;
483    /// let value = r.read_value()?.ready().flatten().unwrap_or_default();
484    /// assert_eq!(value["a"][1], 2);
485    /// # Ok::<(), jsonx_lib::Error>(())
486    /// ```
487    pub fn read_value(&mut self) -> Result<Step<Option<Value<'_>>>, Error> {
488        if let Some(e) = self.failed {
489            return Err(e);
490        }
491        let Step::Ready(span) = self.raw_span()? else {
492            return Ok(Step::NeedInput);
493        };
494        let Some((start, end)) = span else {
495            return Ok(Step::Ready(None));
496        };
497        let text = self
498            .buf
499            .get(start - self.base..end - self.base)
500            .unwrap_or("");
501        // The budgets, and duplicates under `DuplicateKeys::Error`, were
502        // checked while the value was skipped; the memory budget, and
503        // `First` and `Last`, remain to apply to the tree.
504        let limits = Limits::unlimited().with_max_memory(self.machine.limits.max_memory);
505        let parser = Parser::new()
506            .limits(limits)
507            .duplicate_keys(self.machine.duplicates);
508        match parser.parse(text) {
509            Ok(value) => Ok(Step::Ready(Some(value))),
510            Err(e) => {
511                let e = Error::new(e.kind(), e.offset() + start);
512                self.failed = Some(e);
513                Err(e)
514            }
515        }
516    }
517
518    /// Reads the next value whole as a `T` ([`FromJson`]); positioning as
519    /// for [`read_raw`](Self::read_raw). The value borrows from the buffer,
520    /// as `T` allows.
521    ///
522    /// The value is first skipped (validated, with every budget) as it
523    /// arrives, then read as `T` once it is all there. Byte offsets in an
524    /// error count from the first byte fed.
525    ///
526    /// # Errors
527    ///
528    /// The [`DecodeError`] of `T`, or the reader's error.
529    ///
530    /// # Examples
531    ///
532    /// ```
533    /// use jsonx_lib::{Step, StreamReader};
534    ///
535    /// let mut r = StreamReader::new().documents();
536    /// r.feed(b"[1, 2]\n[3")?;
537    /// assert_eq!(r.decode::<Vec<u8>>()?, Step::Ready(Some(vec![1, 2])));
538    /// assert_eq!(r.decode::<Vec<u8>>()?, Step::NeedInput);
539    /// r.feed(b"]\n")?;
540    /// r.end_input();
541    /// assert_eq!(r.decode::<Vec<u8>>()?, Step::Ready(Some(vec![3])));
542    /// assert_eq!(r.decode::<Vec<u8>>()?, Step::Ready(None));
543    /// # Ok::<(), jsonx_lib::DecodeError>(())
544    /// ```
545    pub fn decode<'s, T: FromJson<'s>>(&'s mut self) -> Result<Step<Option<T>>, DecodeError> {
546        let Step::Ready(span) = self.raw_span()? else {
547            return Ok(Step::NeedInput);
548        };
549        let Some((start, end)) = span else {
550            return Ok(Step::Ready(None));
551        };
552        let text = self
553            .buf
554            .get(start - self.base..end - self.base)
555            .unwrap_or("");
556        // The budgets, and duplicates under `DuplicateKeys::Error`, were
557        // checked while the value was skipped. The depth budget stays, to
558        // bound a recursive `T`, and the memory budget, for what `T` holds.
559        let limits = Limits::unlimited()
560            .with_max_depth(self.machine.limits.max_depth)
561            .with_max_memory(self.machine.limits.max_memory);
562        let parser = Parser::new()
563            .limits(limits)
564            .duplicate_keys(self.machine.duplicates)
565            .numbers(self.machine.numbers);
566        parser
567            .decode(text)
568            .map(|value| Step::Ready(Some(value)))
569            .map_err(|e| e.shifted(start))
570    }
571
572    /// The kind of the next value, without consuming anything: `Ready(None)`
573    /// when no value comes next, or when the next byte cannot start one (the
574    /// next [`next_event`](Self::next_event) reports why).
575    ///
576    /// # Errors
577    ///
578    /// Any error the reader already stopped at.
579    ///
580    /// # Examples
581    ///
582    /// ```
583    /// use jsonx_lib::{Kind, Step, StreamReader};
584    ///
585    /// let mut r = StreamReader::new();
586    /// r.feed(b"[ ")?;
587    /// assert!(r.next_event()?.is_ready());
588    /// assert_eq!(r.peek_kind()?, Step::NeedInput);
589    /// r.feed(b"\"x\"]")?;
590    /// assert_eq!(r.peek_kind()?, Step::Ready(Some(Kind::String)));
591    /// # Ok::<(), jsonx_lib::Error>(())
592    /// ```
593    pub fn peek_kind(&mut self) -> Result<Step<Option<Kind>>, Error> {
594        if let Some(e) = self.failed {
595            return Err(e);
596        }
597        match self.machine.at_value(&self.buf) {
598            Ok(false) => Ok(Step::Ready(None)),
599            Ok(true) => match self.machine.peek_kind(&self.buf) {
600                Some(kind) => Ok(Step::Ready(Some(kind))),
601                // The value's first byte has not arrived.
602                None if self.machine.needed_from() == self.buf.len() && !self.machine.complete => {
603                    self.halt(Error::new(ErrorKind::UnexpectedEnd, self.buf.len()))
604                }
605                None => Ok(Step::Ready(None)),
606            },
607            Err(e) => self.halt(e),
608        }
609    }
610
611    /// The number of arrays and objects open.
612    ///
613    /// # Examples
614    ///
615    /// ```
616    /// let mut r = jsonx_lib::StreamReader::new();
617    /// r.feed(b"[[")?;
618    /// let _ = r.next_event()?;
619    /// let _ = r.next_event()?;
620    /// assert_eq!(r.depth(), 2);
621    /// # Ok::<(), jsonx_lib::Error>(())
622    /// ```
623    #[inline]
624    #[must_use]
625    pub fn depth(&self) -> usize {
626        self.machine.depth()
627    }
628
629    /// The byte offset, from the first byte fed, just past the input
630    /// consumed so far.
631    ///
632    /// # Examples
633    ///
634    /// ```
635    /// let mut r = jsonx_lib::StreamReader::new();
636    /// r.feed(b"[10, 20]")?;
637    /// let _ = r.next_event()?;
638    /// let _ = r.next_event()?;
639    /// assert_eq!(r.offset(), 3);
640    /// # Ok::<(), jsonx_lib::Error>(())
641    /// ```
642    #[inline]
643    #[must_use]
644    pub fn offset(&self) -> usize {
645        self.base + self.machine.pos
646    }
647
648    /// The byte offset, from the first byte fed, where the most recent
649    /// event's token began (see [`Reader::event_offset`](crate::Reader::event_offset)).
650    ///
651    /// # Examples
652    ///
653    /// ```
654    /// let mut r = jsonx_lib::StreamReader::new();
655    /// r.feed(b"[ 7]")?;
656    /// let _ = r.next_event()?;
657    /// let _ = r.next_event()?;
658    /// assert_eq!(r.event_offset(), 2);
659    /// # Ok::<(), jsonx_lib::Error>(())
660    /// ```
661    #[inline]
662    #[must_use]
663    pub fn event_offset(&self) -> usize {
664        self.last
665    }
666
667    /// The bytes of input the reader holds: what has been fed and not yet
668    /// dropped. Consumed input is dropped as more is fed.
669    ///
670    /// # Examples
671    ///
672    /// ```
673    /// let mut r = jsonx_lib::StreamReader::new();
674    /// r.feed(b"[1, 2")?;
675    /// assert_eq!(r.buffered(), 5);
676    /// # Ok::<(), jsonx_lib::Error>(())
677    /// ```
678    #[inline]
679    #[must_use]
680    pub fn buffered(&self) -> usize {
681        self.buf.len() + self.tail_len
682    }
683
684    // ---- internals -----------------------------------------------------------
685
686    /// [`halt`] for `self`.
687    fn halt<T>(&mut self, e: Error) -> Result<Step<T>, Error> {
688        halt(
689            &mut self.failed,
690            &self.machine,
691            self.buf.len(),
692            self.base,
693            self.utf8_error,
694            e,
695        )
696    }
697
698    /// Skips the next value (continuing one already started) and returns
699    /// its absolute span, or `None` when no value comes next.
700    fn raw_span(&mut self) -> Result<Step<Option<(usize, usize)>>, Error> {
701        if let Some(e) = self.failed {
702            return Err(e);
703        }
704        let mut progress = match self.raw {
705            Some(progress) => progress,
706            None => match self.machine.at_value(&self.buf) {
707                Ok(false) => return Ok(Step::Ready(None)),
708                Ok(true) => RawProgress {
709                    depth: self.machine.depth(),
710                    start: None,
711                },
712                Err(e) => return self.halt(e),
713            },
714        };
715        self.machine.skipping = true;
716        let outcome = loop {
717            match self.machine.step(&self.buf) {
718                Ok(Some(_)) => {
719                    self.last = self.base + self.machine.last;
720                    let start = *progress.start.get_or_insert(self.last);
721                    if self.machine.depth() == progress.depth {
722                        break Ok(Some((start, self.base + self.machine.pos)));
723                    }
724                }
725                // `at_value` saw a value, so the document cannot end here.
726                Ok(None) => break Err(Error::new(ErrorKind::UnexpectedEnd, self.buf.len())),
727                Err(e) => break Err(e),
728            }
729        };
730        self.machine.skipping = false;
731        match outcome {
732            Ok(span) => {
733                self.raw = None;
734                Ok(Step::Ready(span))
735            }
736            Err(e) => {
737                self.raw = Some(progress);
738                self.halt(e)
739            }
740        }
741    }
742
743    /// Drops consumed input from the front of the buffer once there is
744    /// enough of it to be worth the copy (amortized constant per byte).
745    fn compact(&mut self) {
746        let mut keep_from = self.machine.needed_from();
747        if let Some(RawProgress {
748            start: Some(start), ..
749        }) = self.raw
750        {
751            keep_from = keep_from.min(start - self.base);
752        }
753        if keep_from < COMPACT_AT
754            || keep_from * 2 < self.buf.len()
755            || !self.buf.is_char_boundary(keep_from)
756        {
757            return;
758        }
759        let _dropped = self.buf.drain(..keep_from);
760        self.base += keep_from;
761        self.machine.rebase(keep_from);
762    }
763
764    /// Records that the input stops being UTF-8 at the end of the buffer.
765    /// No valid byte can follow, so the text so far is all there will be.
766    fn bad_byte(&mut self) {
767        self.utf8_error = Some(self.base + self.buf.len());
768        self.machine.complete = true;
769    }
770
771    /// Appends `bytes` to the buffer, keeping only valid UTF-8 and holding
772    /// back a character cut off at the end.
773    fn append(&mut self, bytes: &[u8]) {
774        let mut rest = bytes;
775        if self.tail_len > 0 {
776            // Complete the character the last chunk cut off.
777            let take = (4 - self.tail_len).min(rest.len());
778            let (head, after) = rest.split_at(take);
779            if let Some(slot) = self.tail.get_mut(self.tail_len..self.tail_len + take) {
780                slot.copy_from_slice(head);
781            }
782            let have = self.tail_len + take;
783            let pending = self.tail.get(..have).unwrap_or_default();
784            match core::str::from_utf8(pending) {
785                Ok(text) => {
786                    self.buf.push_str(text);
787                    self.tail_len = 0;
788                    rest = after;
789                }
790                Err(e) => {
791                    let used = e.valid_up_to();
792                    if used > 0 {
793                        // The cut character completed; the bytes after it
794                        // in `pending` are new input to read normally.
795                        let done = pending.get(..used).unwrap_or_default();
796                        self.buf.push_str(core::str::from_utf8(done).unwrap_or(""));
797                        let extra = used - self.tail_len;
798                        self.tail_len = 0;
799                        rest = bytes.get(extra..).unwrap_or_default();
800                    } else if e.error_len().is_none() && after.is_empty() {
801                        // Still incomplete, and this chunk is used up.
802                        self.tail_len = have;
803                        return;
804                    } else {
805                        self.bad_byte();
806                        return;
807                    }
808                }
809            }
810        }
811        match core::str::from_utf8(rest) {
812            Ok(text) => self.buf.push_str(text),
813            Err(e) => {
814                let (valid, invalid) = rest.split_at(e.valid_up_to());
815                self.buf.push_str(core::str::from_utf8(valid).unwrap_or(""));
816                if e.error_len().is_none() {
817                    // A character cut off by the end of this chunk.
818                    if let Some(slot) = self.tail.get_mut(..invalid.len()) {
819                        slot.copy_from_slice(invalid);
820                    }
821                    self.tail_len = invalid.len();
822                } else {
823                    self.bad_byte();
824                }
825            }
826        }
827    }
828}
829
830/// Turns a machine error into a call's answer: `NeedInput` when the buffer
831/// (of `len` bytes, starting at absolute offset `base`) merely ran out,
832/// otherwise the error, made absolute and kept in `failed`.
833fn halt<T>(
834    failed: &mut Option<Error>,
835    machine: &Machine,
836    len: usize,
837    base: usize,
838    utf8_error: Option<usize>,
839    e: Error,
840) -> Result<Step<T>, Error> {
841    let ran_out = e.kind() == ErrorKind::UnexpectedEnd && e.offset() == len;
842    let e = match (ran_out, utf8_error) {
843        (true, _) if !machine.complete => return Ok(Step::NeedInput),
844        // The valid text ends where the input stops being UTF-8.
845        (true, Some(at)) => Error::new(ErrorKind::InvalidUtf8, at),
846        // Wrapping: a repeated key's quote can lie in the dropped front of
847        // the buffer, where its text offset wrapped below zero (see
848        // `Machine::check_object`); adding `base` back gives its exact
849        // offset in the input.
850        _ => Error::new(e.kind(), e.offset().wrapping_add(base)),
851    };
852    *failed = Some(e);
853    Err(e)
854}
855
856impl Parser {
857    /// A [`StreamReader`] with this parser's budgets and duplicate-key
858    /// policy (which [`StreamReader::read_value`] applies).
859    ///
860    /// # Examples
861    ///
862    /// ```
863    /// use jsonx_lib::{ErrorKind, Limits, Parser};
864    ///
865    /// let mut r = Parser::new().limits(Limits::new().with_max_depth(1)).stream_reader();
866    /// r.feed(b"[[]]")?;
867    /// assert!(r.next_event().is_ok());
868    /// assert_eq!(r.next_event().unwrap_err().kind(), ErrorKind::DepthLimitExceeded);
869    /// # Ok::<(), jsonx_lib::Error>(())
870    /// ```
871    #[must_use]
872    pub fn stream_reader(&self) -> StreamReader {
873        let mut reader = StreamReader::with_config(self.get_limits(), self.get_duplicate_keys());
874        reader.machine.numbers = self.get_numbers();
875        reader
876    }
877}
878
879#[cfg(test)]
880mod tests {
881    use super::*;
882
883    #[test]
884    fn test_utf8_split_at_every_byte_of_a_character() {
885        let text = "[\"a\u{e9}\u{20ac}\u{1f600}\"]";
886        for cut in 0..=text.len() {
887            let (a, b) = text.as_bytes().split_at(cut);
888            let mut r = StreamReader::new();
889            assert!(r.feed(a).is_ok() && r.feed(b).is_ok());
890            r.end_input();
891            let _start = r.next_event();
892            let got = r.next_event().ok().and_then(Step::ready).flatten();
893            assert_eq!(
894                got,
895                Some(Event::String("a\u{e9}\u{20ac}\u{1f600}".into())),
896                "cut {cut}"
897            );
898        }
899        // One byte at a time.
900        let mut r = StreamReader::new();
901        for &byte in text.as_bytes() {
902            assert!(r.feed(&[byte]).is_ok());
903        }
904        assert_eq!(r.buffered(), text.len());
905    }
906
907    #[test]
908    fn test_invalid_utf8_is_reported_where_the_reader_reaches_it() {
909        let mut r = StreamReader::new();
910        assert!(r.feed(b"[1, \"a\xC3").is_ok());
911        assert!(r.feed(b"(\"]").is_ok()); // 0xC3 0x28 is not UTF-8
912        let _start = r.next_event();
913        let _one = r.next_event();
914        let err = r.next_event().err();
915        assert_eq!(
916            err.map(|e| (e.kind(), e.offset())),
917            Some((ErrorKind::InvalidUtf8, 6))
918        );
919        // A character cut off by the end of the input.
920        let mut r = StreamReader::new();
921        assert!(r.feed(b"\"\xE2\x82").is_ok());
922        assert_eq!(r.next_event().ok(), Some(Step::NeedInput));
923        r.end_input();
924        let err = r.next_event().err();
925        assert_eq!(
926            err.map(|e| (e.kind(), e.offset())),
927            Some((ErrorKind::InvalidUtf8, 1))
928        );
929        // A syntax error before the bad byte comes first.
930        let mut r = StreamReader::new();
931        assert!(r.feed(b"[1 2, \xFF]").is_ok());
932        let _start = r.next_event();
933        let _one = r.next_event();
934        let err = r.next_event().err();
935        assert_eq!(
936            err.map(|e| (e.kind(), e.offset())),
937            Some((ErrorKind::ExpectedCommaOrBracket, 3))
938        );
939        // An invalid continuation completing a cut character.
940        let mut r = StreamReader::new();
941        assert!(r.feed(b"\"\xE2").is_ok());
942        assert!(r.feed(b"\x41\"").is_ok());
943        let err = r.next_event().err();
944        assert_eq!(
945            err.map(|e| (e.kind(), e.offset())),
946            Some((ErrorKind::InvalidUtf8, 1))
947        );
948        // A cut character completed, with more input after it in the chunk.
949        let mut r = StreamReader::new();
950        assert!(r.feed(b"\"\xE2\x82").is_ok());
951        assert!(r.feed(b"\xACabc\"").is_ok());
952        assert_eq!(
953            r.next_event().ok().and_then(Step::ready).flatten(),
954            Some(Event::String("\u{20ac}abc".into()))
955        );
956    }
957
958    #[test]
959    fn test_feed_after_end_and_errors_latch() {
960        let mut r = StreamReader::new();
961        assert!(r.feed(b"[").is_ok());
962        r.end_input();
963        r.end_input();
964        assert_eq!(
965            r.feed(b"]").err().map(|e| e.kind()),
966            Some(ErrorKind::InputAfterEnd)
967        );
968        let _start = r.next_event();
969        let first = r.next_event().err();
970        assert_eq!(first.map(|e| e.kind()), Some(ErrorKind::UnexpectedEnd));
971        assert_eq!(r.next_event().err(), first);
972        assert_eq!(r.read_raw().err(), first);
973        assert_eq!(r.read_value().err(), first);
974        assert_eq!(r.peek_kind().err(), first);
975        assert_eq!(r.feed(b"x").err(), first);
976    }
977
978    #[test]
979    fn test_compaction_keeps_memory_to_a_chunk_and_offsets_absolute() {
980        let mut r = StreamReader::new();
981        let item = b"12345, ";
982        let mut total = 1;
983        assert!(r.feed(b"[").is_ok());
984        let mut max_buffered = 0;
985        for _ in 0..20_000 {
986            assert!(r.feed(item).is_ok());
987            total += item.len();
988            while let Ok(Step::Ready(Some(_))) = r.next_event() {}
989            max_buffered = max_buffered.max(r.buffered());
990        }
991        assert!(max_buffered < 3 * COMPACT_AT, "{max_buffered}");
992        assert!(r.feed(b"]").is_ok());
993        let err = r.next_event().err();
994        assert_eq!(
995            err.map(|e| (e.kind(), e.offset())),
996            Some((ErrorKind::TrailingComma, total))
997        );
998    }
999
1000    #[test]
1001    fn test_raw_value_spanning_many_chunks_survives_compaction() {
1002        let mut r = StreamReader::new();
1003        assert!(r.feed(b"[0, ").is_ok());
1004        let _start = r.next_event();
1005        assert_eq!(
1006            r.read_raw().ok().map(|s| s.map(|o| o.map(str::len))),
1007            Some(Step::Ready(Some(1)))
1008        );
1009        assert!(r.feed(b"{\"k\": [").is_ok());
1010        assert_eq!(r.read_raw().ok(), Some(Step::NeedInput));
1011        let mut expected = String::from("{\"k\": [");
1012        for i in 0..5_000 {
1013            let piece = alloc::format!("{i}, ");
1014            expected.push_str(&piece);
1015            assert!(r.feed(piece.as_bytes()).is_ok());
1016            assert_eq!(r.read_raw().ok(), Some(Step::NeedInput));
1017        }
1018        assert!(r.feed(b"0]}]").is_ok());
1019        expected.push_str("0]}");
1020        let raw = r
1021            .read_raw()
1022            .ok()
1023            .and_then(Step::ready)
1024            .flatten()
1025            .map(String::from);
1026        assert_eq!(raw.as_deref(), Some(expected.as_str()));
1027        assert_eq!(r.event_offset(), 4 + expected.len() - 1);
1028    }
1029}