jsonx_lib/stream.rs
1//! The incremental reader: a document fed in chunks as they arrive.
2
3use alloc::string::String;
4
5use crate::{
6 DecodeError, DuplicateKeys, FromJson, Kind, Limits, Parser, Value,
7 error::{Error, ErrorKind},
8 machine::Machine,
9 reader::Event,
10};
11
12/// The answer of a [`StreamReader`] call: ready, or waiting for more input.
13///
14/// `NeedInput` is not an error: the input so far is a valid beginning, and
15/// the call can be made again after [`feed`](StreamReader::feed) or
16/// [`end_input`](StreamReader::end_input).
17///
18/// # Examples
19///
20/// ```
21/// use jsonx_lib::{Event, Step, StreamReader};
22///
23/// let mut r = StreamReader::new();
24/// assert_eq!(r.next_event()?, Step::NeedInput);
25/// r.feed(b"[tr")?;
26/// assert_eq!(r.next_event()?, Step::Ready(Some(Event::StartArray)));
27/// assert_eq!(r.next_event()?, Step::NeedInput); // `tr` is not a value yet
28/// r.feed(b"ue]")?;
29/// assert_eq!(r.next_event()?.ready(), Some(Some(Event::Bool(true))));
30/// # Ok::<(), jsonx_lib::Error>(())
31/// ```
32#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash)]
33#[must_use]
34pub enum Step<T> {
35 /// The call completed with this result.
36 Ready(T),
37 /// The input so far ends inside what the call needs: feed more, or end
38 /// the input, and call again.
39 NeedInput,
40}
41
42impl<T> Step<T> {
43 /// The result, if ready.
44 ///
45 /// # Examples
46 ///
47 /// ```
48 /// use jsonx_lib::Step;
49 ///
50 /// assert_eq!(Step::Ready(3).ready(), Some(3));
51 /// assert_eq!(Step::<u8>::NeedInput.ready(), None);
52 /// ```
53 #[inline]
54 pub fn ready(self) -> Option<T> {
55 match self {
56 Self::Ready(value) => Some(value),
57 Self::NeedInput => None,
58 }
59 }
60
61 /// Whether the call completed.
62 ///
63 /// # Examples
64 ///
65 /// ```
66 /// assert!(jsonx_lib::Step::Ready(()).is_ready());
67 /// ```
68 #[inline]
69 #[must_use]
70 pub const fn is_ready(&self) -> bool {
71 matches!(self, Self::Ready(_))
72 }
73
74 /// Applies `f` to a ready result.
75 ///
76 /// # Examples
77 ///
78 /// ```
79 /// use jsonx_lib::Step;
80 ///
81 /// assert_eq!(Step::Ready(2).map(|n| n * 10), Step::Ready(20));
82 /// ```
83 #[inline]
84 pub fn map<U>(self, f: impl FnOnce(T) -> U) -> Step<U> {
85 match self {
86 Self::Ready(value) => Step::Ready(f(value)),
87 Self::NeedInput => Step::NeedInput,
88 }
89 }
90}
91
92/// A raw value being skipped across calls.
93#[derive(Clone, Copy, Debug)]
94struct RawProgress {
95 /// The depth the value started at; it is complete when the reader is back.
96 depth: usize,
97 /// The value's first byte (absolute), once its first event was read.
98 start: Option<usize>,
99}
100
101/// How much consumed text the buffer may keep before it is compacted.
102const COMPACT_AT: usize = 4 * 1024;
103
104/// A pull reader over input that arrives in pieces: bytes go in with
105/// [`feed`](Self::feed) as they arrive, events come out as soon as they are
106/// complete.
107///
108/// It is the same reader as [`Reader`](crate::Reader), over a buffer: the
109/// same grammar, the same budgets, and the same errors at the same offsets
110/// (counted from the first byte ever fed), whatever the input's chunking.
111/// When a call needs bytes that have not arrived it returns
112/// [`Step::NeedInput`] and changes nothing a later call would see
113/// differently; after [`end_input`](Self::end_input), running out is the
114/// [`ErrorKind::UnexpectedEnd`] that [`Reader`](crate::Reader) reports.
115///
116/// Events borrow from the reader's buffer, so each one must be dropped
117/// before the next call. In particular, feed more input after the borrow
118/// ends: in an `else` after `if let Step::Ready(..) = reader.next_event()?`
119/// (Rust 2024 drops the `if let` temporaries before the `else`), or after
120/// the loop that drains the ready events, as below; a `match` keeps the
121/// event borrowed in every arm. Consumed input is dropped from the buffer as more
122/// is fed, so a long document read as it arrives needs memory for one token
123/// and one chunk, not for the document. A token cut by a chunk boundary is
124/// continued where it stopped, never scanned again from its start: a string
125/// fed one byte at a time costs linear time.
126///
127/// The input must be UTF-8, checked as it arrives (a character may be split
128/// across chunks). The first invalid byte is reported as
129/// [`ErrorKind::InvalidUtf8`] when the reader reaches it: a syntax error
130/// earlier in the document is reported first. (A complete document given to
131/// [`parse_bytes`](crate::parse_bytes) checks the encoding before the
132/// syntax, so the two can name different errors for a document that has
133/// both.)
134///
135/// [`Limits::max_input_bytes`] bounds the total fed; the other budgets apply
136/// exactly as for a complete document. After an error the reader is
137/// finished: every later call returns the same error.
138///
139/// # Examples
140///
141/// ```
142/// use jsonx_lib::{Event, Step, StreamReader};
143///
144/// let mut reader = StreamReader::new();
145/// let mut keys = Vec::new();
146/// for chunk in [&b"{\"jsonr"[..], b"pc\": \"2.0\", \"i", b"d\": 12", b"}"] {
147/// reader.feed(chunk)?;
148/// while let Step::Ready(Some(event)) = reader.next_event()? {
149/// if let Event::Key(key) = event {
150/// keys.push(key.into_owned());
151/// }
152/// }
153/// }
154/// reader.end_input();
155/// assert_eq!(reader.next_event()?, Step::Ready(None)); // the document is complete
156/// assert_eq!(keys, ["jsonrpc", "id"]);
157/// # Ok::<(), jsonx_lib::Error>(())
158/// ```
159#[derive(Clone, Debug)]
160pub struct StreamReader {
161 /// The input not yet dropped: valid UTF-8, starting at absolute offset
162 /// `base`.
163 buf: String,
164 base: usize,
165 /// An incomplete UTF-8 sequence at the end of the input so far.
166 tail: [u8; 4],
167 tail_len: usize,
168 /// The absolute offset of the first byte that is not UTF-8. Nothing
169 /// from it on is kept.
170 utf8_error: Option<usize>,
171 /// Bytes fed in total.
172 fed: usize,
173 ended: bool,
174 machine: Machine,
175 failed: Option<Error>,
176 /// The absolute offset where the most recent event's token began.
177 last: usize,
178 raw: Option<RawProgress>,
179}
180
181impl Default for StreamReader {
182 fn default() -> Self {
183 Self::new()
184 }
185}
186
187impl StreamReader {
188 /// A reader with the default [`Limits`], refusing duplicate keys in what
189 /// [`read_value`](Self::read_value) builds. For others, use
190 /// [`Parser::stream_reader`].
191 ///
192 /// # Examples
193 ///
194 /// ```
195 /// let r = jsonx_lib::StreamReader::new();
196 /// assert_eq!(r.offset(), 0);
197 /// ```
198 #[must_use]
199 pub fn new() -> Self {
200 Self::with_config(Limits::new(), DuplicateKeys::Error)
201 }
202
203 /// A reader with the given budgets and duplicate-key policy.
204 pub(crate) fn with_config(limits: Limits, duplicates: DuplicateKeys) -> Self {
205 Self {
206 buf: String::new(),
207 base: 0,
208 tail: [0; 4],
209 tail_len: 0,
210 utf8_error: None,
211 fed: 0,
212 ended: false,
213 machine: {
214 let mut machine = Machine::new(limits, duplicates, false);
215 // Keys outlive the front of the buffer, which is dropped.
216 machine.copy_keys = true;
217 machine
218 },
219 failed: None,
220 last: 0,
221 raw: None,
222 }
223 }
224
225 /// Reads a sequence of documents instead of one, as
226 /// [`Reader::documents`](crate::Reader::documents) does: NDJSON, JSON
227 /// Lines, or concatenated JSON, fed as it arrives. Whole-value reads
228 /// return one document each, and `Ready(None)` once the input has ended
229 /// after the last.
230 ///
231 /// # Examples
232 ///
233 /// ```
234 /// use jsonx_lib::{Step, StreamReader};
235 ///
236 /// let mut r = StreamReader::new().documents();
237 /// let mut seen = Vec::new();
238 /// for chunk in [&b"{\"id\": 1}\n{\"i"[..], b"d\": 2}\n"] {
239 /// r.feed(chunk)?;
240 /// while let Step::Ready(Some(doc)) = r.read_value()? {
241 /// seen.push(doc["id"].as_i64());
242 /// }
243 /// }
244 /// r.end_input();
245 /// assert_eq!(r.read_value()?, Step::Ready(None));
246 /// assert_eq!(seen, [Some(1), Some(2)]);
247 /// # Ok::<(), jsonx_lib::Error>(())
248 /// ```
249 #[must_use]
250 pub fn documents(mut self) -> Self {
251 self.machine.documents = true;
252 self
253 }
254
255 /// Adds the next piece of input. A chunk may end anywhere, even inside a
256 /// token or a UTF-8 character.
257 ///
258 /// # Errors
259 ///
260 /// [`ErrorKind::InputTooLarge`] when the input in total grows past
261 /// [`Limits::max_input_bytes`] (the reader is then finished);
262 /// [`ErrorKind::InputAfterEnd`] after [`end_input`](Self::end_input);
263 /// any error the reader already stopped at.
264 ///
265 /// # Examples
266 ///
267 /// ```
268 /// use jsonx_lib::{ErrorKind, Limits, Parser};
269 ///
270 /// let mut r = Parser::new().limits(Limits::new().with_max_input_bytes(4)).stream_reader();
271 /// assert!(r.feed(b"[1,").is_ok());
272 /// assert_eq!(r.feed(b"22]").unwrap_err().kind(), ErrorKind::InputTooLarge);
273 /// ```
274 pub fn feed(&mut self, bytes: &[u8]) -> Result<(), Error> {
275 if let Some(e) = self.failed {
276 return Err(e);
277 }
278 if self.ended {
279 return Err(Error::new(ErrorKind::InputAfterEnd, self.fed));
280 }
281 let max = self.machine.limits.max_input_bytes;
282 if bytes.len() > max - self.fed {
283 let e = Error::new(ErrorKind::InputTooLarge, max);
284 self.failed = Some(e);
285 return Err(e);
286 }
287 self.fed += bytes.len();
288 self.compact();
289 if self.utf8_error.is_none() {
290 self.append(bytes);
291 }
292 Ok(())
293 }
294
295 /// Says that the input has ended: nothing more will be fed. From now on,
296 /// running out of input is an error, as for a complete document, and a
297 /// number or whitespace at the very end is final.
298 ///
299 /// # Examples
300 ///
301 /// ```
302 /// use jsonx_lib::{Event, Step, StreamReader};
303 ///
304 /// let mut r = StreamReader::new();
305 /// r.feed(b"12")?;
306 /// assert_eq!(r.next_event()?, Step::NeedInput); // `123` may follow
307 /// r.end_input();
308 /// assert_eq!(r.next_event()?, Step::Ready(Some(Event::Number(12.into()))));
309 /// assert_eq!(r.next_event()?, Step::Ready(None));
310 /// # Ok::<(), jsonx_lib::Error>(())
311 /// ```
312 pub fn end_input(&mut self) {
313 if self.ended {
314 return;
315 }
316 self.ended = true;
317 if self.tail_len > 0 && self.utf8_error.is_none() {
318 // A character cut off by the end of the input.
319 self.utf8_error = Some(self.base + self.buf.len());
320 }
321 self.machine.complete = true;
322 }
323
324 /// Reads up to 8 KiB from `source` and feeds it, or calls
325 /// [`end_input`](Self::end_input) when `source` is at its end. Returns
326 /// the number of bytes read (`0` at the end). Interrupted reads are
327 /// retried.
328 ///
329 /// A budget error from [`feed`](Self::feed) is not an I/O error: it is
330 /// kept, and returned by the next reading call.
331 ///
332 /// # Errors
333 ///
334 /// Any error from `source` other than [`std::io::ErrorKind::Interrupted`].
335 ///
336 /// # Examples
337 ///
338 /// ```
339 /// use jsonx_lib::{Step, StreamReader};
340 ///
341 /// let mut source: &[u8] = br#"{"big": [1, 2, 3]}"#;
342 /// let mut r = StreamReader::new();
343 /// let mut events = 0;
344 /// loop {
345 /// if let Step::Ready(event) = r.next_event()? {
346 /// match event {
347 /// Some(_) => events += 1,
348 /// None => break,
349 /// }
350 /// } else {
351 /// r.read_from(&mut source)?;
352 /// }
353 /// }
354 /// assert_eq!(events, 8); // {, "big", [, 1, 2, 3, ], }
355 /// # Ok::<(), Box<dyn std::error::Error>>(())
356 /// ```
357 #[cfg(feature = "std")]
358 #[cfg_attr(docsrs, doc(cfg(feature = "std")))]
359 pub fn read_from(&mut self, source: &mut dyn std::io::Read) -> std::io::Result<usize> {
360 let mut chunk = [0u8; 8 * 1024];
361 loop {
362 match source.read(&mut chunk) {
363 Ok(0) => {
364 self.end_input();
365 return Ok(0);
366 }
367 Ok(n) => {
368 // A refused feed is kept in `failed` and reported by the
369 // next reading call.
370 let _kept = self.feed(chunk.get(..n).unwrap_or_default());
371 return Ok(n);
372 }
373 Err(e) if e.kind() == std::io::ErrorKind::Interrupted => {}
374 Err(e) => return Err(e),
375 }
376 }
377 }
378
379 /// Reads the next event: `Ready(Some(event))`, `Ready(None)` once the
380 /// document is complete and the input has ended with nothing but
381 /// whitespace after it, or `NeedInput`.
382 ///
383 /// # Errors
384 ///
385 /// The first syntax error, exceeded budget, or invalid UTF-8, at its
386 /// byte offset. Once an error is returned, every later call returns it.
387 ///
388 /// # Examples
389 ///
390 /// ```
391 /// use jsonx_lib::{ErrorKind, Event, Step, StreamReader};
392 ///
393 /// let mut r = StreamReader::new();
394 /// r.feed(b"[1,")?;
395 /// assert_eq!(r.next_event()?, Step::Ready(Some(Event::StartArray)));
396 /// assert!(r.next_event()?.is_ready()); // 1
397 /// r.feed(b"]")?;
398 /// let err = r.next_event().unwrap_err();
399 /// assert_eq!((err.kind(), err.offset()), (ErrorKind::TrailingComma, 3));
400 /// # Ok::<(), jsonx_lib::Error>(())
401 /// ```
402 pub fn next_event(&mut self) -> Result<Step<Option<Event<'_>>>, Error> {
403 if let Some(e) = self.failed {
404 return Err(e);
405 }
406 let Self {
407 buf,
408 machine,
409 base,
410 last,
411 failed,
412 utf8_error,
413 ..
414 } = self;
415 match machine.step(buf) {
416 Ok(Some(event)) => {
417 *last = *base + machine.last;
418 Ok(Step::Ready(Some(event)))
419 }
420 Ok(None) => match utf8_error {
421 // The valid text holds a whole document, then bad bytes.
422 Some(at) => Err(*failed.insert(Error::new(ErrorKind::InvalidUtf8, *at))),
423 None => Ok(Step::Ready(None)),
424 },
425 Err(e) => halt(failed, machine, buf.len(), *base, *utf8_error, e),
426 }
427 }
428
429 /// Reads the next value whole and returns its exact source text, without
430 /// decoding or allocating anything, as [`Reader::read_raw`](crate::Reader::read_raw)
431 /// does, with the same positioning rules: `Ready(None)` when no value
432 /// comes next.
433 ///
434 /// The value must be in the buffer whole before it can be returned, so
435 /// a large value costs memory for all of it. `NeedInput` keeps the
436 /// progress: the next call continues the same value.
437 ///
438 /// # Errors
439 ///
440 /// As [`next_event`](Self::next_event), for any error inside the value.
441 ///
442 /// # Examples
443 ///
444 /// ```
445 /// use jsonx_lib::{Event, Step, StreamReader};
446 ///
447 /// let mut r = StreamReader::new();
448 /// r.feed(br#"{"id": 1, "params": {"line""#)?;
449 /// assert!(r.next_event()?.is_ready()); // {
450 /// assert!(r.next_event()?.is_ready()); // "id"
451 /// assert_eq!(r.read_raw()?, Step::Ready(Some("1")));
452 /// assert!(r.next_event()?.is_ready()); // "params"
453 /// assert_eq!(r.read_raw()?, Step::NeedInput);
454 /// r.feed(br#": 3}}"#)?;
455 /// assert_eq!(r.read_raw()?, Step::Ready(Some(r#"{"line": 3}"#)));
456 /// # Ok::<(), jsonx_lib::Error>(())
457 /// ```
458 pub fn read_raw(&mut self) -> Result<Step<Option<&str>>, Error> {
459 Ok(self.raw_span()?.map(|span| {
460 span.and_then(|(start, end)| self.buf.get(start - self.base..end - self.base))
461 }))
462 }
463
464 /// Reads the next value whole and builds it as a [`Value`], applying the
465 /// reader's [`DuplicateKeys`] policy; positioning as for
466 /// [`read_raw`](Self::read_raw). The value borrows from the buffer:
467 /// call [`Value::into_owned`] to keep it.
468 ///
469 /// # Errors
470 ///
471 /// As [`read_raw`](Self::read_raw), plus [`ErrorKind::DuplicateKey`]
472 /// under [`DuplicateKeys::Error`].
473 ///
474 /// # Examples
475 ///
476 /// ```
477 /// use jsonx_lib::{Step, StreamReader};
478 ///
479 /// let mut r = StreamReader::new();
480 /// r.feed(br#"{"a": [1, "#)?;
481 /// assert_eq!(r.read_value()?, Step::NeedInput);
482 /// r.feed(br#"2]}"#)?;
483 /// let value = r.read_value()?.ready().flatten().unwrap_or_default();
484 /// assert_eq!(value["a"][1], 2);
485 /// # Ok::<(), jsonx_lib::Error>(())
486 /// ```
487 pub fn read_value(&mut self) -> Result<Step<Option<Value<'_>>>, Error> {
488 if let Some(e) = self.failed {
489 return Err(e);
490 }
491 let Step::Ready(span) = self.raw_span()? else {
492 return Ok(Step::NeedInput);
493 };
494 let Some((start, end)) = span else {
495 return Ok(Step::Ready(None));
496 };
497 let text = self
498 .buf
499 .get(start - self.base..end - self.base)
500 .unwrap_or("");
501 // The budgets, and duplicates under `DuplicateKeys::Error`, were
502 // checked while the value was skipped; the memory budget, and
503 // `First` and `Last`, remain to apply to the tree.
504 let limits = Limits::unlimited().with_max_memory(self.machine.limits.max_memory);
505 let parser = Parser::new()
506 .limits(limits)
507 .duplicate_keys(self.machine.duplicates);
508 match parser.parse(text) {
509 Ok(value) => Ok(Step::Ready(Some(value))),
510 Err(e) => {
511 let e = Error::new(e.kind(), e.offset() + start);
512 self.failed = Some(e);
513 Err(e)
514 }
515 }
516 }
517
518 /// Reads the next value whole as a `T` ([`FromJson`]); positioning as
519 /// for [`read_raw`](Self::read_raw). The value borrows from the buffer,
520 /// as `T` allows.
521 ///
522 /// The value is first skipped (validated, with every budget) as it
523 /// arrives, then read as `T` once it is all there. Byte offsets in an
524 /// error count from the first byte fed.
525 ///
526 /// # Errors
527 ///
528 /// The [`DecodeError`] of `T`, or the reader's error.
529 ///
530 /// # Examples
531 ///
532 /// ```
533 /// use jsonx_lib::{Step, StreamReader};
534 ///
535 /// let mut r = StreamReader::new().documents();
536 /// r.feed(b"[1, 2]\n[3")?;
537 /// assert_eq!(r.decode::<Vec<u8>>()?, Step::Ready(Some(vec![1, 2])));
538 /// assert_eq!(r.decode::<Vec<u8>>()?, Step::NeedInput);
539 /// r.feed(b"]\n")?;
540 /// r.end_input();
541 /// assert_eq!(r.decode::<Vec<u8>>()?, Step::Ready(Some(vec![3])));
542 /// assert_eq!(r.decode::<Vec<u8>>()?, Step::Ready(None));
543 /// # Ok::<(), jsonx_lib::DecodeError>(())
544 /// ```
545 pub fn decode<'s, T: FromJson<'s>>(&'s mut self) -> Result<Step<Option<T>>, DecodeError> {
546 let Step::Ready(span) = self.raw_span()? else {
547 return Ok(Step::NeedInput);
548 };
549 let Some((start, end)) = span else {
550 return Ok(Step::Ready(None));
551 };
552 let text = self
553 .buf
554 .get(start - self.base..end - self.base)
555 .unwrap_or("");
556 // The budgets, and duplicates under `DuplicateKeys::Error`, were
557 // checked while the value was skipped. The depth budget stays, to
558 // bound a recursive `T`, and the memory budget, for what `T` holds.
559 let limits = Limits::unlimited()
560 .with_max_depth(self.machine.limits.max_depth)
561 .with_max_memory(self.machine.limits.max_memory);
562 let parser = Parser::new()
563 .limits(limits)
564 .duplicate_keys(self.machine.duplicates)
565 .numbers(self.machine.numbers);
566 parser
567 .decode(text)
568 .map(|value| Step::Ready(Some(value)))
569 .map_err(|e| e.shifted(start))
570 }
571
572 /// The kind of the next value, without consuming anything: `Ready(None)`
573 /// when no value comes next, or when the next byte cannot start one (the
574 /// next [`next_event`](Self::next_event) reports why).
575 ///
576 /// # Errors
577 ///
578 /// Any error the reader already stopped at.
579 ///
580 /// # Examples
581 ///
582 /// ```
583 /// use jsonx_lib::{Kind, Step, StreamReader};
584 ///
585 /// let mut r = StreamReader::new();
586 /// r.feed(b"[ ")?;
587 /// assert!(r.next_event()?.is_ready());
588 /// assert_eq!(r.peek_kind()?, Step::NeedInput);
589 /// r.feed(b"\"x\"]")?;
590 /// assert_eq!(r.peek_kind()?, Step::Ready(Some(Kind::String)));
591 /// # Ok::<(), jsonx_lib::Error>(())
592 /// ```
593 pub fn peek_kind(&mut self) -> Result<Step<Option<Kind>>, Error> {
594 if let Some(e) = self.failed {
595 return Err(e);
596 }
597 match self.machine.at_value(&self.buf) {
598 Ok(false) => Ok(Step::Ready(None)),
599 Ok(true) => match self.machine.peek_kind(&self.buf) {
600 Some(kind) => Ok(Step::Ready(Some(kind))),
601 // The value's first byte has not arrived.
602 None if self.machine.needed_from() == self.buf.len() && !self.machine.complete => {
603 self.halt(Error::new(ErrorKind::UnexpectedEnd, self.buf.len()))
604 }
605 None => Ok(Step::Ready(None)),
606 },
607 Err(e) => self.halt(e),
608 }
609 }
610
611 /// The number of arrays and objects open.
612 ///
613 /// # Examples
614 ///
615 /// ```
616 /// let mut r = jsonx_lib::StreamReader::new();
617 /// r.feed(b"[[")?;
618 /// let _ = r.next_event()?;
619 /// let _ = r.next_event()?;
620 /// assert_eq!(r.depth(), 2);
621 /// # Ok::<(), jsonx_lib::Error>(())
622 /// ```
623 #[inline]
624 #[must_use]
625 pub fn depth(&self) -> usize {
626 self.machine.depth()
627 }
628
629 /// The byte offset, from the first byte fed, just past the input
630 /// consumed so far.
631 ///
632 /// # Examples
633 ///
634 /// ```
635 /// let mut r = jsonx_lib::StreamReader::new();
636 /// r.feed(b"[10, 20]")?;
637 /// let _ = r.next_event()?;
638 /// let _ = r.next_event()?;
639 /// assert_eq!(r.offset(), 3);
640 /// # Ok::<(), jsonx_lib::Error>(())
641 /// ```
642 #[inline]
643 #[must_use]
644 pub fn offset(&self) -> usize {
645 self.base + self.machine.pos
646 }
647
648 /// The byte offset, from the first byte fed, where the most recent
649 /// event's token began (see [`Reader::event_offset`](crate::Reader::event_offset)).
650 ///
651 /// # Examples
652 ///
653 /// ```
654 /// let mut r = jsonx_lib::StreamReader::new();
655 /// r.feed(b"[ 7]")?;
656 /// let _ = r.next_event()?;
657 /// let _ = r.next_event()?;
658 /// assert_eq!(r.event_offset(), 2);
659 /// # Ok::<(), jsonx_lib::Error>(())
660 /// ```
661 #[inline]
662 #[must_use]
663 pub fn event_offset(&self) -> usize {
664 self.last
665 }
666
667 /// The bytes of input the reader holds: what has been fed and not yet
668 /// dropped. Consumed input is dropped as more is fed.
669 ///
670 /// # Examples
671 ///
672 /// ```
673 /// let mut r = jsonx_lib::StreamReader::new();
674 /// r.feed(b"[1, 2")?;
675 /// assert_eq!(r.buffered(), 5);
676 /// # Ok::<(), jsonx_lib::Error>(())
677 /// ```
678 #[inline]
679 #[must_use]
680 pub fn buffered(&self) -> usize {
681 self.buf.len() + self.tail_len
682 }
683
684 // ---- internals -----------------------------------------------------------
685
686 /// [`halt`] for `self`.
687 fn halt<T>(&mut self, e: Error) -> Result<Step<T>, Error> {
688 halt(
689 &mut self.failed,
690 &self.machine,
691 self.buf.len(),
692 self.base,
693 self.utf8_error,
694 e,
695 )
696 }
697
698 /// Skips the next value (continuing one already started) and returns
699 /// its absolute span, or `None` when no value comes next.
700 fn raw_span(&mut self) -> Result<Step<Option<(usize, usize)>>, Error> {
701 if let Some(e) = self.failed {
702 return Err(e);
703 }
704 let mut progress = match self.raw {
705 Some(progress) => progress,
706 None => match self.machine.at_value(&self.buf) {
707 Ok(false) => return Ok(Step::Ready(None)),
708 Ok(true) => RawProgress {
709 depth: self.machine.depth(),
710 start: None,
711 },
712 Err(e) => return self.halt(e),
713 },
714 };
715 self.machine.skipping = true;
716 let outcome = loop {
717 match self.machine.step(&self.buf) {
718 Ok(Some(_)) => {
719 self.last = self.base + self.machine.last;
720 let start = *progress.start.get_or_insert(self.last);
721 if self.machine.depth() == progress.depth {
722 break Ok(Some((start, self.base + self.machine.pos)));
723 }
724 }
725 // `at_value` saw a value, so the document cannot end here.
726 Ok(None) => break Err(Error::new(ErrorKind::UnexpectedEnd, self.buf.len())),
727 Err(e) => break Err(e),
728 }
729 };
730 self.machine.skipping = false;
731 match outcome {
732 Ok(span) => {
733 self.raw = None;
734 Ok(Step::Ready(span))
735 }
736 Err(e) => {
737 self.raw = Some(progress);
738 self.halt(e)
739 }
740 }
741 }
742
743 /// Drops consumed input from the front of the buffer once there is
744 /// enough of it to be worth the copy (amortized constant per byte).
745 fn compact(&mut self) {
746 let mut keep_from = self.machine.needed_from();
747 if let Some(RawProgress {
748 start: Some(start), ..
749 }) = self.raw
750 {
751 keep_from = keep_from.min(start - self.base);
752 }
753 if keep_from < COMPACT_AT
754 || keep_from * 2 < self.buf.len()
755 || !self.buf.is_char_boundary(keep_from)
756 {
757 return;
758 }
759 let _dropped = self.buf.drain(..keep_from);
760 self.base += keep_from;
761 self.machine.rebase(keep_from);
762 }
763
764 /// Records that the input stops being UTF-8 at the end of the buffer.
765 /// No valid byte can follow, so the text so far is all there will be.
766 fn bad_byte(&mut self) {
767 self.utf8_error = Some(self.base + self.buf.len());
768 self.machine.complete = true;
769 }
770
771 /// Appends `bytes` to the buffer, keeping only valid UTF-8 and holding
772 /// back a character cut off at the end.
773 fn append(&mut self, bytes: &[u8]) {
774 let mut rest = bytes;
775 if self.tail_len > 0 {
776 // Complete the character the last chunk cut off.
777 let take = (4 - self.tail_len).min(rest.len());
778 let (head, after) = rest.split_at(take);
779 if let Some(slot) = self.tail.get_mut(self.tail_len..self.tail_len + take) {
780 slot.copy_from_slice(head);
781 }
782 let have = self.tail_len + take;
783 let pending = self.tail.get(..have).unwrap_or_default();
784 match core::str::from_utf8(pending) {
785 Ok(text) => {
786 self.buf.push_str(text);
787 self.tail_len = 0;
788 rest = after;
789 }
790 Err(e) => {
791 let used = e.valid_up_to();
792 if used > 0 {
793 // The cut character completed; the bytes after it
794 // in `pending` are new input to read normally.
795 let done = pending.get(..used).unwrap_or_default();
796 self.buf.push_str(core::str::from_utf8(done).unwrap_or(""));
797 let extra = used - self.tail_len;
798 self.tail_len = 0;
799 rest = bytes.get(extra..).unwrap_or_default();
800 } else if e.error_len().is_none() && after.is_empty() {
801 // Still incomplete, and this chunk is used up.
802 self.tail_len = have;
803 return;
804 } else {
805 self.bad_byte();
806 return;
807 }
808 }
809 }
810 }
811 match core::str::from_utf8(rest) {
812 Ok(text) => self.buf.push_str(text),
813 Err(e) => {
814 let (valid, invalid) = rest.split_at(e.valid_up_to());
815 self.buf.push_str(core::str::from_utf8(valid).unwrap_or(""));
816 if e.error_len().is_none() {
817 // A character cut off by the end of this chunk.
818 if let Some(slot) = self.tail.get_mut(..invalid.len()) {
819 slot.copy_from_slice(invalid);
820 }
821 self.tail_len = invalid.len();
822 } else {
823 self.bad_byte();
824 }
825 }
826 }
827 }
828}
829
830/// Turns a machine error into a call's answer: `NeedInput` when the buffer
831/// (of `len` bytes, starting at absolute offset `base`) merely ran out,
832/// otherwise the error, made absolute and kept in `failed`.
833fn halt<T>(
834 failed: &mut Option<Error>,
835 machine: &Machine,
836 len: usize,
837 base: usize,
838 utf8_error: Option<usize>,
839 e: Error,
840) -> Result<Step<T>, Error> {
841 let ran_out = e.kind() == ErrorKind::UnexpectedEnd && e.offset() == len;
842 let e = match (ran_out, utf8_error) {
843 (true, _) if !machine.complete => return Ok(Step::NeedInput),
844 // The valid text ends where the input stops being UTF-8.
845 (true, Some(at)) => Error::new(ErrorKind::InvalidUtf8, at),
846 // Wrapping: a repeated key's quote can lie in the dropped front of
847 // the buffer, where its text offset wrapped below zero (see
848 // `Machine::check_object`); adding `base` back gives its exact
849 // offset in the input.
850 _ => Error::new(e.kind(), e.offset().wrapping_add(base)),
851 };
852 *failed = Some(e);
853 Err(e)
854}
855
856impl Parser {
857 /// A [`StreamReader`] with this parser's budgets and duplicate-key
858 /// policy (which [`StreamReader::read_value`] applies).
859 ///
860 /// # Examples
861 ///
862 /// ```
863 /// use jsonx_lib::{ErrorKind, Limits, Parser};
864 ///
865 /// let mut r = Parser::new().limits(Limits::new().with_max_depth(1)).stream_reader();
866 /// r.feed(b"[[]]")?;
867 /// assert!(r.next_event().is_ok());
868 /// assert_eq!(r.next_event().unwrap_err().kind(), ErrorKind::DepthLimitExceeded);
869 /// # Ok::<(), jsonx_lib::Error>(())
870 /// ```
871 #[must_use]
872 pub fn stream_reader(&self) -> StreamReader {
873 let mut reader = StreamReader::with_config(self.get_limits(), self.get_duplicate_keys());
874 reader.machine.numbers = self.get_numbers();
875 reader
876 }
877}
878
879#[cfg(test)]
880mod tests {
881 use super::*;
882
883 #[test]
884 fn test_utf8_split_at_every_byte_of_a_character() {
885 let text = "[\"a\u{e9}\u{20ac}\u{1f600}\"]";
886 for cut in 0..=text.len() {
887 let (a, b) = text.as_bytes().split_at(cut);
888 let mut r = StreamReader::new();
889 assert!(r.feed(a).is_ok() && r.feed(b).is_ok());
890 r.end_input();
891 let _start = r.next_event();
892 let got = r.next_event().ok().and_then(Step::ready).flatten();
893 assert_eq!(
894 got,
895 Some(Event::String("a\u{e9}\u{20ac}\u{1f600}".into())),
896 "cut {cut}"
897 );
898 }
899 // One byte at a time.
900 let mut r = StreamReader::new();
901 for &byte in text.as_bytes() {
902 assert!(r.feed(&[byte]).is_ok());
903 }
904 assert_eq!(r.buffered(), text.len());
905 }
906
907 #[test]
908 fn test_invalid_utf8_is_reported_where_the_reader_reaches_it() {
909 let mut r = StreamReader::new();
910 assert!(r.feed(b"[1, \"a\xC3").is_ok());
911 assert!(r.feed(b"(\"]").is_ok()); // 0xC3 0x28 is not UTF-8
912 let _start = r.next_event();
913 let _one = r.next_event();
914 let err = r.next_event().err();
915 assert_eq!(
916 err.map(|e| (e.kind(), e.offset())),
917 Some((ErrorKind::InvalidUtf8, 6))
918 );
919 // A character cut off by the end of the input.
920 let mut r = StreamReader::new();
921 assert!(r.feed(b"\"\xE2\x82").is_ok());
922 assert_eq!(r.next_event().ok(), Some(Step::NeedInput));
923 r.end_input();
924 let err = r.next_event().err();
925 assert_eq!(
926 err.map(|e| (e.kind(), e.offset())),
927 Some((ErrorKind::InvalidUtf8, 1))
928 );
929 // A syntax error before the bad byte comes first.
930 let mut r = StreamReader::new();
931 assert!(r.feed(b"[1 2, \xFF]").is_ok());
932 let _start = r.next_event();
933 let _one = r.next_event();
934 let err = r.next_event().err();
935 assert_eq!(
936 err.map(|e| (e.kind(), e.offset())),
937 Some((ErrorKind::ExpectedCommaOrBracket, 3))
938 );
939 // An invalid continuation completing a cut character.
940 let mut r = StreamReader::new();
941 assert!(r.feed(b"\"\xE2").is_ok());
942 assert!(r.feed(b"\x41\"").is_ok());
943 let err = r.next_event().err();
944 assert_eq!(
945 err.map(|e| (e.kind(), e.offset())),
946 Some((ErrorKind::InvalidUtf8, 1))
947 );
948 // A cut character completed, with more input after it in the chunk.
949 let mut r = StreamReader::new();
950 assert!(r.feed(b"\"\xE2\x82").is_ok());
951 assert!(r.feed(b"\xACabc\"").is_ok());
952 assert_eq!(
953 r.next_event().ok().and_then(Step::ready).flatten(),
954 Some(Event::String("\u{20ac}abc".into()))
955 );
956 }
957
958 #[test]
959 fn test_feed_after_end_and_errors_latch() {
960 let mut r = StreamReader::new();
961 assert!(r.feed(b"[").is_ok());
962 r.end_input();
963 r.end_input();
964 assert_eq!(
965 r.feed(b"]").err().map(|e| e.kind()),
966 Some(ErrorKind::InputAfterEnd)
967 );
968 let _start = r.next_event();
969 let first = r.next_event().err();
970 assert_eq!(first.map(|e| e.kind()), Some(ErrorKind::UnexpectedEnd));
971 assert_eq!(r.next_event().err(), first);
972 assert_eq!(r.read_raw().err(), first);
973 assert_eq!(r.read_value().err(), first);
974 assert_eq!(r.peek_kind().err(), first);
975 assert_eq!(r.feed(b"x").err(), first);
976 }
977
978 #[test]
979 fn test_compaction_keeps_memory_to_a_chunk_and_offsets_absolute() {
980 let mut r = StreamReader::new();
981 let item = b"12345, ";
982 let mut total = 1;
983 assert!(r.feed(b"[").is_ok());
984 let mut max_buffered = 0;
985 for _ in 0..20_000 {
986 assert!(r.feed(item).is_ok());
987 total += item.len();
988 while let Ok(Step::Ready(Some(_))) = r.next_event() {}
989 max_buffered = max_buffered.max(r.buffered());
990 }
991 assert!(max_buffered < 3 * COMPACT_AT, "{max_buffered}");
992 assert!(r.feed(b"]").is_ok());
993 let err = r.next_event().err();
994 assert_eq!(
995 err.map(|e| (e.kind(), e.offset())),
996 Some((ErrorKind::TrailingComma, total))
997 );
998 }
999
1000 #[test]
1001 fn test_raw_value_spanning_many_chunks_survives_compaction() {
1002 let mut r = StreamReader::new();
1003 assert!(r.feed(b"[0, ").is_ok());
1004 let _start = r.next_event();
1005 assert_eq!(
1006 r.read_raw().ok().map(|s| s.map(|o| o.map(str::len))),
1007 Some(Step::Ready(Some(1)))
1008 );
1009 assert!(r.feed(b"{\"k\": [").is_ok());
1010 assert_eq!(r.read_raw().ok(), Some(Step::NeedInput));
1011 let mut expected = String::from("{\"k\": [");
1012 for i in 0..5_000 {
1013 let piece = alloc::format!("{i}, ");
1014 expected.push_str(&piece);
1015 assert!(r.feed(piece.as_bytes()).is_ok());
1016 assert_eq!(r.read_raw().ok(), Some(Step::NeedInput));
1017 }
1018 assert!(r.feed(b"0]}]").is_ok());
1019 expected.push_str("0]}");
1020 let raw = r
1021 .read_raw()
1022 .ok()
1023 .and_then(Step::ready)
1024 .flatten()
1025 .map(String::from);
1026 assert_eq!(raw.as_deref(), Some(expected.as_str()));
1027 assert_eq!(r.event_offset(), 4 + expected.len() - 1);
1028 }
1029}