use alloc::string::String;
use crate::{
DecodeError, DuplicateKeys, FromJson, Kind, Limits, Parser, Value,
error::{Error, ErrorKind},
machine::Machine,
reader::Event,
};
#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash)]
#[must_use]
pub enum Step<T> {
Ready(T),
NeedInput,
}
impl<T> Step<T> {
#[inline]
pub fn ready(self) -> Option<T> {
match self {
Self::Ready(value) => Some(value),
Self::NeedInput => None,
}
}
#[inline]
#[must_use]
pub const fn is_ready(&self) -> bool {
matches!(self, Self::Ready(_))
}
#[inline]
pub fn map<U>(self, f: impl FnOnce(T) -> U) -> Step<U> {
match self {
Self::Ready(value) => Step::Ready(f(value)),
Self::NeedInput => Step::NeedInput,
}
}
}
#[derive(Clone, Copy, Debug)]
struct RawProgress {
depth: usize,
start: Option<usize>,
}
const COMPACT_AT: usize = 4 * 1024;
#[derive(Clone, Debug)]
pub struct StreamReader {
buf: String,
base: usize,
tail: [u8; 4],
tail_len: usize,
utf8_error: Option<usize>,
fed: usize,
ended: bool,
machine: Machine,
failed: Option<Error>,
last: usize,
raw: Option<RawProgress>,
}
impl Default for StreamReader {
fn default() -> Self {
Self::new()
}
}
impl StreamReader {
#[must_use]
pub fn new() -> Self {
Self::with_config(Limits::new(), DuplicateKeys::Error)
}
pub(crate) fn with_config(limits: Limits, duplicates: DuplicateKeys) -> Self {
Self {
buf: String::new(),
base: 0,
tail: [0; 4],
tail_len: 0,
utf8_error: None,
fed: 0,
ended: false,
machine: {
let mut machine = Machine::new(limits, duplicates, false);
machine.copy_keys = true;
machine
},
failed: None,
last: 0,
raw: None,
}
}
#[must_use]
pub fn documents(mut self) -> Self {
self.machine.documents = true;
self
}
pub fn feed(&mut self, bytes: &[u8]) -> Result<(), Error> {
if let Some(e) = self.failed {
return Err(e);
}
if self.ended {
return Err(Error::new(ErrorKind::InputAfterEnd, self.fed));
}
let start = self.machine.document_start();
let max = self.machine.limits.max_input_bytes;
if bytes.len() > max.saturating_sub(self.fed.saturating_sub(start)) {
let e = Error::new(ErrorKind::InputTooLarge, start.saturating_add(max));
return Err(*self.failed.insert(e));
}
if let Some(progress) = self.raw {
let start = progress
.start
.unwrap_or(self.base + self.machine.needed_from());
let max = self.machine.limits.max_value_bytes;
if bytes.len() > max.saturating_sub(self.fed.saturating_sub(start)) {
let e = Error::new(ErrorKind::InputTooLarge, start.saturating_add(max));
return Err(*self.failed.insert(e));
}
}
self.fed += bytes.len();
self.compact();
if self.utf8_error.is_none() {
self.append(bytes);
}
Ok(())
}
pub fn end_input(&mut self) {
if self.ended {
return;
}
self.ended = true;
if self.tail_len > 0 && self.utf8_error.is_none() {
self.utf8_error = Some(self.base + self.buf.len());
}
self.machine.complete = true;
}
#[cfg(feature = "std")]
#[cfg_attr(docsrs, doc(cfg(feature = "std")))]
pub fn read_from(&mut self, source: &mut dyn std::io::Read) -> std::io::Result<usize> {
let mut chunk = [0u8; 8 * 1024];
loop {
match source.read(&mut chunk) {
Ok(0) => {
self.end_input();
return Ok(0);
}
Ok(n) => {
let _kept = self.feed(chunk.get(..n).unwrap_or_default());
return Ok(n);
}
Err(e) if e.kind() == std::io::ErrorKind::Interrupted => {}
Err(e) => return Err(e),
}
}
}
pub fn next_event(&mut self) -> Result<Step<Option<Event<'_>>>, Error> {
if let Some(e) = self.failed {
return Err(e);
}
let Self {
buf,
machine,
base,
last,
failed,
utf8_error,
..
} = self;
match machine.step(buf) {
Ok(Some(event)) => {
*last = *base + machine.last;
Ok(Step::Ready(Some(event)))
}
Ok(None) => match utf8_error {
Some(at) => Err(*failed.insert(Error::new(ErrorKind::InvalidUtf8, *at))),
None => Ok(Step::Ready(None)),
},
Err(e) => halt(failed, machine, buf.len(), *base, *utf8_error, e),
}
}
pub fn read_raw(&mut self) -> Result<Step<Option<&str>>, Error> {
Ok(self.raw_span()?.map(|span| {
span.and_then(|(start, end)| self.buf.get(start - self.base..end - self.base))
}))
}
pub fn read_value(&mut self) -> Result<Step<Option<Value<'_>>>, Error> {
if let Some(e) = self.failed {
return Err(e);
}
let Step::Ready(span) = self.raw_span()? else {
return Ok(Step::NeedInput);
};
let Some((start, end)) = span else {
return Ok(Step::Ready(None));
};
let text = self
.buf
.get(start - self.base..end - self.base)
.unwrap_or("");
let limits = Limits::unlimited().with_max_memory(self.machine.limits.max_memory);
let parser = Parser::new()
.limits(limits)
.duplicate_keys(self.machine.duplicates);
match parser.parse(text) {
Ok(value) => Ok(Step::Ready(Some(value))),
Err(e) => {
let e = Error::new(e.kind(), e.offset() + start);
self.failed = Some(e);
Err(e)
}
}
}
pub fn decode<'s, T: FromJson<'s>>(&'s mut self) -> Result<Step<Option<T>>, DecodeError> {
let Step::Ready(span) = self.raw_span()? else {
return Ok(Step::NeedInput);
};
let Some((start, end)) = span else {
return Ok(Step::Ready(None));
};
let text = self
.buf
.get(start - self.base..end - self.base)
.unwrap_or("");
let limits = Limits::unlimited()
.with_max_depth(self.machine.limits.max_depth)
.with_max_memory(self.machine.limits.max_memory);
let parser = Parser::new()
.limits(limits)
.duplicate_keys(self.machine.duplicates)
.numbers(self.machine.numbers);
parser
.decode(text)
.map(|value| Step::Ready(Some(value)))
.map_err(|e| e.shifted(start))
}
pub fn finish(&mut self) -> Result<Step<()>, Error> {
self.raw = None;
loop {
match self.next_event()? {
Step::Ready(Some(_)) => {}
Step::Ready(None) => return Ok(Step::Ready(())),
Step::NeedInput => return Ok(Step::NeedInput),
}
}
}
pub fn peek_kind(&mut self) -> Result<Step<Option<Kind>>, Error> {
if let Some(e) = self.failed {
return Err(e);
}
match self.machine.at_value(&self.buf) {
Ok(false) => Ok(Step::Ready(None)),
Ok(true) => match self.machine.peek_kind(&self.buf) {
Some(kind) => Ok(Step::Ready(Some(kind))),
None if self.machine.needed_from() == self.buf.len() && !self.machine.complete => {
self.halt(Error::new(ErrorKind::UnexpectedEnd, self.buf.len()))
}
None => Ok(Step::Ready(None)),
},
Err(e) => self.halt(e),
}
}
#[inline]
#[must_use]
pub fn depth(&self) -> usize {
self.machine.depth()
}
#[inline]
#[must_use]
pub fn offset(&self) -> usize {
self.base + self.machine.pos
}
#[inline]
#[must_use]
pub fn event_offset(&self) -> usize {
self.last
}
#[inline]
#[must_use]
pub fn buffered(&self) -> usize {
self.buf.len() + self.tail_len
}
fn halt<T>(&mut self, e: Error) -> Result<Step<T>, Error> {
halt(
&mut self.failed,
&self.machine,
self.buf.len(),
self.base,
self.utf8_error,
e,
)
}
fn raw_span(&mut self) -> Result<Step<Option<(usize, usize)>>, Error> {
if let Some(e) = self.failed {
return Err(e);
}
let mut progress = match self.raw {
Some(progress) => progress,
None => match self.machine.at_value(&self.buf) {
Ok(false) => {
return match self.machine.check_trailing(&self.buf) {
Ok(()) => Ok(Step::Ready(None)),
Err(e) => self.halt(e),
};
}
Ok(true) => RawProgress {
depth: self.machine.depth(),
start: None,
},
Err(e) => return self.halt(e),
},
};
self.machine.skipping = true;
let outcome = loop {
match self.machine.step(&self.buf) {
Ok(Some(_)) => {
self.last = self.base + self.machine.last;
let start = *progress.start.get_or_insert(self.last);
if self.machine.depth() == progress.depth {
break Ok(Some((start, self.base + self.machine.pos)));
}
}
Ok(None) => break Err(Error::new(ErrorKind::UnexpectedEnd, self.buf.len())),
Err(e) => break Err(e),
}
};
self.machine.skipping = false;
match outcome {
Ok(span) => {
self.raw = None;
Ok(Step::Ready(span))
}
Err(e) => {
self.raw = Some(progress);
self.halt(e)
}
}
}
fn compact(&mut self) {
let mut keep_from = self.machine.needed_from();
if let Some(RawProgress {
start: Some(start), ..
}) = self.raw
{
keep_from = keep_from.min(start - self.base);
}
if keep_from < COMPACT_AT
|| keep_from * 2 < self.buf.len()
|| !self.buf.is_char_boundary(keep_from)
{
return;
}
let _dropped = self.buf.drain(..keep_from);
self.base += keep_from;
self.machine.rebase(keep_from);
}
fn bad_byte(&mut self) {
self.utf8_error = Some(self.base + self.buf.len());
self.machine.complete = true;
}
fn append(&mut self, bytes: &[u8]) {
let mut rest = bytes;
if self.tail_len > 0 {
let take = (4 - self.tail_len).min(rest.len());
let (head, after) = rest.split_at(take);
if let Some(slot) = self.tail.get_mut(self.tail_len..self.tail_len + take) {
slot.copy_from_slice(head);
}
let have = self.tail_len + take;
let pending = self.tail.get(..have).unwrap_or_default();
match core::str::from_utf8(pending) {
Ok(text) => {
self.buf.push_str(text);
self.tail_len = 0;
rest = after;
}
Err(e) => {
let used = e.valid_up_to();
if used > 0 {
let done = pending.get(..used).unwrap_or_default();
self.buf.push_str(core::str::from_utf8(done).unwrap_or(""));
let extra = used - self.tail_len;
self.tail_len = 0;
rest = bytes.get(extra..).unwrap_or_default();
} else if e.error_len().is_none() && after.is_empty() {
self.tail_len = have;
return;
} else {
self.bad_byte();
return;
}
}
}
}
match core::str::from_utf8(rest) {
Ok(text) => self.buf.push_str(text),
Err(e) => {
let (valid, invalid) = rest.split_at(e.valid_up_to());
self.buf.push_str(core::str::from_utf8(valid).unwrap_or(""));
if e.error_len().is_none() {
if let Some(slot) = self.tail.get_mut(..invalid.len()) {
slot.copy_from_slice(invalid);
}
self.tail_len = invalid.len();
} else {
self.bad_byte();
}
}
}
}
}
fn halt<T>(
failed: &mut Option<Error>,
machine: &Machine,
len: usize,
base: usize,
utf8_error: Option<usize>,
e: Error,
) -> Result<Step<T>, Error> {
let ran_out = e.kind() == ErrorKind::UnexpectedEnd && e.offset() == len;
let e = match (ran_out, utf8_error) {
(true, _) if !machine.complete => return Ok(Step::NeedInput),
(true, Some(at)) => Error::new(ErrorKind::InvalidUtf8, at),
_ => Error::new(e.kind(), e.offset().wrapping_add(base)),
};
*failed = Some(e);
Err(e)
}
impl Parser {
#[must_use]
pub fn stream_reader(&self) -> StreamReader {
let mut reader = StreamReader::with_config(self.get_limits(), self.get_duplicate_keys());
reader.machine.numbers = self.get_numbers();
reader
}
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn test_utf8_split_at_every_byte_of_a_character() {
let text = "[\"a\u{e9}\u{20ac}\u{1f600}\"]";
for cut in 0..=text.len() {
let (a, b) = text.as_bytes().split_at(cut);
let mut r = StreamReader::new();
assert!(r.feed(a).is_ok() && r.feed(b).is_ok());
r.end_input();
let _start = r.next_event();
let got = r.next_event().ok().and_then(Step::ready).flatten();
assert_eq!(
got,
Some(Event::String("a\u{e9}\u{20ac}\u{1f600}".into())),
"cut {cut}"
);
}
let mut r = StreamReader::new();
for &byte in text.as_bytes() {
assert!(r.feed(&[byte]).is_ok());
}
assert_eq!(r.buffered(), text.len());
}
#[test]
fn test_invalid_utf8_is_reported_where_the_reader_reaches_it() {
let mut r = StreamReader::new();
assert!(r.feed(b"[1, \"a\xC3").is_ok());
assert!(r.feed(b"(\"]").is_ok()); let _start = r.next_event();
let _one = r.next_event();
let err = r.next_event().err();
assert_eq!(
err.map(|e| (e.kind(), e.offset())),
Some((ErrorKind::InvalidUtf8, 6))
);
let mut r = StreamReader::new();
assert!(r.feed(b"\"\xE2\x82").is_ok());
assert_eq!(r.next_event().ok(), Some(Step::NeedInput));
r.end_input();
let err = r.next_event().err();
assert_eq!(
err.map(|e| (e.kind(), e.offset())),
Some((ErrorKind::InvalidUtf8, 1))
);
let mut r = StreamReader::new();
assert!(r.feed(b"[1 2, \xFF]").is_ok());
let _start = r.next_event();
let _one = r.next_event();
let err = r.next_event().err();
assert_eq!(
err.map(|e| (e.kind(), e.offset())),
Some((ErrorKind::ExpectedCommaOrBracket, 3))
);
let mut r = StreamReader::new();
assert!(r.feed(b"\"\xE2").is_ok());
assert!(r.feed(b"\x41\"").is_ok());
let err = r.next_event().err();
assert_eq!(
err.map(|e| (e.kind(), e.offset())),
Some((ErrorKind::InvalidUtf8, 1))
);
let mut r = StreamReader::new();
assert!(r.feed(b"\"\xE2\x82").is_ok());
assert!(r.feed(b"\xACabc\"").is_ok());
assert_eq!(
r.next_event().ok().and_then(Step::ready).flatten(),
Some(Event::String("\u{20ac}abc".into()))
);
}
#[test]
fn test_feed_after_end_and_errors_latch() {
let mut r = StreamReader::new();
assert!(r.feed(b"[").is_ok());
r.end_input();
r.end_input();
assert_eq!(
r.feed(b"]").err().map(|e| e.kind()),
Some(ErrorKind::InputAfterEnd)
);
let _start = r.next_event();
let first = r.next_event().err();
assert_eq!(first.map(|e| e.kind()), Some(ErrorKind::UnexpectedEnd));
assert_eq!(r.next_event().err(), first);
assert_eq!(r.read_raw().err(), first);
assert_eq!(r.read_value().err(), first);
assert_eq!(r.peek_kind().err(), first);
assert_eq!(r.feed(b"x").err(), first);
}
#[test]
fn test_compaction_keeps_memory_to_a_chunk_and_offsets_absolute() {
let mut r = StreamReader::new();
let item = b"12345, ";
let mut total = 1;
assert!(r.feed(b"[").is_ok());
let mut max_buffered = 0;
for _ in 0..20_000 {
assert!(r.feed(item).is_ok());
total += item.len();
while let Ok(Step::Ready(Some(_))) = r.next_event() {}
max_buffered = max_buffered.max(r.buffered());
}
assert!(max_buffered < 3 * COMPACT_AT, "{max_buffered}");
assert!(r.feed(b"]").is_ok());
let err = r.next_event().err();
assert_eq!(
err.map(|e| (e.kind(), e.offset())),
Some((ErrorKind::TrailingComma, total))
);
}
#[test]
fn test_raw_value_spanning_many_chunks_survives_compaction() {
let mut r = StreamReader::new();
assert!(r.feed(b"[0, ").is_ok());
let _start = r.next_event();
assert_eq!(
r.read_raw().ok().map(|s| s.map(|o| o.map(str::len))),
Some(Step::Ready(Some(1)))
);
assert!(r.feed(b"{\"k\": [").is_ok());
assert_eq!(r.read_raw().ok(), Some(Step::NeedInput));
let mut expected = String::from("{\"k\": [");
for i in 0..5_000 {
let piece = alloc::format!("{i}, ");
expected.push_str(&piece);
assert!(r.feed(piece.as_bytes()).is_ok());
assert_eq!(r.read_raw().ok(), Some(Step::NeedInput));
}
assert!(r.feed(b"0]}]").is_ok());
expected.push_str("0]}");
let raw = r
.read_raw()
.ok()
.and_then(Step::ready)
.flatten()
.map(String::from);
assert_eq!(raw.as_deref(), Some(expected.as_str()));
assert_eq!(r.event_offset(), 4 + expected.len() - 1);
}
}