use crate::escape_processor::{EscapeProcessor, UnicodeEscapeCollector};
use crate::shared::{ContentRange, Event, ParserState, State, UnexpectedState};
use crate::ujson::{EventToken, Tokenizer};
use crate::{ujson, ParseError};
pub struct ParserCore<T: ujson::BitBucket, C: ujson::DepthCounter> {
pub tokenizer: Tokenizer<T, C>,
pub parser_state: ParserState,
in_escape_sequence: bool,
handles_chunked_input: bool,
}
impl<T: ujson::BitBucket, C: ujson::DepthCounter> ParserCore<T, C> {
pub fn new() -> Self {
Self {
tokenizer: Tokenizer::new(),
parser_state: ParserState::new(),
in_escape_sequence: false,
handles_chunked_input: false,
}
}
pub fn new_chunked() -> Self {
Self {
tokenizer: Tokenizer::new(),
parser_state: ParserState::new(),
in_escape_sequence: false,
handles_chunked_input: true,
}
}
pub fn next_event_impl<'a, P, F>(
&mut self,
provider: &'a mut P,
escape_timing: EscapeTiming,
byte_accumulator: F,
) -> Result<Event<'a, 'a>, ParseError>
where
P: ContentExtractor,
F: FnMut(&mut P, u8) -> Result<(), ParseError>,
{
self.next_event_impl_with_flags(provider, escape_timing, byte_accumulator, false)
}
pub fn next_event_impl_with_flags<'a, P, F>(
&mut self,
provider: &'a mut P,
escape_timing: EscapeTiming,
mut byte_accumulator: F,
always_accumulate_during_escapes: bool,
) -> Result<Event<'a, 'a>, ParseError>
where
P: ContentExtractor,
F: FnMut(&mut P, u8) -> Result<(), ParseError>,
{
loop {
while !have_events(&self.parser_state.evts) {
if let Some(byte) = provider.next_byte()? {
{
clear_events(&mut self.parser_state.evts);
let mut callback = create_tokenizer_callback(&mut self.parser_state.evts);
self.tokenizer
.parse_chunk(&[byte], &mut callback)
.map_err(ParseError::TokenizerError)?;
}
let should_accumulate = if always_accumulate_during_escapes {
if self.in_escape_sequence {
true } else {
!have_events(&self.parser_state.evts) }
} else {
!have_events(&self.parser_state.evts) && !self.in_escape_sequence
};
if should_accumulate {
byte_accumulator(provider, byte)?;
}
} else {
if self.handles_chunked_input {
return Err(ParseError::EndOfData);
} else {
{
let mut finish_callback =
create_tokenizer_callback(&mut self.parser_state.evts);
let _bytes_processed = self.tokenizer.finish(&mut finish_callback)?;
}
if !have_events(&self.parser_state.evts) {
return Ok(Event::EndDocument);
}
}
}
}
let taken_event = take_first_event(&mut self.parser_state.evts);
let Some(taken) = taken_event else {
return Err(UnexpectedState::StateMismatch.into());
};
if let Some(result) =
process_simple_events(&taken).or_else(|| provider.process_begin_events(&taken))
{
match result {
EventResult::Complete(event) => return Ok(event),
EventResult::ExtractString => return provider.validate_and_extract_string(),
EventResult::ExtractKey => return provider.validate_and_extract_key(),
EventResult::ExtractNumber(from_container_end) => {
return provider.validate_and_extract_number(from_container_end)
}
EventResult::Continue => continue,
}
}
match taken {
ujson::Event::Begin(EventToken::EscapeSequence) => {
self.in_escape_sequence = true;
provider.process_begin_escape_sequence_event()?;
}
ujson::Event::Begin(EventToken::UnicodeEscape) => {
self.in_escape_sequence = true;
provider.process_unicode_escape_events(&taken)?;
}
ujson::Event::End(EventToken::UnicodeEscape) => {
self.in_escape_sequence = false;
provider.process_unicode_escape_events(&taken)?;
}
ujson::Event::Begin(
escape_token @ (EventToken::EscapeQuote
| EventToken::EscapeBackslash
| EventToken::EscapeSlash
| EventToken::EscapeBackspace
| EventToken::EscapeFormFeed
| EventToken::EscapeNewline
| EventToken::EscapeCarriageReturn
| EventToken::EscapeTab),
) if escape_timing == EscapeTiming::OnBegin => {
self.in_escape_sequence = true;
provider.process_simple_escape_event(&escape_token)?;
self.in_escape_sequence = false;
}
ujson::Event::End(
escape_token @ (EventToken::EscapeQuote
| EventToken::EscapeBackslash
| EventToken::EscapeSlash
| EventToken::EscapeBackspace
| EventToken::EscapeFormFeed
| EventToken::EscapeNewline
| EventToken::EscapeCarriageReturn
| EventToken::EscapeTab),
) if escape_timing == EscapeTiming::OnEnd => {
provider.process_simple_escape_event(&escape_token)?;
self.in_escape_sequence = false;
}
_ => {
}
}
}
}
}
impl<T: ujson::BitBucket, C: ujson::DepthCounter> Default for ParserCore<T, C> {
fn default() -> Self {
Self::new()
}
}
#[derive(Debug, Clone, Copy, PartialEq)]
pub enum EscapeTiming {
OnBegin,
OnEnd,
}
#[derive(Debug)]
pub enum EventResult<'a, 'b> {
Complete(Event<'a, 'b>),
Continue,
ExtractString,
ExtractKey,
ExtractNumber(bool),
}
pub trait ContentExtractor {
fn next_byte(&mut self) -> Result<Option<u8>, ParseError>;
fn current_position(&self) -> usize;
fn begin_string_content(&mut self, pos: usize);
fn parser_state_mut(&mut self) -> &mut State;
fn unicode_escape_collector_mut(&mut self) -> &mut UnicodeEscapeCollector;
fn extract_string_content(&mut self, start_pos: usize) -> Result<Event<'_, '_>, ParseError>;
fn extract_key_content(&mut self, start_pos: usize) -> Result<Event<'_, '_>, ParseError>;
fn extract_number(
&mut self,
start_pos: usize,
from_container_end: bool,
finished: bool,
) -> Result<Event<'_, '_>, ParseError>;
fn validate_and_extract_string(&mut self) -> Result<Event<'_, '_>, ParseError> {
let start_pos = match *self.parser_state() {
State::String(pos) => pos,
_ => return Err(crate::shared::UnexpectedState::StateMismatch.into()),
};
if self
.unicode_escape_collector_mut()
.has_pending_high_surrogate()
{
return Err(ParseError::InvalidUnicodeCodepoint);
}
*self.parser_state_mut() = State::None;
self.extract_string_content(start_pos)
}
fn validate_and_extract_key(&mut self) -> Result<Event<'_, '_>, ParseError> {
let start_pos = match *self.parser_state() {
State::Key(pos) => pos,
_ => return Err(crate::shared::UnexpectedState::StateMismatch.into()),
};
if self
.unicode_escape_collector_mut()
.has_pending_high_surrogate()
{
return Err(ParseError::InvalidUnicodeCodepoint);
}
*self.parser_state_mut() = State::None;
self.extract_key_content(start_pos)
}
fn is_finished(&self) -> bool {
true
}
fn validate_and_extract_number(
&mut self,
from_container_end: bool,
) -> Result<Event<'_, '_>, ParseError> {
let start_pos = match *self.parser_state() {
State::Number(pos) => pos,
_ => return Err(crate::shared::UnexpectedState::StateMismatch.into()),
};
*self.parser_state_mut() = State::None;
self.extract_number(start_pos, from_container_end, self.is_finished())
}
fn parser_state(&self) -> &State;
fn process_unicode_escape_with_collector(&mut self) -> Result<(), ParseError>;
fn handle_simple_escape_char(&mut self, escape_char: u8) -> Result<(), ParseError>;
fn begin_escape_sequence(&mut self) -> Result<(), ParseError>;
fn begin_unicode_escape(&mut self) -> Result<(), ParseError>;
fn process_begin_events(
&mut self,
event: &ujson::Event,
) -> Option<EventResult<'static, 'static>> {
match event {
ujson::Event::Begin(EventToken::Key) => {
let pos = self.current_position();
*self.parser_state_mut() = State::Key(pos);
self.begin_string_content(pos);
Some(EventResult::Continue)
}
ujson::Event::Begin(EventToken::String) => {
let pos = self.current_position();
*self.parser_state_mut() = State::String(pos);
self.begin_string_content(pos);
Some(EventResult::Continue)
}
ujson::Event::Begin(
EventToken::Number | EventToken::NumberAndArray | EventToken::NumberAndObject,
) => {
let pos = self.current_position();
let number_start = ContentRange::number_start_from_current(pos);
*self.parser_state_mut() = State::Number(number_start);
Some(EventResult::Continue)
}
ujson::Event::Begin(EventToken::True | EventToken::False | EventToken::Null) => {
Some(EventResult::Continue)
}
_ => None,
}
}
fn process_begin_escape_sequence_event(&mut self) -> Result<(), ParseError> {
match self.parser_state() {
State::String(_) | State::Key(_) => {
self.begin_escape_sequence()?;
}
_ => {} }
Ok(())
}
fn process_simple_escape_event(&mut self, escape_token: &EventToken) -> Result<(), ParseError> {
self.unicode_escape_collector_mut().reset_all();
let unescaped_char = EscapeProcessor::process_escape_token(escape_token)?;
match self.parser_state() {
State::String(_) | State::Key(_) => {
self.handle_simple_escape_char(unescaped_char)?;
}
_ => {} }
Ok(())
}
fn process_unicode_escape_events(&mut self, event: &ujson::Event) -> Result<bool, ParseError> {
match event {
ujson::Event::Begin(EventToken::UnicodeEscape) => {
match self.parser_state() {
State::String(_) | State::Key(_) => {
self.unicode_escape_collector_mut().reset();
self.begin_unicode_escape()?;
}
_ => {} }
Ok(true) }
ujson::Event::End(EventToken::UnicodeEscape) => {
match self.parser_state() {
State::String(_) | State::Key(_) => {
self.process_unicode_escape_with_collector()?;
}
_ => {} }
Ok(true) }
_ => Ok(false), }
}
}
pub fn clear_events(event_storage: &mut [Option<ujson::Event>; 2]) {
event_storage[0] = None;
event_storage[1] = None;
}
pub fn create_tokenizer_callback(
event_storage: &mut [Option<ujson::Event>; 2],
) -> impl FnMut(ujson::Event, usize) + '_ {
|event, _len| {
for evt in event_storage.iter_mut() {
if evt.is_none() {
*evt = Some(event);
return;
}
}
}
}
pub fn have_events(event_storage: &[Option<ujson::Event>; 2]) -> bool {
event_storage.iter().any(|evt| evt.is_some())
}
pub fn take_first_event(event_storage: &mut [Option<ujson::Event>; 2]) -> Option<ujson::Event> {
event_storage.iter_mut().find_map(|e| e.take())
}
pub fn process_simple_events(event: &ujson::Event) -> Option<EventResult<'static, 'static>> {
match event {
ujson::Event::ObjectStart => Some(EventResult::Complete(Event::StartObject)),
ujson::Event::ObjectEnd => Some(EventResult::Complete(Event::EndObject)),
ujson::Event::ArrayStart => Some(EventResult::Complete(Event::StartArray)),
ujson::Event::ArrayEnd => Some(EventResult::Complete(Event::EndArray)),
ujson::Event::End(EventToken::True) => Some(EventResult::Complete(Event::Bool(true))),
ujson::Event::End(EventToken::False) => Some(EventResult::Complete(Event::Bool(false))),
ujson::Event::End(EventToken::Null) => Some(EventResult::Complete(Event::Null)),
ujson::Event::End(EventToken::String) => Some(EventResult::ExtractString),
ujson::Event::End(EventToken::Key) => Some(EventResult::ExtractKey),
ujson::Event::End(EventToken::Number) => Some(EventResult::ExtractNumber(false)),
ujson::Event::End(EventToken::NumberAndArray) => Some(EventResult::ExtractNumber(true)),
ujson::Event::End(EventToken::NumberAndObject) => Some(EventResult::ExtractNumber(true)),
_ => None,
}
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn test_container_events() {
assert!(matches!(
process_simple_events(&ujson::Event::ObjectStart),
Some(EventResult::Complete(Event::StartObject))
));
assert!(matches!(
process_simple_events(&ujson::Event::ArrayEnd),
Some(EventResult::Complete(Event::EndArray))
));
}
#[test]
fn test_primitive_events() {
assert!(matches!(
process_simple_events(&ujson::Event::End(EventToken::True)),
Some(EventResult::Complete(Event::Bool(true)))
));
assert!(matches!(
process_simple_events(&ujson::Event::End(EventToken::Null)),
Some(EventResult::Complete(Event::Null))
));
}
#[test]
fn test_extraction_triggers() {
assert!(matches!(
process_simple_events(&ujson::Event::End(EventToken::String)),
Some(EventResult::ExtractString)
));
assert!(matches!(
process_simple_events(&ujson::Event::End(EventToken::Number)),
Some(EventResult::ExtractNumber(false))
));
assert!(matches!(
process_simple_events(&ujson::Event::End(EventToken::NumberAndArray)),
Some(EventResult::ExtractNumber(true))
));
}
#[test]
fn test_complex_events_not_handled() {
assert!(process_simple_events(&ujson::Event::Begin(EventToken::String)).is_none());
assert!(process_simple_events(&ujson::Event::Begin(EventToken::EscapeQuote)).is_none());
}
struct MockContentExtractor {
position: usize,
state: State,
string_begin_calls: Vec<usize>,
}
impl MockContentExtractor {
fn new() -> Self {
Self {
position: 42,
state: State::None,
string_begin_calls: Vec::new(),
}
}
}
impl ContentExtractor for MockContentExtractor {
fn next_byte(&mut self) -> Result<Option<u8>, ParseError> {
Ok(None)
}
fn current_position(&self) -> usize {
self.position
}
fn begin_string_content(&mut self, pos: usize) {
self.string_begin_calls.push(pos);
}
fn parser_state_mut(&mut self) -> &mut State {
&mut self.state
}
fn unicode_escape_collector_mut(&mut self) -> &mut UnicodeEscapeCollector {
unimplemented!("Mock doesn't need unicode collector")
}
fn extract_string_content(
&mut self,
_start_pos: usize,
) -> Result<Event<'_, '_>, ParseError> {
unimplemented!("Mock doesn't need extraction")
}
fn extract_key_content(&mut self, _start_pos: usize) -> Result<Event<'_, '_>, ParseError> {
unimplemented!("Mock doesn't need extraction")
}
fn extract_number(
&mut self,
_start_pos: usize,
_from_container_end: bool,
_finished: bool,
) -> Result<Event<'_, '_>, ParseError> {
unimplemented!("Mock doesn't need extraction")
}
fn parser_state(&self) -> &State {
&self.state
}
fn process_unicode_escape_with_collector(&mut self) -> Result<(), ParseError> {
Ok(())
}
fn handle_simple_escape_char(&mut self, _escape_char: u8) -> Result<(), ParseError> {
Ok(())
}
fn begin_unicode_escape(&mut self) -> Result<(), ParseError> {
Ok(())
}
fn begin_escape_sequence(&mut self) -> Result<(), ParseError> {
Ok(())
}
}
#[test]
fn test_begin_events_key() {
let mut context = MockContentExtractor::new();
let event = ujson::Event::Begin(EventToken::Key);
let result = context.process_begin_events(&event);
assert!(matches!(result, Some(EventResult::Continue)));
assert!(matches!(context.state, State::Key(42)));
assert_eq!(context.string_begin_calls, vec![42]);
}
#[test]
fn test_begin_events_string() {
let mut context = MockContentExtractor::new();
let event = ujson::Event::Begin(EventToken::String);
let result = context.process_begin_events(&event);
assert!(matches!(result, Some(EventResult::Continue)));
assert!(matches!(context.state, State::String(42)));
assert_eq!(context.string_begin_calls, vec![42]);
}
#[test]
fn test_begin_events_number() {
let mut context = MockContentExtractor::new();
let event = ujson::Event::Begin(EventToken::Number);
let result = context.process_begin_events(&event);
assert!(matches!(result, Some(EventResult::Continue)));
assert!(matches!(context.state, State::Number(_)));
assert_eq!(context.string_begin_calls, Vec::<usize>::new()); }
#[test]
fn test_begin_events_primitives() {
let mut context = MockContentExtractor::new();
for token in [EventToken::True, EventToken::False, EventToken::Null] {
let event = ujson::Event::Begin(token);
let result = context.process_begin_events(&event);
assert!(matches!(result, Some(EventResult::Continue)));
}
assert!(matches!(context.state, State::None));
assert!(context.string_begin_calls.is_empty());
}
#[test]
fn test_begin_events_not_handled() {
let mut context = MockContentExtractor::new();
let event = ujson::Event::Begin(EventToken::EscapeQuote);
let result = context.process_begin_events(&event);
assert!(result.is_none());
assert!(matches!(context.state, State::None));
assert!(context.string_begin_calls.is_empty());
}
#[test]
fn test_tokenizer_callback() {
let mut event_storage = [None, None];
assert!(!have_events(&event_storage));
{
let mut callback = create_tokenizer_callback(&mut event_storage);
callback(ujson::Event::ObjectStart, 1);
}
assert!(have_events(&event_storage));
assert!(event_storage[0].is_some());
assert!(event_storage[1].is_none());
{
let mut callback = create_tokenizer_callback(&mut event_storage);
callback(ujson::Event::ArrayStart, 1);
}
assert!(event_storage[0].is_some());
assert!(event_storage[1].is_some());
{
let mut callback = create_tokenizer_callback(&mut event_storage);
callback(ujson::Event::ObjectEnd, 1);
}
assert!(event_storage[0].is_some());
assert!(event_storage[1].is_some());
}
#[test]
fn test_event_extraction() {
let mut event_storage = [
Some(ujson::Event::ObjectStart),
Some(ujson::Event::ArrayStart),
];
let first = take_first_event(&mut event_storage);
assert!(matches!(first, Some(ujson::Event::ObjectStart)));
assert!(event_storage[0].is_none());
assert!(event_storage[1].is_some());
let second = take_first_event(&mut event_storage);
assert!(matches!(second, Some(ujson::Event::ArrayStart)));
assert!(event_storage[0].is_none());
assert!(event_storage[1].is_none());
let none = take_first_event(&mut event_storage);
assert!(none.is_none());
assert!(!have_events(&event_storage));
}
}