use crate::{
error,
path::{Element, Path},
};
use std::{
collections::{vec_deque::Drain, VecDeque},
str::from_utf8,
};
#[derive(Clone, Copy, Debug, PartialEq)]
pub enum ParsedKind {
Obj,
Arr,
Str,
Num,
Null,
Bool,
}
#[derive(Clone, Debug, PartialEq)]
pub enum Token {
Start(usize, ParsedKind),
End(usize, ParsedKind),
Separator(usize),
Pending,
}
impl Token {
pub fn is_end(&self) -> bool {
matches!(self, Self::End(_, _))
}
}
#[derive(Debug)]
enum ObjectKeyState {
Init,
Parse(StringState),
}
#[derive(Debug, PartialEq)]
enum StringState {
Normal,
Escaped,
}
#[derive(Debug)]
enum States {
Value(Option<Element>),
Str(StringState),
Number,
Bool,
Null,
Array(usize),
Object,
ObjectKey(ObjectKeyState),
Colon,
RemoveWhitespaces,
}
#[derive(Debug)]
pub struct Streamer {
path: Path,
states: Vec<States>,
pending: VecDeque<u8>,
pending_idx: usize,
total_idx: usize,
pop_path: bool,
}
impl Default for Streamer {
fn default() -> Self {
Self {
path: Path::default(),
states: vec![States::Value(None), States::RemoveWhitespaces],
pending: VecDeque::new(),
pending_idx: 0,
total_idx: 0,
pop_path: false,
}
}
}
impl Streamer {
pub fn new() -> Self {
Self::default()
}
pub fn current_path(&mut self) -> &mut Path {
&mut self.path
}
fn peek(&mut self) -> Option<u8> {
if self.pending.len() > self.pending_idx {
Some(self.pending[self.pending_idx])
} else {
None
}
}
fn forward(&mut self) {
if self.peek().is_some() {
self.pending_idx += 1;
}
}
fn advance(&mut self) -> Drain<u8> {
let to_remove = self.pending_idx;
if self.pending_idx > 0 {
self.total_idx += self.pending_idx;
self.pending_idx = 0;
}
self.pending.drain(0..to_remove)
}
pub fn feed(&mut self, input: &[u8]) {
self.pending.extend(input);
}
fn process_remove_whitespace(&mut self) -> Option<Token> {
while let Some(byte) = self.peek() {
if !byte.is_ascii_whitespace() {
self.advance();
return None;
}
self.forward();
}
self.states.push(States::RemoveWhitespaces);
Some(Token::Pending)
}
fn process_value(&mut self, element: Option<Element>) -> Result<Option<Token>, error::General> {
if let Some(byte) = self.peek() {
match byte {
b'"' => {
self.states.push(States::Str(StringState::Normal));
self.advance();
self.forward();
if let Some(element) = element {
self.path.push(element);
}
Ok(Some(Token::Start(self.total_idx, ParsedKind::Str)))
}
b'0'..=b'9' => {
self.states.push(States::Number);
self.advance();
if let Some(element) = element {
self.path.push(element);
}
Ok(Some(Token::Start(self.total_idx, ParsedKind::Num)))
}
b't' | b'f' => {
self.states.push(States::Bool);
self.advance();
if let Some(element) = element {
self.path.push(element);
}
Ok(Some(Token::Start(self.total_idx, ParsedKind::Bool)))
}
b'n' => {
self.states.push(States::Null);
self.advance();
if let Some(element) = element {
self.path.push(element);
}
Ok(Some(Token::Start(self.total_idx, ParsedKind::Null)))
}
b'[' => {
self.states.push(States::Array(0));
self.states.push(States::RemoveWhitespaces);
self.states.push(States::Value(Some(Element::Index(0))));
self.states.push(States::RemoveWhitespaces);
self.advance();
self.forward();
if let Some(element) = element {
self.path.push(element);
}
Ok(Some(Token::Start(self.total_idx, ParsedKind::Arr)))
}
b'{' => {
self.states.push(States::Object);
self.states.push(States::RemoveWhitespaces);
self.states.push(States::ObjectKey(ObjectKeyState::Init));
self.states.push(States::RemoveWhitespaces);
self.advance();
self.forward();
if let Some(element) = element {
self.path.push(element);
}
Ok(Some(Token::Start(self.total_idx, ParsedKind::Obj)))
}
b']' | b'}' => {
Ok(None)
}
byte => {
Err(error::IncorrectInput::new(byte, self.total_idx + self.pending_idx).into())
}
}
} else {
self.states.push(States::Value(element));
Ok(Some(Token::Pending))
}
}
fn process_str(&mut self, state: StringState) -> Option<Token> {
if let Some(byte) = self.peek() {
match byte {
b'"' => {
if state == StringState::Normal {
self.forward();
self.advance();
Some(Token::End(self.total_idx, ParsedKind::Str))
} else {
self.forward();
self.states.push(States::Str(StringState::Normal));
None
}
}
b'\\' => {
self.forward();
let new_state = match state {
StringState::Escaped => StringState::Normal,
StringState::Normal => StringState::Escaped,
};
self.states.push(States::Str(new_state));
None
}
_ => {
self.forward();
self.states.push(States::Str(StringState::Normal));
None
}
}
} else {
self.states.push(States::Str(state));
Some(Token::Pending)
}
}
fn process_number(&mut self) -> Option<Token> {
if let Some(byte) = self.peek() {
if byte.is_ascii_digit() || byte == b'.' {
self.forward();
self.states.push(States::Number);
None
} else {
self.advance();
Some(Token::End(self.total_idx, ParsedKind::Num))
}
} else {
self.states.push(States::Number);
Some(Token::Pending)
}
}
fn process_bool(&mut self) -> Option<Token> {
if let Some(byte) = self.peek() {
if byte.is_ascii_alphabetic() {
self.forward();
self.states.push(States::Bool);
None
} else {
self.advance();
Some(Token::End(self.total_idx, ParsedKind::Bool))
}
} else {
self.states.push(States::Bool);
Some(Token::Pending)
}
}
fn process_null(&mut self) -> Option<Token> {
if let Some(byte) = self.peek() {
if byte.is_ascii_alphabetic() {
self.forward();
self.states.push(States::Null);
None
} else {
self.advance();
Some(Token::End(self.total_idx, ParsedKind::Null))
}
} else {
self.states.push(States::Null);
Some(Token::Pending)
}
}
fn process_array(&mut self, idx: usize) -> Result<Option<Token>, error::General> {
if let Some(byte) = self.peek() {
match byte {
b']' => {
self.forward();
self.advance();
Ok(Some(Token::End(self.total_idx, ParsedKind::Arr)))
}
b',' => {
self.forward();
self.states.push(States::Array(idx + 1));
self.states.push(States::RemoveWhitespaces);
self.states
.push(States::Value(Some(Element::Index(idx + 1))));
self.states.push(States::RemoveWhitespaces);
Ok(Some(Token::Separator(self.total_idx)))
}
byte => {
Err(error::IncorrectInput::new(byte, self.total_idx + self.pending_idx).into())
}
}
} else {
self.states.push(States::Array(idx));
Ok(Some(Token::Pending))
}
}
fn process_object(&mut self) -> Result<Option<Token>, error::General> {
if let Some(byte) = self.peek() {
match byte {
b'}' => {
self.forward();
self.advance();
Ok(Some(Token::End(self.total_idx, ParsedKind::Obj)))
}
b',' => {
self.forward();
self.states.push(States::Object);
self.states.push(States::RemoveWhitespaces);
self.states.push(States::ObjectKey(ObjectKeyState::Init));
self.states.push(States::RemoveWhitespaces);
Ok(Some(Token::Separator(self.total_idx)))
}
byte => {
Err(error::IncorrectInput::new(byte, self.total_idx + self.pending_idx).into())
}
}
} else {
self.states.push(States::Object);
Ok(Some(Token::Pending))
}
}
fn process_object_key(
&mut self,
state: ObjectKeyState,
) -> Result<Option<Token>, error::General> {
match state {
ObjectKeyState::Init => {
if let Some(byte) = self.peek() {
match byte {
b'"' => {
self.advance();
self.forward();
self.states.push(States::ObjectKey(ObjectKeyState::Parse(
StringState::Normal,
)));
Ok(None)
}
b'}' => Ok(None),
byte => Err(error::IncorrectInput::new(
byte,
self.total_idx + self.pending_idx,
)
.into()),
}
} else {
self.states.push(States::ObjectKey(state));
Ok(Some(Token::Pending))
}
}
ObjectKeyState::Parse(string_state) => {
if let Some(byte) = self.peek() {
self.forward();
match string_state {
StringState::Normal => match byte {
b'\"' => {
let idx = self.pending_idx;
let slice = &self.advance().collect::<Vec<u8>>()[1..idx - 1];
let key = from_utf8(slice)?.to_string();
self.states.push(States::Value(Some(Element::Key(key))));
self.states.push(States::RemoveWhitespaces);
self.states.push(States::Colon);
self.states.push(States::RemoveWhitespaces);
Ok(None)
}
b'\\' => {
self.states.push(States::ObjectKey(ObjectKeyState::Parse(
StringState::Escaped,
)));
Ok(None)
}
_ => {
self.states.push(States::ObjectKey(ObjectKeyState::Parse(
StringState::Normal,
)));
Ok(None)
}
},
StringState::Escaped => {
self.states.push(States::ObjectKey(ObjectKeyState::Parse(
StringState::Normal,
)));
Ok(None)
}
}
} else {
self.states
.push(States::ObjectKey(ObjectKeyState::Parse(string_state)));
Ok(Some(Token::Pending))
}
}
}
}
fn process_colon(&mut self) -> Result<Option<Token>, error::General> {
if let Some(byte) = self.peek() {
if byte != b':' {
return Err(
error::IncorrectInput::new(byte, self.total_idx + self.pending_idx).into(),
);
}
self.forward();
Ok(None)
} else {
self.states.push(States::Colon);
Ok(Some(Token::Pending))
}
}
pub fn read(&mut self) -> Result<Token, error::General> {
loop {
while let Some(state) = self.states.pop() {
if self.pop_path {
self.path.pop();
self.pop_path = false;
}
match state {
States::RemoveWhitespaces => {
if let Some(output) = self.process_remove_whitespace() {
return Ok(output);
}
}
States::Value(element) => {
if let Some(output) = self.process_value(element)? {
return Ok(output);
}
if self.states.is_empty() {
return Ok(Token::Pending);
}
}
States::Str(state) => {
if let Some(output) = self.process_str(state) {
self.pop_path = output.is_end();
return Ok(output);
}
}
States::Number => {
if let Some(output) = self.process_number() {
self.pop_path = output.is_end();
return Ok(output);
}
}
States::Bool => {
if let Some(output) = self.process_bool() {
self.pop_path = output.is_end();
return Ok(output);
}
}
States::Null => {
if let Some(output) = self.process_null() {
self.pop_path = output.is_end();
return Ok(output);
}
}
States::Array(idx) => {
if let Some(output) = self.process_array(idx)? {
self.pop_path = output.is_end();
return Ok(output);
}
}
States::Object => {
if let Some(output) = self.process_object()? {
self.pop_path = output.is_end();
return Ok(output);
}
}
States::ObjectKey(state) => {
if let Some(output) = self.process_object_key(state)? {
return Ok(output);
}
}
States::Colon => {
if let Some(output) = self.process_colon()? {
return Ok(output);
}
}
}
}
self.states.push(States::Value(None));
self.states.push(States::RemoveWhitespaces);
}
}
}
#[cfg(test)]
mod test {
use super::{ParsedKind, Streamer, Token};
use crate::path::Path;
use std::convert::TryFrom;
fn make_path(path: &str) -> Path {
Path::try_from(path).unwrap()
}
#[test]
fn test_spaces() {
let mut streamer = Streamer::new();
streamer.feed(br#" "#);
assert_eq!(streamer.read().unwrap(), Token::Pending);
}
#[test]
fn test_string() {
let mut streamer = Streamer::new();
streamer.feed(br#" "test string \" \\\" [ ] {} , :\\""#);
assert_eq!(streamer.read().unwrap(), Token::Start(2, ParsedKind::Str));
assert_eq!(streamer.current_path(), &make_path(""));
assert_eq!(streamer.read().unwrap(), Token::End(36, ParsedKind::Str));
assert_eq!(streamer.current_path(), &make_path(""));
assert_eq!(streamer.read().unwrap(), Token::Pending);
let mut streamer = Streamer::new();
streamer.feed(br#"" another one " "#);
assert_eq!(streamer.read().unwrap(), Token::Start(0, ParsedKind::Str));
assert_eq!(streamer.current_path(), &make_path(""));
assert_eq!(streamer.read().unwrap(), Token::End(15, ParsedKind::Str));
assert_eq!(streamer.current_path(), &make_path(""));
assert_eq!(streamer.read().unwrap(), Token::Pending);
}
#[test]
fn test_number() {
let mut streamer = Streamer::new();
streamer.feed(br#" 3.24 "#);
assert_eq!(streamer.read().unwrap(), Token::Start(1, ParsedKind::Num));
assert_eq!(streamer.current_path(), &make_path(""));
assert_eq!(streamer.read().unwrap(), Token::End(5, ParsedKind::Num));
assert_eq!(streamer.current_path(), &make_path(""));
assert_eq!(streamer.read().unwrap(), Token::Pending);
}
#[test]
fn test_bool() {
let mut streamer = Streamer::new();
streamer.feed(br#" true "#);
assert_eq!(streamer.read().unwrap(), Token::Start(2, ParsedKind::Bool));
assert_eq!(streamer.current_path(), &make_path(""));
assert_eq!(streamer.read().unwrap(), Token::End(6, ParsedKind::Bool));
assert_eq!(streamer.current_path(), &make_path(""));
assert_eq!(streamer.read().unwrap(), Token::Pending);
}
#[test]
fn test_null() {
let mut streamer = Streamer::new();
streamer.feed(br#"null"#);
assert_eq!(streamer.read().unwrap(), Token::Start(0, ParsedKind::Null));
assert_eq!(streamer.current_path(), &make_path(""));
assert_eq!(streamer.read().unwrap(), Token::Pending);
let mut streamer = Streamer::new();
streamer.feed(br#"null "#);
assert_eq!(streamer.read().unwrap(), Token::Start(0, ParsedKind::Null));
assert_eq!(streamer.current_path(), &make_path(""));
assert_eq!(streamer.read().unwrap(), Token::End(4, ParsedKind::Null));
assert_eq!(streamer.current_path(), &make_path(""));
assert_eq!(streamer.read().unwrap(), Token::Pending);
}
#[test]
fn test_array() {
let mut streamer = Streamer::new();
streamer.feed(br#"[ null, 33, "string" ]"#);
assert_eq!(streamer.read().unwrap(), Token::Start(0, ParsedKind::Arr));
assert_eq!(streamer.current_path(), &make_path(""));
assert_eq!(streamer.read().unwrap(), Token::Start(2, ParsedKind::Null));
assert_eq!(streamer.current_path(), &make_path("[0]"));
assert_eq!(streamer.read().unwrap(), Token::End(6, ParsedKind::Null));
assert_eq!(streamer.current_path(), &make_path("[0]"));
assert_eq!(streamer.read().unwrap(), Token::Separator(6));
assert_eq!(streamer.read().unwrap(), Token::Start(8, ParsedKind::Num));
assert_eq!(streamer.current_path(), &make_path("[1]"));
assert_eq!(streamer.read().unwrap(), Token::End(10, ParsedKind::Num));
assert_eq!(streamer.current_path(), &make_path("[1]"));
assert_eq!(streamer.read().unwrap(), Token::Separator(10));
assert_eq!(streamer.read().unwrap(), Token::Start(12, ParsedKind::Str));
assert_eq!(streamer.current_path(), &make_path("[2]"));
assert_eq!(streamer.read().unwrap(), Token::End(20, ParsedKind::Str));
assert_eq!(streamer.current_path(), &make_path("[2]"));
assert_eq!(streamer.read().unwrap(), Token::End(22, ParsedKind::Arr));
assert_eq!(streamer.current_path(), &make_path(""));
assert_eq!(streamer.read().unwrap(), Token::Pending);
}
#[test]
fn test_array_pending() {
let mut streamer = Streamer::new();
streamer.feed(br#"[ null, 3"#);
assert_eq!(streamer.read().unwrap(), Token::Start(0, ParsedKind::Arr));
assert_eq!(streamer.current_path(), &make_path(""));
assert_eq!(streamer.read().unwrap(), Token::Start(2, ParsedKind::Null));
assert_eq!(streamer.current_path(), &make_path("[0]"));
assert_eq!(streamer.read().unwrap(), Token::End(6, ParsedKind::Null));
assert_eq!(streamer.current_path(), &make_path("[0]"));
assert_eq!(streamer.read().unwrap(), Token::Separator(6));
assert_eq!(streamer.read().unwrap(), Token::Start(8, ParsedKind::Num));
assert_eq!(streamer.current_path(), &make_path("[1]"));
assert_eq!(streamer.read().unwrap(), Token::Pending);
assert_eq!(streamer.current_path(), &make_path("[1]"));
streamer.feed(br#"3,"#);
assert_eq!(streamer.read().unwrap(), Token::End(10, ParsedKind::Num));
assert_eq!(streamer.current_path(), &make_path("[1]"));
assert_eq!(streamer.read().unwrap(), Token::Separator(10));
assert_eq!(streamer.read().unwrap(), Token::Pending);
assert_eq!(streamer.current_path(), &make_path(""));
streamer.feed(br#" "string" ]"#);
assert_eq!(streamer.read().unwrap(), Token::Start(12, ParsedKind::Str));
assert_eq!(streamer.current_path(), &make_path("[2]"));
assert_eq!(streamer.read().unwrap(), Token::End(20, ParsedKind::Str));
assert_eq!(streamer.current_path(), &make_path("[2]"));
assert_eq!(streamer.read().unwrap(), Token::End(22, ParsedKind::Arr));
assert_eq!(streamer.current_path(), &make_path(""));
assert_eq!(streamer.read().unwrap(), Token::Pending);
}
#[test]
fn test_empty_array() {
let mut streamer = Streamer::new();
streamer.feed(br#"[]"#);
assert_eq!(streamer.read().unwrap(), Token::Start(0, ParsedKind::Arr));
assert_eq!(streamer.current_path(), &make_path(""));
assert_eq!(streamer.read().unwrap(), Token::End(2, ParsedKind::Arr));
assert_eq!(streamer.current_path(), &make_path(""));
assert_eq!(streamer.read().unwrap(), Token::Pending);
}
#[test]
fn test_array_in_array() {
let mut streamer = Streamer::new();
streamer.feed(br#"[ [], 33, ["string" , 44], [ ]]"#);
assert_eq!(streamer.read().unwrap(), Token::Start(0, ParsedKind::Arr));
assert_eq!(streamer.current_path(), &make_path(""));
assert_eq!(streamer.read().unwrap(), Token::Start(2, ParsedKind::Arr));
assert_eq!(streamer.current_path(), &make_path("[0]"));
assert_eq!(streamer.read().unwrap(), Token::End(4, ParsedKind::Arr));
assert_eq!(streamer.current_path(), &make_path("[0]"));
assert_eq!(streamer.read().unwrap(), Token::Separator(4));
assert_eq!(streamer.read().unwrap(), Token::Start(6, ParsedKind::Num));
assert_eq!(streamer.current_path(), &make_path("[1]"));
assert_eq!(streamer.read().unwrap(), Token::End(8, ParsedKind::Num));
assert_eq!(streamer.current_path(), &make_path("[1]"));
assert_eq!(streamer.read().unwrap(), Token::Separator(8));
assert_eq!(streamer.read().unwrap(), Token::Start(10, ParsedKind::Arr));
assert_eq!(streamer.current_path(), &make_path("[2]"));
assert_eq!(streamer.read().unwrap(), Token::Start(11, ParsedKind::Str));
assert_eq!(streamer.current_path(), &make_path("[2][0]"));
assert_eq!(streamer.read().unwrap(), Token::End(19, ParsedKind::Str));
assert_eq!(streamer.current_path(), &make_path("[2][0]"));
assert_eq!(streamer.read().unwrap(), Token::Separator(20));
assert_eq!(streamer.read().unwrap(), Token::Start(22, ParsedKind::Num));
assert_eq!(streamer.current_path(), &make_path("[2][1]"));
assert_eq!(streamer.read().unwrap(), Token::End(24, ParsedKind::Num));
assert_eq!(streamer.current_path(), &make_path("[2][1]"));
assert_eq!(streamer.read().unwrap(), Token::End(25, ParsedKind::Arr));
assert_eq!(streamer.current_path(), &make_path("[2]"));
assert_eq!(streamer.read().unwrap(), Token::Separator(25));
assert_eq!(streamer.read().unwrap(), Token::Start(27, ParsedKind::Arr));
assert_eq!(streamer.current_path(), &make_path("[3]"));
assert_eq!(streamer.read().unwrap(), Token::End(31, ParsedKind::Arr));
assert_eq!(streamer.current_path(), &make_path("[3]"));
assert_eq!(streamer.read().unwrap(), Token::End(32, ParsedKind::Arr));
assert_eq!(streamer.current_path(), &make_path(""));
assert_eq!(streamer.read().unwrap(), Token::Pending);
}
#[test]
fn test_object() {
let mut streamer = Streamer::new();
streamer.feed(br#"{"a":"a", "b" : true , "c": null, " \" \\\" \\": 33}"#);
assert_eq!(streamer.read().unwrap(), Token::Start(0, ParsedKind::Obj));
assert_eq!(streamer.current_path(), &make_path(""));
assert_eq!(streamer.read().unwrap(), Token::Start(5, ParsedKind::Str));
assert_eq!(streamer.current_path(), &make_path("{\"a\"}"));
assert_eq!(streamer.read().unwrap(), Token::End(8, ParsedKind::Str));
assert_eq!(streamer.current_path(), &make_path("{\"a\"}"));
assert_eq!(streamer.read().unwrap(), Token::Separator(8));
assert_eq!(streamer.read().unwrap(), Token::Start(17, ParsedKind::Bool));
assert_eq!(streamer.current_path(), &make_path("{\"b\"}"));
assert_eq!(streamer.read().unwrap(), Token::End(21, ParsedKind::Bool));
assert_eq!(streamer.current_path(), &make_path("{\"b\"}"));
assert_eq!(streamer.read().unwrap(), Token::Separator(22));
assert_eq!(streamer.read().unwrap(), Token::Start(29, ParsedKind::Null));
assert_eq!(streamer.current_path(), &make_path("{\"c\"}"));
assert_eq!(streamer.read().unwrap(), Token::End(33, ParsedKind::Null));
assert_eq!(streamer.current_path(), &make_path("{\"c\"}"));
assert_eq!(streamer.read().unwrap(), Token::Separator(33));
assert_eq!(streamer.read().unwrap(), Token::Start(50, ParsedKind::Num));
assert_eq!(streamer.current_path(), &make_path(r#"{" \" \\\" \\"}"#));
assert_eq!(streamer.read().unwrap(), Token::End(52, ParsedKind::Num));
assert_eq!(streamer.current_path(), &make_path(r#"{" \" \\\" \\"}"#));
assert_eq!(streamer.read().unwrap(), Token::End(53, ParsedKind::Obj));
assert_eq!(streamer.current_path(), &make_path(""));
assert_eq!(streamer.read().unwrap(), Token::Pending);
}
#[test]
fn test_empty_object() {
let mut streamer = Streamer::new();
streamer.feed(br#"{}"#);
assert_eq!(streamer.read().unwrap(), Token::Start(0, ParsedKind::Obj));
assert_eq!(streamer.current_path(), &make_path(""));
assert_eq!(streamer.read().unwrap(), Token::End(2, ParsedKind::Obj));
assert_eq!(streamer.current_path(), &make_path(""));
assert_eq!(streamer.read().unwrap(), Token::Pending);
}
#[test]
fn test_object_in_object() {
let mut streamer = Streamer::new();
streamer.feed(br#" {"u": {}, "j": {"x": { }, "y": 10}} "#);
assert_eq!(streamer.read().unwrap(), Token::Start(1, ParsedKind::Obj));
assert_eq!(streamer.current_path(), &make_path(""));
assert_eq!(streamer.read().unwrap(), Token::Start(7, ParsedKind::Obj));
assert_eq!(streamer.current_path(), &make_path("{\"u\"}"));
assert_eq!(streamer.read().unwrap(), Token::End(9, ParsedKind::Obj));
assert_eq!(streamer.current_path(), &make_path("{\"u\"}"));
assert_eq!(streamer.read().unwrap(), Token::Separator(9));
assert_eq!(streamer.read().unwrap(), Token::Start(16, ParsedKind::Obj));
assert_eq!(streamer.current_path(), &make_path("{\"j\"}"));
assert_eq!(streamer.read().unwrap(), Token::Start(22, ParsedKind::Obj));
assert_eq!(streamer.current_path(), &make_path("{\"j\"}{\"x\"}"));
assert_eq!(streamer.read().unwrap(), Token::End(26, ParsedKind::Obj));
assert_eq!(streamer.current_path(), &make_path("{\"j\"}{\"x\"}"));
assert_eq!(streamer.read().unwrap(), Token::Separator(26));
assert_eq!(streamer.read().unwrap(), Token::Start(33, ParsedKind::Num));
assert_eq!(streamer.current_path(), &make_path("{\"j\"}{\"y\"}"));
assert_eq!(streamer.read().unwrap(), Token::End(35, ParsedKind::Num));
assert_eq!(streamer.current_path(), &make_path("{\"j\"}{\"y\"}"));
assert_eq!(streamer.read().unwrap(), Token::End(36, ParsedKind::Obj));
assert_eq!(streamer.current_path(), &make_path("{\"j\"}"));
assert_eq!(streamer.read().unwrap(), Token::End(37, ParsedKind::Obj));
assert_eq!(streamer.current_path(), &make_path(""));
assert_eq!(streamer.read().unwrap(), Token::Pending);
}
#[test]
fn test_complex_with_pending() {
const COMPLEX_DATA: &[u8] = br#" [{"aha y": {}, "j": {"x": [{ }, [ {}, null ]], "y" : 10}}, null, 43, [ {"a": false} ] ]"#;
for i in 0..COMPLEX_DATA.len() {
let start_data = &COMPLEX_DATA[0..i];
let end_data = &COMPLEX_DATA[i..];
let mut streamer = Streamer::new();
streamer.feed(start_data);
let mut terminate = false;
let mut get_item = |path: Option<&str>| loop {
match streamer.read() {
Ok(Token::Pending) => {
if terminate {
break Token::Pending;
} else {
terminate = true;
streamer.feed(end_data);
}
continue;
}
Ok(e) => {
if let Some(pth) = path {
assert_eq!(streamer.current_path(), &make_path(pth));
}
return e;
}
Err(_) => panic!("Error occured"),
}
};
assert_eq!(get_item(Some("")), Token::Start(1, ParsedKind::Arr));
assert_eq!(get_item(Some("[0]")), Token::Start(2, ParsedKind::Obj));
assert_eq!(
get_item(Some("[0]{\"aha y\"}")),
Token::Start(12, ParsedKind::Obj)
);
assert_eq!(
get_item(Some("[0]{\"aha y\"}")),
Token::End(14, ParsedKind::Obj)
);
assert_eq!(get_item(None), Token::Separator(14));
assert_eq!(
get_item(Some("[0]{\"j\"}")),
Token::Start(21, ParsedKind::Obj)
);
assert_eq!(
get_item(Some("[0]{\"j\"}{\"x\"}")),
Token::Start(27, ParsedKind::Arr)
);
assert_eq!(
get_item(Some("[0]{\"j\"}{\"x\"}[0]")),
Token::Start(28, ParsedKind::Obj)
);
assert_eq!(
get_item(Some("[0]{\"j\"}{\"x\"}[0]")),
Token::End(32, ParsedKind::Obj)
);
assert_eq!(get_item(None), Token::Separator(32));
assert_eq!(
get_item(Some("[0]{\"j\"}{\"x\"}[1]")),
Token::Start(34, ParsedKind::Arr)
);
assert_eq!(
get_item(Some("[0]{\"j\"}{\"x\"}[1][0]")),
Token::Start(36, ParsedKind::Obj)
);
assert_eq!(
get_item(Some("[0]{\"j\"}{\"x\"}[1][0]")),
Token::End(38, ParsedKind::Obj)
);
assert_eq!(get_item(None), Token::Separator(38));
assert_eq!(
get_item(Some("[0]{\"j\"}{\"x\"}[1][1]")),
Token::Start(40, ParsedKind::Null)
);
assert_eq!(
get_item(Some("[0]{\"j\"}{\"x\"}[1][1]")),
Token::End(44, ParsedKind::Null)
);
assert_eq!(
get_item(Some("[0]{\"j\"}{\"x\"}[1]")),
Token::End(46, ParsedKind::Arr)
);
assert_eq!(
get_item(Some("[0]{\"j\"}{\"x\"}")),
Token::End(47, ParsedKind::Arr)
);
assert_eq!(get_item(None), Token::Separator(47));
assert_eq!(
get_item(Some("[0]{\"j\"}{\"y\"}")),
Token::Start(55, ParsedKind::Num)
);
assert_eq!(
get_item(Some("[0]{\"j\"}{\"y\"}")),
Token::End(57, ParsedKind::Num)
);
assert_eq!(
get_item(Some("[0]{\"j\"}")),
Token::End(58, ParsedKind::Obj)
);
assert_eq!(get_item(Some("[0]")), Token::End(59, ParsedKind::Obj));
assert_eq!(get_item(None), Token::Separator(59));
assert_eq!(get_item(Some("[1]")), Token::Start(61, ParsedKind::Null));
assert_eq!(get_item(Some("[1]")), Token::End(65, ParsedKind::Null));
assert_eq!(get_item(None), Token::Separator(65));
assert_eq!(get_item(Some("[2]")), Token::Start(67, ParsedKind::Num));
assert_eq!(get_item(Some("[2]")), Token::End(69, ParsedKind::Num));
assert_eq!(get_item(None), Token::Separator(69));
assert_eq!(get_item(Some("[3]")), Token::Start(71, ParsedKind::Arr));
assert_eq!(get_item(Some("[3][0]")), Token::Start(73, ParsedKind::Obj));
assert_eq!(
get_item(Some("[3][0]{\"a\"}")),
Token::Start(79, ParsedKind::Bool)
);
assert_eq!(
get_item(Some("[3][0]{\"a\"}")),
Token::End(84, ParsedKind::Bool)
);
assert_eq!(get_item(Some("[3][0]")), Token::End(85, ParsedKind::Obj));
assert_eq!(get_item(Some("[3]")), Token::End(87, ParsedKind::Arr));
assert_eq!(get_item(Some("")), Token::End(89, ParsedKind::Arr));
assert_eq!(get_item(None), Token::Pending);
}
}
#[test]
fn test_utf8() {
let utf8_data: Vec<u8> = r#"[{"š𐍈€": "€š𐍈"}, "𐍈€š"]"#.to_string().into_bytes();
for i in 0..utf8_data.len() {
let start_data = &utf8_data[0..i];
let end_data = &utf8_data[i..];
let mut streamer = Streamer::new();
streamer.feed(start_data);
let mut terminate = false;
let mut get_item = |path: Option<&str>| loop {
match streamer.read() {
Ok(Token::Pending) => {
if terminate {
break Token::Pending;
} else {
terminate = true;
streamer.feed(end_data);
}
continue;
}
Ok(e) => {
if let Some(pth) = path {
assert_eq!(streamer.current_path(), &make_path(pth));
}
return e;
}
Err(_) => panic!("Error occured"),
}
};
assert_eq!(get_item(Some("")), Token::Start(0, ParsedKind::Arr));
assert_eq!(get_item(Some("[0]")), Token::Start(1, ParsedKind::Obj));
assert_eq!(
get_item(Some("[0]{\"š𐍈€\"}")),
Token::Start(15, ParsedKind::Str)
);
assert_eq!(
get_item(Some("[0]{\"š𐍈€\"}")),
Token::End(26, ParsedKind::Str)
);
assert_eq!(get_item(Some("[0]")), Token::End(27, ParsedKind::Obj));
assert_eq!(get_item(None), Token::Separator(27));
assert_eq!(get_item(Some("[1]")), Token::Start(29, ParsedKind::Str));
assert_eq!(get_item(Some("[1]")), Token::End(40, ParsedKind::Str));
assert_eq!(get_item(Some("")), Token::End(41, ParsedKind::Arr));
assert_eq!(get_item(None), Token::Pending);
}
}
#[test]
fn test_multiple_input_flat() {
let mut streamer = Streamer::new();
streamer.feed(br#""first" "second""third""#);
assert_eq!(streamer.read().unwrap(), Token::Start(0, ParsedKind::Str));
assert_eq!(streamer.current_path(), &make_path(""));
assert_eq!(streamer.read().unwrap(), Token::End(7, ParsedKind::Str));
assert_eq!(streamer.current_path(), &make_path(""));
assert_eq!(streamer.read().unwrap(), Token::Start(8, ParsedKind::Str));
assert_eq!(streamer.current_path(), &make_path(""));
assert_eq!(streamer.read().unwrap(), Token::End(16, ParsedKind::Str));
assert_eq!(streamer.current_path(), &make_path(""));
assert_eq!(streamer.read().unwrap(), Token::Start(16, ParsedKind::Str));
assert_eq!(streamer.current_path(), &make_path(""));
assert_eq!(streamer.read().unwrap(), Token::End(23, ParsedKind::Str));
assert_eq!(streamer.current_path(), &make_path(""));
assert_eq!(streamer.read().unwrap(), Token::Pending);
}
#[test]
fn test_newlines() {
let mut streamer = Streamer::new();
streamer.feed(
br#" {
"u": {},
"j": {
"x": { } ,
"y":10
}
} "#,
);
assert_eq!(streamer.read().unwrap(), Token::Start(1, ParsedKind::Obj));
assert_eq!(streamer.current_path(), &make_path(""));
assert_eq!(streamer.read().unwrap(), Token::Start(24, ParsedKind::Obj));
assert_eq!(streamer.current_path(), &make_path("{\"u\"}"));
assert_eq!(streamer.read().unwrap(), Token::End(26, ParsedKind::Obj));
assert_eq!(streamer.current_path(), &make_path("{\"u\"}"));
assert_eq!(streamer.read().unwrap(), Token::Separator(26));
assert_eq!(streamer.read().unwrap(), Token::Start(49, ParsedKind::Obj));
assert_eq!(streamer.current_path(), &make_path("{\"j\"}"));
assert_eq!(streamer.read().unwrap(), Token::Start(76, ParsedKind::Obj));
assert_eq!(streamer.current_path(), &make_path("{\"j\"}{\"x\"}"));
assert_eq!(streamer.read().unwrap(), Token::End(80, ParsedKind::Obj));
assert_eq!(streamer.current_path(), &make_path("{\"j\"}{\"x\"}"));
assert_eq!(streamer.read().unwrap(), Token::Separator(81));
assert_eq!(streamer.read().unwrap(), Token::Start(107, ParsedKind::Num));
assert_eq!(streamer.current_path(), &make_path("{\"j\"}{\"y\"}"));
assert_eq!(streamer.read().unwrap(), Token::End(109, ParsedKind::Num));
assert_eq!(streamer.current_path(), &make_path("{\"j\"}{\"y\"}"));
assert_eq!(streamer.read().unwrap(), Token::End(127, ParsedKind::Obj));
assert_eq!(streamer.current_path(), &make_path("{\"j\"}"));
assert_eq!(streamer.read().unwrap(), Token::End(141, ParsedKind::Obj));
assert_eq!(streamer.current_path(), &make_path(""));
assert_eq!(streamer.read().unwrap(), Token::Pending);
}
}