expry 0.1.2

#![allow(dead_code)]
#![cfg_attr(not(feature = "std"), no_std)]

// useful to check binary size of parser
// cargo bloat -n 20 --bin bo-expr-parse --filter expry --profile minsize --features mini

extern crate alloc;
use alloc::format;
use alloc::vec;
use alloc::vec::Vec;
use alloc::string::String;
use alloc::boxed::Box;
use alloc::string::ToString;

use core::cmp::min;
use core::cmp::Ordering;

use core::fmt;
use core::convert::TryFrom;
use core::str::Chars;
use core::str::Utf8Error;
use std::collections::BTreeMap;
use std::fmt::Write;

pub mod memorypool;
pub mod raw_utils;
pub mod parser;
pub mod termcolors;
pub mod macros;
pub mod stringparser;

pub use crate::raw_utils::EncodingError;
pub use crate::memorypool::*;
pub use crate::raw_utils::*;
pub use crate::termcolors::*;
pub use crate::parser::*;
#[allow(unused_imports)]
pub use crate::stringparser::*;

// FIXME: decide if we want to support this syntax (downside is creating a new value context, which can be slow and memory expensive, alternative is having a stack or registers of values, and replace those variable names with stack/register positions). Stack can work in the same place as globals are kept, as every closure adds/pop a value on the stack. During parsing names of stack variables must be kept.
// x.map(x -> 2*x)
// x.map(i -> {x: i})
// x.map(|i| {x: i})
// x.filter(i -> i > 0)
// x.filter(|i| i > 0)
// x.filter(i -> i > other_variable)
// x.filter(|i| i > other_variable)

const WIDTH_OF_JSON_TYPE_MASK: usize = 3;
const JSON_TYPE_MASK: usize = (1 << WIDTH_OF_JSON_TYPE_MASK) - 1;
const KEY_HASH_SIZE: usize = 1;

/// Described the different types of value `expry` supports.
#[derive(PartialEq, Eq)]
pub enum ValueType {
    Null = 0,      // length = 0
    String = 1,    // length = 0+
    BoolFalse = 2, // length = 0
    BoolTrue = 3,  // length = 0
    Float = 4,     // length = 4 or 8 bytes of IEEE 754, length = 0 is used by the CreateObjectDiff algorithm to encode 'key is removed'.
    Int = 5,       // length = [0..]
    Object = 6,    // length = [0..]
    Array = 7,     // length = [0..]
}

impl ValueType {
    pub fn type_string(&self) -> &'static str {
        match self {
            ValueType::Null => "null",
            ValueType::BoolFalse => "bool-false",
            ValueType::BoolTrue => "bool-true",
            ValueType::Int => "int",
            ValueType::Float => "float/double",
            ValueType::String => "string",
            ValueType::Object => "object",
            ValueType::Array => "array",
        }
    }
}

/// keys in Binary uses a simple 8-bit hash to speed up look ups.
pub type KeyHash = u8;

/// the text for Binary keys (in key-values).
pub type Key<'a> = (KeyHash, &'a [u8]);

/// Returns empty key.
pub const fn key_empty() -> Key<'static> {
    (0, b"")
}

/// The central data type of Binary. It is modelled similar to JSON, however, contains a couple of
/// noteworthy differences.
///
/// The differences with JSON:
/// - Floats are either 32-bits or 64-bits;
/// - All strings are u8: they can contain both text or binary data.
#[derive(PartialOrd, PartialEq, Clone)]
pub enum DecodedValue<'a> {
    Null(),
    Bool(bool),
    /// Signed 64-bits integer (that is packed into smaller value in the wire format if possible).
    Int(i64),
    /// 32-bits float type.
    Float(f32),
    /// 64-bits float type.
    Double(f64),
    /// Can be both binary data as regular strings. So not necessary UTF-8 (like Rust strings).
    String(&'a [u8]), // not necessary UTF-8 (so in Rust `str` can not be used)
    Object(DecodedObject<'a>),
    Array(DecodedArray<'a>),
}

impl<'a> Default for DecodedValue<'a> {
    fn default() -> Self {
        Self::Null()
    }
}

impl<'a> DecodedValue<'a> {
    pub fn is_valid_json(&self) -> bool {
        match self {
            DecodedValue::Null() => true,
            DecodedValue::Bool(_) => true,
            DecodedValue::Int(_) => true,
            DecodedValue::Float(_) => true,
            DecodedValue::Double(_) => true,
            DecodedValue::String(s) => core::str::from_utf8(s).is_ok(),
            DecodedValue::Object(m) => {
                for ((_,k),v) in m {
                    if core::str::from_utf8(k).is_err() || !v.is_valid_json() {
                        return false;
                    }
                }
                true
            },
            DecodedValue::Array(a) => {
                for v in a {
                    if !v.is_valid_json() {
                        return false;
                    }
                }
                true
            },
        }
    }
}

impl<'a> From<&DecodedValue<'a>> for DecodedValue<'a> {
    fn from(v: &DecodedValue<'a>) -> Self {
        v.clone()
    }
}
impl<'a, T: 'a> From<&'a Option<T>> for DecodedValue<'a>
where
DecodedValue<'a>: From<&'a T>,
{
    fn from(v: &'a Option<T>) -> Self {
        match v {
            Some(v) => v.into(),
            None => DecodedValue::Null(),
        }
    }
}
impl<'a> From<&bool> for DecodedValue<'a> {
    fn from(v: &bool) -> Self {
        Self::Bool(*v)
    }
}
impl<'a> From<&i64> for DecodedValue<'a> {
    fn from(v: &i64) -> Self {
        Self::Int(*v)
    }
}
impl<'a> From<&f32> for DecodedValue<'a> {
    fn from(v: &f32) -> Self {
        Self::Float(*v)
    }
}
impl<'a> From<&f64> for DecodedValue<'a> {
    fn from(v: &f64) -> Self {
        Self::Double(*v)
    }
}
impl<'a, const N: usize> From<&'a [u8; N]> for DecodedValue<'a> {
    fn from(v: &'a [u8; N]) -> Self {
        Self::String(v)
    }
}
impl<'a> From<&'a [u8]> for DecodedValue<'a> {
    fn from(v: &'a [u8]) -> Self {
        Self::String(v)
    }
}
impl<'a> From<&'a mut [u8]> for DecodedValue<'a> {
    fn from(v: &'a mut [u8]) -> Self {
        Self::String(v)
    }
}
impl<'a> From<&'a str> for DecodedValue<'a> {
    fn from(v: &'a str) -> Self {
        Self::String(v.as_bytes())
    }
}
impl<'a> From<&'a mut str> for DecodedValue<'a> {
    fn from(v: &'a mut str) -> Self {
        Self::String(v.as_bytes())
    }
}
impl<'a> From<&'a &'a str> for DecodedValue<'a> {
    fn from(v: &'a &'a str) -> Self {
        Self::String(v.as_bytes())
    }
}
impl<'a> From<&'a String> for DecodedValue<'a> {
    fn from(v: &'a String) -> Self {
        Self::String(v.as_bytes())
    }
}
impl<'a> From<&'a &'a String> for DecodedValue<'a> {
    fn from(v: &'a &'a String) -> Self {
        Self::String(v.as_bytes())
    }
}
impl<'a> From<&'a BytecodeVec> for DecodedValue<'a> {
    fn from(v: &'a BytecodeVec) -> Self {
        Self::String(v.get())
    }
}
impl<'a> From<&'_ BytecodeRef<'a>> for DecodedValue<'a> {
    fn from(v: &'_ BytecodeRef<'a>) -> Self {
        Self::String(v.get())
    }
}
impl<'a, T: 'a, const N: usize> From<&'a &'a [T; N]> for DecodedValue<'a>
where
    DecodedValue<'a>: From<&'a T>,
    T: Clone,
{
    fn from(v: &'a &'a [T; N]) -> Self {
        let mut retval = DecodedArray::new();
        for item in *v {
            retval.push(item.into());
        }
        Self::Array(retval)
    }
}
impl<'a, T> From<&'a &'a [T]> for DecodedValue<'a>
where
    DecodedValue<'a>: From<&'a T>,
{
    fn from(v: &'a &'a [T]) -> Self {
        Self::Array(v.iter().map(|x| x.into()).collect())
    }
}
impl<'a, T> From<&'a Vec<T>> for DecodedValue<'a>
where
    DecodedValue<'a>: From<&'a T>,
{
    fn from(v: &'a Vec<T>) -> Self {
        Self::Array(v.iter().map(|x| x.into()).collect())
    }
}

pub type DecodedObject<'a> = BTreeMap<Key<'a>, DecodedValue<'a>>;
pub type DecodedArray<'a> = Vec<DecodedValue<'a>>;

pub trait CloneInMemoryScope<'c, T> {
    fn clone_in(&self, scope: &mut MemoryScope<'c>) -> T;
}

impl<'a, 'b, 'c> CloneInMemoryScope<'c, DecodedObject<'b>> for DecodedObject<'a> where 'c: 'b {
    fn clone_in(&self, scope: &mut MemoryScope<'c>) -> DecodedObject<'b> {
        let mut retval = DecodedObject::new();
        for ((hash, k),v) in self {
            retval.insert((*hash, scope.copy_u8(k)), v.clone_in(scope));
        }
        retval
    }
}
impl<'a, 'b, 'c> CloneInMemoryScope<'c, DecodedArray<'b>> for DecodedArray<'a> where 'c: 'b {
    fn clone_in(&self, scope: &mut MemoryScope<'c>) -> DecodedArray<'b> {
        let mut retval = DecodedArray::new();
        for v in self {
            retval.push(v.clone_in(scope));
        }
        retval
    }
}

impl<'a> fmt::Debug for DecodedValue<'a> {
    fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result {
        match self {
            DecodedValue::Null() => write!(f, "null"),
            DecodedValue::Bool(false) => write!(f, "false"),
            DecodedValue::Bool(true) => write!(f, "true"),
            DecodedValue::Int(i) => write!(f, "{}", i),
            DecodedValue::Float(d) => write!(f, "{}f", d),
            DecodedValue::Double(d) => write!(f, "{}", d),
            DecodedValue::String(s) => {
                    if let Ok(s) = core::str::from_utf8(s) {
                        write!(f, "{}", s)
                    } else {
                        write!(f, "{:?}", s)
                    }
            }
            DecodedValue::Object(obj) => {
                write!(f, "{{")?;
                let mut first = true;
                for ((_,k),v) in obj {
                    if !first {
                        write!(f, ", ")?;
                    }
                    if let Ok(key) = core::str::from_utf8(k) {
                        write!(f, "{}: {:?}", key, v)?;
                    } else {
                        write!(f, "{:?}: {:?}", k, v)?;
                    }
                    first = false;
                }
                write!(f, "}}")
            },
            DecodedValue::Array(arr) => write!(f, "{:?}", arr),
        }
    }
}

fn json_escaped_write(f: &mut core::fmt::Formatter<'_>, src: &str) -> Result<(),core::fmt::Error> {
    let mut utf16_buf = [0u16; 2];
    for c in src.chars() {
        match c {
            '\x08' => write!(f, "\\b")?,
            '\x0c' => write!(f, "\\f")?,
            '\n' => write!(f, "\\n")?,
            '\r' => write!(f, "\\r")?,
            '\t' => write!(f, "\\t")?,
            '"' => write!(f, "\\\"")?,
            '\\' => write!(f, "\\\\")?,
            ' ' => write!(f, " ")?,
            c if (c as u32) < 0x20 => {
                let encoded = c.encode_utf16(&mut utf16_buf);
                for utf16 in encoded {
                    write!(f, "\\u{:04X}", utf16)?;
                }
            },
            c => write!(f, "{}", c)?,
        }
    }
    Ok(())
}
impl<'a> core::fmt::Display for DecodedValue<'a> {
    fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result {
        match self {
            DecodedValue::Null() => write!(f, "null"),
            DecodedValue::Bool(false) => write!(f, "false"),
            DecodedValue::Bool(true) => write!(f, "true"),
            DecodedValue::Int(i) => write!(f, "{}", i),
            DecodedValue::Float(d) => write!(f, "{}", d),
            DecodedValue::Double(d) => write!(f, "{}", d),
            DecodedValue::String(s) => {
                    // because format! does no error handling, throwing an error on utf8
                    // problems will cause panics in format! with a decoded value. We want to avoid
                    // this, so we have to ignore non utf8 chars
                    let s = core::str::from_utf8(s);
                    if let Ok(s) = s {
                        write!(f, "\"")?;
                        json_escaped_write(f, s)?;
                        write!(f, "\"")
                    } else {
                        write!(f, "null")
                    }
            }
            DecodedValue::Object(obj) => {
                write!(f, "{{")?;
                let mut first = true;
                for ((_,k),v) in obj {
                    // because format! does no error handling, throwing an error on utf8
                    // problems will cause panics in format! with a decoded value. We want to avoid
                    // this, so we have to ignore non utf8 chars
                    let key = core::str::from_utf8(k);
                    if let Ok(key) = key {
                        if !first {
                            write!(f, ",")?;
                        }
                        write!(f, "\"")?;
                        json_escaped_write(f, key)?;
                        write!(f, "\":{}", v)?;
                        first = false;
                    } else {
                        write!(f, "null")?;
                    }
                }
                write!(f, "}}")
            },
            DecodedValue::Array(arr) => {
                write!(f, "[")?;
                let mut first = true;
                for v in arr {
                    if !first {
                        write!(f, ",")?;
                    }
                    write!(f, "{}", v)?;
                    first = false;
                }
                write!(f, "]")
            },
        }
    }
}

/// Returns the hash for a name.
fn key_stable_hash(s: &[u8]) -> KeyHash {
    let mut state = 0;
    for (i,b) in s.iter().enumerate() {
        state ^= b.wrapping_add(i as u8);
    }
    state
}

impl<'a> DecodedValue<'a> {
    pub const fn type_string(&self) -> &'static str {
        match self {
            DecodedValue::Null() => "null",
            DecodedValue::Bool(_) => "bool",
            DecodedValue::Int(_) => "int",
            DecodedValue::Float(_) => "float",
            DecodedValue::Double(_) => "double",
            DecodedValue::String(_) => "string",
            DecodedValue::Object(_) => "object",
            DecodedValue::Array(_) => "array",
        }
    }
    /// Returns the size needed if we serialize the Binary to a binary wire-format.
    /// See [`DecodedValue::print_binary()`] for explanation of the `compact` argument.
    pub fn size_of_binary(&self, compact: bool) -> usize {
        let mut length_collector = RawWriterLength::new();
        self.print_binary(&mut length_collector, compact).unwrap_infallible();
        length_collector.length()
    }

    /// Converts a Binary to a wire representation of that Binary. This wire representation can be sent
    /// to other implementations as it is stable and 'endian safe'.
    ///
    /// For objects we have a compact
    /// representation and one that inserts hints how to process the wire-representation faster when
    /// using an expression to evaluate the Binary to another value. This is controlled with the
    /// `compact` argument.
    ///
    /// This function expects that the `writer` has enough room. The needed room can be queried
    /// beforehand by using the [`DecodedValue::size_of_binary`].

    pub fn print_binary<E, Out: RawOutput<E>>(&self, writer: &mut Out, compact: bool) -> Result<(),E> {
        match self {
            DecodedValue::Null() => {
                writer.write_u8(ValueType::Null as u8)?;
                Ok(())
            }
            DecodedValue::Bool(b) => {
                if *b {
                    writer.write_u8(ValueType::BoolTrue as u8)?;
                } else {
                    writer.write_u8(ValueType::BoolFalse as u8)?;
                }
                Ok(())
            }
            DecodedValue::Int(i) => {
                if *i == 0 {
                    writer.write_u8(ValueType::Int as u8)?;
                    return Ok(());
                }
                let size = size_of_i64(*i);
                debug_assert!(size <= 8);
                writer.write_u8((size << WIDTH_OF_JSON_TYPE_MASK) as u8 | ValueType::Int as u8)?;
                writer.write_i64(*i, size)?;
                Ok(())
            }
            DecodedValue::Float(f) => {
                writer.write_u8((4 << WIDTH_OF_JSON_TYPE_MASK) as u8 | ValueType::Float as u8)?;
                writer.write_f32(*f)?;
                Ok(())
            }
            DecodedValue::Double(d) => {
                writer.write_u8((8 << WIDTH_OF_JSON_TYPE_MASK) as u8 | ValueType::Float as u8)?;
                writer.write_f64(*d)?;
                Ok(())
            }
            DecodedValue::String(s) => {
                writer.write_var_u64((s.len() << WIDTH_OF_JSON_TYPE_MASK) as u64 | ValueType::String as u64)?;
                writer.write_bytes(s)?;
                Ok(())
            }
            DecodedValue::Array(arr) => {
                if arr.is_empty() {
                    writer.write_u8(ValueType::Array as u8)?;
                    return Ok(());
                }
                write_with_header(writer, |writer,total| {
                    writer.write_var_u64((total as u64) << WIDTH_OF_JSON_TYPE_MASK | ValueType::Array as u64)
                }, |writer| {
                    writer.write_var_u64((arr.len() as u64) << 1)?;
                    for e in arr {
                        e.print_binary(writer, compact)?;
                    }
                    Ok(())
                })
            }
            DecodedValue::Object(obj) => {
                if obj.is_empty() {
                    writer.write_u8(ValueType::Object as u8)?;
                    return Ok(());
                }
                let count = obj.len();
                let local_compact = compact || count <= 4;
                if !cfg!(feature = "mini") && !local_compact {
                    write_with_header(writer, |writer,total| {
                        writer.write_var_u64((total as u64) << WIDTH_OF_JSON_TYPE_MASK | ValueType::Object as u64)
                    }, |writer| {
                        let mut positions : Vec<(usize,u8)> = Vec::with_capacity(count); // (pos, hash)
                        let mut jumps : Vec<(usize,usize)> = Vec::new(); // from_pos, to_index
                        for (i,((hash,key), value)) in obj.iter().enumerate() {
                            for j in 0..i.trailing_zeros() {
                                let to_index = i + (2 << j);
                                if to_index >= count {
                                    break;
                                }
                                // checks if the entry before has the same hash, if so, jumping has no use,
                                // so skip this jump
                                if obj.iter().nth(to_index-1).unwrap().0.0 == obj.iter().nth(to_index).unwrap().0.0 {
                                    continue;
                                }
                                jumps.push((writer.pos(), to_index));
                            }
                            positions.push((writer.pos(),*hash));
                            let key_size = KEY_HASH_SIZE + key.len();
                            writer.write_var_u64((key_size as u64) << 1)?;
                            writer.write_u8(*hash)?;
                            writer.write_bytes(key)?;
                            value.print_binary(writer, compact)?;
                        }
                        // in reverse order pop jumps and insert headers with the new construct
                        for (from_pos, to_index) in jumps.iter().rev() {
                            let current = writer.pos();
                            let to_pos = positions[*to_index].0;
                            let jump = to_pos - from_pos;
                            writer.write_var_u64(((jump as u64) << 1) | 1)?;
                            writer.write_u8(positions[*to_index].1)?;
                            let extra = writer.pos() - current;
                            writer.swap_range(*from_pos, current)?;
                            // HINT: can be made a bit faster if we also store the from_index (part of
                            // the loop can be skipped)
                            for (pos,_) in positions.iter_mut() {
                                if *pos >= *from_pos {
                                    *pos += extra;
                                }
                            }
                        }
                        Ok(())
                    })?;
                    return Ok(());
                }
                write_with_header(writer, |writer,total| {
                    writer.write_var_u64((total as u64) << WIDTH_OF_JSON_TYPE_MASK | ValueType::Object as u64)
                }, |writer| {
                    for ((hash,key), value) in obj {
                        let key_size = KEY_HASH_SIZE + key.len();
                        writer.write_var_u64((key_size as u64) << 1)?;
                        writer.write_u8(*hash)?;
                        writer.write_bytes(key)?;
                        value.print_binary(writer, compact)?;
                    }
                    Ok(())
                })?;
                Ok(())
            }
        }
    }

    /// Converts a Binary to a wire representation of that Binary, in a self contained [`ValueVec`] unsing [`DecodedValue::print_binary`].
    pub fn to_vec(&self, compact: bool) -> ValueVec {
        if cfg!(feature = "mini") {
            let mut writer = RawString::new();
            self.print_binary(&mut writer, compact).expect("expry: problem during actual write");
            ValueVec(writer.data)
        } else {
            let len = self.size_of_binary(compact);
            let mut retval : Vec<u8> = vec![0u8; len];
            let mut writer = RawWriter::with(&mut retval[..]);
            self.print_binary(&mut writer, compact).expect("expry: calculated size differs from actual size");
            debug_assert!(writer.left() == 0);
            ValueVec(retval)
        }
    }

    /// Converts a Binary to a wire representation of that Binary, in a self contained [`ValueVec`] unsing [`DecodedValue::print_binary`].
    pub fn to_scope<'c>(&self, scope: &mut MemoryScope<'c>, compact: bool) -> ValueRef<'c> {
        if cfg!(feature = "mini") {
            // FIXME: convert to delayed deallocation function of MemoryPool
            ValueRef(scope.copy_u8(self.to_vec(compact).get()))
        } else {
            let len = self.size_of_binary(compact);
            let retval = scope.alloc(len);
            let mut writer = RawWriter::with(retval);
            self.print_binary(&mut writer, compact).expect("expry: calculated size differs from actual size");
            debug_assert!(writer.left() == 0);
            ValueRef(retval)
        }
    }

    /// Convert a Binary value to a JSON output.
    ///
    /// Note that some Binary values can not be encoded, because not all strings can be encoded as UTF-8. If this is the case, this method will result in a [`Utf8Error`].
    pub fn print_json(&self, writer: &mut Vec<u8>) -> Result<(),Utf8Error> {
        match self {
            DecodedValue::Null() => {
                writer.extend_from_slice(b"null");
            }
            DecodedValue::Bool(b) => {
                if *b {
                    writer.extend_from_slice(b"true");
                } else {
                    writer.extend_from_slice(b"false");
                }
            }
            DecodedValue::Int(i) => {
                writer.extend_from_slice(i.to_string().as_bytes());
            }
            DecodedValue::Float(f) => {
                writer.extend_from_slice(format!("{:.99}", f).as_bytes());
            }
            DecodedValue::Double(d) => {
                writer.extend_from_slice(format!("{:.99}", d).as_bytes());
            }
            DecodedValue::String(s) => {
                writer.extend_from_slice(b"\"");
                writer.extend_from_slice(&json_escape(core::str::from_utf8(s)?));
                writer.extend_from_slice(b"\"");
            }
            DecodedValue::Array(arr) => {
                writer.extend_from_slice(b"[");
                let mut comma = false;
                for e in arr {
                    if comma {
                        writer.extend_from_slice(b",");
                    }
                    e.print_json(writer)?;
                    comma = true;
                }
                writer.extend_from_slice(b"]");
            }
            DecodedValue::Object(obj) => {
                writer.extend_from_slice(b"{");
                let mut comma = false;
                for ((_,k),v) in obj {
                    if comma {
                        writer.extend_from_slice(b",");
                    }
                    writer.extend_from_slice(b"\"");
                    writer.extend_from_slice(&json_escape(core::str::from_utf8(k)?));
                    writer.extend_from_slice(b"\":");
                    v.print_json(writer)?;
                    comma = true;
                }
                writer.extend_from_slice(b"}");
            }
        };
        Ok(())
    }

    fn lookup_field(&self, key: Key) -> Result<Option<DecodedValue<'a>>,EncodingError> {
        if let DecodedValue::Object(values) = self {
            Ok(values.get(&key).cloned())
        } else {
            Err(EncodingError{line_nr: line!()}) // not an object
        }
    }
}

impl<'a, 'b, 'c> CloneInMemoryScope<'c, DecodedValue<'b>> for DecodedValue<'a> where 'c: 'b {
    fn clone_in(&self, scope: &mut MemoryScope<'c>) -> DecodedValue<'b> {
        match self {
            DecodedValue::Null() => DecodedValue::Null(),
            DecodedValue::Bool(v) => DecodedValue::Bool(*v),
            DecodedValue::Int(v) => DecodedValue::Int(*v),
            DecodedValue::Float(v) => DecodedValue::Float(*v),
            DecodedValue::Double(v) => DecodedValue::Double(*v),
            DecodedValue::String(s) => DecodedValue::String(scope.copy_u8(s)),
            DecodedValue::Object(values) => {
                let mut retval = DecodedObject::new();
                for ((hash, k),v) in values {
                    retval.insert((*hash, scope.copy_u8(k)), v.clone_in(scope));
                }
                DecodedValue::Object(retval)
            },
            DecodedValue::Array(values) => {
                let mut retval = DecodedArray::new();
                for v in values {
                    retval.push(v.clone_in(scope));
                }
                DecodedValue::Array(retval)
            }
        }
    }
}

fn write_field(key: &[u8], value: &[u8], writer: &mut RawWriter) -> Result<(),EncodingError> {
    let hash = key_stable_hash(key);
    write_field_key((hash, key), value, writer)
}

fn write_field_key(key: Key, value: &[u8], writer: &mut RawWriter) -> Result<(),EncodingError> {
    let key_size = KEY_HASH_SIZE + key.1.len();
    writer.write_var_u64((key_size as u64) << 1)?;
    writer.write_u8(key.0)?;
    writer.write_bytes(key.1)?;
    writer.write_bytes(value)?;
    Ok(())
}

/// Creates an object with a single key-value pair inside. Expects `value` to be binary encoded.
pub fn expry_object_with_single_field(key: &[u8], value: &[u8]) -> Result<Vec<u8>,EncodingError> {
    let key_size = key.len() + KEY_HASH_SIZE;
    let entry = size_of_var_u64((key_size as u64) << 1) + key_size + value.len();
    let object_size = size_of_var_u64((entry << WIDTH_OF_JSON_TYPE_MASK) as u64) + entry;
    let mut retval : Vec<u8> = vec![0u8; object_size];
    let mut writer = RawWriter::with(&mut retval[..]);
    write_with_header(&mut writer, |writer,total| {
        writer.write_var_u64((total as u64) << WIDTH_OF_JSON_TYPE_MASK | ValueType::Object as u64)
    }, |writer| {
        write_field(key, value, writer)
    })?;
    assert_eq!(writer.left(), 0);
    Ok(retval)
}

/// create a binary object from individual fields
pub fn expry_object_raw<'c>(scope: &mut MemoryScope<'c>, slice: &[(Key<'_>, &[u8])]) -> Result<&'c [u8],EncodingError> {
    let mut content_size = 0;
    for (key,value) in slice {
        let key_size = key.1.len() + KEY_HASH_SIZE;
        let entry = size_of_var_u64((key_size as u64) << 1) + key_size + value.len();
        content_size += entry;
    }
    let object_size = size_of_var_u64((content_size << WIDTH_OF_JSON_TYPE_MASK) as u64) + content_size;
    let retval = scope.alloc(object_size);
    let mut writer = RawWriter::with(retval);
    write_with_header(&mut writer, |writer,total| {
        writer.write_var_u64((total as u64) << WIDTH_OF_JSON_TYPE_MASK | ValueType::Object as u64)
    }, |writer| {
        for (key,value) in slice {
                write_field_key(*key, value, writer)?;
        }
        Ok(())
    })?;
    assert_eq!(writer.left(), 0);
    Ok(retval)
}
fn json_escape(src: &str) -> Vec<u8> {
    let mut escaped : Vec<u8> = Vec::with_capacity(src.len());
    for c in src.chars() {
        match c {
            '\x08' => escaped.extend_from_slice(b"\\b"),
            '\x0c' => escaped.extend_from_slice(b"\\f"),
            '\n' => escaped.extend_from_slice(b"\\n"),
            '\r' => escaped.extend_from_slice(b"\\r"),
            '\t' => escaped.extend_from_slice(b"\\t"),
            '"' => escaped.extend_from_slice(b"\\\""),
            '\\' => escaped.extend_from_slice(b"\\\\"),
            ' ' => escaped.extend_from_slice(b" "),
            // c.is_ascii_graphic() 
            c if (c as u32) < 0x20 => {
                let mut utf16_buf = [0u16; 2];
                let encoded = c.encode_utf16(&mut utf16_buf);
                for utf16 in encoded {
                    let s = format!("\\u{:04X}", utf16);
                    escaped.extend_from_slice(s.as_bytes());
                }
            },
            c => {
                let mut utf8_buf = [0u8; 4];
                escaped.extend_from_slice(c.encode_utf8(&mut utf8_buf).as_bytes());
            },
        }
    }
    escaped
}

/// Calculates the key as used in this crate from a str. There is also a `[u8]` version.
pub fn key_str(key: &str) -> Key<'_> {
    let hash = key_stable_hash(key.as_bytes());
    (hash, key.as_bytes())
}
/// Calculates the key as used in this crate from an u8 slice. There is also a `str` version.
pub fn key_u8(key: &[u8]) -> Key<'_> {
    let hash = key_stable_hash(key);
    (hash, key)
}

impl TryFrom<usize> for ValueType {
    type Error = EncodingError;

    fn try_from(v: usize) -> Result<Self, Self::Error> {
        match v {
            x if x == ValueType::Null as usize => Ok(ValueType::Null),
            x if x == ValueType::String as usize => Ok(ValueType::String),
            x if x == ValueType::BoolFalse as usize => Ok(ValueType::BoolFalse),
            x if x == ValueType::BoolTrue as usize => Ok(ValueType::BoolTrue),
            x if x == ValueType::Float as usize => Ok(ValueType::Float),
            x if x == ValueType::Int as usize => Ok(ValueType::Int),
            x if x == ValueType::Object as usize => Ok(ValueType::Object),
            x if x == ValueType::Array as usize => Ok(ValueType::Array),
            _ => Err(EncodingError { line_nr: line!() }),
        }
    }
}
impl<'a> DecodedValue<'a> {
    pub fn decoded_type_of(type_and_length: u64) -> Result<ValueType, EncodingError> {
        ((type_and_length as usize) & JSON_TYPE_MASK).try_into()
    }
    /// Based on a variable length integer read for the wire, return the [`ValueType`] raw number.
    pub const fn type_of_binary(type_and_length: u64) -> u8 {
        ((type_and_length as usize) & JSON_TYPE_MASK) as u8
    }
    /// Based on a variable length integer read for the wire, return the length of the current
    /// entry.
    pub const fn length_of_binary(type_and_length: u64) -> usize {
        (type_and_length >> WIDTH_OF_JSON_TYPE_MASK) as usize
    }
    pub fn decoded_type_and_length(type_and_length: u64) -> Result<(ValueType, usize), EncodingError> {
        Ok((Self::decoded_type_of(type_and_length)?, Self::length_of_binary(type_and_length)))
    }

    /// Parse the wire-format.
    pub fn parse(reader: &mut RawReader<'a>) -> Result<DecodedValue<'a>, EncodingError> {
        let type_and_length = reader.read_var_u64()?;
        let (t, len) = Self::decoded_type_and_length(type_and_length)?;
        match t {
            ValueType::Null => {
                if len == 0 {
                    Ok(DecodedValue::Null())
                } else {
                    Err(EncodingError { line_nr: line!() })
                }
            }
            ValueType::String => {
                let value = reader.read_bytes(len)?;
                Ok(DecodedValue::String(value))
            },
            ValueType::BoolFalse => {
                if len == 0 {
                    Ok(DecodedValue::Bool(false))
                } else {
                    Err(EncodingError { line_nr: line!() })
                }
            }
            ValueType::BoolTrue => {
                if len == 0 {
                    Ok(DecodedValue::Bool(true))
                } else {
                    Err(EncodingError { line_nr: line!() })
                }
            }
            ValueType::Float => {
                if len == 4 {
                    let value = reader.read_f32()?;
                    Ok(DecodedValue::Float(value))
                } else if len == 8 {
                    let value = reader.read_f64()?;
                    Ok(DecodedValue::Double(value))
                } else {
                    Err(EncodingError { line_nr: line!() })
                }
            }
            ValueType::Int => {
                let v = reader.read_i64(len)?;
                Ok(DecodedValue::Int(v))
            }
            ValueType::Array => {
                if len == 0 {
                    return Ok(DecodedValue::Array(vec![]));
                }
                let mut subreader = RawReader::with(reader.read_bytes(len)?);
                let mut count = subreader.read_var_u64()?;
                count >>= 1;
                let mut retval : Vec<DecodedValue> = Vec::with_capacity(min(16384, count as usize)); // limit the max allocation to prevent rouge input data leading to out of memory problems
                for _ in 0 .. count {
                    let v = DecodedValue::parse(&mut subreader)?;
                    retval.push(v);
                }
                Ok(DecodedValue::Array(retval))
            },
            ValueType::Object => {
                if len == 0 {
                    return Ok(DecodedValue::Object(DecodedObject::new()));
                }
                let mut last : (KeyHash, &[u8]) = (0, b"");
                let mut retval = DecodedObject::new();
                let mut subreader = RawReader::with(reader.read_bytes(len)?);
                while !subreader.is_empty() {
                    let mut key_length= subreader.read_var_u64()?;
                    if key_length == 0 {
                        return Err(EncodingError{ line_nr: line!() });
                    }
                    if key_length & 0x1 != 0 {
                        subreader.read_u8()?;
                        continue;
                    }
                    key_length = (key_length >> 1) - 1; // - 1 to ignore hash (we handle it differently)
                    if key_length as usize > subreader.len() {
                        return Err(EncodingError{ line_nr: line!() });
                    }
                    let hash = subreader.read_u8()?;
                    let key = (hash, subreader.read_bytes(key_length as usize)?);
                    //let hash = binary_object_stable_hash(&key);

                    let v = DecodedValue::parse(&mut subreader)?;
                    retval.insert(key, v);
                    if key < last {
                        // object not sorted, that is an encoding error
                        return Err(EncodingError{line_nr: line!(), });
                    }
                    last = key;
                }
                debug_assert!(subreader.is_empty());
                Ok(DecodedValue::Object(retval))
            },
        }
    }
}

#[derive(PartialEq, Copy, Clone)]
pub enum LazyDecodedValue<'a> {
    Null(),
    Bool(bool),
    Int(i64),
    Float(f32),
    Double(f64),
    String(&'a [u8]), // ugly in Rust, because a String is forced to be Unicode
    Object(LazyDecodedObject<'a>),
    Array(LazyDecodedArray<'a>),
}

#[derive(PartialEq, Eq, Copy, Clone)]
pub struct LazyDecodedObject<'a> {
    reader: RawReader<'a>,
}

impl<'a> LazyDecodedObject<'a> {

    fn _read_value(reader: &mut RawReader<'a>) -> Result<&'a [u8],EncodingError> {
        let mut reader_type = *reader;
        let type_and_length = reader_type.read_var_u64()?;
        let len = DecodedValue::length_of_binary(type_and_length);
        reader_type.skip(len)?;
        let v = reader.read_bytes(reader.len() - reader_type.len())?;
        Ok(v)
    }

    fn _read_key(reader: &mut RawReader<'a>) -> Result<Key<'a>,EncodingError> {
        let mut key_length = reader.read_var_u64()?;
        if key_length == 0  || key_length & 0x1 != 0 {
            // skip lists should have been read away by the the is_empty()
            return Err(EncodingError{ line_nr: line!() });
        }
        key_length = (key_length >> 1) - 1; // - 1 to ignore hash (we handle it differently)
        let hash = reader.read_u8()?;
        let key = reader.read_bytes(key_length as usize)?;
        Ok((hash, key))
    }
    fn _read(reader: &mut RawReader<'a>) -> Result<(Key<'a>, &'a [u8]),EncodingError> {

        Ok((Self::_read_key(reader)?, Self::_read_value(reader)?))
    }

    pub fn lookup_binary<'b>(&self, key: (KeyHash, &'b [u8])) -> Result<Option<&'a [u8]>,EncodingError> where 'a: 'b {
        let mut value_reader = self.reader;
        match evaluate_seek(key.0, key.1, &mut value_reader) {
            Ok(_) => {},
            Err(EvalError::FieldNotFound(_)) => return Ok(None),
            Err(_) => return Err(EncodingError{ line_nr: line!(), }),

        }
        let binary_value = Self::_read(&mut value_reader)?.1;
        Ok(Some(binary_value))
    }
    pub fn lookup_value<'b>(&self, key: (KeyHash, &'b [u8])) -> Result<Option<LazyDecodedValue<'a>>,EncodingError> where 'a: 'b {
        let binary_value = self.lookup_binary(key)?;
        if let Some(binary_value) = binary_value {
            Ok(Some(LazyDecodedValue::parse(&mut RawReader::with(binary_value))?))
        } else {
            Ok(None)
        }
    }
    pub fn get(&mut self) -> Result<(Key<'a>,LazyDecodedValue),EncodingError> {
        let (key, value) = LazyDecodedObject::<'a>::_read(&mut self.reader)?;
        Ok((key, LazyDecodedValue::parse(&mut RawReader::with(value))?))
    }
    pub fn is_empty(&mut self) -> Result<bool,EncodingError> {
        while self.reader.len() > 0 && self.reader.clone().read_var_u64()? & 0x1 != 0 {
            self.reader.read_var_u64()?;
            self.reader.read_u8()?;
        }
        Ok(self.reader.is_empty())
    }

    pub fn new(reader: RawReader<'a>) -> Self {
        Self {
            reader,
        }
    }

}

#[derive(PartialEq, Eq, Copy, Clone)]
pub struct LazyDecodedArray<'a> {
    reader: RawReader<'a>,
    count: u64,
}

impl<'a> LazyDecodedArray<'a> {
    pub fn get(&mut self) -> Result<LazyDecodedValue,EncodingError> {
        if self.count == 0 {
            return Err(EncodingError{ line_nr: line!() });
        }
        debug_assert!(!self.reader.is_empty());
        self.count -= 1;
        LazyDecodedValue::parse(&mut self.reader)
    }
    pub fn get_raw(&mut self) -> Result<ValueRef<'a>,EncodingError> {
        self.count -= 1;
        let type_and_length = self.reader.clone().read_var_u64()?;
        let len = DecodedValue::length_of_binary(type_and_length);
        Ok(ValueRef(self.reader.read_bytes(len + size_of_var_u64(type_and_length))?))
    }
    pub fn remaining(&self) -> u64 {
        self.count
    }
    pub fn is_empty(&self) -> bool {
        self.count == 0
    }
}

impl<'a> LazyDecodedValue<'a> {
    pub fn type_string(&self) -> &'static str {
        match self {
            LazyDecodedValue::Null() => "null",
            LazyDecodedValue::Bool(_) => "bool",
            LazyDecodedValue::Int(_) => "int",
            LazyDecodedValue::Float(_) => "float",
            LazyDecodedValue::Double(_) => "double",
            LazyDecodedValue::String(_) => "string",
            LazyDecodedValue::Object(_) => "object",
            LazyDecodedValue::Array(_) => "array",
        }
    }
    pub fn parse(reader: &mut RawReader<'a>) -> Result<LazyDecodedValue<'a>, EncodingError> {
        let type_and_length = reader.read_var_u64()?;
        let (t, len) = DecodedValue::decoded_type_and_length(type_and_length)?;
        match t {
            ValueType::Null => {
                if len == 0 {
                    Ok(LazyDecodedValue::Null())
                } else {
                    Err(EncodingError { line_nr: line!() })
                }
            }
            ValueType::String => {
                let value = reader.read_bytes(len)?;
                Ok(LazyDecodedValue::String(value))
            },
            ValueType::BoolFalse => {
                if len == 0 {
                    Ok(LazyDecodedValue::Bool(false))
                } else {
                    Err(EncodingError { line_nr: line!() })
                }
            }
            ValueType::BoolTrue => {
                if len == 0 {
                    Ok(LazyDecodedValue::Bool(true))
                } else {
                    Err(EncodingError { line_nr: line!() })
                }
            }
            ValueType::Float => {
                if len == 4 {
                    let value = reader.read_f32()?;
                    Ok(LazyDecodedValue::Float(value))
                } else if len == 8 {
                    let value = reader.read_f64()?;
                    Ok(LazyDecodedValue::Double(value))
                } else {
                    Err(EncodingError { line_nr: line!() })
                }
            }
            ValueType::Int => {
                let v = reader.read_i64(len)?;
                Ok(LazyDecodedValue::Int(v))
            }
            ValueType::Array => {
                if len == 0 {
                    return Ok(LazyDecodedValue::Array(LazyDecodedArray { count: 0, reader: RawReader::with(b""), }));
                }
                let mut subreader = RawReader::with(reader.read_bytes(len)?);
                let mut count = subreader.read_var_u64()?;
                count >>= 1;

                let retval = LazyDecodedArray {
                    count,
                    reader: subreader,
                };
                Ok(LazyDecodedValue::Array(retval))
            },
            ValueType::Object => {
                if len == 0 {
                    return Ok(LazyDecodedValue::Object(LazyDecodedObject { reader: RawReader::with(b""), }));
                }
                Ok(LazyDecodedValue::Object(LazyDecodedObject::new(RawReader::with(reader.read_bytes(len)?))))
            },
        }
    }
}

/// Reference to a expry value.
#[derive(PartialEq, Eq, Copy, Clone, Debug)]
pub struct ValueRef<'a>(pub &'a [u8]);

impl<'a> ValueRef<'a> {
    fn lookup_field(&self, key: Key) -> Result<Option<DecodedValue<'a>>,EncodingError> {
        if self.get().is_empty() {
            return Ok(None);
        }
        let mut value_reader = RawReader::with(self.get());
        if DecodedValue::type_of_binary(value_reader.read_var_u64()?) != ValueType::Object as u8 {
            return Err(EncodingError{line_nr: line!()});
        }
        match evaluate_seek(key.0, key.1, &mut value_reader) {
            Ok(_) => {},
            Err(EvalError::FieldNotFound(_)) => return Ok(None),
            Err(_) => return Err(EncodingError{line_nr: line!()}),
        }
        let key_length = value_reader.read_var_u64()?;
        if key_length == 0 || (key_length & 0x1) != 0 {
            return Err(EncodingError{line_nr: line!()});
        }
        let _key_name = &value_reader.read_bytes(key_length as usize >> 1)?[1..];
        Ok(Some(DecodedValue::parse(&mut value_reader)?))
    }

    pub fn get(&self) -> &'a [u8] {
        self.0
    }
    pub fn is_empty(&self) -> bool {
        self.0.is_empty()
    }
    pub fn new() -> Self {
        Self(b"")
    }
}

impl<'a> Default for ValueRef<'a> {
    fn default() -> Self {
        Self::new()
    }
}

/// Self-contained expry value (in encoded form).
#[derive(PartialEq, Eq, Clone, Debug)]
pub struct ValueVec(pub Vec<u8>);

/// Reference to expry expression bytecode.
#[derive(PartialEq, Eq, Copy, Clone, Debug)]
pub struct BytecodeRef<'a>(pub &'a [u8]);

impl<'a> BytecodeRef<'a> {
    pub fn get(&self) -> &'a [u8] {
        self.0
    }
    pub fn is_empty(&self) -> bool {
        self.0.is_empty()
    }
    pub fn new() -> Self {
        Self(b"")
    }
}

impl<'a> Default for BytecodeRef<'a> {
    fn default() -> Self {
        Self::new()
    }
}

/// Self-contained expry expression bytecode.
#[derive(PartialEq, Eq, Clone, Debug)]
pub struct BytecodeVec(pub Vec<u8>);

impl BytecodeVec {
    pub fn len(&self) -> usize {
        self.0.len()
    }
    pub fn to_ref(&self) -> BytecodeRef {
        BytecodeRef(&self.0)
    }
    pub fn get(&self) -> &[u8] {
        &self.0
    }
    pub fn is_empty(&self) -> bool {
        self.0.is_empty()
    }
}
impl ValueVec {
    pub fn len(&self) -> usize {
        self.0.len()
    }
    pub fn to_ref(&self) -> ValueRef {
        ValueRef(&self.0)
    }
    pub fn get(&self) -> &[u8] {
        &self.0
    }
    pub fn is_empty(&self) -> bool {
        self.0.is_empty()
    }
}

// we can use the .into() method to convert (but this is an explicit action)
impl Into<ValueVec> for Vec<u8> {
    fn into(self) -> ValueVec {
        ValueVec(self)
    }
}
impl Into<BytecodeVec> for Vec<u8> {
    fn into(self) -> BytecodeVec {
        BytecodeVec(self)
    }
}

// safe of misuse to convert from value/bytecode to vec
impl Into<Vec<u8>> for ValueVec {
    fn into(self) -> Vec<u8> {
        self.0
    }
}
impl Into<Vec<u8>> for BytecodeVec {
    fn into(self) -> Vec<u8> {
        self.0
    }
}


// reads away skip lists
// returns current key, entry length, and the payload type+length
fn peek_key_and_entry<'a>(reader: &'_ mut RawReader<'a>) -> Result<(KeyHash,&'a [u8],usize,usize),EncodingError> {
    let mut peeker = *reader;
    while !peeker.is_empty() {
        let mut key_length = peeker.read_var_u64()?;
        if key_length == 0 {
            return Err(EncodingError{ line_nr: line!() });
        }
        if (key_length & 0x1) != 0 {
            let _hash_of_next_entry = peeker.read_u8()?; // read hash of forward reference
            *reader = peeker;
            continue;
        }
        key_length = (key_length >> 1) - KEY_HASH_SIZE as u64;
        let hash = peeker.read_u8()?;
        let key = peeker.read_bytes(key_length as usize)?;

        let type_and_length = peeker.read_var_u64()?;
        peeker.skip(DecodedValue::length_of_binary(type_and_length))?;
        return Ok((hash, key, reader.len() - peeker.len(), type_and_length as usize));
    }
    Ok((0, b"", 0, 0))
}

/// Merges two objects. Left hand side take preference (so the left hand side value of keys occuring in
/// both is taken). Always producing a compact object
pub fn merge_objects<'b, 'c>(lhs: ValueRef, rhs: ValueRef, allocator: &mut MemoryScope<'c>) -> Result<ValueRef<'b>, EncodingError> where 'c: 'b {
    let mut lhs_reader = RawReader::with(lhs.get());
    let mut rhs_reader = RawReader::with(rhs.get());
    /*
    // Simple version with higher memory usage
    let lhs = DecodedValue::parse(&mut lhs_reader)?;
    let rhs = DecodedValue::parse(&mut rhs_reader)?;
    if let (DecodedValue::Object(mut lhs),DecodedValue::Object(rhs)) = (lhs, rhs) {
        for (k,v) in rhs {
            lhs.entry(k).or_insert(v);
        }
        Ok(DecodedValue::Object(lhs).to_scope(allocator, true))
    } else {
        Err(EncodingError{ line_nr: line!() })
    }
    */
    let lhs_length_and_type = lhs_reader.read_var_u64()?;
    if DecodedValue::type_of_binary(lhs_length_and_type) != ValueType::Object as u8 {
        return Err(EncodingError{ line_nr: line!() });
    }
    if DecodedValue::length_of_binary(lhs_length_and_type) != lhs_reader.len() {
        return Err(EncodingError{ line_nr: line!() });
    }
    let rhs_length_and_type = rhs_reader.read_var_u64()?;
    if DecodedValue::type_of_binary(rhs_length_and_type) != ValueType::Object as u8 {
        return Err(EncodingError{ line_nr: line!() });
    }
    if DecodedValue::length_of_binary(rhs_length_and_type) != rhs_reader.len() {
        return Err(EncodingError{ line_nr: line!() });
    }

    let mut chunks : ScopedArrayBuilder<'_, 'c, &[u8]> = ScopedArrayBuilder::new(allocator);
    chunks.push(b""); // place holder we are going to replace at the end with an header
    let mut total : usize = 0;
    let mut lhs_current : (KeyHash,&[u8],usize,usize) = peek_key_and_entry(&mut lhs_reader)?;
    let mut rhs_current : (KeyHash,&[u8],usize,usize) = peek_key_and_entry(&mut rhs_reader)?;
    while !lhs_current.1.is_empty() || !rhs_current.1.is_empty() {
        let (lhs_hash, lhs_key, lhs_entry_length, lhs_value_length_and_type) = lhs_current;
        let (rhs_hash, rhs_key, rhs_entry_length, _) = rhs_current;
        let mut entry : &[u8];
        if lhs_entry_length > 0 && (lhs_hash, lhs_key) <= (rhs_hash, rhs_key) || rhs_entry_length == 0 {
            entry = lhs_reader.read_bytes(lhs_entry_length)?;
            // special 'remove' marker as generated by CreateObjectDiff
            if lhs_value_length_and_type == ValueType::Float as usize { // float of 0 bytes implies a size of 0
                entry = b"";
            }
            // if key is in both, skip the rhs one
            if (lhs_hash, lhs_key) == (rhs_hash, rhs_key) {
                rhs_reader.skip(rhs_entry_length)?;
                rhs_current = peek_key_and_entry(&mut rhs_reader)?;
            }
            lhs_current = peek_key_and_entry(&mut lhs_reader)?;
        } else {
            entry = rhs_reader.read_bytes(rhs_entry_length)?;
            rhs_current = peek_key_and_entry(&mut rhs_reader)?;
        }
        if !entry.is_empty() {
            chunks.push(entry);
            total += entry.len();
        }
    }
    let retval = chunks.build();
    retval[0] = to_var_length(allocator, total << WIDTH_OF_JSON_TYPE_MASK | ValueType::Object as usize);
    let retval = allocator.concat_u8(retval);
    Ok(ValueRef(retval))
}

#[derive(PartialEq, Copy, Clone, Debug)]
enum JSONToken<'a> {
    End(),
    Bool(bool),
    String(&'a [u8]),
    Float(f32),
    Double(f64),
    Int(i64),
    Comma(),
    ArrayOpen(),
    ArrayClose(),
    ObjectOpen(),
    Colon(),
    ObjectClose(),
    NullObject(),

    Comparison(ExpryComparison),
    Arithmetic(Arithmetic),
    Logical(Logical),
    Bitwise(Bitwise),
    Open(),
    Close(),
    As(),
    Not(),
    Conditional(),
    NullOperator(),
    TryOperator(),
    Concat(),
    Dot(),
    Spread(),
    Field(&'a str),
    FunctionCall(&'a str),
}


#[derive(Debug, Copy, Clone, PartialEq, Eq)]
pub struct LexerError {
    line_nr: u32,
}
#[derive(Debug, Copy, Clone, PartialEq, Eq)]
pub enum CompileErrorDescription<'a> {
    NoToken(),
    Lexer(LexerError),
    Parser(&'a str),
    Optimizer(EvalError<'a>),
}

#[derive(Clone, Debug)]
pub struct CompileError<'a> {
    pub error: CompileErrorDescription<'a>,
    pub start: usize, // counted from the back of the input string
    pub end: usize, // counted from the back of the input string
    pub extra: Option<(usize, usize)>,
}

impl<'a> CompileError<'a> {
    pub fn error_start(&self) -> usize {
        self.start
    }
    pub fn error_end(&self) -> usize {
        self.end
    }
    pub fn error(&self) -> CompileErrorDescription<'a> {
        self.