#![allow(dead_code)]
#![cfg_attr(not(feature = "std"), no_std)]
// useful to check binary size of parser
// cargo bloat -n 20 --bin bo-expr-parse --filter expry --profile minsize --features mini
extern crate alloc;
use alloc::format;
use alloc::vec;
use alloc::vec::Vec;
use alloc::string::String;
use alloc::boxed::Box;
use alloc::string::ToString;
use core::cmp::min;
use core::cmp::Ordering;
use core::fmt;
use core::convert::TryFrom;
use core::str::Chars;
use core::str::Utf8Error;
use std::collections::BTreeMap;
use std::fmt::Write;
pub mod memorypool;
pub mod raw_utils;
pub mod parser;
pub mod termcolors;
pub mod macros;
pub mod stringparser;
pub use crate::raw_utils::EncodingError;
pub use crate::memorypool::*;
pub use crate::raw_utils::*;
pub use crate::termcolors::*;
pub use crate::parser::*;
#[allow(unused_imports)]
pub use crate::stringparser::*;
// FIXME: decide if we want to support this syntax (downside is creating a new value context, which can be slow and memory expensive, alternative is having a stack or registers of values, and replace those variable names with stack/register positions). Stack can work in the same place as globals are kept, as every closure adds/pop a value on the stack. During parsing names of stack variables must be kept.
// x.map(x -> 2*x)
// x.map(i -> {x: i})
// x.map(|i| {x: i})
// x.filter(i -> i > 0)
// x.filter(|i| i > 0)
// x.filter(i -> i > other_variable)
// x.filter(|i| i > other_variable)
const WIDTH_OF_JSON_TYPE_MASK: usize = 3;
const JSON_TYPE_MASK: usize = (1 << WIDTH_OF_JSON_TYPE_MASK) - 1;
const KEY_HASH_SIZE: usize = 1;
/// Described the different types of value `expry` supports.
#[derive(PartialEq, Eq)]
pub enum ValueType {
Null = 0, // length = 0
String = 1, // length = 0+
BoolFalse = 2, // length = 0
BoolTrue = 3, // length = 0
Float = 4, // length = 4 or 8 bytes of IEEE 754, length = 0 is used by the CreateObjectDiff algorithm to encode 'key is removed'.
Int = 5, // length = [0..]
Object = 6, // length = [0..]
Array = 7, // length = [0..]
}
impl ValueType {
pub fn type_string(&self) -> &'static str {
match self {
ValueType::Null => "null",
ValueType::BoolFalse => "bool-false",
ValueType::BoolTrue => "bool-true",
ValueType::Int => "int",
ValueType::Float => "float/double",
ValueType::String => "string",
ValueType::Object => "object",
ValueType::Array => "array",
}
}
}
/// keys in Binary uses a simple 8-bit hash to speed up look ups.
pub type KeyHash = u8;
/// the text for Binary keys (in key-values).
pub type Key<'a> = (KeyHash, &'a [u8]);
/// Returns empty key.
pub const fn key_empty() -> Key<'static> {
(0, b"")
}
/// The central data type of Binary. It is modelled similar to JSON, however, contains a couple of
/// noteworthy differences.
///
/// The differences with JSON:
/// - Floats are either 32-bits or 64-bits;
/// - All strings are u8: they can contain both text or binary data.
#[derive(PartialOrd, PartialEq, Clone)]
pub enum DecodedValue<'a> {
Null(),
Bool(bool),
/// Signed 64-bits integer (that is packed into smaller value in the wire format if possible).
Int(i64),
/// 32-bits float type.
Float(f32),
/// 64-bits float type.
Double(f64),
/// Can be both binary data as regular strings. So not necessary UTF-8 (like Rust strings).
String(&'a [u8]), // not necessary UTF-8 (so in Rust `str` can not be used)
Object(DecodedObject<'a>),
Array(DecodedArray<'a>),
}
impl<'a> Default for DecodedValue<'a> {
fn default() -> Self {
Self::Null()
}
}
impl<'a> DecodedValue<'a> {
pub fn is_valid_json(&self) -> bool {
match self {
DecodedValue::Null() => true,
DecodedValue::Bool(_) => true,
DecodedValue::Int(_) => true,
DecodedValue::Float(_) => true,
DecodedValue::Double(_) => true,
DecodedValue::String(s) => core::str::from_utf8(s).is_ok(),
DecodedValue::Object(m) => {
for ((_,k),v) in m {
if core::str::from_utf8(k).is_err() || !v.is_valid_json() {
return false;
}
}
true
},
DecodedValue::Array(a) => {
for v in a {
if !v.is_valid_json() {
return false;
}
}
true
},
}
}
}
impl<'a> From<&DecodedValue<'a>> for DecodedValue<'a> {
fn from(v: &DecodedValue<'a>) -> Self {
v.clone()
}
}
impl<'a, T: 'a> From<&'a Option<T>> for DecodedValue<'a>
where
DecodedValue<'a>: From<&'a T>,
{
fn from(v: &'a Option<T>) -> Self {
match v {
Some(v) => v.into(),
None => DecodedValue::Null(),
}
}
}
impl<'a> From<&bool> for DecodedValue<'a> {
fn from(v: &bool) -> Self {
Self::Bool(*v)
}
}
impl<'a> From<&i64> for DecodedValue<'a> {
fn from(v: &i64) -> Self {
Self::Int(*v)
}
}
impl<'a> From<&f32> for DecodedValue<'a> {
fn from(v: &f32) -> Self {
Self::Float(*v)
}
}
impl<'a> From<&f64> for DecodedValue<'a> {
fn from(v: &f64) -> Self {
Self::Double(*v)
}
}
impl<'a, const N: usize> From<&'a [u8; N]> for DecodedValue<'a> {
fn from(v: &'a [u8; N]) -> Self {
Self::String(v)
}
}
impl<'a> From<&'a [u8]> for DecodedValue<'a> {
fn from(v: &'a [u8]) -> Self {
Self::String(v)
}
}
impl<'a> From<&'a mut [u8]> for DecodedValue<'a> {
fn from(v: &'a mut [u8]) -> Self {
Self::String(v)
}
}
impl<'a> From<&'a str> for DecodedValue<'a> {
fn from(v: &'a str) -> Self {
Self::String(v.as_bytes())
}
}
impl<'a> From<&'a mut str> for DecodedValue<'a> {
fn from(v: &'a mut str) -> Self {
Self::String(v.as_bytes())
}
}
impl<'a> From<&'a &'a str> for DecodedValue<'a> {
fn from(v: &'a &'a str) -> Self {
Self::String(v.as_bytes())
}
}
impl<'a> From<&'a String> for DecodedValue<'a> {
fn from(v: &'a String) -> Self {
Self::String(v.as_bytes())
}
}
impl<'a> From<&'a &'a String> for DecodedValue<'a> {
fn from(v: &'a &'a String) -> Self {
Self::String(v.as_bytes())
}
}
impl<'a> From<&'a BytecodeVec> for DecodedValue<'a> {
fn from(v: &'a BytecodeVec) -> Self {
Self::String(v.get())
}
}
impl<'a> From<&'_ BytecodeRef<'a>> for DecodedValue<'a> {
fn from(v: &'_ BytecodeRef<'a>) -> Self {
Self::String(v.get())
}
}
impl<'a, T: 'a, const N: usize> From<&'a &'a [T; N]> for DecodedValue<'a>
where
DecodedValue<'a>: From<&'a T>,
T: Clone,
{
fn from(v: &'a &'a [T; N]) -> Self {
let mut retval = DecodedArray::new();
for item in *v {
retval.push(item.into());
}
Self::Array(retval)
}
}
impl<'a, T> From<&'a &'a [T]> for DecodedValue<'a>
where
DecodedValue<'a>: From<&'a T>,
{
fn from(v: &'a &'a [T]) -> Self {
Self::Array(v.iter().map(|x| x.into()).collect())
}
}
impl<'a, T> From<&'a Vec<T>> for DecodedValue<'a>
where
DecodedValue<'a>: From<&'a T>,
{
fn from(v: &'a Vec<T>) -> Self {
Self::Array(v.iter().map(|x| x.into()).collect())
}
}
pub type DecodedObject<'a> = BTreeMap<Key<'a>, DecodedValue<'a>>;
pub type DecodedArray<'a> = Vec<DecodedValue<'a>>;
pub trait CloneInMemoryScope<'c, T> {
fn clone_in(&self, scope: &mut MemoryScope<'c>) -> T;
}
impl<'a, 'b, 'c> CloneInMemoryScope<'c, DecodedObject<'b>> for DecodedObject<'a> where 'c: 'b {
fn clone_in(&self, scope: &mut MemoryScope<'c>) -> DecodedObject<'b> {
let mut retval = DecodedObject::new();
for ((hash, k),v) in self {
retval.insert((*hash, scope.copy_u8(k)), v.clone_in(scope));
}
retval
}
}
impl<'a, 'b, 'c> CloneInMemoryScope<'c, DecodedArray<'b>> for DecodedArray<'a> where 'c: 'b {
fn clone_in(&self, scope: &mut MemoryScope<'c>) -> DecodedArray<'b> {
let mut retval = DecodedArray::new();
for v in self {
retval.push(v.clone_in(scope));
}
retval
}
}
impl<'a> fmt::Debug for DecodedValue<'a> {
fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result {
match self {
DecodedValue::Null() => write!(f, "null"),
DecodedValue::Bool(false) => write!(f, "false"),
DecodedValue::Bool(true) => write!(f, "true"),
DecodedValue::Int(i) => write!(f, "{}", i),
DecodedValue::Float(d) => write!(f, "{}f", d),
DecodedValue::Double(d) => write!(f, "{}", d),
DecodedValue::String(s) => {
if let Ok(s) = core::str::from_utf8(s) {
write!(f, "{}", s)
} else {
write!(f, "{:?}", s)
}
}
DecodedValue::Object(obj) => {
write!(f, "{{")?;
let mut first = true;
for ((_,k),v) in obj {
if !first {
write!(f, ", ")?;
}
if let Ok(key) = core::str::from_utf8(k) {
write!(f, "{}: {:?}", key, v)?;
} else {
write!(f, "{:?}: {:?}", k, v)?;
}
first = false;
}
write!(f, "}}")
},
DecodedValue::Array(arr) => write!(f, "{:?}", arr),
}
}
}
fn json_escaped_write(f: &mut core::fmt::Formatter<'_>, src: &str) -> Result<(),core::fmt::Error> {
let mut utf16_buf = [0u16; 2];
for c in src.chars() {
match c {
'\x08' => write!(f, "\\b")?,
'\x0c' => write!(f, "\\f")?,
'\n' => write!(f, "\\n")?,
'\r' => write!(f, "\\r")?,
'\t' => write!(f, "\\t")?,
'"' => write!(f, "\\\"")?,
'\\' => write!(f, "\\\\")?,
' ' => write!(f, " ")?,
c if (c as u32) < 0x20 => {
let encoded = c.encode_utf16(&mut utf16_buf);
for utf16 in encoded {
write!(f, "\\u{:04X}", utf16)?;
}
},
c => write!(f, "{}", c)?,
}
}
Ok(())
}
impl<'a> core::fmt::Display for DecodedValue<'a> {
fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result {
match self {
DecodedValue::Null() => write!(f, "null"),
DecodedValue::Bool(false) => write!(f, "false"),
DecodedValue::Bool(true) => write!(f, "true"),
DecodedValue::Int(i) => write!(f, "{}", i),
DecodedValue::Float(d) => write!(f, "{}", d),
DecodedValue::Double(d) => write!(f, "{}", d),
DecodedValue::String(s) => {
// because format! does no error handling, throwing an error on utf8
// problems will cause panics in format! with a decoded value. We want to avoid
// this, so we have to ignore non utf8 chars
let s = core::str::from_utf8(s);
if let Ok(s) = s {
write!(f, "\"")?;
json_escaped_write(f, s)?;
write!(f, "\"")
} else {
write!(f, "null")
}
}
DecodedValue::Object(obj) => {
write!(f, "{{")?;
let mut first = true;
for ((_,k),v) in obj {
// because format! does no error handling, throwing an error on utf8
// problems will cause panics in format! with a decoded value. We want to avoid
// this, so we have to ignore non utf8 chars
let key = core::str::from_utf8(k);
if let Ok(key) = key {
if !first {
write!(f, ",")?;
}
write!(f, "\"")?;
json_escaped_write(f, key)?;
write!(f, "\":{}", v)?;
first = false;
} else {
write!(f, "null")?;
}
}
write!(f, "}}")
},
DecodedValue::Array(arr) => {
write!(f, "[")?;
let mut first = true;
for v in arr {
if !first {
write!(f, ",")?;
}
write!(f, "{}", v)?;
first = false;
}
write!(f, "]")
},
}
}
}
/// Returns the hash for a name.
fn key_stable_hash(s: &[u8]) -> KeyHash {
let mut state = 0;
for (i,b) in s.iter().enumerate() {
state ^= b.wrapping_add(i as u8);
}
state
}
impl<'a> DecodedValue<'a> {
pub const fn type_string(&self) -> &'static str {
match self {
DecodedValue::Null() => "null",
DecodedValue::Bool(_) => "bool",
DecodedValue::Int(_) => "int",
DecodedValue::Float(_) => "float",
DecodedValue::Double(_) => "double",
DecodedValue::String(_) => "string",
DecodedValue::Object(_) => "object",
DecodedValue::Array(_) => "array",
}
}
/// Returns the size needed if we serialize the Binary to a binary wire-format.
/// See [`DecodedValue::print_binary()`] for explanation of the `compact` argument.
pub fn size_of_binary(&self, compact: bool) -> usize {
let mut length_collector = RawWriterLength::new();
self.print_binary(&mut length_collector, compact).unwrap_infallible();
length_collector.length()
}
/// Converts a Binary to a wire representation of that Binary. This wire representation can be sent
/// to other implementations as it is stable and 'endian safe'.
///
/// For objects we have a compact
/// representation and one that inserts hints how to process the wire-representation faster when
/// using an expression to evaluate the Binary to another value. This is controlled with the
/// `compact` argument.
///
/// This function expects that the `writer` has enough room. The needed room can be queried
/// beforehand by using the [`DecodedValue::size_of_binary`].
pub fn print_binary<E, Out: RawOutput<E>>(&self, writer: &mut Out, compact: bool) -> Result<(),E> {
match self {
DecodedValue::Null() => {
writer.write_u8(ValueType::Null as u8)?;
Ok(())
}
DecodedValue::Bool(b) => {
if *b {
writer.write_u8(ValueType::BoolTrue as u8)?;
} else {
writer.write_u8(ValueType::BoolFalse as u8)?;
}
Ok(())
}
DecodedValue::Int(i) => {
if *i == 0 {
writer.write_u8(ValueType::Int as u8)?;
return Ok(());
}
let size = size_of_i64(*i);
debug_assert!(size <= 8);
writer.write_u8((size << WIDTH_OF_JSON_TYPE_MASK) as u8 | ValueType::Int as u8)?;
writer.write_i64(*i, size)?;
Ok(())
}
DecodedValue::Float(f) => {
writer.write_u8((4 << WIDTH_OF_JSON_TYPE_MASK) as u8 | ValueType::Float as u8)?;
writer.write_f32(*f)?;
Ok(())
}
DecodedValue::Double(d) => {
writer.write_u8((8 << WIDTH_OF_JSON_TYPE_MASK) as u8 | ValueType::Float as u8)?;
writer.write_f64(*d)?;
Ok(())
}
DecodedValue::String(s) => {
writer.write_var_u64((s.len() << WIDTH_OF_JSON_TYPE_MASK) as u64 | ValueType::String as u64)?;
writer.write_bytes(s)?;
Ok(())
}
DecodedValue::Array(arr) => {
if arr.is_empty() {
writer.write_u8(ValueType::Array as u8)?;
return Ok(());
}
write_with_header(writer, |writer,total| {
writer.write_var_u64((total as u64) << WIDTH_OF_JSON_TYPE_MASK | ValueType::Array as u64)
}, |writer| {
writer.write_var_u64((arr.len() as u64) << 1)?;
for e in arr {
e.print_binary(writer, compact)?;
}
Ok(())
})
}
DecodedValue::Object(obj) => {
if obj.is_empty() {
writer.write_u8(ValueType::Object as u8)?;
return Ok(());
}
let count = obj.len();
let local_compact = compact || count <= 4;
if !cfg!(feature = "mini") && !local_compact {
write_with_header(writer, |writer,total| {
writer.write_var_u64((total as u64) << WIDTH_OF_JSON_TYPE_MASK | ValueType::Object as u64)
}, |writer| {
let mut positions : Vec<(usize,u8)> = Vec::with_capacity(count); // (pos, hash)
let mut jumps : Vec<(usize,usize)> = Vec::new(); // from_pos, to_index
for (i,((hash,key), value)) in obj.iter().enumerate() {
for j in 0..i.trailing_zeros() {
let to_index = i + (2 << j);
if to_index >= count {
break;
}
// checks if the entry before has the same hash, if so, jumping has no use,
// so skip this jump
if obj.iter().nth(to_index-1).unwrap().0.0 == obj.iter().nth(to_index).unwrap().0.0 {
continue;
}
jumps.push((writer.pos(), to_index));
}
positions.push((writer.pos(),*hash));
let key_size = KEY_HASH_SIZE + key.len();
writer.write_var_u64((key_size as u64) << 1)?;
writer.write_u8(*hash)?;
writer.write_bytes(key)?;
value.print_binary(writer, compact)?;
}
// in reverse order pop jumps and insert headers with the new construct
for (from_pos, to_index) in jumps.iter().rev() {
let current = writer.pos();
let to_pos = positions[*to_index].0;
let jump = to_pos - from_pos;
writer.write_var_u64(((jump as u64) << 1) | 1)?;
writer.write_u8(positions[*to_index].1)?;
let extra = writer.pos() - current;
writer.swap_range(*from_pos, current)?;
// HINT: can be made a bit faster if we also store the from_index (part of
// the loop can be skipped)
for (pos,_) in positions.iter_mut() {
if *pos >= *from_pos {
*pos += extra;
}
}
}
Ok(())
})?;
return Ok(());
}
write_with_header(writer, |writer,total| {
writer.write_var_u64((total as u64) << WIDTH_OF_JSON_TYPE_MASK | ValueType::Object as u64)
}, |writer| {
for ((hash,key), value) in obj {
let key_size = KEY_HASH_SIZE + key.len();
writer.write_var_u64((key_size as u64) << 1)?;
writer.write_u8(*hash)?;
writer.write_bytes(key)?;
value.print_binary(writer, compact)?;
}
Ok(())
})?;
Ok(())
}
}
}
/// Converts a Binary to a wire representation of that Binary, in a self contained [`ValueVec`] unsing [`DecodedValue::print_binary`].
pub fn to_vec(&self, compact: bool) -> ValueVec {
if cfg!(feature = "mini") {
let mut writer = RawString::new();
self.print_binary(&mut writer, compact).expect("expry: problem during actual write");
ValueVec(writer.data)
} else {
let len = self.size_of_binary(compact);
let mut retval : Vec<u8> = vec![0u8; len];
let mut writer = RawWriter::with(&mut retval[..]);
self.print_binary(&mut writer, compact).expect("expry: calculated size differs from actual size");
debug_assert!(writer.left() == 0);
ValueVec(retval)
}
}
/// Converts a Binary to a wire representation of that Binary, in a self contained [`ValueVec`] unsing [`DecodedValue::print_binary`].
pub fn to_scope<'c>(&self, scope: &mut MemoryScope<'c>, compact: bool) -> ValueRef<'c> {
if cfg!(feature = "mini") {
// FIXME: convert to delayed deallocation function of MemoryPool
ValueRef(scope.copy_u8(self.to_vec(compact).get()))
} else {
let len = self.size_of_binary(compact);
let retval = scope.alloc(len);
let mut writer = RawWriter::with(retval);
self.print_binary(&mut writer, compact).expect("expry: calculated size differs from actual size");
debug_assert!(writer.left() == 0);
ValueRef(retval)
}
}
/// Convert a Binary value to a JSON output.
///
/// Note that some Binary values can not be encoded, because not all strings can be encoded as UTF-8. If this is the case, this method will result in a [`Utf8Error`].
pub fn print_json(&self, writer: &mut Vec<u8>) -> Result<(),Utf8Error> {
match self {
DecodedValue::Null() => {
writer.extend_from_slice(b"null");
}
DecodedValue::Bool(b) => {
if *b {
writer.extend_from_slice(b"true");
} else {
writer.extend_from_slice(b"false");
}
}
DecodedValue::Int(i) => {
writer.extend_from_slice(i.to_string().as_bytes());
}
DecodedValue::Float(f) => {
writer.extend_from_slice(format!("{:.99}", f).as_bytes());
}
DecodedValue::Double(d) => {
writer.extend_from_slice(format!("{:.99}", d).as_bytes());
}
DecodedValue::String(s) => {
writer.extend_from_slice(b"\"");
writer.extend_from_slice(&json_escape(core::str::from_utf8(s)?));
writer.extend_from_slice(b"\"");
}
DecodedValue::Array(arr) => {
writer.extend_from_slice(b"[");
let mut comma = false;
for e in arr {
if comma {
writer.extend_from_slice(b",");
}
e.print_json(writer)?;
comma = true;
}
writer.extend_from_slice(b"]");
}
DecodedValue::Object(obj) => {
writer.extend_from_slice(b"{");
let mut comma = false;
for ((_,k),v) in obj {
if comma {
writer.extend_from_slice(b",");
}
writer.extend_from_slice(b"\"");
writer.extend_from_slice(&json_escape(core::str::from_utf8(k)?));
writer.extend_from_slice(b"\":");
v.print_json(writer)?;
comma = true;
}
writer.extend_from_slice(b"}");
}
};
Ok(())
}
fn lookup_field(&self, key: Key) -> Result<Option<DecodedValue<'a>>,EncodingError> {
if let DecodedValue::Object(values) = self {
Ok(values.get(&key).cloned())
} else {
Err(EncodingError{line_nr: line!()}) // not an object
}
}
}
impl<'a, 'b, 'c> CloneInMemoryScope<'c, DecodedValue<'b>> for DecodedValue<'a> where 'c: 'b {
fn clone_in(&self, scope: &mut MemoryScope<'c>) -> DecodedValue<'b> {
match self {
DecodedValue::Null() => DecodedValue::Null(),
DecodedValue::Bool(v) => DecodedValue::Bool(*v),
DecodedValue::Int(v) => DecodedValue::Int(*v),
DecodedValue::Float(v) => DecodedValue::Float(*v),
DecodedValue::Double(v) => DecodedValue::Double(*v),
DecodedValue::String(s) => DecodedValue::String(scope.copy_u8(s)),
DecodedValue::Object(values) => {
let mut retval = DecodedObject::new();
for ((hash, k),v) in values {
retval.insert((*hash, scope.copy_u8(k)), v.clone_in(scope));
}
DecodedValue::Object(retval)
},
DecodedValue::Array(values) => {
let mut retval = DecodedArray::new();
for v in values {
retval.push(v.clone_in(scope));
}
DecodedValue::Array(retval)
}
}
}
}
fn write_field(key: &[u8], value: &[u8], writer: &mut RawWriter) -> Result<(),EncodingError> {
let hash = key_stable_hash(key);
write_field_key((hash, key), value, writer)
}
fn write_field_key(key: Key, value: &[u8], writer: &mut RawWriter) -> Result<(),EncodingError> {
let key_size = KEY_HASH_SIZE + key.1.len();
writer.write_var_u64((key_size as u64) << 1)?;
writer.write_u8(key.0)?;
writer.write_bytes(key.1)?;
writer.write_bytes(value)?;
Ok(())
}
/// Creates an object with a single key-value pair inside. Expects `value` to be binary encoded.
pub fn expry_object_with_single_field(key: &[u8], value: &[u8]) -> Result<Vec<u8>,EncodingError> {
let key_size = key.len() + KEY_HASH_SIZE;
let entry = size_of_var_u64((key_size as u64) << 1) + key_size + value.len();
let object_size = size_of_var_u64((entry << WIDTH_OF_JSON_TYPE_MASK) as u64) + entry;
let mut retval : Vec<u8> = vec![0u8; object_size];
let mut writer = RawWriter::with(&mut retval[..]);
write_with_header(&mut writer, |writer,total| {
writer.write_var_u64((total as u64) << WIDTH_OF_JSON_TYPE_MASK | ValueType::Object as u64)
}, |writer| {
write_field(key, value, writer)
})?;
assert_eq!(writer.left(), 0);
Ok(retval)
}
/// create a binary object from individual fields
pub fn expry_object_raw<'c>(scope: &mut MemoryScope<'c>, slice: &[(Key<'_>, &[u8])]) -> Result<&'c [u8],EncodingError> {
let mut content_size = 0;
for (key,value) in slice {
let key_size = key.1.len() + KEY_HASH_SIZE;
let entry = size_of_var_u64((key_size as u64) << 1) + key_size + value.len();
content_size += entry;
}
let object_size = size_of_var_u64((content_size << WIDTH_OF_JSON_TYPE_MASK) as u64) + content_size;
let retval = scope.alloc(object_size);
let mut writer = RawWriter::with(retval);
write_with_header(&mut writer, |writer,total| {
writer.write_var_u64((total as u64) << WIDTH_OF_JSON_TYPE_MASK | ValueType::Object as u64)
}, |writer| {
for (key,value) in slice {
write_field_key(*key, value, writer)?;
}
Ok(())
})?;
assert_eq!(writer.left(), 0);
Ok(retval)
}
fn json_escape(src: &str) -> Vec<u8> {
let mut escaped : Vec<u8> = Vec::with_capacity(src.len());
for c in src.chars() {
match c {
'\x08' => escaped.extend_from_slice(b"\\b"),
'\x0c' => escaped.extend_from_slice(b"\\f"),
'\n' => escaped.extend_from_slice(b"\\n"),
'\r' => escaped.extend_from_slice(b"\\r"),
'\t' => escaped.extend_from_slice(b"\\t"),
'"' => escaped.extend_from_slice(b"\\\""),
'\\' => escaped.extend_from_slice(b"\\\\"),
' ' => escaped.extend_from_slice(b" "),
// c.is_ascii_graphic()
c if (c as u32) < 0x20 => {
let mut utf16_buf = [0u16; 2];
let encoded = c.encode_utf16(&mut utf16_buf);
for utf16 in encoded {
let s = format!("\\u{:04X}", utf16);
escaped.extend_from_slice(s.as_bytes());
}
},
c => {
let mut utf8_buf = [0u8; 4];
escaped.extend_from_slice(c.encode_utf8(&mut utf8_buf).as_bytes());
},
}
}
escaped
}
/// Calculates the key as used in this crate from a str. There is also a `[u8]` version.
pub fn key_str(key: &str) -> Key<'_> {
let hash = key_stable_hash(key.as_bytes());
(hash, key.as_bytes())
}
/// Calculates the key as used in this crate from an u8 slice. There is also a `str` version.
pub fn key_u8(key: &[u8]) -> Key<'_> {
let hash = key_stable_hash(key);
(hash, key)
}
impl TryFrom<usize> for ValueType {
type Error = EncodingError;
fn try_from(v: usize) -> Result<Self, Self::Error> {
match v {
x if x == ValueType::Null as usize => Ok(ValueType::Null),
x if x == ValueType::String as usize => Ok(ValueType::String),
x if x == ValueType::BoolFalse as usize => Ok(ValueType::BoolFalse),
x if x == ValueType::BoolTrue as usize => Ok(ValueType::BoolTrue),
x if x == ValueType::Float as usize => Ok(ValueType::Float),
x if x == ValueType::Int as usize => Ok(ValueType::Int),
x if x == ValueType::Object as usize => Ok(ValueType::Object),
x if x == ValueType::Array as usize => Ok(ValueType::Array),
_ => Err(EncodingError { line_nr: line!() }),
}
}
}
impl<'a> DecodedValue<'a> {
pub fn decoded_type_of(type_and_length: u64) -> Result<ValueType, EncodingError> {
((type_and_length as usize) & JSON_TYPE_MASK).try_into()
}
/// Based on a variable length integer read for the wire, return the [`ValueType`] raw number.
pub const fn type_of_binary(type_and_length: u64) -> u8 {
((type_and_length as usize) & JSON_TYPE_MASK) as u8
}
/// Based on a variable length integer read for the wire, return the length of the current
/// entry.
pub const fn length_of_binary(type_and_length: u64) -> usize {
(type_and_length >> WIDTH_OF_JSON_TYPE_MASK) as usize
}
pub fn decoded_type_and_length(type_and_length: u64) -> Result<(ValueType, usize), EncodingError> {
Ok((Self::decoded_type_of(type_and_length)?, Self::length_of_binary(type_and_length)))
}
/// Parse the wire-format.
pub fn parse(reader: &mut RawReader<'a>) -> Result<DecodedValue<'a>, EncodingError> {
let type_and_length = reader.read_var_u64()?;
let (t, len) = Self::decoded_type_and_length(type_and_length)?;
match t {
ValueType::Null => {
if len == 0 {
Ok(DecodedValue::Null())
} else {
Err(EncodingError { line_nr: line!() })
}
}
ValueType::String => {
let value = reader.read_bytes(len)?;
Ok(DecodedValue::String(value))
},
ValueType::BoolFalse => {
if len == 0 {
Ok(DecodedValue::Bool(false))
} else {
Err(EncodingError { line_nr: line!() })
}
}
ValueType::BoolTrue => {
if len == 0 {
Ok(DecodedValue::Bool(true))
} else {
Err(EncodingError { line_nr: line!() })
}
}
ValueType::Float => {
if len == 4 {
let value = reader.read_f32()?;
Ok(DecodedValue::Float(value))
} else if len == 8 {
let value = reader.read_f64()?;
Ok(DecodedValue::Double(value))
} else {
Err(EncodingError { line_nr: line!() })
}
}
ValueType::Int => {
let v = reader.read_i64(len)?;
Ok(DecodedValue::Int(v))
}
ValueType::Array => {
if len == 0 {
return Ok(DecodedValue::Array(vec![]));
}
let mut subreader = RawReader::with(reader.read_bytes(len)?);
let mut count = subreader.read_var_u64()?;
count >>= 1;
let mut retval : Vec<DecodedValue> = Vec::with_capacity(min(16384, count as usize)); // limit the max allocation to prevent rouge input data leading to out of memory problems
for _ in 0 .. count {
let v = DecodedValue::parse(&mut subreader)?;
retval.push(v);
}
Ok(DecodedValue::Array(retval))
},
ValueType::Object => {
if len == 0 {
return Ok(DecodedValue::Object(DecodedObject::new()));
}
let mut last : (KeyHash, &[u8]) = (0, b"");
let mut retval = DecodedObject::new();
let mut subreader = RawReader::with(reader.read_bytes(len)?);
while !subreader.is_empty() {
let mut key_length= subreader.read_var_u64()?;
if key_length == 0 {
return Err(EncodingError{ line_nr: line!() });
}
if key_length & 0x1 != 0 {
subreader.read_u8()?;
continue;
}
key_length = (key_length >> 1) - 1; // - 1 to ignore hash (we handle it differently)
if key_length as usize > subreader.len() {
return Err(EncodingError{ line_nr: line!() });
}
let hash = subreader.read_u8()?;
let key = (hash, subreader.read_bytes(key_length as usize)?);
//let hash = binary_object_stable_hash(&key);
let v = DecodedValue::parse(&mut subreader)?;
retval.insert(key, v);
if key < last {
// object not sorted, that is an encoding error
return Err(EncodingError{line_nr: line!(), });
}
last = key;
}
debug_assert!(subreader.is_empty());
Ok(DecodedValue::Object(retval))
},
}
}
}
#[derive(PartialEq, Copy, Clone)]
pub enum LazyDecodedValue<'a> {
Null(),
Bool(bool),
Int(i64),
Float(f32),
Double(f64),
String(&'a [u8]), // ugly in Rust, because a String is forced to be Unicode
Object(LazyDecodedObject<'a>),
Array(LazyDecodedArray<'a>),
}
#[derive(PartialEq, Eq, Copy, Clone)]
pub struct LazyDecodedObject<'a> {
reader: RawReader<'a>,
}
impl<'a> LazyDecodedObject<'a> {
fn _read_value(reader: &mut RawReader<'a>) -> Result<&'a [u8],EncodingError> {
let mut reader_type = *reader;
let type_and_length = reader_type.read_var_u64()?;
let len = DecodedValue::length_of_binary(type_and_length);
reader_type.skip(len)?;
let v = reader.read_bytes(reader.len() - reader_type.len())?;
Ok(v)
}
fn _read_key(reader: &mut RawReader<'a>) -> Result<Key<'a>,EncodingError> {
let mut key_length = reader.read_var_u64()?;
if key_length == 0 || key_length & 0x1 != 0 {
// skip lists should have been read away by the the is_empty()
return Err(EncodingError{ line_nr: line!() });
}
key_length = (key_length >> 1) - 1; // - 1 to ignore hash (we handle it differently)
let hash = reader.read_u8()?;
let key = reader.read_bytes(key_length as usize)?;
Ok((hash, key))
}
fn _read(reader: &mut RawReader<'a>) -> Result<(Key<'a>, &'a [u8]),EncodingError> {
Ok((Self::_read_key(reader)?, Self::_read_value(reader)?))
}
pub fn lookup_binary<'b>(&self, key: (KeyHash, &'b [u8])) -> Result<Option<&'a [u8]>,EncodingError> where 'a: 'b {
let mut value_reader = self.reader;
match evaluate_seek(key.0, key.1, &mut value_reader) {
Ok(_) => {},
Err(EvalError::FieldNotFound(_)) => return Ok(None),
Err(_) => return Err(EncodingError{ line_nr: line!(), }),
}
let binary_value = Self::_read(&mut value_reader)?.1;
Ok(Some(binary_value))
}
pub fn lookup_value<'b>(&self, key: (KeyHash, &'b [u8])) -> Result<Option<LazyDecodedValue<'a>>,EncodingError> where 'a: 'b {
let binary_value = self.lookup_binary(key)?;
if let Some(binary_value) = binary_value {
Ok(Some(LazyDecodedValue::parse(&mut RawReader::with(binary_value))?))
} else {
Ok(None)
}
}
pub fn get(&mut self) -> Result<(Key<'a>,LazyDecodedValue),EncodingError> {
let (key, value) = LazyDecodedObject::<'a>::_read(&mut self.reader)?;
Ok((key, LazyDecodedValue::parse(&mut RawReader::with(value))?))
}
pub fn is_empty(&mut self) -> Result<bool,EncodingError> {
while self.reader.len() > 0 && self.reader.clone().read_var_u64()? & 0x1 != 0 {
self.reader.read_var_u64()?;
self.reader.read_u8()?;
}
Ok(self.reader.is_empty())
}
pub fn new(reader: RawReader<'a>) -> Self {
Self {
reader,
}
}
}
#[derive(PartialEq, Eq, Copy, Clone)]
pub struct LazyDecodedArray<'a> {
reader: RawReader<'a>,
count: u64,
}
impl<'a> LazyDecodedArray<'a> {
pub fn get(&mut self) -> Result<LazyDecodedValue,EncodingError> {
if self.count == 0 {
return Err(EncodingError{ line_nr: line!() });
}
debug_assert!(!self.reader.is_empty());
self.count -= 1;
LazyDecodedValue::parse(&mut self.reader)
}
pub fn get_raw(&mut self) -> Result<ValueRef<'a>,EncodingError> {
self.count -= 1;
let type_and_length = self.reader.clone().read_var_u64()?;
let len = DecodedValue::length_of_binary(type_and_length);
Ok(ValueRef(self.reader.read_bytes(len + size_of_var_u64(type_and_length))?))
}
pub fn remaining(&self) -> u64 {
self.count
}
pub fn is_empty(&self) -> bool {
self.count == 0
}
}
impl<'a> LazyDecodedValue<'a> {
pub fn type_string(&self) -> &'static str {
match self {
LazyDecodedValue::Null() => "null",
LazyDecodedValue::Bool(_) => "bool",
LazyDecodedValue::Int(_) => "int",
LazyDecodedValue::Float(_) => "float",
LazyDecodedValue::Double(_) => "double",
LazyDecodedValue::String(_) => "string",
LazyDecodedValue::Object(_) => "object",
LazyDecodedValue::Array(_) => "array",
}
}
pub fn parse(reader: &mut RawReader<'a>) -> Result<LazyDecodedValue<'a>, EncodingError> {
let type_and_length = reader.read_var_u64()?;
let (t, len) = DecodedValue::decoded_type_and_length(type_and_length)?;
match t {
ValueType::Null => {
if len == 0 {
Ok(LazyDecodedValue::Null())
} else {
Err(EncodingError { line_nr: line!() })
}
}
ValueType::String => {
let value = reader.read_bytes(len)?;
Ok(LazyDecodedValue::String(value))
},
ValueType::BoolFalse => {
if len == 0 {
Ok(LazyDecodedValue::Bool(false))
} else {
Err(EncodingError { line_nr: line!() })
}
}
ValueType::BoolTrue => {
if len == 0 {
Ok(LazyDecodedValue::Bool(true))
} else {
Err(EncodingError { line_nr: line!() })
}
}
ValueType::Float => {
if len == 4 {
let value = reader.read_f32()?;
Ok(LazyDecodedValue::Float(value))
} else if len == 8 {
let value = reader.read_f64()?;
Ok(LazyDecodedValue::Double(value))
} else {
Err(EncodingError { line_nr: line!() })
}
}
ValueType::Int => {
let v = reader.read_i64(len)?;
Ok(LazyDecodedValue::Int(v))
}
ValueType::Array => {
if len == 0 {
return Ok(LazyDecodedValue::Array(LazyDecodedArray { count: 0, reader: RawReader::with(b""), }));
}
let mut subreader = RawReader::with(reader.read_bytes(len)?);
let mut count = subreader.read_var_u64()?;
count >>= 1;
let retval = LazyDecodedArray {
count,
reader: subreader,
};
Ok(LazyDecodedValue::Array(retval))
},
ValueType::Object => {
if len == 0 {
return Ok(LazyDecodedValue::Object(LazyDecodedObject { reader: RawReader::with(b""), }));
}
Ok(LazyDecodedValue::Object(LazyDecodedObject::new(RawReader::with(reader.read_bytes(len)?))))
},
}
}
}
/// Reference to a expry value.
#[derive(PartialEq, Eq, Copy, Clone, Debug)]
pub struct ValueRef<'a>(pub &'a [u8]);
impl<'a> ValueRef<'a> {
fn lookup_field(&self, key: Key) -> Result<Option<DecodedValue<'a>>,EncodingError> {
if self.get().is_empty() {
return Ok(None);
}
let mut value_reader = RawReader::with(self.get());
if DecodedValue::type_of_binary(value_reader.read_var_u64()?) != ValueType::Object as u8 {
return Err(EncodingError{line_nr: line!()});
}
match evaluate_seek(key.0, key.1, &mut value_reader) {
Ok(_) => {},
Err(EvalError::FieldNotFound(_)) => return Ok(None),
Err(_) => return Err(EncodingError{line_nr: line!()}),
}
let key_length = value_reader.read_var_u64()?;
if key_length == 0 || (key_length & 0x1) != 0 {
return Err(EncodingError{line_nr: line!()});
}
let _key_name = &value_reader.read_bytes(key_length as usize >> 1)?[1..];
Ok(Some(DecodedValue::parse(&mut value_reader)?))
}
pub fn get(&self) -> &'a [u8] {
self.0
}
pub fn is_empty(&self) -> bool {
self.0.is_empty()
}
pub fn new() -> Self {
Self(b"")
}
}
impl<'a> Default for ValueRef<'a> {
fn default() -> Self {
Self::new()
}
}
/// Self-contained expry value (in encoded form).
#[derive(PartialEq, Eq, Clone, Debug)]
pub struct ValueVec(pub Vec<u8>);
/// Reference to expry expression bytecode.
#[derive(PartialEq, Eq, Copy, Clone, Debug)]
pub struct BytecodeRef<'a>(pub &'a [u8]);
impl<'a> BytecodeRef<'a> {
pub fn get(&self) -> &'a [u8] {
self.0
}
pub fn is_empty(&self) -> bool {
self.0.is_empty()
}
pub fn new() -> Self {
Self(b"")
}
}
impl<'a> Default for BytecodeRef<'a> {
fn default() -> Self {
Self::new()
}
}
/// Self-contained expry expression bytecode.
#[derive(PartialEq, Eq, Clone, Debug)]
pub struct BytecodeVec(pub Vec<u8>);
impl BytecodeVec {
pub fn len(&self) -> usize {
self.0.len()
}
pub fn to_ref(&self) -> BytecodeRef {
BytecodeRef(&self.0)
}
pub fn get(&self) -> &[u8] {
&self.0
}
pub fn is_empty(&self) -> bool {
self.0.is_empty()
}
}
impl ValueVec {
pub fn len(&self) -> usize {
self.0.len()
}
pub fn to_ref(&self) -> ValueRef {
ValueRef(&self.0)
}
pub fn get(&self) -> &[u8] {
&self.0
}
pub fn is_empty(&self) -> bool {
self.0.is_empty()
}
}
// we can use the .into() method to convert (but this is an explicit action)
impl Into<ValueVec> for Vec<u8> {
fn into(self) -> ValueVec {
ValueVec(self)
}
}
impl Into<BytecodeVec> for Vec<u8> {
fn into(self) -> BytecodeVec {
BytecodeVec(self)
}
}
// safe of misuse to convert from value/bytecode to vec
impl Into<Vec<u8>> for ValueVec {
fn into(self) -> Vec<u8> {
self.0
}
}
impl Into<Vec<u8>> for BytecodeVec {
fn into(self) -> Vec<u8> {
self.0
}
}
// reads away skip lists
// returns current key, entry length, and the payload type+length
fn peek_key_and_entry<'a>(reader: &'_ mut RawReader<'a>) -> Result<(KeyHash,&'a [u8],usize,usize),EncodingError> {
let mut peeker = *reader;
while !peeker.is_empty() {
let mut key_length = peeker.read_var_u64()?;
if key_length == 0 {
return Err(EncodingError{ line_nr: line!() });
}
if (key_length & 0x1) != 0 {
let _hash_of_next_entry = peeker.read_u8()?; // read hash of forward reference
*reader = peeker;
continue;
}
key_length = (key_length >> 1) - KEY_HASH_SIZE as u64;
let hash = peeker.read_u8()?;
let key = peeker.read_bytes(key_length as usize)?;
let type_and_length = peeker.read_var_u64()?;
peeker.skip(DecodedValue::length_of_binary(type_and_length))?;
return Ok((hash, key, reader.len() - peeker.len(), type_and_length as usize));
}
Ok((0, b"", 0, 0))
}
/// Merges two objects. Left hand side take preference (so the left hand side value of keys occuring in
/// both is taken). Always producing a compact object
pub fn merge_objects<'b, 'c>(lhs: ValueRef, rhs: ValueRef, allocator: &mut MemoryScope<'c>) -> Result<ValueRef<'b>, EncodingError> where 'c: 'b {
let mut lhs_reader = RawReader::with(lhs.get());
let mut rhs_reader = RawReader::with(rhs.get());
/*
// Simple version with higher memory usage
let lhs = DecodedValue::parse(&mut lhs_reader)?;
let rhs = DecodedValue::parse(&mut rhs_reader)?;
if let (DecodedValue::Object(mut lhs),DecodedValue::Object(rhs)) = (lhs, rhs) {
for (k,v) in rhs {
lhs.entry(k).or_insert(v);
}
Ok(DecodedValue::Object(lhs).to_scope(allocator, true))
} else {
Err(EncodingError{ line_nr: line!() })
}
*/
let lhs_length_and_type = lhs_reader.read_var_u64()?;
if DecodedValue::type_of_binary(lhs_length_and_type) != ValueType::Object as u8 {
return Err(EncodingError{ line_nr: line!() });
}
if DecodedValue::length_of_binary(lhs_length_and_type) != lhs_reader.len() {
return Err(EncodingError{ line_nr: line!() });
}
let rhs_length_and_type = rhs_reader.read_var_u64()?;
if DecodedValue::type_of_binary(rhs_length_and_type) != ValueType::Object as u8 {
return Err(EncodingError{ line_nr: line!() });
}
if DecodedValue::length_of_binary(rhs_length_and_type) != rhs_reader.len() {
return Err(EncodingError{ line_nr: line!() });
}
let mut chunks : ScopedArrayBuilder<'_, 'c, &[u8]> = ScopedArrayBuilder::new(allocator);
chunks.push(b""); // place holder we are going to replace at the end with an header
let mut total : usize = 0;
let mut lhs_current : (KeyHash,&[u8],usize,usize) = peek_key_and_entry(&mut lhs_reader)?;
let mut rhs_current : (KeyHash,&[u8],usize,usize) = peek_key_and_entry(&mut rhs_reader)?;
while !lhs_current.1.is_empty() || !rhs_current.1.is_empty() {
let (lhs_hash, lhs_key, lhs_entry_length, lhs_value_length_and_type) = lhs_current;
let (rhs_hash, rhs_key, rhs_entry_length, _) = rhs_current;
let mut entry : &[u8];
if lhs_entry_length > 0 && (lhs_hash, lhs_key) <= (rhs_hash, rhs_key) || rhs_entry_length == 0 {
entry = lhs_reader.read_bytes(lhs_entry_length)?;
// special 'remove' marker as generated by CreateObjectDiff
if lhs_value_length_and_type == ValueType::Float as usize { // float of 0 bytes implies a size of 0
entry = b"";
}
// if key is in both, skip the rhs one
if (lhs_hash, lhs_key) == (rhs_hash, rhs_key) {
rhs_reader.skip(rhs_entry_length)?;
rhs_current = peek_key_and_entry(&mut rhs_reader)?;
}
lhs_current = peek_key_and_entry(&mut lhs_reader)?;
} else {
entry = rhs_reader.read_bytes(rhs_entry_length)?;
rhs_current = peek_key_and_entry(&mut rhs_reader)?;
}
if !entry.is_empty() {
chunks.push(entry);
total += entry.len();
}
}
let retval = chunks.build();
retval[0] = to_var_length(allocator, total << WIDTH_OF_JSON_TYPE_MASK | ValueType::Object as usize);
let retval = allocator.concat_u8(retval);
Ok(ValueRef(retval))
}
#[derive(PartialEq, Copy, Clone, Debug)]
enum JSONToken<'a> {
End(),
Bool(bool),
String(&'a [u8]),
Float(f32),
Double(f64),
Int(i64),
Comma(),
ArrayOpen(),
ArrayClose(),
ObjectOpen(),
Colon(),
ObjectClose(),
NullObject(),
Comparison(ExpryComparison),
Arithmetic(Arithmetic),
Logical(Logical),
Bitwise(Bitwise),
Open(),
Close(),
As(),
Not(),
Conditional(),
NullOperator(),
TryOperator(),
Concat(),
Dot(),
Spread(),
Field(&'a str),
FunctionCall(&'a str),
}
#[derive(Debug, Copy, Clone, PartialEq, Eq)]
pub struct LexerError {
line_nr: u32,
}
#[derive(Debug, Copy, Clone, PartialEq, Eq)]
pub enum CompileErrorDescription<'a> {
NoToken(),
Lexer(LexerError),
Parser(&'a str),
Optimizer(EvalError<'a>),
}
#[derive(Clone, Debug)]
pub struct CompileError<'a> {
pub error: CompileErrorDescription<'a>,
pub start: usize, // counted from the back of the input string
pub end: usize, // counted from the back of the input string
pub extra: Option<(usize, usize)>,
}
impl<'a> CompileError<'a> {
pub fn error_start(&self) -> usize {
self.start
}
pub fn error_end(&self) -> usize {
self.end
}
pub fn error(&self) -> CompileErrorDescription<'a> {
self.error
}
pub fn extra(&self) -> Option<(usize, usize)> {
self.extra
}
}
impl<'a> core::fmt::Display for CompileErrorDescription<'a> {
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
match self {
CompileErrorDescription::NoToken() => write!(f, "could not tokenize remaining value"),
CompileErrorDescription::Lexer(err) => write!(f, "{:?} (in lexer)", err),
CompileErrorDescription::Parser(msg) => write!(f, "{}", msg),
CompileErrorDescription::Optimizer(msg) => write!(f, "{} (during optimizing)", msg),
}
}
}
fn get_codepoint_from_hex(chars: &mut Chars) -> Result<u32,LexerError> {
let first = chars.next().and_then(|x| from_hex(x).ok());
let second = chars.next().and_then(|x| from_hex(x).ok());
let third = chars.next().and_then(|x| from_hex(x).ok());
let fourth = chars.next().and_then(|x| from_hex(x).ok());
if let (Some(first), Some(second), Some(third), Some(fourth)) = (first, second, third, fourth) {
let unicode = (first as u32) << 12 | (second as u32) << 8 | (third as u32) << 4 | (fourth as u32);
Ok(unicode)
} else {
Err(LexerError{ line_nr: line!() })
}
}
fn json_unescape<'b>(str: &'_ str, allocator: &'_ mut MemoryScope<'b>, extended: bool) -> Result<&'b [u8], (LexerError,usize)> {
// see spec http://www.ietf.org/rfc/rfc4627.txt; section 2.5:
let mut retval = ScopedStringBuilder::new(allocator);
let mut chars = str.chars();
let mut err_pos = 0;
while let Some(c) = chars.next() {
err_pos += 1;
if c != '\\' {
retval.push_char(c);
continue;
}
match chars.next() {
Some('b') => retval.push_char('\x08'),
Some('f') => retval.push_char('\x0c'),
Some('n') => retval.push_char('\n'),
Some('r') => retval.push_char('\r'),
Some('t') => retval.push_char('\t'),
Some('\\') => retval.push_char('\\'),
Some('"') => retval.push_char('"'),
Some('\'') if extended => retval.push_char('\''),
Some('u') => {
let unicode = get_codepoint_from_hex(&mut chars).map_err(|x| (x,err_pos))?;
err_pos += 4;
if unicode <= 0x7F {
retval.push_char((unicode & 0x7F) as u8 as char);
} else if unicode <= 0x7FF {
retval.push_char(char::from_u32(unicode).ok_or((LexerError{ line_nr: line!()}, err_pos))?);
} else if (0xD800..=0xDBFF).contains(&unicode) {
// surrogate pair
if let (Some('\\'), Some('u')) = (chars.next(), chars.next()) {
} else {
return Err((LexerError{ line_nr: line!()}, err_pos));
}
let unicode2 = get_codepoint_from_hex(&mut chars).map_err(|x| (x, err_pos))?;
err_pos += 4;
let u = (((unicode & 0x3FF) as u32) << 10 | (unicode2 & 0x3FF) as u32) + 0x10000;
retval.push_char(char::from_u32(u).ok_or((LexerError{ line_nr: line!()}, err_pos))?);
} else {
retval.push_char(char::from_u32(unicode).ok_or((LexerError{ line_nr: line!()}, err_pos))?);
}
},
_ => return Err((LexerError{ line_nr: line!()}, err_pos)),
}
}
Ok(retval.build().as_bytes())
}
fn json_tokenize<'b,'c>(reader: &mut &'b str, context: &mut JSONParserContext<'_,'b,'c>) -> Result<(JSONToken<'b>,usize), (CompileErrorDescription<'b>,usize,usize)>
where 'c: 'b
{
// might be faster using https://github.com/maciejhirsz/logos
*reader = reader.trim_start();
let remaining = reader.len();
if reader.is_empty() {
return Ok((JSONToken::End(), remaining));
}
if reader.accept("null") {
return Ok((JSONToken::NullObject(), remaining));
}
if reader.accept("true") {
return Ok((JSONToken::Bool(true), remaining));
}
if reader.accept("false") {
return Ok((JSONToken::Bool(false), remaining));
}
if reader.accept(",") {
return Ok((JSONToken::Comma(), remaining));
}
if reader.accept("[") {
return Ok((JSONToken::ArrayOpen(), remaining));
}
if reader.accept("]") {
return Ok((JSONToken::ArrayClose(), remaining));
}
if reader.accept("{") {
return Ok((JSONToken::ObjectOpen(), remaining));
}
if reader.accept(":") {
return Ok((JSONToken::Colon(), remaining));
}
if reader.accept("}") {
return Ok((JSONToken::ObjectClose(), remaining));
}
let mut string_matcher = StringLiteralSpanner::new('"');
if let Some(value) = string_matcher.span(reader) {
debug_assert!(value.len() >= 2);
if string_matcher.unescape_needed {
let err = |x,offset| (CompileErrorDescription::Lexer(x), remaining-1-offset,reader.len());
let unescaped = json_unescape(&value[1..value.len()-1], context.allocator, false).map_err(|(x,offset)| err(x, offset))?;
return Ok((JSONToken::String(unescaped), remaining));
}
return Ok((JSONToken::String(value[1..value.len()-1].as_bytes()), remaining));
}
// FIXME:
// 0 is a special case; numbers can not start with 0 execpt if it is the number zero or a real number with only digits after the comma
// FIXME: -0000000.1 is not accounted for (is not allowed)
// -0 is allowed!
let mut number_matcher = NumberSpanner::new();
if let Some(value) = number_matcher.span(reader) {
let err = || (CompileErrorDescription::Lexer(LexerError{ line_nr: line!() }), remaining,reader.len());
if !number_matcher.float {
// given the number spanner, we know that value is UTF8
let number : i64 = value.parse().map_err(|_| err())?;
return Ok((JSONToken::Int(number), remaining));
}
// given the number spanner, we know that value is UTF8
let number : f64 = value.parse().map_err(|_| err())?;
return Ok((JSONToken::Double(number), remaining));
}
Err((CompileErrorDescription::NoToken(), remaining,reader.len()))
}
struct JSONParserContext<'a,'b,'c> {
allocator: &'a mut MemoryScope<'c>,
static_value: &'a DecodedValue<'b>,
}
type JSONParserState<'b,'a,'c> = ParserState<'b,JSONToken<'b>,CompileErrorDescription<'b>, JSONParserContext<'a,'b,'c>>;
type JSONParserResult<'b,T> = ParserResult<T,CompileErrorDescription<'b>>;
fn json_expr<'b>(parser: &mut JSONParserState<'b,'_,'_>) -> JSONParserResult<'b,DecodedValue<'b>> {
match parser.get()? {
(JSONToken::NullObject(),_) => Ok(DecodedValue::Null()),
(JSONToken::Bool(b),_) => Ok(DecodedValue::Bool(b)),
(JSONToken::Int(b),_) => Ok(DecodedValue::Int(b)),
// strictly not part of the JSON spec, so depends of the lexer produces a float. This is used in the predicate parsing
(JSONToken::Float(b),_) => Ok(DecodedValue::Float(b)),
(JSONToken::Double(b),_) => Ok(DecodedValue::Double(b)),
(JSONToken::String(b),_) => Ok(DecodedValue::String(b)),
(JSONToken::ObjectOpen(), open) => {
let mut retval = DecodedObject::new();
parser.opt(|parser| {
json_object_entry(parser, &mut retval)?;
parser.repeat(|parser| {
parser.accept(JSONToken::Comma(), None, || CompileErrorDescription::Parser(""))?;
json_object_entry(parser, &mut retval)?;
Ok(true)
})
})?;
parser.accept(JSONToken::ObjectClose(), Some(&open), || CompileErrorDescription::Parser("expected '}' to close object"))?;
Ok(DecodedValue::Object(retval))
},
(JSONToken::ArrayOpen(), open) => {
let mut retval : Vec<DecodedValue<'b>> = Vec::new();
parser.opt(|parser| {
json_array_entry(parser, &mut retval)?;
parser.repeat(|parser| {
parser.accept(JSONToken::Comma(), None, || CompileErrorDescription::Parser(""))?;
json_array_entry(parser, &mut retval)?;
Ok(true)
})
})?;
parser.accept(JSONToken::ArrayClose(), Some(&open), || CompileErrorDescription::Parser("expected ']' to close array"))?;
Ok(DecodedValue::Array(retval))
}
(token,info) => Err(parser.error_token(token, info, |_| CompileErrorDescription::Parser("expected primary"))),
}
}
fn json_object_entry<'b>(parser: &mut JSONParserState<'b,'_,'_>, value: &mut DecodedObject<'b>) -> JSONParserResult<'b,()> {
match parser.get()? {
(JSONToken::String(k),_) => {
parser.accept(JSONToken::Colon(), None, || CompileErrorDescription::Parser("expected ':' as field delimeter for an object field"))?;
let v = json_expr(parser)?;
value.insert(key_u8(k), v);
Ok(())
},
(token,info) => Err(parser.error_token(token, info, |_| CompileErrorDescription::Parser("expected string as object field"))),
}
}
fn json_array_entry<'b>(parser: &mut JSONParserState<'b,'_,'_>, value: &mut Vec<DecodedValue<'b>>) -> JSONParserResult<'b,()> {
let v = json_expr(parser)?;
value.push(v);
Ok(())
}
fn json_top_level<'b>(parser: &mut JSONParserState<'b,'_,'_>) -> JSONParserResult<'b,DecodedValue<'b>> {
let retval = json_expr(parser)?;
parser.accept(JSONToken::End(), None, || CompileErrorDescription::Parser("expected end of input (possibly an operator was missing, or a '{'/'[')"))?;
Ok(retval)
}
impl<'a> DecodedValue<'a> {
pub fn parse_json<'c>(reader: &'a str, allocator: &'_ mut MemoryScope<'c>) -> Result<DecodedValue<'a>,CompileError<'a>> where 'c: 'a {
let static_value = DecodedValue::Object(DecodedObject::new());
let mut parser = JSONParserState::new_with(reader, json_tokenize, JSONParserContext{allocator, static_value: &static_value});
parser.parse(json_top_level, CompileErrorDescription::Parser("unexpected token"), CompileErrorDescription::Parser("too much parser recursion")).map_err(|(error, start, end, extra)| CompileError{ error, start, end, extra })
}
pub fn parse_expry_value<'c>(reader: &'a str, scope: &'_ mut MemoryScope<'c>) -> Result<DecodedValue<'a>,CompileError<'a>> where 'c: 'a {
let bytecode = expry_compile(reader, None, scope)?;
expry_eval(bytecode, ValueRef::new(), scope).map_err(|err| err.into())
}
}
#[derive(PartialEq, Eq, Copy, Clone, Debug)]
enum ExpryComparison {
Equal,
NotEqual,
LessEqual,
GreaterEqual,
Less,
Greater,
Contains,
StartsWith,
EndsWith,
}
impl ExpryComparison {
fn reverse(&self) -> Option<ExpryComparison> {
match self {
ExpryComparison::LessEqual => Some(ExpryComparison::GreaterEqual),
ExpryComparison::Less => Some(ExpryComparison::Greater),
ExpryComparison::GreaterEqual => Some(ExpryComparison::LessEqual),
ExpryComparison::Greater => Some(ExpryComparison::Less),
ExpryComparison::Equal => Some(ExpryComparison::Equal),
ExpryComparison::NotEqual => Some(ExpryComparison::NotEqual),
_ => None,
}
}
}
#[derive(PartialEq, Eq, Copy, Clone, Debug)]
enum Arithmetic {
Add,
Sub,
Mul,
Div,
Mod,
#[cfg(not(feature = "mini"))]
Pow,
}
#[derive(PartialEq, Eq, Copy, Clone, Debug)]
enum Logical {
And,
Or,
}
#[derive(PartialEq, Eq, Copy, Clone, Debug)]
enum Bitwise {
And,
Or,
Xor,
ShiftLeft,
ShiftRight,
}
fn expry_tokenize<'b,'c>(reader: &mut &'b str, context: &mut JSONParserContext<'_,'b,'c>) -> Result<(JSONToken<'b>,usize), (CompileErrorDescription<'b>,usize,usize)> where 'c: 'b,
{
// might be faster using https://github.com/maciejhirsz/logos
loop {
*reader = reader.trim_start();
let start_of_comment = reader.len();
if reader.accept("/*") {
if let Some((_comment, tail)) = reader.split_once("*/") {
*reader = tail;
} else {
return Err((CompileErrorDescription::Lexer(LexerError{ line_nr: line!()}), start_of_comment,reader.len()))
}
continue;
}
if reader.accept("//") {
if let Some((_comment, tail)) = reader.split_once('\n') {
*reader = tail;
} else {
*reader = "";
}
continue;
}
break;
}
let remaining = reader.len();
if reader.is_empty() {
return Ok((JSONToken::End(), remaining));
}
// parse hex values 0x
if reader.accept("0x") {
if let Some(value) = reader.span_fn(&mut |x| from_hex(x).is_ok()) {
let err = || (CompileErrorDescription::Lexer(LexerError{ line_nr: line!()}), remaining,reader.len());
return Ok((JSONToken::Int(i64::from_str_radix(value, 16).map_err(|_| err())?), remaining));
} else {
return Err((CompileErrorDescription::Lexer(LexerError{ line_nr: line!()}), remaining,reader.len()));
}
}
// parse binary values 0b
if reader.accept("0b") {
if let Some(value) = reader.span_fn(&mut |x| x == '0' || x == '1') {
let err = || (CompileErrorDescription::Lexer(LexerError{ line_nr: line!()}), remaining,reader.len());
return Ok((JSONToken::Int(i64::from_str_radix(value, 2).map_err(|_| err())?), remaining));
} else {
return Err((CompileErrorDescription::Lexer(LexerError{ line_nr: line!()}), remaining,reader.len()));
}
}
// parse single precision floats which have the suffix 'f' (this is not part of the JSON spec)
let mut reader_copy = *reader;
let mut number_matcher = NumberSpanner::new();
if let Some(value) = number_matcher.span(&mut reader_copy) {
if reader_copy.accept("f") {
*reader = reader_copy;
let err = || (CompileErrorDescription::Lexer(LexerError{ line_nr: line!()}), remaining,reader.len());
// given the number spanner, we know that value is UTF8
let number : f32 = value.parse().map_err(|_| err())?;
return Ok((JSONToken::Float(number), remaining));
}
}
// call regular JSON lexer
match json_tokenize(reader, context) {
Ok(v) => return Ok(v),
Err((CompileErrorDescription::NoToken(),_,_)) => {},
Err(err) => return Err(err),
}
// parse single quoted strings
let mut string_matcher = StringLiteralSpanner::new('\'');
if let Some(value) = string_matcher.span(reader) {
debug_assert!(value.len() >= 2);
if string_matcher.unescape_needed {
let err = || (CompileErrorDescription::Lexer(LexerError{ line_nr: line!()}), remaining,reader.len());
return Ok((JSONToken::String(json_unescape(&value[1..value.len()-1], context.allocator, true).map_err(|_| err())?),remaining));
}
return Ok((JSONToken::String(value[1..value.len()-1].as_bytes()),remaining));
}
// parse strings starting with 'x"'
let mut alt = false;
if reader.accept("x\"") || (alt = true, reader.accept("x'")).1 {
if let Some((hex,tail)) = reader.split_once(if alt { "'" } else { "\"" }) {
*reader = tail;
let err = || (CompileErrorDescription::Lexer(LexerError{ line_nr: line!()}), remaining,reader.len());
return Ok((JSONToken::String(context.allocator.copy_unhex(hex.as_bytes()).map_err(|_| err())?),remaining));
} else {
return Err((CompileErrorDescription::Lexer(LexerError{ line_nr: line!()}), remaining,reader.len()));
}
}
// parse strings starting with `r##"..."##`
if reader.starts_with('r') && reader.len() >= 2 {
let count = reader[1..].chars().take_while(|x| *x == '#').count();
if let Some(x) = reader.chars().nth(1+count) {
if x == '"' || x == '\'' {
let delim : String = format!("{}{}", x, &reader[1..1+count]);
*reader = &reader[2+count..];
if let Some((string,tail)) = reader.split_once(&delim) {
*reader = tail;
return Ok((JSONToken::String(string.as_bytes()), remaining));
} else {
return Err((CompileErrorDescription::Lexer(LexerError{ line_nr: line!()}), remaining,reader.len()));
}
}
}
}
if reader.accept("$=") {
return Ok((JSONToken::Comparison(ExpryComparison::EndsWith), remaining));
}
// parse functions with '(' after (with a variable string spanner)
// parse fields (with a variable string spanner) (handle `this` differently)
let mut matcher = |c: char| c == '$' || c == '_' || c == '\\' || c.is_ascii_alphanumeric();
if let Some(value) = reader.span_fn(&mut matcher) {
// field name or function call (peek at next)
if !reader.is_empty() && reader.starts_with('(') {
return Ok((JSONToken::FunctionCall(value), remaining));
} else {
return Ok((JSONToken::Field(value), remaining));
}
}
if reader.accept("<=") {
return Ok((JSONToken::Comparison(ExpryComparison::LessEqual), remaining));
}
if reader.accept("<<") {
return Ok((JSONToken::Bitwise(Bitwise::ShiftLeft), remaining));
}
if reader.accept("<") {
return Ok((JSONToken::Comparison(ExpryComparison::Less), remaining));
}
if reader.accept(">=") {
return Ok((JSONToken::Comparison(ExpryComparison::GreaterEqual), remaining));
}
if reader.accept(">>") {
return Ok((JSONToken::Bitwise(Bitwise::ShiftRight), remaining));
}
if reader.accept(">") {
return Ok((JSONToken::Comparison(ExpryComparison::Greater), remaining));
}
if reader.accept("==") || reader.accept("=") {
return Ok((JSONToken::Comparison(ExpryComparison::Equal), remaining));
}
if reader.accept("!=") {
return Ok((JSONToken::Comparison(ExpryComparison::NotEqual), remaining));
}
if reader.accept("!") {
return Ok((JSONToken::Not(), remaining));
}
#[cfg(not(feature = "mini"))]
if reader.accept("**") {
return Ok((JSONToken::Arithmetic(Arithmetic::Pow), remaining));
}
if reader.accept("*=") {
return Ok((JSONToken::Comparison(ExpryComparison::Contains), remaining));
}
if reader.accept("^=") {
return Ok((JSONToken::Comparison(ExpryComparison::StartsWith), remaining));
}
if reader.accept("&&") {
return Ok((JSONToken::Logical(Logical::And), remaining));
}
if reader.accept("||") {
return Ok((JSONToken::Logical(Logical::Or), remaining));
}
if reader.accept("&") {
return Ok((JSONToken::Bitwise(Bitwise::And), remaining));
}
if reader.accept("|") {
return Ok((JSONToken::Bitwise(Bitwise::Or), remaining));
}
if reader.accept("^") {
return Ok((JSONToken::Bitwise(Bitwise::Xor), remaining));
}
if reader.accept("(") {
return Ok((JSONToken::Open(), remaining));
}
if reader.accept(")") {
return Ok((JSONToken::Close(), remaining));
}
if reader.accept("???") {
return Ok((JSONToken::TryOperator(), remaining));
}
if reader.accept("??") {
return Ok((JSONToken::NullOperator(), remaining));
}
if reader.accept("?") {
return Ok((JSONToken::Conditional(), remaining));
}
if reader.accept("...") {
return Ok((JSONToken::Spread(), remaining));
}
if reader.accept("..") {
return Ok((JSONToken::Concat(), remaining));
}
if reader.accept(".") {
return Ok((JSONToken::Dot(), remaining));
}
if reader.accept("+") {
return Ok((JSONToken::Arithmetic(Arithmetic::Add), remaining));
}
if reader.accept("-") {
return Ok((JSONToken::Arithmetic(Arithmetic::Sub), remaining));
}
if reader.accept("*") {
return Ok((JSONToken::Arithmetic(Arithmetic::Mul), remaining));
}
if reader.accept("/") {
return Ok((JSONToken::Arithmetic(Arithmetic::Div), remaining));
}
if reader.accept("%") {
return Ok((JSONToken::Arithmetic(Arithmetic::Mod), remaining));
}
Err((CompileErrorDescription::NoToken(), remaining,reader.len()))
}
type ExpryParserState<'a,'b,'c> = JSONParserState<'a,'b,'c>;
type ExpryParserResult<'b, T> = ParserResult<T,CompileErrorDescription<'b>>;
#[derive(Debug, Copy, Clone)]
enum UnaryOperation {
Not,
Negate,
}
#[derive(Eq, PartialEq, Debug, Copy, Clone)]
enum ValueSource {
Computed, // can be either one of the others, but that is only dynamically known
Value,
Expression,
}
#[derive(Debug, Copy, Clone)]
struct ASTProperties<'a> {
minimal_field: Key<'a>,
#[cfg(not(feature = "mini"))]
needs_dynamic_eval: bool,
#[cfg(not(feature = "mini"))]
source: ValueSource,
}
impl<'a> ASTProperties<'a> {
fn constant() -> Self {
Self {
minimal_field: key_empty(),
#[cfg(not(feature = "mini"))]
needs_dynamic_eval: false,
#[cfg(not(feature = "mini"))]
source: ValueSource::Expression,
}
}
fn field(minimal_field: &'a str) -> Self {
Self {
minimal_field: key_str(minimal_field),
#[cfg(not(feature = "mini"))]
needs_dynamic_eval: true,
#[cfg(not(feature = "mini"))]
source: ValueSource::Value,
}
}
fn dynamic() -> Self {
Self {
minimal_field: key_empty(),
#[cfg(not(feature = "mini"))]
needs_dynamic_eval: true,
#[cfg(not(feature = "mini"))]
source: ValueSource::Computed,
}
}
}
#[derive(Debug, Clone)]
enum ExpryAST<'b> {
Constant(DecodedValue<'b>),
Field(&'b str),
All(),
ObjectAccessWithString(Box<ExpryAST<'b>>, &'b str, ASTProperties<'b>),
ObjectAccessWithExpr(Box<ExpryAST<'b>>, Box<ExpryAST<'b>>, ASTProperties<'b>),
Arithmetic(Box<ExpryAST<'b>>, Arithmetic, Box<ExpryAST<'b>>, ASTProperties<'b>),
Comparison(Box<ExpryAST<'b>>, ExpryComparison, Box<ExpryAST<'b>>, ASTProperties<'b>),
Logical(Box<ExpryAST<'b>>, Logical, Box<ExpryAST<'b>>, ASTProperties<'b>),
//Logical(Logical, Vec<ExpryAST<'b>>),
Bitwise(Box<ExpryAST<'b>>, Bitwise, Box<ExpryAST<'b>>, ASTProperties<'b>),
Concat(Box<ExpryAST<'b>>, Box<ExpryAST<'b>>, ASTProperties<'b>),
UnaryExpression(UnaryOperation, Box<ExpryAST<'b>>, ASTProperties<'b>),
Function(&'b str, Vec<ExpryAST<'b>>, ASTProperties<'b>),
Method(Box<ExpryAST<'b>>, u8, Vec<ExpryAST<'b>>, ASTProperties<'b>),
Conditional(Box<ExpryAST<'b>>, Box<ExpryAST<'b>>, Box<ExpryAST<'b>>, ASTProperties<'b>),
Object(Vec<(ExpryAST<'b>, ExpryAST<'b>)>, ASTProperties<'b>),
MergeObjects(Box<ExpryAST<'b>>, Box<ExpryAST<'b>>, ASTProperties<'b>),
Array(Vec<ExpryAST<'b>>, ASTProperties<'b>),
Try(Box<ExpryAST<'b>>, Box<ExpryAST<'b>>, ASTProperties<'b>),
Null(Box<ExpryAST<'b>>, Box<ExpryAST<'b>>, ASTProperties<'b>),
}
fn fast_path(ast: &ExpryAST) -> bool {
match ast {
ExpryAST::Constant(_) => true,
ExpryAST::Field(_) => true,
ExpryAST::All() => true,
ExpryAST::ObjectAccessWithString(lhs, _, _) => fast_path(lhs),
ExpryAST::ObjectAccessWithExpr(lhs, rhs, _) => fast_path(lhs) && fast_path(rhs),
_ => false,
}
}
fn ast_properties<'b>(ast: &ExpryAST<'b>) -> ASTProperties<'b> {
match ast {
ExpryAST::Constant(_) => ASTProperties::constant(),
ExpryAST::Field(field) => ASTProperties::field(field),
ExpryAST::All() => ASTProperties::dynamic(),
ExpryAST::ObjectAccessWithString(_, _, prop) => *prop,
ExpryAST::ObjectAccessWithExpr(_, _, prop) => *prop,
ExpryAST::Arithmetic(_, _, _, prop) => *prop,
ExpryAST::Comparison(_, _, _, prop) => *prop,
ExpryAST::Logical(_, _, _, prop) => *prop,
ExpryAST::Bitwise(_, _, _, prop) => *prop,
ExpryAST::Concat(_, _, prop) => *prop,
ExpryAST::UnaryExpression(_, _, prop) => *prop,
ExpryAST::Function(_, _, prop) => *prop,
ExpryAST::Method(_, _, _, prop) => *prop,
ExpryAST::Conditional(_, _, _, prop) => *prop,
ExpryAST::Object(_, prop) => *prop,
ExpryAST::Array(_, prop) => *prop,
ExpryAST::Try(_, _, prop) => *prop,
ExpryAST::Null(_, _, prop) => *prop,
ExpryAST::MergeObjects(_, _, prop) => *prop,
}
}
fn ast_properties_plus<'a>(x: &ASTProperties<'a>, y: &ASTProperties<'a>) -> ASTProperties<'a> {
ASTProperties {
minimal_field: expry_min(x.minimal_field, y.minimal_field),
#[cfg(not(feature = "mini"))]
needs_dynamic_eval: x.needs_dynamic_eval || y.needs_dynamic_eval,
#[cfg(not(feature = "mini"))]
source: ValueSource::Computed,
}
}
fn ast_properties_plus_ast<'a>(x: &ExpryAST<'a>, y: &ExpryAST<'a>) -> ASTProperties<'a> {
ast_properties_plus(&ast_properties(x), &ast_properties(y))
}
fn ast_properties_combine<'a>(args: &[ExpryAST<'a>]) -> ASTProperties<'a> {
args.iter().fold(ASTProperties::constant(), |x,y| {
let y = ast_properties(y);
ast_properties_plus(&x,&y)
})
}
fn ast_properties_iter<'a,'b, Iter>(args: Iter) -> ASTProperties<'a>
where Iter: Iterator<Item=&'b &'b ExpryAST<'a>>, 'a: 'b
{
args.fold(ASTProperties::constant(), |x,y| {
let y = ast_properties(y);
ASTProperties {
minimal_field: expry_min(x.minimal_field, y.minimal_field),
#[cfg(not(feature = "mini"))]
needs_dynamic_eval: x.needs_dynamic_eval || y.needs_dynamic_eval,
#[cfg(not(feature = "mini"))]
source: ValueSource::Computed,
}
})
}
fn expry_parse_primary_not<'b>(parser: &mut ExpryParserState<'b,'_,'_>) -> ExpryParserResult<'b, ExpryAST<'b>> {
parser.accept(JSONToken::Not(), None, || CompileErrorDescription::Parser(""))?;
let lhs = expry_parse_primary(parser)?;
let lhs_prop = ast_properties(&lhs);
Ok(ExpryAST::UnaryExpression(UnaryOperation::Not, Box::new(lhs), lhs_prop))
}
fn expry_parse_primary_minus<'b>(parser: &mut ExpryParserState<'b,'_,'_>) -> ExpryParserResult<'b, ExpryAST<'b>> {
parser.accept(JSONToken::Arithmetic(Arithmetic::Sub), None, || CompileErrorDescription::Parser(""))?;
let info = parser.token_info()?;
let lhs = expry_parse_primary(parser)?;
match lhs {
ExpryAST::Constant(DecodedValue::Double(number)) => {
return Ok(ExpryAST::Constant(DecodedValue::Double(-number)));
},
ExpryAST::Constant(DecodedValue::Float(number)) => {
return Ok(ExpryAST::Constant(DecodedValue::Float(-number)));
},
ExpryAST::Constant(DecodedValue::Int(number)) => {
let value = number.checked_neg();
if let Some(value) = value {
return Ok(ExpryAST::Constant(DecodedValue::Int(value)));
}
return Err(parser.error_other(&info, CompileErrorDescription::Parser("could not negate the int number")));
},
_ => {},
}
let lhs_prop = ast_properties(&lhs);
Ok(ExpryAST::UnaryExpression(UnaryOperation::Negate, Box::new(lhs), lhs_prop))
}
fn expry_parse_primary_function_call<'b>(parser: &mut ExpryParserState<'b,'_,'_>) -> ExpryParserResult<'b, ExpryAST<'b>> {
match parser.get()? {
(JSONToken::FunctionCall(fname),_) => {
let open = parser.accept(JSONToken::Open(), None, || CompileErrorDescription::Parser("expected '(' to open function call"))?;
let mut args : Vec<ExpryAST<'b>> = vec!();
parser.opt(|parser| {
args.push(expry_parse_expr(parser)?);
parser.repeat(|parser| {
parser.accept(JSONToken::Comma(), None, || CompileErrorDescription::Parser(""))?;
args.push(expry_parse_expr(parser)?);
Ok(true)
})?;
Ok(())
})?;
parser.accept(JSONToken::Close(), Some(&open), || CompileErrorDescription::Parser("expected ')' to close function call"))?;
#[allow(unused_mut)]
let mut prop = ast_properties_combine(&args);
#[cfg(not(feature = "mini"))]
{
prop.needs_dynamic_eval = internal_function(fname, args.len()).is_none();
}
Ok(ExpryAST::Function(fname, args, prop))
},
(token,info) => Err(parser.error_token(token, info, |_| CompileErrorDescription::Parser(""))),
}
}
fn expry_parse_primary_field<'b>(parser: &mut ExpryParserState<'b,'_,'_>) -> ExpryParserResult<'b, ExpryAST<'b>> {
match parser.get()? {
(JSONToken::Field(x),info) => {
resolve(x, parser).map_err(|_| parser.error_other(&info, CompileErrorDescription::Parser("error decoding static value")))
},
(token,info) => Err(parser.error_token(token, info, |_| CompileErrorDescription::Parser(""))),
}
}
fn resolve<'b>(x: &'b str, parser: &mut ExpryParserState<'b,'_,'_>) -> Result<ExpryAST<'b>,EncodingError> {
if x == "$this" {
return Ok(ExpryAST::Constant(parser.context.static_value.clone()));
}
if x == "this" {
return Ok(ExpryAST::All())
}
if let Some(value) = parser.context.static_value.lookup_field(key_str(x))? {
return Ok(ExpryAST::Constant(value));
}
Ok(ExpryAST::Field(x))
}
fn expry_parse_primary_field_raw<'b>(parser: &mut ExpryParserState<'b,'_,'_>) -> ExpryParserResult<'b, ExpryAST<'b>> {
match parser.get()? {
(JSONToken::Field(x),_info) => {
Ok(ExpryAST::Field(x))
},
(token,info) => Err(parser.error_token(token, info, |_| CompileErrorDescription::Parser(""))),
}
}
fn expry_parse_json<'b>(parser: &mut ExpryParserState<'b,'_,'_>) -> ExpryParserResult<'b, ExpryAST<'b>> {
json_expr(parser).map(ExpryAST::Constant)
}
fn expry_parse_primary_parenthesis<'b>(parser: &mut ExpryParserState<'b,'_,'_>) -> ExpryParserResult<'b, ExpryAST<'b>> {
let open = parser.accept(JSONToken::Open(), None, || CompileErrorDescription::Parser(""))?;
let value = expry_parse_expr(parser)?;
parser.accept(JSONToken::Close(), Some(&open), || CompileErrorDescription::Parser("expected ')' to close '('"))?;
Ok(value)
}
fn expry_parse_primary<'b>(parser: &mut ExpryParserState<'b,'_,'_>) -> ExpryParserResult<'b, ExpryAST<'b>> {
parser.choose(&[
expry_parse_array,
expry_parse_object,
expry_parse_primary_not,
expry_parse_primary_minus,
expry_parse_primary_function_call,
expry_parse_primary_field,
expry_parse_json,
expry_parse_primary_parenthesis,
][..],
|| CompileErrorDescription::Parser("expected expr primary (possibly mismatched closing '}'/']')"))
}
fn expry_parse_primary_object_access<'b>(parser: &mut ExpryParserState<'b,'_,'_>) -> ExpryParserResult<'b, ExpryAST<'b>> {
let mut retval = expry_parse_primary(parser)?;
// implement static (with .) and dynamic (with []) object access
parser.repeat(|parser| {
match parser.get()? {
(JSONToken::Dot(), _) => {
match parser.get()? {
(JSONToken::Field(name),_) => {
let lhs = core::mem::replace(&mut retval, ExpryAST::Constant(DecodedValue::Null()));
let lhs_prop = ast_properties(&lhs);
retval = ExpryAST::ObjectAccessWithString(Box::new(lhs), name, lhs_prop);
Ok(true)
},
(JSONToken::FunctionCall(name), open) => {
parser.accept(JSONToken::Open(), None, || CompileErrorDescription::Parser(""))?;
let mut args : Vec<ExpryAST<'b>> = Vec::new();
parser.repeat(|parser| {
let value = expry_parse_expr(parser)?;
args.push(value);
Ok(parser.accept(JSONToken::Comma(), None, || CompileErrorDescription::Parser("")).is_ok())
})?;
parser.accept(JSONToken::Close(), Some(&open), || CompileErrorDescription::Parser("expected a ')' to close this '('."))?;
let lhs = core::mem::replace(&mut retval, ExpryAST::Constant(DecodedValue::Null()));
let prop = ast_properties_plus(&ast_properties(&lhs), &ast_properties_combine(&args));
let opcode = internal_method(name, args.len()).map_err(|err| parser.error_other(&open, err))?;
retval = ExpryAST::Method(Box::new(lhs), opcode, args, prop);
Ok(true)
},
(token,info) => Err(parser.error_token(token, info, |_| CompileErrorDescription::Parser("expected field name to access object"))),
}
},
(JSONToken::ArrayOpen(), open) => {
let rhs = expry_parse_expr(parser)?;
parser.accept(JSONToken::ArrayClose(), Some(&open), || CompileErrorDescription::Parser("expected ']' to close this dynamic field access"))?;
let lhs = core::mem::replace(&mut retval, ExpryAST::Constant(DecodedValue::Null()));
let prop = ast_properties_iter([&lhs, &rhs].iter());
retval = ExpryAST::ObjectAccessWithExpr(Box::new(lhs), Box::new(rhs), prop);
Ok(true)
},
(token,info) => Err(parser.error_token(token, info, |_| CompileErrorDescription::Parser("expected field name to access object"))),
}
})?;
Ok(retval)
}
fn expry_parse_exponentation<'b>(parser: &mut ExpryParserState<'b,'_,'_>) -> ExpryParserResult<'b, ExpryAST<'b>> {
#[allow(unused_mut)]
let mut retval = expry_parse_primary_object_access(parser)?;
#[cfg(not(feature = "mini"))]
parser.repeat(|parser| {
match parser.get()? {
(JSONToken::Arithmetic(x),_) if matches!(x, Arithmetic::Pow) => {
let lhs = core::mem::replace(&mut retval, ExpryAST::Constant(DecodedValue::Null()));
let rhs = expry_parse_primary_object_access(parser)?;
let prop = ast_properties_iter([&lhs, &rhs].iter());
retval = ExpryAST::Arithmetic(Box::new(lhs), x, Box::new(rhs), prop);
Ok(true)
}
(token,info) => Err(parser.error_token(token, info, |_| CompileErrorDescription::Parser("expected primary as rhs for exponentation"))),
}
})?;
Ok(retval)
}
fn expry_parse_multiplication<'b>(parser: &mut ExpryParserState<'b,'_,'_>) -> ExpryParserResult<'b, ExpryAST<'b>> {
let mut retval = expry_parse_exponentation(parser)?;
parser.repeat(|parser| {
match parser.get()? {
(JSONToken::Arithmetic(x),_) if matches!(x, Arithmetic::Mul | Arithmetic::Div | Arithmetic::Mod) => {
let lhs = core::mem::replace(&mut retval, ExpryAST::Constant(DecodedValue::Null()));
let rhs = expry_parse_exponentation(parser)?;
let prop = ast_properties_iter([&lhs, &rhs].iter());
retval = ExpryAST::Arithmetic(Box::new(lhs), x, Box::new(rhs), prop);
Ok(true)
}
(token, info) => Err(parser.error_token(token, info, |_| CompileErrorDescription::Parser("expected exponentation as rhs for multiplication"))),
}
})?;
Ok(retval)
}
fn expry_parse_addition<'b>(parser: &mut ExpryParserState<'b,'_,'_>) -> ExpryParserResult<'b, ExpryAST<'b>> {
let mut retval = expry_parse_multiplication(parser)?;
parser.repeat(|parser| {
match parser.get()? {
(JSONToken::Arithmetic(x),_) if matches!(x, Arithmetic::Add | Arithmetic::Sub) => {
let lhs = core::mem::replace(&mut retval, ExpryAST::Constant(DecodedValue::Null()));
let rhs = expry_parse_multiplication(parser)?;
let prop = ast_properties_iter([&lhs, &rhs].iter());
retval = ExpryAST::Arithmetic(Box::new(lhs), x, Box::new(rhs), prop);
Ok(true)
}
(token, info) => Err(parser.error_token(token, info, |_| CompileErrorDescription::Parser("expected multiplication as rhs for an addition"))),
}
})?;
Ok(retval)
}
fn expry_parse_bitwise_shift<'b>(parser: &mut ExpryParserState<'b,'_,'_>) -> ExpryParserResult<'b, ExpryAST<'b>> {
let mut retval = expry_parse_addition(parser)?;
parser.repeat(|parser| {
match parser.get()? {
(JSONToken::Bitwise(x),_) if matches!(x, Bitwise::ShiftLeft | Bitwise::ShiftRight) => {
let lhs = core::mem::replace(&mut retval, ExpryAST::Constant(DecodedValue::Null()));
let rhs = expry_parse_addition(parser)?;
let prop = ast_properties_iter([&lhs, &rhs].iter());
retval = ExpryAST::Bitwise(Box::new(lhs), x, Box::new(rhs), prop);
Ok(true)
}
(token, info) => Err(parser.error_token(token, info, |_| CompileErrorDescription::Parser("expected addition as rhs for an bitwise shift"))),
}
})?;
Ok(retval)
}
fn expry_parse_bitwise_logical<'b>(parser: &mut ExpryParserState<'b,'_,'_>) -> ExpryParserResult<'b, ExpryAST<'b>> {
let mut retval = expry_parse_bitwise_shift(parser)?;
parser.repeat(|parser| {
match parser.get()? {
(JSONToken::Bitwise(x),_) if matches!(x, Bitwise::And | Bitwise::Or | Bitwise::Xor) => {
let lhs = core::mem::replace(&mut retval, ExpryAST::Constant(DecodedValue::Null()));
let rhs = expry_parse_bitwise_shift(parser)?;
let prop = ast_properties_iter([&lhs, &rhs].iter());
retval = ExpryAST::Bitwise(Box::new(lhs), x, Box::new(rhs), prop);
Ok(true)
}
(token, info) => Err(parser.error_token(token, info, |_| CompileErrorDescription::Parser("expected bitwise-shift as rhs for an bitwise-logical"))),
}
})?;
Ok(retval)
}
fn expry_parse_concat<'b>(parser: &mut ExpryParserState<'b,'_,'_>) -> ExpryParserResult<'b, ExpryAST<'b>> {
let mut retval = expry_parse_bitwise_logical(parser)?;
parser.repeat(|parser| {
match parser.get()? {
(JSONToken::Concat(),_) => {
let lhs = core::mem::replace(&mut retval, ExpryAST::Constant(DecodedValue::Null()));
let rhs = expry_parse_bitwise_logical(parser)?;
let prop = ast_properties_iter([&lhs, &rhs].iter());
retval = ExpryAST::Concat(Box::new(lhs), Box::new(rhs), prop);
Ok(true)
}
(token,info) => Err(parser.error_token(token, info, |_| CompileErrorDescription::Parser("expected bitwise-logical as rhs for a concat"))),
}
})?;
Ok(retval)
}
fn expry_parse_comparison<'b>(parser: &mut ExpryParserState<'b,'_,'_>) -> ExpryParserResult<'b, ExpryAST<'b>> {
let mut retval = expry_parse_concat(parser)?;
parser.repeat(|parser| {
match parser.get()? {
(JSONToken::Comparison(c),_) => {
let lhs = core::mem::replace(&mut retval, ExpryAST::Constant(DecodedValue::Null()));
let rhs = expry_parse_concat(parser)?;
let prop = ast_properties_iter([&lhs, &rhs].iter());
retval = ExpryAST::Comparison(Box::new(lhs), c, Box::new(rhs), prop);
Ok(true)
}
(token, info) => Err(parser.error_token(token, info, |_| CompileErrorDescription::Parser("expected concat as rhs for a comparison"))),
}
})?;
Ok(retval)
}
fn expry_parse_logical<'b>(parser: &mut ExpryParserState<'b,'_,'_>) -> ExpryParserResult<'b, ExpryAST<'b>> {
let mut retval = expry_parse_comparison(parser)?;
parser.repeat(|parser| {
match parser.get()? {
(JSONToken::Logical(c),_) => {
let lhs = core::mem::replace(&mut retval, ExpryAST::Constant(DecodedValue::Null()));
let rhs = expry_parse_comparison(parser)?;
let prop = ast_properties_iter([&lhs, &rhs].iter());
retval = ExpryAST::Logical(Box::new(lhs), c, Box::new(rhs), prop);
Ok(true)
}
(token,info) => Err(parser.error_token(token, info, |_| CompileErrorDescription::Parser("expected concat as rhs for a comparison"))),
}
})?;
Ok(retval)
}
fn expry_parse_try_catch<'b>(parser: &mut ExpryParserState<'b,'_,'_>) -> ExpryParserResult<'b, ExpryAST<'b>> {
let mut retval = expry_parse_logical(parser)?;
parser.repeat(|parser| {
match parser.get()? {
(JSONToken::TryOperator(),_) => {
// changes here should also be applied to expry_parse_slice_key
let rhs = parser.opt(expry_parse_logical)?;
let lhs = core::mem::replace(&mut retval, ExpryAST::Constant(DecodedValue::Null()));
if let Some(rhs) = rhs {
let prop = ast_properties_iter([&lhs, &rhs].iter());
retval = ExpryAST::Try(Box::new(lhs), Box::new(rhs), prop);
} else {
let prop = ast_properties(&lhs);
retval = ExpryAST::Try(Box::new(lhs), Box::new(ExpryAST::Constant(DecodedValue::Null())), prop);
}
Ok(true)
},
(JSONToken::NullOperator(),_) => {
let lhs = core::mem::replace(&mut retval, ExpryAST::Constant(DecodedValue::Null()));
let rhs = expry_parse_logical(parser)?;
let prop = ast_properties_iter([&lhs, &rhs].iter());
retval = ExpryAST::Null(Box::new(lhs), Box::new(rhs), prop);
Ok(true)
},
(token, info) => Err(parser.error_token(token, info, |_| CompileErrorDescription::Parser(""))),
}
})?;
Ok(retval)
}
fn expry_parse_conditional<'b>(parser: &mut ExpryParserState<'b,'_,'_>) -> ExpryParserResult<'b, ExpryAST<'b>> {
let value = expry_parse_try_catch(parser)?;
if let Some(info) = parser.opt(|parser| {
parser.accept(JSONToken::Conditional(), None, || CompileErrorDescription::Parser(""))
})? {
let value_then = expry_parse_try_catch(parser)?;
parser.accept(JSONToken::Colon(), Some(&info), || CompileErrorDescription::Parser(""))?;
let value_else = expry_parse_conditional(parser)?;
let prop = ast_properties_iter([&value, &value_then, &value_else].iter());
return Ok(ExpryAST::Conditional(Box::new(value), Box::new(value_then), Box::new(value_else), prop))
}
Ok(value)
}
fn expry_parse_expr<'b>(parser: &mut ExpryParserState<'b,'_,'_>) -> ExpryParserResult<'b, ExpryAST<'b>> {
expry_parse_conditional(parser)
}
fn expry_parse_slice_key<'b>(parser: &mut ExpryParserState<'b,'_,'_>) -> ExpryParserResult<'b, ExpryAST<'b>> {
let mut retval = parser.choose(&[
expry_parse_primary_field_raw,
expry_parse_json,
][..],
|| CompileErrorDescription::Parser("expected constant or field"))?;
parser.repeat(|parser| {
match parser.get()? {
(JSONToken::TryOperator(),_) => {
let rhs = parser.opt(expry_parse_logical)?;
let lhs = core::mem::replace(&mut retval, ExpryAST::Constant(DecodedValue::Null()));
if let Some(rhs) = rhs {
let prop = ast_properties_iter([&lhs, &rhs].iter());
retval = ExpryAST::Try(Box::new(lhs), Box::new(rhs), prop);
} else {
let prop = ast_properties(&lhs);
retval = ExpryAST::Try(Box::new(lhs), Box::new(ExpryAST::Constant(DecodedValue::Null())), prop);
}
Ok(true)
},
(token, info) => Err(parser.error_token(token, info, |_| CompileErrorDescription::Parser(""))),
}
})?;
Ok(retval)
}
// spread syntax returns (None, expr)
// 'key: value' syntax returns (Some(expr), expr)
fn expry_parse_slice<'b>(parser: &mut ExpryParserState<'b,'_,'_>) -> ExpryParserResult<'b, (Option<ExpryAST<'b>>, ExpryAST<'b>)> {
if let Some(value_expr) = parser.opt(|parser| {
parser.accept(JSONToken::Spread(), None, || CompileErrorDescription::Parser(""))?;
let value_expr = expry_parse_expr(parser)?;
Ok(value_expr)
})? {
return Ok((None, value_expr));
}
if let Some((key_expr,value_expr)) = parser.opt(|parser| {
let open = parser.accept(JSONToken::Open(), None, || CompileErrorDescription::Parser(""))?;
let key_expr = expry_parse_expr(parser)?;
parser.accept(JSONToken::Close(), Some(&open), || CompileErrorDescription::Parser("expected ')' to close this dynamic field specifier"))?;
parser.accept(JSONToken::Colon(), None, || CompileErrorDescription::Parser("expected ':' as field delimeter for an object field"))?;
let value_expr = expry_parse_expr(parser)?;
Ok((key_expr, value_expr))
})? {
return Ok((Some(key_expr), value_expr));
}
// we need to match expry_parse_expr so we can also match with the try operator (which is specified
// last)
let err_info = parser.token_info()?;
let key = expry_parse_slice_key(parser)?;
// label
if let Some((label, expr)) = parser.opt(|parser| {
parser.accept(JSONToken::Colon(), None, || CompileErrorDescription::Parser("expected ':' as field delimeter for an object field"))?;
let value = expry_parse_expr(parser)?;
match key {
ExpryAST::Constant(DecodedValue::String(name)) => Ok((name, value)),
ExpryAST::Field(name) => Ok((name.as_bytes(), value)),
_ => Err(parser.error_other(&err_info, CompileErrorDescription::Parser("field name should be a constant string or a variable name"))),
}
})? {
return Ok((Some(ExpryAST::Constant(DecodedValue::String(label))), expr));
}
// implicit (so only a field, string, or try operating with one of both)
match key {
ExpryAST::Field(name) => Ok((Some(ExpryAST::Constant(DecodedValue::String(name.as_bytes()))), resolve(name, parser).map_err(|_| parser.error_other(&err_info, CompileErrorDescription::Parser("error decoding static value")))?)),
ExpryAST::Try(lhs, expr, prop) => {
match *lhs {
ExpryAST::Field(name) => Ok((Some(ExpryAST::Constant(DecodedValue::String(name.as_bytes()))), ExpryAST::Try(Box::new(ExpryAST::Field(name)), expr, prop))),
_ => Err(parser.error_other(&err_info, CompileErrorDescription::Parser("implicit field names only work for the try operator if the left-hand-side of the try operator is a variable name"))),
}
}
_ => Err(parser.error_other(&err_info, CompileErrorDescription::Parser("field without a name should be a variable name"))),
}
}
fn expry_parse_object<'b>(parser: &mut ExpryParserState<'b,'_,'_>) -> ExpryParserResult<'b, ExpryAST<'b>> {
let open = parser.accept(JSONToken::ObjectOpen(), None, || CompileErrorDescription::Parser("expected '{' to open object"))?;
// previous operations such as SliceObject and MergeObjects
let mut lhs : Option<ExpryAST> = None;
// lazy accumulator of current SliceObject
let mut accumulator : Vec<(ExpryAST, ExpryAST)> = Vec::new();
parser.repeat(|parser| {
let (key,value) = expry_parse_slice(parser)?;
process_slice(&mut lhs, &mut accumulator, key, value);
Ok(parser.accept(JSONToken::Comma(), None, || CompileErrorDescription::Parser("")).is_ok())
})?;
parser.accept(JSONToken::ObjectClose(), Some(&open), || CompileErrorDescription::Parser("expected a '}' to close this object"))?;
Ok(combine(lhs, accumulator))
}
// combine lhs and accumulator into one value and return that value
fn combine<'a>(lhs: Option<ExpryAST<'a>>, accumulator: Vec<(ExpryAST<'a>, ExpryAST<'a>)>) -> ExpryAST<'a> {
if let Some(lhs) = lhs {
if accumulator.is_empty() {
// ends with spread syntax
return lhs;
}
// earlier a spread syntax occured (which should be in in lhs)
let current = make_object(accumulator);
let prop = ast_properties_plus_ast(&lhs, ¤t);
ExpryAST::MergeObjects(Box::new(lhs), Box::new(current), prop)
} else if accumulator.is_empty() {
// nothing, so empty object
ExpryAST::Constant(DecodedValue::Object(DecodedObject::new()))
} else {
// only fields, no spread syntax
make_object(accumulator)
}
}
fn process_slice<'a>(lhs: &mut Option<ExpryAST<'a>>, accumulator: &mut Vec<(ExpryAST<'a>, ExpryAST<'a>)>, key: Option<ExpryAST<'a>>, value: ExpryAST<'a>) {
if let Some(key) = key {
accumulator.push((key,value));
return;
}
let l = lhs.take();
let acc = core::mem::take(accumulator);
// check if there is something to combine
if l.is_none() && acc.is_empty() {
*lhs = Some(value);
return;
}
let l = combine(l, acc);
let prop = ast_properties_plus_ast(&l, &value);
*lhs = Some(ExpryAST::MergeObjects(Box::new(l), Box::new(value), prop));
}
fn make_object<'a>(retval: Vec<(ExpryAST<'a>, ExpryAST<'a>)>) -> ExpryAST<'a> {
let prop = retval.iter().map(|(x,y)| ast_properties_plus_ast(x, y)).fold(ASTProperties::constant(), |x,y| {
ast_properties_plus(&x,&y)
});
ExpryAST::Object(retval, prop)
}
fn expry_parse_array<'b>(parser: &mut ExpryParserState<'b,'_,'_>) -> ExpryParserResult<'b, ExpryAST<'b>> {
let open = parser.accept(JSONToken::ArrayOpen(), None, || CompileErrorDescription::Parser("expected '[' to open array"))?;
let mut retval : Vec<ExpryAST> = Vec::new();
parser.repeat(|parser| {
if let Some(value) = parser.opt(expry_parse_expr)? {
retval.push(value);
Ok(parser.opt(|parser| {
parser.accept(JSONToken::Comma(), None, || CompileErrorDescription::Parser(""))
})?.is_some())
} else {
Ok(false)
}
})?;
parser.accept(JSONToken::ArrayClose(), Some(&open), || CompileErrorDescription::Parser("expected ']' to close array"))?;
let prop = ast_properties_combine(&retval);
Ok(ExpryAST::Array(retval, prop))
}
fn expry_parse_top_level<'b>(parser: &mut ExpryParserState<'b,'_,'_>) -> ExpryParserResult<'b, ExpryAST<'b>> {
let retval = expry_parse_expr(parser)?;
parser.accept(JSONToken::End(), None, || CompileErrorDescription::Parser("expected end of input (possibly an operator was missing)"))?;
Ok(retval)
}
/// Parse expressions that can be executed with an optional binary value as argument. These
/// expressions can be compiled to bytecode and executed very fast so it is suitable as a query
/// language, performing selection and modification of values on the server.
fn parse_expry_expr<'a,'b,'c>(reader: &'b str, static_value: &DecodedValue<'b>, allocator: &'a mut MemoryScope<'c>) -> Result<ExpryAST<'b>,(CompileErrorDescription<'b>,usize,usize,Option<(usize,usize)>)> where 'c: 'b, 'b: 'a {
let mut parser = ExpryParserState::new_with(reader, expry_tokenize, JSONParserContext{ allocator, static_value});
parser.parse(expry_parse_top_level, CompileErrorDescription::Parser("unexpected token"), CompileErrorDescription::Parser("too much parser recursion"))
}
#[repr(u8)]
enum ExpryBytecode {
Invalid = 0,
Find = 1,
SliceObjectStatic = 2,
SliceArray = 3,
Conditional = 4,
FindAndField = 5, Field = 6, All = 7,
MergeObjects = 8,
SliceObjectDynamic = 9,
Concrete = 10,
FieldAccessFastPath = 11,
FieldAccessCompute = 12,
Call = 16,
ArithmeticPlus = 20, ArithmeticMin, ArithmeticMul, ArithmeticDiv, ArithmeticMod, ArithmeticShiftLeft, ArithmeticShiftRight, ArithmeticUnaryMin, ArithmeticBitwiseAnd, ArithmeticBitwiseOr, ArithmeticBitwiseXor,
Not = 32, And, Or,
Equal = 35, NotEqual,
ParseEqual = 40, ParseNotEqual,
NumericLess = 45, NumericGreaterEqual, NumericGreater, NumericLessEqual,
StringContains = 55, StringStartsWith, StringEndsWith,
StringConcat = 65,
OpenObject = 70,
NotNullElse = 76,
Try = 77,
Defined = 78,
Undefined = 79,
ArithmeticSin = 80, ArithmeticTan, ArithmeticCos, ArithmeticSqrt, ArithmeticAbs, ArithmeticLog, ArithmeticCeil, ArithmeticFloor, ArithmeticTrunc, ArithmeticRound, ArithmeticLog10, ArithmeticLog2,
#[cfg(not(feature = "mini"))]
ArithmeticPower,
StringUpper = 100, StringLower, StringTrim, StringHex, StringXxhash32, StringXxhash64, StringSha1, StringSha512, StringHtmlescape, StringUrlescape,
StringDirname = 111, StringBasename, StringSubstringFromTo, StringCount, StringSplit, StringSubstringFrom,
ToInteger = 124, ToFloat, ToDouble, ToString,
TypeIsNull = 128, TypeIsBool, TypeIsNumber, TypeIsInteger, TypeIsFloat, TypeIsDouble, TypeIsString, TypeIsArray, TypeIsObject,
Length = 140,
}
fn expry_cmp<'a>(lhs: Key<'a>, rhs: Key<'a>) -> Ordering {
if lhs.1.is_empty() {
return Ordering::Greater;
}
if rhs.1.is_empty() {
return Ordering::Less;
}
lhs.cmp(&rhs)
}
fn expry_min<'a>(lhs: Key<'a>, rhs: Key<'a>) -> Key<'a> {
if lhs.1.is_empty() {
return rhs;
}
if rhs.1.is_empty() {
return lhs;
}
core::cmp::min(lhs, rhs)
}
#[cfg(not(feature = "mini"))]
fn expry_ast_optimize<'b,'c>(ast: &'_ mut ExpryAST<'b>) -> Result<(),EvalError<'b>> where 'c: 'b
{
match ast {
ExpryAST::Constant(_) => {},
ExpryAST::Field(_) => {},
ExpryAST::All() => {},
ExpryAST::ObjectAccessWithString(lhs, _, _) => {
expry_ast_optimize(lhs)?
},
ExpryAST::ObjectAccessWithExpr(lhs, rhs, _) => {
if let ExpryAST::All() = **lhs {
if let ExpryAST::Constant(DecodedValue::String(name)) = **rhs {
*ast = ExpryAST::Field(core::str::from_utf8(name).map_err(|_| EvalError::Other())?);
return expry_ast_optimize(&mut *ast);
}
}
expry_ast_optimize(lhs)?;
expry_ast_optimize(rhs)?;
},
ExpryAST::Arithmetic(lhs, op, rhs, _) => {
expry_ast_optimize(lhs)?;
expry_ast_optimize(rhs)?;
if matches!(op, Arithmetic::Add | Arithmetic::Mul) {
if ast_properties(rhs).minimal_field < ast_properties(lhs).minimal_field {
core::mem::swap(lhs, rhs);
}
// (first_key + last_key) + middle_key should be converted to
// (first_key + middle_key) + last_key (+ -> all ops for this branch)
match lhs.as_mut() {
ExpryAST::Arithmetic(_, subop, subrhs, _) if subop == op => {
if ast_properties(rhs).minimal_field < ast_properties(subrhs).minimal_field {
core::mem::swap(subrhs, rhs);
}
}
_ => (),
};
}
},
ExpryAST::Comparison(lhs, op, rhs, _) => {
expry_ast_optimize(lhs)?;
expry_ast_optimize(rhs)?;
if ast_properties(rhs).minimal_field < ast_properties(lhs).minimal_field {
if let Some(rop) = op.reverse() {
core::mem::swap(lhs, rhs);
*op = rop;
}
}
},
ExpryAST::Logical(lhs, op, rhs, _) => {
expry_ast_optimize(lhs)?;
expry_ast_optimize(rhs)?;
if matches!(op, Logical::And | Logical::Or) {
if ast_properties(rhs).minimal_field < ast_properties(lhs).minimal_field {
core::mem::swap(lhs, rhs);
}
// (first_key + last_key) + middle_key should be converted to
// (first_key + middle_key) + last_key (+ -> all ops for this branch)
match lhs.as_mut() {
ExpryAST::Logical(_, subop, subrhs, _) if subop == op => {
if ast_properties(rhs).minimal_field < ast_properties(subrhs).minimal_field {
core::mem::swap(subrhs, rhs);
}
}
_ => (),
};
}
},
ExpryAST::Bitwise(lhs, _, rhs, _) => {
expry_ast_optimize(lhs)?;
expry_ast_optimize(rhs)?;
},
ExpryAST::Concat(lhs, rhs, _) => {
expry_ast_optimize(lhs)?;
expry_ast_optimize(rhs)?;
},
ExpryAST::UnaryExpression(_, lhs, _) => {
expry_ast_optimize(lhs)?
},
ExpryAST::Function(_, args, _) => {
for v in args {
expry_ast_optimize(v)?;
}
},
ExpryAST::Method(lhs, _, args, _) => {
for v in args {
expry_ast_optimize(v)?;
}
expry_ast_optimize(lhs)?
},
ExpryAST::Conditional(c, t, e, _) => {
expry_ast_optimize(c)?;
expry_ast_optimize(t)?;
expry_ast_optimize(e)?;
},
// object creation optimisations (> etc can be reversed)
// object creation can be optimized in two ways:
// 1) sort on the output field,
// 2) sort on the field accesses
// Probably (2) is better, because they are more expensive than keeping something sorted
// (that we should anyhow)
ExpryAST::Object(fields, _) => {
let mut ordered : Vec<(Key, ExpryAST, ExpryAST)> = vec!();
for (k,v) in fields.iter_mut() {
expry_ast_optimize(k)?;
expry_ast_optimize(v)?;
let minimal = ast_properties_plus(&ast_properties(k), &ast_properties(v));
ordered.push((
minimal.minimal_field,
core::mem::replace(k, ExpryAST::Constant(DecodedValue::Null())),
core::mem::replace(v, ExpryAST::Constant(DecodedValue::Null()))
));
}
ordered.sort_by(|x,y| x.0.cmp(&y.0));
*fields = ordered.iter_mut().map(|(_,k,v)| (
core::mem::replace(k, ExpryAST::Constant(DecodedValue::Null())),
core::mem::replace(v, ExpryAST::Constant(DecodedValue::Null()))
)).collect();
},
ExpryAST::MergeObjects(lhs, rhs, _) => {
expry_ast_optimize(lhs)?;
expry_ast_optimize(rhs)?;
},
ExpryAST::Array(arr, _) => {
for v in arr {
expry_ast_optimize(v)?;
}
},
ExpryAST::Try(lhs, rhs, _) => {
expry_ast_optimize(lhs)?;
expry_ast_optimize(rhs)?;
},
ExpryAST::Null(lhs, rhs, _) => {
expry_ast_optimize(lhs)?;
expry_ast_optimize(rhs)?;
},
};
Ok(())
}
#[cfg(not(feature = "mini"))]
fn expry_ast_fold<'b,'c>(ast: &'_ mut ExpryAST<'b>, scope: &mut MemoryScope<'c>) -> Result<(),EvalError<'b>> where 'c: 'b
{
// if this is a 'static' subtree, replace with a constant
if !ast_properties(ast).needs_dynamic_eval {
let bytecode = expry_ast_to_bytecode(scope, ast).map_err(EvalError::Expression)?;
// storing bytecode in allocator, as evaluating might create a reference to the bytecode
let mut bytecode_reader = RawReader::with(bytecode.get());
let mut value_reader = RawReader::with(b"");
let mut context = Context { original: ValueRef(b""), custom: &mut NoCustomFuncs{} };
let value = evaluate_to_decoded_value(&mut bytecode_reader, &mut value_reader, &mut context, scope)?;
*ast = ExpryAST::Constant(value);
return Ok(());
}
match ast {
ExpryAST::ObjectAccessWithString(lhs, _, _) => {
expry_ast_fold(lhs, scope)?;
},
ExpryAST::ObjectAccessWithExpr(lhs, rhs, _) => {
expry_ast_fold(lhs, scope)?;
expry_ast_fold(rhs, scope)?;
},
ExpryAST::Arithmetic(lhs, _, rhs, _) => {
expry_ast_fold(lhs, scope)?;
expry_ast_fold(rhs, scope)?;
},
ExpryAST::Comparison(lhs, _, rhs, _) => {
expry_ast_fold(lhs, scope)?;
expry_ast_fold(rhs, scope)?;
},
ExpryAST::Logical(lhs, _, rhs, _) => {
expry_ast_fold(lhs, scope)?;
expry_ast_fold(rhs, scope)?;
},
ExpryAST::Bitwise(lhs, _, rhs, _) => {
expry_ast_fold(lhs, scope)?;
expry_ast_fold(rhs, scope)?;
},
ExpryAST::Concat(lhs, rhs, _) => {
expry_ast_fold(lhs, scope)?;
expry_ast_fold(rhs, scope)?;
},
ExpryAST::UnaryExpression(_, lhs, _) => {
expry_ast_fold(lhs, scope)?;
},
ExpryAST::Function(_, args, _) => {
for v in args {
expry_ast_fold(v, scope)?;
}
},
ExpryAST::Method(lhs, _, args, _) => {
for v in args {
expry_ast_fold(v, scope)?;
}
expry_ast_fold(lhs, scope)?;
},
ExpryAST::Conditional(c, t, e, _) => {
expry_ast_fold(c, scope)?;
expry_ast_fold(t, scope)?;
expry_ast_fold(e, scope)?;
},
ExpryAST::Object(fields, _) => {
for (k,v) in fields.iter_mut() {
expry_ast_fold(k, scope)?;
expry_ast_fold(v, scope)?;
}
},
ExpryAST::Array(arr, _) => {
for v in arr {
expry_ast_fold(v, scope)?;
}
},
ExpryAST::Try(lhs, rhs, _) => {
expry_ast_fold(lhs, scope)?;
expry_ast_fold(rhs, scope)?;
},
ExpryAST::Null(lhs, rhs, _) => {
expry_ast_fold(lhs, scope)?;
expry_ast_fold(rhs, scope)?;
},
_ => {},
};
Ok(())
}
fn to_var_length<'c>(scope: &mut MemoryScope<'c>, len: usize) -> &'c [u8] {
let header_size = size_of_var_u64(len as u64);
let header = scope.alloc(header_size);
let mut writer = RawWriter::with(header);
writer.write_var_u64(len as u64).unwrap();
debug_assert!(writer.left() == 0);
header
}
fn internal_method_expects(_name: &str, _args: usize) -> CompileErrorDescription<'static> {
CompileErrorDescription::Parser("wrong number of arguments")
}
fn internal_method(name: &str, args: usize) -> Result<u8,CompileErrorDescription<'static>> {
match name {
"len" => if args == 0 {
Ok(ExpryBytecode::Length as u8)
} else {
Err(internal_method_expects(name, 0))
},
"sub" => if args == 1 {
Ok(ExpryBytecode::StringSubstringFrom as u8)
} else if args == 2 {
Ok(ExpryBytecode::StringSubstringFromTo as u8)
} else {
Err(CompileErrorDescription::Parser(".sub() expects 1 or 2 arguments"))
},
"upper" => if args == 0 {
Ok(ExpryBytecode::StringUpper as u8)
} else {
Err(internal_method_expects(name, 0))
},
"lower" => if args == 0 {
Ok(ExpryBytecode::StringLower as u8)
} else {
Err(internal_method_expects(name, 0))
},
"trim" => if args == 0 {
Ok(ExpryBytecode::StringTrim as u8)
} else {
Err(internal_method_expects(name, 0))
},
"hex" => if args == 0 {
Ok(ExpryBytecode::StringHex as u8)
} else {
Err(internal_method_expects(name, 0))
},
"urlescape" => if args == 0 {
Ok(ExpryBytecode::StringUrlescape as u8)
} else {
Err(internal_method_expects(name, 0))
},
"htmlescape" => if args == 0 {
Ok(ExpryBytecode::StringHtmlescape as u8)
} else {
Err(internal_method_expects(name, 0))
},
"dirname" => if args == 0 {
Ok(ExpryBytecode::StringDirname as u8)
} else {
Err(internal_method_expects(name, 0))
},
"basename" => if args == 0 {
Ok(ExpryBytecode::StringBasename as u8)
} else {
Err(internal_method_expects(name, 0))
},
"splitn" => if args == 2 {
Ok(ExpryBytecode::StringSplit as u8)
} else {
Err(internal_method_expects(name, 2))
},
"tostring" => if args == 0 {
Ok(ExpryBytecode::ToString as u8)
} else {
Err(internal_method_expects(name, 0))
},
"tointeger" => if args == 0 {
Ok(ExpryBytecode::ToInteger as u8)
} else {
Err(internal_method_expects(name, 0))
},
"tofloat" => if args == 0 {
Ok(ExpryBytecode::ToFloat as u8)
} else {
Err(internal_method_expects(name, 0))
},
"todouble" => if args == 0 {
Ok(ExpryBytecode::ToDouble as u8)
} else {
Err(internal_method_expects(name, 0))
},
// do not call defined/undefined here, as it will mess up the interpreter due to early abort on error (so need to write a fixed length)
// on error
_ => Err(CompileErrorDescription::Parser("only built-in methods are supported")),
}
}
fn internal_function(_name: &str, _args: usize) -> Option<u8> {
None
}
fn expry_ast_to_bytecode<'a,'c>(scope: &mut MemoryScope<'c>, ast: &'a ExpryAST<'a>) -> Result<BytecodeRef<'c>,&'static str> {
const MAX_DEPTH: usize = 128;
if cfg!(feature = "mini") {
let next_position: Key<'a> = key_empty();
let mut current_position: Key<'a> = key_empty();
let mut writer = RawString::new();
if !_expry_ast_to_bytecode(ast, &mut current_position, next_position, &mut writer, MAX_DEPTH).expect("binary encoding error") {
return Err("too complex AST to convert to bytecode");
}
// FIXME: use delayed deallocation function of MemoryScope here
Ok(BytecodeRef(scope.copy_u8(&writer.data)))
} else {
let length;
{
let next_position: Key<'a> = key_empty();
let mut current_position: Key<'a> = key_empty();
let mut length_collector = RawWriterLength::new();
if !_expry_ast_to_bytecode(ast, &mut current_position, next_position, &mut length_collector, MAX_DEPTH).unwrap_infallible() {
return Err("too complex AST to convert to bytecode");
}
length = length_collector.length();
}
let next_position: Key<'a> = key_empty();
let mut current_position: Key<'a> = key_empty();
let retval = scope.alloc(length);
let mut writer = RawWriter::with(retval);
if !_expry_ast_to_bytecode(ast, &mut current_position, next_position, &mut writer, MAX_DEPTH).expect("expry: calculated size differs from actual size") {
return Err("too complex AST to convert to bytecode");
}
debug_assert!(writer.left() == 0);
Ok(BytecodeRef(retval))
}
}
fn _expry_ast_to_bytecode<'a, 'b, E, Out>(ast: &'b ExpryAST<'b>, current_position: &mut Key<'a>, next_position: Key<'a>, writer: &mut Out, depth: usize) -> Result<bool,E>
where
Out: RawOutput<E>,
'b: 'a,
{
if depth == 0 {
return Ok(false);
}
let mut retval = true;
match ast {
ExpryAST::Constant(v) => {
writer.write_u8(ExpryBytecode::Concrete as u8)?;
v.print_binary(writer, false)?;
},
ExpryAST::All() => {
writer.write_u8(ExpryBytecode::All as u8)?;
},
ExpryAST::Field(name) => {
let sorted_name = key_str(name);
debug_assert!(current_position.1.is_empty() || expry_min(*current_position,sorted_name) == *current_position); // ensure that currentPosition <= field
let min_this_and_next = expry_min(next_position, sorted_name);
if min_this_and_next == sorted_name {
// later operation do not need a field earlier as our field, so we can do FIND_AND_FIELD
writer.write_u8(ExpryBytecode::FindAndField as u8)?;
writer.write_u8(sorted_name.0)?;
writer.write_var_bytes(sorted_name.1)?;
*current_position = min_this_and_next;
} else if *current_position != min_this_and_next {
// later operation needs a field before our current field, so first seek (FIND) to that
// field for a later operation, and then perform a FIELD operation for our own
writer.write_u8(ExpryBytecode::Find as u8)?;
writer.write_u8(min_this_and_next.0)?;
writer.write_var_bytes(min_this_and_next.1)?;
writer.write_u8(ExpryBytecode::Field as u8)?;
writer.write_u8(sorted_name.0)?;
writer.write_var_bytes(sorted_name.1)?;
*current_position = min_this_and_next;
} else {
// later operation needs the current field, so only FIND
writer.write_u8(ExpryBytecode::Field as u8)?;
writer.write_u8(sorted_name.0)?;
writer.write_var_bytes(sorted_name.1)?;
}
},
ExpryAST::ObjectAccessWithExpr(lhs, rhs, _) => {
if !cfg!(feature = "mini") && fast_path(lhs) && fast_path(rhs) {
writer.write_u8(ExpryBytecode::FieldAccessFastPath as u8)?;
} else {
writer.write_u8(ExpryBytecode::FieldAccessCompute as u8)?;
}
retval &= _expry_ast_to_bytecode(lhs, current_position, expry_min(ast_properties(rhs).minimal_field, next_position), writer, depth-1)?;
retval &= _expry_ast_to_bytecode(rhs, current_position, next_position, writer, depth-1)?;
},
ExpryAST::ObjectAccessWithString(lhs, rhs, _) => {
if !cfg!(feature = "mini") && fast_path(lhs) {
writer.write_u8(ExpryBytecode::FieldAccessFastPath as u8)?;
} else {
writer.write_u8(ExpryBytecode::FieldAccessCompute as u8)?;
}
retval &= _expry_ast_to_bytecode(lhs, current_position, next_position, writer, depth-1)?;
writer.write_u8(ExpryBytecode::Concrete as u8)?;
DecodedValue::String(rhs.as_bytes()).print_binary(writer, false)?;
},
ExpryAST::Arithmetic(lhs, op, rhs, _) => {
let op_bin : u8 = match op {
Arithmetic::Add => ExpryBytecode::ArithmeticPlus as u8,
Arithmetic::Sub => ExpryBytecode::ArithmeticMin as u8,
Arithmetic::Mul => ExpryBytecode::ArithmeticMul as u8,
Arithmetic::Div => ExpryBytecode::ArithmeticDiv as u8,
Arithmetic::Mod => ExpryBytecode::ArithmeticMod as u8,
#[cfg(not(feature = "mini"))]
Arithmetic::Pow => ExpryBytecode::ArithmeticPower as u8,
};
writer.write_u8(op_bin)?;
retval &= _expry_ast_to_bytecode(lhs, current_position, expry_min(ast_properties(rhs).minimal_field, next_position), writer, depth-1)?;
retval &= _expry_ast_to_bytecode(rhs, current_position, next_position, writer, depth-1)?;
},
ExpryAST::Try(lhs, rhs, _) |
ExpryAST::Null(lhs, rhs, _) => {
// Value is reset on dynamic error, so we have two independent current_positions.
// This will speed up evaluation, because all field accesses will be faster.
if matches!(ast, ExpryAST::Try(_, _, _)) {
writer.write_u8(ExpryBytecode::Try as u8)?;
} else {
writer.write_u8(ExpryBytecode::NotNullElse as u8)?;
}
let mut lhs_current = *current_position;
retval &= write_with_header(writer, Out::write_var_u64, |writer| {
_expry_ast_to_bytecode(lhs, &mut lhs_current, next_position, writer, depth-1)
})?;
let mut rhs_current = *current_position;
retval &= write_with_header(writer, Out::write_var_u64, |writer| {
_expry_ast_to_bytecode(rhs, &mut rhs_current, next_position, writer, depth-1)
})?;
*current_position = expry_min(lhs_current, rhs_current);
},
ExpryAST::Method(obj, opcode, args, _) => {
writer.write_u8(*opcode)?;
retval &= _expry_ast_to_bytecode(obj, current_position, next_position, writer, depth-1)?;
for arg in args {
retval &= _expry_ast_to_bytecode(arg, current_position, next_position, writer, depth-1)?;
}
},
ExpryAST::Comparison(lhs, op, rhs, _) => {
// equal and unequal
#[allow(unused_mut)]
let mut op_bin : u8 = match op {
ExpryComparison::Equal => ExpryBytecode::ParseEqual as u8,
ExpryComparison::NotEqual => ExpryBytecode::ParseNotEqual as u8,
ExpryComparison::Less => ExpryBytecode::NumericLess as u8,
ExpryComparison::LessEqual => ExpryBytecode::NumericLessEqual as u8,
ExpryComparison::Greater => ExpryBytecode::NumericGreater as u8,
ExpryComparison::GreaterEqual => ExpryBytecode::NumericGreaterEqual as u8,
ExpryComparison::Contains => ExpryBytecode::StringContains as u8,
ExpryComparison::StartsWith => ExpryBytecode::StringStartsWith as u8,
ExpryComparison::EndsWith => ExpryBytecode::StringEndsWith as u8,
};
// if the source is the same, the encoding is probably the same and we can use a raw compare
#[cfg(not(feature = "mini"))]
if ast_properties(lhs).source != ValueSource::Computed && ast_properties(lhs).source == ast_properties(rhs).source {
if op_bin == ExpryBytecode::ParseEqual as u8 {
op_bin = ExpryBytecode::Equal as u8;
} else if op_bin == ExpryBytecode::ParseNotEqual as u8 {
op_bin = ExpryBytecode::NotEqual as u8;
}
}
writer.write_u8(op_bin)?;
retval &= _expry_ast_to_bytecode(lhs, current_position, expry_min(ast_properties(rhs).minimal_field, next_position), writer, depth-1)?;
retval &= _expry_ast_to_bytecode(rhs, current_position, next_position, writer, depth-1)?;
},
ExpryAST::Logical(lhs, op, rhs, _) => {
let op_bin : u8 = match op {
Logical::And => ExpryBytecode::And as u8,
Logical::Or => ExpryBytecode::Or as u8,
};
writer.write_u8(op_bin)?;
retval &= write_with_header(writer, Out::write_var_u64, |writer| {
_expry_ast_to_bytecode(lhs, current_position, expry_min(ast_properties(rhs).minimal_field, next_position), writer, depth-1)?;
_expry_ast_to_bytecode(rhs, current_position, next_position, writer, depth-1)
})?;
},
ExpryAST::Bitwise(lhs, op, rhs, _) => {
let op_bin : u8 = match op {
Bitwise::And => ExpryBytecode::ArithmeticBitwiseAnd as u8,
Bitwise::Or => ExpryBytecode::ArithmeticBitwiseOr as u8,
Bitwise::Xor => ExpryBytecode::ArithmeticBitwiseXor as u8,
Bitwise::ShiftLeft => ExpryBytecode::ArithmeticShiftLeft as u8,
Bitwise::ShiftRight => ExpryBytecode::ArithmeticShiftRight as u8,
};
writer.write_u8(op_bin)?;
retval &= _expry_ast_to_bytecode(lhs, current_position, expry_min(ast_properties(rhs).minimal_field, next_position), writer, depth-1)?;
retval &= _expry_ast_to_bytecode(rhs, current_position, next_position, writer, depth-1)?;
},
ExpryAST::Concat(lhs, rhs, _) => {
writer.write_u8(ExpryBytecode::StringConcat as u8)?;
retval &= _expry_ast_to_bytecode(lhs, current_position, expry_min(ast_properties(rhs).minimal_field, next_position), writer, depth-1)?;
retval &= _expry_ast_to_bytecode(rhs, current_position, next_position, writer, depth-1)?;
},
ExpryAST::UnaryExpression(op, lhs, _) => {
let op_bin : u8 = match op {
UnaryOperation::Not => ExpryBytecode::Not as u8,
UnaryOperation::Negate => ExpryBytecode::ArithmeticUnaryMin as u8,
};
writer.write_u8(op_bin)?;
retval &= _expry_ast_to_bytecode(lhs, current_position, next_position, writer, depth-1)?;
},
ExpryAST::Function(name, args, _) => {
if *name == "undefined" && args.len() == 1 {
// special handling because length of expression is needed (because of early abort
// on error)
writer.write_u8(ExpryBytecode::Undefined as u8)?;
retval &= write_with_header(writer, Out::write_var_u64, |writer| {
_expry_ast_to_bytecode(&args[0], current_position, next_position, writer, depth-1)
})?;
} else if *name == "defined" && args.len() == 1 {
// special handling because length of expression is needed (because of early abort
// on error)
writer.write_u8(ExpryBytecode::Defined as u8)?;
retval &= write_with_header(writer, Out::write_var_u64, |writer| {
_expry_ast_to_bytecode(&args[0], current_position, next_position, writer, depth-1)
})?;
} else if let Some(opcode) = internal_function(*name, args.len()) {
writer.write_u8(opcode)?;
for arg in args {
retval &= _expry_ast_to_bytecode(arg, current_position, next_position, writer, depth-1)?;
}
} else {
writer.write_u8(ExpryBytecode::Call as u8)?;
writer.write_var_bytes(name.as_bytes())?;
writer.write_var_u64(args.len() as u64)?;
for i in 0..args.len() {
let next = expry_min(ast_properties_combine(&args[i+1..]).minimal_field, next_position);
_expry_ast_to_bytecode(&args[i], current_position, next, writer, depth-1)?;
}
}
},
ExpryAST::Conditional(c, t, e, _) => {
writer.write_u8(ExpryBytecode::Conditional as u8)?;
retval &= write_with_header(writer, Out::write_var_u64, |writer| {
_expry_ast_to_bytecode(c, current_position, expry_min(ast_properties(t).minimal_field, expry_min(ast_properties(e).minimal_field, next_position)), writer, depth-1)
})?;
let mut t_current = *current_position;
retval &= write_with_header(writer, Out::write_var_u64, |writer| {
_expry_ast_to_bytecode(t, &mut t_current, next_position, writer, depth-1)
})?;
let mut e_current = *current_position;
retval &= write_with_header(writer, Out::write_var_u64, |writer| {
_expry_ast_to_bytecode(e, &mut e_current, next_position, writer, depth-1)
})?;
*current_position = expry_min(t_current, e_current);
},
ExpryAST::Object(fields, _) => {
let slice_static = fields.iter().all(|(k,_)| matches!(k, ExpryAST::Constant(DecodedValue::String(_))));
if slice_static {
writer.write_u8(ExpryBytecode::SliceObjectStatic as u8)?;
} else {
writer.write_u8(ExpryBytecode::SliceObjectDynamic as u8)?;
}
writer.write_var_u64(fields.len() as u64)?;
for i in 0..fields.len() {
// FIXME: benchmark the code, becuase it might be slow because it calculates a
// property over the remaining entries.
let next = expry_min(fields[i+1..].iter().map(|(x,y)| ast_properties_plus_ast(x, y)).fold(ASTProperties::constant(), |x,y| {
ast_properties_plus(&x,&y)
}).minimal_field, next_position);
if slice_static {
if let ExpryAST::Constant(DecodedValue::String(key)) = fields[i].0 {
writer.write_u8(key_stable_hash(key))?;
writer.write_var_bytes(key)?;
}
} else {
let next_with_value = expry_min(next, ast_properties(&fields[i].1).minimal_field);
retval &= _expry_ast_to_bytecode(&fields[i].0, current_position, next_with_value, writer, depth-1)?;
}
retval &= _expry_ast_to_bytecode(&fields[i].1, current_position, next, writer, depth-1)?;
}
},
ExpryAST::MergeObjects(lhs, rhs, _) => {
writer.write_u8(ExpryBytecode::MergeObjects as u8)?;
retval &= _expry_ast_to_bytecode(lhs, current_position, expry_min(ast_properties(rhs).minimal_field, next_position), writer, depth-1)?;
retval &= _expry_ast_to_bytecode(rhs, current_position, next_position, writer, depth-1)?;
},
ExpryAST::Array(fields, _) => {
writer.write_u8(ExpryBytecode::SliceArray as u8)?;
writer.write_var_u64(fields.len() as u64)?;
for i in 0..fields.len() {
let next = expry_min(ast_properties_combine(&fields[i+1..]).minimal_field, next_position);
retval &= _expry_ast_to_bytecode(&fields[i], current_position, next, writer, depth-1)?;
}
},
}
Ok(retval)
}
#[derive(Copy, Clone, PartialEq, Eq)]
pub enum EvalError<'a> {
// errors in the _encoding_ of the expression bytecode
// errors in the static types of the expression (e.g. if a certain expression always produces a boolean, but a different type was expected this implies an error during generation of the expression)
Expression(&'a str),
// errors in the _encoding_ of the value
Value(&'a str),
// type errors, division by zero
// these errors depends on the data given
Dynamic(&'a str),
// errors if a field is not found
FieldNotFound(&'a str),
// other errors (if it is unclear what is the source of the error)
Other(),
}
impl<'a> core::fmt::Debug for EvalError<'a> {
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
write!(f, "{}", self)
}
}
impl<'a> core::fmt::Display for EvalError<'a> {
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
match self {
EvalError::Expression(msg) => write!(f, "error in decoding expression: {}", msg),
EvalError::Value(msg) => write!(f, "error in decoding value: {}", msg),
EvalError::Dynamic(msg) => write!(f, "dynamic error: {}", msg),
EvalError::FieldNotFound(msg) => write!(f, "field '{}' not found", msg),
EvalError::Other() => write!(f, "unknown error"),
}
}
}
impl<'a> From<EvalError<'a>> for CompileError<'a> {
fn from(err: EvalError<'a>) -> Self {
Self {
error: CompileErrorDescription::Optimizer(err),
start: 0,
end: 0,
extra: None,
}
}
}
/// Trait that can be used to pass a custom user-provided function handler to `expry` eval functions.
pub trait CustomFuncs {
fn call<'b,'c>(&'_ mut self, name: &'_ str, args: &'_ [DecodedValue<'b>], scope: &'_ mut MemoryScope<'c>) -> Result<DecodedValue<'b>,&'b str> where 'c: 'b;
}
/// An provided implementation of the custom function handler, that directly throws an error when
/// invoked.
pub struct NoCustomFuncs {
}
impl CustomFuncs for NoCustomFuncs {
fn call<'b,'c>(&'_ mut self, _name: &'_ str, _args: &'_ [DecodedValue<'b>], _scope: &'_ mut MemoryScope<'c>) -> Result<DecodedValue<'b>,&'b str> where 'c: 'b {
Err("no custom functions defined")
}
}
struct Context<'a,'b> {
original: ValueRef<'b>, // needed for the `this` input
custom: &'a mut dyn CustomFuncs,
}
// does not throw an error if the object is not sorted, because we can not reliable detect all
// cases (if first item after previous call to evaluate_seek is less, we can not detect it without
// adding state).
fn evaluate_seek<'a,'b>(looking_for_hash: KeyHash, looking_for_key: &'b [u8], value: &mut RawReader<'a>) -> Result<(),EvalError<'b>> {
//eprintln!("seeking {}/{}", sorted_key.hash, String::from_utf8_lossy(sorted_key.key));
while !value.is_empty() {
let mut peeker = *value;
let mut key_length = peeker.read_var_u64().map_err(|_| EvalError::Value("Could not read additional field"))?;
//eprintln!("got key length {} -> {}", key_length, key_length & 0x1);
if (key_length & 0x1) != 0 {
let hash_of_next_entry = peeker.read_u8().map_err(|_| EvalError::Value("Invalid skip hash"))?;
if looking_for_hash >= hash_of_next_entry {
peeker.skip(key_length as usize >> 1).map_err(|_| EvalError::Value("seek yielded invalid index in value"))?;
}
*value = peeker;
continue;
}
key_length = (key_length >> 1) - 1;
let current_hash = peeker.read_u8().map_err(|_| EvalError::Value("Invalid hash"))?;
let current_key = peeker.read_bytes(key_length as usize).map_err(|_| EvalError::Value("Invalid key"))?;
//eprintln!("looking at {}/{} -> {:?}", hash_of_key, String::from_utf8_lossy(key.key), sorted_key.cmp(&key));
match (looking_for_hash,looking_for_key).cmp(&(current_hash, current_key)) {
Ordering::Less => break,
Ordering::Equal => return Ok(()),
Ordering::Greater => {},
}
let length_and_type = peeker.read_var_u64().map_err(|_| EvalError::Value("Could not read additional field"))?;
let length = DecodedValue::length_of_binary(length_and_type);
peeker.skip(length as usize).map_err(|_| EvalError::Value("seek yielded invalid index in value"))?;
*value = peeker;
}
Err(EvalError::FieldNotFound(core::str::from_utf8(looking_for_key).map_err(|_| EvalError::Dynamic("invalid UTF8 for key"))?))
}
fn read_hashed_string<'a>(expression: &mut RawReader<'a>) -> Result<(KeyHash,&'a [u8]),EvalError<'a>> {
let hash = expression.read_u8().map_err(|_| EvalError::Expression("read hashed string resulted in error"))?;
let key = expression.read_var_string().map_err(|_| EvalError::Expression("read hashed string resulted in error"))?;
Ok((hash, key))
}
fn evaluate_find<'a>(expression: &mut RawReader<'a>, value: &mut RawReader<'a>) -> Result<(),EvalError<'a>> {
let (looking_for_hash, looking_for_key) = read_hashed_string(expression)?;
evaluate_seek(looking_for_hash, looking_for_key, value)
}
fn get_double<'a,'b>(v: &DecodedValue<'a>) -> Result<f64,EvalError<'b>> {
match v {
DecodedValue::Int(n) => Ok(*n as f64),
DecodedValue::Float(n) => Ok(*n as f64),
DecodedValue::Double(n) => Ok(*n),
_ => Err(EvalError::Dynamic("expected a number, found something else")),
}
}
fn compare<'a,'b,'c>(lhs: &DecodedValue<'a>, rhs: &DecodedValue<'a>, scope: &mut MemoryScope<'c>) -> Result<Ordering,EvalError<'b>> where 'c: 'b {
if let (DecodedValue::Int(l), DecodedValue::Int(r)) = (&lhs, &rhs) {
return Ok(l.cmp(r));
}
if let (DecodedValue::Float(l), DecodedValue::Float(r)) = (&lhs, &rhs) {
return match l.partial_cmp(r) {
None => Err(EvalError::Dynamic("uncomparable floats")),
Some(v) => Ok(v),
};
}
if let (DecodedValue::Double(l), DecodedValue::Double(r)) = (&lhs, &rhs) {
return match l.partial_cmp(r) {
None => Err(EvalError::Dynamic("uncomparable doubles")),
Some(v) => Ok(v),
};
}
Err(EvalError::Dynamic(write!(scope, "trying to compare two values that differ in type: {} and {}", lhs.type_string(), rhs.type_string())))
}
fn find_subsequence<T>(haystack: &[T], needle: &[T]) -> Option<usize>
where for<'a> &'a [T]: PartialEq
{
haystack.windows(needle.len()).position(|window| window == needle)
}
fn evaluate_to_decoded_value<'a,'b,'c>(expression: &mut RawReader<'b>, value: &mut RawReader<'b>, context: &mut Context<'a,'b>, allocator: &mut MemoryScope<'c>) -> Result<DecodedValue<'b>,EvalError<'b>>
where 'c: 'b, 'b: 'a
{
let opcode = expression.read_u8().map_err(|_| EvalError::Expression("Expected more bytes in expression"))?;
if opcode == ExpryBytecode::Find as u8 {
match evaluate_find(expression, value) {
Err(EvalError::FieldNotFound(_)) => {},
Err(x) => return Err(x),
_ => {},
}
return evaluate_to_decoded_value(expression, value, context, allocator);
}
if opcode == ExpryBytecode::And as u8 {
let mut sub = RawReader::with(expression.read_var_string().map_err(|_| EvalError::Expression("corrupted AND"))?);
while !sub.is_empty() {
let value = evaluate_to_decoded_value(&mut sub, value, context, allocator)?;
if let DecodedValue::Bool(v) = value {
if !v {
return Ok(DecodedValue::Bool(false));
}
} else {
return Err(EvalError::Dynamic("value for AND was not a boolean"));
}
}
return Ok(DecodedValue::Bool(true));
}
if opcode == ExpryBytecode::Or as u8 {
let mut sub = RawReader::with(expression.read_var_string().map_err(|_| EvalError::Expression("corrupted OR"))?);
while !sub.is_empty() {
let value = evaluate_to_decoded_value(&mut sub, value, context, allocator)?;
if let DecodedValue::Bool(v) = value {
if v {
return Ok(DecodedValue::Bool(true));
}
} else {
return Err(EvalError::Dynamic("value for OR was not a boolean"));
}
}
return Ok(DecodedValue::Bool(false));
}
if opcode == ExpryBytecode::Call as u8 {
let function_name = expression.read_var_string().map_err(|_| EvalError::Expression("invalid CALL"))?;
let arg_count = expression.read_var_u64().map_err(|_| EvalError::Expression("invalid CALL"))?;
let mut args : Vec<DecodedValue> = Vec::new();
for _ in 0..arg_count {
let v = evaluate_to_decoded_value(expression, value, context, allocator)?;
args.push(v);
}
return context.custom.call(core::str::from_utf8(function_name).map_err(|_| EvalError::Expression("function name is not UTF8"))?, &args, allocator).map_err(EvalError::Dynamic);
}
if opcode == ExpryBytecode::All as u8 {
let mut reader = RawReader::with(context.original.get());
return DecodedValue::parse(&mut reader).map_err(|_| EvalError::Value("Corrupted original input"));
}
if opcode == ExpryBytecode::Field as u8 ||
opcode == ExpryBytecode::FindAndField as u8 {
let mut value_copy : RawReader;
if opcode == ExpryBytecode::Field as u8 {
value_copy = *value;
evaluate_find(expression, &mut value_copy)?;
} else {
evaluate_find(expression, value)?;
value_copy = *value;
}
let key_length = value_copy.read_var_u64().map_err(|_| EvalError::Value("expected key_length"))?;
if key_length == 0 || (key_length & 0x1) != 0 {
return Err(EvalError::Value("invalid key_length"));
}
let _key_name = &value_copy.read_bytes(key_length as usize >> 1).map_err(|_| EvalError::Value(""))?[1..];
//eprintln!("using field {}", String::from_utf8_lossy(_key_name));
let retval = DecodedValue::parse(&mut value_copy).map_err(|_| EvalError::Value("Corrupted field from value"))?;
return Ok(retval);
}
#[cfg(not(feature = "mini"))]
if opcode == ExpryBytecode::FieldAccessFastPath as u8 {
// This op can be implemented faster because the value does not need to be decoded at all for the field access to work.
// fast_path should only be triggered for operations that return a whole value in output
// (such as Concrete, All, Field, FieldAccess*)
let mut output = Vec::new();
evaluate_to_binary(expression, value, context, allocator, &mut output)?;
debug_assert!(output.len() == 1);
if let Some(first) = output.first() {
let rhs = evaluate_to_decoded_value(expression, value, context, allocator)?;
if let DecodedValue::String(key) = rhs {
if let LazyDecodedValue::Object(obj) = LazyDecodedValue::parse(&mut RawReader::with(*first)).map_err(|_| EvalError::Value("Corrupted lhs for field access"))? {
if let Some(value) = obj.lookup_binary(key_u8(key)).map_err(|_| EvalError::Value("Corrupted lhs for field access"))? {
let retval = DecodedValue::parse(&mut RawReader::with(value)).map_err(|_| EvalError::Value("Corrupted field from value"))?;
return Ok(retval);
} else {
return Err(EvalError::FieldNotFound(core::str::from_utf8(key).map_err(|_| EvalError::Dynamic("invalid UTF8 for key"))?));
}
} else {
return Err(EvalError::Dynamic("field access (fast path) for a left-hand-side that resolves to something different than an object"));
}
} else {
return Err(EvalError::Dynamic("field access (fast path) for a right-hand-side that resolves to something different than a string"));
}
} else {
return Err(EvalError::Dynamic("field access (fast path) for a left-hand-side that resolves to something different than an object"));
}
}
if opcode == ExpryBytecode::FieldAccessCompute as u8 {
let lhs = evaluate_to_decoded_value(expression, value, context, allocator)?;
if let DecodedValue::Object(mut obj) = lhs {
let rhs = evaluate_to_decoded_value(expression, value, context, allocator)?;
if let DecodedValue::String(key) = rhs {
if let Some(value) = obj.get_mut(&key_u8(key)) {
let value = core::mem::replace(value, DecodedValue::Null());
return Ok(value);
}
return Err(EvalError::FieldNotFound(core::str::from_utf8(key).map_err(|_| EvalError::Dynamic("invalid UTF8 for key"))?));
} else {
return Err(EvalError::Dynamic("field access (compute) for a right-hand-side that resolves to something different than a string"));
}
} else {
return Err(EvalError::Dynamic("field access (compute) for a left-hand-side that resolves to something different than an object"));
}
}
if opcode == ExpryBytecode::Concrete as u8 {
let value = DecodedValue::parse(expression).map_err(|_| {
EvalError::Expression("Corrupted concrete value")
})?;
return Ok(value)
}
// can be disabled for smaller binary size (no tables to make operation faster are compiled in)
#[cfg(not(feature = "mini"))]
if opcode == ExpryBytecode::ArithmeticPower as u8 {
let lhs = evaluate_to_decoded_value(expression, value, context, allocator)?;
let rhs = evaluate_to_decoded_value(expression, value, context, allocator)?;
let l = get_double(&lhs)?;
let r = get_double(&rhs)?;
return Ok(DecodedValue::Double(l.powf(r)));
}
if opcode == ExpryBytecode::ArithmeticPlus as u8 ||
opcode == ExpryBytecode::ArithmeticMin as u8 ||
opcode == ExpryBytecode::ArithmeticMul as u8 ||
opcode == ExpryBytecode::ArithmeticDiv as u8 ||
opcode == ExpryBytecode::ArithmeticMod as u8 ||
opcode == ExpryBytecode::ArithmeticShiftLeft as u8 ||
opcode == ExpryBytecode::ArithmeticShiftRight as u8 ||
opcode == ExpryBytecode::ArithmeticBitwiseOr as u8 ||
opcode == ExpryBytecode::ArithmeticBitwiseXor as u8 ||
opcode == ExpryBytecode::ArithmeticBitwiseAnd as u8
{
let lhs = evaluate_to_decoded_value(expression, value, context, allocator)?;
let rhs = evaluate_to_decoded_value(expression, value, context, allocator)?;
if let (DecodedValue::Int(l), DecodedValue::Int(r)) = (&lhs, &rhs) {
if opcode == ExpryBytecode::ArithmeticPlus as u8 {
return Ok(DecodedValue::Int(
l.checked_add(*r).ok_or(EvalError::Dynamic("int add resulted in overflow"))?
));
}
if opcode == ExpryBytecode::ArithmeticMin as u8 {
return Ok(DecodedValue::Int(
l.checked_sub(*r).ok_or(EvalError::Dynamic("int sub resulted in overflow"))?
));
}
if opcode == ExpryBytecode::ArithmeticMul as u8 {
return Ok(DecodedValue::Int(
l.checked_mul(*r).ok_or(EvalError::Dynamic("int mul resulted in overflow"))?
));
}
if opcode == ExpryBytecode::ArithmeticDiv as u8 {
if *r == 0 {
return Err(EvalError::Dynamic("Division by zero"));
}
return Ok(DecodedValue::Int(
l.checked_div(*r).ok_or(EvalError::Dynamic("int div resulted in overflow"))?
));
}
if opcode == ExpryBytecode::ArithmeticMod as u8 {
if *r == 0 {
return Err(EvalError::Dynamic("Modulo zero"));
}
return Ok(DecodedValue::Int(l % r));
}
if opcode == ExpryBytecode::ArithmeticShiftLeft as u8 {
return Ok(DecodedValue::Int(
l.checked_shl((*r).try_into().map_err(|_| EvalError::Dynamic("shift-left resulted in overflow"))?).ok_or(EvalError::Dynamic("shift-left resulted in overflow"))?
));
}
if opcode == ExpryBytecode::ArithmeticShiftRight as u8 {
return Ok(DecodedValue::Int(
l.checked_shr((*r).try_into().map_err(|_| EvalError::Dynamic("shift-right resulted in overflow"))?).ok_or(EvalError::Dynamic("shift-right resulted in overflow"))?
));
}
if opcode == ExpryBytecode::ArithmeticBitwiseOr as u8 {
return Ok(DecodedValue::Int(l | r));
}
if opcode == ExpryBytecode::ArithmeticBitwiseXor as u8 {
return Ok(DecodedValue::Int(l ^ r));
}
if opcode == ExpryBytecode::ArithmeticBitwiseAnd as u8 {
return Ok(DecodedValue::Int(l & r));
}
}
let l = get_double(&lhs)?;
let r = get_double(&rhs)?;
if opcode == ExpryBytecode::ArithmeticPlus as u8 {
return Ok(DecodedValue::Double(l + r));
}
if opcode == ExpryBytecode::ArithmeticMin as u8 {
return Ok(DecodedValue::Double(l - r));
}
if opcode == ExpryBytecode::ArithmeticMul as u8 {
return Ok(DecodedValue::Double(l * r));
}
if opcode == ExpryBytecode::ArithmeticDiv as u8 {
if r == 0.0 {
return Err(EvalError::Dynamic("Division by zero"));
}
return Ok(DecodedValue::Double(l / r));
}
// disabled for smaller binary size
#[cfg(not(feature = "mini"))]
if opcode == ExpryBytecode::ArithmeticMod as u8 {
if r == 0.0 {
return Err(EvalError::Dynamic("Modulo zero"));
}
return Ok(DecodedValue::Double(l % r));
}
return Err(EvalError::Dynamic("Arithmetic operation without proper arguments (maybe a bitwise operation on a float or a comparison between a float and int?)"));
}
#[cfg(not(feature = "mini"))]
if opcode == ExpryBytecode::Equal as u8 ||
opcode == ExpryBytecode::NotEqual as u8
{
let mut lhs_output = Vec::new();
evaluate_to_binary(expression, value, context, allocator, &mut lhs_output)?;
let mut rhs_output = Vec::new();
evaluate_to_binary(expression, value, context, allocator, &mut rhs_output)?;
if opcode == ExpryBytecode::NotEqual as u8 {
return Ok(DecodedValue::Bool(lhs_output != rhs_output));
}
return Ok(DecodedValue::Bool(lhs_output == rhs_output));
}
if opcode == ExpryBytecode::Not as u8 {
let lhs = evaluate_to_decoded_value(expression, value, context, allocator)?;
if let DecodedValue::Bool(b) = &lhs {
return Ok(DecodedValue::Bool(!b));
}
return Err(EvalError::Dynamic("NOT expects a boolean"));
}
if opcode == ExpryBytecode::ParseEqual as u8 ||
opcode == ExpryBytecode::ParseNotEqual as u8 ||
opcode == ExpryBytecode::Equal as u8 ||
opcode == ExpryBytecode::NotEqual as u8
{
let lhs = evaluate_to_decoded_value(expression, value, context, allocator)?;
let rhs = evaluate_to_decoded_value(expression, value, context, allocator)?;
if opcode == ExpryBytecode::ParseNotEqual as u8 ||
opcode == ExpryBytecode::NotEqual as u8 {
return Ok(DecodedValue::Bool(lhs != rhs));
}
return Ok(DecodedValue::Bool(lhs == rhs));
}
if opcode == ExpryBytecode::NumericLess as u8 ||
opcode == ExpryBytecode::NumericLessEqual as u8 ||
opcode == ExpryBytecode::NumericGreater as u8 ||
opcode == ExpryBytecode::NumericGreaterEqual as u8
{
let lhs = evaluate_to_decoded_value(expression, value, context, allocator)?;
let rhs = evaluate_to_decoded_value(expression, value, context, allocator)?;
if opcode == ExpryBytecode::NumericLess as u8 {
return Ok(DecodedValue::Bool(compare(&lhs, &rhs, allocator)? == Ordering::Less));
}
if opcode == ExpryBytecode::NumericLessEqual as u8 {
return Ok(DecodedValue::Bool(compare(&lhs, &rhs, allocator)? != Ordering::Greater));
}
if opcode == ExpryBytecode::NumericGreater as u8 {
return Ok(DecodedValue::Bool(compare(&lhs, &rhs, allocator)? == Ordering::Greater));
}
if opcode == ExpryBytecode::NumericGreaterEqual as u8 {
return Ok(DecodedValue::Bool(compare(&lhs, &rhs, allocator)? != Ordering::Less));
}
unimplemented!();
}
if opcode == ExpryBytecode::StringContains as u8 ||
opcode == ExpryBytecode::StringStartsWith as u8 ||
opcode == ExpryBytecode::StringEndsWith as u8
{
let lhs = evaluate_to_decoded_value(expression, value, context, allocator)?;
let rhs = evaluate_to_decoded_value(expression, value, context, allocator)?;
match (lhs, opcode, rhs) {
(DecodedValue::String(lhs), op, DecodedValue::String(rhs)) if op == ExpryBytecode::StringContains as u8 => {
if rhs.is_empty() {
return Err(EvalError::Dynamic("string contains operation with zero-size right hand size"));
}
return Ok(DecodedValue::Bool(find_subsequence(lhs, rhs).is_some()));
},
(DecodedValue::String(lhs), op, DecodedValue::String(rhs)) if op == ExpryBytecode::StringStartsWith as u8 => {
return Ok(DecodedValue::Bool(lhs.starts_with(rhs)));
},
(DecodedValue::String(lhs), op, DecodedValue::String(rhs)) if op == ExpryBytecode::StringEndsWith as u8 => {
return Ok(DecodedValue::Bool(lhs.ends_with(rhs)));
},
_ => return Err(EvalError::Dynamic("string operation (contains, starts-with, ends-with) expects two string arguments")),
}
}
if opcode == ExpryBytecode::StringConcat as u8 {
let lhs = evaluate_to_decoded_value(expression, value, context, allocator)?;
let rhs = evaluate_to_decoded_value(expression, value, context, allocator)?;
match (lhs, rhs) {
(DecodedValue::String(lhs), DecodedValue::String(rhs)) => {
let value = allocator.concat_u8(&[lhs, rhs]);
return Ok(DecodedValue::String(value));
},
(lhs, rhs) => return Err(EvalError::Dynamic(write!(allocator, "string concat expects two string arguments, not {} and {}", lhs.type_string(), rhs.type_string()))),
}
}
if opcode == ExpryBytecode::Length as u8 {
let lhs = evaluate_to_decoded_value(expression, value, context, allocator)?;
match lhs {
DecodedValue::String(s) => return Ok(DecodedValue::Int(s.len() as i64)),
DecodedValue::Array(arr) => return Ok(DecodedValue::Int(arr.len() as i64)),
DecodedValue::Object(obj) => return Ok(DecodedValue::Int(obj.len() as i64)),
_ => return Err(EvalError::Dynamic(write!(allocator, "taking len() from non supported value type: {}", lhs.type_string()))),
}
}
if opcode == ExpryBytecode::StringDirname as u8 ||
opcode == ExpryBytecode::StringBasename as u8
{
let string = evaluate_to_decoded_value(expression, value, context, allocator)?;
if let DecodedValue::String(string) = string {
let mut it = string.rsplitn(2, |x| *x == b'/');
if let (Some(basename), dirname) = (it.next(), it.next()) {
return Ok(DecodedValue::String(if opcode == ExpryBytecode::StringBasename as u8 {
basename
} else if let Some(dirname) = dirname {
dirname
} else {
b""
}));
}
} else {
return Err(EvalError::Dynamic("trying to apply basename()/dirname() to non supported value type"));
}
}
if opcode == ExpryBytecode::StringSplit as u8 {
let string = evaluate_to_decoded_value(expression, value, context, allocator)?;
let max = evaluate_to_decoded_value(expression, value, context, allocator)?;
let split_on = evaluate_to_decoded_value(expression, value, context, allocator)?;
if let (DecodedValue::String(mut string), DecodedValue::Int(mut max), DecodedValue::String(split_on)) = (string, max, split_on) {
let mut retval = DecodedArray::new();
if max > 1 {
max -= 1;
while let Some(pos) = find_subsequence(string, split_on) {
let (begin, end) = string.split_at(pos);
retval.push(DecodedValue::String(begin));
string = &end[split_on.len()..];
if retval.len() >= max as usize {
break;
}
}
}
if !string.is_empty() {
retval.push(DecodedValue::String(string));
}
return Ok(DecodedValue::Array(retval));
} else {
return Err(EvalError::Dynamic("splitn() method expects to be called on a string, with a int and string argument"));
}
}
if opcode == ExpryBytecode::StringSubstringFromTo as u8 ||
opcode == ExpryBytecode::StringSubstringFrom as u8 {
let string = evaluate_to_decoded_value(expression, value, context, allocator)?;
let start = evaluate_to_decoded_value(expression, value, context, allocator)?;
let length = if opcode == ExpryBytecode::StringSubstringFromTo as u8 {
evaluate_to_decoded_value(expression, value, context, allocator)?
} else {
DecodedValue::Int(i64::MAX)
};
if let DecodedValue::String(string) = string {
if let DecodedValue::Int(start) = start {
if let DecodedValue::Int(length) = length {
// u32 for 32-bits targets
let start = start.clamp(0, u32::MAX as i64) as usize;
let length = length.clamp(0, u32::MAX as i64) as usize;
let from = core::cmp::min(start, string.len());
let to = core::cmp::min((start).saturating_add(length), string.len());
return Ok(DecodedValue::String(&string[from..to]));
} else {
return Err(EvalError::Dynamic("substring needs an int as third argument"));
}
} else {
return Err(EvalError::Dynamic("substring needs an int as second argument"));
}
} else {
return Err(EvalError::Dynamic("substring from string needs a string as first argument"));
}
}
if opcode == ExpryBytecode::StringUpper as u8 ||
opcode == ExpryBytecode::StringLower as u8 ||
opcode == ExpryBytecode::StringTrim as u8 ||
opcode == ExpryBytecode::StringHex as u8 ||
opcode == ExpryBytecode::StringHtmlescape as u8 ||
opcode == ExpryBytecode::StringUrlescape as u8
{
let lhs = evaluate_to_decoded_value(expression, value, context, allocator)?;
match lhs {
DecodedValue::String(s) if opcode == ExpryBytecode::StringUpper as u8 => {
// full utf-8 to_uppercase() makes the resulting binary a lot larger (~50 KiB)
let retval = allocator.copy_u8(s);
for c in retval.iter_mut() {
c.make_ascii_uppercase();
}
return Ok(DecodedValue::String(retval));
},
DecodedValue::String(s) if opcode == ExpryBytecode::StringLower as u8 => {
// full utf-8 to_lowercase() makes the resulting binary a lot larger (~50 KiB)
let retval = allocator.copy_u8(s);
for c in retval.iter_mut() {
c.make_ascii_lowercase();
}
return Ok(DecodedValue::String(retval));
},
DecodedValue::String(mut s) if opcode == ExpryBytecode::StringTrim as u8 => {
let from = s.iter().position(|x| *x != b' ');
if let Some(from) = from {
let to = s.iter().rev().position(|x| *x != b' ');
if let Some(to) = to {
s = &s[from..s.len()-to];
} else {
s = &s[from..];
}
} else {
s = b"";
}
return Ok(DecodedValue::String(s));
},
DecodedValue::String(s) if opcode == ExpryBytecode::StringHex as u8 => {
return Ok(DecodedValue::String(allocator.copy_hex(s)));
},
DecodedValue::String(s) if opcode == ExpryBytecode::StringHtmlescape as u8 => {
return Ok(DecodedValue::String(allocator.copy_with_replacement(s, html_escape_inside_attribute_u8).map_err(|_| EvalError::Dynamic("string replacement error"))?));
},
DecodedValue::String(s) if opcode == ExpryBytecode::StringUrlescape as u8 => {
return Ok(DecodedValue::String(allocator.copy_with_dynamic_replacement(s, url_escape_u8).map_err(|_| EvalError::Dynamic("string replacement error"))?));
},
_ => return Err(EvalError::Dynamic("string function expects a string argument"))
}
}
if opcode == ExpryBytecode::ToString as u8 ||
opcode == ExpryBytecode::ToInteger as u8 ||
opcode == ExpryBytecode::ToFloat as u8 ||
opcode == ExpryBytecode::ToDouble as u8
{
let lhs = evaluate_to_decoded_value(expression, value, context, allocator)?;
if opcode == ExpryBytecode::ToString as u8 {
if !lhs.is_valid_json() {
return Err(EvalError::Dynamic("TO_STRING resulted in Utf8Error"));
}
return Ok(DecodedValue::String(write!(allocator, "{}", lhs).as_bytes()));
}
if opcode == ExpryBytecode::ToInteger as u8 {
return match lhs {
DecodedValue::Int(v) => Ok(DecodedValue::Int(v as i64)),
DecodedValue::Float(v) => Ok(DecodedValue::Int(v as i64)),
DecodedValue::Double(v) => Ok(DecodedValue::Int(v as i64)),
DecodedValue::String(v) => Ok(DecodedValue::Int(core::str::from_utf8(v).map_err(|_| EvalError::Dynamic("invalid UTF8 for tointeger()"))?.parse().map_err(|_| EvalError::Dynamic("could not parse in tointeger()"))?)),
_ => Err(EvalError::Dynamic("tointeger() expects a number or a string")),
}
}
if opcode == ExpryBytecode::ToFloat as u8 {
return match lhs {
DecodedValue::Int(v) => Ok(DecodedValue::Float(v as f32)),
DecodedValue::Float(v) => Ok(DecodedValue::Float(v as f32)),
DecodedValue::Double(v) => Ok(DecodedValue::Float(v as f32)),
DecodedValue::String(v) => Ok(DecodedValue::Float(core::str::from_utf8(v).map_err(|_| EvalError::Dynamic("invalid UTF8 for tofloat()"))?.parse().map_err(|_| EvalError::Dynamic("could not parse in tofloat()"))?)),
_ => Err(EvalError::Dynamic("tofloat() expects a number or a string")),
}
}
if opcode == ExpryBytecode::ToDouble as u8 {
return match lhs {
DecodedValue::Int(v) => Ok(DecodedValue::Double(v as f64)),
DecodedValue::Float(v) => Ok(DecodedValue::Double(v as f64)),
DecodedValue::Double(v) => Ok(DecodedValue::Double(v as f64)),
DecodedValue::String(v) => Ok(DecodedValue::Double(core::str::from_utf8(v).map_err(|_| EvalError::Dynamic("invalid UTF8 for todouble()"))?.parse().map_err(|_| EvalError::Dynamic("could not parse in todouble()"))?)),
_ => Err(EvalError::Dynamic("todouble() expects a number or a string")),
}
}
unimplemented!();
}
if opcode == ExpryBytecode::ArithmeticUnaryMin as u8 {
let lhs = evaluate_to_decoded_value(expression, value, context, allocator)?;
match lhs {
// signed overflow protection here (generating a dynamic error)
// signed int can overflow if -i64::MIN is calculated
DecodedValue::Int(i) => return Ok(DecodedValue::Int(i.checked_neg().ok_or(EvalError::Dynamic("UNARY_MIN resulted in overflow"))?)),
DecodedValue::Float(i) => return Ok(DecodedValue::Float(-i)),
DecodedValue::Double(i) => return Ok(DecodedValue::Double(-i)),
_ => return Err(EvalError::Dynamic("UNARY_MIN expects a boolean")),
}
}
// FIXME: is_type etc
// FIXME: math functions: ceil, floor, trunc, round, sqrt, abs
if opcode == ExpryBytecode::Defined as u8 ||
opcode == ExpryBytecode::Undefined as u8
{
let mut lhs_reader = RawReader::with(expression.read_var_string().map_err(|_| EvalError::Expression("error in (UN)DEFINED"))?);
let result = evaluate_to_decoded_value(&mut lhs_reader, value, context, allocator);
if opcode == ExpryBytecode::Defined as u8 {
return Ok(DecodedValue::Bool(result.is_ok()));
}
return Ok(DecodedValue::Bool(result.is_err()));
}
if opcode == ExpryBytecode::Try as u8 {
let value_copy = *value;
let mut lhs_reader = RawReader::with(expression.read_var_string().map_err(|_| EvalError::Expression("error in TRY"))?);
let rhs_size = expression.read_var_u64().map_err(|_| EvalError::Expression("error in TRY"))?;
match evaluate_to_decoded_value(&mut lhs_reader, value, context, allocator) {
Ok(v) => {
expression.skip(rhs_size as usize).map_err(|_| EvalError::Expression("error in TRY"))?;
return Ok(v);
}
Err(err) if matches!(err, EvalError::Dynamic(_) | EvalError::FieldNotFound(_)) => {
*value = value_copy;
return evaluate_to_decoded_value(expression, value, context, allocator);
}
Err(err) => {
return Err(err);
}
}
}
if opcode == ExpryBytecode::NotNullElse as u8 {
let value_copy = *value;
let mut lhs_reader = RawReader::with(expression.read_var_string().map_err(|_| EvalError::Expression("error in NOTNULLELSE"))?);
let rhs_size = expression.read_var_u64().map_err(|_| EvalError::Expression("error in NOTNULLELSE"))?;
match evaluate_to_decoded_value(&mut lhs_reader, value, context, allocator) {
Ok(DecodedValue::Null()) => {
*value = value_copy;
return evaluate_to_decoded_value(expression, value, context, allocator);
},
Ok(v) => {
expression.skip(rhs_size as usize).map_err(|_| EvalError::Expression("error in TRY"))?;
return Ok(v);
}
Err(err) => {
return Err(err);
}
}
}
if opcode == ExpryBytecode::SliceObjectDynamic as u8 {
let count = expression.read_var_u64().map_err(|_| EvalError::Expression("error in SLICEOBJECT"))?;
let mut retval = DecodedObject::new();
for _ in 0..count {
match evaluate_to_decoded_value(expression, value, context, allocator)? {
DecodedValue::String(key) => {
let hash = key_stable_hash(key);
retval.insert((hash, key), evaluate_to_decoded_value(expression, value, context, allocator)?);
},
_ => {
return Err(EvalError::Dynamic("expected string as key for a SLICEOBJECT"));
}
}
}
return Ok(DecodedValue::Object(retval));
}
if opcode == ExpryBytecode::SliceObjectStatic as u8 {
let count = expression.read_var_u64().map_err(|_| EvalError::Expression("error in SLICEOBJECT"))?;
let mut retval = DecodedObject::new();
for _ in 0..count {
let hash_of_key = expression.read_u8().map_err(|_| EvalError::Value("invalid hash in SLICEOBJECT"))?;
let key_length = expression.read_var_u64().map_err(|_| EvalError::Expression("error in SLICEOBJECT"))?;
let key = expression.read_bytes(key_length as usize).map_err(|_| EvalError::Expression("Invalid key in slice object"))?;
retval.insert((hash_of_key, key), evaluate_to_decoded_value(expression, value, context, allocator)?);
}
return Ok(DecodedValue::Object(retval));
}
if opcode == ExpryBytecode::MergeObjects as u8 {
let mut lhs = evaluate_to_decoded_value(expression, value, context, allocator)?;
let mut rhs = evaluate_to_decoded_value(expression, value, context, allocator)?;
if let (DecodedValue::Object(lhs), DecodedValue::Object(rhs)) = (&mut lhs, &mut rhs) {
for (k,v) in rhs {
let v = core::mem::replace(v, DecodedValue::Null());
lhs.insert(*k, v);
}
let v = core::mem::take(lhs);
return Ok(DecodedValue::Object(v));
}
return Err(EvalError::Dynamic(write!(allocator, "merging object (with e.g. spread syntax) expects two objects as arguments: lhs={}, rhs={}", lhs.type_string(), rhs.type_string())));
}
if opcode == ExpryBytecode::SliceArray as u8 {
let count = expression.read_var_u64().map_err(|_| EvalError::Expression("error in SLICEARRAY"))?;
let mut retval = DecodedArray::new();
for _ in 0..count {
retval.push(evaluate_to_decoded_value(expression, value, context, allocator)?);
}
return Ok(DecodedValue::Array(retval));
}
if opcode == ExpryBytecode::Conditional as u8 {
let mut c = RawReader::with(expression.read_var_string().map_err(|_| EvalError::Expression("error in CONDITIONAL condition"))?);
let t = RawReader::with(expression.read_var_string().map_err(|_| EvalError::Expression("error in CONDITIONAL then"))?);
let e = RawReader::with(expression.read_var_string().map_err(|_| EvalError::Expression("error in CONDITIONAL else"))?);
match evaluate_to_decoded_value(&mut c, value, context, allocator) {
Ok(DecodedValue::Bool(c)) => {
return evaluate_to_decoded_value(&mut if c { t } else { e }, value, context, allocator);
},
Ok(_) => {
return Err(EvalError::Dynamic("conditional expects a boolean for the condition"));
}
Err(err) => {
return Err(err);
}
}
}
Err(EvalError::Expression(write!(allocator, "Unrecognized opcode {}", opcode)))
}
fn create_object_header<'c>(scope: &mut MemoryScope<'c>, hash_of_key: u8, key: &[u8]) -> &'c [u8] {
let prefix = scope.alloc(size_of_var_u64(((key.len() + KEY_HASH_SIZE) << 1) as u64) + KEY_HASH_SIZE + key.len());
let mut writer = RawWriter::with(prefix);
writer.write_var_u64(((key.len() + KEY_HASH_SIZE) << 1) as u64).unwrap();
writer.write_u8(hash_of_key).unwrap();
writer.write_bytes(key).unwrap();
debug_assert!(writer.left() == 0);
prefix
}
// maybe return an array of [u8], to avoid copying. Only concat the value in the expry_slice_func
// function. Using this approach MergeObjects can also be efficiently implemented.
fn evaluate_to_binary<'a,'b,'c>(expression: &mut RawReader<'b>, value: &mut RawReader<'b>, context: &mut Context<'a,'b>, allocator: &mut MemoryScope<'c>, output: &mut Vec<&'b [u8]>) -> Result<(),EvalError<'b>>
where 'c: 'b, 'b: 'a
{
let expression_original = *expression;
if !cfg!(feature = "mini") {
let opcode = expression.read_u8().map_err(|_| EvalError::Expression("Expected more bytes in expression"))?;
if opcode == ExpryBytecode::Find as u8 {
match evaluate_find(expression, value) {
Err(EvalError::FieldNotFound(_)) => {},
Err(x) => return Err(x),
_ => {},
}
return evaluate_to_binary(expression, value, context, allocator, output);
}
if opcode == ExpryBytecode::All as u8 {
output.push(context.original.get());
return Ok(());
}
if opcode == ExpryBytecode::Field as u8 ||
opcode == ExpryBytecode::FindAndField as u8 {
let mut value_copy : RawReader;
if opcode == ExpryBytecode::Field as u8 {
value_copy = *value;
evaluate_find(expression, &mut value_copy)?;
} else {
evaluate_find(expression, value)?;
value_copy = *value;
}
let key_length = value_copy.read_var_u64().map_err(|_| EvalError::Value("expected key_length"))?;
if key_length == 0 || (key_length & 0x1) != 0 {
return Err(EvalError::Value("invalid key_length"));
}
let _key_name = &value_copy.read_bytes(key_length as usize >> 1).map_err(|_| EvalError::Value(""))?[1..];
// decoded correct length and copy only that (not the remainder)
let mut reader = value_copy;
let type_and_length = reader.read_var_u64().map_err(|_| EvalError::Value("expected type_and_length of object"))?;
let len = (value_copy.len() - reader.len()) + DecodedValue::length_of_binary(type_and_length);
value_copy.truncate(len);
output.push(value_copy.data());
return Ok(());
}
if opcode == ExpryBytecode::Concrete as u8 {
let mut expression_copy = *expression;
let type_and_length = expression_copy.read_var_u64().map_err(|_| {
EvalError::Expression("Corrupted concrete value")
})?;
let (_, len) = DecodedValue::decoded_type_and_length(type_and_length).map_err(|_| {
EvalError::Expression("Corrupted concrete value")
})?;
output.push(expression.read_bytes(len + expression.len() - expression_copy.len()).map_err(|_| {
EvalError::Expression("Corrupted concrete value")
})?);
return Ok(());
}
if opcode == ExpryBytecode::FieldAccessFastPath as u8 {
// fast_path should only be triggered for operations that return a whole value in output
// (such as Concrete, All, Field, FieldAccess*)
let mut lhs_output = Vec::new();
evaluate_to_binary(expression, value, context, allocator, &mut lhs_output)?;
debug_assert!(lhs_output.len() == 1);
if let Some(first) = lhs_output.first() {
let rhs = evaluate_to_decoded_value(expression, value, context, allocator)?;
if let DecodedValue::String(key) = rhs {
if let LazyDecodedValue::Object(obj) = LazyDecodedValue::parse(&mut RawReader::with(*first)).map_err(|_| EvalError::Value("Corrupted lhs for field access"))? {
if let Some(value) = obj.lookup_binary(key_u8(key)).map_err(|_| EvalError::Value("Corrupted lhs for field access"))? {
output.push(value);
return Ok(());
} else {
return Err(EvalError::FieldNotFound(core::str::from_utf8(key).map_err(|_| EvalError::Dynamic("invalid UTF8 for key"))?));
}
} else {
return Err(EvalError::Dynamic("field access for a left-hand-side that resolves to something different than an object"));
}
} else {
return Err(EvalError::Dynamic("field access for a right-hand-side that resolves to something different than a string"));
}
} else {
return Err(EvalError::Dynamic("field access for a left-hand-side that resolves to something different than an object"));
}
}
if opcode == ExpryBytecode::FieldAccessCompute as u8 {
// avoid serializing a complete object before only taking part of it
let lhs = evaluate_to_decoded_value(expression, value, context, allocator)?;
if let DecodedValue::Object(mut obj) = lhs {
let rhs = evaluate_to_decoded_value(expression, value, context, allocator)?;
if let DecodedValue::String(key) = rhs {
if let Some(value) = obj.get_mut(&key_u8(key)) {
let value = core::mem::replace(value, DecodedValue::Null());
let len = value.size_of_binary(false);
let buffer = allocator.alloc(len);
let mut writer = RawWriter::with(buffer);
value.print_binary(&mut writer, false).map_err(|_| EvalError::Other())?;
output.push(buffer);
return Ok(());
}
return Err(EvalError::FieldNotFound(core::str::from_utf8(key).map_err(|_| EvalError::Dynamic("invalid UTF8 for key"))?));
} else {
return Err(EvalError::Dynamic("field access for a right-hand-side that resolves to something different than a string"));
}
} else {
return Err(EvalError::Dynamic("field access for a left-hand-side that resolves to something different than an object"));
}
}
// if opcode == ExpryBytecode::SliceObjectStatic as u8 ||
// opcode == ExpryBytecode::SliceObjectDynamic as u8 {
// let count = expression.read_var_u64().map_err(|_| EvalError::Expression("error in SLICEOBJECT"))?;
// let index = output.len();
// output.push(&[]); // reserve space for object header
// if opcode == ExpryBytecode::SliceObjectStatic as u8 {
// // FIXME: sort output based on key (as input is sorted on accesses and not on
// output key)
// let mut last_key = None;
// for _ in 0..count {
// let hash_of_key = expression.read_u8().map_err(|_| EvalError::Value("invalid hash in SLICEOBJECT"))?;
// let key_length = expression.read_var_u64().map_err(|_| EvalError::Expression("error in SLICEOBJECT"))?;
// let key = expression.read_bytes(key_length as usize).map_err(|_| EvalError::Expression("Invalid key in slice object"))?;
// if let Some(last_key) = last_key {
// debug_assert!(last_key < key);
// }
// last_key = Some(key);
// let prefix = create_object_header(allocator, hash_of_key, key);
// output.push(prefix);
// evaluate_to_binary(expression, value, context, allocator, output)?;
// }
// } else {
// // FIXME: sort output based on key (as input is sorted on accesses and not on
// output key)
// for _ in 0..count {
// match evaluate_to_decoded_value(expression, value, context, allocator)? {
// DecodedValue::String(key) => {
// let hash_of_key = key_stable_hash(key);
// let prefix = create_object_header(allocator, hash_of_key, key);
// output.push(prefix);
// evaluate_to_binary(expression, value, context, allocator, output)?;
// },
// _ => {
// return Err(EvalError::Dynamic("expected string as key for a SLICEOBJECT"));
// }
// }
// }
// }
// let size = output[index+1..].iter().fold(0usize, |sum,val| sum + val.len());
// output[index] = to_var_length(allocator, size << WIDTH_OF_JSON_TYPE_MASK | ValueType::Object as usize);
// return Ok(());
// }
if opcode == ExpryBytecode::Try as u8 {
let value_copy = *value;
let mut lhs_reader = RawReader::with(expression.read_var_string().map_err(|_| EvalError::Expression("error in TRY"))?);
let rhs_size = expression.read_var_u64().map_err(|_| EvalError::Expression("error in TRY"))?;
let output_len = output.len();
match evaluate_to_binary(&mut lhs_reader, value, context, allocator, output) {
Ok(()) => {
expression.skip(rhs_size as usize).map_err(|_| EvalError::Expression("error in TRY"))?;
return Ok(());
}
Err(err) if matches!(err, EvalError::Dynamic(_) | EvalError::FieldNotFound(_)) => {
*value = value_copy;
output.truncate(output_len);
return evaluate_to_binary(expression, value, context, allocator, output);
}
Err(err) => {
return Err(err);
}
}
}
// FIXME: add case for NotNullElse
// FIXME: add case for SliceArray
}
// use dynamic evaluation and convert to a binary value
*expression = expression_original;
let value = evaluate_to_decoded_value(expression, value, context, allocator)?;
let len = value.size_of_binary(false);
let buffer = allocator.alloc(len);
let mut writer = RawWriter::with(buffer);
value.print_binary(&mut writer, false).map_err(|_| EvalError::Other())?;
output.push(buffer);
Ok(())
}
// FIXME: add properties to the result value (as a seperate struct):
// - needs dynamic evaluation;
// - is a super simple slicer (only returns 1 part of the input value or bytecode)
// - is a simple slicer (only returns a composition of multiple values)
/// Compiles an expression to bytecode that can quickly be evaluated using [`expry_eval`] and
/// [`expry_slice`].
pub fn expry_compile<'b,'c>(expr: &'b str, static_value: Option<&DecodedValue<'b>>, scope: &mut MemoryScope<'c>) -> Result<BytecodeRef<'b>, CompileError<'b>> where 'c: 'b {
if expr.len() >= u32::MAX as usize {
return Err(CompileError{error: CompileErrorDescription::Parser("input too large to parse (max 4 GiB)"), start: expr.len(), end: 0, extra: None});
}
let default_value = DecodedValue::Object(DecodedObject::new());
let static_value = static_value.unwrap_or(&default_value);
#[allow(unused_mut)]
let mut object = parse_expry_expr(expr, static_value, scope).map_err(|(error, start, end, extra)| CompileError{ error, start, end, extra })?;
#[cfg(not(feature = "mini"))]
{
// FIXME: return source range where the error is triggered (in the fold/optimize)
expry_ast_fold(&mut object, scope).map_err(|x| CompileError{error: CompileErrorDescription::Optimizer(x), start: 0, end: 0, extra: None})?;
expry_ast_optimize(&mut object).map_err(|x| CompileError{error: CompileErrorDescription::Optimizer(x), start: 0, end: 0, extra: None})?;
}
let bytecode = expry_ast_to_bytecode(scope, &object).map_err(|x| CompileError{error: CompileErrorDescription::Parser(x), start: 0, end: 0, extra: None})?;
Ok(bytecode)
}
/// Formats errors from compilation with a nice layout and underlines on the source lines, so that
/// users can easily understand what to do. Returns the line number of the main error, and a string
/// that can be displayed to the user.
pub fn expry_compile_error_format(expr: &str, e: &CompileError, extra_line_no: u32) -> (u32,String) {
let mut retval = String::new();
let line_context = LineContext::new(expr);
let (line_no, prefix, error_msg) = line_context.format_error_context(expr, e.start, e.end, extra_line_no).map_or((0u32, String::new(), String::new()), |x| x);
if line_no > 0 {
writeln!(&mut retval, "{}{}error at line {}:{} {}", prefix, TERM_BRIGHT_RED, line_no, TERM_RESET, e.error).ok();
retval.push_str(&error_msg);
}
if let Some((start, end)) = e.extra {
let (line_no, prefix, error_msg) = line_context.format_error_context(expr, start, end, extra_line_no).map_or((0u32, String::new(), String::new()), |x| x);
if line_no > 0 {
writeln!(&mut retval, "{}{}related at line {}:{} expected it here (or earlier)", prefix, TERM_DIM_CYAN, line_no, TERM_RESET).ok();
retval.push_str(&error_msg);
}
}
(line_no, retval)
}
pub fn expry_compile_error_format_short(expr: &str, e: &CompileError) -> String {
let mut retval = String::new();
let line_context = LineContext::new(expr);
let error_msg = line_context.format_error_context_short(expr, e.start, e.end).map_or(String::new(), |x| x);
writeln!(&mut retval, "error: {}", e.error).ok();
retval.push_str(&error_msg);
if let Some((start, end)) = e.extra {
let error_msg = line_context.format_error_context_short(expr, start, end).map_or(String::new(), |x| x);
retval.push_str("\nrelated: expected it here (or earlier)\n");
retval.push_str(&error_msg);
}
retval
}
/// Evaluate expression bytecode in the context of the given `value`, resulting in a **decoded**
/// object.
pub fn expry_eval<'a,'c>(bytecode: BytecodeRef<'a>, value: ValueRef<'a>, allocator: &mut MemoryScope<'c>) -> Result<DecodedValue<'a>, EvalError<'a>> where 'c: 'a {
let mut funcs = NoCustomFuncs{};
expry_eval_func(bytecode, value, allocator, &mut funcs)
}
/// Evaluate expression bytecode in the context of the given `value`, resulting in a **decoded**
/// object. This version supports custom user defined functions.
pub fn expry_eval_func<'a,'b,'c>(bytecode: BytecodeRef<'b>, value: ValueRef<'b>, allocator: &mut MemoryScope<'c>, funcs: &'a mut dyn CustomFuncs) -> Result<DecodedValue<'b>, EvalError<'b>>
where 'c: 'b, 'b: 'a
{
let mut bytecode_reader = RawReader::with(bytecode.0);
let mut value_reader = RawReader::with(value.0);
let mut context = Context { original: value, custom: funcs};
let err = || EvalError::Value("value type can not be decoded");
if !value_reader.is_empty() && DecodedValue::type_of_binary(value_reader.read_var_u64().map_err(|_| err())?) != ValueType::Object as u8 {
// still useful, because the `this` field can obtain the int, bool, etc variable
value_reader = RawReader::with(b"");
}
let result = evaluate_to_decoded_value(&mut bytecode_reader, &mut value_reader, &mut context, allocator)?;
Ok(result)
}
/// Evaluate expression bytecode in the context of the given `value`, resulting in an **encoded**
/// object. This version supports custom user defined functions.
///
/// This object can be directly transfered over a wire.
///
/// This version is cheaper because values
/// can be included directly in encoded form, so a decoding step is skipped.
pub fn expry_slice_func<'a,'b,'c>(bytecode: BytecodeRef<'b>, value: ValueRef<'b>, allocator: &'a mut MemoryScope<'c>, funcs: &'a mut dyn CustomFuncs) -> Result<ValueRef<'b>, EvalError<'b>>
where 'c: 'b, 'b: 'a
{
let mut bytecode_reader = RawReader::with(bytecode.0);
let mut value_reader = RawReader::with(value.0);
let mut context = Context { original: value, custom: funcs};
let err = || EvalError::Value("value type can not be decoded");
if !value_reader.is_empty() && DecodedValue::type_of_binary(value_reader.read_var_u64().map_err(|_| err())?) != ValueType::Object as u8 {
// still useful, because the `this` field can obtain the int, bool, etc variable
value_reader = RawReader::with(b"");
}
let mut output = Vec::new();
evaluate_to_binary(&mut bytecode_reader, &mut value_reader, &mut context, allocator, &mut output)?;
debug_assert!(!output.is_empty());
let output = allocator.concat_u8(&output);
if cfg!(debug_assertions) {
let mut reader = RawReader::with(output);
let len = reader.read_var_u64().unwrap();
debug_assert_eq!(DecodedValue::length_of_binary(len), reader.len());
debug_assert!(DecodedValue::parse(&mut RawReader::with(output)).is_ok());
}
Ok(ValueRef(output))
}
/// Evaluate expression bytecode in the context of the given `value`, resulting in an **encoded**
/// object.
///
/// The encoded resulting value can be directly transfered over a wire.
///
/// This version is cheaper because values
/// can be included directly in encoded form, so a decoding step is skipped.
pub fn expry_slice<'b,'c>(bytecode: BytecodeRef<'b>, value: ValueRef<'b>, allocator: &mut MemoryScope<'c>) -> Result<ValueRef<'b>, EvalError<'b>> where 'c: 'b {
expry_slice_func(bytecode, value, allocator, &mut NoCustomFuncs{})
}
#[cfg(test)]
extern crate quickcheck;
#[cfg(test)]
#[macro_use(quickcheck)]
extern crate quickcheck_macros;
#[cfg(test)]
mod tests {
use core::time;
use std::{time::{Instant, Duration}, fs::File, io::{BufReader, Read}};
use crate::*;
fn convert_test(v: &DecodedValue) {
println!("testing {:?}", v);
let len = v.size_of_binary(false);
let mut buffer = vec![0u8; len];
let mut writer = RawWriter::with(&mut buffer[..]);
v.print_binary(&mut writer, false).unwrap();
assert_eq!(writer.left(), 0);
let mut reader = RawReader::with(&buffer[..]);
let v2 = DecodedValue::parse(&mut reader).expect("error");
assert_eq!(reader.len(), 0);
assert_eq!(*v, v2);
}
#[quickcheck]
fn quickcheck_binary_int(v: i64) -> bool {
let v = DecodedValue::Int(v);
println!("testing {:?}", v);
let len = v.size_of_binary(false);
let mut buffer = vec![0u8; len];
let mut writer = RawWriter::with(&mut buffer[..]);
v.print_binary(&mut writer, false).unwrap();
let writer_left = writer.left();
if buffer.len() - writer_left != len {
println!("wrong expected size of binary representation");
return false;
}
println!("binary representation: {:?}", buffer);
let mut reader = RawReader::with(&buffer[..]);
let v2 = DecodedValue::parse(&mut reader).expect("error");
println!("left reader: {}; left writer: {}", reader.len(), writer_left);
reader.len() == writer_left && v == v2
}
#[test]
fn basic() {
let mut foo = DecodedValue::Null();
assert_eq!(foo, foo);
println!("testing null");
foo = DecodedValue::Null();
assert_eq!(1, foo.size_of_binary(false));
convert_test(&foo);
println!("testing bool");
foo = DecodedValue::Bool(false);
assert_eq!(1, foo.size_of_binary(false));
convert_test(&foo);
foo = DecodedValue::Bool(true);
assert_eq!(1, foo.size_of_binary(false));
convert_test(&foo);
println!("testing float");
foo = DecodedValue::Float(0.0);
assert_eq!(5, foo.size_of_binary(false));
convert_test(&foo);
println!("testing double");
foo = DecodedValue::Double(0.0);
assert_eq!(9, foo.size_of_binary(false));
convert_test(&foo);
println!("testing string");
foo = DecodedValue::String("foobar".as_bytes());
assert_eq!(7, foo.size_of_binary(false));
convert_test(&foo);
println!("foo string = {:?}", foo);
println!("testing int");
foo = DecodedValue::Int(0);
assert_eq!(1, foo.size_of_binary(false));
convert_test(&foo);
foo = DecodedValue::Int(1);
assert_eq!(2, foo.size_of_binary(false));
convert_test(&foo);
foo = DecodedValue::Int(2);
assert_eq!(2, foo.size_of_binary(false));
convert_test(&foo);
foo = DecodedValue::Int(-1);
assert_eq!(2, foo.size_of_binary(false));
convert_test(&foo);
println!("testing array");
foo = DecodedValue::Array(vec![]);
assert_eq!(1, foo.size_of_binary(false));
convert_test(&foo);
let arr = vec![DecodedValue::Null()];
foo = DecodedValue::Array(arr);
assert_eq!(3, foo.size_of_binary(false));
convert_test(&foo);
println!("testing object");
foo = value!({
"null": null,
"one": 1,
"foo": {
"null": null,
"one": 1,
},
});
assert_eq!(36, foo.size_of_binary(false));
convert_test(&foo);
}
pub fn throughput(dur: time::Duration, bytes: usize) -> u64 {
let mut megabytes_per_second = bytes as u64 / dur.as_micros() as u64;
// Round to two significant digits.
if megabytes_per_second > 100 {
if megabytes_per_second % 10 >= 5 {
megabytes_per_second += 10;
}
megabytes_per_second = megabytes_per_second / 10 * 10;
}
megabytes_per_second
}
#[test] #[ignore]
fn twitter_binary_parse_fat() {
// let _guard = sentry::init(("https://examplePublicKey@o0.ingest.sentry.io/0", sentry::ClientOptions {
// release: sentry::release_name!(),
// ..Default::default()
// }));
// panic!("Everything is on fire!");
let mut allocator = MemoryPool::new();
let mut scope = allocator.rewind();
let v = get_data(&mut scope);
let bytes = v.to_vec(false);
let before = Instant::now();
let count = 256;
for _ in 0..count {
let mut reader = RawReader::with(bytes.get());
let _ = DecodedValue::parse(&mut reader).expect("error");
assert_eq!(0, reader.len());
}
let after = Instant::now();
let dur = after - before;
//println!("{}x in {} ms -> {} MB/s", count, dur.as_millis(), throughput(dur, count*bytes.len()));
println!("{}x in {} ms -> {} MB/s", count, dur.as_millis(), ((count*bytes.len()) as u128)/dur.as_micros());
let mut reader = RawReader::with(bytes.get());
let v = DecodedValue::parse(&mut reader).expect("error");
let mut buffer : Vec<u8> = vec!();
v.print_json(&mut buffer).unwrap();
// let mut f = File::create("twitter.json").unwrap();
// f.write_all(&buffer).unwrap();
// drop(f);
/*
let (v, pos) = parse(&bytes[..]).expect("error");
assert_eq!(pos, bytes.len());
let len = size_of_binary(&v, false);
let mut writer = vec![0u8; len];
let used_len = print(&v, &mut writer[..], false);
assert_eq!(used_len, len);
let (v2, pos) = parse(&writer[..]).expect("error");
assert_eq!(pos, writer.len());
assert_eq!(v, v2);
// println!("{:?}", v);
*/
assert!(false);
}
#[test] #[ignore]
fn twitter_binary_parse_lazy() {
let mut allocator = MemoryPool::new();
let mut scope = allocator.rewind();
let v = get_data(&mut scope);
let bytes = v.to_vec(false);
let before = Instant::now();
let count = 256;
for _ in 0..count {
let mut reader = RawReader::with(bytes.get());
let _ = walk_lazy_binary_object(LazyDecodedValue::parse(&mut reader).expect("error")).expect("error2");
assert_eq!(0, reader.len());
}
let after = Instant::now();
let dur = after - before;
//println!("{}x in {} ms -> {} MB/s", count, dur.as_millis(), throughput(dur, count*bytes.len()));
println!("{}x in {} ms -> {} MB/s", count, dur.as_millis(), ((count*bytes.len()) as u128)/dur.as_micros());
assert!(false);
}
fn walk_lazy_binary_object(value: LazyDecodedValue) -> Result<(),EncodingError> {
match value {
LazyDecodedValue::Null() => Ok(()),
LazyDecodedValue::Bool(_) => Ok(()),
LazyDecodedValue::Int(_) => Ok(()),
LazyDecodedValue::Float(_) => Ok(()),
LazyDecodedValue::Double(_) => Ok(()),
LazyDecodedValue::String(_) => Ok(()),
LazyDecodedValue::Object(mut obj) => {
while !obj.is_empty()? {
let ((_,_),v) = obj.get()?;
walk_lazy_binary_object(v)?;
}
Ok(())
},
LazyDecodedValue::Array(mut arr) => {
while !arr.is_empty() {
let v = arr.get()?;
walk_lazy_binary_object(v)?;
}
if !arr.reader.is_empty() {
return Err(EncodingError{ line_nr: line!(), });
}
Ok(())
},
}
}
#[test] #[ignore]
fn twitter_binary_print() {
let mut allocator = MemoryPool::new();
let mut scope = allocator.rewind();
let v = get_data(&mut scope);
let len = v.size_of_binary(false);
let count = 256;
let before = Instant::now();
for _ in 0..count {
let len = v.size_of_binary(false);
let mut buffer = vec![0u8; len];
let mut writer = RawWriter::with(&mut buffer[..]);
v.print_binary(&mut writer, false).unwrap();
assert!(!buffer.is_empty());
}
let after = Instant::now();
let dur = after - before;
//println!("{}x in {} ms -> {} MB/s", count, dur.as_millis(), throughput(dur, count*bytes.len()));
println!("{}x in {} ms -> {} MB/s", count, dur.as_millis(), ((count*len) as u128)/dur.as_micros());
assert!(false);
}
#[test] #[ignore]
fn twitter_json_print() {
let mut allocator = MemoryPool::new();
let mut scope = allocator.rewind();
let v = get_data(&mut scope);
let count = 256;
let len = 466908;
let before = Instant::now();
for _ in 0..count {
let mut buffer : Vec<u8> = Vec::with_capacity(len);
v.print_json(&mut buffer).unwrap();
assert!(buffer.len() > 0);
}
let after = Instant::now();
let dur = after - before;
//println!("{}x in {} ms -> {} MB/s", count, dur.as_millis(), throughput(dur, count*bytes.len()));
println!("{}x in {} ms -> {} MB/s", count, dur.as_millis(), ((count*len) as u128)/dur.as_micros());
assert!(false);
}
fn get_data<'a,'c>(scope: &'a mut MemoryScope<'c>) -> DecodedValue<'a> where 'c: 'a {
let f = File::open("twitter.json").unwrap();
let mut reader = BufReader::new(f);
let mut bytes = String::new();
reader.read_to_string(&mut bytes).unwrap();
let bytes = scope.copy_str(&bytes);
println!("read {} bytes", bytes.len());
DecodedValue::parse_json(bytes, scope).expect("error")
}
#[test] #[ignore]
fn twitter_json_parse() {
let f = File::open("twitter.json").unwrap();
let mut reader = BufReader::new(f);
let mut bytes = String::new();
reader.read_to_string(&mut bytes).unwrap();
println!("read {} bytes", bytes.len());
let before = Instant::now();
let count = 256;
let mut min_loop : Duration = Duration::MAX;
for _ in 0..count {
let before_loop_body = Instant::now();
let mut allocator = MemoryPool::new();
let mut scope = allocator.rewind();
let parsed = DecodedValue::parse_json(&bytes, &mut scope);
let after_loop_body = Instant::now();
assert!(parsed.is_ok());
let dur = after_loop_body - before_loop_body;
min_loop = Duration::min(min_loop, dur);
}
let after = Instant::now();
let dur = after - before;
println!("{}x in {} ms -> {} MB/s", count, dur.as_millis(), ((count*bytes.len()) as u128)/dur.as_micros());
println!("fastest loop body in {} ms -> {} MB/s", min_loop.as_millis(), ((bytes.len()) as u128)/min_loop.as_micros());
assert!(false);
}
#[test] #[ignore]
fn twitter_json_eval() {
let f = File::open("twitter.json").unwrap();
let mut reader = BufReader::new(f);
let mut bytes = String::new();
reader.read_to_string(&mut bytes).unwrap();
println!("read {} bytes", bytes.len());
let before = Instant::now();
let count = 256;
let mut min_loop : Duration = Duration::MAX;
for _ in 0..count {
let before_loop_body = Instant::now();
let mut allocator = MemoryPool::new();
let mut scope = allocator.rewind();
let mut parsed = parse_expry_expr(&bytes, &DecodedValue::Object(DecodedObject::new()), &mut scope);
if let Ok(parsed) = &mut parsed {
#[cfg(not(feature = "mini"))]
expry_ast_fold(parsed, &mut scope).unwrap();
#[cfg(not(feature = "mini"))]
expry_ast_optimize(parsed).unwrap();
let bytecode = expry_ast_to_bytecode(&mut scope, &parsed).expect("bytecode generator error");
assert!(!bytecode.is_empty());
} else {
assert!(false);
}
let after_loop_body = Instant::now();
assert!(parsed.is_ok());
let dur = after_loop_body - before_loop_body;
min_loop = Duration::min(min_loop, dur);
}
let after = Instant::now();
let dur = after - before;
println!("{}x in {} ms -> {} MB/s", count, dur.as_millis(), ((count*bytes.len()) as u128)/dur.as_micros());
println!("fastest loop body in {} ms -> {} MB/s", min_loop.as_millis(), ((bytes.len()) as u128)/min_loop.as_micros());
assert!(false);
}
#[test]
fn insert_jumps() {
let size : u32 = 9;
for i in 0..size {
for j in 0..i.trailing_zeros() {
let to = i + (2 << j);
if to > size {
break;
}
println!("jump to {}", to);
}
println!("{}", i);
}
//assert!(false);
}
#[test]
fn parsing_json() {
//let test = br#"[PI,123,456,789,1e9,0.001,-42,"foobar",{"a":1,"b":2}]"#;
let test = r#"{"a":1,"b":2}"#;
let mut allocator = MemoryPool::new();
let mut scope = allocator.rewind();
let object = DecodedValue::parse_json(test, &mut scope).unwrap();
println!("Parsed object: {:?}", object);
}
fn expression_tester<'a>(v: &ValueVec, expr: &str, expected: Result<DecodedValue<'a>,EvalError<'a>>) {
println!("testing expr {}, expecting {:?}", expr, expected);
let mut allocator = MemoryPool::new();
let mut scope = allocator.rewind();
let bytecode = expry_compile(expr, None, &mut scope);
if let Ok(bytecode) = bytecode {
let result = expry_eval(bytecode, v.to_ref(), &mut scope);
assert_eq!(expected, result);
} else if let Err(error) = bytecode {
println!("{}", expry_compile_error_format(expr, &error, 0).1);
assert!(false);
}
}
fn expression_tester_parser_error<'a>(expr: &str, expected: CompileErrorDescription<'a>) {
println!("testing expr {}, expecting {:?}", expr, expected);
let mut allocator = MemoryPool::new();
let mut scope = allocator.rewind();
let bytecode = expry_compile(expr, None, &mut scope);
if let Err(CompileError{error: got_error, ..}) = bytecode {
assert_eq!(expected, got_error);
} else {
println!("instead got: {:?}", bytecode);
assert!(false);
}
}
#[test]
fn basic_expressions() {
let value = value!({
"null": null,
"zero": 0,
"one": 1,
"two": 2,
"two_f32": 2.25f32,
"two_f64": 2.55555555,
"three": 3,
"sub": {
"subnull": null,
"subzero": 0,
"subone": 1,
"subtwo": 2,
"subthree": 3,
},
"x": 1,
"y": 1,
});
let v = value.to_vec(false);
expression_tester(&v, r#"2.14"#, Ok(DecodedValue::Double(2.14)));
expression_tester(&v, r#"2.14f"#, Ok(DecodedValue::Float(2.14f32)));
expression_tester(&v, r#"-42"#, Ok(DecodedValue::Int(-42)));
expression_tester(&v, r#"-2.14"#, Ok(DecodedValue::Double(-2.14)));
//expression_tester(&v, br#"-one"#, Ok(Binary::Int(-1));
//expression_tester(&v, br#"-two_f32"#, Ok(Binary::Float(-2.25f32));
expression_tester(&v, r#"one + two"#, Ok(DecodedValue::Int(3)));
expression_tester(&v, r#"one - two"#, Ok(DecodedValue::Int(-1)));
expression_tester(&v, r#"two * two"#, Ok(DecodedValue::Int(4)));
expression_tester(&v, r#"three / two"#, Ok(DecodedValue::Int(1)));
expression_tester(&v, r#"three % two"#, Ok(DecodedValue::Int(1)));
//expression_tester(&v, r#"three ** two"#, Ok(Binary::Double(9.0)));
expression_tester(&v, r#""a" .. "b""#, Ok(DecodedValue::String(b"ab")));
expression_tester(&v, r#""a" .. /* "c" .. */ "b""#, Ok(DecodedValue::String(b"ab")));
expression_tester(&v, r#""a" .. "b" // .. "c""#, Ok(DecodedValue::String(b"ab")));
expression_tester(&v, r##" r#"foo"# "##, Ok(DecodedValue::String(b"foo")));
expression_tester(&v, r##" "\uD83D\uDE00" "##, Ok("😀".into()));
expression_tester(&v, r##" '\uD83D\uDE00' "##, Ok("😀".into()));
expression_tester(&v, r##" 'fo\'o' "##, Ok(b"fo'o".into()));
expression_tester(&v, r##" 'fo\'o'.upper() "##, Ok(DecodedValue::String(b"FO'O")));
expression_tester(&v, r##" 'fo\'o'.upper() "##, Ok(DecodedValue::String(b"FO'O")));
expression_tester(&v, r##" '<html>'.htmlescape() "##, Ok(DecodedValue::String(b"<html>")));
expression_tester(&v, r##" 'fo\'o'.sub(1, 2) "##, Ok(DecodedValue::String(b"o'")));
expression_tester(&v, r##" 'foobar'.sub(0, 1000) "##, Ok(DecodedValue::String(b"foobar")));
expression_tester(&v, r##" 'foobar'.sub(1001, 1000) "##, Ok(DecodedValue::String(b"")));
expression_tester(&v, r##" 'foobar'.sub(3, -1) "##, Ok(DecodedValue::String(b"")));
expression_tester(&v, r##" 'foobar'.sub(3) "##, Ok(DecodedValue::String(b"bar")));
// in mini mode, the code is not folded, so this error is not triggered
#[cfg(not(feature = "mini"))]
{
expression_tester_parser_error(r##"!1"##, CompileErrorDescription::Optimizer(EvalError::Dynamic("NOT expects a boolean")));
expression_tester_parser_error(r##"1f>>3"##, CompileErrorDescription::Optimizer(EvalError::Dynamic("Arithmetic operation without proper arguments (maybe a bitwise operation on a float or a comparison between a float and int?)")));
}
expression_tester(&v, r##"1=!!!!faalse"##, Err(EvalError::FieldNotFound("faalse")));
expression_tester(&v, r#"0???"#, Ok(DecodedValue::Int(0)));
expression_tester_parser_error(r#"--9223372036854775808"#, CompileErrorDescription::Parser("could not negate the int number"));
expression_tester(&v, r#"foo(1)"#, Err(EvalError::Dynamic("no custom functions defined")));
expression_tester(&v, r##" one < two "##, Ok(value!(true)));
expression_tester(&v, r##" two > one "##, Ok(value!(true)));
expression_tester(&v, r##" "foo".basename() "##, Ok(value!("foo")));
expression_tester(&v, r##" "foo".dirname() "##, Ok(value!("")));
expression_tester(&v, r##" "/foo".basename() "##, Ok(value!("foo")));
expression_tester(&v, r##" "/foo".dirname() "##, Ok(value!("")));
expression_tester(&v, r##" "x/foo".basename() "##, Ok(value!("foo")));
expression_tester(&v, r##" "x/foo".dirname() "##, Ok(value!("x")));
expression_tester(&v, r##" "x/y/foo".basename() "##, Ok(value!("foo")));
expression_tester(&v, r##" "x/y/foo".dirname() "##, Ok(value!("x/y")));
expression_tester(&v, r##" "x/y/foo".splitn(999, "/") "##, Ok(value!(["x", "y", "foo"])));
expression_tester(&v, r##" {...this, x:2}.x "##, Ok(value!(2)));
expression_tester(&v, r##" {x:2, ...this}.x "##, Ok(value!(1)));
expression_tester(&v, r##" {x:1,y:2}.tostring() "##, Ok(value!(r#"{"x":1,"y":2}"#)));
expression_tester(&v, r##" (42).tointeger() "##, Ok(value!(42)));
expression_tester(&v, r##" (42.5).tointeger() "##, Ok(value!(42)));
expression_tester(&v, r##" (42.5f).tointeger() "##, Ok(value!(42)));
expression_tester(&v, r##" (42).tofloat() "##, Ok(value!(42f32)));
expression_tester(&v, r##" (42.0).tofloat() "##, Ok(value!(42f32)));
expression_tester(&v, r##" (42.0f).tofloat() "##, Ok(value!(42f32)));
expression_tester(&v, r##" "42.5".tofloat() "##, Ok(value!(42.5f32)));
expression_tester(&v, r##" (42).todouble() "##, Ok(value!(42f64)));
expression_tester(&v, r##" (42.0).todouble() "##, Ok(value!(42f64)));
expression_tester(&v, r##" (42.0f).todouble() "##, Ok(value!(42f64)));
expression_tester(&v, r##" "42.5".todouble() "##, Ok(value!(42.5f64)));
expression_tester(&v, r##" {("foo" .. (2).tostring()): 42} "##, Ok(value!({"foo2":42})));
expression_tester(&v, r##" undefined(foobarfoobar)"##, Ok(value!(true)));
expression_tester(&v, r##" undefined(null)"##, Ok(value!(false)));
expression_tester(&v, r##" undefined(one)"##, Ok(value!(false)));
expression_tester(&v, r##" defined(foobarfoobar)"##, Ok(value!(false)));
expression_tester(&v, r##" defined(null)"##, Ok(value!(true)));
expression_tester(&v, r##" defined(one)"##, Ok(value!(true)));
expression_tester(&v, r##" ' foo '.trim() "##, Ok(value!("foo")));
expression_tester(&v, r##" ' foo '.trim() == 'foo' "##, Ok(value!(true)));
expression_tester(&v, r#"[]"#, Ok(value!([])));
expression_tester(&v, r#"[1]"#, Ok(value!([1])));
expression_tester(&v, r#"[1,]"#, Ok(value!([1,])));
expression_tester(&v, r#"[1,2]"#, Ok(value!([1,2])));
expression_tester(&v, r#"[1,2,]"#, Ok(value!([1,2,])));
expression_tester(&v, r#"{}"#, Ok(value!({})));
expression_tester(&v, r#"{a:1}"#, Ok(value!({"a":1})));
expression_tester(&v, r#"{a:1,}"#, Ok(value!({"a":1,})));
expression_tester(&v, r#"{a:1,b:2}"#, Ok(value!({"a":1,"b":2})));
expression_tester(&v, r#"{a:1,b:2,}"#, Ok(value!({"a":1,"b":2,})));
// based on fuzzing some weird test cases
expression_tester_parser_error(r#"0f.F**fa"#, CompileErrorDescription::Optimizer(EvalError::Dynamic("field access (fast path) for a left-hand-side that resolves to something different than an object")));
// triggers a stack overflow without the depth protection that is build in
expression_tester_parser_error(r#"[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[["#, CompileErrorDescription::Parser("too much parser recursion"));
expression_tester_parser_error(r#"!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!1"#, CompileErrorDescription::Parser("too much parser recursion"));
expression_tester_parser_error(r#"j>$this.t"#, CompileErrorDescription::Optimizer(EvalError::FieldNotFound("t")));
expression_tester_parser_error("m/d..d8666*k666*6*=6*=6*333*6*=666*6*=6=66*6*=666*6*=6=6*6*6$=666*6*=6*=6*6&&6*6*=666*6*=6=66*6*=666*6*=6=6*6*6$=666*6*=6*=6*6$=666*6*=6*=6*666*66=66**6*=6=6*6*66*6*=666*6*=6=6*6*6*=6*=66*=6*666*65=66**6*=6=6*6*6*=6=6*6*666*6*=6*=6*66*6*6*=6=6*6*6*=6*=66*=6*666*66=66**6*=6=6**6*6$=666*6*=6*=6*6$=666*6*=6*=6*666*66=66**6*=6=6*6*66*6$=666*6*=6*=6*666*66=66**6*=6*66=66**6*=6=6*6*66*6*=666*6*=6=6*6*6*=6*=66*=6**65=66**6*=6=6*6*6*=6=6*6*6*=6*=66*=6*666*66=66**6*=6=6**6*6$=666*6*6=6*6*6*=6*=66*=6*666*66=66**6*=6=6**6*6$=666*6*=6*=6*6$=666*6*=6*=6*6*6*6*=6*=5*=6*=6=6*6*=6*6*6$=666*6*=6*=6*6$=666*6*=6*=6*666*66=66**6*=6*66=66**6*=6=6*6*66*6*=666*6*=6=6*6*6*=6*=66*=6**65=66**6*=6=6*6*6*=6=6*6*6*=6*=66*=6*666*66=66**6*=6=6**6*6$=666*6*=6*=6*6$=666*6**6*66*6*=666*6*=6=6*6*6*=6*=66*=6**65=66**6*=6=6*6*6*=6=6*6*6*=6*=66*=6*666*66=66**6*=6=6**6*6$=666*6*6=6*6*6*=6*=66*=6*666*66=66**6*=6=6**6*6$=666*6*=6*=6*6$=666*6*=6*=6*6*6*6*=6*=5*=6*=6*=666*6*=6=6*6*6*=6*=66*=6*666*66=66**6*=6=6*6*66*6*=666*6*=6=66*6*=6=6*6*6*=6*=66*=6*666*65=66**6*=6=6*6*6*=6=6*6*666*6*=6*=6*66*6*6*=6=6*6*6*=6*=66*=6*666*65=66**6*=6=6*6*6*8888888=6=6*6*666*6*=6*=6*66*6*6*=6=6*6*6*=6*=66*=6*666*66=66**6*=6=6**0*6$=666*6*=6*=6*6$=666*6*=6*=6*666*66=66**6*=6=6*6*66*6*=666*6*=6=6*=6=6**6*6$=666*6*6=6*6*6*=6*=66*=6*666*66=66**6*=6=6**6*6$=666*6*=6*=6*6$=666*6*=6*=6*6*6*6*=6*=5*=6*=6=6*6*6*=6*=66*=6*666*66=66**6*=6=6*6*66*6*=666*6*=6=66*6*=6=6*6*6*=6*=66*=6*666*65=66**6*=6=6*6*6*=6=6*6*666*6*=6*=6*66*6*6*=6=6*6*6*=6*=66*=6*666*66=66**6*=6=6**6*6$=666*6*=6*=6*6$=666*6*=6*=6*6*6*6*=6*=5*=6*=6=6*6*=6*6*6$=666*6*=6*=6*6$=666*6*=6*=6*666*66=66**6*=6*66=66**6*=6=6666*6*=6*=6*66*68*6*=6=6*6*6*=6*=66*=6*666*66=66**6*=6=6**6*6$=666*6*=6*=6*6$=666*6*=6*=6*6*6*6*=6*=5*=6*=6=6*6*=6*6*6$=666*6*=6*=6*6$=666*6*=6*=86*666*66=66**6*=6*66=66**6*=6=6*6*66*6*=666*6*=6=6*6*6*=6*=66*=6**65=66**6*=6=6*6*6*=6=6*6*6*=6*=66*=6*666*66=66**6*=6=6**6*6$=666*6*6=6*6*6*=6*=66*=6*666*66=66**6*=6=6**6*6$=666*6*=6*=6*6$=666*6*=6*=6*6*6*6*=6*=5*=6*=6=6*6*=6*6*6$=666*6*=6*=6*6$=666*6*=6*=6*666*66=66**6*=6*66=66**6*=6=6*6*66*6*=666*6*=6=6*6*6*=6*=66*=6**65=66**6*=6=6*6*6*=6=6*6*6*=6*=66*=6*666*66=66**6*=6=6**6*6$=666*6*=6*=6*6$=666*6**6*66*6*=666*6*=6=6*6*6*=6*=66*=6**65=66**6*=6=6*6*6*=6=6*6*6*=6*=66*=6*666*66=66**6*=6=6**6*6$=666*6*6=6*6*666*6*=6*=6*6$=666*6*=6*=6*6*6*6*=6*=5*=6*=6*=666*6*=6=6*6*6*=6*=66*=6*666*66=66**6*=6=6*6*66*6*=666*6*=6=66*6*=6=6*6*6*=6*=66*=6*666*65=66**6*=6=6*6*6*=6=6*6*666*6*=6*=6*66*6*6*=6=6*6*6*=6*=66*=6*666*65=66**6*=6*=66*=6*666*66=66**6*=6=6**6*6$=666*6*=6*=6*6$=666*6*=6*=6*6*6*6*=6*=5*=6*=6=6*6*=6*6*6$=666*6*=6*=6*6$=666*6*=6*=6*666*66=66**6*=6*66=66**6*=6=6*6*66*6*=666*6*=6=6*6*6*=6*=66*=6**65=66**6*=6=6*6*6*=6=6*6*6*=6*=66*=6*666*66=66**6*=6=6**6*6$=666*6*=6*=6*6$=666*6*=6*=6*666*66=66**6*=6=6*6*66*6*=666*6*=6=6*6*6*=6*=66*=6*666*66=66**6*=6=6*6*66*6*=666*6*=6=6*6*6*=6*=5*=6*=6==66*=6*666*66=66**6*=6=6*6*66*6*=666*6*=6=6*6*6*=6*=6", CompileErrorDescription::Optimizer(EvalError::Expression("too complex AST to convert to bytecode")));
expression_tester_parser_error(r#"00404&04=664000<04&000<04&04=664004=04=66>004=00404&04=664000<04&000<04&04=664000=04=66>004=46>4=04=66>00004=04=4&000<04&46>4=04=46>004=4628=664004=04=66>004=46>4=04=664004=04==00404&04=664000<04&000<04&04=664000=04=66>004=46>4=04=66>00004=04=4&000<04&46>4=04=46>004=4628=664004=04=66>004=46>4=04=664004=04=66>004=00404&04=664000<04&000<04&04=664000=04=66>004=46>4=04=66>00004=04=4&000<04&46>4=04=46>004=4664=664004=04=66>004=46>4=04=646>4=04=66>004=46>4004=04=66>004=46>4=04=66>0044=646>4=04=66>004=46>4004=04=66>004=46>4=04=66>004=46>004=466*=t66>66>004=46>4=04=66>004=46>004=466*=t"#, CompileErrorDescription::Optimizer(EvalError::Expression("too complex AST to convert to bytecode")));
}
#[test]
fn merge() {
let s = String::from("foo");
let lhs_value = value!({
"null": null,
"zero": 0,
"one": 1,
"two_f32": 2.25f32,
"two_f64": 2.55555555,
"three": 3,
});
let lhs = lhs_value.to_vec(false);
let rhs_value = value!({
"zero": 1, // this is not zero, so we can test the merge function
"two": 2,
"foo": s,
});
let rhs = rhs_value.to_vec(false);
let result_value = value!({
"null": null,
"zero": 0,
"one": 1,
"two": 2,
"two_f32": 2.25f32,
"two_f64": 2.55555555,
"three": 3,
"foo": s,
});
let expected = result_value.to_vec(true);
let mut allocator = MemoryPool::new();
let mut scope = allocator.rewind();
let result = merge_objects(lhs.to_ref(), rhs.to_ref(), &mut scope).unwrap();
assert_eq!(expected.to_ref(), result);
}
#[test]
fn parsing_objects() {
let subthree_str = "subthree";
let sub = value!({
"subnull": null,
"subzero": 0,
"subone": 1,
"subtwo": 1+1,
subthree_str: 3,
});
let value = value!({
"null": null,
"zero": 0,
"one": 1,
"two": 2,
"three": 3,
"sub": sub,
});
let value_packed = value.to_vec(false);
println!("Using value {:?}", value);
println!("Packed value {:?}", value_packed);
//let test = br#"1+2+3"#;
//let test = br#"f(1*2+3*4,5,6,"foo", "bar")"#;
//let test = br#"f("foo",field) >= foo && true"#;
//let test = br#"a ? b : c ? d : e"#;
//let test = br#"{name: 1+2*3+4, field2: 42, "field3": 37}"#;
//let test = br#"a.b???c??d"#;
//let test = br#"{a:1f,foo???}"#;
// let test = br#"c + (b + a)"#;
// println!("a = {:?}", key("a"));
// println!("b = {:?}", key("b"));
// println!("c = {:?}", key("c"));
//let test = br#"10.5 % 2.25"#;
//let test = br#"10.5 / 0 ??? 42"#;
//let test = br#"zero + one + [1,2,3].len()"#;
//let test = br#"zero + sub.subone + sub.subtwo"#;
let test_expr = r#"zero + sub["subone"] + sub.subtwo"#;
//let test = br#"null ?? 42"#;
//let test = br#" "foo".len()"#;
//let test = br#"one == 0 ? "true" : "false""#;
//let test = br#"[1,2,3]"#;
//let test = br#"{"one":one,"two":two,"three":three}"#;
//let test = br#"{"one":1,"two":2,"three":3}"#;
let mut allocator = MemoryPool::new();
let mut scope2 = allocator.rewind();
{
let mut scope1 = scope2.rewind();
let mut ast = parse_expry_expr(test_expr, &value!({}), &mut scope1);
if let Ok(object) = &mut ast {
let mut scope = scope1.rewind();
println!("Parsed eval {:?}", object);
#[cfg(not(feature = "mini"))]
expry_ast_optimize(object).unwrap();
let bytecode = expry_ast_to_bytecode(&mut scope, object).expect("bytecode generator error");
println!("Bytecode {:?}", bytecode);
let mut bytecode_reader = RawReader::with(bytecode.get());
let mut value_reader = RawReader::with(value_packed.get());
let mut context = Context { original: value_packed.to_ref(), custom: &mut NoCustomFuncs{}, };
value_reader.read_var_u64().unwrap();
let result = evaluate_to_decoded_value(&mut bytecode_reader, &mut value_reader, &mut context, &mut scope).unwrap();
println!("Result {:?}", result);
assert_eq!(result, value!(3));
} else {
println!("Parse error {:?}", ast);
assert!(false);
}
// will trigger a compiler warning: because expry_parse_ast_optimize will modify the ast in
// place and that will result in a dependency on the lifetime of `scope` (which is not
// available here anymore).
// let mut scope3 = scope1.scope();
// expry_parse_ast_optimize(&mut object.unwrap(), &mut scope3).unwrap();
}
}
// helper data type to type check a binary expression
#[derive(PartialEq, Clone)]
pub enum TypeValue<'a> {
Null,
Bool,
Int,
Float,
Double,
String,
Object(TypeObject<'a>),
Array(TypeArray<'a>),
}
pub type TypeObject<'a> = BTreeMap<Key<'a>, TypeValue<'a>>;
pub type TypeArray<'a> = Vec<TypeValue<'a>>;
#[derive(Debug)]
enum TypeError<'a> {
Expression(&'a str),
Opcode,
Other(&'a str),
}
fn get_type<'b>(expression: &mut RawReader<'b>, types: &TypeValue<'b>) -> Result<TypeValue<'b>,TypeError<'b>>
{
let opcode = expression.read_u8().map_err(|_| TypeError::Expression("Expected more bytes in expression"))?;
// FIXME: find?
if opcode == ExpryBytecode::And as u8 {
let mut sub = RawReader::with(expression.read_var_string().map_err(|_| TypeError::Expression("corrupted AND"))?);
while !sub.is_empty() {
let clause = get_type(&mut sub, types)?;
if !matches!(clause, TypeValue::Bool) {
return Err(TypeError::Other("value for AND was not a boolean"));
}
}
return Ok(TypeValue::Bool);
}
if opcode == ExpryBytecode::Or as u8 {
let mut sub = RawReader::with(expression.read_var_string().map_err(|_| TypeError::Expression("corrupted OR"))?);
while !sub.is_empty() {
let clause = get_type(&mut sub, types)?;
if !matches!(clause, TypeValue::Bool) {
return Err(TypeError::Other("value for OR was not a boolean"));
}
}
return Ok(TypeValue::Bool);
}
/*
if opcode == ExpryBytecode::Call as u8 {
let function_name = expression.read_var_string().map_err(|_| TypeError::Expression("invalid CALL"))?;
let arg_count = expression.read_var_u64().map_err(|_| TypeError::Expression("invalid CALL"))?;
let mut args : Vec<TypeValue> = Vec::new();
for _ in 0..arg_count {
let v = get_type(expression, types)?;
args.push(v);
}
return context.custom.call(core::str::from_utf8(function_name).map_err(|_| TypeError::Expression("function name is not UTF8"))?, &args).map_err(TypeError::Expression);
}
*/
if opcode == ExpryBytecode::All as u8 {
return Ok(types.clone());
}
if opcode == ExpryBytecode::Field as u8 ||
opcode == ExpryBytecode::FindAndField as u8 {
let key = read_hashed_string(expression).map_err(|_| TypeError::Expression("Expected more bytes in expression"))?;
if let TypeValue::Object(object) = types {
if let Some(subtypes) = object.get(&key) {
return Ok(subtypes.clone());
}
}
return Err(TypeError::Other("current value should be an object"));
}
/*
// FIXME: implement fast path for FieldAccessFastPath. This op can be implemented faster because
// the value does not need to be decoded at all for the field access to work.
if opcode == ExpryBytecode::FieldAccessCompute as u8
|| opcode == ExpryBytecode::FieldAccessFastPath as u8 {
let lhs = evaluate_to_object(expression, value, context, allocator)?;
if let DecodedValue::Object(mut obj) = lhs {
let rhs = evaluate_to_object(expression, value, context, allocator)?;
if let DecodedValue::String(key) = rhs {
if let Some(value) = obj.get_mut(&key_u8(key)) {
let value = core::mem::replace(value, DecodedValue::Null());
return Ok(value);
}
return Err(TypeError::FieldNotFound(core::str::from_utf8(key).map_err(|_| TypeError::Dynamic("invalid UTF8 for key"))?));
} else {
return Err(TypeError::Dynamic("field access for a right-hand-side that resolves to something different than a string"));
}
} else {
return Err(TypeError::Dynamic("field access for a left-hand-side that resolves to something different than an object"));
}
}
*/
if opcode == ExpryBytecode::Concrete as u8 {
let value = DecodedValue::parse(expression).map_err(|_| {
TypeError::Expression("Corrupted concrete value")
})?;
return Ok(to_type(value));
}
if opcode == ExpryBytecode::ArithmeticPlus as u8 {
let clause = get_type(expression, types)?;
if !matches!(clause, TypeValue::Int) {
return Err(TypeError::Other("value for + should be an int"));
}
let clause = get_type(expression, types)?;
if !matches!(clause, TypeValue::Int) {
return Err(TypeError::Other("value for + should be an int"));
}
return Ok(TypeValue::Int);
}
/*
if opcode == ExpryBytecode::ArithmeticPlus as u8 ||
opcode == ExpryBytecode::ArithmeticMin as u8 ||
opcode == ExpryBytecode::ArithmeticMul as u8 ||
opcode == ExpryBytecode::ArithmeticDiv as u8 ||
opcode == ExpryBytecode::ArithmeticMod as u8 ||
opcode == ExpryBytecode::ArithmeticPower as u8 ||
opcode == ExpryBytecode::ArithmeticShiftLeft as u8 ||
opcode == ExpryBytecode::ArithmeticShiftRight as u8 ||
opcode == ExpryBytecode::ArithmeticBitwiseOr as u8 ||
opcode == ExpryBytecode::ArithmeticBitwiseXor as u8 ||
opcode == ExpryBytecode::ArithmeticBitwiseAnd as u8
{
let lhs = evaluate_to_object(expression, value, context, allocator)?;
let rhs = evaluate_to_object(expression, value, context, allocator)?;
// disabled for smaller binary size
// if opcode == ExpryBytecode::ArithmeticPower as u8 {
// let l = get_double(&lhs)?;
// let r = get_double(&rhs)?;
// return Ok(Binary::Double(l.powf(r)));
// }
if let (DecodedValue::Int(l), DecodedValue::Int(r)) = (&lhs, &rhs) {
if opcode == ExpryBytecode::ArithmeticPlus as u8 {
return Ok(DecodedValue::Int(
l.checked_add(*r).ok_or(TypeError::Dynamic("int add resulted in overflow"))?
));
}
if opcode == ExpryBytecode::ArithmeticMin as u8 {
return Ok(DecodedValue::Int(
l.checked_sub(*r).ok_or(TypeError::Dynamic("int sub resulted in overflow"))?
));
}
if opcode == ExpryBytecode::ArithmeticMul as u8 {
return Ok(DecodedValue::Int(
l.checked_mul(*r).ok_or(TypeError::Dynamic("int mul resulted in overflow"))?
));
}
if opcode == ExpryBytecode::ArithmeticDiv as u8 {
if *r == 0 {
return Err(TypeError::Dynamic("Division by zero"));
}
return Ok(DecodedValue::Int(
l.checked_div(*r).ok_or(TypeError::Dynamic("int div resulted in overflow"))?
));
}
if opcode == ExpryBytecode::ArithmeticMod as u8 {
if *r == 0 {
return Err(TypeError::Dynamic("Modulo zero"));
}
return Ok(DecodedValue::Int(l % r));
}
if opcode == ExpryBytecode::ArithmeticShiftLeft as u8 {
return Ok(DecodedValue::Int(
l.checked_shl((*r).try_into().map_err(|_| TypeError::Dynamic("shift-left resulted in overflow"))?).ok_or(TypeError::Dynamic("shift-left resulted in overflow"))?
));
}
if opcode == ExpryBytecode::ArithmeticShiftRight as u8 {
return Ok(DecodedValue::Int(
l.checked_shr((*r).try_into().map_err(|_| TypeError::Dynamic("shift-right resulted in overflow"))?).ok_or(TypeError::Dynamic("shift-right resulted in overflow"))?
));
}
if opcode == ExpryBytecode::ArithmeticBitwiseOr as u8 {
return Ok(DecodedValue::Int(l | r));
}
if opcode == ExpryBytecode::ArithmeticBitwiseXor as u8 {
return Ok(DecodedValue::Int(l ^ r));
}
if opcode == ExpryBytecode::ArithmeticBitwiseAnd as u8 {
return Ok(DecodedValue::Int(l & r));
}
}
let l = get_double(&lhs)?;
let r = get_double(&rhs)?;
if opcode == ExpryBytecode::ArithmeticPlus as u8 {
return Ok(DecodedValue::Double(l + r));
}
if opcode == ExpryBytecode::ArithmeticMin as u8 {
return Ok(DecodedValue::Double(l - r));
}
if opcode == ExpryBytecode::ArithmeticMul as u8 {
return Ok(DecodedValue::Double(l * r));
}
if opcode == ExpryBytecode::ArithmeticDiv as u8 {
if r == 0.0 {
return Err(TypeError::Dynamic("Division by zero"));
}
return Ok(DecodedValue::Double(l / r));
}
// disabled for smaller binary size
// if opcode == ExpryBytecode::ArithmeticMod as u8 {
// if r == 0.0 {
// return Err(ExpryTypeError::Dynamic("Modulo zero"));
// }
// return Ok(Binary::Double(l % r));
// }
return Err(TypeError::Dynamic("Arithmetic operation without proper arguments (maybe a bitwise operation on a float or a comparison between a float and int?)"));
}
*/
Err(TypeError::Opcode)
}
fn to_type(value: DecodedValue) -> TypeValue {
match value {
DecodedValue::Null() => TypeValue::Null,
DecodedValue::Bool(_) => TypeValue::Bool,
DecodedValue::Int(_) => TypeValue::Int,
DecodedValue::Float(_) => TypeValue::Float,
DecodedValue::Double(_) => TypeValue::Double,
DecodedValue::String(_) => TypeValue::String,
DecodedValue::Object(obj) => {
let mut retval = TypeObject::new();
for (k,v) in obj {
retval.insert(k, to_type(v));
}
TypeValue::Object(retval)
},
DecodedValue::Array(arr) => {
let mut retval = TypeArray::new();
for v in arr {
retval.push(to_type(v));
}
TypeValue::Array(retval)
},
}
}
/*
* value!() knows all the types, even if something is None the full type should be
* available (CHECK THIS). If this is the case, value!() can return a tuple, one with the
* concrete value, and one with the type info. This second tuple can then be used for
* checking types compile time on invocation sites. These needs a macro, e.g.
* expry_static_eval!("a+b", val); with let val = value!(..);
* This only works with static available expressions.
*
* The same can be done for hair templates if they are known statically.
* The type checker for hair templates can work with get_type(), because types
* are derived from binary expressions. And then assigned to loop variables and such.
*
* We can also have a function to dynamically type check an expression. But I don't know
* how useful that will be. It gives a bit more certainty, but is there a big advantage
* besides just running it the template/expry? There is a bit of a mismatch between one
* type instance or all possible types. But I can not imagine when to use that.
*/
#[test]
fn type_check() {
let expr = "a+b";
let mut allocator = MemoryPool::new();
let mut scope = allocator.rewind();
let bytecode = expry_compile(expr, None, &mut scope);
if let Ok(bytecode) = bytecode {
//let result = expry_eval(bytecode, v.to_ref(), &mut scope);
let mut types = TypeObject::new();
types.insert(key_str("a"), TypeValue::Int);
types.insert(key_str("b"), TypeValue::Int);
get_type(&mut RawReader::with(bytecode.get()), &TypeValue::Object(types)).unwrap();
}
}
}