use std::sync::Arc;
use datafusion::arrow::array::{Array, ArrayRef, AsArray, BooleanArray, Float64Array, NullArray, StringArray};
use datafusion::arrow::datatypes::{
DataType, Float16Type, Float32Type, Float64Type, Int16Type, Int32Type, Int64Type, Int8Type, UInt16Type, UInt32Type,
UInt64Type, UInt8Type,
};
use datafusion::error::{DataFusionError, Result};
use fv_value::Value;
pub fn has_value_form(t: &DataType) -> bool {
match t {
DataType::Null
| DataType::Boolean
| DataType::Int8
| DataType::Int16
| DataType::Int32
| DataType::Int64
| DataType::UInt8
| DataType::UInt16
| DataType::UInt32
| DataType::UInt64
| DataType::Float16
| DataType::Float32
| DataType::Float64
| DataType::Utf8
| DataType::LargeUtf8
| DataType::Utf8View => true,
DataType::Dictionary(_, values) => has_value_form(values),
DataType::List(f) | DataType::LargeList(f) | DataType::FixedSizeList(f, _) => has_value_form(f.data_type()),
DataType::Struct(fields) => fields.iter().all(|f| has_value_form(f.data_type())),
_ => false,
}
}
pub fn cell_to_value(array: &ArrayRef, i: usize) -> Result<Value> {
Ok(Cells::of(array)?.value(i))
}
#[derive(Clone, Copy, Debug)]
pub struct Cells<'a>(&'a ArrayRef);
impl<'a> Cells<'a> {
pub fn of(array: &'a ArrayRef) -> Result<Self> {
if has_value_form(array.data_type()) {
Ok(Cells(array))
} else {
Err(DataFusionError::Execution(format!(
"fv-value: a {} value has no form in the dialect",
array.data_type()
)))
}
}
#[inline]
pub fn value(&self, i: usize) -> Value {
value_at(self.0, i)
}
}
pub(crate) fn value_at(array: &ArrayRef, i: usize) -> Value {
if array.is_null(i) {
return Value::Null;
}
match array.data_type() {
DataType::Null => Value::Null,
DataType::Boolean => Value::Bool(array.as_boolean().value(i)),
DataType::Float64 => Value::Num(array.as_primitive::<Float64Type>().value(i)),
DataType::Float32 => Value::Num(f64::from(array.as_primitive::<Float32Type>().value(i))),
DataType::Float16 => Value::Num(f64::from(array.as_primitive::<Float16Type>().value(i))),
DataType::Int64 => Value::Num(array.as_primitive::<Int64Type>().value(i) as f64),
DataType::Int32 => Value::Num(f64::from(array.as_primitive::<Int32Type>().value(i))),
DataType::Int16 => Value::Num(f64::from(array.as_primitive::<Int16Type>().value(i))),
DataType::Int8 => Value::Num(f64::from(array.as_primitive::<Int8Type>().value(i))),
DataType::UInt64 => Value::Num(array.as_primitive::<UInt64Type>().value(i) as f64),
DataType::UInt32 => Value::Num(f64::from(array.as_primitive::<UInt32Type>().value(i))),
DataType::UInt16 => Value::Num(f64::from(array.as_primitive::<UInt16Type>().value(i))),
DataType::UInt8 => Value::Num(f64::from(array.as_primitive::<UInt8Type>().value(i))),
DataType::Utf8 => Value::Str(array.as_string::<i32>().value(i).to_string()),
DataType::LargeUtf8 => Value::Str(array.as_string::<i64>().value(i).to_string()),
DataType::Utf8View => Value::Str(array.as_string_view().value(i).to_string()),
DataType::Dictionary(_, _) => {
let dict = array.as_any_dictionary();
value_at(dict.values(), dict.normalized_keys()[i])
}
DataType::List(_) => list(&array.as_list::<i32>().value(i)),
DataType::LargeList(_) => list(&array.as_list::<i64>().value(i)),
DataType::FixedSizeList(_, _) => list(&array.as_fixed_size_list().value(i)),
DataType::Struct(fields) => {
let s = array.as_struct();
Value::Obj(
fields
.iter()
.zip(s.columns())
.map(|(f, c)| (f.name().clone(), value_at(c, i)))
.collect(),
)
}
_ => Value::Null,
}
}
fn list(items: &ArrayRef) -> Value {
Value::List((0..items.len()).map(|j| value_at(items, j)).collect())
}
pub fn data_type_for(kind: Kind) -> DataType {
match kind {
Kind::Num => DataType::Float64,
Kind::Str => DataType::Utf8,
Kind::Bool => DataType::Boolean,
Kind::Null => DataType::Null,
}
}
#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
pub enum Kind {
Num,
Str,
Bool,
Null,
}
impl Kind {
pub fn of_data_type(t: &DataType) -> Option<Kind> {
match t {
DataType::Null => Some(Kind::Null),
DataType::Boolean => Some(Kind::Bool),
DataType::Int8
| DataType::Int16
| DataType::Int32
| DataType::Int64
| DataType::UInt8
| DataType::UInt16
| DataType::UInt32
| DataType::UInt64
| DataType::Float32
| DataType::Float64 => Some(Kind::Num),
DataType::Utf8 | DataType::LargeUtf8 | DataType::Utf8View => Some(Kind::Str),
DataType::Dictionary(_, values) => Kind::of_data_type(values),
_ => None,
}
}
}
pub fn values_to_array(values: &[Value], data_type: &DataType) -> Result<ArrayRef> {
let wrong = |v: &Value| {
DataFusionError::Execution(format!(
"fv-value: expression produced a {} where the plan declared {data_type}",
v.type_name()
))
};
Ok(match data_type {
DataType::Float64 => Arc::new(
values
.iter()
.map(|v| match v {
Value::Null => Ok(None),
Value::Num(n) => Ok(Some(*n)),
other => Err(wrong(other)),
})
.collect::<Result<Float64Array>>()?,
),
DataType::Utf8 => Arc::new(
values
.iter()
.map(|v| match v {
Value::Null => Ok(None),
Value::Str(s) => Ok(Some(s.clone())),
other => Err(wrong(other)),
})
.collect::<Result<StringArray>>()?,
),
DataType::Boolean => Arc::new(
values
.iter()
.map(|v| match v {
Value::Null => Ok(None),
Value::Bool(b) => Ok(Some(*b)),
other => Err(wrong(other)),
})
.collect::<Result<BooleanArray>>()?,
),
DataType::Null => Arc::new(NullArray::new(values.len())),
other => {
return Err(DataFusionError::NotImplemented(format!(
"fv-value: cannot materialise dialect values as {other}"
)))
}
})
}
#[cfg(test)]
mod tests {
use super::*;
use datafusion::arrow::array::{Int32Array, ListBuilder, StringViewArray};
#[test]
fn cells_convert_by_type() {
let ints: ArrayRef = Arc::new(Int32Array::from(vec![Some(7), None]));
assert_eq!(cell_to_value(&ints, 0).unwrap(), Value::Num(7.0));
assert_eq!(cell_to_value(&ints, 1).unwrap(), Value::Null);
let views: ArrayRef = Arc::new(StringViewArray::from(vec!["x"]));
assert_eq!(cell_to_value(&views, 0).unwrap(), Value::from("x"));
let mut lb = ListBuilder::new(Float64Array::builder(2));
lb.values().append_value(1.0);
lb.values().append_value(2.0);
lb.append(true);
let list: ArrayRef = Arc::new(lb.finish());
assert_eq!(cell_to_value(&list, 0).unwrap(), Value::from(vec![1.0, 2.0]));
}
#[test]
fn cells_check_the_type_once_and_refuse_one_with_no_dialect_value() {
use datafusion::arrow::array::BinaryArray;
let words: ArrayRef = Arc::new(StringViewArray::from(vec!["a", "b"]));
let cells = Cells::of(&words).unwrap();
assert_eq!((cells.value(0), cells.value(1)), (Value::from("a"), Value::from("b")));
let bytes: ArrayRef = Arc::new(BinaryArray::from(vec![b"x".as_ref()]));
let err = Cells::of(&bytes).unwrap_err().to_string();
assert!(
err.contains("Binary") && err.contains("no form in the dialect"),
"{err}"
);
assert!(cell_to_value(&bytes, 0).is_err());
assert!(!has_value_form(&DataType::Date32) && has_value_form(&DataType::Float16));
}
#[test]
fn kinds_and_materialisation() {
assert_eq!(Kind::of_data_type(&DataType::Int64), Some(Kind::Num));
assert_eq!(Kind::of_data_type(&DataType::Utf8View), Some(Kind::Str));
assert_eq!(Kind::of_data_type(&DataType::Date32), None);
let a = values_to_array(&[Value::Num(1.5), Value::Null], &DataType::Float64).unwrap();
assert_eq!(a.len(), 2);
assert!(a.is_null(1));
assert!(values_to_array(&[Value::from("s")], &DataType::Float64).is_err());
assert_eq!(data_type_for(Kind::Bool), DataType::Boolean);
}
}