fv-value-datafusion 0.2.0

The fv-value expression dialect as DataFusion expressions: exact translation for pushdown and vectorized evaluation, with a per-row UDF fallback.
Documentation
//! Arrow ↔ [`Value`] conversion for the per-row fallback.

use std::sync::Arc;

use datafusion::arrow::array::{Array, ArrayRef, AsArray, BooleanArray, Float64Array, NullArray, StringArray};
use datafusion::arrow::datatypes::{
    DataType, Float16Type, Float32Type, Float64Type, Int16Type, Int32Type, Int64Type, Int8Type, UInt16Type, UInt32Type,
    UInt64Type, UInt8Type,
};
use datafusion::error::{DataFusionError, Result};
use fv_value::Value;

/// Whether a value of Arrow type `t` has an exact form in the dialect: every number, string and
/// boolean type (a dictionary of them reads as its values), lists of such values, and structs of
/// them (as objects). Anything else — binary, decimal, temporal, map — has none, and is refused
/// by name rather than read as a value it is not.
pub fn has_value_form(t: &DataType) -> bool {
    match t {
        DataType::Null
        | DataType::Boolean
        | DataType::Int8
        | DataType::Int16
        | DataType::Int32
        | DataType::Int64
        | DataType::UInt8
        | DataType::UInt16
        | DataType::UInt32
        | DataType::UInt64
        | DataType::Float16
        | DataType::Float32
        | DataType::Float64
        | DataType::Utf8
        | DataType::LargeUtf8
        | DataType::Utf8View => true,
        DataType::Dictionary(_, values) => has_value_form(values),
        DataType::List(f) | DataType::LargeList(f) | DataType::FixedSizeList(f, _) => has_value_form(f.data_type()),
        DataType::Struct(fields) => fields.iter().all(|f| has_value_form(f.data_type())),
        _ => false,
    }
}

/// One cell of an Arrow array as a dialect [`Value`]: every numeric width becomes `Num`, every
/// string flavour `Str`, booleans `Bool`, a dictionary its value, lists `List` and structs `Obj`
/// (recursively). A type with no form in the dialect ([`has_value_form`]) is an error, never a
/// value it is not.
pub fn cell_to_value(array: &ArrayRef, i: usize) -> Result<Value> {
    Ok(Cells::of(array)?.value(i))
}

/// A column read as dialect values, its type checked once: each cell then reads with no error
/// path. What a caller converting whole batches uses (`cell_to_value` checks the type per cell).
#[derive(Clone, Copy, Debug)]
pub struct Cells<'a>(&'a ArrayRef);

impl<'a> Cells<'a> {
    /// The column, if its type has a value in the dialect ([`has_value_form`]).
    pub fn of(array: &'a ArrayRef) -> Result<Self> {
        if has_value_form(array.data_type()) {
            Ok(Cells(array))
        } else {
            Err(DataFusionError::Execution(format!(
                "fv-value: a {} value has no form in the dialect",
                array.data_type()
            )))
        }
    }

    /// Row `i` as a dialect value.
    #[inline]
    pub fn value(&self, i: usize) -> Value {
        value_at(self.0, i)
    }
}

/// [`cell_to_value`] for an array whose type [`has_value_form`] — checked once per column, so
/// reading each cell carries no error path.
pub(crate) fn value_at(array: &ArrayRef, i: usize) -> Value {
    if array.is_null(i) {
        return Value::Null;
    }
    match array.data_type() {
        DataType::Null => Value::Null,
        DataType::Boolean => Value::Bool(array.as_boolean().value(i)),
        DataType::Float64 => Value::Num(array.as_primitive::<Float64Type>().value(i)),
        DataType::Float32 => Value::Num(f64::from(array.as_primitive::<Float32Type>().value(i))),
        DataType::Float16 => Value::Num(f64::from(array.as_primitive::<Float16Type>().value(i))),
        DataType::Int64 => Value::Num(array.as_primitive::<Int64Type>().value(i) as f64),
        DataType::Int32 => Value::Num(f64::from(array.as_primitive::<Int32Type>().value(i))),
        DataType::Int16 => Value::Num(f64::from(array.as_primitive::<Int16Type>().value(i))),
        DataType::Int8 => Value::Num(f64::from(array.as_primitive::<Int8Type>().value(i))),
        DataType::UInt64 => Value::Num(array.as_primitive::<UInt64Type>().value(i) as f64),
        DataType::UInt32 => Value::Num(f64::from(array.as_primitive::<UInt32Type>().value(i))),
        DataType::UInt16 => Value::Num(f64::from(array.as_primitive::<UInt16Type>().value(i))),
        DataType::UInt8 => Value::Num(f64::from(array.as_primitive::<UInt8Type>().value(i))),
        DataType::Utf8 => Value::Str(array.as_string::<i32>().value(i).to_string()),
        DataType::LargeUtf8 => Value::Str(array.as_string::<i64>().value(i).to_string()),
        DataType::Utf8View => Value::Str(array.as_string_view().value(i).to_string()),
        DataType::Dictionary(_, _) => {
            let dict = array.as_any_dictionary();
            value_at(dict.values(), dict.normalized_keys()[i])
        }
        DataType::List(_) => list(&array.as_list::<i32>().value(i)),
        DataType::LargeList(_) => list(&array.as_list::<i64>().value(i)),
        DataType::FixedSizeList(_, _) => list(&array.as_fixed_size_list().value(i)),
        DataType::Struct(fields) => {
            let s = array.as_struct();
            Value::Obj(
                fields
                    .iter()
                    .zip(s.columns())
                    .map(|(f, c)| (f.name().clone(), value_at(c, i)))
                    .collect(),
            )
        }
        // unreachable once the type is checked: `has_value_form` admits exactly the arms above
        _ => Value::Null,
    }
}

fn list(items: &ArrayRef) -> Value {
    Value::List((0..items.len()).map(|j| value_at(items, j)).collect())
}

/// The Arrow type a dialect result kind is materialised as.
pub fn data_type_for(kind: Kind) -> DataType {
    match kind {
        Kind::Num => DataType::Float64,
        Kind::Str => DataType::Utf8,
        Kind::Bool => DataType::Boolean,
        Kind::Null => DataType::Null,
    }
}

/// The static kind of a dialect expression's result, when it can be known before evaluation.
#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
pub enum Kind {
    /// A number (`Float64`).
    Num,
    /// A string (`Utf8`).
    Str,
    /// A boolean.
    Bool,
    /// Always null.
    Null,
}

impl Kind {
    /// The kind a column of this Arrow type reads as, or `None` if the translator does not handle
    /// it (lists, structs, temporal types, …).
    pub fn of_data_type(t: &DataType) -> Option<Kind> {
        match t {
            DataType::Null => Some(Kind::Null),
            DataType::Boolean => Some(Kind::Bool),
            DataType::Int8
            | DataType::Int16
            | DataType::Int32
            | DataType::Int64
            | DataType::UInt8
            | DataType::UInt16
            | DataType::UInt32
            | DataType::UInt64
            | DataType::Float32
            | DataType::Float64 => Some(Kind::Num),
            DataType::Utf8 | DataType::LargeUtf8 | DataType::Utf8View => Some(Kind::Str),
            DataType::Dictionary(_, values) => Kind::of_data_type(values),
            _ => None,
        }
    }
}

/// Materialise per-row results as an Arrow array of `data_type`. A value of the wrong kind for the
/// declared type is an execution error (the dialect returned something the plan did not declare).
pub fn values_to_array(values: &[Value], data_type: &DataType) -> Result<ArrayRef> {
    let wrong = |v: &Value| {
        DataFusionError::Execution(format!(
            "fv-value: expression produced a {} where the plan declared {data_type}",
            v.type_name()
        ))
    };
    Ok(match data_type {
        DataType::Float64 => Arc::new(
            values
                .iter()
                .map(|v| match v {
                    Value::Null => Ok(None),
                    Value::Num(n) => Ok(Some(*n)),
                    other => Err(wrong(other)),
                })
                .collect::<Result<Float64Array>>()?,
        ),
        DataType::Utf8 => Arc::new(
            values
                .iter()
                .map(|v| match v {
                    Value::Null => Ok(None),
                    Value::Str(s) => Ok(Some(s.clone())),
                    other => Err(wrong(other)),
                })
                .collect::<Result<StringArray>>()?,
        ),
        DataType::Boolean => Arc::new(
            values
                .iter()
                .map(|v| match v {
                    Value::Null => Ok(None),
                    Value::Bool(b) => Ok(Some(*b)),
                    other => Err(wrong(other)),
                })
                .collect::<Result<BooleanArray>>()?,
        ),
        DataType::Null => Arc::new(NullArray::new(values.len())),
        other => {
            return Err(DataFusionError::NotImplemented(format!(
                "fv-value: cannot materialise dialect values as {other}"
            )))
        }
    })
}

#[cfg(test)]
mod tests {
    use super::*;
    use datafusion::arrow::array::{Int32Array, ListBuilder, StringViewArray};

    #[test]
    fn cells_convert_by_type() {
        let ints: ArrayRef = Arc::new(Int32Array::from(vec![Some(7), None]));
        assert_eq!(cell_to_value(&ints, 0).unwrap(), Value::Num(7.0));
        assert_eq!(cell_to_value(&ints, 1).unwrap(), Value::Null);
        let views: ArrayRef = Arc::new(StringViewArray::from(vec!["x"]));
        assert_eq!(cell_to_value(&views, 0).unwrap(), Value::from("x"));
        let mut lb = ListBuilder::new(Float64Array::builder(2));
        lb.values().append_value(1.0);
        lb.values().append_value(2.0);
        lb.append(true);
        let list: ArrayRef = Arc::new(lb.finish());
        assert_eq!(cell_to_value(&list, 0).unwrap(), Value::from(vec![1.0, 2.0]));
    }

    #[test]
    fn cells_check_the_type_once_and_refuse_one_with_no_dialect_value() {
        use datafusion::arrow::array::BinaryArray;
        let words: ArrayRef = Arc::new(StringViewArray::from(vec!["a", "b"]));
        let cells = Cells::of(&words).unwrap();
        assert_eq!((cells.value(0), cells.value(1)), (Value::from("a"), Value::from("b")));
        let bytes: ArrayRef = Arc::new(BinaryArray::from(vec![b"x".as_ref()]));
        let err = Cells::of(&bytes).unwrap_err().to_string();
        assert!(
            err.contains("Binary") && err.contains("no form in the dialect"),
            "{err}"
        );
        assert!(cell_to_value(&bytes, 0).is_err());
        assert!(!has_value_form(&DataType::Date32) && has_value_form(&DataType::Float16));
    }

    #[test]
    fn kinds_and_materialisation() {
        assert_eq!(Kind::of_data_type(&DataType::Int64), Some(Kind::Num));
        assert_eq!(Kind::of_data_type(&DataType::Utf8View), Some(Kind::Str));
        assert_eq!(Kind::of_data_type(&DataType::Date32), None);
        let a = values_to_array(&[Value::Num(1.5), Value::Null], &DataType::Float64).unwrap();
        assert_eq!(a.len(), 2);
        assert!(a.is_null(1));
        assert!(values_to_array(&[Value::from("s")], &DataType::Float64).is_err());
        assert_eq!(data_type_for(Kind::Bool), DataType::Boolean);
    }
}