fv-value-datafusion 0.1.1

The fv-value expression dialect as DataFusion expressions: exact translation for pushdown and vectorized evaluation, with a per-row UDF fallback.
Documentation
//! Arrow ↔ [`Value`] conversion for the per-row fallback.

use std::sync::Arc;

use datafusion::arrow::array::{
    Array, ArrayRef, AsArray, BooleanArray, Float64Array, ListArray, NullArray, StringArray,
};
use datafusion::arrow::datatypes::{
    DataType, Float32Type, Float64Type, Int16Type, Int32Type, Int64Type, Int8Type, UInt16Type, UInt32Type, UInt64Type,
    UInt8Type,
};
use datafusion::error::{DataFusionError, Result};
use fv_value::Value;

/// One cell of an Arrow array as a dialect [`Value`]. Every numeric width becomes `Num`, every
/// string flavour `Str`, booleans `Bool`, lists `List` (recursively), nulls and anything else `Null`.
pub fn cell_to_value(array: &ArrayRef, i: usize) -> Value {
    if array.is_null(i) {
        return Value::Null;
    }
    match array.data_type() {
        DataType::Null => Value::Null,
        DataType::Boolean => Value::Bool(array.as_boolean().value(i)),
        DataType::Float64 => Value::Num(array.as_primitive::<Float64Type>().value(i)),
        DataType::Float32 => Value::Num(f64::from(array.as_primitive::<Float32Type>().value(i))),
        DataType::Int64 => Value::Num(array.as_primitive::<Int64Type>().value(i) as f64),
        DataType::Int32 => Value::Num(f64::from(array.as_primitive::<Int32Type>().value(i))),
        DataType::Int16 => Value::Num(f64::from(array.as_primitive::<Int16Type>().value(i))),
        DataType::Int8 => Value::Num(f64::from(array.as_primitive::<Int8Type>().value(i))),
        DataType::UInt64 => Value::Num(array.as_primitive::<UInt64Type>().value(i) as f64),
        DataType::UInt32 => Value::Num(f64::from(array.as_primitive::<UInt32Type>().value(i))),
        DataType::UInt16 => Value::Num(f64::from(array.as_primitive::<UInt16Type>().value(i))),
        DataType::UInt8 => Value::Num(f64::from(array.as_primitive::<UInt8Type>().value(i))),
        DataType::Utf8 => Value::Str(array.as_string::<i32>().value(i).to_string()),
        DataType::LargeUtf8 => Value::Str(array.as_string::<i64>().value(i).to_string()),
        DataType::Utf8View => Value::Str(array.as_string_view().value(i).to_string()),
        DataType::List(_) => {
            let inner = array.as_list::<i32>().value(i);
            Value::List((0..inner.len()).map(|j| cell_to_value(&inner, j)).collect())
        }
        _ => Value::Null,
    }
}

/// The Arrow type a dialect result kind is materialised as.
pub fn data_type_for(kind: Kind) -> DataType {
    match kind {
        Kind::Num => DataType::Float64,
        Kind::Str => DataType::Utf8,
        Kind::Bool => DataType::Boolean,
        Kind::Null => DataType::Null,
    }
}

/// The static kind of a dialect expression's result, when it can be known before evaluation.
#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
pub enum Kind {
    /// A number (`Float64`).
    Num,
    /// A string (`Utf8`).
    Str,
    /// A boolean.
    Bool,
    /// Always null.
    Null,
}

impl Kind {
    /// The kind a column of this Arrow type reads as, or `None` if the translator does not handle
    /// it (lists, structs, temporal types, …).
    pub fn of_data_type(t: &DataType) -> Option<Kind> {
        match t {
            DataType::Null => Some(Kind::Null),
            DataType::Boolean => Some(Kind::Bool),
            DataType::Int8
            | DataType::Int16
            | DataType::Int32
            | DataType::Int64
            | DataType::UInt8
            | DataType::UInt16
            | DataType::UInt32
            | DataType::UInt64
            | DataType::Float32
            | DataType::Float64 => Some(Kind::Num),
            DataType::Utf8 | DataType::LargeUtf8 | DataType::Utf8View => Some(Kind::Str),
            _ => None,
        }
    }
}

/// Materialise per-row results as an Arrow array of `data_type`. A value of the wrong kind for the
/// declared type is an execution error (the dialect returned something the plan did not declare).
pub fn values_to_array(values: &[Value], data_type: &DataType) -> Result<ArrayRef> {
    let wrong = |v: &Value| {
        DataFusionError::Execution(format!(
            "fv-value: expression produced a {} where the plan declared {data_type}",
            v.type_name()
        ))
    };
    Ok(match data_type {
        DataType::Float64 => Arc::new(
            values
                .iter()
                .map(|v| match v {
                    Value::Null => Ok(None),
                    Value::Num(n) => Ok(Some(*n)),
                    other => Err(wrong(other)),
                })
                .collect::<Result<Float64Array>>()?,
        ),
        DataType::Utf8 => Arc::new(
            values
                .iter()
                .map(|v| match v {
                    Value::Null => Ok(None),
                    Value::Str(s) => Ok(Some(s.clone())),
                    other => Err(wrong(other)),
                })
                .collect::<Result<StringArray>>()?,
        ),
        DataType::Boolean => Arc::new(
            values
                .iter()
                .map(|v| match v {
                    Value::Null => Ok(None),
                    Value::Bool(b) => Ok(Some(*b)),
                    other => Err(wrong(other)),
                })
                .collect::<Result<BooleanArray>>()?,
        ),
        DataType::Null => Arc::new(NullArray::new(values.len())),
        other => {
            return Err(DataFusionError::NotImplemented(format!(
                "fv-value: cannot materialise dialect values as {other}"
            )))
        }
    })
}

#[allow(dead_code)]
fn _list_type_marker(_: &ListArray) {}

#[cfg(test)]
mod tests {
    use super::*;
    use datafusion::arrow::array::{Int32Array, ListBuilder, StringViewArray};

    #[test]
    fn cells_convert_by_type() {
        let ints: ArrayRef = Arc::new(Int32Array::from(vec![Some(7), None]));
        assert_eq!(cell_to_value(&ints, 0), Value::Num(7.0));
        assert_eq!(cell_to_value(&ints, 1), Value::Null);
        let views: ArrayRef = Arc::new(StringViewArray::from(vec!["x"]));
        assert_eq!(cell_to_value(&views, 0), Value::from("x"));
        let mut lb = ListBuilder::new(Float64Array::builder(2));
        lb.values().append_value(1.0);
        lb.values().append_value(2.0);
        lb.append(true);
        let list: ArrayRef = Arc::new(lb.finish());
        assert_eq!(cell_to_value(&list, 0), Value::from(vec![1.0, 2.0]));
    }

    #[test]
    fn kinds_and_materialisation() {
        assert_eq!(Kind::of_data_type(&DataType::Int64), Some(Kind::Num));
        assert_eq!(Kind::of_data_type(&DataType::Utf8View), Some(Kind::Str));
        assert_eq!(Kind::of_data_type(&DataType::Date32), None);
        let a = values_to_array(&[Value::Num(1.5), Value::Null], &DataType::Float64).unwrap();
        assert_eq!(a.len(), 2);
        assert!(a.is_null(1));
        assert!(values_to_array(&[Value::from("s")], &DataType::Float64).is_err());
        assert_eq!(data_type_for(Kind::Bool), DataType::Boolean);
    }
}