fv-value-datafusion 0.2.0

The fv-value expression dialect as DataFusion expressions: exact translation for pushdown and vectorized evaluation, with a per-row UDF fallback.
Documentation
//! The per-row fallback and the two helpers that carry the dialect's run-time errors into a plan —
//! raising one, and checking a computed number is finite — all as DataFusion scalar UDFs.

use std::collections::hash_map::DefaultHasher;
use std::hash::{Hash, Hasher};

use datafusion::arrow::array::{ArrayRef, AsArray};
use datafusion::arrow::datatypes::DataType;
use datafusion::arrow::datatypes::Float64Type;
use datafusion::common::Column;
use datafusion::common::Result;
use datafusion::error::DataFusionError;
use datafusion::logical_expr::{
    ColumnarValue, Expr as DfExpr, ScalarFunctionArgs, ScalarUDF, ScalarUDFImpl, Signature, Volatility,
};
use datafusion::scalar::ScalarValue;
use fv_value::{Expr, Value};

use crate::convert::{has_value_form, values_to_array, Cells};

/// A dialect expression evaluated per row by DataFusion, over the columns it reads. The fallback
/// for anything the translator cannot express exactly; also usable on its own.
///
/// `return_type` is what the plan declares for the result; a row producing a value of another kind
/// is an execution error.
#[derive(Debug)]
pub struct EvalUdf {
    name: String,
    expr: Expr,
    columns: Vec<String>,
    return_type: DataType,
    signature: Signature,
}

impl EvalUdf {
    /// A UDF over `columns` (the expression's identifiers, in argument order). Its name embeds a
    /// hash of the source so two different expressions never look like the same function to the
    /// planner.
    pub fn new(expr: Expr, columns: Vec<String>, return_type: DataType) -> Self {
        let mut h = DefaultHasher::new();
        expr.source().hash(&mut h);
        columns.hash(&mut h);
        Self {
            name: format!("fv_eval_{:016x}", h.finish()),
            signature: if columns.is_empty() {
                Signature::nullary(Volatility::Immutable)
            } else {
                Signature::variadic_any(Volatility::Immutable)
            },
            expr,
            columns,
            return_type,
        }
    }

    /// The call expression: the UDF applied to its columns.
    pub fn call_expr(self) -> DfExpr {
        let args: Vec<DfExpr> = self
            .columns
            .iter()
            .map(|c| DfExpr::Column(Column::from_name(c)))
            .collect();
        ScalarUDF::from(self).call(args)
    }
}

// `Expr` is not `Hash`/`Eq`; identity is the name, which hashes the source and the columns.
impl PartialEq for EvalUdf {
    fn eq(&self, o: &Self) -> bool {
        self.name == o.name
    }
}
impl Eq for EvalUdf {}
impl Hash for EvalUdf {
    fn hash<H: Hasher>(&self, h: &mut H) {
        self.name.hash(h);
    }
}

impl ScalarUDFImpl for EvalUdf {
    fn name(&self) -> &str {
        &self.name
    }
    fn signature(&self) -> &Signature {
        &self.signature
    }
    fn return_type(&self, _: &[DataType]) -> Result<DataType> {
        Ok(self.return_type.clone())
    }
    fn invoke_with_args(&self, args: ScalarFunctionArgs) -> Result<ColumnarValue> {
        let n = args.number_rows;
        let arrays: Vec<ArrayRef> = args
            .args
            .into_iter()
            .map(|a| a.into_array(n))
            .collect::<Result<Vec<_>>>()?;
        let mut out = Vec::with_capacity(n);
        // each column's type is checked once per batch; every cell is then read with no error path
        let cells = self
            .columns
            .iter()
            .zip(&arrays)
            .map(|(name, arr)| Cells::of(arr).map_err(|e| DataFusionError::Execution(format!("column '{name}': {e}"))))
            .collect::<Result<Vec<_>>>()?;
        let mut row: Vec<(String, Value)> = self.columns.iter().map(|c| (c.clone(), Value::Null)).collect();
        for i in 0..n {
            for (slot, cells) in row.iter_mut().zip(&cells) {
                slot.1 = cells.value(i);
            }
            let v = self
                .expr
                .eval(row.as_slice())
                .map_err(|e| DataFusionError::Execution(format!("fv-value: {e}")))?;
            out.push(v);
        }
        Ok(ColumnarValue::Array(values_to_array(&out, &self.return_type)?))
    }
}

/// A function that fails with its message when evaluated. Placed inside `CASE WHEN … THEN
/// fv_error('…') ELSE … END`, it reproduces the dialect's run-time errors exactly: DataFusion
/// evaluates a `CASE` branch only for the rows that select it.
#[derive(Debug, PartialEq, Eq, Hash)]
pub struct ErrorUdf {
    return_type: DataType,
    signature: Signature,
}

impl ErrorUdf {
    /// An error function whose (never produced) result has `return_type`, so it can sit in a `CASE`
    /// branch of that type.
    pub fn new(return_type: DataType) -> Self {
        Self {
            return_type,
            // Volatile, so the planner's constant folder never evaluates it ahead of time: it
            // must fire only for the rows that reach it at execution.
            signature: Signature::exact(vec![DataType::Utf8], Volatility::Volatile),
        }
    }

    /// The call expression `fv_error('message')`.
    pub fn call_expr(return_type: DataType, message: &str) -> DfExpr {
        ScalarUDF::from(Self::new(return_type)).call(vec![DfExpr::Literal(
            ScalarValue::Utf8(Some(message.to_string())),
            None,
        )])
    }
}

impl ScalarUDFImpl for ErrorUdf {
    fn name(&self) -> &str {
        match self.return_type {
            DataType::Float64 => "fv_error_num",
            DataType::Utf8 => "fv_error_str",
            DataType::Boolean => "fv_error_bool",
            _ => "fv_error",
        }
    }
    fn signature(&self) -> &Signature {
        &self.signature
    }
    fn return_type(&self, _: &[DataType]) -> Result<DataType> {
        Ok(self.return_type.clone())
    }
    fn invoke_with_args(&self, args: ScalarFunctionArgs) -> Result<ColumnarValue> {
        if args.number_rows == 0 {
            return Ok(ColumnarValue::Array(values_to_array(&[], &self.return_type)?));
        }
        let message = match args.args.first() {
            Some(ColumnarValue::Scalar(ScalarValue::Utf8(Some(m)))) => m.clone(),
            _ => "fv-value: error".to_string(),
        };
        Err(DataFusionError::Execution(format!("fv-value: {message}")))
    }
}

/// The dialect's rule for a computed number: NaN or ±infinity is never a value but an error naming
/// what computed it. `fv_finite(x, 'what')` returns `x` itself — the same array, not a copy — after
/// one pass over it; a null passes. It costs one comparison per value, and the null mask is read
/// only when some slot is not finite (a null slot may hold anything).
#[derive(Debug, PartialEq, Eq, Hash)]
pub struct FiniteUdf {
    signature: Signature,
}

impl Default for FiniteUdf {
    fn default() -> Self {
        Self {
            signature: Signature::exact(vec![DataType::Float64, DataType::Utf8], Volatility::Immutable),
        }
    }
}

impl FiniteUdf {
    /// The call expression `fv_finite(x, 'what')`; `what` names the function or operator (`sqrt`,
    /// `'+'`) as the dialect's error does.
    pub fn call_expr(x: DfExpr, what: &str) -> DfExpr {
        ScalarUDF::from(Self::default()).call(vec![
            x,
            DfExpr::Literal(ScalarValue::Utf8(Some(what.to_string())), None),
        ])
    }
}

/// Whether every valid value of `a` is finite: one branch-free pass, reading the null mask only
/// when some slot is not finite.
pub fn all_finite(a: &datafusion::arrow::array::Float64Array) -> bool {
    // A float is NaN or infinite exactly when its exponent bits are all ones. Tested on the bits
    // as integers and OR-reduced, the loop has no branch and no early exit, so it vectorises.
    const EXP: u64 = 0x7FF0_0000_0000_0000;
    let any_nonfinite = a
        .values()
        .iter()
        .fold(0u64, |acc, v| acc | u64::from(v.to_bits() & EXP == EXP));
    if any_nonfinite == 0 {
        return true;
    }
    // some slot is not finite: it matters only if that slot is valid (a null slot holds anything)
    a.iter().all(|v| v.is_none_or(f64::is_finite))
}

impl ScalarUDFImpl for FiniteUdf {
    fn name(&self) -> &str {
        "fv_finite"
    }
    fn signature(&self) -> &Signature {
        &self.signature
    }
    fn return_type(&self, _: &[DataType]) -> Result<DataType> {
        Ok(DataType::Float64)
    }
    fn invoke_with_args(&self, args: ScalarFunctionArgs) -> Result<ColumnarValue> {
        let [x, what] = args.args.as_slice() else {
            return Err(DataFusionError::Internal("fv_finite takes a number and a name".into()));
        };
        let ok = match x {
            ColumnarValue::Array(a) => all_finite(a.as_primitive::<Float64Type>()),
            ColumnarValue::Scalar(ScalarValue::Float64(v)) => v.is_none_or(f64::is_finite),
            ColumnarValue::Scalar(other) => {
                return Err(DataFusionError::Internal(format!("fv_finite on {}", other.data_type())))
            }
        };
        if ok {
            return Ok(x.clone());
        }
        let what = match what {
            ColumnarValue::Scalar(ScalarValue::Utf8(Some(w))) => w.as_str(),
            _ => "a computation",
        };
        Err(DataFusionError::Execution(format!(
            "fv-value: {what}: result is not a finite number"
        )))
    }
}

/// Build a fallback call for `expr` returning `return_type`, over the expression's identifiers
/// that exist in `schema` (an identifier with no column reads as `null`, as the dialect defines).
///
/// A column whose type has no form in the dialect ([`has_value_form`]) is refused here, by name,
/// when the plan is built — never read row by row as a null it is not.
pub fn fallback(
    expr: &Expr,
    return_type: DataType,
    schema: &datafusion::arrow::datatypes::Schema,
) -> std::result::Result<DfExpr, crate::Unsupported> {
    let mut columns = Vec::new();
    for c in expr.identifiers() {
        let Ok(field) = schema.field_with_name(c) else { continue };
        if !has_value_form(field.data_type()) {
            return Err(crate::Unsupported {
                reason: format!(
                    "column '{c}' has type {}, which has no value in the dialect (numbers, strings, booleans, lists and structs of them); cast it in the source",
                    field.data_type()
                ),
            });
        }
        columns.push(c.clone());
    }
    Ok(EvalUdf::new(expr.clone(), columns, return_type).call_expr())
}

#[cfg(test)]
mod tests {
    use super::*;

    #[test]
    fn eval_udf_names_hash_the_source() {
        let a = EvalUdf::new(fv_value::compile("x + 1").unwrap(), vec!["x".into()], DataType::Float64);
        let b = EvalUdf::new(fv_value::compile("x + 2").unwrap(), vec!["x".into()], DataType::Float64);
        assert_ne!(a.name(), b.name());
        assert!(a.name().starts_with("fv_eval_"));
        assert_eq!(a.return_type(&[]).unwrap(), DataType::Float64);
    }

    #[test]
    fn finite_udf_passes_its_input_through_and_names_what_overflowed() {
        use datafusion::arrow::array::{Array, Float64Array};
        use datafusion::arrow::datatypes::Field;
        use std::sync::Arc;
        let call = |values: Vec<Option<f64>>| {
            let input: ArrayRef = Arc::new(Float64Array::from(values));
            let n = input.len();
            let out = FiniteUdf::default().invoke_with_args(ScalarFunctionArgs {
                args: vec![
                    ColumnarValue::Array(Arc::clone(&input)),
                    ColumnarValue::Scalar(ScalarValue::Utf8(Some("'*'".into()))),
                ],
                arg_fields: vec![
                    Arc::new(Field::new("x", DataType::Float64, true)),
                    Arc::new(Field::new("w", DataType::Utf8, false)),
                ],
                number_rows: n,
                return_field: Arc::new(Field::new("f", DataType::Float64, true)),
                config_options: Arc::new(Default::default()),
            });
            (input, out)
        };
        // the same array back, not a copy
        let (input, out) = call(vec![Some(1.0), None, Some(-2.5)]);
        match out.unwrap() {
            ColumnarValue::Array(a) => assert!(Arc::ptr_eq(&a, &input)),
            other => panic!("{other:?}"),
        }
        for bad in [f64::INFINITY, f64::NEG_INFINITY, f64::NAN] {
            let err = call(vec![Some(1.0), Some(bad)]).1.unwrap_err().to_string();
            assert!(err.contains("fv-value: '*': result is not a finite number"), "{err}");
        }
    }

    #[test]
    fn a_non_finite_value_under_a_null_is_not_an_error() {
        use datafusion::arrow::array::Float64Array;
        use datafusion::arrow::buffer::{NullBuffer, ScalarBuffer};
        // a kernel may leave anything in a null slot
        let a = Float64Array::new(
            ScalarBuffer::from(vec![1.0, f64::NAN, f64::INFINITY]),
            Some(NullBuffer::from(vec![true, false, false])),
        );
        assert!(all_finite(&a));
        let b = Float64Array::new(
            ScalarBuffer::from(vec![1.0, f64::NAN, f64::INFINITY]),
            Some(NullBuffer::from(vec![true, false, true])),
        );
        assert!(!all_finite(&b));
    }

    #[test]
    fn error_udf_is_named_by_type() {
        assert_eq!(ErrorUdf::new(DataType::Float64).name(), "fv_error_num");
        assert_eq!(ErrorUdf::new(DataType::Utf8).name(), "fv_error_str");
        assert_eq!(ErrorUdf::new(DataType::Boolean).name(), "fv_error_bool");
    }
}