use std::collections::hash_map::DefaultHasher;
use std::hash::{Hash, Hasher};
use datafusion::arrow::array::{ArrayRef, AsArray};
use datafusion::arrow::datatypes::DataType;
use datafusion::arrow::datatypes::Float64Type;
use datafusion::common::Column;
use datafusion::common::Result;
use datafusion::error::DataFusionError;
use datafusion::logical_expr::{
ColumnarValue, Expr as DfExpr, ScalarFunctionArgs, ScalarUDF, ScalarUDFImpl, Signature, Volatility,
};
use datafusion::scalar::ScalarValue;
use fv_value::{Expr, Value};
use crate::convert::{has_value_form, values_to_array, Cells};
#[derive(Debug)]
pub struct EvalUdf {
name: String,
expr: Expr,
columns: Vec<String>,
return_type: DataType,
signature: Signature,
}
impl EvalUdf {
pub fn new(expr: Expr, columns: Vec<String>, return_type: DataType) -> Self {
let mut h = DefaultHasher::new();
expr.source().hash(&mut h);
columns.hash(&mut h);
Self {
name: format!("fv_eval_{:016x}", h.finish()),
signature: if columns.is_empty() {
Signature::nullary(Volatility::Immutable)
} else {
Signature::variadic_any(Volatility::Immutable)
},
expr,
columns,
return_type,
}
}
pub fn call_expr(self) -> DfExpr {
let args: Vec<DfExpr> = self
.columns
.iter()
.map(|c| DfExpr::Column(Column::from_name(c)))
.collect();
ScalarUDF::from(self).call(args)
}
}
impl PartialEq for EvalUdf {
fn eq(&self, o: &Self) -> bool {
self.name == o.name
}
}
impl Eq for EvalUdf {}
impl Hash for EvalUdf {
fn hash<H: Hasher>(&self, h: &mut H) {
self.name.hash(h);
}
}
impl ScalarUDFImpl for EvalUdf {
fn name(&self) -> &str {
&self.name
}
fn signature(&self) -> &Signature {
&self.signature
}
fn return_type(&self, _: &[DataType]) -> Result<DataType> {
Ok(self.return_type.clone())
}
fn invoke_with_args(&self, args: ScalarFunctionArgs) -> Result<ColumnarValue> {
let n = args.number_rows;
let arrays: Vec<ArrayRef> = args
.args
.into_iter()
.map(|a| a.into_array(n))
.collect::<Result<Vec<_>>>()?;
let mut out = Vec::with_capacity(n);
let cells = self
.columns
.iter()
.zip(&arrays)
.map(|(name, arr)| Cells::of(arr).map_err(|e| DataFusionError::Execution(format!("column '{name}': {e}"))))
.collect::<Result<Vec<_>>>()?;
let mut row: Vec<(String, Value)> = self.columns.iter().map(|c| (c.clone(), Value::Null)).collect();
for i in 0..n {
for (slot, cells) in row.iter_mut().zip(&cells) {
slot.1 = cells.value(i);
}
let v = self
.expr
.eval(row.as_slice())
.map_err(|e| DataFusionError::Execution(format!("fv-value: {e}")))?;
out.push(v);
}
Ok(ColumnarValue::Array(values_to_array(&out, &self.return_type)?))
}
}
#[derive(Debug, PartialEq, Eq, Hash)]
pub struct ErrorUdf {
return_type: DataType,
signature: Signature,
}
impl ErrorUdf {
pub fn new(return_type: DataType) -> Self {
Self {
return_type,
signature: Signature::exact(vec![DataType::Utf8], Volatility::Volatile),
}
}
pub fn call_expr(return_type: DataType, message: &str) -> DfExpr {
ScalarUDF::from(Self::new(return_type)).call(vec![DfExpr::Literal(
ScalarValue::Utf8(Some(message.to_string())),
None,
)])
}
}
impl ScalarUDFImpl for ErrorUdf {
fn name(&self) -> &str {
match self.return_type {
DataType::Float64 => "fv_error_num",
DataType::Utf8 => "fv_error_str",
DataType::Boolean => "fv_error_bool",
_ => "fv_error",
}
}
fn signature(&self) -> &Signature {
&self.signature
}
fn return_type(&self, _: &[DataType]) -> Result<DataType> {
Ok(self.return_type.clone())
}
fn invoke_with_args(&self, args: ScalarFunctionArgs) -> Result<ColumnarValue> {
if args.number_rows == 0 {
return Ok(ColumnarValue::Array(values_to_array(&[], &self.return_type)?));
}
let message = match args.args.first() {
Some(ColumnarValue::Scalar(ScalarValue::Utf8(Some(m)))) => m.clone(),
_ => "fv-value: error".to_string(),
};
Err(DataFusionError::Execution(format!("fv-value: {message}")))
}
}
#[derive(Debug, PartialEq, Eq, Hash)]
pub struct FiniteUdf {
signature: Signature,
}
impl Default for FiniteUdf {
fn default() -> Self {
Self {
signature: Signature::exact(vec![DataType::Float64, DataType::Utf8], Volatility::Immutable),
}
}
}
impl FiniteUdf {
pub fn call_expr(x: DfExpr, what: &str) -> DfExpr {
ScalarUDF::from(Self::default()).call(vec![
x,
DfExpr::Literal(ScalarValue::Utf8(Some(what.to_string())), None),
])
}
}
pub fn all_finite(a: &datafusion::arrow::array::Float64Array) -> bool {
const EXP: u64 = 0x7FF0_0000_0000_0000;
let any_nonfinite = a
.values()
.iter()
.fold(0u64, |acc, v| acc | u64::from(v.to_bits() & EXP == EXP));
if any_nonfinite == 0 {
return true;
}
a.iter().all(|v| v.is_none_or(f64::is_finite))
}
impl ScalarUDFImpl for FiniteUdf {
fn name(&self) -> &str {
"fv_finite"
}
fn signature(&self) -> &Signature {
&self.signature
}
fn return_type(&self, _: &[DataType]) -> Result<DataType> {
Ok(DataType::Float64)
}
fn invoke_with_args(&self, args: ScalarFunctionArgs) -> Result<ColumnarValue> {
let [x, what] = args.args.as_slice() else {
return Err(DataFusionError::Internal("fv_finite takes a number and a name".into()));
};
let ok = match x {
ColumnarValue::Array(a) => all_finite(a.as_primitive::<Float64Type>()),
ColumnarValue::Scalar(ScalarValue::Float64(v)) => v.is_none_or(f64::is_finite),
ColumnarValue::Scalar(other) => {
return Err(DataFusionError::Internal(format!("fv_finite on {}", other.data_type())))
}
};
if ok {
return Ok(x.clone());
}
let what = match what {
ColumnarValue::Scalar(ScalarValue::Utf8(Some(w))) => w.as_str(),
_ => "a computation",
};
Err(DataFusionError::Execution(format!(
"fv-value: {what}: result is not a finite number"
)))
}
}
pub fn fallback(
expr: &Expr,
return_type: DataType,
schema: &datafusion::arrow::datatypes::Schema,
) -> std::result::Result<DfExpr, crate::Unsupported> {
let mut columns = Vec::new();
for c in expr.identifiers() {
let Ok(field) = schema.field_with_name(c) else { continue };
if !has_value_form(field.data_type()) {
return Err(crate::Unsupported {
reason: format!(
"column '{c}' has type {}, which has no value in the dialect (numbers, strings, booleans, lists and structs of them); cast it in the source",
field.data_type()
),
});
}
columns.push(c.clone());
}
Ok(EvalUdf::new(expr.clone(), columns, return_type).call_expr())
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn eval_udf_names_hash_the_source() {
let a = EvalUdf::new(fv_value::compile("x + 1").unwrap(), vec!["x".into()], DataType::Float64);
let b = EvalUdf::new(fv_value::compile("x + 2").unwrap(), vec!["x".into()], DataType::Float64);
assert_ne!(a.name(), b.name());
assert!(a.name().starts_with("fv_eval_"));
assert_eq!(a.return_type(&[]).unwrap(), DataType::Float64);
}
#[test]
fn finite_udf_passes_its_input_through_and_names_what_overflowed() {
use datafusion::arrow::array::{Array, Float64Array};
use datafusion::arrow::datatypes::Field;
use std::sync::Arc;
let call = |values: Vec<Option<f64>>| {
let input: ArrayRef = Arc::new(Float64Array::from(values));
let n = input.len();
let out = FiniteUdf::default().invoke_with_args(ScalarFunctionArgs {
args: vec![
ColumnarValue::Array(Arc::clone(&input)),
ColumnarValue::Scalar(ScalarValue::Utf8(Some("'*'".into()))),
],
arg_fields: vec![
Arc::new(Field::new("x", DataType::Float64, true)),
Arc::new(Field::new("w", DataType::Utf8, false)),
],
number_rows: n,
return_field: Arc::new(Field::new("f", DataType::Float64, true)),
config_options: Arc::new(Default::default()),
});
(input, out)
};
let (input, out) = call(vec![Some(1.0), None, Some(-2.5)]);
match out.unwrap() {
ColumnarValue::Array(a) => assert!(Arc::ptr_eq(&a, &input)),
other => panic!("{other:?}"),
}
for bad in [f64::INFINITY, f64::NEG_INFINITY, f64::NAN] {
let err = call(vec![Some(1.0), Some(bad)]).1.unwrap_err().to_string();
assert!(err.contains("fv-value: '*': result is not a finite number"), "{err}");
}
}
#[test]
fn a_non_finite_value_under_a_null_is_not_an_error() {
use datafusion::arrow::array::Float64Array;
use datafusion::arrow::buffer::{NullBuffer, ScalarBuffer};
let a = Float64Array::new(
ScalarBuffer::from(vec![1.0, f64::NAN, f64::INFINITY]),
Some(NullBuffer::from(vec![true, false, false])),
);
assert!(all_finite(&a));
let b = Float64Array::new(
ScalarBuffer::from(vec![1.0, f64::NAN, f64::INFINITY]),
Some(NullBuffer::from(vec![true, false, true])),
);
assert!(!all_finite(&b));
}
#[test]
fn error_udf_is_named_by_type() {
assert_eq!(ErrorUdf::new(DataType::Float64).name(), "fv_error_num");
assert_eq!(ErrorUdf::new(DataType::Utf8).name(), "fv_error_str");
assert_eq!(ErrorUdf::new(DataType::Boolean).name(), "fv_error_bool");
}
}