use std::collections::hash_map::DefaultHasher;
use std::hash::{Hash, Hasher};
use datafusion::arrow::array::ArrayRef;
use datafusion::arrow::datatypes::DataType;
use datafusion::common::Column;
use datafusion::common::Result;
use datafusion::error::DataFusionError;
use datafusion::logical_expr::{
ColumnarValue, Expr as DfExpr, ScalarFunctionArgs, ScalarUDF, ScalarUDFImpl, Signature, Volatility,
};
use datafusion::scalar::ScalarValue;
use fv_value::{Expr, Value};
use crate::convert::{cell_to_value, values_to_array};
#[derive(Debug)]
pub struct EvalUdf {
name: String,
expr: Expr,
columns: Vec<String>,
return_type: DataType,
signature: Signature,
}
impl EvalUdf {
pub fn new(expr: Expr, columns: Vec<String>, return_type: DataType) -> Self {
let mut h = DefaultHasher::new();
expr.source().hash(&mut h);
columns.hash(&mut h);
Self {
name: format!("fv_eval_{:016x}", h.finish()),
signature: if columns.is_empty() {
Signature::nullary(Volatility::Immutable)
} else {
Signature::variadic_any(Volatility::Immutable)
},
expr,
columns,
return_type,
}
}
pub fn call_expr(self) -> DfExpr {
let args: Vec<DfExpr> = self
.columns
.iter()
.map(|c| DfExpr::Column(Column::from_name(c)))
.collect();
ScalarUDF::from(self).call(args)
}
}
impl PartialEq for EvalUdf {
fn eq(&self, o: &Self) -> bool {
self.name == o.name
}
}
impl Eq for EvalUdf {}
impl Hash for EvalUdf {
fn hash<H: Hasher>(&self, h: &mut H) {
self.name.hash(h);
}
}
impl ScalarUDFImpl for EvalUdf {
fn name(&self) -> &str {
&self.name
}
fn signature(&self) -> &Signature {
&self.signature
}
fn return_type(&self, _: &[DataType]) -> Result<DataType> {
Ok(self.return_type.clone())
}
fn invoke_with_args(&self, args: ScalarFunctionArgs) -> Result<ColumnarValue> {
let n = args.number_rows;
let arrays: Vec<ArrayRef> = args
.args
.into_iter()
.map(|a| a.into_array(n))
.collect::<Result<Vec<_>>>()?;
let mut out = Vec::with_capacity(n);
let mut row: Vec<(String, Value)> = self.columns.iter().map(|c| (c.clone(), Value::Null)).collect();
for i in 0..n {
for (slot, arr) in row.iter_mut().zip(&arrays) {
slot.1 = cell_to_value(arr, i);
}
let v = self
.expr
.eval(row.as_slice())
.map_err(|e| DataFusionError::Execution(format!("fv-value: {e}")))?;
out.push(v);
}
Ok(ColumnarValue::Array(values_to_array(&out, &self.return_type)?))
}
}
#[derive(Debug, PartialEq, Eq, Hash)]
pub struct ErrorUdf {
return_type: DataType,
signature: Signature,
}
impl ErrorUdf {
pub fn new(return_type: DataType) -> Self {
Self {
return_type,
signature: Signature::exact(vec![DataType::Utf8], Volatility::Volatile),
}
}
pub fn call_expr(return_type: DataType, message: &str) -> DfExpr {
ScalarUDF::from(Self::new(return_type)).call(vec![DfExpr::Literal(
ScalarValue::Utf8(Some(message.to_string())),
None,
)])
}
}
impl ScalarUDFImpl for ErrorUdf {
fn name(&self) -> &str {
match self.return_type {
DataType::Float64 => "fv_error_num",
DataType::Utf8 => "fv_error_str",
DataType::Boolean => "fv_error_bool",
_ => "fv_error",
}
}
fn signature(&self) -> &Signature {
&self.signature
}
fn return_type(&self, _: &[DataType]) -> Result<DataType> {
Ok(self.return_type.clone())
}
fn invoke_with_args(&self, args: ScalarFunctionArgs) -> Result<ColumnarValue> {
if args.number_rows == 0 {
return Ok(ColumnarValue::Array(values_to_array(&[], &self.return_type)?));
}
let message = match args.args.first() {
Some(ColumnarValue::Scalar(ScalarValue::Utf8(Some(m)))) => m.clone(),
_ => "fv-value: error".to_string(),
};
Err(DataFusionError::Execution(format!("fv-value: {message}")))
}
}
pub fn fallback(expr: &Expr, return_type: DataType, schema: &datafusion::arrow::datatypes::Schema) -> DfExpr {
let columns: Vec<String> = expr
.identifiers()
.iter()
.filter(|c| schema.field_with_name(c).is_ok())
.cloned()
.collect();
EvalUdf::new(expr.clone(), columns, return_type).call_expr()
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn eval_udf_names_hash_the_source() {
let a = EvalUdf::new(fv_value::compile("x + 1").unwrap(), vec!["x".into()], DataType::Float64);
let b = EvalUdf::new(fv_value::compile("x + 2").unwrap(), vec!["x".into()], DataType::Float64);
assert_ne!(a.name(), b.name());
assert!(a.name().starts_with("fv_eval_"));
assert_eq!(a.return_type(&[]).unwrap(), DataType::Float64);
}
#[test]
fn error_udf_is_named_by_type() {
assert_eq!(ErrorUdf::new(DataType::Float64).name(), "fv_error_num");
assert_eq!(ErrorUdf::new(DataType::Utf8).name(), "fv_error_str");
assert_eq!(ErrorUdf::new(DataType::Boolean).name(), "fv_error_bool");
}
}