use datafusion::arrow::datatypes::DataType;
use datafusion::logical_expr::{col, Expr as DfExpr, ScalarUDF};
use datafusion::prelude::SessionContext;
use fv_value::{compile, Expr, Value};
use fv_value_datafusion::{fallback, EvalUdf, Translator};
use crate::convert::{batches_to_rows, rows_to_batch};
use crate::row::Row;
pub fn fv_scalar_udf(
_name: &str,
expr: &str,
ref_cols: &[String],
_input_types: Vec<DataType>,
return_type: DataType,
) -> Result<ScalarUDF, String> {
let compiled = compile(expr).map_err(|e| e.to_string())?;
Ok(ScalarUDF::from(EvalUdf::new(compiled, ref_cols.to_vec(), return_type)))
}
fn infer_type(vals: &[Value]) -> DataType {
let (mut num, mut boolean, mut other) = (false, false, false);
for v in vals {
match v {
Value::Num(_) => num = true,
Value::Bool(_) => boolean = true,
Value::Null => {}
_ => other = true,
}
}
if other {
DataType::Utf8
} else if boolean && !num {
DataType::Boolean
} else {
DataType::Float64
}
}
fn eval_all(expr: &Expr, rows: &[Row]) -> Result<Vec<Value>, String> {
rows.iter()
.map(|r| expr.eval(r.lookup()).map_err(|e| e.to_string()))
.collect()
}
async fn collect(df: datafusion::dataframe::DataFrame) -> Result<Vec<Row>, String> {
let batches = df.collect().await.map_err(|e| e.to_string())?;
batches_to_rows(&batches)
}
pub async fn filter_via_datafusion(rows: &[Row], columns: &[String], expr: &str) -> Result<Vec<Row>, String> {
if rows.is_empty() {
return Ok(vec![]);
}
let compiled = compile(expr).map_err(|e| e.to_string())?;
if compiled.is_constant() {
return match eval_all(&compiled, &rows[..1])?.into_iter().next() {
Some(Value::Bool(true)) => Ok(rows.to_vec()),
Some(Value::Bool(false)) => Ok(vec![]),
other => Err(format!("filter: predicate must be a boolean, got {other:?}")),
};
}
let batch = rows_to_batch(rows, columns)?;
let schema = batch.schema();
let predicate: DfExpr = match Translator::new(schema.clone()).translate_predicate_or_fallback(&compiled) {
Ok(t) => t.expr,
Err(_) => fallback(&compiled, DataType::Boolean, &schema).map_err(|e| e.to_string())?,
};
let ctx = SessionContext::new();
let df = ctx
.read_batch(batch)
.map_err(|e| e.to_string())?
.filter(predicate)
.map_err(|e| e.to_string())?;
collect(df).await
}
pub async fn apply_expression_via_datafusion(
rows: &[Row],
columns: &[String],
out_col: &str,
expr: &str,
) -> Result<Vec<Row>, String> {
if rows.is_empty() {
return Ok(vec![]);
}
let compiled = compile(expr).map_err(|e| e.to_string())?;
let batch = rows_to_batch(rows, columns)?;
let schema = batch.schema();
let translator = Translator::new(schema.clone());
let value: DfExpr = match translator.translate_or_fallback(&compiled) {
Ok(t) => t.expr,
Err(_) => fallback(&compiled, infer_type(&eval_all(&compiled, rows)?), &schema).map_err(|e| e.to_string())?,
};
let mut selects: Vec<DfExpr> = Vec::new();
let mut replaced = false;
for c in columns {
if c == out_col {
selects.push(value.clone().alias(out_col));
replaced = true;
} else {
selects.push(col(format!("\"{}\"", c.replace('"', "\"\""))));
}
}
if !replaced {
selects.push(value.alias(out_col));
}
let ctx = SessionContext::new();
let df = ctx
.read_batch(batch)
.map_err(|e| e.to_string())?
.select(selects)
.map_err(|e| e.to_string())?;
collect(df).await
}