fv-plan 0.3.0

The FusionVault transform executor on Apache DataFusion: inline row steps and relational reshapes (join, aggregate, window, union) over Arrow, with the value dialect translated into the plan, spatial SQL, file sources, and build orchestration checked against the published conformance vectors.
Documentation
//! The value dialect inside a DataFusion plan.
//!
//! `filter` / `applyExpression` steps run INSIDE the DataFusion plan — fusable with the relational
//! ops, vectorized, and pushdown-capable — by translating the compiled dialect expression into a
//! DataFusion expression (`fv-value-datafusion`) wherever the semantics are provably identical, and
//! evaluating the rest per row through its UDF fallback. Results are identical to the row-wise
//! executor either way; the conformance vectors replayed through DataFusion are the proof.
//! Output typing comes from the translator when it can be known statically, else from a one-time
//! probe of the values.

use datafusion::arrow::datatypes::DataType;
use datafusion::logical_expr::{col, Expr as DfExpr, ScalarUDF};
use datafusion::prelude::SessionContext;
use fv_value::{compile, Expr, Value};
use fv_value_datafusion::{fallback, EvalUdf, Translator};

use crate::convert::{batches_to_rows, rows_to_batch};
use crate::row::Row;

/// A `ScalarUDF` that evaluates `expr` per row over `ref_cols` (the per-row fallback, exposed for
/// callers that register functions by name — e.g. a streaming query). `name` is ignored: the UDF
/// names itself from a hash of the expression so distinct expressions never collide.
pub fn fv_scalar_udf(
    _name: &str,
    expr: &str,
    ref_cols: &[String],
    _input_types: Vec<DataType>,
    return_type: DataType,
) -> Result<ScalarUDF, String> {
    let compiled = compile(expr).map_err(|e| e.to_string())?;
    Ok(ScalarUDF::from(EvalUdf::new(compiled, ref_cols.to_vec(), return_type)))
}

/// Infer an output Arrow type from computed values (numbers → Float64, booleans → Boolean, else Utf8).
fn infer_type(vals: &[Value]) -> DataType {
    let (mut num, mut boolean, mut other) = (false, false, false);
    for v in vals {
        match v {
            Value::Num(_) => num = true,
            Value::Bool(_) => boolean = true,
            Value::Null => {}
            _ => other = true,
        }
    }
    if other {
        DataType::Utf8
    } else if boolean && !num {
        DataType::Boolean
    } else {
        DataType::Float64
    }
}

fn eval_all(expr: &Expr, rows: &[Row]) -> Result<Vec<Value>, String> {
    rows.iter()
        .map(|r| expr.eval(r.lookup()).map_err(|e| e.to_string()))
        .collect()
}

async fn collect(df: datafusion::dataframe::DataFrame) -> Result<Vec<Row>, String> {
    let batches = df.collect().await.map_err(|e| e.to_string())?;
    batches_to_rows(&batches)
}

/// Keep rows whose `expr` is strictly true, executed via the DataFusion plan. The predicate is
/// translated into a DataFusion filter where exact (so it can push down and vectorize) and falls
/// back to per-row evaluation where not. A constant predicate is applied directly.
pub async fn filter_via_datafusion(rows: &[Row], columns: &[String], expr: &str) -> Result<Vec<Row>, String> {
    if rows.is_empty() {
        return Ok(vec![]);
    }
    let compiled = compile(expr).map_err(|e| e.to_string())?;
    if compiled.is_constant() {
        return match eval_all(&compiled, &rows[..1])?.into_iter().next() {
            Some(Value::Bool(true)) => Ok(rows.to_vec()),
            Some(Value::Bool(false)) => Ok(vec![]),
            other => Err(format!("filter: predicate must be a boolean, got {other:?}")),
        };
    }
    let batch = rows_to_batch(rows, columns)?;
    let schema = batch.schema();
    // Rows may carry numbers computed by earlier steps, so NaN stays possible: strict options.
    let predicate: DfExpr = match Translator::new(schema.clone()).translate_predicate_or_fallback(&compiled) {
        Ok(t) => t.expr,
        Err(_) => fallback(&compiled, DataType::Boolean, &schema).map_err(|e| e.to_string())?,
    };
    let ctx = SessionContext::new();
    let df = ctx
        .read_batch(batch)
        .map_err(|e| e.to_string())?
        .filter(predicate)
        .map_err(|e| e.to_string())?;
    collect(df).await
}

/// Add/overwrite `out_col` from `expr`, executed via the DataFusion plan (a translated projection,
/// or the per-row fallback). Output column order matches the row-wise executor: overwrite in
/// place, else append.
pub async fn apply_expression_via_datafusion(
    rows: &[Row],
    columns: &[String],
    out_col: &str,
    expr: &str,
) -> Result<Vec<Row>, String> {
    if rows.is_empty() {
        return Ok(vec![]);
    }
    let compiled = compile(expr).map_err(|e| e.to_string())?;
    let batch = rows_to_batch(rows, columns)?;
    let schema = batch.schema();
    let translator = Translator::new(schema.clone());
    let value: DfExpr = match translator.translate_or_fallback(&compiled) {
        Ok(t) => t.expr,
        Err(_) => fallback(&compiled, infer_type(&eval_all(&compiled, rows)?), &schema).map_err(|e| e.to_string())?,
    };

    // The projection: overwrite out_col in place, else append it.
    let mut selects: Vec<DfExpr> = Vec::new();
    let mut replaced = false;
    for c in columns {
        if c == out_col {
            selects.push(value.clone().alias(out_col));
            replaced = true;
        } else {
            selects.push(col(format!("\"{}\"", c.replace('"', "\"\""))));
        }
    }
    if !replaced {
        selects.push(value.alias(out_col));
    }
    let ctx = SessionContext::new();
    let df = ctx
        .read_batch(batch)
        .map_err(|e| e.to_string())?
        .select(selects)
        .map_err(|e| e.to_string())?;
    collect(df).await
}