fv-plan 0.4.0

The FusionVault transform executor on Apache DataFusion: inline row steps and relational reshapes (join, aggregate, window, union) over Arrow, with the value dialect translated into the plan, spatial SQL, file sources, and build orchestration checked against the published conformance vectors.
Documentation
//! Row ⇄ Arrow conversion for the DataFusion execution path (reshapes). Column types are inferred
//! from the values (real pipelines will pass declared schemas later); the value dialect stays dynamic.

use std::sync::Arc;

use datafusion::arrow::array::{Array, BooleanArray, Float64Array, StringArray};
use datafusion::arrow::datatypes::{DataType, Field, Schema};
use datafusion::arrow::record_batch::RecordBatch;
use fv_value::Value;

use crate::row::Row;

/// Infer the Arrow type for a column across rows: numbers → Float64, booleans → Boolean, anything
/// with strings or mixed/complex values → Utf8 (JSON-encoded), all-null → Float64 (round-trips Null).
pub fn infer_type(rows: &[Row], col: &str) -> DataType {
    infer(rows, col)
}

fn infer(rows: &[Row], col: &str) -> DataType {
    let (mut num, mut boolean, mut string, mut other) = (false, false, false, false);
    for r in rows {
        match r.get(col) {
            Value::Null => {}
            Value::Num(_) => num = true,
            Value::Bool(_) => boolean = true,
            Value::Str(_) => string = true,
            _ => other = true,
        }
    }
    if other || string {
        DataType::Utf8
    } else if boolean && !num {
        DataType::Boolean
    } else {
        DataType::Float64
    }
}

fn scalar_to_string(v: &Value) -> Option<String> {
    match v {
        Value::Null => None,
        Value::Str(s) => Some(s.clone()),
        Value::Bool(b) => Some(if *b { "true" } else { "false" }.to_string()),
        Value::Num(n) => Some(if n.fract() == 0.0 && n.is_finite() {
            format!("{}", *n as i64)
        } else {
            format!("{n}")
        }),
        // lists / objects → a stable JSON-ish encoding so grouping/joining on them is well-defined
        other => Some(format!("{other:?}")),
    }
}

/// Build a RecordBatch from rows in the given column order (types inferred).
pub fn rows_to_batch(rows: &[Row], columns: &[String]) -> Result<RecordBatch, String> {
    let mut fields = Vec::with_capacity(columns.len());
    let mut arrays: Vec<Arc<dyn Array>> = Vec::with_capacity(columns.len());
    for col in columns {
        let dt = infer(rows, col);
        fields.push(Field::new(col, dt.clone(), true));
        let arr: Arc<dyn Array> = match dt {
            DataType::Float64 => Arc::new(Float64Array::from(
                rows.iter().map(|r| r.get(col).as_num()).collect::<Vec<_>>(),
            )),
            DataType::Boolean => Arc::new(BooleanArray::from(
                rows.iter()
                    .map(|r| match r.get(col) {
                        Value::Bool(b) => Some(b),
                        _ => None,
                    })
                    .collect::<Vec<_>>(),
            )),
            _ => Arc::new(StringArray::from(
                rows.iter().map(|r| scalar_to_string(&r.get(col))).collect::<Vec<_>>(),
            )),
        };
        arrays.push(arr);
    }
    let schema = Arc::new(Schema::new(fields));
    RecordBatch::try_new(schema, arrays).map_err(|e| e.to_string())
}

/// Read a RecordBatch back into ordered rows (schema field order preserved), through
/// fv-value-datafusion's bridge: every number width becomes a number, every string flavour a
/// string, a dictionary its values, lists and structs lists and objects. A column whose type has no
/// value in the dialect (binary, decimal, temporal) is an error naming it — never a null it is not.
pub fn batch_to_rows(batch: &RecordBatch) -> Result<Vec<Row>, String> {
    let schema = batch.schema();
    let mut rows = vec![Row::new(); batch.num_rows()];
    for (c, field) in schema.fields().iter().enumerate() {
        let name = field.name();
        let cells = fv_value_datafusion::Cells::of(batch.column(c)).map_err(|e| format!("column '{name}': {e}"))?;
        for (i, row) in rows.iter_mut().enumerate() {
            row.0.push((name.clone(), cells.value(i)));
        }
    }
    Ok(rows)
}

/// Every row of `batches`, in order (see [`batch_to_rows`]).
pub fn batches_to_rows(batches: &[RecordBatch]) -> Result<Vec<Row>, String> {
    let mut rows = Vec::new();
    for b in batches {
        rows.extend(batch_to_rows(b)?);
    }
    Ok(rows)
}

#[cfg(test)]
mod tests {
    use datafusion::arrow::array::{ListArray, StructArray};

    /// One cell through the bridge `batch_to_rows` uses.
    fn array_value(col: &std::sync::Arc<dyn Array>, i: usize) -> Value {
        fv_value_datafusion::cell_to_value(col, i).unwrap()
    }

    use super::*;
    use datafusion::arrow::array::ArrayRef;
    use datafusion::arrow::datatypes::{Field, Float64Type};

    fn num(n: f64) -> Value {
        Value::Num(n)
    }

    #[test]
    fn decodes_a_list_column_to_value_list() {
        let list: ArrayRef = Arc::new(ListArray::from_iter_primitive::<Float64Type, _, _>(vec![
            Some(vec![Some(1.0), Some(2.0)]),
            Some(vec![Some(3.0)]),
        ]));
        assert_eq!(array_value(&list, 0), Value::List(vec![num(1.0), num(2.0)]));
        assert_eq!(array_value(&list, 1), Value::List(vec![num(3.0)]));
    }

    #[test]
    fn decodes_a_struct_column_to_value_obj() {
        let ty: ArrayRef = Arc::new(StringArray::from(vec!["Polygon"]));
        let coords: ArrayRef = Arc::new(ListArray::from_iter_primitive::<Float64Type, _, _>(vec![Some(vec![
            Some(1.0),
            Some(2.0),
        ])]));
        let s: ArrayRef = Arc::new(StructArray::from(vec![
            (Arc::new(Field::new("type", DataType::Utf8, true)), ty),
            (
                Arc::new(Field::new("coordinates", coords.data_type().clone(), true)),
                coords,
            ),
        ]));
        let got = array_value(&s, 0);
        let want = Value::Obj(
            [
                ("type".to_string(), Value::Str("Polygon".to_string())),
                ("coordinates".to_string(), Value::List(vec![num(1.0), num(2.0)])),
            ]
            .into_iter()
            .collect(),
        );
        assert_eq!(got, want);
    }

    #[test]
    fn decodes_nested_lists_of_lists() {
        // A GeoJSON-shaped List<List<Float64>> (e.g. a ring of positions).
        let inner = ListArray::from_iter_primitive::<Float64Type, _, _>(vec![
            Some(vec![Some(1.0), Some(2.0)]),
            Some(vec![Some(3.0), Some(4.0)]),
        ]);
        // Wrap the two inner lists into one outer list element.
        use datafusion::arrow::buffer::OffsetBuffer;
        let field = Arc::new(Field::new("item", inner.data_type().clone(), true));
        let outer: ArrayRef = Arc::new(ListArray::new(
            field,
            OffsetBuffer::new(vec![0, 2].into()),
            Arc::new(inner),
            None,
        ));
        assert_eq!(
            array_value(&outer, 0),
            Value::List(vec![
                Value::List(vec![num(1.0), num(2.0)]),
                Value::List(vec![num(3.0), num(4.0)]),
            ])
        );
    }

    #[test]
    fn a_null_nested_cell_reads_null() {
        let list: ArrayRef = Arc::new(ListArray::from_iter_primitive::<Float64Type, _, _>(vec![
            Some(vec![Some(1.0)]),
            None,
        ]));
        assert_eq!(array_value(&list, 1), Value::Null);
    }
}