use std::sync::Arc;
use datafusion::arrow::array::{Array, BooleanArray, Float64Array, StringArray};
use datafusion::arrow::datatypes::{DataType, Field, Schema};
use datafusion::arrow::record_batch::RecordBatch;
use fv_value::Value;
use crate::row::Row;
pub fn infer_type(rows: &[Row], col: &str) -> DataType {
infer(rows, col)
}
fn infer(rows: &[Row], col: &str) -> DataType {
let (mut num, mut boolean, mut string, mut other) = (false, false, false, false);
for r in rows {
match r.get(col) {
Value::Null => {}
Value::Num(_) => num = true,
Value::Bool(_) => boolean = true,
Value::Str(_) => string = true,
_ => other = true,
}
}
if other || string {
DataType::Utf8
} else if boolean && !num {
DataType::Boolean
} else {
DataType::Float64
}
}
fn scalar_to_string(v: &Value) -> Option<String> {
match v {
Value::Null => None,
Value::Str(s) => Some(s.clone()),
Value::Bool(b) => Some(if *b { "true" } else { "false" }.to_string()),
Value::Num(n) => Some(if n.fract() == 0.0 && n.is_finite() {
format!("{}", *n as i64)
} else {
format!("{n}")
}),
other => Some(format!("{other:?}")),
}
}
pub fn rows_to_batch(rows: &[Row], columns: &[String]) -> Result<RecordBatch, String> {
let mut fields = Vec::with_capacity(columns.len());
let mut arrays: Vec<Arc<dyn Array>> = Vec::with_capacity(columns.len());
for col in columns {
let dt = infer(rows, col);
fields.push(Field::new(col, dt.clone(), true));
let arr: Arc<dyn Array> = match dt {
DataType::Float64 => Arc::new(Float64Array::from(
rows.iter().map(|r| r.get(col).as_num()).collect::<Vec<_>>(),
)),
DataType::Boolean => Arc::new(BooleanArray::from(
rows.iter()
.map(|r| match r.get(col) {
Value::Bool(b) => Some(b),
_ => None,
})
.collect::<Vec<_>>(),
)),
_ => Arc::new(StringArray::from(
rows.iter().map(|r| scalar_to_string(&r.get(col))).collect::<Vec<_>>(),
)),
};
arrays.push(arr);
}
let schema = Arc::new(Schema::new(fields));
RecordBatch::try_new(schema, arrays).map_err(|e| e.to_string())
}
pub fn batch_to_rows(batch: &RecordBatch) -> Result<Vec<Row>, String> {
let schema = batch.schema();
let mut rows = vec![Row::new(); batch.num_rows()];
for (c, field) in schema.fields().iter().enumerate() {
let name = field.name();
let cells = fv_value_datafusion::Cells::of(batch.column(c)).map_err(|e| format!("column '{name}': {e}"))?;
for (i, row) in rows.iter_mut().enumerate() {
row.0.push((name.clone(), cells.value(i)));
}
}
Ok(rows)
}
pub fn batches_to_rows(batches: &[RecordBatch]) -> Result<Vec<Row>, String> {
let mut rows = Vec::new();
for b in batches {
rows.extend(batch_to_rows(b)?);
}
Ok(rows)
}
#[cfg(test)]
mod tests {
use datafusion::arrow::array::{ListArray, StructArray};
fn array_value(col: &std::sync::Arc<dyn Array>, i: usize) -> Value {
fv_value_datafusion::cell_to_value(col, i).unwrap()
}
use super::*;
use datafusion::arrow::array::ArrayRef;
use datafusion::arrow::datatypes::{Field, Float64Type};
fn num(n: f64) -> Value {
Value::Num(n)
}
#[test]
fn decodes_a_list_column_to_value_list() {
let list: ArrayRef = Arc::new(ListArray::from_iter_primitive::<Float64Type, _, _>(vec![
Some(vec![Some(1.0), Some(2.0)]),
Some(vec![Some(3.0)]),
]));
assert_eq!(array_value(&list, 0), Value::List(vec![num(1.0), num(2.0)]));
assert_eq!(array_value(&list, 1), Value::List(vec![num(3.0)]));
}
#[test]
fn decodes_a_struct_column_to_value_obj() {
let ty: ArrayRef = Arc::new(StringArray::from(vec!["Polygon"]));
let coords: ArrayRef = Arc::new(ListArray::from_iter_primitive::<Float64Type, _, _>(vec![Some(vec![
Some(1.0),
Some(2.0),
])]));
let s: ArrayRef = Arc::new(StructArray::from(vec![
(Arc::new(Field::new("type", DataType::Utf8, true)), ty),
(
Arc::new(Field::new("coordinates", coords.data_type().clone(), true)),
coords,
),
]));
let got = array_value(&s, 0);
let want = Value::Obj(
[
("type".to_string(), Value::Str("Polygon".to_string())),
("coordinates".to_string(), Value::List(vec![num(1.0), num(2.0)])),
]
.into_iter()
.collect(),
);
assert_eq!(got, want);
}
#[test]
fn decodes_nested_lists_of_lists() {
let inner = ListArray::from_iter_primitive::<Float64Type, _, _>(vec![
Some(vec![Some(1.0), Some(2.0)]),
Some(vec![Some(3.0), Some(4.0)]),
]);
use datafusion::arrow::buffer::OffsetBuffer;
let field = Arc::new(Field::new("item", inner.data_type().clone(), true));
let outer: ArrayRef = Arc::new(ListArray::new(
field,
OffsetBuffer::new(vec![0, 2].into()),
Arc::new(inner),
None,
));
assert_eq!(
array_value(&outer, 0),
Value::List(vec![
Value::List(vec![num(1.0), num(2.0)]),
Value::List(vec![num(3.0), num(4.0)]),
])
);
}
#[test]
fn a_null_nested_cell_reads_null() {
let list: ArrayRef = Arc::new(ListArray::from_iter_primitive::<Float64Type, _, _>(vec![
Some(vec![Some(1.0)]),
None,
]));
assert_eq!(array_value(&list, 1), Value::Null);
}
}