use arrow::array::{
Array, ArrayRef, BooleanArray, Float32Array, Float64Array, Int32Array, Int64Array, RecordBatch, StringArray,
};
use fv_compute::{ComputeError, FvType, SchemaSpec};
use serde_json::{Map, Value as J};
use std::sync::Arc;
fn schema_err(id: &str, msg: impl Into<String>) -> ComputeError {
ComputeError::Schema {
id: id.to_string(),
msg: msg.into(),
}
}
fn cell_to_json(col: &ArrayRef, i: usize) -> J {
if col.is_null(i) {
return J::Null;
}
use arrow::datatypes::DataType as D;
let a = col.as_any();
match col.data_type() {
D::Boolean => J::Bool(a.downcast_ref::<BooleanArray>().unwrap().value(i)),
D::Int64 => J::from(a.downcast_ref::<Int64Array>().unwrap().value(i)),
D::Int32 => J::from(a.downcast_ref::<Int32Array>().unwrap().value(i)),
D::Float64 => serde_json::Number::from_f64(a.downcast_ref::<Float64Array>().unwrap().value(i))
.map(J::Number)
.unwrap_or(J::Null),
D::Float32 => serde_json::Number::from_f64(a.downcast_ref::<Float32Array>().unwrap().value(i) as f64)
.map(J::Number)
.unwrap_or(J::Null),
D::Utf8 => J::String(a.downcast_ref::<StringArray>().unwrap().value(i).to_string()),
_ => J::Null,
}
}
pub fn batch_to_json_rows(rb: &RecordBatch) -> Vec<J> {
let schema = rb.schema();
(0..rb.num_rows())
.map(|i| {
let mut m = Map::new();
for (c, f) in schema.fields().iter().enumerate() {
m.insert(f.name().clone(), cell_to_json(rb.column(c), i));
}
J::Object(m)
})
.collect()
}
fn build_col(id: &str, dtype: FvType, name: &str, rows: &[J]) -> Result<ArrayRef, ComputeError> {
let get = |r: &J| r.get(name).cloned().unwrap_or(J::Null);
Ok(match dtype {
FvType::Int64 => Arc::new(rows.iter().map(|r| get(r).as_i64()).collect::<Int64Array>()),
FvType::Int32 => Arc::new(
rows.iter()
.map(|r| get(r).as_i64().map(|n| n as i32))
.collect::<Int32Array>(),
),
FvType::Float64 => Arc::new(rows.iter().map(|r| get(r).as_f64()).collect::<Float64Array>()),
FvType::Float32 => Arc::new(
rows.iter()
.map(|r| get(r).as_f64().map(|n| n as f32))
.collect::<Float32Array>(),
),
FvType::Bool => Arc::new(rows.iter().map(|r| get(r).as_bool()).collect::<BooleanArray>()),
FvType::Utf8 => Arc::new(
rows.iter()
.map(|r| get(r).as_str().map(String::from))
.collect::<StringArray>(),
),
other => {
return Err(schema_err(
id,
format!("json protocol: column `{name}` type {other:?} unsupported"),
))
}
})
}
pub fn json_rows_to_batch(id: &str, rows: &[J], out: &SchemaSpec) -> Result<RecordBatch, ComputeError> {
let arrays: Vec<ArrayRef> = out
.columns
.iter()
.map(|c| build_col(id, c.dtype, &c.name, rows))
.collect::<Result<_, _>>()?;
RecordBatch::try_new(out.to_arrow_schema_ref(), arrays).map_err(|e| schema_err(id, e.to_string()))
}
#[cfg(test)]
mod tests {
use super::*;
use arrow::datatypes::{DataType, Field, Schema};
use fv_compute::ColumnSpec;
#[test]
fn batch_to_rows_and_back() {
let schema = Arc::new(Schema::new(vec![
Field::new("id", DataType::Int64, true),
Field::new("name", DataType::Utf8, true),
]));
let rb = RecordBatch::try_new(
schema,
vec![
Arc::new(Int64Array::from(vec![1, 2])),
Arc::new(StringArray::from(vec!["a", "b"])),
],
)
.unwrap();
let rows = batch_to_json_rows(&rb);
assert_eq!(rows[0]["id"], J::from(1));
assert_eq!(rows[1]["name"], J::from("b"));
let out = SchemaSpec {
name: None,
columns: vec![
ColumnSpec {
name: "id".into(),
dtype: FvType::Int64,
},
ColumnSpec {
name: "name".into(),
dtype: FvType::Utf8,
},
],
};
assert_eq!(json_rows_to_batch("t", &rows, &out).unwrap(), rb);
}
}