use arrow::array::RecordBatch;
use arrow::datatypes::SchemaRef;
use fv_compute::ComputeError;
fn err(id: &str, e: impl std::fmt::Display) -> ComputeError {
ComputeError::Run {
id: id.to_string(),
msg: e.to_string(),
}
}
pub fn write_ipc(id: &str, schema: &SchemaRef, batches: &[RecordBatch]) -> Result<Vec<u8>, ComputeError> {
let mut buf = Vec::new();
{
let mut w = arrow::ipc::writer::StreamWriter::try_new(&mut buf, schema).map_err(|e| err(id, e))?;
for b in batches {
w.write(b).map_err(|e| err(id, e))?;
}
w.finish().map_err(|e| err(id, e))?;
}
Ok(buf)
}
pub fn read_ipc(id: &str, bytes: &[u8]) -> Result<Vec<RecordBatch>, ComputeError> {
let reader =
arrow::ipc::reader::StreamReader::try_new(std::io::Cursor::new(bytes), None).map_err(|e| err(id, e))?;
reader.collect::<Result<Vec<_>, _>>().map_err(|e| err(id, e))
}
#[cfg(test)]
mod tests {
use super::*;
use arrow::array::{Float64Array, Int64Array};
use arrow::datatypes::{DataType, Field, Schema};
use std::sync::Arc;
#[test]
fn ipc_roundtrips_a_batch() {
let schema: SchemaRef = Arc::new(Schema::new(vec![
Field::new("id", DataType::Int64, true),
Field::new("amount", DataType::Float64, true),
]));
let rb = RecordBatch::try_new(
schema.clone(),
vec![
Arc::new(Int64Array::from(vec![1, 2, 3])),
Arc::new(Float64Array::from(vec![Some(1.5), None, Some(3.0)])),
],
)
.unwrap();
let bytes = write_ipc("t", &schema, std::slice::from_ref(&rb)).unwrap();
let back = read_ipc("t", &bytes).unwrap();
assert_eq!(back.len(), 1);
assert_eq!(back[0], rb, "IPC must preserve schema + values (incl nulls)");
}
}