fv-compute-container 0.2.0

The container backend for fv-compute: a transform is a process or image that reads Arrow IPC (or row JSON) on stdin and writes it on stdout — for heavy, GPU or arbitrary-dependency transforms.
Documentation
//! Arrow IPC stream <-> bytes — the container boundary "Arrow handoff": whole
//! `RecordBatch`es cross as Arrow IPC (columnar, zero-copy framing), NOT row-JSON. Any language with
//! Arrow (Python+pyarrow, C++, Go, …) reads/writes this over stdio.

use arrow::array::RecordBatch;
use arrow::datatypes::SchemaRef;
use fv_compute::ComputeError;

fn err(id: &str, e: impl std::fmt::Display) -> ComputeError {
    ComputeError::Run {
        id: id.to_string(),
        msg: e.to_string(),
    }
}

/// Serialize batches as one Arrow IPC stream (schema header + record batches + EOS).
pub fn write_ipc(id: &str, schema: &SchemaRef, batches: &[RecordBatch]) -> Result<Vec<u8>, ComputeError> {
    let mut buf = Vec::new();
    {
        let mut w = arrow::ipc::writer::StreamWriter::try_new(&mut buf, schema).map_err(|e| err(id, e))?;
        for b in batches {
            w.write(b).map_err(|e| err(id, e))?;
        }
        w.finish().map_err(|e| err(id, e))?;
    }
    Ok(buf)
}

/// Read an Arrow IPC stream back into batches.
pub fn read_ipc(id: &str, bytes: &[u8]) -> Result<Vec<RecordBatch>, ComputeError> {
    let reader =
        arrow::ipc::reader::StreamReader::try_new(std::io::Cursor::new(bytes), None).map_err(|e| err(id, e))?;
    reader.collect::<Result<Vec<_>, _>>().map_err(|e| err(id, e))
}

#[cfg(test)]
mod tests {
    use super::*;
    use arrow::array::{Float64Array, Int64Array};
    use arrow::datatypes::{DataType, Field, Schema};
    use std::sync::Arc;

    #[test]
    fn ipc_roundtrips_a_batch() {
        let schema: SchemaRef = Arc::new(Schema::new(vec![
            Field::new("id", DataType::Int64, true),
            Field::new("amount", DataType::Float64, true),
        ]));
        let rb = RecordBatch::try_new(
            schema.clone(),
            vec![
                Arc::new(Int64Array::from(vec![1, 2, 3])),
                Arc::new(Float64Array::from(vec![Some(1.5), None, Some(3.0)])),
            ],
        )
        .unwrap();
        let bytes = write_ipc("t", &schema, std::slice::from_ref(&rb)).unwrap();
        let back = read_ipc("t", &bytes).unwrap();
        assert_eq!(back.len(), 1);
        assert_eq!(back[0], rb, "IPC must preserve schema + values (incl nulls)");
    }
}