rs-ipc2ints 0.1.0

Converts the column of the arrow ipc to raw ints.
Documentation
use std::io;

use io::BufWriter;
use io::Write;

use io::BufRead;
use io::Read;

use arrow_schema::DataType;

use arrow_buffer::buffer::Buffer;
use arrow_buffer::buffer::ScalarBuffer;

use arrow_array::RecordBatch;
use arrow_array::array::Array;
use arrow_array::array::ArrayRef;

use arrow_array::array::PrimitiveArray;

use arrow_array::types::ArrowPrimitiveType;

use arrow_array::cast::AsArray;

use arrow_array::array::Int8Array;
use arrow_array::array::Int16Array;
use arrow_array::array::Int32Array;

use arrow_array::array::UInt8Array;
use arrow_array::array::UInt16Array;
use arrow_array::array::UInt32Array;

use arrow_ipc::reader::StreamReader;

pub trait PrimitiveSink {
    fn sink_bytes(&mut self, data: &[u8]) -> Result<(), io::Error>;

    fn sink_array<T>(&mut self, arr: &PrimitiveArray<T>) -> Result<(), io::Error>
    where
        T: ArrowPrimitiveType,
    {
        let sbuf: &ScalarBuffer<T::Native> = arr.values();
        let ibuf: &Buffer = sbuf.inner();
        let data: &[u8] = ibuf.as_slice();
        self.sink_bytes(data)
    }

    fn sink_ints8i(&mut self, arr: &Int8Array) -> Result<(), io::Error> {
        self.sink_array(arr)
    }

    fn sink_ints16i(&mut self, arr: &Int16Array) -> Result<(), io::Error> {
        self.sink_array(arr)
    }

    fn sink_ints32i(&mut self, arr: &Int32Array) -> Result<(), io::Error> {
        self.sink_array(arr)
    }

    fn sink_uints8(&mut self, arr: &UInt8Array) -> Result<(), io::Error> {
        self.sink_array(arr)
    }

    fn sink_uints16(&mut self, arr: &UInt16Array) -> Result<(), io::Error> {
        self.sink_array(arr)
    }

    fn sink_uints32(&mut self, arr: &UInt32Array) -> Result<(), io::Error> {
        self.sink_array(arr)
    }

    fn sink_ref(&mut self, aref: &ArrayRef) -> Result<(), io::Error> {
        let arr: &dyn Array = aref;
        let dtyp: &DataType = arr.data_type();

        match dtyp {
            DataType::Int8 => {
                let oarr: Option<&Int8Array> = aref.as_primitive_opt();
                let arr: &_ = oarr.ok_or(io::Error::other("not int8"))?;
                self.sink_ints8i(arr)
            }
            DataType::Int16 => {
                let oarr: Option<&Int16Array> = aref.as_primitive_opt();
                let arr: &_ = oarr.ok_or(io::Error::other("not int16"))?;
                self.sink_ints16i(arr)
            }
            DataType::Int32 => {
                let oarr: Option<&Int32Array> = aref.as_primitive_opt();
                let arr: &_ = oarr.ok_or(io::Error::other("not int32"))?;
                self.sink_ints32i(arr)
            }
            DataType::UInt8 => {
                let oarr: Option<&UInt8Array> = aref.as_primitive_opt();
                let arr: &_ = oarr.ok_or(io::Error::other("not uint8"))?;
                self.sink_uints8(arr)
            }
            DataType::UInt16 => {
                let oarr: Option<&UInt16Array> = aref.as_primitive_opt();
                let arr: &_ = oarr.ok_or(io::Error::other("not uint16"))?;
                self.sink_uints16(arr)
            }
            DataType::UInt32 => {
                let oarr: Option<&UInt32Array> = aref.as_primitive_opt();
                let arr: &_ = oarr.ok_or(io::Error::other("not uint32"))?;
                self.sink_uints32(arr)
            }
            _ => Err(io::Error::other(format!(
                "unsupported type in sink_ref: {dtyp}"
            ))),
        }
    }

    fn sink_bat(&mut self, bat: &RecordBatch, colix: usize) -> Result<(), io::Error> {
        let aref: &ArrayRef = bat.column(colix);
        self.sink_ref(aref)
    }
}

impl<F> PrimitiveSink for F
where
    F: FnMut(&[u8]) -> Result<(), io::Error>,
{
    fn sink_bytes(&mut self, data: &[u8]) -> Result<(), io::Error> {
        self(data)
    }
}

pub struct IpcReader<R>(pub StreamReader<R>);

impl<R> IpcReader<R>
where
    R: Read,
{
    pub fn into_sink<S>(self, mut sink: S, colix: usize) -> Result<(), io::Error>
    where
        S: PrimitiveSink,
    {
        for rslt in self.0 {
            let rbat: RecordBatch = rslt.map_err(io::Error::other)?;
            sink.sink_bat(&rbat, colix)?;
        }
        Ok(())
    }
}

impl<R> IpcReader<R>
where
    R: Read,
{
    pub fn into_writer<W>(self, mut wtr: W, colix: usize) -> Result<(), io::Error>
    where
        W: Write,
    {
        let wref = &mut wtr;
        let sink = |dat: &[u8]| wref.write_all(dat);
        self.into_sink(sink, colix)?;
        wtr.flush()
    }
}

pub fn reader2writer<R, W>(rdr: R, mut wtr: W, colix4projection: usize) -> Result<(), io::Error>
where
    R: BufRead,
    W: Write,
{
    let srdr: StreamReader<_> =
        StreamReader::try_new(rdr, Some(vec![colix4projection])).map_err(io::Error::other)?;
    IpcReader(srdr).into_writer(BufWriter::new(&mut wtr), 0)?;
    wtr.flush()
}

pub fn stdin2ipc2ints2stdout(colix4projection: usize) -> Result<(), io::Error> {
    reader2writer(io::stdin().lock(), io::stdout().lock(), colix4projection)
}