Skip to main content

rs_ipc2ints/
lib.rs

1use std::io;
2
3use io::BufWriter;
4use io::Write;
5
6use io::BufRead;
7use io::Read;
8
9use arrow_schema::DataType;
10
11use arrow_buffer::buffer::Buffer;
12use arrow_buffer::buffer::ScalarBuffer;
13
14use arrow_array::RecordBatch;
15use arrow_array::array::Array;
16use arrow_array::array::ArrayRef;
17
18use arrow_array::array::PrimitiveArray;
19
20use arrow_array::types::ArrowPrimitiveType;
21
22use arrow_array::cast::AsArray;
23
24use arrow_array::array::Int8Array;
25use arrow_array::array::Int16Array;
26use arrow_array::array::Int32Array;
27
28use arrow_array::array::UInt8Array;
29use arrow_array::array::UInt16Array;
30use arrow_array::array::UInt32Array;
31
32use arrow_ipc::reader::StreamReader;
33
34pub trait PrimitiveSink {
35    fn sink_bytes(&mut self, data: &[u8]) -> Result<(), io::Error>;
36
37    fn sink_array<T>(&mut self, arr: &PrimitiveArray<T>) -> Result<(), io::Error>
38    where
39        T: ArrowPrimitiveType,
40    {
41        let sbuf: &ScalarBuffer<T::Native> = arr.values();
42        let ibuf: &Buffer = sbuf.inner();
43        let data: &[u8] = ibuf.as_slice();
44        self.sink_bytes(data)
45    }
46
47    fn sink_ints8i(&mut self, arr: &Int8Array) -> Result<(), io::Error> {
48        self.sink_array(arr)
49    }
50
51    fn sink_ints16i(&mut self, arr: &Int16Array) -> Result<(), io::Error> {
52        self.sink_array(arr)
53    }
54
55    fn sink_ints32i(&mut self, arr: &Int32Array) -> Result<(), io::Error> {
56        self.sink_array(arr)
57    }
58
59    fn sink_uints8(&mut self, arr: &UInt8Array) -> Result<(), io::Error> {
60        self.sink_array(arr)
61    }
62
63    fn sink_uints16(&mut self, arr: &UInt16Array) -> Result<(), io::Error> {
64        self.sink_array(arr)
65    }
66
67    fn sink_uints32(&mut self, arr: &UInt32Array) -> Result<(), io::Error> {
68        self.sink_array(arr)
69    }
70
71    fn sink_ref(&mut self, aref: &ArrayRef) -> Result<(), io::Error> {
72        let arr: &dyn Array = aref;
73        let dtyp: &DataType = arr.data_type();
74
75        match dtyp {
76            DataType::Int8 => {
77                let oarr: Option<&Int8Array> = aref.as_primitive_opt();
78                let arr: &_ = oarr.ok_or(io::Error::other("not int8"))?;
79                self.sink_ints8i(arr)
80            }
81            DataType::Int16 => {
82                let oarr: Option<&Int16Array> = aref.as_primitive_opt();
83                let arr: &_ = oarr.ok_or(io::Error::other("not int16"))?;
84                self.sink_ints16i(arr)
85            }
86            DataType::Int32 => {
87                let oarr: Option<&Int32Array> = aref.as_primitive_opt();
88                let arr: &_ = oarr.ok_or(io::Error::other("not int32"))?;
89                self.sink_ints32i(arr)
90            }
91            DataType::UInt8 => {
92                let oarr: Option<&UInt8Array> = aref.as_primitive_opt();
93                let arr: &_ = oarr.ok_or(io::Error::other("not uint8"))?;
94                self.sink_uints8(arr)
95            }
96            DataType::UInt16 => {
97                let oarr: Option<&UInt16Array> = aref.as_primitive_opt();
98                let arr: &_ = oarr.ok_or(io::Error::other("not uint16"))?;
99                self.sink_uints16(arr)
100            }
101            DataType::UInt32 => {
102                let oarr: Option<&UInt32Array> = aref.as_primitive_opt();
103                let arr: &_ = oarr.ok_or(io::Error::other("not uint32"))?;
104                self.sink_uints32(arr)
105            }
106            _ => Err(io::Error::other(format!(
107                "unsupported type in sink_ref: {dtyp}"
108            ))),
109        }
110    }
111
112    fn sink_bat(&mut self, bat: &RecordBatch, colix: usize) -> Result<(), io::Error> {
113        let aref: &ArrayRef = bat.column(colix);
114        self.sink_ref(aref)
115    }
116}
117
118impl<F> PrimitiveSink for F
119where
120    F: FnMut(&[u8]) -> Result<(), io::Error>,
121{
122    fn sink_bytes(&mut self, data: &[u8]) -> Result<(), io::Error> {
123        self(data)
124    }
125}
126
127pub struct IpcReader<R>(pub StreamReader<R>);
128
129impl<R> IpcReader<R>
130where
131    R: Read,
132{
133    pub fn into_sink<S>(self, mut sink: S, colix: usize) -> Result<(), io::Error>
134    where
135        S: PrimitiveSink,
136    {
137        for rslt in self.0 {
138            let rbat: RecordBatch = rslt.map_err(io::Error::other)?;
139            sink.sink_bat(&rbat, colix)?;
140        }
141        Ok(())
142    }
143}
144
145impl<R> IpcReader<R>
146where
147    R: Read,
148{
149    pub fn into_writer<W>(self, mut wtr: W, colix: usize) -> Result<(), io::Error>
150    where
151        W: Write,
152    {
153        let wref = &mut wtr;
154        let sink = |dat: &[u8]| wref.write_all(dat);
155        self.into_sink(sink, colix)?;
156        wtr.flush()
157    }
158}
159
160pub fn reader2writer<R, W>(rdr: R, mut wtr: W, colix4projection: usize) -> Result<(), io::Error>
161where
162    R: BufRead,
163    W: Write,
164{
165    let srdr: StreamReader<_> =
166        StreamReader::try_new(rdr, Some(vec![colix4projection])).map_err(io::Error::other)?;
167    IpcReader(srdr).into_writer(BufWriter::new(&mut wtr), 0)?;
168    wtr.flush()
169}
170
171pub fn stdin2ipc2ints2stdout(colix4projection: usize) -> Result<(), io::Error> {
172    reader2writer(io::stdin().lock(), io::stdout().lock(), colix4projection)
173}