Skip to main content

rs_ints2arrow/
sync.rs

1#[cfg(feature = "fs")]
2pub mod fs;
3
4use std::io;
5
6use io::Read;
7
8use arrow::datatypes::ArrowPrimitiveType;
9
10use arrow::datatypes::Int8Type;
11use arrow::datatypes::Int16Type;
12use arrow::datatypes::Int32Type;
13use arrow::datatypes::Int64Type;
14
15use arrow::datatypes::UInt8Type;
16use arrow::datatypes::UInt16Type;
17use arrow::datatypes::UInt32Type;
18use arrow::datatypes::UInt64Type;
19
20use arrow::array::PrimitiveArray;
21use arrow::array::PrimitiveBuilder;
22
23macro_rules! ints2arrow {
24    ($fname: ident, $ity: ty) => {
25        /// Converts the iterator of integers to an array of integers.
26        pub fn $fname<I>(ints: I) -> Result<PrimitiveArray<$ity>, io::Error>
27        where
28            I: Iterator<Item = Result<<$ity as ArrowPrimitiveType>::Native, io::Error>>,
29        {
30            let mut bldr = PrimitiveBuilder::new();
31            for ri in ints {
32                let i = ri?;
33                bldr.append_value(i);
34            }
35            Ok(bldr.finish())
36        }
37    };
38}
39
40ints2arrow!(ints2arrow8, Int8Type);
41ints2arrow!(ints2arrow16, Int16Type);
42ints2arrow!(ints2arrow32, Int32Type);
43ints2arrow!(ints2arrow64, Int64Type);
44
45ints2arrow!(uints2arrow8, UInt8Type);
46ints2arrow!(uints2arrow16, UInt16Type);
47ints2arrow!(uints2arrow32, UInt32Type);
48ints2arrow!(uints2arrow64, UInt64Type);
49
50pub fn raw2ints_rtfn<R, T, F, const N: usize>(
51    mut rdr: R,
52    buf2t: F,
53) -> impl Iterator<Item = Result<T, io::Error>>
54where
55    R: Read,
56    F: Fn([u8; N]) -> T,
57{
58    let mut buf: [u8; N] = [0; N];
59    std::iter::from_fn(move || {
60        let rslt = rdr.read_exact(&mut buf);
61        match rslt {
62            Ok(_) => Some(Ok(buf2t(buf))),
63            Err(e) => match e.kind() {
64                io::ErrorKind::UnexpectedEof => None,
65                _ => Some(Err(e)),
66            },
67        }
68    })
69}
70
71macro_rules! raw2ints {
72    ($fname: ident, $ity: ty, $buf2i: expr, $bufsz: literal) => {
73        /// Converts the reader to an iterator of integers.
74        pub fn $fname<R>(rdr: R) -> impl Iterator<Item = Result<$ity, io::Error>>
75        where
76            R: Read,
77        {
78            raw2ints_rtfn(rdr, $buf2i)
79        }
80    };
81}
82
83raw2ints!(raw2ints8, i8, |buf: [u8; 1]| buf[0] as i8, 1);
84raw2ints!(raw2uints8, u8, |buf: [u8; 1]| buf[0], 1);
85
86raw2ints!(raw2ints16le, i16, i16::from_le_bytes, 2);
87raw2ints!(raw2ints32le, i32, i32::from_le_bytes, 4);
88raw2ints!(raw2ints64le, i64, i64::from_le_bytes, 8);
89
90raw2ints!(raw2uints16le, u16, u16::from_le_bytes, 2);
91raw2ints!(raw2uints32le, u32, u32::from_le_bytes, 4);
92raw2ints!(raw2uints64le, u64, u64::from_le_bytes, 8);
93
94raw2ints!(raw2ints16be, i16, i16::from_be_bytes, 2);
95raw2ints!(raw2ints32be, i32, i32::from_be_bytes, 4);
96raw2ints!(raw2ints64be, i64, i64::from_be_bytes, 8);
97
98raw2ints!(raw2uints16be, u16, u16::from_be_bytes, 2);
99raw2ints!(raw2uints32be, u32, u32::from_be_bytes, 4);
100raw2ints!(raw2uints64be, u64, u64::from_be_bytes, 8);
101
102macro_rules! raw2ints2arrow {
103    ($fname: ident, $rdr2i: ident, $i2arr: ident, $aty: ty) => {
104        /// Converts the reader to an array of integers.
105        pub fn $fname<R>(rdr: R) -> Result<PrimitiveArray<$aty>, io::Error>
106        where
107            R: Read,
108        {
109            let i = $rdr2i(rdr);
110            $i2arr(i)
111        }
112    };
113}
114
115raw2ints2arrow!(raw2ints2arrow8, raw2ints8, ints2arrow8, Int8Type);
116raw2ints2arrow!(raw2uints2arrow8, raw2uints8, uints2arrow8, UInt8Type);
117
118raw2ints2arrow!(raw2ints2arrow16le, raw2ints16le, ints2arrow16, Int16Type);
119raw2ints2arrow!(raw2ints2arrow32le, raw2ints32le, ints2arrow32, Int32Type);
120raw2ints2arrow!(raw2ints2arrow64le, raw2ints64le, ints2arrow64, Int64Type);
121raw2ints2arrow!(raw2ints2arrow16be, raw2ints16be, ints2arrow16, Int16Type);
122raw2ints2arrow!(raw2ints2arrow32be, raw2ints32be, ints2arrow32, Int32Type);
123raw2ints2arrow!(raw2ints2arrow64be, raw2ints64be, ints2arrow64, Int64Type);
124
125raw2ints2arrow!(
126    raw2uints2arrow16le,
127    raw2uints16le,
128    uints2arrow16,
129    UInt16Type
130);
131raw2ints2arrow!(
132    raw2uints2arrow32le,
133    raw2uints32le,
134    uints2arrow32,
135    UInt32Type
136);
137raw2ints2arrow!(
138    raw2uints2arrow64le,
139    raw2uints64le,
140    uints2arrow64,
141    UInt64Type
142);
143raw2ints2arrow!(
144    raw2uints2arrow16be,
145    raw2uints16be,
146    uints2arrow16,
147    UInt16Type
148);
149raw2ints2arrow!(
150    raw2uints2arrow32be,
151    raw2uints32be,
152    uints2arrow32,
153    UInt32Type
154);
155raw2ints2arrow!(
156    raw2uints2arrow64be,
157    raw2uints64be,
158    uints2arrow64,
159    UInt64Type
160);