Skip to main content

rs_parquet2ints/
lib.rs

1use core::any::Any;
2
3use std::io;
4
5use std::fs::File;
6
7use io::BufWriter;
8use io::Write;
9
10use arrow_array::ArrayRef;
11use arrow_array::RecordBatch;
12use arrow_array::array::Int32Array;
13
14use parquet::arrow::ProjectionMask;
15use parquet::arrow::arrow_reader::ParquetRecordBatchReader;
16use parquet::arrow::arrow_reader::ParquetRecordBatchReaderBuilder;
17use parquet::schema::types::SchemaDescriptor;
18
19#[derive(Default, Debug, Clone, Copy)]
20pub enum Endian {
21    #[default]
22    Lit,
23    Big,
24}
25
26impl std::str::FromStr for Endian {
27    type Err = io::Error;
28    fn from_str(s: &str) -> Result<Self, Self::Err> {
29        match s {
30            "l" => Ok(Self::Lit),
31            "lit" => Ok(Self::Lit),
32            "Lit" => Ok(Self::Lit),
33            "little" => Ok(Self::Lit),
34            "Little" => Ok(Self::Lit),
35            "b" => Ok(Self::Big),
36            "big" => Ok(Self::Big),
37            "Big" => Ok(Self::Big),
38            _ => Err(io::Error::other("invalid endian string")),
39        }
40    }
41}
42
43impl Endian {
44    pub fn wtr2int_consumer<W>(&self, mut wtr: W) -> impl FnMut(i32) -> Result<(), io::Error>
45    where
46        W: Write,
47    {
48        let i2b = match self {
49            Self::Lit => |i: i32| i.to_le_bytes(),
50            Self::Big => |i: i32| i.to_be_bytes(),
51        };
52
53        move |i: i32| {
54            let b: [u8; 4] = i2b(i);
55            wtr.write_all(&b)
56        }
57    }
58}
59
60pub struct ProjMask(pub ProjectionMask);
61
62impl ProjMask {
63    pub fn new(colix: usize, sdesc: &SchemaDescriptor) -> Self {
64        Self(ProjectionMask::leaves(sdesc, vec![colix]))
65    }
66}
67
68pub struct ArrowBatchReader(pub ParquetRecordBatchReader);
69
70impl ArrowBatchReader {
71    pub fn into_batches(self) -> impl Iterator<Item = Result<RecordBatch, io::Error>> {
72        self.0.map(|r| r.map_err(io::Error::other))
73    }
74}
75
76pub struct Batch(pub RecordBatch);
77
78impl Batch {
79    pub fn to_sink<S>(&self, sink: &mut S, colix: usize) -> Result<(), io::Error>
80    where
81        S: FnMut(i32) -> Result<(), io::Error>,
82    {
83        let col: &ArrayRef = self.0.column(colix);
84        let acol: &dyn Any = col.as_any();
85        let ocol: Option<&Int32Array> = acol.downcast_ref::<Int32Array>();
86        let Some(i3a) = ocol else {
87            return Err(io::Error::other("not an i32 column"));
88        };
89        let sbuf = i3a.values();
90        let si: &[i32] = sbuf;
91        for i in si {
92            sink(*i)?;
93        }
94        Ok(())
95    }
96}
97
98impl ArrowBatchReader {
99    pub fn into_sink<S>(self, colix: usize, mut sink: S) -> Result<(), io::Error>
100    where
101        S: FnMut(i32) -> Result<(), io::Error>,
102    {
103        let irbat = self.into_batches();
104        for rslt in irbat {
105            let rbat: RecordBatch = rslt?;
106            Batch(rbat).to_sink(&mut sink, colix)?;
107        }
108        Ok(())
109    }
110}
111
112pub struct ParquetFile(pub File);
113
114impl ParquetFile {
115    pub fn into_builder(self) -> Result<ParquetRecordBatchReaderBuilder<File>, io::Error> {
116        ParquetRecordBatchReaderBuilder::try_new(self.0).map_err(io::Error::other)
117    }
118}
119
120pub struct Cfg {
121    pub filename: String,
122    pub columnix: usize,
123    pub batch_sz: usize,
124    pub endian4i: Endian,
125}
126
127impl Cfg {
128    pub fn to_reader(&self) -> Result<ParquetRecordBatchReader, io::Error> {
129        let f: File = File::open(&self.filename)?;
130        let bldr: ParquetRecordBatchReaderBuilder<_> = ParquetFile(f).into_builder()?;
131        let sch: &SchemaDescriptor = bldr.parquet_schema();
132        let msk: ProjMask = ProjMask::new(self.columnix, sch);
133        bldr.with_projection(msk.0)
134            .with_batch_size(self.batch_sz)
135            .build()
136            .map_err(io::Error::other)
137    }
138}
139
140impl Cfg {
141    pub fn pfile2brdr2sink<S>(&self, sink: S) -> Result<(), io::Error>
142    where
143        S: FnMut(i32) -> Result<(), io::Error>,
144    {
145        let rdr: ParquetRecordBatchReader = self.to_reader()?;
146        let abr = ArrowBatchReader(rdr);
147        abr.into_sink(0, sink)
148    }
149}
150
151impl Cfg {
152    pub fn pfile2brdr2wtr<W>(&self, mut wtr: W) -> Result<(), io::Error>
153    where
154        W: Write,
155    {
156        let sink = self.endian4i.wtr2int_consumer(&mut wtr);
157        self.pfile2brdr2sink(sink)?;
158        wtr.flush()
159    }
160}
161
162impl Cfg {
163    pub fn pfile2brdr2stdout(&self) -> Result<(), io::Error> {
164        let o = io::stdout();
165        let mut ol = o.lock();
166        self.pfile2brdr2wtr(BufWriter::new(&mut ol))?;
167        ol.flush()
168    }
169}