1use core::any::Any;
2
3use std::io;
4
5use std::fs::File;
6
7use io::BufWriter;
8use io::Write;
9
10use arrow_array::ArrayRef;
11use arrow_array::RecordBatch;
12use arrow_array::array::Int32Array;
13
14use parquet::arrow::ProjectionMask;
15use parquet::arrow::arrow_reader::ParquetRecordBatchReader;
16use parquet::arrow::arrow_reader::ParquetRecordBatchReaderBuilder;
17use parquet::schema::types::SchemaDescriptor;
18
19#[derive(Default, Debug, Clone, Copy)]
20pub enum Endian {
21 #[default]
22 Lit,
23 Big,
24}
25
26impl std::str::FromStr for Endian {
27 type Err = io::Error;
28 fn from_str(s: &str) -> Result<Self, Self::Err> {
29 match s {
30 "l" => Ok(Self::Lit),
31 "lit" => Ok(Self::Lit),
32 "Lit" => Ok(Self::Lit),
33 "little" => Ok(Self::Lit),
34 "Little" => Ok(Self::Lit),
35 "b" => Ok(Self::Big),
36 "big" => Ok(Self::Big),
37 "Big" => Ok(Self::Big),
38 _ => Err(io::Error::other("invalid endian string")),
39 }
40 }
41}
42
43impl Endian {
44 pub fn wtr2int_consumer<W>(&self, mut wtr: W) -> impl FnMut(i32) -> Result<(), io::Error>
45 where
46 W: Write,
47 {
48 let i2b = match self {
49 Self::Lit => |i: i32| i.to_le_bytes(),
50 Self::Big => |i: i32| i.to_be_bytes(),
51 };
52
53 move |i: i32| {
54 let b: [u8; 4] = i2b(i);
55 wtr.write_all(&b)
56 }
57 }
58}
59
60pub struct ProjMask(pub ProjectionMask);
61
62impl ProjMask {
63 pub fn new(colix: usize, sdesc: &SchemaDescriptor) -> Self {
64 Self(ProjectionMask::leaves(sdesc, vec![colix]))
65 }
66}
67
68pub struct ArrowBatchReader(pub ParquetRecordBatchReader);
69
70impl ArrowBatchReader {
71 pub fn into_batches(self) -> impl Iterator<Item = Result<RecordBatch, io::Error>> {
72 self.0.map(|r| r.map_err(io::Error::other))
73 }
74}
75
76pub struct Batch(pub RecordBatch);
77
78impl Batch {
79 pub fn to_sink<S>(&self, sink: &mut S, colix: usize) -> Result<(), io::Error>
80 where
81 S: FnMut(i32) -> Result<(), io::Error>,
82 {
83 let col: &ArrayRef = self.0.column(colix);
84 let acol: &dyn Any = col.as_any();
85 let ocol: Option<&Int32Array> = acol.downcast_ref::<Int32Array>();
86 let Some(i3a) = ocol else {
87 return Err(io::Error::other("not an i32 column"));
88 };
89 let sbuf = i3a.values();
90 let si: &[i32] = sbuf;
91 for i in si {
92 sink(*i)?;
93 }
94 Ok(())
95 }
96}
97
98impl ArrowBatchReader {
99 pub fn into_sink<S>(self, colix: usize, mut sink: S) -> Result<(), io::Error>
100 where
101 S: FnMut(i32) -> Result<(), io::Error>,
102 {
103 let irbat = self.into_batches();
104 for rslt in irbat {
105 let rbat: RecordBatch = rslt?;
106 Batch(rbat).to_sink(&mut sink, colix)?;
107 }
108 Ok(())
109 }
110}
111
112pub struct ParquetFile(pub File);
113
114impl ParquetFile {
115 pub fn into_builder(self) -> Result<ParquetRecordBatchReaderBuilder<File>, io::Error> {
116 ParquetRecordBatchReaderBuilder::try_new(self.0).map_err(io::Error::other)
117 }
118}
119
120pub struct Cfg {
121 pub filename: String,
122 pub columnix: usize,
123 pub batch_sz: usize,
124 pub endian4i: Endian,
125}
126
127impl Cfg {
128 pub fn to_reader(&self) -> Result<ParquetRecordBatchReader, io::Error> {
129 let f: File = File::open(&self.filename)?;
130 let bldr: ParquetRecordBatchReaderBuilder<_> = ParquetFile(f).into_builder()?;
131 let sch: &SchemaDescriptor = bldr.parquet_schema();
132 let msk: ProjMask = ProjMask::new(self.columnix, sch);
133 bldr.with_projection(msk.0)
134 .with_batch_size(self.batch_sz)
135 .build()
136 .map_err(io::Error::other)
137 }
138}
139
140impl Cfg {
141 pub fn pfile2brdr2sink<S>(&self, sink: S) -> Result<(), io::Error>
142 where
143 S: FnMut(i32) -> Result<(), io::Error>,
144 {
145 let rdr: ParquetRecordBatchReader = self.to_reader()?;
146 let abr = ArrowBatchReader(rdr);
147 abr.into_sink(0, sink)
148 }
149}
150
151impl Cfg {
152 pub fn pfile2brdr2wtr<W>(&self, mut wtr: W) -> Result<(), io::Error>
153 where
154 W: Write,
155 {
156 let sink = self.endian4i.wtr2int_consumer(&mut wtr);
157 self.pfile2brdr2sink(sink)?;
158 wtr.flush()
159 }
160}
161
162impl Cfg {
163 pub fn pfile2brdr2stdout(&self) -> Result<(), io::Error> {
164 let o = io::stdout();
165 let mut ol = o.lock();
166 self.pfile2brdr2wtr(BufWriter::new(&mut ol))?;
167 ol.flush()
168 }
169}