1use std::fs::File;
20use std::io::{BufReader, BufWriter, Seek};
21use std::sync::Arc;
22
23use arrow::array::RecordBatch;
24use arrow::datatypes::SchemaRef;
25use arrow::error::ArrowError;
26
27pub type BatchIter = Box<dyn Iterator<Item = Result<RecordBatch, ArrowError>> + Send>;
29
30pub trait FileFormat: Send + Sync {
32 fn name(&self) -> &'static str;
34 fn ext(&self) -> &'static str;
36 fn writer(&self, file: File, schema: &SchemaRef) -> Result<Box<dyn FormatWriter>, ArrowError>;
38 fn reader(&self, file: File) -> Result<BatchIter, ArrowError>;
40 fn reader_bytes(&self, bytes: bytes::Bytes) -> Result<BatchIter, ArrowError>;
42}
43
44pub trait FormatWriter: Send {
46 fn write(&mut self, batch: &RecordBatch) -> Result<(), ArrowError>;
47 fn finish(self: Box<Self>) -> Result<File, ArrowError>;
49}
50
51pub const NAMES: &[&str] = &["arrow", "parquet", "csv", "json"];
53
54pub fn by_extension(ext: &str) -> Option<Arc<dyn FileFormat>> {
56 NAMES
57 .iter()
58 .filter_map(|name| by_name(name).ok())
59 .find(|f| f.ext() == ext)
60}
61
62pub fn by_name(name: &str) -> Result<Arc<dyn FileFormat>, String> {
64 Ok(match name {
65 "arrow" => Arc::new(ArrowIpc),
66 "parquet" => Arc::new(Parquet::default()),
67 "csv" => Arc::new(Csv),
68 "json" => Arc::new(JsonLines::default()),
69 other => return Err(format!("format = \"{other}\" is not known ({})", NAMES.join(", "))),
70 })
71}
72
73fn unbuffer(w: BufWriter<File>) -> Result<File, ArrowError> {
74 w.into_inner()
75 .map_err(|e| ArrowError::IoError(e.to_string(), e.into_error()))
76}
77
78pub struct ArrowIpc;
83
84struct ArrowIpcWriter(arrow::ipc::writer::StreamWriter<BufWriter<File>>);
85
86impl FileFormat for ArrowIpc {
87 fn name(&self) -> &'static str {
88 "arrow"
89 }
90 fn ext(&self) -> &'static str {
91 "arrow"
92 }
93 fn writer(&self, file: File, schema: &SchemaRef) -> Result<Box<dyn FormatWriter>, ArrowError> {
94 Ok(Box::new(ArrowIpcWriter(arrow::ipc::writer::StreamWriter::try_new(
95 BufWriter::new(file),
96 schema,
97 )?)))
98 }
99 fn reader(&self, file: File) -> Result<BatchIter, ArrowError> {
100 Ok(Box::new(arrow::ipc::reader::StreamReader::try_new(file, None)?))
101 }
102 fn reader_bytes(&self, bytes: bytes::Bytes) -> Result<BatchIter, ArrowError> {
103 Ok(Box::new(arrow::ipc::reader::StreamReader::try_new(
104 std::io::Cursor::new(bytes),
105 None,
106 )?))
107 }
108}
109
110impl FormatWriter for ArrowIpcWriter {
111 fn write(&mut self, batch: &RecordBatch) -> Result<(), ArrowError> {
112 self.0.write(batch)
113 }
114 fn finish(mut self: Box<Self>) -> Result<File, ArrowError> {
115 self.0.finish()?;
116 unbuffer(self.0.into_inner()?)
117 }
118}
119
120pub struct Parquet {
126 pub compression: parquet::basic::Compression,
127 plain: fn(&arrow::datatypes::Field) -> bool,
129}
130
131impl Default for Parquet {
132 fn default() -> Self {
133 Parquet {
134 compression: parquet::basic::Compression::SNAPPY,
135 plain: |_| false,
136 }
137 }
138}
139
140impl Parquet {
141 pub fn with_compression(name: &str) -> Result<Self, String> {
143 use parquet::basic::{Compression, ZstdLevel};
144 let compression = match name {
145 "snappy" => Compression::SNAPPY,
146 "zstd" => Compression::ZSTD(ZstdLevel::default()),
147 "none" => Compression::UNCOMPRESSED,
148 other => return Err(format!("compression = \"{other}\" is not known (snappy, zstd, none)")),
149 };
150 Ok(Parquet {
151 compression,
152 ..Parquet::default()
153 })
154 }
155
156 pub fn without_dictionary_for(self, plain: fn(&arrow::datatypes::Field) -> bool) -> Self {
159 Parquet { plain, ..self }
160 }
161}
162
163struct ParquetWriter(parquet::arrow::ArrowWriter<BufWriter<File>>);
164
165impl FileFormat for Parquet {
166 fn name(&self) -> &'static str {
167 "parquet"
168 }
169 fn ext(&self) -> &'static str {
170 "parquet"
171 }
172 fn writer(&self, file: File, schema: &SchemaRef) -> Result<Box<dyn FormatWriter>, ArrowError> {
173 geo_stats::install();
175 let mut props = parquet::file::properties::WriterProperties::builder().set_compression(self.compression);
176 for f in schema.fields().iter().filter(|f| (self.plain)(f)) {
177 props =
178 props.set_column_dictionary_enabled(parquet::schema::types::ColumnPath::from(f.name().as_str()), false);
179 }
180 let props = props.build();
181 let w = parquet::arrow::ArrowWriter::try_new(BufWriter::new(file), Arc::clone(schema), Some(props))?;
182 Ok(Box::new(ParquetWriter(w)))
183 }
184 fn reader(&self, file: File) -> Result<BatchIter, ArrowError> {
185 let reader = parquet::arrow::arrow_reader::ParquetRecordBatchReaderBuilder::try_new(file)?.build()?;
186 Ok(Box::new(reader))
187 }
188 fn reader_bytes(&self, bytes: bytes::Bytes) -> Result<BatchIter, ArrowError> {
189 let reader = parquet::arrow::arrow_reader::ParquetRecordBatchReaderBuilder::try_new(bytes)?.build()?;
190 Ok(Box::new(reader))
191 }
192}
193
194impl FormatWriter for ParquetWriter {
195 fn write(&mut self, batch: &RecordBatch) -> Result<(), ArrowError> {
196 Ok(self.0.write(batch)?)
197 }
198 fn finish(self: Box<Self>) -> Result<File, ArrowError> {
199 unbuffer(self.0.into_inner()?)
201 }
202}
203
204pub struct Csv;
208
209struct CsvWriter(arrow::csv::Writer<BufWriter<File>>);
210
211impl FileFormat for Csv {
212 fn name(&self) -> &'static str {
213 "csv"
214 }
215 fn ext(&self) -> &'static str {
216 "csv"
217 }
218 fn writer(&self, file: File, _schema: &SchemaRef) -> Result<Box<dyn FormatWriter>, ArrowError> {
219 Ok(Box::new(CsvWriter(
220 arrow::csv::WriterBuilder::new()
221 .with_header(true)
222 .build(BufWriter::new(file)),
223 )))
224 }
225 fn reader(&self, file: File) -> Result<BatchIter, ArrowError> {
226 csv_reader(file)
227 }
228 fn reader_bytes(&self, bytes: bytes::Bytes) -> Result<BatchIter, ArrowError> {
229 csv_reader(std::io::Cursor::new(bytes))
230 }
231}
232
233fn csv_reader<R: std::io::Read + Seek + Send + 'static>(mut r: R) -> Result<BatchIter, ArrowError> {
234 let format = arrow::csv::reader::Format::default().with_header(true);
235 let (schema, _) = format.infer_schema(&mut r, None)?;
236 r.rewind().map_err(|e| ArrowError::IoError(e.to_string(), e))?;
237 let reader = arrow::csv::ReaderBuilder::new(Arc::new(schema))
238 .with_format(format)
239 .build(r)?;
240 Ok(Box::new(reader))
241}
242
243impl FormatWriter for CsvWriter {
244 fn write(&mut self, batch: &RecordBatch) -> Result<(), ArrowError> {
245 self.0.write(batch)
246 }
247 fn finish(self: Box<Self>) -> Result<File, ArrowError> {
248 unbuffer(self.0.into_inner())
249 }
250}
251
252#[derive(Default)]
258pub struct JsonLines {
259 encoders: Option<Arc<dyn arrow::json::EncoderFactory>>,
260}
261
262impl JsonLines {
263 pub fn with_encoders(encoders: Arc<dyn arrow::json::EncoderFactory>) -> Self {
265 JsonLines {
266 encoders: Some(encoders),
267 }
268 }
269}
270
271struct JsonWriter(arrow::json::LineDelimitedWriter<BufWriter<File>>);
272
273impl FileFormat for JsonLines {
274 fn name(&self) -> &'static str {
275 "json"
276 }
277 fn ext(&self) -> &'static str {
278 "json"
279 }
280 fn writer(&self, file: File, _schema: &SchemaRef) -> Result<Box<dyn FormatWriter>, ArrowError> {
281 let mut builder = arrow::json::WriterBuilder::new();
282 if let Some(e) = &self.encoders {
283 builder = builder.with_encoder_factory(Arc::clone(e));
284 }
285 Ok(Box::new(JsonWriter(builder.build(BufWriter::new(file)))))
286 }
287 fn reader(&self, file: File) -> Result<BatchIter, ArrowError> {
288 json_reader(BufReader::new(file))
289 }
290 fn reader_bytes(&self, bytes: bytes::Bytes) -> Result<BatchIter, ArrowError> {
291 json_reader(std::io::Cursor::new(bytes))
292 }
293}
294
295fn json_reader<R: std::io::BufRead + Seek + Send + 'static>(mut r: R) -> Result<BatchIter, ArrowError> {
296 let (schema, _) = arrow::json::reader::infer_json_schema_from_seekable(&mut r, None)?;
297 r.rewind().map_err(|e| ArrowError::IoError(e.to_string(), e))?;
298 let reader = arrow::json::ReaderBuilder::new(Arc::new(schema)).build(r)?;
299 Ok(Box::new(reader))
300}
301
302impl FormatWriter for JsonWriter {
303 fn write(&mut self, batch: &RecordBatch) -> Result<(), ArrowError> {
304 self.0.write(batch)
305 }
306 fn finish(mut self: Box<Self>) -> Result<File, ArrowError> {
307 self.0.finish()?;
308 unbuffer(self.0.into_inner())
309 }
310}
311
312mod geo_stats;
313
314#[cfg(test)]
315mod tests;