Skip to main content

fv_streams_formats/
lib.rs

1//! THE FILE FORMATS: bytes ⇄ `RecordBatch` for whole files — the *format* axis of the connector frame
2//! (`Format × Connector × Role`), separate from *where* a file lives (a local directory, an object
3//! store — the engine's part store) and from *what role* it plays (a sink's part, a
4//! source's input). Nothing here is hand-rolled: every format is the arrow-rs codec for it —
5//! `arrow::ipc` (the Arrow IPC stream), `parquet`, `arrow::csv`, `arrow::json` — wrapped behind one
6//! writer/reader shape so a sink or a source is written once and works for all of them.
7//!
8//! A [`FormatWriter`] streams batches into a file the caller staged and hands the file back
9//! finished (its footer / end marker written, its buffers flushed — not synced; the file sink's
10//! committer syncs it as part of the two-phase commit). A reader iterates a file's batches.
11//!
12//! Exactness: Arrow IPC and Parquet round-trip a batch exactly (schema and values). CSV and JSON
13//! lines are *text* formats — a reader infers their schema from the data, so types round-trip only
14//! as far as the text carries them (integers, floats, strings, booleans do; a timestamp comes back as
15//! text). They are here because a downstream tool wants them, not for our own round trip.
16//!
17//! Formats work on whole batches and whole files: nothing here touches the per-row path.
18
19use std::fs::File;
20use std::io::{BufReader, BufWriter, Seek};
21use std::sync::Arc;
22
23use arrow::array::RecordBatch;
24use arrow::datatypes::SchemaRef;
25use arrow::error::ArrowError;
26
27/// One batch iterator over a file.
28pub type BatchIter = Box<dyn Iterator<Item = Result<RecordBatch, ArrowError>> + Send>;
29
30/// A file format: how batches become one file and back.
31pub trait FileFormat: Send + Sync {
32    /// The name a pipeline selects it by (`format = "…"`).
33    fn name(&self) -> &'static str;
34    /// The file extension its parts carry.
35    fn ext(&self) -> &'static str;
36    /// A writer streaming batches of `schema` into `file` (staged by the caller).
37    fn writer(&self, file: File, schema: &SchemaRef) -> Result<Box<dyn FormatWriter>, ArrowError>;
38    /// The batches of a finished file, in order.
39    fn reader(&self, file: File) -> Result<BatchIter, ArrowError>;
40    /// The batches of a finished file held in memory (an object fetched from a store), in order.
41    fn reader_bytes(&self, bytes: bytes::Bytes) -> Result<BatchIter, ArrowError>;
42}
43
44/// A writer for one file: batches in, the finished file out.
45pub trait FormatWriter: Send {
46    fn write(&mut self, batch: &RecordBatch) -> Result<(), ArrowError>;
47    /// Write the format's footer / end marker, flush, and hand the file back for the commit.
48    fn finish(self: Box<Self>) -> Result<File, ArrowError>;
49}
50
51/// Every format's name, which is also its files' extension.
52pub const NAMES: &[&str] = &["arrow", "parquet", "csv", "json"];
53
54/// The format a file extension names (`parquet` for `.parquet`), if any.
55pub fn by_extension(ext: &str) -> Option<Arc<dyn FileFormat>> {
56    NAMES
57        .iter()
58        .filter_map(|name| by_name(name).ok())
59        .find(|f| f.ext() == ext)
60}
61
62/// The format named `name`, or the list of the ones there are.
63pub fn by_name(name: &str) -> Result<Arc<dyn FileFormat>, String> {
64    Ok(match name {
65        "arrow" => Arc::new(ArrowIpc),
66        "parquet" => Arc::new(Parquet::default()),
67        "csv" => Arc::new(Csv),
68        "json" => Arc::new(JsonLines::default()),
69        other => return Err(format!("format = \"{other}\" is not known ({})", NAMES.join(", "))),
70    })
71}
72
73fn unbuffer(w: BufWriter<File>) -> Result<File, ArrowError> {
74    w.into_inner()
75        .map_err(|e| ArrowError::IoError(e.to_string(), e.into_error()))
76}
77
78// ── Arrow IPC stream ─────────────────────────────────────────────────────────────────────────────
79
80/// The Arrow IPC *stream* format: the engine's own representation, byte-for-byte; the fastest
81/// edge there is (no encode beyond a memcpy) for an Arrow-speaking peer.
82pub struct ArrowIpc;
83
84struct ArrowIpcWriter(arrow::ipc::writer::StreamWriter<BufWriter<File>>);
85
86impl FileFormat for ArrowIpc {
87    fn name(&self) -> &'static str {
88        "arrow"
89    }
90    fn ext(&self) -> &'static str {
91        "arrow"
92    }
93    fn writer(&self, file: File, schema: &SchemaRef) -> Result<Box<dyn FormatWriter>, ArrowError> {
94        Ok(Box::new(ArrowIpcWriter(arrow::ipc::writer::StreamWriter::try_new(
95            BufWriter::new(file),
96            schema,
97        )?)))
98    }
99    fn reader(&self, file: File) -> Result<BatchIter, ArrowError> {
100        Ok(Box::new(arrow::ipc::reader::StreamReader::try_new(file, None)?))
101    }
102    fn reader_bytes(&self, bytes: bytes::Bytes) -> Result<BatchIter, ArrowError> {
103        Ok(Box::new(arrow::ipc::reader::StreamReader::try_new(
104            std::io::Cursor::new(bytes),
105            None,
106        )?))
107    }
108}
109
110impl FormatWriter for ArrowIpcWriter {
111    fn write(&mut self, batch: &RecordBatch) -> Result<(), ArrowError> {
112        self.0.write(batch)
113    }
114    fn finish(mut self: Box<Self>) -> Result<File, ArrowError> {
115        self.0.finish()?;
116        unbuffer(self.0.into_inner()?)
117    }
118}
119
120// ── Parquet ──────────────────────────────────────────────────────────────────────────────────────
121
122/// Parquet, through `parquet::arrow::ArrowWriter` — the writer DataFusion's own Parquet sink wraps.
123/// One row group per batch group the writer decides; the compression is a choice (`snappy` by
124/// default — the fast, universally read one; `zstd` for smaller files; `none`).
125pub struct Parquet {
126    pub compression: parquet::basic::Compression,
127    /// The columns written without a dictionary: values that rarely repeat, such as geometries.
128    plain: fn(&arrow::datatypes::Field) -> bool,
129}
130
131impl Default for Parquet {
132    fn default() -> Self {
133        Parquet {
134            compression: parquet::basic::Compression::SNAPPY,
135            plain: |_| false,
136        }
137    }
138}
139
140impl Parquet {
141    /// `snappy` | `zstd` | `none`.
142    pub fn with_compression(name: &str) -> Result<Self, String> {
143        use parquet::basic::{Compression, ZstdLevel};
144        let compression = match name {
145            "snappy" => Compression::SNAPPY,
146            "zstd" => Compression::ZSTD(ZstdLevel::default()),
147            "none" => Compression::UNCOMPRESSED,
148            other => return Err(format!("compression = \"{other}\" is not known (snappy, zstd, none)")),
149        };
150        Ok(Parquet {
151            compression,
152            ..Parquet::default()
153        })
154    }
155
156    /// The columns `plain` names written without a dictionary: values that rarely repeat, which a
157    /// dictionary only makes larger and slower to write.
158    pub fn without_dictionary_for(self, plain: fn(&arrow::datatypes::Field) -> bool) -> Self {
159        Parquet { plain, ..self }
160    }
161}
162
163struct ParquetWriter(parquet::arrow::ArrowWriter<BufWriter<File>>);
164
165impl FileFormat for Parquet {
166    fn name(&self) -> &'static str {
167        "parquet"
168    }
169    fn ext(&self) -> &'static str {
170        "parquet"
171    }
172    fn writer(&self, file: File, schema: &SchemaRef) -> Result<Box<dyn FormatWriter>, ArrowError> {
173        // a GEOMETRY column's bbox, a point at a time
174        geo_stats::install();
175        let mut props = parquet::file::properties::WriterProperties::builder().set_compression(self.compression);
176        for f in schema.fields().iter().filter(|f| (self.plain)(f)) {
177            props =
178                props.set_column_dictionary_enabled(parquet::schema::types::ColumnPath::from(f.name().as_str()), false);
179        }
180        let props = props.build();
181        let w = parquet::arrow::ArrowWriter::try_new(BufWriter::new(file), Arc::clone(schema), Some(props))?;
182        Ok(Box::new(ParquetWriter(w)))
183    }
184    fn reader(&self, file: File) -> Result<BatchIter, ArrowError> {
185        let reader = parquet::arrow::arrow_reader::ParquetRecordBatchReaderBuilder::try_new(file)?.build()?;
186        Ok(Box::new(reader))
187    }
188    fn reader_bytes(&self, bytes: bytes::Bytes) -> Result<BatchIter, ArrowError> {
189        let reader = parquet::arrow::arrow_reader::ParquetRecordBatchReaderBuilder::try_new(bytes)?.build()?;
190        Ok(Box::new(reader))
191    }
192}
193
194impl FormatWriter for ParquetWriter {
195    fn write(&mut self, batch: &RecordBatch) -> Result<(), ArrowError> {
196        Ok(self.0.write(batch)?)
197    }
198    fn finish(self: Box<Self>) -> Result<File, ArrowError> {
199        // `into_inner` closes the file (footer written) and hands the writer back
200        unbuffer(self.0.into_inner()?)
201    }
202}
203
204// ── CSV ──────────────────────────────────────────────────────────────────────────────────────────
205
206/// CSV with a header row per file, through `arrow::csv`. Text: the reader infers types.
207pub struct Csv;
208
209struct CsvWriter(arrow::csv::Writer<BufWriter<File>>);
210
211impl FileFormat for Csv {
212    fn name(&self) -> &'static str {
213        "csv"
214    }
215    fn ext(&self) -> &'static str {
216        "csv"
217    }
218    fn writer(&self, file: File, _schema: &SchemaRef) -> Result<Box<dyn FormatWriter>, ArrowError> {
219        Ok(Box::new(CsvWriter(
220            arrow::csv::WriterBuilder::new()
221                .with_header(true)
222                .build(BufWriter::new(file)),
223        )))
224    }
225    fn reader(&self, file: File) -> Result<BatchIter, ArrowError> {
226        csv_reader(file)
227    }
228    fn reader_bytes(&self, bytes: bytes::Bytes) -> Result<BatchIter, ArrowError> {
229        csv_reader(std::io::Cursor::new(bytes))
230    }
231}
232
233fn csv_reader<R: std::io::Read + Seek + Send + 'static>(mut r: R) -> Result<BatchIter, ArrowError> {
234    let format = arrow::csv::reader::Format::default().with_header(true);
235    let (schema, _) = format.infer_schema(&mut r, None)?;
236    r.rewind().map_err(|e| ArrowError::IoError(e.to_string(), e))?;
237    let reader = arrow::csv::ReaderBuilder::new(Arc::new(schema))
238        .with_format(format)
239        .build(r)?;
240    Ok(Box::new(reader))
241}
242
243impl FormatWriter for CsvWriter {
244    fn write(&mut self, batch: &RecordBatch) -> Result<(), ArrowError> {
245        self.0.write(batch)
246    }
247    fn finish(self: Box<Self>) -> Result<File, ArrowError> {
248        unbuffer(self.0.into_inner())
249    }
250}
251
252// ── JSON lines ───────────────────────────────────────────────────────────────────────────────────
253
254/// Newline-delimited JSON (one object per row), through `arrow::json`. Text: the reader infers types.
255/// `encoders` writes the columns it knows (an extension type) its own way; arrow-json writes the
256/// rest.
257#[derive(Default)]
258pub struct JsonLines {
259    encoders: Option<Arc<dyn arrow::json::EncoderFactory>>,
260}
261
262impl JsonLines {
263    /// JSON lines whose columns `encoders` knows are written by it.
264    pub fn with_encoders(encoders: Arc<dyn arrow::json::EncoderFactory>) -> Self {
265        JsonLines {
266            encoders: Some(encoders),
267        }
268    }
269}
270
271struct JsonWriter(arrow::json::LineDelimitedWriter<BufWriter<File>>);
272
273impl FileFormat for JsonLines {
274    fn name(&self) -> &'static str {
275        "json"
276    }
277    fn ext(&self) -> &'static str {
278        "json"
279    }
280    fn writer(&self, file: File, _schema: &SchemaRef) -> Result<Box<dyn FormatWriter>, ArrowError> {
281        let mut builder = arrow::json::WriterBuilder::new();
282        if let Some(e) = &self.encoders {
283            builder = builder.with_encoder_factory(Arc::clone(e));
284        }
285        Ok(Box::new(JsonWriter(builder.build(BufWriter::new(file)))))
286    }
287    fn reader(&self, file: File) -> Result<BatchIter, ArrowError> {
288        json_reader(BufReader::new(file))
289    }
290    fn reader_bytes(&self, bytes: bytes::Bytes) -> Result<BatchIter, ArrowError> {
291        json_reader(std::io::Cursor::new(bytes))
292    }
293}
294
295fn json_reader<R: std::io::BufRead + Seek + Send + 'static>(mut r: R) -> Result<BatchIter, ArrowError> {
296    let (schema, _) = arrow::json::reader::infer_json_schema_from_seekable(&mut r, None)?;
297    r.rewind().map_err(|e| ArrowError::IoError(e.to_string(), e))?;
298    let reader = arrow::json::ReaderBuilder::new(Arc::new(schema)).build(r)?;
299    Ok(Box::new(reader))
300}
301
302impl FormatWriter for JsonWriter {
303    fn write(&mut self, batch: &RecordBatch) -> Result<(), ArrowError> {
304        self.0.write(batch)
305    }
306    fn finish(mut self: Box<Self>) -> Result<File, ArrowError> {
307        self.0.finish()?;
308        unbuffer(self.0.into_inner())
309    }
310}
311
312mod geo_stats;
313
314#[cfg(test)]
315mod tests;