deltalake_core/datafile/mod.rs
1//! Data-file read/write abstractions, in two tiers:
2//!
3//! * **File tier** ([`DataFileWriter`], [`DataFileReader`]) — the per-file
4//! seam where parquet `WriterProperties` attach.
5//! Impl: [`writer::PartitionWriter`].
6//! * **Dataset tier** ([`DeltaDataWriter`], [`DeltaDataReader`]) — composes the
7//! file tier across a table. Impl: [`writer::DeltaWriter`].
8//!
9//! Both tiers operate on a DataFusion-free fallible stream of [`RecordBatch`]es.
10//! The DataFusion-capable surface lives in the gated [`datafusion_ext`] module.
11
12use arrow_array::RecordBatch;
13use futures::stream::{BoxStream, Stream, StreamExt as _};
14
15use crate::errors::{DeltaResult, DeltaTableError};
16use crate::kernel::Add;
17
18pub mod properties;
19pub mod reader;
20pub mod writer;
21
22#[cfg(feature = "datafusion")]
23pub mod datafusion_ext;
24
25pub use properties::ReaderProperties;
26
27/// A fallible stream of [`RecordBatch`]es — the common currency of both tiers.
28/// Producers that parallelize (concurrent file opens, partitioned scans) do so
29/// upstream and merge into this stream.
30pub type BatchStream = BoxStream<'static, DeltaResult<RecordBatch>>;
31
32/// Adapt a fallible `RecordBatch` stream into a [`BatchStream`], mapping its
33/// error into [`DeltaTableError`]. Shared by the parquet file reader
34/// ([`reader`]) and the DataFusion stream adapters ([`datafusion_ext`]).
35pub(crate) fn results_to_stream<S, E>(stream: S) -> BatchStream
36where
37 S: Stream<Item = Result<RecordBatch, E>> + Send + 'static,
38 E: Into<DeltaTableError> + Send + 'static,
39{
40 stream.map(|res| res.map_err(Into::into)).boxed()
41}
42
43/// File tier: writes a single Delta data file (or size-split set for one
44/// partition). The per-file seam where parquet `WriterProperties`/encryption
45/// attach. Impl: [`writer::PartitionWriter`].
46#[async_trait::async_trait]
47pub trait DataFileWriter: Send {
48 /// Buffer a record batch, writing to one or more parquet files as needed.
49 /// The batch must match the writer's (partition-stripped) file schema.
50 async fn write(&mut self, batch: &RecordBatch) -> DeltaResult<()>;
51
52 /// Finish writing and return the uncommitted [`Add`] actions for the files
53 /// that were produced.
54 async fn close(self: Box<Self>) -> DeltaResult<Vec<Add>>;
55
56 /// Abandon the writer after an error, aborting its in-progress multipart
57 /// uploads. Dropping the writer instead leaks upload parts vacuum cannot
58 /// see, and `close`-ing it finalizes files for partially-written data;
59 /// error-path callers should use this.
60 async fn abort(self: Box<Self>) -> DeltaResult<()>;
61}
62
63/// File tier: reads a single parquet data file (the per-file decryption seam,
64/// mirroring [`DataFileWriter`]). Impl: [`reader::ParquetFileReader`].
65#[async_trait::async_trait]
66pub trait DataFileReader: Send + Sync {
67 /// Read the parquet data file at `path` into a stream of record batches.
68 async fn read_file(&self, path: object_store::path::Path) -> DeltaResult<BatchStream>;
69}
70
71/// Options for a basic (DataFusion-free) read. Richer predicate/projection
72/// pushdown is the DataFusion extension's job ([`datafusion_ext::DeltaDataReaderExt`]).
73#[derive(Debug, Default, Clone)]
74pub struct ReadOptions {
75 /// Project to this subset of (logical) column names. `None` reads all columns.
76 pub projection: Option<Vec<String>>,
77 /// Stop after returning at least this many rows. `None` reads the whole table.
78 pub limit: Option<usize>,
79}
80
81impl ReadOptions {
82 /// Project to the given logical column names.
83 pub fn with_projection(mut self, projection: impl Into<Vec<String>>) -> Self {
84 self.projection = Some(projection.into());
85 self
86 }
87
88 /// Limit the number of rows returned.
89 pub fn with_limit(mut self, limit: usize) -> Self {
90 self.limit = Some(limit);
91 self
92 }
93}
94
95/// Dataset tier: a DataFusion-free writer that drains a batch stream into a
96/// table's data files (partitioning and composing a [`DataFileWriter`] per
97/// partition). Batches must already conform to the table schema and constraints
98/// (callers on the basic path validate themselves).
99#[async_trait::async_trait]
100pub trait DeltaDataWriter: Send {
101 /// Drain the batch stream into data files, returning the uncommitted
102 /// [`Add`] actions (still to be committed via a transaction).
103 async fn write_all(self: Box<Self>, batches: BatchStream) -> DeltaResult<Vec<Add>>;
104}
105
106/// Dataset tier: a DataFusion-free reader that composes the file tier
107/// ([`DataFileReader`]) across a table's data files, applying deletion
108/// vectors, partition values, and column-mapping transforms.
109#[async_trait::async_trait]
110pub trait DeltaDataReader: Send + Sync {
111 /// Read the selected data as a batch stream.
112 async fn read(&self, options: ReadOptions) -> DeltaResult<BatchStream>;
113}
114
115#[cfg(test)]
116mod tests {
117 use super::*;
118
119 #[test]
120 fn test_read_options_builders() {
121 let options = ReadOptions::default()
122 .with_projection(vec!["a".to_string(), "b".to_string()])
123 .with_limit(5);
124 assert_eq!(
125 options.projection,
126 Some(vec!["a".to_string(), "b".to_string()])
127 );
128 assert_eq!(options.limit, Some(5));
129 }
130}