Skip to main content

deltalake_core/datafile/
reader.rs

1//! DataFusion-free data-file readers (file and dataset tiers).
2//!
3//! This module hosts two kinds of reader:
4//!
5//! * [`ParquetFileReader`] / [`ParquetTableReader`] — the first concrete
6//!   implementations: they read raw parquet directly from object storage with no
7//!   DataFusion, and are intentionally minimal, rejecting tables that need
8//!   deletion-vector application, column mapping, or partition-value reconstruction.
9//! * [`KernelDataFileReader`] / [`KernelDataReader`] — placeholders for the
10//!   later, full-fidelity reader backed by `delta-kernel`'s scan engine (which
11//!   applies deletion vectors, partition values, and column-mapping transforms).
12//!
13//! In a `datafusion` build, full reads go through
14//! [`crate::datafile::datafusion_ext::DeltaDataReaderExt`].
15
16use std::sync::Arc;
17
18use delta_kernel::table_features::ColumnMappingMode;
19use futures::stream::{StreamExt as _, TryStreamExt as _};
20use object_store::ObjectStore;
21use object_store::path::Path;
22use parquet::arrow::async_reader::{ParquetObjectReader, ParquetRecordBatchStreamBuilder};
23
24use crate::DeltaTable;
25use crate::errors::{DeltaResult, DeltaTableError};
26
27use super::{BatchStream, DataFileReader, DeltaDataReader, ReadOptions, results_to_stream};
28
29fn not_yet_implemented(what: &str) -> DeltaTableError {
30    DeltaTableError::Generic(format!(
31        "The kernel-backed read path ({what}) is not yet implemented. For a plain \
32         table, use ParquetTableReader (a DataFusion-free reader); for full Delta \
33         read semantics (deletion vectors, column mapping, partition values) \
34         enable the `datafusion` feature."
35    ))
36}
37
38fn not_supported(feature: &str) -> DeltaTableError {
39    DeltaTableError::Generic(format!(
40        "ParquetTableReader cannot read a table that uses {feature}; \
41         use the DataFusion read path (DeltaDataReaderExt) for such tables."
42    ))
43}
44
45// ---------------------------------------------------------------------------
46// Concrete, DataFusion-free parquet readers (file and dataset tiers).
47// ---------------------------------------------------------------------------
48
49/// File-tier reader that reads a single parquet data file directly from object
50/// storage, with no DataFusion. A concrete [`DataFileReader`] — the per-file read
51/// seam where parquet decryption will later attach, mirroring the write side.
52#[derive(Debug, Clone)]
53pub struct ParquetFileReader {
54    store: Arc<dyn ObjectStore>,
55}
56
57impl ParquetFileReader {
58    /// Create a reader over the given object store.
59    pub fn new(store: Arc<dyn ObjectStore>) -> Self {
60        Self { store }
61    }
62
63    /// Read a data file, optionally passing its known size so the parquet reader
64    /// can skip the extra `HEAD` request it would otherwise make to discover it.
65    async fn read_file_sized(&self, path: Path, size: Option<u64>) -> DeltaResult<BatchStream> {
66        let mut reader = ParquetObjectReader::new(self.store.clone(), path);
67        if let Some(size) = size {
68            reader = reader.with_file_size(size);
69        }
70        let stream = ParquetRecordBatchStreamBuilder::new(reader)
71            .await?
72            .build()?;
73        Ok(results_to_stream(stream))
74    }
75}
76
77#[async_trait::async_trait]
78impl DataFileReader for ParquetFileReader {
79    async fn read_file(&self, path: Path) -> DeltaResult<BatchStream> {
80        self.read_file_sized(path, None).await
81    }
82}
83
84/// Dataset-tier reader that reads all of a table's parquet data files directly —
85/// no DataFusion, no predicate — composing [`ParquetFileReader`] across the table.
86///
87/// Minimal by design: it reads raw parquet, so [`ParquetTableReader::try_new`]
88/// rejects tables that use deletion vectors, column mapping, or partition columns
89/// (those are the kernel-backed [`KernelDataReader`]'s job), and it does not unify
90/// schemas across files — returned batches reflect each file's physical schema as
91/// written. On a table whose schema evolved, older and newer files yield batches
92/// with differing schemas, and the caller must reconcile them (e.g. cast to a
93/// common schema before `concat`).
94#[derive(Debug, Clone)]
95pub struct ParquetTableReader {
96    file_reader: ParquetFileReader,
97    /// Data files as (object-store path, size in bytes); `Arc` so `read`/`Clone`
98    /// share the list instead of deep-copying every path.
99    files: Arc<[(Path, u64)]>,
100}
101
102impl ParquetTableReader {
103    /// Build a reader over the table's current data files.
104    ///
105    /// Errors if the table uses a feature this raw reader cannot honor
106    /// (deletion vectors, column mapping, or partition columns).
107    pub async fn try_new(table: &DeltaTable) -> DeltaResult<Self> {
108        let snapshot = table.snapshot()?;
109
110        // Guard: column mapping would mean physical (not logical) column names.
111        // Use the resolved mode (the property is only honored when the protocol
112        // actually enables the feature) to avoid rejecting a table that merely
113        // carries a stale, ignored `delta.columnMapping.mode` property.
114        if snapshot
115            .snapshot()
116            .table_configuration()
117            .column_mapping_mode()
118            != ColumnMappingMode::None
119        {
120            return Err(not_supported("column mapping"));
121        }
122        // Guard: partition column values live in the path, not the parquet file.
123        if !snapshot.metadata().partition_columns().is_empty() {
124            return Err(not_supported("partition columns"));
125        }
126
127        let log_store = table.log_store();
128        let mut files = Vec::new();
129        let mut views = snapshot.snapshot().file_views(log_store.as_ref(), None);
130        while let Some(view) = views.try_next().await? {
131            // Guard: a raw read would return rows that a deletion vector removes.
132            if view.deletion_vector_descriptor().is_some() {
133                return Err(not_supported("deletion vectors"));
134            }
135            // Canonical object-store path (preserves percent encoding), matching
136            // every other read path; carry the size so the reader can skip a
137            // per-file `HEAD`. A non-positive size (absent/corrupt log entry)
138            // becomes 0 = "unknown", and the reader HEADs for it rather than
139            // trusting a wrapped-around length.
140            let size = u64::try_from(view.size()).unwrap_or(0);
141            files.push((view.object_store_path(), size));
142        }
143
144        let store = log_store.object_store(None);
145        Ok(Self {
146            file_reader: ParquetFileReader::new(store),
147            files: files.into(),
148        })
149    }
150}
151
152#[async_trait::async_trait]
153impl DeltaDataReader for ParquetTableReader {
154    async fn read(&self, options: ReadOptions) -> DeltaResult<BatchStream> {
155        // Projection / limit pushdown is a follow-up; reject rather than silently
156        // ignore so callers don't get more data than they asked for.
157        if options.projection.is_some() || options.limit.is_some() {
158            return Err(DeltaTableError::Generic(
159                "projection and limit are not yet supported by the basic parquet reader".into(),
160            ));
161        }
162
163        let file_reader = self.file_reader.clone();
164        let files = Arc::clone(&self.files);
165        // Read up to `available_parallelism` files concurrently and interleave their batches
166        // (`flat_map_unordered`), overlapping per-file open/read latency. Row order
167        // across files isn't preserved, which is fine for a no-predicate scan.
168        let concurrency = std::thread::available_parallelism()
169            .map(|n| n.get())
170            .unwrap_or(1);
171        let stream = futures::stream::iter(0..files.len())
172            .flat_map_unordered(concurrency, move |i| {
173                let (path, size) = files[i].clone();
174                futures::stream::once(open_file_stream(file_reader.clone(), path, size))
175                    .flatten()
176                    .boxed()
177            })
178            .boxed();
179        Ok(stream)
180    }
181}
182
183/// Open one data file, turning an open error into a single failing batch item
184/// so it surfaces while draining rather than being silently dropped.
185async fn open_file_stream(reader: ParquetFileReader, path: Path, size: u64) -> BatchStream {
186    // size 0 means "unknown" (see try_new): let the reader HEAD for it.
187    match reader
188        .read_file_sized(path, (size > 0).then_some(size))
189        .await
190    {
191        Ok(file_stream) => file_stream,
192        Err(err) => futures::stream::once(std::future::ready(Err(err))).boxed(),
193    }
194}
195
196// ---------------------------------------------------------------------------
197// Placeholders for the future kernel-backed (full-fidelity) reader.
198// ---------------------------------------------------------------------------
199
200/// File-tier reader backed by `delta-kernel`'s parquet handler (placeholder).
201#[derive(Debug, Clone, Default)]
202pub struct KernelDataFileReader;
203
204#[async_trait::async_trait]
205impl DataFileReader for KernelDataFileReader {
206    async fn read_file(&self, _path: Path) -> DeltaResult<BatchStream> {
207        Err(not_yet_implemented("KernelDataFileReader"))
208    }
209}
210
211/// Dataset-tier reader backed by `delta-kernel`'s scan engine (placeholder).
212///
213/// Unlike [`ParquetTableReader`], this will apply deletion vectors, partition
214/// values, and column-mapping transforms — the full Delta read semantics.
215#[derive(Debug, Clone, Default)]
216pub struct KernelDataReader;
217
218#[async_trait::async_trait]
219impl DeltaDataReader for KernelDataReader {
220    async fn read(&self, _options: ReadOptions) -> DeltaResult<BatchStream> {
221        Err(not_yet_implemented("KernelDataReader"))
222    }
223}
224
225#[cfg(test)]
226mod tests {
227    use super::*;
228    use crate::operations::create::CreateBuilder;
229    use crate::writer::DeltaWriter as _;
230    use crate::writer::RecordBatchWriter;
231    use crate::writer::test_utils::{get_delta_schema, get_record_batch};
232
233    #[tokio::test]
234    async fn test_parquet_table_reader_reads_all_files() {
235        // Prove the read traits end-to-end: write a plain (unpartitioned, no-DV,
236        // no-column-mapping) table, then read every parquet file back with no
237        // predicate through the DataFusion-free reader.
238        let schema = get_delta_schema();
239        let tmp = tempfile::tempdir().unwrap();
240        let mut table = CreateBuilder::new()
241            .with_location(tmp.path().to_str().unwrap())
242            .with_columns(schema.fields().cloned())
243            .await
244            .unwrap();
245
246        let batch = get_record_batch(None, false);
247        let expected_rows = batch.num_rows();
248        let mut writer = RecordBatchWriter::for_table(&table).unwrap();
249        writer.write(batch).await.unwrap();
250        writer.flush_and_commit(&mut table).await.unwrap();
251
252        let reader = ParquetTableReader::try_new(&table).await.unwrap();
253        let stream = reader.read(ReadOptions::default()).await.unwrap();
254        let batches: Vec<_> = stream.try_collect().await.unwrap();
255
256        assert!(!batches.is_empty(), "expected at least one batch");
257        let total: usize = batches.iter().map(|b| b.num_rows()).sum();
258        assert_eq!(total, expected_rows);
259    }
260
261    #[tokio::test]
262    async fn test_parquet_table_reader_rejects_partitioned_table() {
263        // The raw reader cannot reconstruct partition columns, so it must refuse
264        // a partitioned table rather than return rows missing those columns.
265        let schema = get_delta_schema();
266        let tmp = tempfile::tempdir().unwrap();
267        let table = CreateBuilder::new()
268            .with_location(tmp.path().to_str().unwrap())
269            .with_columns(schema.fields().cloned())
270            .with_partition_columns(vec!["modified".to_string()])
271            .await
272            .unwrap();
273
274        let err = ParquetTableReader::try_new(&table).await.unwrap_err();
275        assert!(matches!(err, DeltaTableError::Generic(_)), "got: {err:?}");
276    }
277
278    #[tokio::test]
279    async fn test_kernel_readers_not_yet_implemented() {
280        // The kernel-backed readers are placeholders; both report a clear
281        // not-yet-implemented error rather than panicking or returning empty data.
282        let file_err = KernelDataFileReader
283            .read_file(Path::from("data/file.parquet"))
284            .await
285            .err()
286            .expect("KernelDataFileReader is a placeholder and must error");
287        assert!(
288            matches!(file_err, DeltaTableError::Generic(_)),
289            "got: {file_err:?}"
290        );
291
292        let table_err = KernelDataReader
293            .read(ReadOptions::default())
294            .await
295            .err()
296            .expect("KernelDataReader is a placeholder and must error");
297        assert!(
298            matches!(table_err, DeltaTableError::Generic(_)),
299            "got: {table_err:?}"
300        );
301    }
302
303    #[tokio::test]
304    async fn test_parquet_file_reader_reads_single_file() {
305        // Exercise the file-tier reader's `read_file` (size-unknown) path directly,
306        // independent of the dataset-tier reader.
307        let schema = get_delta_schema();
308        let tmp = tempfile::tempdir().unwrap();
309        let mut table = CreateBuilder::new()
310            .with_location(tmp.path().to_str().unwrap())
311            .with_columns(schema.fields().cloned())
312            .await
313            .unwrap();
314
315        let batch = get_record_batch(None, false);
316        let expected_rows = batch.num_rows();
317        let mut writer = RecordBatchWriter::for_table(&table).unwrap();
318        writer.write(batch).await.unwrap();
319        writer.flush_and_commit(&mut table).await.unwrap();
320
321        // Locate a data file in the table root.
322        let store = table.object_store();
323        let mut listing = store.list(None);
324        let mut data_file = None;
325        while let Some(meta) = listing.next().await {
326            let location = meta.unwrap().location;
327            if location.filename().is_some_and(|f| f.ends_with(".parquet")) {
328                data_file = Some(location);
329                break;
330            }
331        }
332        let path = data_file.expect("table should have a parquet data file");
333
334        let reader = ParquetFileReader::new(store);
335        let stream = reader.read_file(path).await.unwrap();
336        let batches: Vec<_> = stream.try_collect().await.unwrap();
337
338        let total: usize = batches.iter().map(|b| b.num_rows()).sum();
339        assert_eq!(total, expected_rows);
340    }
341
342    #[tokio::test]
343    async fn test_parquet_table_reader_rejects_projection() {
344        // The basic reader has no pushdown yet, so it rejects projection/limit
345        // rather than silently returning unfiltered data.
346        let schema = get_delta_schema();
347        let tmp = tempfile::tempdir().unwrap();
348        let mut table = CreateBuilder::new()
349            .with_location(tmp.path().to_str().unwrap())
350            .with_columns(schema.fields().cloned())
351            .await
352            .unwrap();
353
354        let mut writer = RecordBatchWriter::for_table(&table).unwrap();
355        writer.write(get_record_batch(None, false)).await.unwrap();
356        writer.flush_and_commit(&mut table).await.unwrap();
357
358        let reader = ParquetTableReader::try_new(&table).await.unwrap();
359        let options = ReadOptions::default().with_projection(vec!["id".to_string()]);
360        let err = reader
361            .read(options)
362            .await
363            .err()
364            .expect("the basic reader should reject projection");
365        assert!(matches!(err, DeltaTableError::Generic(_)), "got: {err:?}");
366    }
367}