deltalake_core/datafile/
reader.rs1use std::sync::Arc;
17
18use delta_kernel::table_features::ColumnMappingMode;
19use futures::stream::{StreamExt as _, TryStreamExt as _};
20use object_store::ObjectStore;
21use object_store::path::Path;
22use parquet::arrow::async_reader::{ParquetObjectReader, ParquetRecordBatchStreamBuilder};
23
24use crate::DeltaTable;
25use crate::errors::{DeltaResult, DeltaTableError};
26
27use super::{BatchStream, DataFileReader, DeltaDataReader, ReadOptions, results_to_stream};
28
29fn not_yet_implemented(what: &str) -> DeltaTableError {
30 DeltaTableError::Generic(format!(
31 "The kernel-backed read path ({what}) is not yet implemented. For a plain \
32 table, use ParquetTableReader (a DataFusion-free reader); for full Delta \
33 read semantics (deletion vectors, column mapping, partition values) \
34 enable the `datafusion` feature."
35 ))
36}
37
38fn not_supported(feature: &str) -> DeltaTableError {
39 DeltaTableError::Generic(format!(
40 "ParquetTableReader cannot read a table that uses {feature}; \
41 use the DataFusion read path (DeltaDataReaderExt) for such tables."
42 ))
43}
44
45#[derive(Debug, Clone)]
53pub struct ParquetFileReader {
54 store: Arc<dyn ObjectStore>,
55}
56
57impl ParquetFileReader {
58 pub fn new(store: Arc<dyn ObjectStore>) -> Self {
60 Self { store }
61 }
62
63 async fn read_file_sized(&self, path: Path, size: Option<u64>) -> DeltaResult<BatchStream> {
66 let mut reader = ParquetObjectReader::new(self.store.clone(), path);
67 if let Some(size) = size {
68 reader = reader.with_file_size(size);
69 }
70 let stream = ParquetRecordBatchStreamBuilder::new(reader)
71 .await?
72 .build()?;
73 Ok(results_to_stream(stream))
74 }
75}
76
77#[async_trait::async_trait]
78impl DataFileReader for ParquetFileReader {
79 async fn read_file(&self, path: Path) -> DeltaResult<BatchStream> {
80 self.read_file_sized(path, None).await
81 }
82}
83
84#[derive(Debug, Clone)]
95pub struct ParquetTableReader {
96 file_reader: ParquetFileReader,
97 files: Arc<[(Path, u64)]>,
100}
101
102impl ParquetTableReader {
103 pub async fn try_new(table: &DeltaTable) -> DeltaResult<Self> {
108 let snapshot = table.snapshot()?;
109
110 if snapshot
115 .snapshot()
116 .table_configuration()
117 .column_mapping_mode()
118 != ColumnMappingMode::None
119 {
120 return Err(not_supported("column mapping"));
121 }
122 if !snapshot.metadata().partition_columns().is_empty() {
124 return Err(not_supported("partition columns"));
125 }
126
127 let log_store = table.log_store();
128 let mut files = Vec::new();
129 let mut views = snapshot.snapshot().file_views(log_store.as_ref(), None);
130 while let Some(view) = views.try_next().await? {
131 if view.deletion_vector_descriptor().is_some() {
133 return Err(not_supported("deletion vectors"));
134 }
135 let size = u64::try_from(view.size()).unwrap_or(0);
141 files.push((view.object_store_path(), size));
142 }
143
144 let store = log_store.object_store(None);
145 Ok(Self {
146 file_reader: ParquetFileReader::new(store),
147 files: files.into(),
148 })
149 }
150}
151
152#[async_trait::async_trait]
153impl DeltaDataReader for ParquetTableReader {
154 async fn read(&self, options: ReadOptions) -> DeltaResult<BatchStream> {
155 if options.projection.is_some() || options.limit.is_some() {
158 return Err(DeltaTableError::Generic(
159 "projection and limit are not yet supported by the basic parquet reader".into(),
160 ));
161 }
162
163 let file_reader = self.file_reader.clone();
164 let files = Arc::clone(&self.files);
165 let concurrency = std::thread::available_parallelism()
169 .map(|n| n.get())
170 .unwrap_or(1);
171 let stream = futures::stream::iter(0..files.len())
172 .flat_map_unordered(concurrency, move |i| {
173 let (path, size) = files[i].clone();
174 futures::stream::once(open_file_stream(file_reader.clone(), path, size))
175 .flatten()
176 .boxed()
177 })
178 .boxed();
179 Ok(stream)
180 }
181}
182
183async fn open_file_stream(reader: ParquetFileReader, path: Path, size: u64) -> BatchStream {
186 match reader
188 .read_file_sized(path, (size > 0).then_some(size))
189 .await
190 {
191 Ok(file_stream) => file_stream,
192 Err(err) => futures::stream::once(std::future::ready(Err(err))).boxed(),
193 }
194}
195
196#[derive(Debug, Clone, Default)]
202pub struct KernelDataFileReader;
203
204#[async_trait::async_trait]
205impl DataFileReader for KernelDataFileReader {
206 async fn read_file(&self, _path: Path) -> DeltaResult<BatchStream> {
207 Err(not_yet_implemented("KernelDataFileReader"))
208 }
209}
210
211#[derive(Debug, Clone, Default)]
216pub struct KernelDataReader;
217
218#[async_trait::async_trait]
219impl DeltaDataReader for KernelDataReader {
220 async fn read(&self, _options: ReadOptions) -> DeltaResult<BatchStream> {
221 Err(not_yet_implemented("KernelDataReader"))
222 }
223}
224
225#[cfg(test)]
226mod tests {
227 use super::*;
228 use crate::operations::create::CreateBuilder;
229 use crate::writer::DeltaWriter as _;
230 use crate::writer::RecordBatchWriter;
231 use crate::writer::test_utils::{get_delta_schema, get_record_batch};
232
233 #[tokio::test]
234 async fn test_parquet_table_reader_reads_all_files() {
235 let schema = get_delta_schema();
239 let tmp = tempfile::tempdir().unwrap();
240 let mut table = CreateBuilder::new()
241 .with_location(tmp.path().to_str().unwrap())
242 .with_columns(schema.fields().cloned())
243 .await
244 .unwrap();
245
246 let batch = get_record_batch(None, false);
247 let expected_rows = batch.num_rows();
248 let mut writer = RecordBatchWriter::for_table(&table).unwrap();
249 writer.write(batch).await.unwrap();
250 writer.flush_and_commit(&mut table).await.unwrap();
251
252 let reader = ParquetTableReader::try_new(&table).await.unwrap();
253 let stream = reader.read(ReadOptions::default()).await.unwrap();
254 let batches: Vec<_> = stream.try_collect().await.unwrap();
255
256 assert!(!batches.is_empty(), "expected at least one batch");
257 let total: usize = batches.iter().map(|b| b.num_rows()).sum();
258 assert_eq!(total, expected_rows);
259 }
260
261 #[tokio::test]
262 async fn test_parquet_table_reader_rejects_partitioned_table() {
263 let schema = get_delta_schema();
266 let tmp = tempfile::tempdir().unwrap();
267 let table = CreateBuilder::new()
268 .with_location(tmp.path().to_str().unwrap())
269 .with_columns(schema.fields().cloned())
270 .with_partition_columns(vec!["modified".to_string()])
271 .await
272 .unwrap();
273
274 let err = ParquetTableReader::try_new(&table).await.unwrap_err();
275 assert!(matches!(err, DeltaTableError::Generic(_)), "got: {err:?}");
276 }
277
278 #[tokio::test]
279 async fn test_kernel_readers_not_yet_implemented() {
280 let file_err = KernelDataFileReader
283 .read_file(Path::from("data/file.parquet"))
284 .await
285 .err()
286 .expect("KernelDataFileReader is a placeholder and must error");
287 assert!(
288 matches!(file_err, DeltaTableError::Generic(_)),
289 "got: {file_err:?}"
290 );
291
292 let table_err = KernelDataReader
293 .read(ReadOptions::default())
294 .await
295 .err()
296 .expect("KernelDataReader is a placeholder and must error");
297 assert!(
298 matches!(table_err, DeltaTableError::Generic(_)),
299 "got: {table_err:?}"
300 );
301 }
302
303 #[tokio::test]
304 async fn test_parquet_file_reader_reads_single_file() {
305 let schema = get_delta_schema();
308 let tmp = tempfile::tempdir().unwrap();
309 let mut table = CreateBuilder::new()
310 .with_location(tmp.path().to_str().unwrap())
311 .with_columns(schema.fields().cloned())
312 .await
313 .unwrap();
314
315 let batch = get_record_batch(None, false);
316 let expected_rows = batch.num_rows();
317 let mut writer = RecordBatchWriter::for_table(&table).unwrap();
318 writer.write(batch).await.unwrap();
319 writer.flush_and_commit(&mut table).await.unwrap();
320
321 let store = table.object_store();
323 let mut listing = store.list(None);
324 let mut data_file = None;
325 while let Some(meta) = listing.next().await {
326 let location = meta.unwrap().location;
327 if location.filename().is_some_and(|f| f.ends_with(".parquet")) {
328 data_file = Some(location);
329 break;
330 }
331 }
332 let path = data_file.expect("table should have a parquet data file");
333
334 let reader = ParquetFileReader::new(store);
335 let stream = reader.read_file(path).await.unwrap();
336 let batches: Vec<_> = stream.try_collect().await.unwrap();
337
338 let total: usize = batches.iter().map(|b| b.num_rows()).sum();
339 assert_eq!(total, expected_rows);
340 }
341
342 #[tokio::test]
343 async fn test_parquet_table_reader_rejects_projection() {
344 let schema = get_delta_schema();
347 let tmp = tempfile::tempdir().unwrap();
348 let mut table = CreateBuilder::new()
349 .with_location(tmp.path().to_str().unwrap())
350 .with_columns(schema.fields().cloned())
351 .await
352 .unwrap();
353
354 let mut writer = RecordBatchWriter::for_table(&table).unwrap();
355 writer.write(get_record_batch(None, false)).await.unwrap();
356 writer.flush_and_commit(&mut table).await.unwrap();
357
358 let reader = ParquetTableReader::try_new(&table).await.unwrap();
359 let options = ReadOptions::default().with_projection(vec!["id".to_string()]);
360 let err = reader
361 .read(options)
362 .await
363 .err()
364 .expect("the basic reader should reject projection");
365 assert!(matches!(err, DeltaTableError::Generic(_)), "got: {err:?}");
366 }
367}