deltalake_core/kernel/snapshot/
scan.rs1use std::cell::RefCell;
2use std::pin::Pin;
3use std::rc::Rc;
4use std::sync::Arc;
5
6use arrow_array::RecordBatch;
7use delta_kernel::engine::arrow_data::ArrowEngineData;
8use delta_kernel::scan::{Scan as KernelScan, ScanBuilder as KernelScanBuilder, ScanMetadata};
9use delta_kernel::schema::SchemaRef;
10use delta_kernel::snapshot::Snapshot as KernelSnapshot;
11use delta_kernel::{Engine, EngineData, PredicateRef, SnapshotRef, Version};
12use futures::Stream;
13use futures::future::ready;
14use futures::stream::once;
15use url::Url;
16
17#[cfg(feature = "datafusion")]
18use super::MaterializedFiles;
19use super::stats_projection::{FileStatsMaterialization, StatsProjection, StatsSourcePolicy};
20use crate::kernel::{ReceiverStreamBuilder, scan_row_in_eval};
21use crate::{DeltaResult, DeltaTableError};
22
23pub type SendableScanMetadataStream = Pin<Box<dyn Stream<Item = DeltaResult<ScanMetadata>> + Send>>;
25
26#[derive(Debug)]
28pub struct ScanBuilder {
29 snapshot: Arc<KernelSnapshot>,
30 schema: Option<SchemaRef>,
31 predicate: Option<PredicateRef>,
32 stats_materialization: Option<FileStatsMaterialization>,
33}
34
35impl ScanBuilder {
36 pub fn new(snapshot: impl Into<Arc<KernelSnapshot>>) -> Self {
38 Self {
39 snapshot: snapshot.into(),
40 schema: None,
41 predicate: None,
42 stats_materialization: None,
43 }
44 }
45
46 pub fn with_schema(mut self, schema: SchemaRef) -> Self {
54 self.schema = Some(schema);
55 self
56 }
57
58 pub fn with_schema_opt(mut self, schema_opt: Option<SchemaRef>) -> Self {
63 if let Some(schema) = schema_opt {
64 self.schema = Some(schema);
65 }
66 self
67 }
68
69 pub fn with_predicate(mut self, predicate: impl Into<Option<PredicateRef>>) -> Self {
76 self.predicate = predicate.into();
77 self
78 }
79
80 pub fn with_skip_stats(mut self, skip_stats: bool) -> Self {
88 if skip_stats {
89 self.stats_materialization = Some(FileStatsMaterialization::without_stats());
90 } else {
91 self.stats_materialization = None;
92 }
93 self
94 }
95
96 pub(crate) fn with_stats_materialization(
100 mut self,
101 stats_materialization: FileStatsMaterialization,
102 ) -> Self {
103 self.stats_materialization = Some(stats_materialization);
104 self
105 }
106
107 pub fn build(self) -> DeltaResult<Scan> {
109 let Self {
110 snapshot,
111 schema,
112 predicate,
113 stats_materialization,
114 } = self;
115
116 let stats_materialization = match stats_materialization {
117 Some(stats_materialization) => stats_materialization,
118 None => FileStatsMaterialization::query(StatsProjection::for_scan_inputs(
119 snapshot.as_ref(),
120 schema.as_ref(),
121 predicate.as_ref(),
122 )?),
123 };
124
125 let inner = build_kernel_scan(snapshot, schema, predicate, Some(&stats_materialization))?;
126
127 Ok(Scan::new(Arc::new(inner), stats_materialization))
128 }
129
130 pub fn with_kernel_all_struct_stats(mut self) -> Self {
133 self.stats_materialization = Some(FileStatsMaterialization::query(StatsProjection::Full));
136 self
137 }
138}
139
140fn build_kernel_scan(
141 snapshot: Arc<KernelSnapshot>,
142 schema: Option<SchemaRef>,
143 predicate: Option<PredicateRef>,
144 stats_materialization: Option<&FileStatsMaterialization>,
145) -> DeltaResult<KernelScan> {
146 let mut builder = KernelScanBuilder::new(snapshot)
147 .with_schema_opt(schema)
148 .with_predicate(predicate);
149
150 if let Some(stats_materialization) = stats_materialization {
151 builder = with_kernel_stats_output(builder, stats_materialization);
152 }
153
154 Ok(builder.build()?)
155}
156
157fn with_kernel_stats_output(
158 builder: KernelScanBuilder,
159 materialization: &FileStatsMaterialization,
160) -> KernelScanBuilder {
161 let stats_opts = match materialization.stats_source_policy() {
162 StatsSourcePolicy::None => delta_kernel::scan::StatsOptions::none(),
163 StatsSourcePolicy::ParsedWithJsonFallback => match materialization.stats_projection() {
164 StatsProjection::None => delta_kernel::scan::StatsOptions::json_only(),
165 StatsProjection::Full | StatsProjection::PredicateColumns(_) => {
166 if materialization.preserves_raw_stats() {
168 delta_kernel::scan::StatsOptions::all()
169 } else {
170 delta_kernel::scan::StatsOptions::all_struct()
172 }
173 }
174 StatsProjection::NumRecordsOnly => delta_kernel::scan::StatsOptions::json_only(),
175 },
176 };
177
178 builder.with_stats(stats_opts)
179}
180
181#[cfg(test)]
182mod tests {
183 use delta_kernel::expressions::{ColumnName, Scalar};
184 use delta_kernel::schema::{DataType, StructField, StructType};
185 use delta_kernel::{Expression, PredicateRef};
186 use futures::TryStreamExt;
187
188 use super::super::stats_projection::{
189 FileStatsMaterialization, StatsProjection, StatsSourcePolicy,
190 };
191 use super::*;
192 use crate::kernel::Action;
193 use crate::test_utils::make_test_add;
194 use crate::{DeltaTable, kernel::snapshot::Snapshot};
195
196 async fn synthetic_snapshot() -> DeltaResult<super::super::Snapshot> {
197 let nested = StructType::try_new([
198 StructField::nullable("leaf", DataType::INTEGER),
199 StructField::nullable("other_leaf", DataType::STRING),
200 ])?;
201 let table = DeltaTable::new_in_memory()
202 .create()
203 .with_columns([
204 StructField::nullable("value", DataType::INTEGER),
205 StructField::nullable("unreferenced_col", DataType::STRING),
206 StructField::nullable("part", DataType::STRING),
207 StructField::nullable("nested", DataType::Struct(Box::new(nested))),
208 ])
209 .with_partition_columns(["part"])
210 .await?;
211 super::super::Snapshot::try_new(table.log_store().as_ref(), None).await
212 }
213
214 #[tokio::test]
215 async fn scan_builder_infers_num_records_only_for_default_query_scan() -> DeltaResult<()> {
216 let snapshot = synthetic_snapshot().await?;
217 let scan = snapshot.scan_builder().build()?;
218
219 assert_eq!(
220 scan.stats_materialization().stats_projection(),
221 &StatsProjection::NumRecordsOnly
222 );
223 assert!(!scan.stats_materialization().preserves_raw_stats());
224 assert_eq!(
225 scan.stats_materialization().stats_source_policy(),
226 StatsSourcePolicy::ParsedWithJsonFallback
227 );
228
229 Ok(())
230 }
231
232 #[tokio::test]
233 async fn scan_builder_infers_predicate_columns_for_data_predicate() -> DeltaResult<()> {
234 let snapshot = synthetic_snapshot().await?;
235 let predicate: PredicateRef =
236 Arc::new(Expression::column(["value"]).gt(Scalar::Integer(10)));
237 let scan = snapshot.scan_builder().with_predicate(predicate).build()?;
238
239 assert_eq!(
240 scan.stats_materialization().stats_projection(),
241 &StatsProjection::PredicateColumns([ColumnName::new(["value"])].into())
242 );
243
244 Ok(())
245 }
246
247 #[tokio::test]
248 async fn scan_builder_rejects_predicate_on_unknown_column() -> DeltaResult<()> {
249 let snapshot = synthetic_snapshot().await?;
250 let predicate: PredicateRef =
251 Arc::new(Expression::column(["missing"]).gt(Scalar::Integer(10)));
252
253 let err = snapshot
254 .scan_builder()
255 .with_predicate(predicate)
256 .build()
257 .expect_err("predicate on an unknown column should fail scan planning");
258
259 assert!(
260 err.to_string().to_lowercase().contains("missing")
261 || err.to_string().to_lowercase().contains("unknown"),
262 "unexpected error: {err}"
263 );
264
265 Ok(())
266 }
267
268 #[tokio::test]
269 async fn scan_builder_preserves_explicit_compatibility_materialization() -> DeltaResult<()> {
270 let snapshot = synthetic_snapshot().await?;
271 let materialization = FileStatsMaterialization::compatibility(StatsProjection::full());
272 let scan = snapshot
273 .scan_builder()
274 .with_stats_materialization(materialization.clone())
275 .build()?;
276
277 assert_eq!(scan.stats_materialization(), &materialization);
278 assert!(scan.stats_materialization().preserves_raw_stats());
279
280 Ok(())
281 }
282
283 #[tokio::test]
284 async fn scan_builder_preserves_explicit_without_stats_materialization() -> DeltaResult<()> {
285 let snapshot = synthetic_snapshot().await?;
286 let scan = snapshot
287 .scan_builder()
288 .with_stats_materialization(FileStatsMaterialization::without_stats())
289 .build()?;
290
291 assert_eq!(
292 scan.stats_materialization().stats_projection(),
293 &StatsProjection::none()
294 );
295 assert_eq!(
296 scan.stats_materialization().stats_source_policy(),
297 StatsSourcePolicy::None
298 );
299
300 Ok(())
301 }
302
303 #[tokio::test]
304 async fn scan_builder_with_skip_stats_false_clears_prior_skip_stats() -> DeltaResult<()> {
305 let snapshot = synthetic_snapshot().await?;
306 let scan = snapshot
307 .scan_builder()
308 .with_stats_materialization(FileStatsMaterialization::without_stats())
309 .with_skip_stats(false)
310 .build()?;
311
312 assert_eq!(
313 scan.stats_materialization().stats_projection(),
314 &StatsProjection::NumRecordsOnly
315 );
316 assert_eq!(
317 scan.stats_materialization().stats_source_policy(),
318 StatsSourcePolicy::ParsedWithJsonFallback
319 );
320 assert!(!scan.stats_materialization().preserves_raw_stats());
321
322 Ok(())
323 }
324
325 #[tokio::test]
326 async fn scan_builder_skip_stats_disables_partition_pruning() -> DeltaResult<()> {
327 let table = DeltaTable::new_in_memory()
328 .create()
329 .with_columns([
330 StructField::nullable("value", DataType::INTEGER),
331 StructField::nullable("part", DataType::STRING),
332 ])
333 .with_partition_columns(["part"])
334 .with_actions([
335 Action::Add(make_test_add("part=a/part-a.parquet", &[("part", "a")], 0)),
336 Action::Add(make_test_add("part=b/part-b.parquet", &[("part", "b")], 0)),
337 ])
338 .await?;
339 let log_store = table.log_store();
340 let snapshot = Snapshot::try_new(log_store.as_ref(), None).await?;
341 let predicate: PredicateRef =
342 Arc::new(Expression::column(["part"]).eq(Scalar::String("a".to_string())));
343 let scan = snapshot
344 .scan_builder()
345 .with_predicate(predicate)
346 .with_skip_stats(true)
347 .build()?;
348
349 let mut paths = Vec::new();
350 let mut metadata = scan.scan_metadata(log_store.engine(None));
351 while let Some(batch) = metadata.try_next().await? {
352 paths = batch.visit_scan_files(paths, |paths, file| paths.push(file.path))?;
353 }
354 paths.sort();
355
356 assert_eq!(paths, ["part=a/part-a.parquet", "part=b/part-b.parquet"]);
357
358 Ok(())
359 }
360
361 #[tokio::test]
362 async fn scan_metadata_from_returns_error_for_malformed_cache() -> DeltaResult<()> {
363 let table = DeltaTable::new_in_memory()
364 .create()
365 .with_columns([StructField::nullable("value", DataType::INTEGER)])
366 .await?;
367 let log_store = table.log_store();
368 let snapshot = Snapshot::try_new(log_store.as_ref(), None).await?;
369 let scan = snapshot.scan_builder().build()?;
370 let malformed = RecordBatch::new_empty(Arc::new(arrow_schema::Schema::empty()));
371
372 assert!(
373 scan.scan_metadata_from(
374 log_store.engine(None),
375 snapshot.version(),
376 Box::new(std::iter::once(malformed)),
377 None,
378 )
379 .try_collect::<Vec<_>>()
380 .await
381 .is_err(),
382 "malformed cached data must return an error"
383 );
384 Ok(())
385 }
386}
387
388#[derive(Debug)]
393pub struct Scan {
394 inner: Arc<KernelScan>,
395 stats_materialization: FileStatsMaterialization,
396}
397
398impl From<KernelScan> for Scan {
399 fn from(inner: KernelScan) -> Self {
400 Self::new(
401 Arc::new(inner),
402 FileStatsMaterialization::compatibility(StatsProjection::full()),
403 )
404 }
405}
406
407impl From<Arc<KernelScan>> for Scan {
408 fn from(inner: Arc<KernelScan>) -> Self {
409 Self::new(
410 inner,
411 FileStatsMaterialization::compatibility(StatsProjection::full()),
412 )
413 }
414}
415
416impl Scan {
417 fn new(inner: Arc<KernelScan>, stats_materialization: FileStatsMaterialization) -> Self {
418 Self {
419 inner,
420 stats_materialization,
421 }
422 }
423
424 #[cfg(feature = "datafusion")]
426 pub(crate) fn inner(&self) -> &Arc<KernelScan> {
427 &self.inner
428 }
429
430 pub(crate) fn stats_materialization(&self) -> &FileStatsMaterialization {
434 &self.stats_materialization
435 }
436
437 pub fn table_root(&self) -> &Url {
444 self.inner.table_root()
445 }
446
447 pub fn snapshot(&self) -> &SnapshotRef {
451 self.inner.snapshot()
452 }
453
454 pub fn logical_schema(&self) -> &SchemaRef {
460 self.inner.logical_schema()
461 }
462
463 pub fn physical_schema(&self) -> &SchemaRef {
468 self.inner.physical_schema()
469 }
470
471 pub fn physical_predicate(&self) -> Option<PredicateRef> {
474 self.inner.physical_predicate()
475 }
476
477 pub fn scan_metadata(&self, engine: Arc<dyn Engine>) -> SendableScanMetadataStream {
479 let mut builder = ReceiverStreamBuilder::<ScanMetadata>::new(100);
481 let tx = builder.tx();
482
483 let inner = self.inner.clone();
484 let blocking_iter = move || {
485 for res in inner.scan_metadata(engine.as_ref())? {
486 if tx.blocking_send(Ok(res?)).is_err() {
487 break;
488 }
489 }
490 Ok(())
491 };
492
493 builder.spawn_blocking(blocking_iter);
494 builder.build()
495 }
496
497 #[cfg(feature = "datafusion")]
498 pub(crate) fn scan_metadata_seeded(
499 &self,
500 engine: Arc<dyn Engine>,
501 materialized_files: Option<&Arc<MaterializedFiles>>,
502 ) -> SendableScanMetadataStream {
503 match materialized_files.and_then(|materialized_files| materialized_files.full_table_seed())
504 {
505 Some(materialized_seed) => {
506 let (existing_version, existing_data, existing_predicate) =
507 materialized_seed.into_parts();
508 self.scan_metadata_from(
509 engine,
510 existing_version,
511 Box::new(existing_data),
512 existing_predicate,
513 )
514 }
515 None => self.scan_metadata(engine),
516 }
517 }
518
519 pub fn scan_metadata_from<T: Iterator<Item = RecordBatch> + Send + 'static>(
525 &self,
526 engine: Arc<dyn Engine>,
527 existing_version: Version,
528 existing_data: Box<T>,
529 existing_predicate: Option<PredicateRef>,
530 ) -> SendableScanMetadataStream {
531 let inner = self.inner.clone();
532 let snapshot = self.inner.snapshot().clone();
533
534 let evaluator = match scan_row_in_eval(&snapshot) {
536 Ok(scan_row_in_eval) => scan_row_in_eval,
537 Err(err) => return Box::pin(once(ready(Err(err)))),
538 };
539 let mut builder = ReceiverStreamBuilder::<ScanMetadata>::new(100);
541 let tx = builder.tx();
542 let scan_inner = move || {
543 let evaluation_error: Rc<RefCell<Option<DeltaTableError>>> =
544 Rc::new(RefCell::new(None));
545 let error_slot = Rc::clone(&evaluation_error);
546 let scan_row_data = existing_data
547 .map(|batch| Box::new(ArrowEngineData::new(batch)) as Box<dyn EngineData>)
548 .map_while(move |batch| match evaluator.evaluate(batch.as_ref()) {
549 Ok(data) => Some(data),
550 Err(err) => {
551 *error_slot.borrow_mut() = Some(err.into());
552 None
553 }
554 })
555 .fuse();
556
557 for res in inner.scan_metadata_from(
558 engine.as_ref(),
559 existing_version,
560 Box::new(scan_row_data),
561 existing_predicate,
562 )? {
563 if let Some(err) = evaluation_error.borrow_mut().take() {
564 return Err(err);
565 }
566 if tx.blocking_send(Ok(res?)).is_err() {
567 break;
568 }
569 }
570 if let Some(err) = evaluation_error.borrow_mut().take() {
571 return Err(err);
572 }
573 Ok(())
574 };
575
576 builder.spawn_blocking(scan_inner);
577 builder.build()
578 }
579}