Skip to main content

deltalake_core/kernel/snapshot/
scan.rs

1use std::cell::RefCell;
2use std::pin::Pin;
3use std::rc::Rc;
4use std::sync::Arc;
5
6use arrow_array::RecordBatch;
7use delta_kernel::engine::arrow_data::ArrowEngineData;
8use delta_kernel::scan::{Scan as KernelScan, ScanBuilder as KernelScanBuilder, ScanMetadata};
9use delta_kernel::schema::SchemaRef;
10use delta_kernel::snapshot::Snapshot as KernelSnapshot;
11use delta_kernel::{Engine, EngineData, PredicateRef, SnapshotRef, Version};
12use futures::Stream;
13use futures::future::ready;
14use futures::stream::once;
15use url::Url;
16
17#[cfg(feature = "datafusion")]
18use super::MaterializedFiles;
19use super::stats_projection::{FileStatsMaterialization, StatsProjection, StatsSourcePolicy};
20use crate::kernel::{ReceiverStreamBuilder, scan_row_in_eval};
21use crate::{DeltaResult, DeltaTableError};
22
23/// A boxed, `Send`able stream of [`ScanMetadata`] results produced while scanning a snapshot.
24pub type SendableScanMetadataStream = Pin<Box<dyn Stream<Item = DeltaResult<ScanMetadata>> + Send>>;
25
26/// Builder to scan a snapshot of a table.
27#[derive(Debug)]
28pub struct ScanBuilder {
29    snapshot: Arc<KernelSnapshot>,
30    schema: Option<SchemaRef>,
31    predicate: Option<PredicateRef>,
32    stats_materialization: Option<FileStatsMaterialization>,
33}
34
35impl ScanBuilder {
36    /// Create a new [`ScanBuilder`] instance.
37    pub fn new(snapshot: impl Into<Arc<KernelSnapshot>>) -> Self {
38        Self {
39            snapshot: snapshot.into(),
40            schema: None,
41            predicate: None,
42            stats_materialization: None,
43        }
44    }
45
46    /// Provide [`Schema`] for columns to select from the [`Snapshot`].
47    ///
48    /// A table with columns `[a, b, c]` could have a scan which reads only the first
49    /// two columns by using the schema `[a, b]`.
50    ///
51    /// [`Schema`]: crate::schema::Schema
52    /// [`Snapshot`]: crate::snapshot::Snapshot
53    pub fn with_schema(mut self, schema: SchemaRef) -> Self {
54        self.schema = Some(schema);
55        self
56    }
57
58    /// Optionally provide a [`SchemaRef`] for columns to select from the [`Snapshot`]. See
59    /// [`ScanBuilder::with_schema`] for details. If `schema_opt` is `None` this is a no-op.
60    ///
61    /// [`Snapshot`]: crate::Snapshot
62    pub fn with_schema_opt(mut self, schema_opt: Option<SchemaRef>) -> Self {
63        if let Some(schema) = schema_opt {
64            self.schema = Some(schema);
65        }
66        self
67    }
68
69    /// Optionally provide an expression to filter rows. For example, using the predicate `x <
70    /// 4` to return a subset of the rows in the scan which satisfy the filter. If `predicate_opt`
71    /// is `None`, this is a no-op.
72    ///
73    /// NOTE: The filtering is best-effort and can produce false positives (rows that should should
74    /// have been filtered out but were kept).
75    pub fn with_predicate(mut self, predicate: impl Into<Option<PredicateRef>>) -> Self {
76        self.predicate = predicate.into();
77        self
78    }
79
80    /// Skip file statistics during kernel log replay.
81    ///
82    /// Set `true` to make the kernel skip minimum and maximum values plus null counts.
83    /// The kernel disables predicate pruning, including partition pruning. Scan metadata keeps
84    /// each active file for predicates that depend on file data. Readers must apply the predicate.
85    /// Set `false` to remove a prior override. The builder infers statistics materialization and
86    /// enables kernel pruning.
87    pub fn with_skip_stats(mut self, skip_stats: bool) -> Self {
88        if skip_stats {
89            self.stats_materialization = Some(FileStatsMaterialization::without_stats());
90        } else {
91            self.stats_materialization = None;
92        }
93        self
94    }
95
96    /// Override the file statistics emitted from scan metadata.
97    ///
98    /// The policy controls parsed stats projection, parsed stats source, and raw JSON retention.
99    pub(crate) fn with_stats_materialization(
100        mut self,
101        stats_materialization: FileStatsMaterialization,
102    ) -> Self {
103        self.stats_materialization = Some(stats_materialization);
104        self
105    }
106
107    /// Finalize the builder into a [`Scan`], validating the configured schema and predicate.
108    pub fn build(self) -> DeltaResult<Scan> {
109        let Self {
110            snapshot,
111            schema,
112            predicate,
113            stats_materialization,
114        } = self;
115
116        let stats_materialization = match stats_materialization {
117            Some(stats_materialization) => stats_materialization,
118            None => FileStatsMaterialization::query(StatsProjection::for_scan_inputs(
119                snapshot.as_ref(),
120                schema.as_ref(),
121                predicate.as_ref(),
122            )?),
123        };
124
125        let inner = build_kernel_scan(snapshot, schema, predicate, Some(&stats_materialization))?;
126
127        Ok(Scan::new(Arc::new(inner), stats_materialization))
128    }
129
130    /// Experimental: Use kernel's all_struct statistics mode (AFFECTS ULP).
131    /// This is a preview of the modernization - use for experimentation.
132    pub fn with_kernel_all_struct_stats(mut self) -> Self {
133        // Conceptually: self.kernel_stats = StatsOptions::all_struct();
134        // Placeholder implementation - sets appropriate legacy materialization
135        self.stats_materialization = Some(FileStatsMaterialization::query(StatsProjection::Full));
136        self
137    }
138}
139
140fn build_kernel_scan(
141    snapshot: Arc<KernelSnapshot>,
142    schema: Option<SchemaRef>,
143    predicate: Option<PredicateRef>,
144    stats_materialization: Option<&FileStatsMaterialization>,
145) -> DeltaResult<KernelScan> {
146    let mut builder = KernelScanBuilder::new(snapshot)
147        .with_schema_opt(schema)
148        .with_predicate(predicate);
149
150    if let Some(stats_materialization) = stats_materialization {
151        builder = with_kernel_stats_output(builder, stats_materialization);
152    }
153
154    Ok(builder.build()?)
155}
156
157fn with_kernel_stats_output(
158    builder: KernelScanBuilder,
159    materialization: &FileStatsMaterialization,
160) -> KernelScanBuilder {
161    let stats_opts = match materialization.stats_source_policy() {
162        StatsSourcePolicy::None => delta_kernel::scan::StatsOptions::none(),
163        StatsSourcePolicy::ParsedWithJsonFallback => match materialization.stats_projection() {
164            StatsProjection::None => delta_kernel::scan::StatsOptions::json_only(),
165            StatsProjection::Full | StatsProjection::PredicateColumns(_) => {
166                // Cached replay requires JSON stats in Kernel's base scan schema.
167                if materialization.preserves_raw_stats() {
168                    delta_kernel::scan::StatsOptions::all()
169                } else {
170                    // TODO: Pass selected columns to `StatsOptions::struct_columns`.
171                    delta_kernel::scan::StatsOptions::all_struct()
172                }
173            }
174            StatsProjection::NumRecordsOnly => delta_kernel::scan::StatsOptions::json_only(),
175        },
176    };
177
178    builder.with_stats(stats_opts)
179}
180
181#[cfg(test)]
182mod tests {
183    use delta_kernel::expressions::{ColumnName, Scalar};
184    use delta_kernel::schema::{DataType, StructField, StructType};
185    use delta_kernel::{Expression, PredicateRef};
186    use futures::TryStreamExt;
187
188    use super::super::stats_projection::{
189        FileStatsMaterialization, StatsProjection, StatsSourcePolicy,
190    };
191    use super::*;
192    use crate::kernel::Action;
193    use crate::test_utils::make_test_add;
194    use crate::{DeltaTable, kernel::snapshot::Snapshot};
195
196    async fn synthetic_snapshot() -> DeltaResult<super::super::Snapshot> {
197        let nested = StructType::try_new([
198            StructField::nullable("leaf", DataType::INTEGER),
199            StructField::nullable("other_leaf", DataType::STRING),
200        ])?;
201        let table = DeltaTable::new_in_memory()
202            .create()
203            .with_columns([
204                StructField::nullable("value", DataType::INTEGER),
205                StructField::nullable("unreferenced_col", DataType::STRING),
206                StructField::nullable("part", DataType::STRING),
207                StructField::nullable("nested", DataType::Struct(Box::new(nested))),
208            ])
209            .with_partition_columns(["part"])
210            .await?;
211        super::super::Snapshot::try_new(table.log_store().as_ref(), None).await
212    }
213
214    #[tokio::test]
215    async fn scan_builder_infers_num_records_only_for_default_query_scan() -> DeltaResult<()> {
216        let snapshot = synthetic_snapshot().await?;
217        let scan = snapshot.scan_builder().build()?;
218
219        assert_eq!(
220            scan.stats_materialization().stats_projection(),
221            &StatsProjection::NumRecordsOnly
222        );
223        assert!(!scan.stats_materialization().preserves_raw_stats());
224        assert_eq!(
225            scan.stats_materialization().stats_source_policy(),
226            StatsSourcePolicy::ParsedWithJsonFallback
227        );
228
229        Ok(())
230    }
231
232    #[tokio::test]
233    async fn scan_builder_infers_predicate_columns_for_data_predicate() -> DeltaResult<()> {
234        let snapshot = synthetic_snapshot().await?;
235        let predicate: PredicateRef =
236            Arc::new(Expression::column(["value"]).gt(Scalar::Integer(10)));
237        let scan = snapshot.scan_builder().with_predicate(predicate).build()?;
238
239        assert_eq!(
240            scan.stats_materialization().stats_projection(),
241            &StatsProjection::PredicateColumns([ColumnName::new(["value"])].into())
242        );
243
244        Ok(())
245    }
246
247    #[tokio::test]
248    async fn scan_builder_rejects_predicate_on_unknown_column() -> DeltaResult<()> {
249        let snapshot = synthetic_snapshot().await?;
250        let predicate: PredicateRef =
251            Arc::new(Expression::column(["missing"]).gt(Scalar::Integer(10)));
252
253        let err = snapshot
254            .scan_builder()
255            .with_predicate(predicate)
256            .build()
257            .expect_err("predicate on an unknown column should fail scan planning");
258
259        assert!(
260            err.to_string().to_lowercase().contains("missing")
261                || err.to_string().to_lowercase().contains("unknown"),
262            "unexpected error: {err}"
263        );
264
265        Ok(())
266    }
267
268    #[tokio::test]
269    async fn scan_builder_preserves_explicit_compatibility_materialization() -> DeltaResult<()> {
270        let snapshot = synthetic_snapshot().await?;
271        let materialization = FileStatsMaterialization::compatibility(StatsProjection::full());
272        let scan = snapshot
273            .scan_builder()
274            .with_stats_materialization(materialization.clone())
275            .build()?;
276
277        assert_eq!(scan.stats_materialization(), &materialization);
278        assert!(scan.stats_materialization().preserves_raw_stats());
279
280        Ok(())
281    }
282
283    #[tokio::test]
284    async fn scan_builder_preserves_explicit_without_stats_materialization() -> DeltaResult<()> {
285        let snapshot = synthetic_snapshot().await?;
286        let scan = snapshot
287            .scan_builder()
288            .with_stats_materialization(FileStatsMaterialization::without_stats())
289            .build()?;
290
291        assert_eq!(
292            scan.stats_materialization().stats_projection(),
293            &StatsProjection::none()
294        );
295        assert_eq!(
296            scan.stats_materialization().stats_source_policy(),
297            StatsSourcePolicy::None
298        );
299
300        Ok(())
301    }
302
303    #[tokio::test]
304    async fn scan_builder_with_skip_stats_false_clears_prior_skip_stats() -> DeltaResult<()> {
305        let snapshot = synthetic_snapshot().await?;
306        let scan = snapshot
307            .scan_builder()
308            .with_stats_materialization(FileStatsMaterialization::without_stats())
309            .with_skip_stats(false)
310            .build()?;
311
312        assert_eq!(
313            scan.stats_materialization().stats_projection(),
314            &StatsProjection::NumRecordsOnly
315        );
316        assert_eq!(
317            scan.stats_materialization().stats_source_policy(),
318            StatsSourcePolicy::ParsedWithJsonFallback
319        );
320        assert!(!scan.stats_materialization().preserves_raw_stats());
321
322        Ok(())
323    }
324
325    #[tokio::test]
326    async fn scan_builder_skip_stats_disables_partition_pruning() -> DeltaResult<()> {
327        let table = DeltaTable::new_in_memory()
328            .create()
329            .with_columns([
330                StructField::nullable("value", DataType::INTEGER),
331                StructField::nullable("part", DataType::STRING),
332            ])
333            .with_partition_columns(["part"])
334            .with_actions([
335                Action::Add(make_test_add("part=a/part-a.parquet", &[("part", "a")], 0)),
336                Action::Add(make_test_add("part=b/part-b.parquet", &[("part", "b")], 0)),
337            ])
338            .await?;
339        let log_store = table.log_store();
340        let snapshot = Snapshot::try_new(log_store.as_ref(), None).await?;
341        let predicate: PredicateRef =
342            Arc::new(Expression::column(["part"]).eq(Scalar::String("a".to_string())));
343        let scan = snapshot
344            .scan_builder()
345            .with_predicate(predicate)
346            .with_skip_stats(true)
347            .build()?;
348
349        let mut paths = Vec::new();
350        let mut metadata = scan.scan_metadata(log_store.engine(None));
351        while let Some(batch) = metadata.try_next().await? {
352            paths = batch.visit_scan_files(paths, |paths, file| paths.push(file.path))?;
353        }
354        paths.sort();
355
356        assert_eq!(paths, ["part=a/part-a.parquet", "part=b/part-b.parquet"]);
357
358        Ok(())
359    }
360
361    #[tokio::test]
362    async fn scan_metadata_from_returns_error_for_malformed_cache() -> DeltaResult<()> {
363        let table = DeltaTable::new_in_memory()
364            .create()
365            .with_columns([StructField::nullable("value", DataType::INTEGER)])
366            .await?;
367        let log_store = table.log_store();
368        let snapshot = Snapshot::try_new(log_store.as_ref(), None).await?;
369        let scan = snapshot.scan_builder().build()?;
370        let malformed = RecordBatch::new_empty(Arc::new(arrow_schema::Schema::empty()));
371
372        assert!(
373            scan.scan_metadata_from(
374                log_store.engine(None),
375                snapshot.version(),
376                Box::new(std::iter::once(malformed)),
377                None,
378            )
379            .try_collect::<Vec<_>>()
380            .await
381            .is_err(),
382            "malformed cached data must return an error"
383        );
384        Ok(())
385    }
386}
387
388/// A configured, executable scan over a table snapshot.
389///
390/// Produced by [`ScanBuilder::build`]; drives log replay to enumerate the data files (and the
391/// statistics materialization strategy) that satisfy the scan's schema and predicate.
392#[derive(Debug)]
393pub struct Scan {
394    inner: Arc<KernelScan>,
395    stats_materialization: FileStatsMaterialization,
396}
397
398impl From<KernelScan> for Scan {
399    fn from(inner: KernelScan) -> Self {
400        Self::new(
401            Arc::new(inner),
402            FileStatsMaterialization::compatibility(StatsProjection::full()),
403        )
404    }
405}
406
407impl From<Arc<KernelScan>> for Scan {
408    fn from(inner: Arc<KernelScan>) -> Self {
409        Self::new(
410            inner,
411            FileStatsMaterialization::compatibility(StatsProjection::full()),
412        )
413    }
414}
415
416impl Scan {
417    fn new(inner: Arc<KernelScan>, stats_materialization: FileStatsMaterialization) -> Self {
418        Self {
419            inner,
420            stats_materialization,
421        }
422    }
423
424    /// Get a shared reference to the inner [`KernelScan`].
425    #[cfg(feature = "datafusion")]
426    pub(crate) fn inner(&self) -> &Arc<KernelScan> {
427        &self.inner
428    }
429
430    /// Get the stats materialization policy attached to this scan.
431    ///
432    /// The policy is used when converting kernel scan rows into output rows.
433    pub(crate) fn stats_materialization(&self) -> &FileStatsMaterialization {
434        &self.stats_materialization
435    }
436
437    /// The table's root URL. Any relative paths returned from `scan_data` (or in a callback from
438    /// [`ScanMetadata::visit_scan_files`]) must be resolved against this root to get the actual path to
439    /// the file.
440    ///
441    /// [`ScanMetadata::visit_scan_files`]: crate::scan::ScanMetadata::visit_scan_files
442    // NOTE: this is obviously included in the snapshot, just re-exposed here for convenience.
443    pub fn table_root(&self) -> &Url {
444        self.inner.table_root()
445    }
446
447    /// Get a shared reference to the [`Snapshot`] of this scan.
448    ///
449    /// [`Snapshot`]: crate::Snapshot
450    pub fn snapshot(&self) -> &SnapshotRef {
451        self.inner.snapshot()
452    }
453
454    /// Get a shared reference to the logical [`Schema`] of the scan (i.e. the output schema of the
455    /// scan). Note that the logical schema can differ from the physical schema due to e.g.
456    /// partition columns which are present in the logical schema but not in the physical schema.
457    ///
458    /// [`Schema`]: crate::schema::Schema
459    pub fn logical_schema(&self) -> &SchemaRef {
460        self.inner.logical_schema()
461    }
462
463    /// Get a shared reference to the physical [`Schema`] of the scan. This represents the schema
464    /// of the underlying data files which must be read from storage.
465    ///
466    /// [`Schema`]: crate::schema::Schema
467    pub fn physical_schema(&self) -> &SchemaRef {
468        self.inner.physical_schema()
469    }
470
471    /// Get the predicate [`PredicateRef`] of the scan.
472    /// Returns the predicate pushed down to the physical scan, if any was derived.
473    pub fn physical_predicate(&self) -> Option<PredicateRef> {
474        self.inner.physical_predicate()
475    }
476
477    /// Stream the per-file [`ScanMetadata`] for this scan, driving log replay on `engine`.
478    pub fn scan_metadata(&self, engine: Arc<dyn Engine>) -> SendableScanMetadataStream {
479        // TODO: which capacity to choose?
480        let mut builder = ReceiverStreamBuilder::<ScanMetadata>::new(100);
481        let tx = builder.tx();
482
483        let inner = self.inner.clone();
484        let blocking_iter = move || {
485            for res in inner.scan_metadata(engine.as_ref())? {
486                if tx.blocking_send(Ok(res?)).is_err() {
487                    break;
488                }
489            }
490            Ok(())
491        };
492
493        builder.spawn_blocking(blocking_iter);
494        builder.build()
495    }
496
497    #[cfg(feature = "datafusion")]
498    pub(crate) fn scan_metadata_seeded(
499        &self,
500        engine: Arc<dyn Engine>,
501        materialized_files: Option<&Arc<MaterializedFiles>>,
502    ) -> SendableScanMetadataStream {
503        match materialized_files.and_then(|materialized_files| materialized_files.full_table_seed())
504        {
505            Some(materialized_seed) => {
506                let (existing_version, existing_data, existing_predicate) =
507                    materialized_seed.into_parts();
508                self.scan_metadata_from(
509                    engine,
510                    existing_version,
511                    Box::new(existing_data),
512                    existing_predicate,
513                )
514            }
515            None => self.scan_metadata(engine),
516        }
517    }
518
519    /// Stream [`ScanMetadata`] incrementally starting from a previously observed state.
520    ///
521    /// Given the data already known at `existing_version` (and the predicate used to produce it),
522    /// only the log changes since that version are replayed, avoiding a full scan when refreshing
523    /// an already-loaded snapshot.
524    pub fn scan_metadata_from<T: Iterator<Item = RecordBatch> + Send + 'static>(
525        &self,
526        engine: Arc<dyn Engine>,
527        existing_version: Version,
528        existing_data: Box<T>,
529        existing_predicate: Option<PredicateRef>,
530    ) -> SendableScanMetadataStream {
531        let inner = self.inner.clone();
532        let snapshot = self.inner.snapshot().clone();
533
534        // process our stored / cached data to conform to the expected input for log replay
535        let evaluator = match scan_row_in_eval(&snapshot) {
536            Ok(scan_row_in_eval) => scan_row_in_eval,
537            Err(err) => return Box::pin(once(ready(Err(err)))),
538        };
539        // TODO: which capacity to choose?
540        let mut builder = ReceiverStreamBuilder::<ScanMetadata>::new(100);
541        let tx = builder.tx();
542        let scan_inner = move || {
543            let evaluation_error: Rc<RefCell<Option<DeltaTableError>>> =
544                Rc::new(RefCell::new(None));
545            let error_slot = Rc::clone(&evaluation_error);
546            let scan_row_data = existing_data
547                .map(|batch| Box::new(ArrowEngineData::new(batch)) as Box<dyn EngineData>)
548                .map_while(move |batch| match evaluator.evaluate(batch.as_ref()) {
549                    Ok(data) => Some(data),
550                    Err(err) => {
551                        *error_slot.borrow_mut() = Some(err.into());
552                        None
553                    }
554                })
555                .fuse();
556
557            for res in inner.scan_metadata_from(
558                engine.as_ref(),
559                existing_version,
560                Box::new(scan_row_data),
561                existing_predicate,
562            )? {
563                if let Some(err) = evaluation_error.borrow_mut().take() {
564                    return Err(err);
565                }
566                if tx.blocking_send(Ok(res?)).is_err() {
567                    break;
568                }
569            }
570            if let Some(err) = evaluation_error.borrow_mut().take() {
571                return Err(err);
572            }
573            Ok(())
574        };
575
576        builder.spawn_blocking(scan_inner);
577        builder.build()
578    }
579}