Skip to main content

deltalake_core/delta_datafusion/
mod.rs

1//! Datafusion integration for Delta Table
2//!
3//! Example:
4//!
5//! ```rust
6//! use std::sync::Arc;
7//! use datafusion::execution::context::SessionContext;
8//!
9//! async {
10//!   let mut ctx = SessionContext::new();
11//!   let table = deltalake_core::open_table_with_storage_options(
12//!       url::Url::parse("memory://").unwrap(),
13//!       std::collections::HashMap::new()
14//!   )
15//!       .await
16//!       .unwrap();
17//!   ctx.register_table("demo", table.table_provider().await.unwrap()).unwrap();
18//!
19//!   let batches = ctx
20//!       .sql("SELECT * FROM demo").await.unwrap()
21//!       .collect()
22//!       .await.unwrap();
23//! };
24//! ```
25
26use std::fmt::Debug;
27use std::sync::Arc;
28
29use arrow::array::types::UInt16Type;
30use arrow::array::{Array, DictionaryArray, RecordBatch, StringArray, TypedDictionaryArray};
31use arrow_cast::{CastOptions, cast_with_options};
32use arrow_schema::{
33    DataType as ArrowDataType, Schema as ArrowSchema, SchemaRef, SchemaRef as ArrowSchemaRef,
34    TimeUnit,
35};
36use datafusion::catalog::{Session, TableProviderFactory};
37use datafusion::common::scalar::ScalarValue;
38use datafusion::common::{
39    Column, DFSchema, DataFusionError, Result as DataFusionResult, TableReference, ToDFSchema,
40};
41use datafusion::datasource::TableProvider;
42use datafusion::datasource::physical_plan::wrap_partition_type_in_dict;
43use datafusion::execution::TaskContext;
44use datafusion::execution::context::SessionContext;
45use datafusion::logical_expr::logical_plan::CreateExternalTable;
46use datafusion::logical_expr::utils::conjunction;
47use datafusion::logical_expr::{Expr, Extension, LogicalPlan};
48use datafusion::physical_optimizer::pruning::PruningPredicateBuilder;
49use datafusion::physical_plan::ExecutionPlan;
50use datafusion_proto::logical_plan::LogicalExtensionCodec;
51use datafusion_proto::physical_plan::{PhysicalExtensionCodec, PhysicalProtoConverterExtension};
52use delta_kernel::engine::arrow_conversion::TryIntoArrow as _;
53use either::Either;
54
55use crate::delta_datafusion::expr::parse_predicate_expression;
56use crate::delta_datafusion::table_provider::{DeltaScan, DeltaScanWire};
57use crate::ensure_table_uri;
58use crate::errors::{DeltaResult, DeltaTableError};
59use crate::kernel::{Add, EagerSnapshot, LogDataHandler, Snapshot};
60use crate::{open_table, open_table_with_storage_options};
61
62pub(crate) use self::session::DeltaSessionExt;
63pub use self::session::{
64    DeltaParserOptions, DeltaRuntimeEnvBuilder, DeltaSessionConfig, DeltaSessionContext,
65    create_session, create_session_state_with_spill_config,
66};
67pub use self::table_provider::next::{
68    DeletionVectorSelection, DeltaScan as DeltaScanNext, FileSelection, MissingSelectedFilePolicy,
69};
70pub(crate) use self::utils::*;
71pub use cdf::scan::DeltaCdfTableProvider;
72pub(crate) use column_mapping::ColumnMappingState;
73pub(crate) use data_validation::{DataValidationExec, constraints_to_exprs, validation_predicates};
74pub(crate) use find_files::*;
75pub(crate) use table_provider::next::RuntimeFileFilter;
76pub(crate) use table_provider::next::normalize_path_as_file_id;
77pub use table_provider::{
78    DeltaScanConfig, DeltaScanConfigBuilder, TableProviderBuilder, next::DeltaScanExec,
79};
80pub(crate) use table_provider::{
81    next::FILE_ID_COLUMN_DEFAULT, resolve_file_column_name, update_datafusion_session,
82};
83
84pub(crate) const PATH_COLUMN: &str = "__delta_rs_path";
85
86#[doc(hidden)]
87pub mod bench_support;
88pub mod cdf;
89mod column_mapping;
90mod data_validation;
91/// DataFusion-backed kernel engine and its storage/format handlers.
92pub mod engine;
93pub mod expr;
94mod file_id;
95mod find_files;
96pub mod logical;
97pub mod physical;
98pub mod planner;
99mod session;
100pub use session::SessionFallbackPolicy;
101pub(crate) use session::{SessionResolveContext, resolve_session_state};
102mod table_provider;
103pub(crate) mod utils;
104
105impl From<DeltaTableError> for DataFusionError {
106    fn from(err: DeltaTableError) -> Self {
107        match err {
108            DeltaTableError::Arrow { source } => DataFusionError::from(source),
109            DeltaTableError::Io { source } => DataFusionError::IoError(source),
110            DeltaTableError::ObjectStore { source } => DataFusionError::from(source),
111            DeltaTableError::Parquet { source } => DataFusionError::from(source),
112            _ => DataFusionError::External(Box::new(err)),
113        }
114    }
115}
116
117impl From<DataFusionError> for DeltaTableError {
118    fn from(err: DataFusionError) -> Self {
119        match err {
120            DataFusionError::ArrowError(source, _) => DeltaTableError::from(*source),
121            DataFusionError::IoError(source) => DeltaTableError::Io { source },
122            DataFusionError::ObjectStore(source) => DeltaTableError::from(*source),
123            DataFusionError::ParquetError(source) => DeltaTableError::from(*source),
124            _ => DeltaTableError::Generic(err.to_string()),
125        }
126    }
127}
128
129/// Convenience trait for calling common methods on snapshot hierarchies
130pub trait DataFusionMixins {
131    /// The physical datafusion schema of a table
132    fn read_schema(&self) -> ArrowSchemaRef;
133
134    /// Get the table schema as an [`ArrowSchemaRef`]
135    fn input_schema(&self) -> ArrowSchemaRef;
136
137    /// Parse an expression string into a datafusion [`Expr`]
138    fn parse_predicate_expression(
139        &self,
140        expr: impl AsRef<str>,
141        session: &dyn Session,
142    ) -> DeltaResult<Expr>;
143}
144
145impl DataFusionMixins for Snapshot {
146    fn read_schema(&self) -> ArrowSchemaRef {
147        _arrow_schema(
148            self.arrow_schema(),
149            self.metadata().partition_columns(),
150            true,
151        )
152    }
153
154    fn input_schema(&self) -> ArrowSchemaRef {
155        _arrow_schema(
156            self.arrow_schema(),
157            self.metadata().partition_columns(),
158            false,
159        )
160    }
161
162    fn parse_predicate_expression(
163        &self,
164        expr: impl AsRef<str>,
165        session: &dyn Session,
166    ) -> DeltaResult<Expr> {
167        let schema = DFSchema::try_from(self.read_schema().as_ref().to_owned())?;
168        parse_predicate_expression(&schema, expr, session)
169    }
170}
171
172impl DataFusionMixins for LogDataHandler<'_> {
173    fn read_schema(&self) -> ArrowSchemaRef {
174        _arrow_schema(
175            Arc::new(
176                self.table_configuration()
177                    .logical_schema()
178                    .as_ref()
179                    .try_into_arrow()
180                    .unwrap(),
181            ),
182            self.table_configuration().metadata().partition_columns(),
183            true,
184        )
185    }
186
187    fn input_schema(&self) -> ArrowSchemaRef {
188        _arrow_schema(
189            Arc::new(
190                self.table_configuration()
191                    .logical_schema()
192                    .as_ref()
193                    .try_into_arrow()
194                    .unwrap(),
195            ),
196            self.table_configuration().metadata().partition_columns(),
197            false,
198        )
199    }
200
201    fn parse_predicate_expression(
202        &self,
203        expr: impl AsRef<str>,
204        session: &dyn Session,
205    ) -> DeltaResult<Expr> {
206        let schema = DFSchema::try_from(self.read_schema().as_ref().to_owned())?;
207        parse_predicate_expression(&schema, expr, session)
208    }
209}
210
211impl DataFusionMixins for EagerSnapshot {
212    fn read_schema(&self) -> ArrowSchemaRef {
213        self.snapshot().read_schema()
214    }
215
216    fn input_schema(&self) -> ArrowSchemaRef {
217        self.snapshot().input_schema()
218    }
219
220    fn parse_predicate_expression(
221        &self,
222        expr: impl AsRef<str>,
223        session: &dyn Session,
224    ) -> DeltaResult<Expr> {
225        self.snapshot().parse_predicate_expression(expr, session)
226    }
227}
228
229fn _arrow_schema(
230    schema: SchemaRef,
231    partition_columns: &[String],
232    wrap_partitions: bool,
233) -> ArrowSchemaRef {
234    let fields = schema
235        .fields()
236        .into_iter()
237        .filter(|f| !partition_columns.contains(&f.name().to_string()))
238        .cloned()
239        .chain(
240            // We need stable order between logical and physical schemas, but the order of
241            // partitioning columns is not always the same in the json schema and the array
242            partition_columns.iter().map(|partition_col| {
243                let field = schema.field_with_name(partition_col).unwrap();
244                let corrected = if wrap_partitions {
245                    match field.data_type() {
246                        // Only dictionary-encode types that may be large
247                        // https://github.com/apache/arrow-datafusion/pull/5545
248                        ArrowDataType::Utf8
249                        | ArrowDataType::LargeUtf8
250                        | ArrowDataType::Binary
251                        | ArrowDataType::LargeBinary => {
252                            wrap_partition_type_in_dict(field.data_type().clone())
253                        }
254                        _ => field.data_type().clone(),
255                    }
256                } else {
257                    field.data_type().clone()
258                };
259                Arc::new(field.clone().with_data_type(corrected))
260            }),
261        )
262        .collect::<Vec<_>>();
263    Arc::new(ArrowSchema::new(fields))
264}
265
266pub(crate) fn files_matching_predicate<'a>(
267    log_data: LogDataHandler<'a>,
268    filters: &[Expr],
269) -> DeltaResult<impl Iterator<Item = Add> + 'a> {
270    if let Some(Some(predicate)) =
271        (!filters.is_empty()).then_some(conjunction(filters.iter().cloned()))
272    {
273        let session: SessionContext = create_session().into();
274        let schema = log_data.read_schema();
275        let df_schema = Arc::new(schema.clone().to_dfschema()?);
276        let resolved = Expression::from(predicate).resolve(&session.state(), df_schema.clone())?;
277        let expr = session.create_physical_expr(resolved, &df_schema)?;
278        let pruning_predicate = PruningPredicateBuilder::new()
279            .with_file_schema(schema)
280            .try_build(expr)?;
281        let mask = pruning_predicate.prune(&log_data)?;
282
283        Ok(Either::Left(log_data.into_iter().zip(mask).filter_map(
284            |(file, keep_file)| {
285                if keep_file { Some(file.to_add()) } else { None }
286            },
287        )))
288    } else {
289        Ok(Either::Right(
290            log_data.into_iter().map(|file| file.to_add()),
291        ))
292    }
293}
294
295pub(crate) fn get_path_column<'a>(
296    batch: &'a RecordBatch,
297    path_column: &str,
298) -> DeltaResult<TypedDictionaryArray<'a, UInt16Type, StringArray>> {
299    let err = || DeltaTableError::Generic("Unable to obtain Delta-rs path column".to_string());
300    batch
301        .column_by_name(path_column)
302        .ok_or_else(err)?
303        .as_any()
304        .downcast_ref::<DictionaryArray<UInt16Type>>()
305        .ok_or_else(err)?
306        .downcast_dict::<StringArray>()
307        .ok_or_else(err)
308}
309
310pub(crate) fn get_null_of_arrow_type(t: &ArrowDataType) -> DeltaResult<ScalarValue> {
311    match t {
312        ArrowDataType::Null => Ok(ScalarValue::Null),
313        ArrowDataType::Boolean => Ok(ScalarValue::Boolean(None)),
314        ArrowDataType::Int8 => Ok(ScalarValue::Int8(None)),
315        ArrowDataType::Int16 => Ok(ScalarValue::Int16(None)),
316        ArrowDataType::Int32 => Ok(ScalarValue::Int32(None)),
317        ArrowDataType::Int64 => Ok(ScalarValue::Int64(None)),
318        ArrowDataType::UInt8 => Ok(ScalarValue::UInt8(None)),
319        ArrowDataType::UInt16 => Ok(ScalarValue::UInt16(None)),
320        ArrowDataType::UInt32 => Ok(ScalarValue::UInt32(None)),
321        ArrowDataType::UInt64 => Ok(ScalarValue::UInt64(None)),
322        ArrowDataType::Float32 => Ok(ScalarValue::Float32(None)),
323        ArrowDataType::Float64 => Ok(ScalarValue::Float64(None)),
324        ArrowDataType::Date32 => Ok(ScalarValue::Date32(None)),
325        ArrowDataType::Date64 => Ok(ScalarValue::Date64(None)),
326        ArrowDataType::Binary => Ok(ScalarValue::Binary(None)),
327        ArrowDataType::FixedSizeBinary(size) => {
328            Ok(ScalarValue::FixedSizeBinary(size.to_owned(), None))
329        }
330        ArrowDataType::LargeBinary => Ok(ScalarValue::LargeBinary(None)),
331        ArrowDataType::Utf8 => Ok(ScalarValue::Utf8(None)),
332        ArrowDataType::LargeUtf8 => Ok(ScalarValue::LargeUtf8(None)),
333        ArrowDataType::Decimal128(precision, scale) => Ok(ScalarValue::Decimal128(
334            None,
335            precision.to_owned(),
336            scale.to_owned(),
337        )),
338        ArrowDataType::Timestamp(unit, tz) => {
339            let tz = tz.to_owned();
340            Ok(match unit {
341                TimeUnit::Second => ScalarValue::TimestampSecond(None, tz),
342                TimeUnit::Millisecond => ScalarValue::TimestampMillisecond(None, tz),
343                TimeUnit::Microsecond => ScalarValue::TimestampMicrosecond(None, tz),
344                TimeUnit::Nanosecond => ScalarValue::TimestampNanosecond(None, tz),
345            })
346        }
347        ArrowDataType::Dictionary(k, v) => Ok(ScalarValue::Dictionary(
348            k.clone(),
349            Box::new(get_null_of_arrow_type(v)?),
350        )),
351        //Unsupported types...
352        ArrowDataType::Float16
353        | ArrowDataType::Decimal32(_, _)
354        | ArrowDataType::Decimal64(_, _)
355        | ArrowDataType::Decimal256(_, _)
356        | ArrowDataType::Union(_, _)
357        | ArrowDataType::LargeList(_)
358        | ArrowDataType::Struct(_)
359        | ArrowDataType::List(_)
360        | ArrowDataType::FixedSizeList(_, _)
361        | ArrowDataType::Time32(_)
362        | ArrowDataType::Time64(_)
363        | ArrowDataType::Duration(_)
364        | ArrowDataType::Interval(_)
365        | ArrowDataType::RunEndEncoded(_, _)
366        | ArrowDataType::BinaryView
367        | ArrowDataType::Utf8View
368        | ArrowDataType::LargeListView(_)
369        | ArrowDataType::ListView(_)
370        | ArrowDataType::Map(_, _) => Err(DeltaTableError::Generic(format!(
371            "Unsupported data type for Delta Lake {t}"
372        ))),
373    }
374}
375
376fn parse_date(
377    stat_val: &serde_json::Value,
378    field_dt: &ArrowDataType,
379) -> DataFusionResult<ScalarValue> {
380    let string = match stat_val {
381        serde_json::Value::String(s) => s.to_owned(),
382        _ => stat_val.to_string(),
383    };
384
385    let time_micro = ScalarValue::try_from_string(string, &ArrowDataType::Date32)?;
386    let cast_arr = cast_with_options(
387        &time_micro.to_array()?,
388        field_dt,
389        &CastOptions {
390            safe: false,
391            ..Default::default()
392        },
393    )?;
394    ScalarValue::try_from_array(&cast_arr, 0)
395}
396
397fn parse_timestamp(
398    stat_val: &serde_json::Value,
399    field_dt: &ArrowDataType,
400) -> DataFusionResult<ScalarValue> {
401    let string = match stat_val {
402        serde_json::Value::String(s) => s.to_owned(),
403        _ => stat_val.to_string(),
404    };
405
406    let time_micro = ScalarValue::try_from_string(
407        string,
408        &ArrowDataType::Timestamp(TimeUnit::Microsecond, None),
409    )?;
410    let cast_arr = cast_with_options(
411        &time_micro.to_array()?,
412        field_dt,
413        &CastOptions {
414            safe: false,
415            ..Default::default()
416        },
417    )?;
418    ScalarValue::try_from_array(&cast_arr, 0)
419}
420
421pub(crate) fn to_correct_scalar_value(
422    stat_val: &serde_json::Value,
423    field_dt: &ArrowDataType,
424) -> DataFusionResult<Option<ScalarValue>> {
425    match stat_val {
426        serde_json::Value::Array(_) => Ok(None),
427        serde_json::Value::Object(_) => Ok(None),
428        serde_json::Value::Null => Ok(Some(get_null_of_arrow_type(field_dt)?)),
429        serde_json::Value::String(string_val) => match field_dt {
430            ArrowDataType::Timestamp(_, _) => Ok(Some(parse_timestamp(stat_val, field_dt)?)),
431            ArrowDataType::Date32 => Ok(Some(parse_date(stat_val, field_dt)?)),
432            _ => Ok(Some(ScalarValue::try_from_string(
433                string_val.to_owned(),
434                field_dt,
435            )?)),
436        },
437        other => match field_dt {
438            ArrowDataType::Timestamp(_, _) => Ok(Some(parse_timestamp(stat_val, field_dt)?)),
439            ArrowDataType::Date32 => Ok(Some(parse_date(stat_val, field_dt)?)),
440            _ => Ok(Some(ScalarValue::try_from_string(
441                other.to_string(),
442                field_dt,
443            )?)),
444        },
445    }
446}
447
448/// Legacy codec for serialized plans that still contain the retired physical
449/// [`DeltaScan`] wrapper.
450#[deprecated(
451    note = "DeltaPhysicalCodec only supports the retired physical DeltaScan wrapper. \
452            Use DeltaLogicalCodec for table-provider serialization until a DeltaScanExec \
453            physical codec is available."
454)]
455#[derive(Debug)]
456pub struct DeltaPhysicalCodec {}
457
458#[allow(deprecated)]
459impl PhysicalExtensionCodec for DeltaPhysicalCodec {
460    fn try_decode(
461        &self,
462        buf: &[u8],
463        inputs: &[Arc<dyn ExecutionPlan>],
464        _registry: &TaskContext,
465        _converter: &dyn PhysicalProtoConverterExtension,
466    ) -> Result<Arc<dyn ExecutionPlan>, DataFusionError> {
467        let wire: DeltaScanWire = serde_json::from_reader(buf)
468            .map_err(|_| DataFusionError::Internal("Unable to decode DeltaScan".to_string()))?;
469        let delta_scan = wire.into_delta_scan((*inputs)[0].clone());
470        Ok(Arc::new(delta_scan))
471    }
472
473    fn try_encode(
474        &self,
475        node: Arc<dyn ExecutionPlan>,
476        buf: &mut Vec<u8>,
477        _converter: &dyn PhysicalProtoConverterExtension,
478    ) -> Result<(), DataFusionError> {
479        let delta_scan = node
480            .downcast_ref::<DeltaScan>()
481            .ok_or_else(|| DataFusionError::Internal("Not a legacy delta scan!".to_string()))?;
482
483        let wire = DeltaScanWire::from(delta_scan);
484        serde_json::to_writer(buf, &wire).map_err(|_| {
485            DataFusionError::Internal("Unable to encode legacy delta scan!".to_string())
486        })?;
487        Ok(())
488    }
489}
490
491/// Does serde on DeltaTables
492#[derive(Debug)]
493pub struct DeltaLogicalCodec {}
494
495impl LogicalExtensionCodec for DeltaLogicalCodec {
496    fn try_decode(
497        &self,
498        _buf: &[u8],
499        _inputs: &[LogicalPlan],
500        _ctx: &TaskContext,
501    ) -> Result<Extension, DataFusionError> {
502        todo!("DeltaLogicalCodec")
503    }
504
505    fn try_encode(&self, _node: &Extension, _buf: &mut Vec<u8>) -> Result<(), DataFusionError> {
506        todo!("DeltaLogicalCodec")
507    }
508
509    fn try_decode_table_provider(
510        &self,
511        buf: &[u8],
512        _table_ref: &TableReference,
513        _schema: SchemaRef,
514        _ctx: &TaskContext,
515    ) -> Result<Arc<dyn TableProvider>, DataFusionError> {
516        let provider: DeltaScanNext = serde_json::from_slice(buf)
517            .map_err(|_| DataFusionError::Internal("Error encoding delta table".to_string()))?;
518        Ok(Arc::new(provider))
519    }
520
521    fn try_encode_table_provider(
522        &self,
523        _table_ref: &TableReference,
524        node: Arc<dyn TableProvider>,
525        buf: &mut Vec<u8>,
526    ) -> Result<(), DataFusionError> {
527        let scan = node.downcast_ref::<DeltaScanNext>().ok_or_else(|| {
528            DataFusionError::Internal("Can't encode non-delta tables".to_string())
529        })?;
530        serde_json::to_writer(buf, scan)
531            .map_err(|_| DataFusionError::Internal("Error encoding delta table".to_string()))
532    }
533}
534
535/// Responsible for creating deltatables
536#[derive(Debug)]
537pub struct DeltaTableFactory {}
538
539#[async_trait::async_trait]
540impl TableProviderFactory for DeltaTableFactory {
541    async fn create(
542        &self,
543        ctx: &dyn Session,
544        cmd: &CreateExternalTable,
545    ) -> datafusion::error::Result<Arc<dyn TableProvider>> {
546        let [location] = cmd.locations.as_slice() else {
547            return Err(DataFusionError::Plan(format!(
548                "Delta tables require one location, got {}",
549                cmd.locations.len()
550            )));
551        };
552        let table = if cmd.options.is_empty() {
553            let table_url = ensure_table_uri(location)?;
554            open_table(table_url).await?
555        } else {
556            let table_url = ensure_table_uri(location)?;
557            open_table_with_storage_options(table_url, cmd.to_owned().options).await?
558        };
559        let table_uri = table.log_store().root_url().clone();
560        let (session_state, _) = resolve_session_state(
561            Some(ctx),
562            SessionFallbackPolicy::DeriveFromTrait,
563            || create_session().state(),
564            SessionResolveContext {
565                operation: "DeltaTableFactory::create",
566                table_uri: Some(&table_uri),
567                cdc: false,
568            },
569        )?;
570
571        Ok(table
572            .table_provider()
573            .with_session(Arc::new(session_state))
574            .await?)
575    }
576}
577
578/// A wrapper for Deltafusion's Column to preserve case-sensitivity during string conversion
579pub struct DeltaColumn {
580    inner: Column,
581}
582
583impl From<&str> for DeltaColumn {
584    fn from(c: &str) -> Self {
585        DeltaColumn {
586            inner: Column::from_qualified_name_ignore_case(c),
587        }
588    }
589}
590
591/// Create a column, cloning the string
592impl From<&String> for DeltaColumn {
593    fn from(c: &String) -> Self {
594        DeltaColumn {
595            inner: Column::from_qualified_name_ignore_case(c),
596        }
597    }
598}
599
600/// Create a column, reusing the existing string
601impl From<String> for DeltaColumn {
602    fn from(c: String) -> Self {
603        DeltaColumn {
604            inner: Column::from_qualified_name_ignore_case(c),
605        }
606    }
607}
608
609impl From<DeltaColumn> for Column {
610    fn from(value: DeltaColumn) -> Self {
611        value.inner
612    }
613}
614
615/// Create a column, reusing the existing datafusion column
616impl From<Column> for DeltaColumn {
617    fn from(c: Column) -> Self {
618        DeltaColumn { inner: c }
619    }
620}
621
622#[cfg(test)]
623mod tests {
624    use crate::DeltaTable;
625    use crate::operations::write::SchemaMode;
626    use crate::test_utils::{
627        make_test_add,
628        object_store::{
629            RecordedObjectStoreOperation as ObjectStoreOperation, RecordedPathKind as PathKind,
630            drain_recorded_object_store_operations as drain_recorded_ops, recording_log_store,
631        },
632        open_fs_path,
633    };
634    use crate::writer::test_utils::get_delta_schema;
635    use arrow::array::StructArray;
636    use arrow::datatypes::{Field, Schema};
637    use datafusion::assert_batches_sorted_eq;
638    use datafusion::physical_plan::empty::EmptyExec;
639    use datafusion::prelude::SessionConfig;
640    use datafusion_proto::physical_plan::AsExecutionPlan;
641    use datafusion_proto::protobuf;
642    use delta_kernel::schema::ArrayType;
643    use futures::{StreamExt, TryStreamExt};
644    use object_store::ObjectStoreExt as _;
645    use serde_json::json;
646    use std::ops::Range;
647    use url::Url;
648
649    use super::*;
650    use crate::delta_datafusion::table_provider::next::{FileSelection, MissingSelectedFilePolicy};
651
652    #[tokio::test]
653    async fn delta_table_factory_rejects_location_counts_other_than_one() {
654        let ctx = SessionContext::new();
655        let factory = DeltaTableFactory {};
656
657        for locations in [vec![], vec!["one".to_string(), "two".to_string()]] {
658            let command = CreateExternalTable::builder(
659                TableReference::bare("invalid_locations"),
660                "unused",
661                "DELTATABLE",
662                Arc::new(DFSchema::empty()),
663            )
664            .with_locations(locations.clone())
665            .build();
666
667            let error = factory
668                .create(&ctx.state(), &command)
669                .await
670                .expect_err("DeltaTableFactory accepted an invalid location count");
671            assert!(matches!(error, DataFusionError::Plan(_)));
672            assert!(
673                error.to_string().contains("require one location"),
674                "unexpected error for {} locations: {error}",
675                locations.len()
676            );
677        }
678    }
679
680    // test deserialization of serialized partition values.
681    // https://github.com/delta-io/delta/blob/master/PROTOCOL.md#partition-value-serialization
682    #[test]
683    fn test_to_correct_scalar_value() {
684        let reference_pairs = &[
685            (
686                json!("2015"),
687                ArrowDataType::Int16,
688                ScalarValue::Int16(Some(2015)),
689            ),
690            (
691                json!("2015"),
692                ArrowDataType::Int32,
693                ScalarValue::Int32(Some(2015)),
694            ),
695            (
696                json!("2015"),
697                ArrowDataType::Int64,
698                ScalarValue::Int64(Some(2015)),
699            ),
700            (
701                json!("2015"),
702                ArrowDataType::Float32,
703                ScalarValue::Float32(Some(2015_f32)),
704            ),
705            (
706                json!("2015"),
707                ArrowDataType::Float64,
708                ScalarValue::Float64(Some(2015_f64)),
709            ),
710            (
711                json!(2015),
712                ArrowDataType::Float64,
713                ScalarValue::Float64(Some(2015_f64)),
714            ),
715            (
716                json!("2015-01-01"),
717                ArrowDataType::Date32,
718                ScalarValue::Date32(Some(16436)),
719            ),
720            // (
721            //     json!("2015-01-01"),
722            //     ArrowDataType::Date64,
723            //     ScalarValue::Date64(Some(16436)),
724            // ),
725            // TODO(roeap) there seem to be differences in how precisions are handled locally and in CI, need to investigate
726            // (
727            //     json!("2020-09-08 13:42:29"),
728            //     ArrowDataType::Timestamp(TimeUnit::Nanosecond, None),
729            //     ScalarValue::TimestampNanosecond(Some(1599565349000000000), None),
730            // ),
731            // (
732            //     json!("2020-09-08 13:42:29"),
733            //     ArrowDataType::Timestamp(TimeUnit::Microsecond, None),
734            //     ScalarValue::TimestampMicrosecond(Some(1599565349000000), None),
735            // ),
736            // (
737            //     json!("2020-09-08 13:42:29"),
738            //     ArrowDataType::Timestamp(TimeUnit::Millisecond, None),
739            //     ScalarValue::TimestampMillisecond(Some(1599565349000), None),
740            // ),
741            (
742                json!(true),
743                ArrowDataType::Boolean,
744                ScalarValue::Boolean(Some(true)),
745            ),
746        ];
747
748        for (raw, data_type, ref_scalar) in reference_pairs {
749            let scalar = to_correct_scalar_value(raw, data_type).unwrap().unwrap();
750            assert_eq!(*ref_scalar, scalar)
751        }
752    }
753
754    // REMOVED: Test used deprecated DeltaPhysicalCodec
755    // The deprecated DeltaPhysicalCodec only supports retired physical DeltaScan wrapper.
756    // This test should be replaced with a test using DeltaLogicalCodec or modern FileScan
757    // configuration if the functionality is still needed.
758    #[allow(dead_code)]
759    fn _removed_roundtrip_test_delta_exec_plan() {
760        // Test was removed due to use of deprecated DeltaPhysicalCodec
761        // Previous functionality tested serialization roundtrip of DeltaScan
762    }
763
764    #[tokio::test]
765    async fn roundtrip_test_delta_logical_codec_preserves_file_selection() {
766        let log_store = crate::test_utils::TestTables::Simple
767            .table_builder()
768            .unwrap()
769            .build_storage()
770            .unwrap();
771        let snapshot = Arc::new(
772            crate::kernel::Snapshot::try_new(&log_store, None)
773                .await
774                .unwrap(),
775        );
776        let table_root = snapshot.inner.table_root().clone();
777        let selected_file_ids: Vec<String> = snapshot
778            .file_views(log_store.as_ref(), None)
779            .take(1)
780            .map_ok(|view| table_root.join(view.path_raw()).unwrap().to_string())
781            .try_collect()
782            .await
783            .unwrap();
784        assert_eq!(selected_file_ids.len(), 1);
785
786        let provider = DeltaScanNext::builder()
787            .with_snapshot(snapshot)
788            .build()
789            .await
790            .unwrap()
791            .with_file_selection(
792                FileSelection::from_file_paths(selected_file_ids.clone())
793                    .with_missing_file_policy(MissingSelectedFilePolicy::Ignore),
794            );
795
796        let codec = DeltaLogicalCodec {};
797        let table_ref = TableReference::bare("delta_table");
798        let mut encoded = Vec::new();
799        codec
800            .try_encode_table_provider(&table_ref, Arc::new(provider), &mut encoded)
801            .unwrap();
802
803        let ctx = SessionContext::new();
804        let decoded = codec
805            .try_decode_table_provider(
806                &encoded,
807                &table_ref,
808                Arc::new(ArrowSchema::empty()),
809                &ctx.task_ctx(),
810            )
811            .unwrap();
812        let decoded_provider = decoded.downcast_ref::<DeltaScanNext>().unwrap();
813
814        let serialized = serde_json::to_value(decoded_provider).unwrap();
815        let decoded_file_ids = serialized
816            .get("file_selection")
817            .and_then(|value| value.get("paths"))
818            .and_then(|value| value.as_array())
819            .unwrap()
820            .iter()
821            .map(|value| value.as_str().unwrap().to_string())
822            .collect::<Vec<_>>();
823        let decoded_policy = serialized
824            .get("file_selection")
825            .and_then(|value| value.get("missing_file_policy"))
826            .and_then(|value| value.as_str())
827            .unwrap();
828
829        assert_eq!(decoded_file_ids, selected_file_ids);
830        assert_eq!(decoded_policy, "Ignore");
831    }
832
833    #[tokio::test]
834    async fn roundtrip_test_delta_logical_codec_serializes_file_selection_adds_as_identity_only() {
835        let log_store = crate::test_utils::TestTables::Simple
836            .table_builder()
837            .unwrap()
838            .build_storage()
839            .unwrap();
840        let snapshot = Arc::new(
841            crate::kernel::Snapshot::try_new(&log_store, None)
842                .await
843                .unwrap(),
844        );
845        let mut add = make_test_add("part=a/part-000.parquet", &[("part", "a")], 123);
846        add.stats = Some(r#"{"numRecords":999}"#.to_string());
847        add.tags = Some(std::collections::HashMap::from([(
848            "secret-tag".to_string(),
849            Some("secret-value".to_string()),
850        )]));
851        add.size = 999;
852
853        let provider = DeltaScanNext::builder()
854            .with_snapshot(snapshot)
855            .build()
856            .await
857            .unwrap()
858            .with_file_selection(FileSelection::from_adds([add]));
859
860        let codec = DeltaLogicalCodec {};
861        let table_ref = TableReference::bare("delta_table");
862        let mut encoded = Vec::new();
863        codec
864            .try_encode_table_provider(&table_ref, Arc::new(provider), &mut encoded)
865            .unwrap();
866
867        let ctx = SessionContext::new();
868        let decoded = codec
869            .try_decode_table_provider(
870                &encoded,
871                &table_ref,
872                Arc::new(ArrowSchema::empty()),
873                &ctx.task_ctx(),
874            )
875            .unwrap();
876        let decoded_provider = decoded.downcast_ref::<DeltaScanNext>().unwrap();
877        let serialized = serde_json::to_value(decoded_provider).unwrap();
878        let file_selection = serialized.get("file_selection").unwrap();
879        let file_selection_json = serde_json::to_string(file_selection).unwrap();
880
881        assert!(file_selection_json.contains("part=a/part-000.parquet"));
882        for forbidden in [
883            "partitionValues",
884            "numRecords",
885            "secret-tag",
886            "secret-value",
887            "modificationTime",
888            "\"size\"",
889        ] {
890            assert!(
891                !file_selection_json.contains(forbidden),
892                "FileSelection::from_adds serialized Add metadata field {forbidden}: {file_selection_json}"
893            );
894        }
895    }
896
897    #[tokio::test]
898    async fn roundtrip_test_delta_logical_codec_strips_file_selection_url_credentials() {
899        let log_store = crate::test_utils::TestTables::Simple
900            .table_builder()
901            .unwrap()
902            .build_storage()
903            .unwrap();
904        let snapshot = Arc::new(
905            crate::kernel::Snapshot::try_new(&log_store, None)
906                .await
907                .unwrap(),
908        );
909        let provider = DeltaScanNext::builder()
910            .with_snapshot(snapshot)
911            .build()
912            .await
913            .unwrap()
914            .with_file_selection(FileSelection::from_file_paths([
915                "https://urluser:urlpassword@example.com/table/part.parquet?token=urltoken#frag",
916            ]));
917
918        let codec = DeltaLogicalCodec {};
919        let table_ref = TableReference::bare("delta_table");
920        let mut encoded = Vec::new();
921        codec
922            .try_encode_table_provider(&table_ref, Arc::new(provider), &mut encoded)
923            .unwrap();
924
925        let ctx = SessionContext::new();
926        let decoded = codec
927            .try_decode_table_provider(
928                &encoded,
929                &table_ref,
930                Arc::new(ArrowSchema::empty()),
931                &ctx.task_ctx(),
932            )
933            .unwrap();
934        let decoded_provider = decoded.downcast_ref::<DeltaScanNext>().unwrap();
935        let serialized = serde_json::to_value(decoded_provider).unwrap();
936        let file_selection = serialized.get("file_selection").unwrap();
937        let file_selection_json = serde_json::to_string(file_selection).unwrap();
938
939        assert!(file_selection_json.contains("https://example.com/table/part.parquet"));
940        for forbidden in ["urluser", "urlpassword", "urltoken", "frag"] {
941            assert!(
942                !file_selection_json.contains(forbidden),
943                "encoded file selection contains {forbidden}: {file_selection_json}"
944            );
945        }
946    }
947
948    #[tokio::test]
949    async fn delta_table_provider_with_config() {
950        let table = open_fs_path("../test/tests/data/delta-2.2.0-partitioned-types");
951        let provider = table
952            .table_provider()
953            .with_file_column("file_source")
954            .await
955            .unwrap();
956        let ctx = SessionContext::new();
957        ctx.register_table("test", provider).unwrap();
958
959        let df = ctx
960            .sql("select c3, c1, c2, right(file_source, 77) as file_source from test")
961            .await
962            .unwrap();
963        let actual = df.collect().await.unwrap();
964        let expected = vec![
965            "+----+----+----+-------------------------------------------------------------------------------+",
966            "| c3 | c1 | c2 | file_source                                                                   |",
967            "+----+----+----+-------------------------------------------------------------------------------+",
968            "| 4  | 6  | a  | c1=6/c2=a/part-00011-10619b10-b691-4fd0-acc4-2a9608499d7c.c000.snappy.parquet |",
969            "| 5  | 4  | c  | c1=4/c2=c/part-00003-f525f459-34f9-46f5-82d6-d42121d883fd.c000.snappy.parquet |",
970            "| 6  | 5  | b  | c1=5/c2=b/part-00007-4e73fa3b-2c88-424a-8051-f8b54328ffdb.c000.snappy.parquet |",
971            "+----+----+----+-------------------------------------------------------------------------------+",
972        ];
973        assert_batches_sorted_eq!(&expected, &actual);
974    }
975
976    #[tokio::test]
977    async fn delta_scan_mixed_partition_order() {
978        // Tests issue (1787) where partition columns were incorrect when they
979        // have a different order in the metadata and table schema
980        let schema = Arc::new(ArrowSchema::new(vec![
981            Field::new("modified", ArrowDataType::Utf8, true),
982            Field::new("id", ArrowDataType::Utf8, true),
983            Field::new("value", ArrowDataType::Int32, true),
984        ]));
985
986        let table = DeltaTable::new_in_memory()
987            .create()
988            .with_columns(get_delta_schema().fields().cloned())
989            .with_partition_columns(["modified", "id"])
990            .await
991            .unwrap();
992        assert_eq!(table.version(), Some(0));
993
994        let batch = RecordBatch::try_new(
995            schema.clone(),
996            vec![
997                Arc::new(arrow::array::StringArray::from(vec![
998                    "2021-02-01",
999                    "2021-02-01",
1000                    "2021-02-02",
1001                    "2021-02-02",
1002                ])),
1003                Arc::new(arrow::array::StringArray::from(vec!["A", "B", "C", "D"])),
1004                Arc::new(arrow::array::Int32Array::from(vec![1, 10, 20, 100])),
1005            ],
1006        )
1007        .unwrap();
1008        // write some data
1009        let table = table
1010            .write(vec![batch.clone()])
1011            .with_save_mode(crate::protocol::SaveMode::Append)
1012            .await
1013            .unwrap();
1014
1015        let provider = table.table_provider().await.unwrap();
1016        let logical_schema = provider.schema();
1017        let ctx = SessionContext::new();
1018        ctx.runtime_env().register_object_store(
1019            table.log_store().root_url(),
1020            table.log_store().object_store(None),
1021        );
1022        ctx.register_table("test", provider).unwrap();
1023
1024        let expected_logical_order = vec!["id", "value", "modified"];
1025        let actual_order: Vec<String> = logical_schema
1026            .fields()
1027            .iter()
1028            .map(|f| f.name().to_owned())
1029            .collect();
1030
1031        let df = ctx.sql("select * from test").await.unwrap();
1032        let actual = df.collect().await.unwrap();
1033        let expected = vec![
1034            "+----+-------+------------+",
1035            "| id | value | modified   |",
1036            "+----+-------+------------+",
1037            "| A  | 1     | 2021-02-01 |",
1038            "| B  | 10    | 2021-02-01 |",
1039            "| C  | 20    | 2021-02-02 |",
1040            "| D  | 100   | 2021-02-02 |",
1041            "+----+-------+------------+",
1042        ];
1043        assert_batches_sorted_eq!(&expected, &actual);
1044        assert_eq!(expected_logical_order, actual_order);
1045    }
1046
1047    #[tokio::test]
1048    async fn delta_scan_case_sensitive() {
1049        let schema = Arc::new(ArrowSchema::new(vec![
1050            Field::new("moDified", ArrowDataType::Utf8, true),
1051            Field::new("ID", ArrowDataType::Utf8, true),
1052            Field::new("vaLue", ArrowDataType::Int32, true),
1053        ]));
1054
1055        let batch = RecordBatch::try_new(
1056            schema.clone(),
1057            vec![
1058                Arc::new(arrow::array::StringArray::from(vec![
1059                    "2021-02-01",
1060                    "2021-02-01",
1061                    "2021-02-02",
1062                    "2021-02-02",
1063                ])),
1064                Arc::new(arrow::array::StringArray::from(vec!["A", "B", "C", "D"])),
1065                Arc::new(arrow::array::Int32Array::from(vec![1, 10, 20, 100])),
1066            ],
1067        )
1068        .unwrap();
1069        // write some data
1070        let table = DeltaTable::new_in_memory()
1071            .write(vec![batch.clone()])
1072            .with_save_mode(crate::protocol::SaveMode::Append)
1073            .await
1074            .unwrap();
1075
1076        let provider = table.table_provider().await.unwrap();
1077        let ctx: SessionContext = DeltaSessionContext::default().into();
1078        ctx.register_table("test", provider).unwrap();
1079
1080        let df = ctx
1081            .sql("select ID, moDified, vaLue from test")
1082            .await
1083            .unwrap();
1084        let actual = df.collect().await.unwrap();
1085        let expected = vec![
1086            "+----+------------+-------+",
1087            "| ID | moDified   | vaLue |",
1088            "+----+------------+-------+",
1089            "| A  | 2021-02-01 | 1     |",
1090            "| B  | 2021-02-01 | 10    |",
1091            "| C  | 2021-02-02 | 20    |",
1092            "| D  | 2021-02-02 | 100   |",
1093            "+----+------------+-------+",
1094        ];
1095        assert_batches_sorted_eq!(&expected, &actual);
1096
1097        /* TODO: Datafusion doesn't have any options to prevent case-sensitivity with the col func */
1098        /*
1099        let df = ctx
1100            .table("test")
1101            .await
1102            .unwrap()
1103            .select(vec![col("ID"), col("moDified"), col("vaLue")])
1104            .unwrap();
1105        let actual = df.collect().await.unwrap();
1106        assert_batches_sorted_eq!(&expected, &actual);
1107        */
1108    }
1109
1110    #[tokio::test]
1111    async fn delta_scan_supports_missing_columns() {
1112        let schema1 = Arc::new(ArrowSchema::new(vec![Field::new(
1113            "col_1",
1114            ArrowDataType::Utf8,
1115            true,
1116        )]));
1117
1118        let batch1 = RecordBatch::try_new(
1119            schema1.clone(),
1120            vec![Arc::new(arrow::array::StringArray::from(vec![
1121                Some("A"),
1122                Some("B"),
1123            ]))],
1124        )
1125        .unwrap();
1126
1127        let schema2 = Arc::new(ArrowSchema::new(vec![
1128            Field::new("col_1", ArrowDataType::Utf8, true),
1129            Field::new("col_2", ArrowDataType::Utf8, true),
1130        ]));
1131
1132        let batch2 = RecordBatch::try_new(
1133            schema2.clone(),
1134            vec![
1135                Arc::new(arrow::array::StringArray::from(vec![
1136                    Some("E"),
1137                    Some("F"),
1138                    Some("G"),
1139                ])),
1140                Arc::new(arrow::array::StringArray::from(vec![
1141                    Some("E2"),
1142                    Some("F2"),
1143                    Some("G2"),
1144                ])),
1145            ],
1146        )
1147        .unwrap();
1148
1149        let table = DeltaTable::new_in_memory()
1150            .write(vec![batch2])
1151            .with_save_mode(crate::protocol::SaveMode::Append)
1152            .await
1153            .unwrap();
1154
1155        let table = table
1156            .write(vec![batch1])
1157            .with_schema_mode(SchemaMode::Merge)
1158            .with_save_mode(crate::protocol::SaveMode::Append)
1159            .await
1160            .unwrap();
1161
1162        let provider = table.table_provider().await.unwrap();
1163        let ctx: SessionContext = DeltaSessionContext::default().into();
1164        ctx.register_table("test", provider).unwrap();
1165
1166        let df = ctx.sql("select col_1, col_2 from test").await.unwrap();
1167        let actual = df.collect().await.unwrap();
1168        let expected = vec![
1169            "+-------+-------+",
1170            "| col_1 | col_2 |",
1171            "+-------+-------+",
1172            "| A     |       |",
1173            "| B     |       |",
1174            "| E     | E2    |",
1175            "| F     | F2    |",
1176            "| G     | G2    |",
1177            "+-------+-------+",
1178        ];
1179        assert_batches_sorted_eq!(&expected, &actual);
1180    }
1181
1182    #[tokio::test]
1183    async fn delta_scan_supports_pushdown() {
1184        let schema = Arc::new(ArrowSchema::new(vec![
1185            Field::new("col_1", ArrowDataType::Utf8, false),
1186            Field::new("col_2", ArrowDataType::Utf8, false),
1187        ]));
1188
1189        let batch = RecordBatch::try_new(
1190            schema.clone(),
1191            vec![
1192                Arc::new(arrow::array::StringArray::from(vec![
1193                    Some("A"),
1194                    Some("B"),
1195                    Some("C"),
1196                ])),
1197                Arc::new(arrow::array::StringArray::from(vec![
1198                    Some("A2"),
1199                    Some("B2"),
1200                    Some("C2"),
1201                ])),
1202            ],
1203        )
1204        .unwrap();
1205
1206        let table = DeltaTable::new_in_memory()
1207            .write(vec![batch])
1208            .with_save_mode(crate::protocol::SaveMode::Append)
1209            .await
1210            .unwrap();
1211
1212        let provider = table.table_provider().await.unwrap();
1213
1214        let mut cfg = SessionConfig::default();
1215        cfg.options_mut().execution.parquet.pushdown_filters = true;
1216        let ctx = SessionContext::new_with_config(cfg);
1217        ctx.register_table("test", provider).unwrap();
1218
1219        let df = ctx
1220            .sql("select col_1, col_2 from test WHERE col_1 = 'A'")
1221            .await
1222            .unwrap();
1223        let actual = df.collect().await.unwrap();
1224        let expected = vec![
1225            "+-------+-------+",
1226            "| col_1 | col_2 |",
1227            "+-------+-------+",
1228            "| A     | A2    |",
1229            "+-------+-------+",
1230        ];
1231        assert_batches_sorted_eq!(&expected, &actual);
1232    }
1233
1234    #[tokio::test]
1235    async fn delta_scan_supports_nested_missing_columns() {
1236        let column1_schema1: arrow::datatypes::Fields =
1237            vec![Field::new("col_1a", ArrowDataType::Utf8, true)].into();
1238        let schema1 = Arc::new(ArrowSchema::new(vec![Field::new(
1239            "col_1",
1240            ArrowDataType::Struct(column1_schema1.clone()),
1241            true,
1242        )]));
1243
1244        let batch1 = RecordBatch::try_new(
1245            schema1.clone(),
1246            vec![Arc::new(StructArray::new(
1247                column1_schema1,
1248                vec![Arc::new(arrow::array::StringArray::from(vec![
1249                    Some("A"),
1250                    Some("B"),
1251                ]))],
1252                None,
1253            ))],
1254        )
1255        .unwrap();
1256
1257        let column1_schema2: arrow_schema::Fields = vec![
1258            Field::new("col_1a", ArrowDataType::Utf8, true),
1259            Field::new("col_1b", ArrowDataType::Utf8, true),
1260        ]
1261        .into();
1262        let schema2 = Arc::new(ArrowSchema::new(vec![Field::new(
1263            "col_1",
1264            ArrowDataType::Struct(column1_schema2.clone()),
1265            true,
1266        )]));
1267
1268        let batch2 = RecordBatch::try_new(
1269            schema2.clone(),
1270            vec![Arc::new(StructArray::new(
1271                column1_schema2,
1272                vec![
1273                    Arc::new(arrow::array::StringArray::from(vec![
1274                        Some("E"),
1275                        Some("F"),
1276                        Some("G"),
1277                    ])),
1278                    Arc::new(arrow::array::StringArray::from(vec![
1279                        Some("E2"),
1280                        Some("F2"),
1281                        Some("G2"),
1282                    ])),
1283                ],
1284                None,
1285            ))],
1286        )
1287        .unwrap();
1288
1289        let table = DeltaTable::new_in_memory()
1290            .write(vec![batch1])
1291            .with_save_mode(crate::protocol::SaveMode::Append)
1292            .await
1293            .unwrap();
1294
1295        let table = table
1296            .write(vec![batch2])
1297            .with_schema_mode(SchemaMode::Merge)
1298            .with_save_mode(crate::protocol::SaveMode::Append)
1299            .await
1300            .unwrap();
1301
1302        let provider = table.table_provider().await.unwrap();
1303        let ctx: SessionContext = DeltaSessionContext::default().into();
1304        ctx.register_table("test", provider).unwrap();
1305
1306        let df = ctx
1307            .sql("select col_1.col_1a, col_1.col_1b from test")
1308            .await
1309            .unwrap();
1310        let actual = df.collect().await.unwrap();
1311        let expected = vec![
1312            "+--------------------+--------------------+",
1313            "| test.col_1[col_1a] | test.col_1[col_1b] |",
1314            "+--------------------+--------------------+",
1315            "| A                  |                    |",
1316            "| B                  |                    |",
1317            "| E                  | E2                 |",
1318            "| F                  | F2                 |",
1319            "| G                  | G2                 |",
1320            "+--------------------+--------------------+",
1321        ];
1322        assert_batches_sorted_eq!(&expected, &actual);
1323    }
1324
1325    #[tokio::test]
1326    async fn test_multiple_predicate_pushdown() {
1327        let schema = Arc::new(ArrowSchema::new(vec![
1328            Field::new("moDified", ArrowDataType::Utf8, true),
1329            Field::new("id", ArrowDataType::Utf8, true),
1330            Field::new("vaLue", ArrowDataType::Int32, true),
1331        ]));
1332
1333        let batch = RecordBatch::try_new(
1334            schema.clone(),
1335            vec![
1336                Arc::new(arrow::array::StringArray::from(vec![
1337                    "2021-02-01",
1338                    "2021-02-01",
1339                    "2021-02-02",
1340                    "2021-02-02",
1341                ])),
1342                Arc::new(arrow::array::StringArray::from(vec!["A", "B", "C", "D"])),
1343                Arc::new(arrow::array::Int32Array::from(vec![1, 10, 20, 100])),
1344            ],
1345        )
1346        .unwrap();
1347        // write some data
1348        let table = DeltaTable::new_in_memory()
1349            .write(vec![batch.clone()])
1350            .with_save_mode(crate::protocol::SaveMode::Append)
1351            .await
1352            .unwrap();
1353
1354        let datafusion = SessionContext::new();
1355        table
1356            .update_datafusion_session(&datafusion.state())
1357            .unwrap();
1358
1359        datafusion
1360            .register_table("snapshot", table.table_provider().await.unwrap())
1361            .unwrap();
1362
1363        let df = datafusion
1364            .sql(r#"select * from snapshot where "vaLue" > 10000 and "vaLue" < 20000"#)
1365            .await
1366            .unwrap();
1367
1368        df.collect().await.unwrap();
1369    }
1370
1371    // Run a query that filters out all files and sorts.
1372    // Verify that it returns an empty set of rows without panicking.
1373    //
1374    // Historically, we had a bug that caused us to emit a query plan with 0 partitions, which
1375    // datafusion rejected.
1376    #[tokio::test]
1377    async fn passes_sanity_checker_when_all_files_filtered() {
1378        let table = open_fs_path("../test/tests/data/delta-2.2.0-partitioned-types");
1379        let ctx = create_session().into_inner();
1380        ctx.register_table("test", table.table_provider().await.unwrap())
1381            .unwrap();
1382
1383        let df = ctx
1384            .sql("select * from test where c3 = 100 ORDER BY c1 ASC")
1385            .await
1386            .unwrap();
1387        let actual = df.collect().await.unwrap();
1388
1389        assert_eq!(actual.len(), 0);
1390    }
1391
1392    #[tokio::test]
1393    async fn test_delta_scan_uses_parquet_column_pruning() {
1394        let small: Arc<dyn Array> = Arc::new(arrow::array::StringArray::from(vec!["a"]));
1395        let large: Arc<dyn Array> = Arc::new(arrow::array::StringArray::from(vec![
1396            "b".repeat(1024).as_str(),
1397        ]));
1398        let batch = RecordBatch::try_from_iter(vec![("small", small), ("large", large)]).unwrap();
1399        let table = DeltaTable::new_in_memory()
1400            .write(vec![batch])
1401            .with_save_mode(crate::protocol::SaveMode::Append)
1402            .await
1403            .unwrap();
1404
1405        let config = DeltaScanConfigBuilder::new()
1406            .build(table.snapshot().unwrap().snapshot())
1407            .unwrap();
1408
1409        let (log_store, mut operations) = recording_log_store(table.log_store());
1410        let provider = DeltaScanNext::new(table.snapshot().unwrap().snapshot().clone(), config)
1411            .unwrap()
1412            .with_log_store(log_store.clone());
1413        let ctx = SessionContext::new();
1414        ctx.register_table("test", Arc::new(provider)).unwrap();
1415        let state = ctx.state();
1416
1417        let df = ctx.sql("select small from test").await.unwrap();
1418        let plan = df.create_physical_plan().await.unwrap();
1419
1420        let mut stream = plan.execute(0, state.task_ctx()).unwrap();
1421        let Some(Ok(batch)) = stream.next().await else {
1422            panic!()
1423        };
1424        assert!(stream.next().await.is_none());
1425        assert_eq!(1, batch.num_columns());
1426        assert_eq!(1, batch.num_rows());
1427        let small = ScalarValue::try_from_array(batch.column_by_name("small").unwrap(), 0).unwrap();
1428        assert_eq!("a", small.to_string());
1429
1430        let files = table.get_files_by_partitions(&[]).await.unwrap();
1431        assert_eq!(1, files.len());
1432        let object_store = table.object_store();
1433        let file_meta = object_store.head(&files[0]).await.unwrap();
1434        let file_reader = parquet::arrow::async_reader::ParquetObjectReader::new(
1435            object_store,
1436            file_meta.location.clone(),
1437        )
1438        .with_file_size(file_meta.size);
1439        let parquet_metadata =
1440            parquet::arrow::async_reader::ParquetRecordBatchStreamBuilder::new(file_reader)
1441                .await
1442                .unwrap()
1443                .metadata()
1444                .as_ref()
1445                .clone();
1446        let (small_start, small_len) = parquet_metadata.row_group(0).column(0).byte_range();
1447        let small_range = small_start..small_start + small_len;
1448        let (large_start, large_len) = parquet_metadata.row_group(0).column(1).byte_range();
1449        let large_range = large_start..large_start + large_len;
1450
1451        let actual = drain_recorded_ops(&mut operations).await;
1452
1453        let data_ranges = actual
1454            .iter()
1455            .flat_map(|operation| match operation {
1456                ObjectStoreOperation::GetRange(PathKind::Data, range) => vec![range.clone()],
1457                ObjectStoreOperation::GetRanges(PathKind::Data, ranges) => ranges.clone(),
1458                _ => Vec::new(),
1459            })
1460            .collect::<Vec<_>>();
1461
1462        let overlaps = |left: &Range<u64>, right: &Range<u64>| {
1463            left.start < right.end && right.start < left.end
1464        };
1465
1466        assert!(
1467            !data_ranges.is_empty(),
1468            "expected ranged parquet data reads, saw {actual:?}"
1469        );
1470        assert!(
1471            data_ranges
1472                .iter()
1473                .any(|range| overlaps(range, &small_range)),
1474            "expected selected column chunk {small_range:?} to be read, saw {actual:?}"
1475        );
1476        assert!(
1477            data_ranges
1478                .iter()
1479                .all(|range| !overlaps(range, &large_range)),
1480            "expected unselected column chunk {large_range:?} to be pruned, saw {actual:?}"
1481        );
1482        assert!(
1483            !actual.iter().any(|operation| matches!(
1484                operation,
1485                ObjectStoreOperation::Get(PathKind::Data)
1486                    | ObjectStoreOperation::GetOpts(PathKind::Data)
1487            )),
1488            "expected no full data file reads, saw {actual:?}"
1489        );
1490    }
1491
1492    #[tokio::test]
1493    async fn test_push_down_filter_panic_2602() -> DeltaResult<()> {
1494        use crate::kernel::schema::{DataType, PrimitiveType};
1495        let ctx = SessionContext::new();
1496        let table = DeltaTable::new_in_memory()
1497            .create()
1498            .with_column("id", DataType::Primitive(PrimitiveType::Long), true, None)
1499            .with_column(
1500                "name",
1501                DataType::Primitive(PrimitiveType::String),
1502                true,
1503                None,
1504            )
1505            .with_column("b", DataType::Primitive(PrimitiveType::Boolean), true, None)
1506            .with_column(
1507                "ts",
1508                DataType::Primitive(PrimitiveType::Timestamp),
1509                true,
1510                None,
1511            )
1512            .with_column("dt", DataType::Primitive(PrimitiveType::Date), true, None)
1513            .with_column(
1514                "zap",
1515                DataType::Array(Box::new(ArrayType::new(
1516                    DataType::Primitive(PrimitiveType::Boolean),
1517                    true,
1518                ))),
1519                true,
1520                None,
1521            )
1522            .await?;
1523        table.update_datafusion_session(&ctx.state()).unwrap();
1524
1525        ctx.register_table("snapshot", table.table_provider().await.unwrap())
1526            .unwrap();
1527
1528        let df = ctx
1529            .sql("select * from snapshot where id > 10000 and id < 20000")
1530            .await
1531            .unwrap();
1532
1533        let _ = df.collect().await?;
1534        Ok(())
1535    }
1536
1537    /// Tests that binary partition columns support equality pruning (= / IN predicates)
1538    /// even though they do not support range pruning (< / > predicates).
1539    ///
1540    /// Resolves issue #1214.
1541    #[tokio::test]
1542    async fn test_files_matching_predicate_binary_partition_not_pruned() {
1543        use crate::delta_datafusion::files_matching_predicate;
1544        use arrow::array::BinaryArray;
1545        use datafusion::common::ScalarValue;
1546        use datafusion::logical_expr::{col, lit};
1547
1548        // Write two files into separate binary partitions.
1549        let arrow_schema = Arc::new(ArrowSchema::new(vec![
1550            Field::new("id", ArrowDataType::Int32, true),
1551            Field::new("bin_part", ArrowDataType::Binary, true),
1552        ]));
1553
1554        let batch_a = RecordBatch::try_new(
1555            arrow_schema.clone(),
1556            vec![
1557                Arc::new(arrow::array::Int32Array::from(vec![1_i32])) as Arc<dyn Array>,
1558                Arc::new(BinaryArray::from_vec(vec![b"aaa".as_ref()])) as Arc<dyn Array>,
1559            ],
1560        )
1561        .unwrap();
1562
1563        let batch_b = RecordBatch::try_new(
1564            arrow_schema.clone(),
1565            vec![
1566                Arc::new(arrow::array::Int32Array::from(vec![2_i32])) as Arc<dyn Array>,
1567                Arc::new(BinaryArray::from_vec(vec![b"bbb".as_ref()])) as Arc<dyn Array>,
1568            ],
1569        )
1570        .unwrap();
1571
1572        // Two separate appends so each lands in its own partition file.
1573        let table = DeltaTable::new_in_memory()
1574            .create()
1575            .with_column(
1576                "id",
1577                delta_kernel::schema::DataType::Primitive(
1578                    delta_kernel::schema::PrimitiveType::Integer,
1579                ),
1580                true,
1581                None,
1582            )
1583            .with_column(
1584                "bin_part",
1585                delta_kernel::schema::DataType::Primitive(
1586                    delta_kernel::schema::PrimitiveType::Binary,
1587                ),
1588                true,
1589                None,
1590            )
1591            .with_partition_columns(["bin_part"])
1592            .await
1593            .unwrap();
1594
1595        let table = table
1596            .write(vec![batch_a])
1597            .with_save_mode(crate::protocol::SaveMode::Append)
1598            .await
1599            .unwrap();
1600
1601        let table = table
1602            .write(vec![batch_b])
1603            .with_save_mode(crate::protocol::SaveMode::Append)
1604            .await
1605            .unwrap();
1606
1607        let snapshot = table.snapshot().unwrap().snapshot().clone();
1608        let log_data = snapshot.log_data();
1609
1610        // Without a predicate all files are returned.
1611        let all_files: Vec<_> = files_matching_predicate(log_data.clone(), &[])
1612            .unwrap()
1613            .collect();
1614        assert_eq!(all_files.len(), 2, "expected 2 files before pruning");
1615
1616        // Apply a predicate that selects only the "aaa" partition.
1617        // `contained()` now handles binary exact-match pruning (issue #1214).
1618        let predicate = col("bin_part").eq(lit(ScalarValue::Binary(Some(b"aaa".to_vec()))));
1619        let kept_files: Vec<_> = files_matching_predicate(log_data.clone(), &[predicate])
1620            .unwrap()
1621            .collect();
1622
1623        assert_eq!(
1624            kept_files.len(),
1625            1,
1626            "expected exactly the 'aaa' partition file to survive equality pruning"
1627        );
1628    }
1629
1630    /// Range predicates (`<`, `>`) on binary partition columns must NOT prune any files
1631    /// because binary values have no natural ordering.  All files are conservatively kept.
1632    #[tokio::test]
1633    async fn test_files_matching_predicate_binary_partition_range_not_pruned() {
1634        use crate::delta_datafusion::files_matching_predicate;
1635        use arrow::array::BinaryArray;
1636        use datafusion::common::ScalarValue;
1637        use datafusion::logical_expr::{col, lit};
1638
1639        let arrow_schema = Arc::new(ArrowSchema::new(vec![
1640            Field::new("id", ArrowDataType::Int32, true),
1641            Field::new("bin_part", ArrowDataType::Binary, true),
1642        ]));
1643
1644        let batch_a = RecordBatch::try_new(
1645            arrow_schema.clone(),
1646            vec![
1647                Arc::new(arrow::array::Int32Array::from(vec![1_i32])) as Arc<dyn Array>,
1648                Arc::new(BinaryArray::from_vec(vec![b"aaa".as_ref()])) as Arc<dyn Array>,
1649            ],
1650        )
1651        .unwrap();
1652
1653        let batch_b = RecordBatch::try_new(
1654            arrow_schema.clone(),
1655            vec![
1656                Arc::new(arrow::array::Int32Array::from(vec![2_i32])) as Arc<dyn Array>,
1657                Arc::new(BinaryArray::from_vec(vec![b"bbb".as_ref()])) as Arc<dyn Array>,
1658            ],
1659        )
1660        .unwrap();
1661
1662        let table = DeltaTable::new_in_memory()
1663            .create()
1664            .with_column(
1665                "id",
1666                delta_kernel::schema::DataType::Primitive(
1667                    delta_kernel::schema::PrimitiveType::Integer,
1668                ),
1669                true,
1670                None,
1671            )
1672            .with_column(
1673                "bin_part",
1674                delta_kernel::schema::DataType::Primitive(
1675                    delta_kernel::schema::PrimitiveType::Binary,
1676                ),
1677                true,
1678                None,
1679            )
1680            .with_partition_columns(["bin_part"])
1681            .await
1682            .unwrap();
1683
1684        let table = table
1685            .write(vec![batch_a])
1686            .with_save_mode(crate::protocol::SaveMode::Append)
1687            .await
1688            .unwrap();
1689        let table = table
1690            .write(vec![batch_b])
1691            .with_save_mode(crate::protocol::SaveMode::Append)
1692            .await
1693            .unwrap();
1694
1695        let snapshot = table.snapshot().unwrap().snapshot().clone();
1696        let log_data = snapshot.log_data();
1697
1698        // A `>` predicate on a binary column must keep all files — there are no
1699        // min/max stats for binary, so datafusion cannot eliminate any file.
1700        let gt_pred = col("bin_part").gt(lit(ScalarValue::Binary(Some(b"aaa".to_vec()))));
1701        let kept = files_matching_predicate(log_data.clone(), &[gt_pred])
1702            .unwrap()
1703            .count();
1704        assert_eq!(
1705            kept, 2,
1706            "range predicate '>' on binary partition must not prune any files"
1707        );
1708
1709        // Likewise for `<`.
1710        let lt_pred = col("bin_part").lt(lit(ScalarValue::Binary(Some(b"bbb".to_vec()))));
1711        let kept = files_matching_predicate(log_data.clone(), &[lt_pred])
1712            .unwrap()
1713            .count();
1714        assert_eq!(
1715            kept, 2,
1716            "range predicate '<' on binary partition must not prune any files"
1717        );
1718    }
1719}