1use std::fmt::Debug;
27use std::sync::Arc;
28
29use arrow::array::types::UInt16Type;
30use arrow::array::{Array, DictionaryArray, RecordBatch, StringArray, TypedDictionaryArray};
31use arrow_cast::{CastOptions, cast_with_options};
32use arrow_schema::{
33 DataType as ArrowDataType, Schema as ArrowSchema, SchemaRef, SchemaRef as ArrowSchemaRef,
34 TimeUnit,
35};
36use datafusion::catalog::{Session, TableProviderFactory};
37use datafusion::common::scalar::ScalarValue;
38use datafusion::common::{
39 Column, DFSchema, DataFusionError, Result as DataFusionResult, TableReference, ToDFSchema,
40};
41use datafusion::datasource::TableProvider;
42use datafusion::datasource::physical_plan::wrap_partition_type_in_dict;
43use datafusion::execution::TaskContext;
44use datafusion::execution::context::SessionContext;
45use datafusion::logical_expr::logical_plan::CreateExternalTable;
46use datafusion::logical_expr::utils::conjunction;
47use datafusion::logical_expr::{Expr, Extension, LogicalPlan};
48use datafusion::physical_optimizer::pruning::PruningPredicateBuilder;
49use datafusion::physical_plan::ExecutionPlan;
50use datafusion_proto::logical_plan::LogicalExtensionCodec;
51use datafusion_proto::physical_plan::{PhysicalExtensionCodec, PhysicalProtoConverterExtension};
52use delta_kernel::engine::arrow_conversion::TryIntoArrow as _;
53use either::Either;
54
55use crate::delta_datafusion::expr::parse_predicate_expression;
56use crate::delta_datafusion::table_provider::{DeltaScan, DeltaScanWire};
57use crate::ensure_table_uri;
58use crate::errors::{DeltaResult, DeltaTableError};
59use crate::kernel::{Add, EagerSnapshot, LogDataHandler, Snapshot};
60use crate::{open_table, open_table_with_storage_options};
61
62pub(crate) use self::session::DeltaSessionExt;
63pub use self::session::{
64 DeltaParserOptions, DeltaRuntimeEnvBuilder, DeltaSessionConfig, DeltaSessionContext,
65 create_session, create_session_state_with_spill_config,
66};
67pub use self::table_provider::next::{
68 DeletionVectorSelection, DeltaScan as DeltaScanNext, FileSelection, MissingSelectedFilePolicy,
69};
70pub(crate) use self::utils::*;
71pub use cdf::scan::DeltaCdfTableProvider;
72pub(crate) use column_mapping::ColumnMappingState;
73pub(crate) use data_validation::{DataValidationExec, constraints_to_exprs, validation_predicates};
74pub(crate) use find_files::*;
75pub(crate) use table_provider::next::RuntimeFileFilter;
76pub(crate) use table_provider::next::normalize_path_as_file_id;
77pub use table_provider::{
78 DeltaScanConfig, DeltaScanConfigBuilder, TableProviderBuilder, next::DeltaScanExec,
79};
80pub(crate) use table_provider::{
81 next::FILE_ID_COLUMN_DEFAULT, resolve_file_column_name, update_datafusion_session,
82};
83
84pub(crate) const PATH_COLUMN: &str = "__delta_rs_path";
85
86#[doc(hidden)]
87pub mod bench_support;
88pub mod cdf;
89mod column_mapping;
90mod data_validation;
91pub mod engine;
93pub mod expr;
94mod file_id;
95mod find_files;
96pub mod logical;
97pub mod physical;
98pub mod planner;
99mod session;
100pub use session::SessionFallbackPolicy;
101pub(crate) use session::{SessionResolveContext, resolve_session_state};
102mod table_provider;
103pub(crate) mod utils;
104
105impl From<DeltaTableError> for DataFusionError {
106 fn from(err: DeltaTableError) -> Self {
107 match err {
108 DeltaTableError::Arrow { source } => DataFusionError::from(source),
109 DeltaTableError::Io { source } => DataFusionError::IoError(source),
110 DeltaTableError::ObjectStore { source } => DataFusionError::from(source),
111 DeltaTableError::Parquet { source } => DataFusionError::from(source),
112 _ => DataFusionError::External(Box::new(err)),
113 }
114 }
115}
116
117impl From<DataFusionError> for DeltaTableError {
118 fn from(err: DataFusionError) -> Self {
119 match err {
120 DataFusionError::ArrowError(source, _) => DeltaTableError::from(*source),
121 DataFusionError::IoError(source) => DeltaTableError::Io { source },
122 DataFusionError::ObjectStore(source) => DeltaTableError::from(*source),
123 DataFusionError::ParquetError(source) => DeltaTableError::from(*source),
124 _ => DeltaTableError::Generic(err.to_string()),
125 }
126 }
127}
128
129pub trait DataFusionMixins {
131 fn read_schema(&self) -> ArrowSchemaRef;
133
134 fn input_schema(&self) -> ArrowSchemaRef;
136
137 fn parse_predicate_expression(
139 &self,
140 expr: impl AsRef<str>,
141 session: &dyn Session,
142 ) -> DeltaResult<Expr>;
143}
144
145impl DataFusionMixins for Snapshot {
146 fn read_schema(&self) -> ArrowSchemaRef {
147 _arrow_schema(
148 self.arrow_schema(),
149 self.metadata().partition_columns(),
150 true,
151 )
152 }
153
154 fn input_schema(&self) -> ArrowSchemaRef {
155 _arrow_schema(
156 self.arrow_schema(),
157 self.metadata().partition_columns(),
158 false,
159 )
160 }
161
162 fn parse_predicate_expression(
163 &self,
164 expr: impl AsRef<str>,
165 session: &dyn Session,
166 ) -> DeltaResult<Expr> {
167 let schema = DFSchema::try_from(self.read_schema().as_ref().to_owned())?;
168 parse_predicate_expression(&schema, expr, session)
169 }
170}
171
172impl DataFusionMixins for LogDataHandler<'_> {
173 fn read_schema(&self) -> ArrowSchemaRef {
174 _arrow_schema(
175 Arc::new(
176 self.table_configuration()
177 .logical_schema()
178 .as_ref()
179 .try_into_arrow()
180 .unwrap(),
181 ),
182 self.table_configuration().metadata().partition_columns(),
183 true,
184 )
185 }
186
187 fn input_schema(&self) -> ArrowSchemaRef {
188 _arrow_schema(
189 Arc::new(
190 self.table_configuration()
191 .logical_schema()
192 .as_ref()
193 .try_into_arrow()
194 .unwrap(),
195 ),
196 self.table_configuration().metadata().partition_columns(),
197 false,
198 )
199 }
200
201 fn parse_predicate_expression(
202 &self,
203 expr: impl AsRef<str>,
204 session: &dyn Session,
205 ) -> DeltaResult<Expr> {
206 let schema = DFSchema::try_from(self.read_schema().as_ref().to_owned())?;
207 parse_predicate_expression(&schema, expr, session)
208 }
209}
210
211impl DataFusionMixins for EagerSnapshot {
212 fn read_schema(&self) -> ArrowSchemaRef {
213 self.snapshot().read_schema()
214 }
215
216 fn input_schema(&self) -> ArrowSchemaRef {
217 self.snapshot().input_schema()
218 }
219
220 fn parse_predicate_expression(
221 &self,
222 expr: impl AsRef<str>,
223 session: &dyn Session,
224 ) -> DeltaResult<Expr> {
225 self.snapshot().parse_predicate_expression(expr, session)
226 }
227}
228
229fn _arrow_schema(
230 schema: SchemaRef,
231 partition_columns: &[String],
232 wrap_partitions: bool,
233) -> ArrowSchemaRef {
234 let fields = schema
235 .fields()
236 .into_iter()
237 .filter(|f| !partition_columns.contains(&f.name().to_string()))
238 .cloned()
239 .chain(
240 partition_columns.iter().map(|partition_col| {
243 let field = schema.field_with_name(partition_col).unwrap();
244 let corrected = if wrap_partitions {
245 match field.data_type() {
246 ArrowDataType::Utf8
249 | ArrowDataType::LargeUtf8
250 | ArrowDataType::Binary
251 | ArrowDataType::LargeBinary => {
252 wrap_partition_type_in_dict(field.data_type().clone())
253 }
254 _ => field.data_type().clone(),
255 }
256 } else {
257 field.data_type().clone()
258 };
259 Arc::new(field.clone().with_data_type(corrected))
260 }),
261 )
262 .collect::<Vec<_>>();
263 Arc::new(ArrowSchema::new(fields))
264}
265
266pub(crate) fn files_matching_predicate<'a>(
267 log_data: LogDataHandler<'a>,
268 filters: &[Expr],
269) -> DeltaResult<impl Iterator<Item = Add> + 'a> {
270 if let Some(Some(predicate)) =
271 (!filters.is_empty()).then_some(conjunction(filters.iter().cloned()))
272 {
273 let session: SessionContext = create_session().into();
274 let schema = log_data.read_schema();
275 let df_schema = Arc::new(schema.clone().to_dfschema()?);
276 let resolved = Expression::from(predicate).resolve(&session.state(), df_schema.clone())?;
277 let expr = session.create_physical_expr(resolved, &df_schema)?;
278 let pruning_predicate = PruningPredicateBuilder::new()
279 .with_file_schema(schema)
280 .try_build(expr)?;
281 let mask = pruning_predicate.prune(&log_data)?;
282
283 Ok(Either::Left(log_data.into_iter().zip(mask).filter_map(
284 |(file, keep_file)| {
285 if keep_file { Some(file.to_add()) } else { None }
286 },
287 )))
288 } else {
289 Ok(Either::Right(
290 log_data.into_iter().map(|file| file.to_add()),
291 ))
292 }
293}
294
295pub(crate) fn get_path_column<'a>(
296 batch: &'a RecordBatch,
297 path_column: &str,
298) -> DeltaResult<TypedDictionaryArray<'a, UInt16Type, StringArray>> {
299 let err = || DeltaTableError::Generic("Unable to obtain Delta-rs path column".to_string());
300 batch
301 .column_by_name(path_column)
302 .ok_or_else(err)?
303 .as_any()
304 .downcast_ref::<DictionaryArray<UInt16Type>>()
305 .ok_or_else(err)?
306 .downcast_dict::<StringArray>()
307 .ok_or_else(err)
308}
309
310pub(crate) fn get_null_of_arrow_type(t: &ArrowDataType) -> DeltaResult<ScalarValue> {
311 match t {
312 ArrowDataType::Null => Ok(ScalarValue::Null),
313 ArrowDataType::Boolean => Ok(ScalarValue::Boolean(None)),
314 ArrowDataType::Int8 => Ok(ScalarValue::Int8(None)),
315 ArrowDataType::Int16 => Ok(ScalarValue::Int16(None)),
316 ArrowDataType::Int32 => Ok(ScalarValue::Int32(None)),
317 ArrowDataType::Int64 => Ok(ScalarValue::Int64(None)),
318 ArrowDataType::UInt8 => Ok(ScalarValue::UInt8(None)),
319 ArrowDataType::UInt16 => Ok(ScalarValue::UInt16(None)),
320 ArrowDataType::UInt32 => Ok(ScalarValue::UInt32(None)),
321 ArrowDataType::UInt64 => Ok(ScalarValue::UInt64(None)),
322 ArrowDataType::Float32 => Ok(ScalarValue::Float32(None)),
323 ArrowDataType::Float64 => Ok(ScalarValue::Float64(None)),
324 ArrowDataType::Date32 => Ok(ScalarValue::Date32(None)),
325 ArrowDataType::Date64 => Ok(ScalarValue::Date64(None)),
326 ArrowDataType::Binary => Ok(ScalarValue::Binary(None)),
327 ArrowDataType::FixedSizeBinary(size) => {
328 Ok(ScalarValue::FixedSizeBinary(size.to_owned(), None))
329 }
330 ArrowDataType::LargeBinary => Ok(ScalarValue::LargeBinary(None)),
331 ArrowDataType::Utf8 => Ok(ScalarValue::Utf8(None)),
332 ArrowDataType::LargeUtf8 => Ok(ScalarValue::LargeUtf8(None)),
333 ArrowDataType::Decimal128(precision, scale) => Ok(ScalarValue::Decimal128(
334 None,
335 precision.to_owned(),
336 scale.to_owned(),
337 )),
338 ArrowDataType::Timestamp(unit, tz) => {
339 let tz = tz.to_owned();
340 Ok(match unit {
341 TimeUnit::Second => ScalarValue::TimestampSecond(None, tz),
342 TimeUnit::Millisecond => ScalarValue::TimestampMillisecond(None, tz),
343 TimeUnit::Microsecond => ScalarValue::TimestampMicrosecond(None, tz),
344 TimeUnit::Nanosecond => ScalarValue::TimestampNanosecond(None, tz),
345 })
346 }
347 ArrowDataType::Dictionary(k, v) => Ok(ScalarValue::Dictionary(
348 k.clone(),
349 Box::new(get_null_of_arrow_type(v)?),
350 )),
351 ArrowDataType::Float16
353 | ArrowDataType::Decimal32(_, _)
354 | ArrowDataType::Decimal64(_, _)
355 | ArrowDataType::Decimal256(_, _)
356 | ArrowDataType::Union(_, _)
357 | ArrowDataType::LargeList(_)
358 | ArrowDataType::Struct(_)
359 | ArrowDataType::List(_)
360 | ArrowDataType::FixedSizeList(_, _)
361 | ArrowDataType::Time32(_)
362 | ArrowDataType::Time64(_)
363 | ArrowDataType::Duration(_)
364 | ArrowDataType::Interval(_)
365 | ArrowDataType::RunEndEncoded(_, _)
366 | ArrowDataType::BinaryView
367 | ArrowDataType::Utf8View
368 | ArrowDataType::LargeListView(_)
369 | ArrowDataType::ListView(_)
370 | ArrowDataType::Map(_, _) => Err(DeltaTableError::Generic(format!(
371 "Unsupported data type for Delta Lake {t}"
372 ))),
373 }
374}
375
376fn parse_date(
377 stat_val: &serde_json::Value,
378 field_dt: &ArrowDataType,
379) -> DataFusionResult<ScalarValue> {
380 let string = match stat_val {
381 serde_json::Value::String(s) => s.to_owned(),
382 _ => stat_val.to_string(),
383 };
384
385 let time_micro = ScalarValue::try_from_string(string, &ArrowDataType::Date32)?;
386 let cast_arr = cast_with_options(
387 &time_micro.to_array()?,
388 field_dt,
389 &CastOptions {
390 safe: false,
391 ..Default::default()
392 },
393 )?;
394 ScalarValue::try_from_array(&cast_arr, 0)
395}
396
397fn parse_timestamp(
398 stat_val: &serde_json::Value,
399 field_dt: &ArrowDataType,
400) -> DataFusionResult<ScalarValue> {
401 let string = match stat_val {
402 serde_json::Value::String(s) => s.to_owned(),
403 _ => stat_val.to_string(),
404 };
405
406 let time_micro = ScalarValue::try_from_string(
407 string,
408 &ArrowDataType::Timestamp(TimeUnit::Microsecond, None),
409 )?;
410 let cast_arr = cast_with_options(
411 &time_micro.to_array()?,
412 field_dt,
413 &CastOptions {
414 safe: false,
415 ..Default::default()
416 },
417 )?;
418 ScalarValue::try_from_array(&cast_arr, 0)
419}
420
421pub(crate) fn to_correct_scalar_value(
422 stat_val: &serde_json::Value,
423 field_dt: &ArrowDataType,
424) -> DataFusionResult<Option<ScalarValue>> {
425 match stat_val {
426 serde_json::Value::Array(_) => Ok(None),
427 serde_json::Value::Object(_) => Ok(None),
428 serde_json::Value::Null => Ok(Some(get_null_of_arrow_type(field_dt)?)),
429 serde_json::Value::String(string_val) => match field_dt {
430 ArrowDataType::Timestamp(_, _) => Ok(Some(parse_timestamp(stat_val, field_dt)?)),
431 ArrowDataType::Date32 => Ok(Some(parse_date(stat_val, field_dt)?)),
432 _ => Ok(Some(ScalarValue::try_from_string(
433 string_val.to_owned(),
434 field_dt,
435 )?)),
436 },
437 other => match field_dt {
438 ArrowDataType::Timestamp(_, _) => Ok(Some(parse_timestamp(stat_val, field_dt)?)),
439 ArrowDataType::Date32 => Ok(Some(parse_date(stat_val, field_dt)?)),
440 _ => Ok(Some(ScalarValue::try_from_string(
441 other.to_string(),
442 field_dt,
443 )?)),
444 },
445 }
446}
447
448#[deprecated(
451 note = "DeltaPhysicalCodec only supports the retired physical DeltaScan wrapper. \
452 Use DeltaLogicalCodec for table-provider serialization until a DeltaScanExec \
453 physical codec is available."
454)]
455#[derive(Debug)]
456pub struct DeltaPhysicalCodec {}
457
458#[allow(deprecated)]
459impl PhysicalExtensionCodec for DeltaPhysicalCodec {
460 fn try_decode(
461 &self,
462 buf: &[u8],
463 inputs: &[Arc<dyn ExecutionPlan>],
464 _registry: &TaskContext,
465 _converter: &dyn PhysicalProtoConverterExtension,
466 ) -> Result<Arc<dyn ExecutionPlan>, DataFusionError> {
467 let wire: DeltaScanWire = serde_json::from_reader(buf)
468 .map_err(|_| DataFusionError::Internal("Unable to decode DeltaScan".to_string()))?;
469 let delta_scan = wire.into_delta_scan((*inputs)[0].clone());
470 Ok(Arc::new(delta_scan))
471 }
472
473 fn try_encode(
474 &self,
475 node: Arc<dyn ExecutionPlan>,
476 buf: &mut Vec<u8>,
477 _converter: &dyn PhysicalProtoConverterExtension,
478 ) -> Result<(), DataFusionError> {
479 let delta_scan = node
480 .downcast_ref::<DeltaScan>()
481 .ok_or_else(|| DataFusionError::Internal("Not a legacy delta scan!".to_string()))?;
482
483 let wire = DeltaScanWire::from(delta_scan);
484 serde_json::to_writer(buf, &wire).map_err(|_| {
485 DataFusionError::Internal("Unable to encode legacy delta scan!".to_string())
486 })?;
487 Ok(())
488 }
489}
490
491#[derive(Debug)]
493pub struct DeltaLogicalCodec {}
494
495impl LogicalExtensionCodec for DeltaLogicalCodec {
496 fn try_decode(
497 &self,
498 _buf: &[u8],
499 _inputs: &[LogicalPlan],
500 _ctx: &TaskContext,
501 ) -> Result<Extension, DataFusionError> {
502 todo!("DeltaLogicalCodec")
503 }
504
505 fn try_encode(&self, _node: &Extension, _buf: &mut Vec<u8>) -> Result<(), DataFusionError> {
506 todo!("DeltaLogicalCodec")
507 }
508
509 fn try_decode_table_provider(
510 &self,
511 buf: &[u8],
512 _table_ref: &TableReference,
513 _schema: SchemaRef,
514 _ctx: &TaskContext,
515 ) -> Result<Arc<dyn TableProvider>, DataFusionError> {
516 let provider: DeltaScanNext = serde_json::from_slice(buf)
517 .map_err(|_| DataFusionError::Internal("Error encoding delta table".to_string()))?;
518 Ok(Arc::new(provider))
519 }
520
521 fn try_encode_table_provider(
522 &self,
523 _table_ref: &TableReference,
524 node: Arc<dyn TableProvider>,
525 buf: &mut Vec<u8>,
526 ) -> Result<(), DataFusionError> {
527 let scan = node.downcast_ref::<DeltaScanNext>().ok_or_else(|| {
528 DataFusionError::Internal("Can't encode non-delta tables".to_string())
529 })?;
530 serde_json::to_writer(buf, scan)
531 .map_err(|_| DataFusionError::Internal("Error encoding delta table".to_string()))
532 }
533}
534
535#[derive(Debug)]
537pub struct DeltaTableFactory {}
538
539#[async_trait::async_trait]
540impl TableProviderFactory for DeltaTableFactory {
541 async fn create(
542 &self,
543 ctx: &dyn Session,
544 cmd: &CreateExternalTable,
545 ) -> datafusion::error::Result<Arc<dyn TableProvider>> {
546 let [location] = cmd.locations.as_slice() else {
547 return Err(DataFusionError::Plan(format!(
548 "Delta tables require one location, got {}",
549 cmd.locations.len()
550 )));
551 };
552 let table = if cmd.options.is_empty() {
553 let table_url = ensure_table_uri(location)?;
554 open_table(table_url).await?
555 } else {
556 let table_url = ensure_table_uri(location)?;
557 open_table_with_storage_options(table_url, cmd.to_owned().options).await?
558 };
559 let table_uri = table.log_store().root_url().clone();
560 let (session_state, _) = resolve_session_state(
561 Some(ctx),
562 SessionFallbackPolicy::DeriveFromTrait,
563 || create_session().state(),
564 SessionResolveContext {
565 operation: "DeltaTableFactory::create",
566 table_uri: Some(&table_uri),
567 cdc: false,
568 },
569 )?;
570
571 Ok(table
572 .table_provider()
573 .with_session(Arc::new(session_state))
574 .await?)
575 }
576}
577
578pub struct DeltaColumn {
580 inner: Column,
581}
582
583impl From<&str> for DeltaColumn {
584 fn from(c: &str) -> Self {
585 DeltaColumn {
586 inner: Column::from_qualified_name_ignore_case(c),
587 }
588 }
589}
590
591impl From<&String> for DeltaColumn {
593 fn from(c: &String) -> Self {
594 DeltaColumn {
595 inner: Column::from_qualified_name_ignore_case(c),
596 }
597 }
598}
599
600impl From<String> for DeltaColumn {
602 fn from(c: String) -> Self {
603 DeltaColumn {
604 inner: Column::from_qualified_name_ignore_case(c),
605 }
606 }
607}
608
609impl From<DeltaColumn> for Column {
610 fn from(value: DeltaColumn) -> Self {
611 value.inner
612 }
613}
614
615impl From<Column> for DeltaColumn {
617 fn from(c: Column) -> Self {
618 DeltaColumn { inner: c }
619 }
620}
621
622#[cfg(test)]
623mod tests {
624 use crate::DeltaTable;
625 use crate::operations::write::SchemaMode;
626 use crate::test_utils::{
627 make_test_add,
628 object_store::{
629 RecordedObjectStoreOperation as ObjectStoreOperation, RecordedPathKind as PathKind,
630 drain_recorded_object_store_operations as drain_recorded_ops, recording_log_store,
631 },
632 open_fs_path,
633 };
634 use crate::writer::test_utils::get_delta_schema;
635 use arrow::array::StructArray;
636 use arrow::datatypes::{Field, Schema};
637 use datafusion::assert_batches_sorted_eq;
638 use datafusion::physical_plan::empty::EmptyExec;
639 use datafusion::prelude::SessionConfig;
640 use datafusion_proto::physical_plan::AsExecutionPlan;
641 use datafusion_proto::protobuf;
642 use delta_kernel::schema::ArrayType;
643 use futures::{StreamExt, TryStreamExt};
644 use object_store::ObjectStoreExt as _;
645 use serde_json::json;
646 use std::ops::Range;
647 use url::Url;
648
649 use super::*;
650 use crate::delta_datafusion::table_provider::next::{FileSelection, MissingSelectedFilePolicy};
651
652 #[tokio::test]
653 async fn delta_table_factory_rejects_location_counts_other_than_one() {
654 let ctx = SessionContext::new();
655 let factory = DeltaTableFactory {};
656
657 for locations in [vec![], vec!["one".to_string(), "two".to_string()]] {
658 let command = CreateExternalTable::builder(
659 TableReference::bare("invalid_locations"),
660 "unused",
661 "DELTATABLE",
662 Arc::new(DFSchema::empty()),
663 )
664 .with_locations(locations.clone())
665 .build();
666
667 let error = factory
668 .create(&ctx.state(), &command)
669 .await
670 .expect_err("DeltaTableFactory accepted an invalid location count");
671 assert!(matches!(error, DataFusionError::Plan(_)));
672 assert!(
673 error.to_string().contains("require one location"),
674 "unexpected error for {} locations: {error}",
675 locations.len()
676 );
677 }
678 }
679
680 #[test]
683 fn test_to_correct_scalar_value() {
684 let reference_pairs = &[
685 (
686 json!("2015"),
687 ArrowDataType::Int16,
688 ScalarValue::Int16(Some(2015)),
689 ),
690 (
691 json!("2015"),
692 ArrowDataType::Int32,
693 ScalarValue::Int32(Some(2015)),
694 ),
695 (
696 json!("2015"),
697 ArrowDataType::Int64,
698 ScalarValue::Int64(Some(2015)),
699 ),
700 (
701 json!("2015"),
702 ArrowDataType::Float32,
703 ScalarValue::Float32(Some(2015_f32)),
704 ),
705 (
706 json!("2015"),
707 ArrowDataType::Float64,
708 ScalarValue::Float64(Some(2015_f64)),
709 ),
710 (
711 json!(2015),
712 ArrowDataType::Float64,
713 ScalarValue::Float64(Some(2015_f64)),
714 ),
715 (
716 json!("2015-01-01"),
717 ArrowDataType::Date32,
718 ScalarValue::Date32(Some(16436)),
719 ),
720 (
742 json!(true),
743 ArrowDataType::Boolean,
744 ScalarValue::Boolean(Some(true)),
745 ),
746 ];
747
748 for (raw, data_type, ref_scalar) in reference_pairs {
749 let scalar = to_correct_scalar_value(raw, data_type).unwrap().unwrap();
750 assert_eq!(*ref_scalar, scalar)
751 }
752 }
753
754 #[allow(dead_code)]
759 fn _removed_roundtrip_test_delta_exec_plan() {
760 }
763
764 #[tokio::test]
765 async fn roundtrip_test_delta_logical_codec_preserves_file_selection() {
766 let log_store = crate::test_utils::TestTables::Simple
767 .table_builder()
768 .unwrap()
769 .build_storage()
770 .unwrap();
771 let snapshot = Arc::new(
772 crate::kernel::Snapshot::try_new(&log_store, None)
773 .await
774 .unwrap(),
775 );
776 let table_root = snapshot.inner.table_root().clone();
777 let selected_file_ids: Vec<String> = snapshot
778 .file_views(log_store.as_ref(), None)
779 .take(1)
780 .map_ok(|view| table_root.join(view.path_raw()).unwrap().to_string())
781 .try_collect()
782 .await
783 .unwrap();
784 assert_eq!(selected_file_ids.len(), 1);
785
786 let provider = DeltaScanNext::builder()
787 .with_snapshot(snapshot)
788 .build()
789 .await
790 .unwrap()
791 .with_file_selection(
792 FileSelection::from_file_paths(selected_file_ids.clone())
793 .with_missing_file_policy(MissingSelectedFilePolicy::Ignore),
794 );
795
796 let codec = DeltaLogicalCodec {};
797 let table_ref = TableReference::bare("delta_table");
798 let mut encoded = Vec::new();
799 codec
800 .try_encode_table_provider(&table_ref, Arc::new(provider), &mut encoded)
801 .unwrap();
802
803 let ctx = SessionContext::new();
804 let decoded = codec
805 .try_decode_table_provider(
806 &encoded,
807 &table_ref,
808 Arc::new(ArrowSchema::empty()),
809 &ctx.task_ctx(),
810 )
811 .unwrap();
812 let decoded_provider = decoded.downcast_ref::<DeltaScanNext>().unwrap();
813
814 let serialized = serde_json::to_value(decoded_provider).unwrap();
815 let decoded_file_ids = serialized
816 .get("file_selection")
817 .and_then(|value| value.get("paths"))
818 .and_then(|value| value.as_array())
819 .unwrap()
820 .iter()
821 .map(|value| value.as_str().unwrap().to_string())
822 .collect::<Vec<_>>();
823 let decoded_policy = serialized
824 .get("file_selection")
825 .and_then(|value| value.get("missing_file_policy"))
826 .and_then(|value| value.as_str())
827 .unwrap();
828
829 assert_eq!(decoded_file_ids, selected_file_ids);
830 assert_eq!(decoded_policy, "Ignore");
831 }
832
833 #[tokio::test]
834 async fn roundtrip_test_delta_logical_codec_serializes_file_selection_adds_as_identity_only() {
835 let log_store = crate::test_utils::TestTables::Simple
836 .table_builder()
837 .unwrap()
838 .build_storage()
839 .unwrap();
840 let snapshot = Arc::new(
841 crate::kernel::Snapshot::try_new(&log_store, None)
842 .await
843 .unwrap(),
844 );
845 let mut add = make_test_add("part=a/part-000.parquet", &[("part", "a")], 123);
846 add.stats = Some(r#"{"numRecords":999}"#.to_string());
847 add.tags = Some(std::collections::HashMap::from([(
848 "secret-tag".to_string(),
849 Some("secret-value".to_string()),
850 )]));
851 add.size = 999;
852
853 let provider = DeltaScanNext::builder()
854 .with_snapshot(snapshot)
855 .build()
856 .await
857 .unwrap()
858 .with_file_selection(FileSelection::from_adds([add]));
859
860 let codec = DeltaLogicalCodec {};
861 let table_ref = TableReference::bare("delta_table");
862 let mut encoded = Vec::new();
863 codec
864 .try_encode_table_provider(&table_ref, Arc::new(provider), &mut encoded)
865 .unwrap();
866
867 let ctx = SessionContext::new();
868 let decoded = codec
869 .try_decode_table_provider(
870 &encoded,
871 &table_ref,
872 Arc::new(ArrowSchema::empty()),
873 &ctx.task_ctx(),
874 )
875 .unwrap();
876 let decoded_provider = decoded.downcast_ref::<DeltaScanNext>().unwrap();
877 let serialized = serde_json::to_value(decoded_provider).unwrap();
878 let file_selection = serialized.get("file_selection").unwrap();
879 let file_selection_json = serde_json::to_string(file_selection).unwrap();
880
881 assert!(file_selection_json.contains("part=a/part-000.parquet"));
882 for forbidden in [
883 "partitionValues",
884 "numRecords",
885 "secret-tag",
886 "secret-value",
887 "modificationTime",
888 "\"size\"",
889 ] {
890 assert!(
891 !file_selection_json.contains(forbidden),
892 "FileSelection::from_adds serialized Add metadata field {forbidden}: {file_selection_json}"
893 );
894 }
895 }
896
897 #[tokio::test]
898 async fn roundtrip_test_delta_logical_codec_strips_file_selection_url_credentials() {
899 let log_store = crate::test_utils::TestTables::Simple
900 .table_builder()
901 .unwrap()
902 .build_storage()
903 .unwrap();
904 let snapshot = Arc::new(
905 crate::kernel::Snapshot::try_new(&log_store, None)
906 .await
907 .unwrap(),
908 );
909 let provider = DeltaScanNext::builder()
910 .with_snapshot(snapshot)
911 .build()
912 .await
913 .unwrap()
914 .with_file_selection(FileSelection::from_file_paths([
915 "https://urluser:urlpassword@example.com/table/part.parquet?token=urltoken#frag",
916 ]));
917
918 let codec = DeltaLogicalCodec {};
919 let table_ref = TableReference::bare("delta_table");
920 let mut encoded = Vec::new();
921 codec
922 .try_encode_table_provider(&table_ref, Arc::new(provider), &mut encoded)
923 .unwrap();
924
925 let ctx = SessionContext::new();
926 let decoded = codec
927 .try_decode_table_provider(
928 &encoded,
929 &table_ref,
930 Arc::new(ArrowSchema::empty()),
931 &ctx.task_ctx(),
932 )
933 .unwrap();
934 let decoded_provider = decoded.downcast_ref::<DeltaScanNext>().unwrap();
935 let serialized = serde_json::to_value(decoded_provider).unwrap();
936 let file_selection = serialized.get("file_selection").unwrap();
937 let file_selection_json = serde_json::to_string(file_selection).unwrap();
938
939 assert!(file_selection_json.contains("https://example.com/table/part.parquet"));
940 for forbidden in ["urluser", "urlpassword", "urltoken", "frag"] {
941 assert!(
942 !file_selection_json.contains(forbidden),
943 "encoded file selection contains {forbidden}: {file_selection_json}"
944 );
945 }
946 }
947
948 #[tokio::test]
949 async fn delta_table_provider_with_config() {
950 let table = open_fs_path("../test/tests/data/delta-2.2.0-partitioned-types");
951 let provider = table
952 .table_provider()
953 .with_file_column("file_source")
954 .await
955 .unwrap();
956 let ctx = SessionContext::new();
957 ctx.register_table("test", provider).unwrap();
958
959 let df = ctx
960 .sql("select c3, c1, c2, right(file_source, 77) as file_source from test")
961 .await
962 .unwrap();
963 let actual = df.collect().await.unwrap();
964 let expected = vec![
965 "+----+----+----+-------------------------------------------------------------------------------+",
966 "| c3 | c1 | c2 | file_source |",
967 "+----+----+----+-------------------------------------------------------------------------------+",
968 "| 4 | 6 | a | c1=6/c2=a/part-00011-10619b10-b691-4fd0-acc4-2a9608499d7c.c000.snappy.parquet |",
969 "| 5 | 4 | c | c1=4/c2=c/part-00003-f525f459-34f9-46f5-82d6-d42121d883fd.c000.snappy.parquet |",
970 "| 6 | 5 | b | c1=5/c2=b/part-00007-4e73fa3b-2c88-424a-8051-f8b54328ffdb.c000.snappy.parquet |",
971 "+----+----+----+-------------------------------------------------------------------------------+",
972 ];
973 assert_batches_sorted_eq!(&expected, &actual);
974 }
975
976 #[tokio::test]
977 async fn delta_scan_mixed_partition_order() {
978 let schema = Arc::new(ArrowSchema::new(vec![
981 Field::new("modified", ArrowDataType::Utf8, true),
982 Field::new("id", ArrowDataType::Utf8, true),
983 Field::new("value", ArrowDataType::Int32, true),
984 ]));
985
986 let table = DeltaTable::new_in_memory()
987 .create()
988 .with_columns(get_delta_schema().fields().cloned())
989 .with_partition_columns(["modified", "id"])
990 .await
991 .unwrap();
992 assert_eq!(table.version(), Some(0));
993
994 let batch = RecordBatch::try_new(
995 schema.clone(),
996 vec![
997 Arc::new(arrow::array::StringArray::from(vec![
998 "2021-02-01",
999 "2021-02-01",
1000 "2021-02-02",
1001 "2021-02-02",
1002 ])),
1003 Arc::new(arrow::array::StringArray::from(vec!["A", "B", "C", "D"])),
1004 Arc::new(arrow::array::Int32Array::from(vec![1, 10, 20, 100])),
1005 ],
1006 )
1007 .unwrap();
1008 let table = table
1010 .write(vec![batch.clone()])
1011 .with_save_mode(crate::protocol::SaveMode::Append)
1012 .await
1013 .unwrap();
1014
1015 let provider = table.table_provider().await.unwrap();
1016 let logical_schema = provider.schema();
1017 let ctx = SessionContext::new();
1018 ctx.runtime_env().register_object_store(
1019 table.log_store().root_url(),
1020 table.log_store().object_store(None),
1021 );
1022 ctx.register_table("test", provider).unwrap();
1023
1024 let expected_logical_order = vec!["id", "value", "modified"];
1025 let actual_order: Vec<String> = logical_schema
1026 .fields()
1027 .iter()
1028 .map(|f| f.name().to_owned())
1029 .collect();
1030
1031 let df = ctx.sql("select * from test").await.unwrap();
1032 let actual = df.collect().await.unwrap();
1033 let expected = vec![
1034 "+----+-------+------------+",
1035 "| id | value | modified |",
1036 "+----+-------+------------+",
1037 "| A | 1 | 2021-02-01 |",
1038 "| B | 10 | 2021-02-01 |",
1039 "| C | 20 | 2021-02-02 |",
1040 "| D | 100 | 2021-02-02 |",
1041 "+----+-------+------------+",
1042 ];
1043 assert_batches_sorted_eq!(&expected, &actual);
1044 assert_eq!(expected_logical_order, actual_order);
1045 }
1046
1047 #[tokio::test]
1048 async fn delta_scan_case_sensitive() {
1049 let schema = Arc::new(ArrowSchema::new(vec![
1050 Field::new("moDified", ArrowDataType::Utf8, true),
1051 Field::new("ID", ArrowDataType::Utf8, true),
1052 Field::new("vaLue", ArrowDataType::Int32, true),
1053 ]));
1054
1055 let batch = RecordBatch::try_new(
1056 schema.clone(),
1057 vec![
1058 Arc::new(arrow::array::StringArray::from(vec![
1059 "2021-02-01",
1060 "2021-02-01",
1061 "2021-02-02",
1062 "2021-02-02",
1063 ])),
1064 Arc::new(arrow::array::StringArray::from(vec!["A", "B", "C", "D"])),
1065 Arc::new(arrow::array::Int32Array::from(vec![1, 10, 20, 100])),
1066 ],
1067 )
1068 .unwrap();
1069 let table = DeltaTable::new_in_memory()
1071 .write(vec![batch.clone()])
1072 .with_save_mode(crate::protocol::SaveMode::Append)
1073 .await
1074 .unwrap();
1075
1076 let provider = table.table_provider().await.unwrap();
1077 let ctx: SessionContext = DeltaSessionContext::default().into();
1078 ctx.register_table("test", provider).unwrap();
1079
1080 let df = ctx
1081 .sql("select ID, moDified, vaLue from test")
1082 .await
1083 .unwrap();
1084 let actual = df.collect().await.unwrap();
1085 let expected = vec![
1086 "+----+------------+-------+",
1087 "| ID | moDified | vaLue |",
1088 "+----+------------+-------+",
1089 "| A | 2021-02-01 | 1 |",
1090 "| B | 2021-02-01 | 10 |",
1091 "| C | 2021-02-02 | 20 |",
1092 "| D | 2021-02-02 | 100 |",
1093 "+----+------------+-------+",
1094 ];
1095 assert_batches_sorted_eq!(&expected, &actual);
1096
1097 }
1109
1110 #[tokio::test]
1111 async fn delta_scan_supports_missing_columns() {
1112 let schema1 = Arc::new(ArrowSchema::new(vec![Field::new(
1113 "col_1",
1114 ArrowDataType::Utf8,
1115 true,
1116 )]));
1117
1118 let batch1 = RecordBatch::try_new(
1119 schema1.clone(),
1120 vec![Arc::new(arrow::array::StringArray::from(vec![
1121 Some("A"),
1122 Some("B"),
1123 ]))],
1124 )
1125 .unwrap();
1126
1127 let schema2 = Arc::new(ArrowSchema::new(vec![
1128 Field::new("col_1", ArrowDataType::Utf8, true),
1129 Field::new("col_2", ArrowDataType::Utf8, true),
1130 ]));
1131
1132 let batch2 = RecordBatch::try_new(
1133 schema2.clone(),
1134 vec![
1135 Arc::new(arrow::array::StringArray::from(vec![
1136 Some("E"),
1137 Some("F"),
1138 Some("G"),
1139 ])),
1140 Arc::new(arrow::array::StringArray::from(vec![
1141 Some("E2"),
1142 Some("F2"),
1143 Some("G2"),
1144 ])),
1145 ],
1146 )
1147 .unwrap();
1148
1149 let table = DeltaTable::new_in_memory()
1150 .write(vec![batch2])
1151 .with_save_mode(crate::protocol::SaveMode::Append)
1152 .await
1153 .unwrap();
1154
1155 let table = table
1156 .write(vec![batch1])
1157 .with_schema_mode(SchemaMode::Merge)
1158 .with_save_mode(crate::protocol::SaveMode::Append)
1159 .await
1160 .unwrap();
1161
1162 let provider = table.table_provider().await.unwrap();
1163 let ctx: SessionContext = DeltaSessionContext::default().into();
1164 ctx.register_table("test", provider).unwrap();
1165
1166 let df = ctx.sql("select col_1, col_2 from test").await.unwrap();
1167 let actual = df.collect().await.unwrap();
1168 let expected = vec![
1169 "+-------+-------+",
1170 "| col_1 | col_2 |",
1171 "+-------+-------+",
1172 "| A | |",
1173 "| B | |",
1174 "| E | E2 |",
1175 "| F | F2 |",
1176 "| G | G2 |",
1177 "+-------+-------+",
1178 ];
1179 assert_batches_sorted_eq!(&expected, &actual);
1180 }
1181
1182 #[tokio::test]
1183 async fn delta_scan_supports_pushdown() {
1184 let schema = Arc::new(ArrowSchema::new(vec![
1185 Field::new("col_1", ArrowDataType::Utf8, false),
1186 Field::new("col_2", ArrowDataType::Utf8, false),
1187 ]));
1188
1189 let batch = RecordBatch::try_new(
1190 schema.clone(),
1191 vec![
1192 Arc::new(arrow::array::StringArray::from(vec![
1193 Some("A"),
1194 Some("B"),
1195 Some("C"),
1196 ])),
1197 Arc::new(arrow::array::StringArray::from(vec![
1198 Some("A2"),
1199 Some("B2"),
1200 Some("C2"),
1201 ])),
1202 ],
1203 )
1204 .unwrap();
1205
1206 let table = DeltaTable::new_in_memory()
1207 .write(vec![batch])
1208 .with_save_mode(crate::protocol::SaveMode::Append)
1209 .await
1210 .unwrap();
1211
1212 let provider = table.table_provider().await.unwrap();
1213
1214 let mut cfg = SessionConfig::default();
1215 cfg.options_mut().execution.parquet.pushdown_filters = true;
1216 let ctx = SessionContext::new_with_config(cfg);
1217 ctx.register_table("test", provider).unwrap();
1218
1219 let df = ctx
1220 .sql("select col_1, col_2 from test WHERE col_1 = 'A'")
1221 .await
1222 .unwrap();
1223 let actual = df.collect().await.unwrap();
1224 let expected = vec![
1225 "+-------+-------+",
1226 "| col_1 | col_2 |",
1227 "+-------+-------+",
1228 "| A | A2 |",
1229 "+-------+-------+",
1230 ];
1231 assert_batches_sorted_eq!(&expected, &actual);
1232 }
1233
1234 #[tokio::test]
1235 async fn delta_scan_supports_nested_missing_columns() {
1236 let column1_schema1: arrow::datatypes::Fields =
1237 vec![Field::new("col_1a", ArrowDataType::Utf8, true)].into();
1238 let schema1 = Arc::new(ArrowSchema::new(vec![Field::new(
1239 "col_1",
1240 ArrowDataType::Struct(column1_schema1.clone()),
1241 true,
1242 )]));
1243
1244 let batch1 = RecordBatch::try_new(
1245 schema1.clone(),
1246 vec![Arc::new(StructArray::new(
1247 column1_schema1,
1248 vec![Arc::new(arrow::array::StringArray::from(vec![
1249 Some("A"),
1250 Some("B"),
1251 ]))],
1252 None,
1253 ))],
1254 )
1255 .unwrap();
1256
1257 let column1_schema2: arrow_schema::Fields = vec![
1258 Field::new("col_1a", ArrowDataType::Utf8, true),
1259 Field::new("col_1b", ArrowDataType::Utf8, true),
1260 ]
1261 .into();
1262 let schema2 = Arc::new(ArrowSchema::new(vec![Field::new(
1263 "col_1",
1264 ArrowDataType::Struct(column1_schema2.clone()),
1265 true,
1266 )]));
1267
1268 let batch2 = RecordBatch::try_new(
1269 schema2.clone(),
1270 vec![Arc::new(StructArray::new(
1271 column1_schema2,
1272 vec![
1273 Arc::new(arrow::array::StringArray::from(vec![
1274 Some("E"),
1275 Some("F"),
1276 Some("G"),
1277 ])),
1278 Arc::new(arrow::array::StringArray::from(vec![
1279 Some("E2"),
1280 Some("F2"),
1281 Some("G2"),
1282 ])),
1283 ],
1284 None,
1285 ))],
1286 )
1287 .unwrap();
1288
1289 let table = DeltaTable::new_in_memory()
1290 .write(vec![batch1])
1291 .with_save_mode(crate::protocol::SaveMode::Append)
1292 .await
1293 .unwrap();
1294
1295 let table = table
1296 .write(vec![batch2])
1297 .with_schema_mode(SchemaMode::Merge)
1298 .with_save_mode(crate::protocol::SaveMode::Append)
1299 .await
1300 .unwrap();
1301
1302 let provider = table.table_provider().await.unwrap();
1303 let ctx: SessionContext = DeltaSessionContext::default().into();
1304 ctx.register_table("test", provider).unwrap();
1305
1306 let df = ctx
1307 .sql("select col_1.col_1a, col_1.col_1b from test")
1308 .await
1309 .unwrap();
1310 let actual = df.collect().await.unwrap();
1311 let expected = vec![
1312 "+--------------------+--------------------+",
1313 "| test.col_1[col_1a] | test.col_1[col_1b] |",
1314 "+--------------------+--------------------+",
1315 "| A | |",
1316 "| B | |",
1317 "| E | E2 |",
1318 "| F | F2 |",
1319 "| G | G2 |",
1320 "+--------------------+--------------------+",
1321 ];
1322 assert_batches_sorted_eq!(&expected, &actual);
1323 }
1324
1325 #[tokio::test]
1326 async fn test_multiple_predicate_pushdown() {
1327 let schema = Arc::new(ArrowSchema::new(vec![
1328 Field::new("moDified", ArrowDataType::Utf8, true),
1329 Field::new("id", ArrowDataType::Utf8, true),
1330 Field::new("vaLue", ArrowDataType::Int32, true),
1331 ]));
1332
1333 let batch = RecordBatch::try_new(
1334 schema.clone(),
1335 vec![
1336 Arc::new(arrow::array::StringArray::from(vec![
1337 "2021-02-01",
1338 "2021-02-01",
1339 "2021-02-02",
1340 "2021-02-02",
1341 ])),
1342 Arc::new(arrow::array::StringArray::from(vec!["A", "B", "C", "D"])),
1343 Arc::new(arrow::array::Int32Array::from(vec![1, 10, 20, 100])),
1344 ],
1345 )
1346 .unwrap();
1347 let table = DeltaTable::new_in_memory()
1349 .write(vec![batch.clone()])
1350 .with_save_mode(crate::protocol::SaveMode::Append)
1351 .await
1352 .unwrap();
1353
1354 let datafusion = SessionContext::new();
1355 table
1356 .update_datafusion_session(&datafusion.state())
1357 .unwrap();
1358
1359 datafusion
1360 .register_table("snapshot", table.table_provider().await.unwrap())
1361 .unwrap();
1362
1363 let df = datafusion
1364 .sql(r#"select * from snapshot where "vaLue" > 10000 and "vaLue" < 20000"#)
1365 .await
1366 .unwrap();
1367
1368 df.collect().await.unwrap();
1369 }
1370
1371 #[tokio::test]
1377 async fn passes_sanity_checker_when_all_files_filtered() {
1378 let table = open_fs_path("../test/tests/data/delta-2.2.0-partitioned-types");
1379 let ctx = create_session().into_inner();
1380 ctx.register_table("test", table.table_provider().await.unwrap())
1381 .unwrap();
1382
1383 let df = ctx
1384 .sql("select * from test where c3 = 100 ORDER BY c1 ASC")
1385 .await
1386 .unwrap();
1387 let actual = df.collect().await.unwrap();
1388
1389 assert_eq!(actual.len(), 0);
1390 }
1391
1392 #[tokio::test]
1393 async fn test_delta_scan_uses_parquet_column_pruning() {
1394 let small: Arc<dyn Array> = Arc::new(arrow::array::StringArray::from(vec!["a"]));
1395 let large: Arc<dyn Array> = Arc::new(arrow::array::StringArray::from(vec![
1396 "b".repeat(1024).as_str(),
1397 ]));
1398 let batch = RecordBatch::try_from_iter(vec![("small", small), ("large", large)]).unwrap();
1399 let table = DeltaTable::new_in_memory()
1400 .write(vec![batch])
1401 .with_save_mode(crate::protocol::SaveMode::Append)
1402 .await
1403 .unwrap();
1404
1405 let config = DeltaScanConfigBuilder::new()
1406 .build(table.snapshot().unwrap().snapshot())
1407 .unwrap();
1408
1409 let (log_store, mut operations) = recording_log_store(table.log_store());
1410 let provider = DeltaScanNext::new(table.snapshot().unwrap().snapshot().clone(), config)
1411 .unwrap()
1412 .with_log_store(log_store.clone());
1413 let ctx = SessionContext::new();
1414 ctx.register_table("test", Arc::new(provider)).unwrap();
1415 let state = ctx.state();
1416
1417 let df = ctx.sql("select small from test").await.unwrap();
1418 let plan = df.create_physical_plan().await.unwrap();
1419
1420 let mut stream = plan.execute(0, state.task_ctx()).unwrap();
1421 let Some(Ok(batch)) = stream.next().await else {
1422 panic!()
1423 };
1424 assert!(stream.next().await.is_none());
1425 assert_eq!(1, batch.num_columns());
1426 assert_eq!(1, batch.num_rows());
1427 let small = ScalarValue::try_from_array(batch.column_by_name("small").unwrap(), 0).unwrap();
1428 assert_eq!("a", small.to_string());
1429
1430 let files = table.get_files_by_partitions(&[]).await.unwrap();
1431 assert_eq!(1, files.len());
1432 let object_store = table.object_store();
1433 let file_meta = object_store.head(&files[0]).await.unwrap();
1434 let file_reader = parquet::arrow::async_reader::ParquetObjectReader::new(
1435 object_store,
1436 file_meta.location.clone(),
1437 )
1438 .with_file_size(file_meta.size);
1439 let parquet_metadata =
1440 parquet::arrow::async_reader::ParquetRecordBatchStreamBuilder::new(file_reader)
1441 .await
1442 .unwrap()
1443 .metadata()
1444 .as_ref()
1445 .clone();
1446 let (small_start, small_len) = parquet_metadata.row_group(0).column(0).byte_range();
1447 let small_range = small_start..small_start + small_len;
1448 let (large_start, large_len) = parquet_metadata.row_group(0).column(1).byte_range();
1449 let large_range = large_start..large_start + large_len;
1450
1451 let actual = drain_recorded_ops(&mut operations).await;
1452
1453 let data_ranges = actual
1454 .iter()
1455 .flat_map(|operation| match operation {
1456 ObjectStoreOperation::GetRange(PathKind::Data, range) => vec![range.clone()],
1457 ObjectStoreOperation::GetRanges(PathKind::Data, ranges) => ranges.clone(),
1458 _ => Vec::new(),
1459 })
1460 .collect::<Vec<_>>();
1461
1462 let overlaps = |left: &Range<u64>, right: &Range<u64>| {
1463 left.start < right.end && right.start < left.end
1464 };
1465
1466 assert!(
1467 !data_ranges.is_empty(),
1468 "expected ranged parquet data reads, saw {actual:?}"
1469 );
1470 assert!(
1471 data_ranges
1472 .iter()
1473 .any(|range| overlaps(range, &small_range)),
1474 "expected selected column chunk {small_range:?} to be read, saw {actual:?}"
1475 );
1476 assert!(
1477 data_ranges
1478 .iter()
1479 .all(|range| !overlaps(range, &large_range)),
1480 "expected unselected column chunk {large_range:?} to be pruned, saw {actual:?}"
1481 );
1482 assert!(
1483 !actual.iter().any(|operation| matches!(
1484 operation,
1485 ObjectStoreOperation::Get(PathKind::Data)
1486 | ObjectStoreOperation::GetOpts(PathKind::Data)
1487 )),
1488 "expected no full data file reads, saw {actual:?}"
1489 );
1490 }
1491
1492 #[tokio::test]
1493 async fn test_push_down_filter_panic_2602() -> DeltaResult<()> {
1494 use crate::kernel::schema::{DataType, PrimitiveType};
1495 let ctx = SessionContext::new();
1496 let table = DeltaTable::new_in_memory()
1497 .create()
1498 .with_column("id", DataType::Primitive(PrimitiveType::Long), true, None)
1499 .with_column(
1500 "name",
1501 DataType::Primitive(PrimitiveType::String),
1502 true,
1503 None,
1504 )
1505 .with_column("b", DataType::Primitive(PrimitiveType::Boolean), true, None)
1506 .with_column(
1507 "ts",
1508 DataType::Primitive(PrimitiveType::Timestamp),
1509 true,
1510 None,
1511 )
1512 .with_column("dt", DataType::Primitive(PrimitiveType::Date), true, None)
1513 .with_column(
1514 "zap",
1515 DataType::Array(Box::new(ArrayType::new(
1516 DataType::Primitive(PrimitiveType::Boolean),
1517 true,
1518 ))),
1519 true,
1520 None,
1521 )
1522 .await?;
1523 table.update_datafusion_session(&ctx.state()).unwrap();
1524
1525 ctx.register_table("snapshot", table.table_provider().await.unwrap())
1526 .unwrap();
1527
1528 let df = ctx
1529 .sql("select * from snapshot where id > 10000 and id < 20000")
1530 .await
1531 .unwrap();
1532
1533 let _ = df.collect().await?;
1534 Ok(())
1535 }
1536
1537 #[tokio::test]
1542 async fn test_files_matching_predicate_binary_partition_not_pruned() {
1543 use crate::delta_datafusion::files_matching_predicate;
1544 use arrow::array::BinaryArray;
1545 use datafusion::common::ScalarValue;
1546 use datafusion::logical_expr::{col, lit};
1547
1548 let arrow_schema = Arc::new(ArrowSchema::new(vec![
1550 Field::new("id", ArrowDataType::Int32, true),
1551 Field::new("bin_part", ArrowDataType::Binary, true),
1552 ]));
1553
1554 let batch_a = RecordBatch::try_new(
1555 arrow_schema.clone(),
1556 vec![
1557 Arc::new(arrow::array::Int32Array::from(vec![1_i32])) as Arc<dyn Array>,
1558 Arc::new(BinaryArray::from_vec(vec![b"aaa".as_ref()])) as Arc<dyn Array>,
1559 ],
1560 )
1561 .unwrap();
1562
1563 let batch_b = RecordBatch::try_new(
1564 arrow_schema.clone(),
1565 vec![
1566 Arc::new(arrow::array::Int32Array::from(vec![2_i32])) as Arc<dyn Array>,
1567 Arc::new(BinaryArray::from_vec(vec![b"bbb".as_ref()])) as Arc<dyn Array>,
1568 ],
1569 )
1570 .unwrap();
1571
1572 let table = DeltaTable::new_in_memory()
1574 .create()
1575 .with_column(
1576 "id",
1577 delta_kernel::schema::DataType::Primitive(
1578 delta_kernel::schema::PrimitiveType::Integer,
1579 ),
1580 true,
1581 None,
1582 )
1583 .with_column(
1584 "bin_part",
1585 delta_kernel::schema::DataType::Primitive(
1586 delta_kernel::schema::PrimitiveType::Binary,
1587 ),
1588 true,
1589 None,
1590 )
1591 .with_partition_columns(["bin_part"])
1592 .await
1593 .unwrap();
1594
1595 let table = table
1596 .write(vec![batch_a])
1597 .with_save_mode(crate::protocol::SaveMode::Append)
1598 .await
1599 .unwrap();
1600
1601 let table = table
1602 .write(vec![batch_b])
1603 .with_save_mode(crate::protocol::SaveMode::Append)
1604 .await
1605 .unwrap();
1606
1607 let snapshot = table.snapshot().unwrap().snapshot().clone();
1608 let log_data = snapshot.log_data();
1609
1610 let all_files: Vec<_> = files_matching_predicate(log_data.clone(), &[])
1612 .unwrap()
1613 .collect();
1614 assert_eq!(all_files.len(), 2, "expected 2 files before pruning");
1615
1616 let predicate = col("bin_part").eq(lit(ScalarValue::Binary(Some(b"aaa".to_vec()))));
1619 let kept_files: Vec<_> = files_matching_predicate(log_data.clone(), &[predicate])
1620 .unwrap()
1621 .collect();
1622
1623 assert_eq!(
1624 kept_files.len(),
1625 1,
1626 "expected exactly the 'aaa' partition file to survive equality pruning"
1627 );
1628 }
1629
1630 #[tokio::test]
1633 async fn test_files_matching_predicate_binary_partition_range_not_pruned() {
1634 use crate::delta_datafusion::files_matching_predicate;
1635 use arrow::array::BinaryArray;
1636 use datafusion::common::ScalarValue;
1637 use datafusion::logical_expr::{col, lit};
1638
1639 let arrow_schema = Arc::new(ArrowSchema::new(vec![
1640 Field::new("id", ArrowDataType::Int32, true),
1641 Field::new("bin_part", ArrowDataType::Binary, true),
1642 ]));
1643
1644 let batch_a = RecordBatch::try_new(
1645 arrow_schema.clone(),
1646 vec![
1647 Arc::new(arrow::array::Int32Array::from(vec![1_i32])) as Arc<dyn Array>,
1648 Arc::new(BinaryArray::from_vec(vec![b"aaa".as_ref()])) as Arc<dyn Array>,
1649 ],
1650 )
1651 .unwrap();
1652
1653 let batch_b = RecordBatch::try_new(
1654 arrow_schema.clone(),
1655 vec![
1656 Arc::new(arrow::array::Int32Array::from(vec![2_i32])) as Arc<dyn Array>,
1657 Arc::new(BinaryArray::from_vec(vec![b"bbb".as_ref()])) as Arc<dyn Array>,
1658 ],
1659 )
1660 .unwrap();
1661
1662 let table = DeltaTable::new_in_memory()
1663 .create()
1664 .with_column(
1665 "id",
1666 delta_kernel::schema::DataType::Primitive(
1667 delta_kernel::schema::PrimitiveType::Integer,
1668 ),
1669 true,
1670 None,
1671 )
1672 .with_column(
1673 "bin_part",
1674 delta_kernel::schema::DataType::Primitive(
1675 delta_kernel::schema::PrimitiveType::Binary,
1676 ),
1677 true,
1678 None,
1679 )
1680 .with_partition_columns(["bin_part"])
1681 .await
1682 .unwrap();
1683
1684 let table = table
1685 .write(vec![batch_a])
1686 .with_save_mode(crate::protocol::SaveMode::Append)
1687 .await
1688 .unwrap();
1689 let table = table
1690 .write(vec![batch_b])
1691 .with_save_mode(crate::protocol::SaveMode::Append)
1692 .await
1693 .unwrap();
1694
1695 let snapshot = table.snapshot().unwrap().snapshot().clone();
1696 let log_data = snapshot.log_data();
1697
1698 let gt_pred = col("bin_part").gt(lit(ScalarValue::Binary(Some(b"aaa".to_vec()))));
1701 let kept = files_matching_predicate(log_data.clone(), &[gt_pred])
1702 .unwrap()
1703 .count();
1704 assert_eq!(
1705 kept, 2,
1706 "range predicate '>' on binary partition must not prune any files"
1707 );
1708
1709 let lt_pred = col("bin_part").lt(lit(ScalarValue::Binary(Some(b"bbb".to_vec()))));
1711 let kept = files_matching_predicate(log_data.clone(), &[lt_pred])
1712 .unwrap()
1713 .count();
1714 assert_eq!(
1715 kept, 2,
1716 "range predicate '<' on binary partition must not prune any files"
1717 );
1718 }
1719}