1use std::borrow::Cow;
4use std::sync::Arc;
5
6use arrow_schema::{
7 DataType as ArrowDataType, Field, Schema as ArrowSchema, SchemaRef as ArrowSchemaRef,
8};
9use delta_kernel::arrow::array::BooleanArray;
10use delta_kernel::arrow::compute::filter_record_batch;
11use delta_kernel::arrow::record_batch::RecordBatch;
12use delta_kernel::engine::arrow_conversion::TryIntoArrow;
13use delta_kernel::engine::arrow_data::ArrowEngineData;
14use delta_kernel::expressions::{ColumnName, Scalar, StructData};
15use delta_kernel::scan::ScanMetadata;
16use delta_kernel::schema::{
17 ArrayType, DataType, MapType, PrimitiveType, Schema, SchemaRef, StructField, StructType,
18};
19use delta_kernel::snapshot::Snapshot;
20use delta_kernel::table_configuration::TableConfiguration;
21use delta_kernel::table_features::ColumnMappingMode;
22use delta_kernel::table_properties::{DataSkippingNumIndexedCols, TableProperties};
23use delta_kernel::transforms::{SchemaTransform, transform_output_type};
24use delta_kernel::{DeltaResult, ExpressionEvaluator};
25
26use crate::errors::{DeltaResult as DeltaResultLocal, DeltaTableError};
27use crate::kernel::{SCAN_ROW_ARROW_SCHEMA, StatsProjection};
28
29pub(crate) trait SnapshotExt {
34 fn stats_schema(&self) -> DeltaResult<SchemaRef>;
36
37 fn partitions_schema(&self) -> DeltaResultLocal<Option<SchemaRef>>;
39
40 fn scan_row_parsed_schema_arrow(&self) -> DeltaResultLocal<ArrowSchemaRef>;
45}
46
47impl SnapshotExt for TableConfiguration {
48 fn stats_schema(&self) -> DeltaResult<SchemaRef> {
49 let (source_schema, table_properties) = stats_inputs(self)?;
50 Ok(Arc::new(stats_schema(
51 &source_schema,
52 table_properties.as_ref(),
53 )))
54 }
55
56 fn partitions_schema(&self) -> DeltaResultLocal<Option<SchemaRef>> {
57 Ok(partitions_schema(
58 self.logical_schema().as_ref(),
59 self.metadata().partition_columns(),
60 )?
61 .map(Arc::new))
62 }
63
64 fn scan_row_parsed_schema_arrow(&self) -> DeltaResultLocal<ArrowSchemaRef> {
65 build_scan_row_parsed_schema_arrow(self.stats_schema()?, self.partitions_schema()?)
66 }
67}
68
69impl SnapshotExt for Snapshot {
70 fn stats_schema(&self) -> DeltaResult<SchemaRef> {
71 self.table_configuration().stats_schema()
72 }
73
74 fn partitions_schema(&self) -> DeltaResultLocal<Option<SchemaRef>> {
75 self.table_configuration().partitions_schema()
76 }
77
78 fn scan_row_parsed_schema_arrow(&self) -> DeltaResultLocal<ArrowSchemaRef> {
79 StatsProjection::Full.parsed_scan_row_schema_arrow(self)
80 }
81}
82
83fn build_scan_row_parsed_schema_arrow(
84 stats_schema: SchemaRef,
85 partition_schema: Option<SchemaRef>,
86) -> DeltaResultLocal<ArrowSchemaRef> {
87 let mut fields = SCAN_ROW_ARROW_SCHEMA.fields().to_vec();
88
89 let stats_schema: ArrowSchema = stats_schema.as_ref().try_into_arrow()?;
90 fields.push(Arc::new(Field::new(
91 "stats_parsed",
92 ArrowDataType::Struct(stats_schema.fields().to_owned()),
93 true,
94 )));
95
96 if let Some(partition_schema) = partition_schema {
97 let partition_schema: ArrowSchema = partition_schema.as_ref().try_into_arrow()?;
98 fields.push(Arc::new(Field::new(
99 "partitionValues_parsed",
100 ArrowDataType::Struct(partition_schema.fields().to_owned()),
101 false,
102 )));
103 }
104
105 Ok(Arc::new(ArrowSchema::new(fields)))
106}
107
108fn partitions_schema(
109 schema: &StructType,
110 partition_columns: &[String],
111) -> DeltaResultLocal<Option<StructType>> {
112 if partition_columns.is_empty() {
113 return Ok(None);
114 }
115 Ok(Some(StructType::try_new(
116 partition_columns
117 .iter()
118 .map(|col| {
119 schema.field(col).cloned().ok_or_else(|| {
120 DeltaTableError::Generic(format!("Partition column {col} not found in schema"))
121 })
122 })
123 .collect::<Result<Vec<_>, _>>()?,
124 )?))
125}
126
127pub(crate) fn stats_inputs(
131 table_configuration: &TableConfiguration,
132) -> DeltaResult<(StructType, Cow<'_, TableProperties>)> {
133 let source_schema = stats_source_schema(
134 table_configuration.logical_schema().as_ref(),
135 table_configuration.metadata().partition_columns(),
136 table_configuration.column_mapping_mode(),
137 )?;
138 let table_properties = stats_table_properties(
139 table_configuration.logical_schema().as_ref(),
140 table_configuration.table_properties(),
141 table_configuration.column_mapping_mode(),
142 );
143 Ok((source_schema, table_properties))
144}
145
146fn stats_source_schema(
153 logical_schema: &StructType,
154 partition_columns: &[String],
155 column_mapping_mode: ColumnMappingMode,
156) -> DeltaResult<StructType> {
157 let fields = logical_schema
158 .fields()
159 .filter(|field| !partition_columns.contains(field.name()))
160 .map(|field| field.make_physical(column_mapping_mode))
161 .collect::<DeltaResult<Vec<_>>>()?;
162 StructType::try_new(fields)
163}
164
165pub(crate) fn stats_table_properties<'a>(
169 logical_schema: &StructType,
170 table_properties: &'a TableProperties,
171 column_mapping_mode: ColumnMappingMode,
172) -> Cow<'a, TableProperties> {
173 if column_mapping_mode == ColumnMappingMode::None
174 || table_properties.data_skipping_stats_columns.is_none()
175 {
176 return Cow::Borrowed(table_properties);
177 }
178
179 let mut table_properties = table_properties.clone();
180 table_properties.data_skipping_stats_columns =
181 table_properties.data_skipping_stats_columns.map(|columns| {
182 columns
183 .iter()
184 .filter_map(|column| {
185 physical_column_name(logical_schema, column, column_mapping_mode)
186 })
187 .collect()
188 });
189 Cow::Owned(table_properties)
190}
191
192fn physical_column_name(
194 logical_schema: &StructType,
195 column: &ColumnName,
196 column_mapping_mode: ColumnMappingMode,
197) -> Option<ColumnName> {
198 let mut path = column.path().iter().peekable();
199 let mut physical_path = Vec::with_capacity(column.path().len());
200 let mut current_struct = logical_schema;
201
202 while let Some(field_name) = path.next() {
203 let field = current_struct.field(field_name)?;
204 physical_path.push(field.physical_name(column_mapping_mode).to_string());
205
206 if path.peek().is_some() {
207 let DataType::Struct(inner) = field.data_type() else {
208 return None;
209 };
210 current_struct = inner;
211 }
212 }
213
214 Some(ColumnName::new(physical_path))
215}
216
217pub(crate) fn stats_schema(
246 stats_source_schema: &Schema,
247 table_properties: &TableProperties,
248) -> Schema {
249 let mut fields = Vec::with_capacity(4);
250 fields.push(StructField::nullable("numRecords", DataType::LONG));
251
252 let mut base_transform = BaseStatsTransform::new(table_properties);
254 if let Some(base_schema) = base_transform.transform_struct(stats_source_schema) {
255 let base_schema = base_schema.into_owned();
256
257 let mut null_count_transform = NullCountStatsTransform;
259 if let Some(null_count_schema) = null_count_transform.transform_struct(&base_schema) {
260 fields.push(StructField::nullable(
261 "nullCount",
262 null_count_schema.into_owned(),
263 ));
264 };
265
266 let mut min_max_transform = MinMaxStatsTransform;
268 if let Some(min_max_schema) = min_max_transform.transform_struct(&base_schema) {
269 let min_max_schema = min_max_schema.into_owned();
270 fields.push(StructField::nullable("minValues", min_max_schema.clone()));
271 fields.push(StructField::nullable("maxValues", min_max_schema));
272 }
273 }
274
275 StructType::try_new(fields).expect("Failed to construct StructType for stats_schema")
276}
277
278pub(crate) struct NullCountStatsTransform;
280impl<'a> SchemaTransform<'a> for NullCountStatsTransform {
281 transform_output_type!(|'a, T| Option<Cow<'a, T>>);
282
283 fn transform_primitive(&mut self, _ptype: &'a PrimitiveType) -> Option<Cow<'a, PrimitiveType>> {
284 Some(Cow::Owned(PrimitiveType::Long))
285 }
286 fn transform_struct_field(&mut self, field: &'a StructField) -> Option<Cow<'a, StructField>> {
287 use Cow::*;
288
289 if matches!(
290 &field.data_type,
291 DataType::Array(_) | DataType::Map(_) | DataType::Variant(_)
292 ) {
293 return Some(Cow::Owned(StructField {
294 name: field.name.clone(),
295 data_type: DataType::LONG,
296 nullable: true,
297 metadata: Default::default(),
298 }));
299 }
300
301 match self.transform(&field.data_type)? {
302 Borrowed(_) => Some(Borrowed(field)),
303 dt => Some(Owned(StructField {
304 name: field.name.clone(),
305 data_type: dt.into_owned(),
306 nullable: true,
307 metadata: Default::default(),
308 })),
309 }
310 }
311}
312
313struct BaseStatsTransform {
327 n_columns: Option<DataSkippingNumIndexedCols>,
328 added_columns: u64,
329 column_names: Option<Vec<ColumnName>>,
330 path: Vec<String>,
331}
332
333impl BaseStatsTransform {
334 fn new(props: &TableProperties) -> Self {
335 if let Some(columns_names) = &props.data_skipping_stats_columns {
338 Self {
339 n_columns: None,
340 added_columns: 0,
341 column_names: Some(columns_names.clone()),
342 path: Vec::new(),
343 }
344 } else {
345 Self {
346 n_columns: Some(
347 props
348 .data_skipping_num_indexed_cols
349 .unwrap_or(DataSkippingNumIndexedCols::NumColumns(32)),
350 ),
351 added_columns: 0,
352 column_names: None,
353 path: Vec::new(),
354 }
355 }
356 }
357}
358
359impl<'a> SchemaTransform<'a> for BaseStatsTransform {
360 transform_output_type!(|'a, T| Option<Cow<'a, T>>);
361
362 fn transform_array(&mut self, atype: &'a ArrayType) -> Option<Cow<'a, ArrayType>> {
369 Some(Cow::Borrowed(atype))
370 }
371 fn transform_map(&mut self, mtype: &'a MapType) -> Option<Cow<'a, MapType>> {
372 Some(Cow::Borrowed(mtype))
373 }
374 fn transform_variant(&mut self, stype: &'a StructType) -> Option<Cow<'a, StructType>> {
375 Some(Cow::Borrowed(stype))
376 }
377
378 fn transform_struct_field(&mut self, field: &'a StructField) -> Option<Cow<'a, StructField>> {
379 use Cow::*;
380
381 if let Some(DataSkippingNumIndexedCols::NumColumns(n_cols)) = self.n_columns
384 && self.added_columns >= n_cols
385 {
386 return None;
387 }
388
389 self.path.push(field.name.clone());
390 let data_type = field.data_type();
391
392 let should_include = matches!(data_type, DataType::Struct(_))
397 || self
398 .column_names
399 .as_ref()
400 .map(|ns| should_include_column(&ColumnName::new(&self.path), ns))
401 .unwrap_or(true);
402
403 if !should_include {
404 self.path.pop();
405 return None;
406 }
407
408 if !matches!(data_type, DataType::Struct(_)) {
410 self.added_columns += 1;
411 }
412
413 let transformed = self.transform(&field.data_type);
414 self.path.pop();
416
417 let field = match transformed? {
418 Borrowed(_) if field.is_nullable() => Borrowed(field),
419 data_type => Owned(StructField {
420 name: field.name.clone(),
421 data_type: data_type.into_owned(),
422 nullable: true,
423 metadata: Default::default(),
424 }),
425 };
426
427 if matches!(field.data_type(), DataType::Struct(dt) if dt.num_fields() == 0) {
429 None
430 } else {
431 Some(field)
432 }
433 }
434}
435
436struct MinMaxStatsTransform;
440
441impl<'a> SchemaTransform<'a> for MinMaxStatsTransform {
442 transform_output_type!(|'a, T| Option<Cow<'a, T>>);
443
444 fn transform_array(&mut self, _: &'a ArrayType) -> Option<Cow<'a, ArrayType>> {
446 None
447 }
448 fn transform_map(&mut self, _: &'a MapType) -> Option<Cow<'a, MapType>> {
449 None
450 }
451 fn transform_variant(&mut self, _: &'a StructType) -> Option<Cow<'a, StructType>> {
452 None
453 }
454
455 fn transform_primitive(&mut self, ptype: &'a PrimitiveType) -> Option<Cow<'a, PrimitiveType>> {
456 if is_public_min_max_stats_eligible_primitive(ptype) {
457 Some(Cow::Borrowed(ptype))
458 } else {
459 None
460 }
461 }
462}
463
464fn should_include_column(column_name: &ColumnName, column_names: &[ColumnName]) -> bool {
470 column_names.iter().any(|name| {
471 name.as_ref().starts_with(column_name) || column_name.as_ref().starts_with(name)
472 })
473}
474
475pub(crate) fn is_public_min_max_stats_eligible_primitive(data_type: &PrimitiveType) -> bool {
483 #[cfg(not(feature = "nanosecond-timestamps"))]
484 let matches_nanos = false;
485 #[cfg(feature = "nanosecond-timestamps")]
486 let matches_nanos = matches!(
487 data_type,
488 &PrimitiveType::TimestampNanos | &PrimitiveType::TimestampNanosNtz
489 );
490 matches!(
491 data_type,
492 &PrimitiveType::Byte
493 | &PrimitiveType::Short
494 | &PrimitiveType::Integer
495 | &PrimitiveType::Long
496 | &PrimitiveType::Float
497 | &PrimitiveType::Double
498 | &PrimitiveType::Boolean
499 | &PrimitiveType::Date
500 | &PrimitiveType::Timestamp
501 | &PrimitiveType::TimestampNtz
502 | &PrimitiveType::String
503 | PrimitiveType::Decimal(_)
504 ) || matches_nanos
505}
506
507pub(crate) fn rb_from_scan_meta(metadata: ScanMetadata) -> DeltaResult<RecordBatch> {
508 let (underlying_data, selection_vector) = metadata.scan_files.into_parts();
509 let batch = ArrowEngineData::try_from_engine_data(underlying_data)?.into();
510 Ok(filter_record_batch(
511 &batch,
512 &BooleanArray::from(selection_vector),
513 )?)
514}
515
516pub(crate) trait ExpressionEvaluatorExt {
521 fn evaluate_arrow(&self, batch: RecordBatch) -> DeltaResult<RecordBatch>;
522}
523
524impl<T: ExpressionEvaluator + ?Sized> ExpressionEvaluatorExt for T {
525 fn evaluate_arrow(&self, batch: RecordBatch) -> DeltaResult<RecordBatch> {
526 let engine_data = ArrowEngineData::new(batch);
527 Ok(ArrowEngineData::try_from_engine_data(T::evaluate(self, &engine_data)?)?.into())
528 }
529}
530
531pub trait StructDataExt {
539 fn field(&self, name: &str) -> Option<&StructField>;
541
542 fn value(&self, index: usize) -> Option<&Scalar>;
544
545 fn index_of(&self, name: &str) -> Option<usize>;
547}
548
549impl StructDataExt for StructData {
550 fn field(&self, name: &str) -> Option<&StructField> {
551 self.fields().iter().find(|f| f.name() == name)
552 }
553
554 fn index_of(&self, name: &str) -> Option<usize> {
555 self.fields().iter().position(|f| f.name() == name)
556 }
557
558 fn value(&self, index: usize) -> Option<&Scalar> {
559 self.values().get(index)
560 }
561}
562
563#[cfg(test)]
564mod tests {
565 use std::collections::HashMap;
566 use std::sync::Arc;
567
568 use super::*;
569
570 use crate::test_utils::{
571 build_test_table_configuration, column_mapping_test_field,
572 column_mapping_test_field_with_type,
573 };
574
575 use delta_kernel::EvaluationHandler;
576 use delta_kernel::arrow::array::Int32Array;
577 use delta_kernel::arrow::datatypes::{DataType as ArrowDataType, Field, Schema};
578 use delta_kernel::arrow::record_batch::RecordBatch;
579 use delta_kernel::engine::arrow_conversion::TryIntoKernel;
580 use delta_kernel::engine::arrow_expression::ArrowEvaluationHandler;
581 use delta_kernel::expressions::*;
582 use delta_kernel::schema::{ArrayType, DataType};
583 use pretty_assertions::assert_eq;
584
585 #[test]
586 fn test_evaluate_arrow() {
587 let handler = ArrowEvaluationHandler;
588
589 let schema = Schema::new(vec![Field::new("a", ArrowDataType::Int32, false)]);
590 let values = Int32Array::from(vec![1, 2, 3]);
591 let batch = RecordBatch::try_new(Arc::new(schema.clone()), vec![Arc::new(values)]).unwrap();
592
593 let expression = column_expr!("a");
594 let expr = handler
595 .new_expression_evaluator(
596 Arc::new((&schema).try_into_kernel().unwrap()),
597 expression.into(),
598 DataType::INTEGER,
599 )
600 .unwrap();
601
602 let result = expr.evaluate_arrow(batch);
603 assert!(result.is_ok());
604 }
605
606 #[test]
607 fn test_should_include_column() {
608 let full_name = vec![ColumnName::new(["lvl1", "lvl2", "lvl3", "lvl4"])];
609 let parent = ColumnName::new(["lvl1", "lvl2", "lvl3"]);
610 assert!(should_include_column(&parent, &full_name));
611 assert!(should_include_column(&full_name[0], &full_name));
612 assert!(should_include_column(&full_name[0], &[parent]));
614
615 let not_parent = ColumnName::new(["lvl1", "lvl2", "lvl3", "lvl5"]);
616 assert!(!should_include_column(¬_parent, &full_name));
617 let not_parent = ColumnName::new(["lvl1", "lvl3", "lvl4"]);
618 assert!(!should_include_column(¬_parent, &full_name));
619
620 let not_parent = ColumnName::new(["lvl1", "lvl2", "lvl4"]);
621 assert!(!should_include_column(¬_parent, &full_name));
622 }
623
624 #[test]
625 fn test_stats_schema_simple() {
626 let properties: TableProperties = [("key", "value")].into();
627 let file_schema =
628 StructType::try_new([StructField::nullable("id", DataType::LONG)]).unwrap();
629
630 let stats_schema = stats_schema(&file_schema, &properties);
631 let expected = StructType::try_new([
632 StructField::nullable("numRecords", DataType::LONG),
633 StructField::nullable("nullCount", file_schema.clone()),
634 StructField::nullable("minValues", file_schema.clone()),
635 StructField::nullable("maxValues", file_schema),
636 ])
637 .unwrap();
638
639 assert_eq!(&expected, &stats_schema);
640 }
641
642 #[test]
643 fn test_stats_schema_with_non_eligible_field() {
644 let properties: TableProperties = [("key", "value")].into();
645
646 let array_type = DataType::Array(Box::new(ArrayType::new(DataType::STRING, false)));
655 let metadata_struct = StructType::try_new([
656 StructField::nullable("name", DataType::STRING),
657 StructField::nullable("tags", array_type),
658 StructField::nullable("score", DataType::DOUBLE),
659 ])
660 .unwrap();
661 let file_schema = StructType::try_new([
662 StructField::nullable("id", DataType::LONG),
663 StructField::nullable(
664 "metadata",
665 DataType::Struct(Box::new(metadata_struct.clone())),
666 ),
667 ])
668 .unwrap();
669
670 let stats_schema = stats_schema(&file_schema, &properties);
671
672 let expected_null_nested = StructType::try_new([
673 StructField::nullable("name", DataType::LONG),
674 StructField::nullable("tags", DataType::LONG),
675 StructField::nullable("score", DataType::LONG),
676 ])
677 .unwrap();
678 let expected_null = StructType::try_new([
679 StructField::nullable("id", DataType::LONG),
680 StructField::nullable("metadata", DataType::Struct(Box::new(expected_null_nested))),
681 ])
682 .unwrap();
683
684 let expected_nested = StructType::try_new([
685 StructField::nullable("name", DataType::STRING),
686 StructField::nullable("score", DataType::DOUBLE),
687 ])
688 .unwrap();
689 let expected_fields = StructType::try_new([
690 StructField::nullable("id", DataType::LONG),
691 StructField::nullable("metadata", DataType::Struct(Box::new(expected_nested))),
692 ])
693 .unwrap();
694
695 let expected = StructType::try_new([
696 StructField::nullable("numRecords", DataType::LONG),
697 StructField::nullable("nullCount", expected_null),
698 StructField::nullable("minValues", expected_fields.clone()),
699 StructField::nullable("maxValues", expected_fields.clone()),
700 ])
701 .unwrap();
702
703 assert_eq!(&expected, &stats_schema);
704 }
705
706 #[test]
707 fn test_stats_schema_col_names() {
708 let properties: TableProperties = [(
709 "delta.dataSkippingStatsColumns".to_string(),
710 "`user.info`.name".to_string(),
711 )]
712 .into();
713
714 let user_struct = StructType::try_new([
715 StructField::nullable("name", DataType::STRING),
716 StructField::nullable("age", DataType::INTEGER),
717 ])
718 .unwrap();
719 let file_schema = StructType::try_new([
720 StructField::nullable("id", DataType::LONG),
721 StructField::nullable("user.info", DataType::Struct(Box::new(user_struct.clone()))),
722 ])
723 .unwrap();
724
725 let stats_schema = stats_schema(&file_schema, &properties);
726
727 let expected_nested =
728 StructType::try_new([StructField::nullable("name", DataType::STRING)]).unwrap();
729 let expected_fields = StructType::try_new([StructField::nullable(
730 "user.info",
731 DataType::Struct(Box::new(expected_nested)),
732 )])
733 .unwrap();
734 let null_count = NullCountStatsTransform
735 .transform_struct(&expected_fields)
736 .unwrap()
737 .into_owned();
738
739 let expected = StructType::try_new([
740 StructField::nullable("numRecords", DataType::LONG),
741 StructField::nullable("nullCount", null_count),
742 StructField::nullable("minValues", expected_fields.clone()),
743 StructField::nullable("maxValues", expected_fields.clone()),
744 ])
745 .unwrap();
746
747 assert_eq!(&expected, &stats_schema);
748 }
749
750 #[test]
753 fn test_stats_schema_col_names_after_excluded_struct() {
754 let properties: TableProperties = [(
755 "delta.dataSkippingStatsColumns".to_string(),
756 "a,b".to_string(),
757 )]
758 .into();
759
760 let st = StructType::try_new([StructField::nullable("x", DataType::LONG)]).unwrap();
761 let file_schema = StructType::try_new([
762 StructField::nullable("a", DataType::LONG),
763 StructField::nullable("st", DataType::Struct(Box::new(st))),
764 StructField::nullable("b", DataType::LONG),
765 ])
766 .unwrap();
767
768 let stats_schema = stats_schema(&file_schema, &properties);
769
770 let expected_fields = StructType::try_new([
771 StructField::nullable("a", DataType::LONG),
772 StructField::nullable("b", DataType::LONG),
773 ])
774 .unwrap();
775 let expected = StructType::try_new([
776 StructField::nullable("numRecords", DataType::LONG),
777 StructField::nullable("nullCount", expected_fields.clone()),
778 StructField::nullable("minValues", expected_fields.clone()),
779 StructField::nullable("maxValues", expected_fields),
780 ])
781 .unwrap();
782
783 assert_eq!(&expected, &stats_schema);
784 }
785
786 #[test]
787 fn test_stats_source_schema_excludes_partition_columns() {
788 let logical_schema = StructType::try_new([
789 StructField::nullable("p", DataType::INTEGER),
790 StructField::nullable("a", DataType::INTEGER),
791 ])
792 .unwrap();
793
794 let stats_schema =
795 stats_source_schema(&logical_schema, &["p".to_string()], ColumnMappingMode::None)
796 .unwrap();
797
798 assert!(stats_schema.field("a").is_some());
799 assert!(stats_schema.field("p").is_none());
800 }
801
802 #[test]
803 fn test_stats_source_schema_applies_column_mapping_physical_names() {
804 let logical_schema = StructType::try_new([
805 column_mapping_test_field("p", "col_p", 1),
806 column_mapping_test_field("a", "col_a", 2),
807 ])
808 .unwrap();
809
810 for column_mapping_mode in [ColumnMappingMode::Name, ColumnMappingMode::Id] {
811 let stats_schema =
812 stats_source_schema(&logical_schema, &["p".to_string()], column_mapping_mode)
813 .unwrap();
814
815 assert!(stats_schema.field("col_a").is_some());
816 assert!(stats_schema.field("a").is_none());
817 assert!(stats_schema.field("col_p").is_none());
818 assert!(stats_schema.field("p").is_none());
819 }
820 }
821
822 #[test]
823 fn test_table_configuration_stats_schema_translates_stats_columns_to_physical_names() {
824 for column_mapping_mode in ["name", "id"] {
826 let logical_schema = StructType::try_new([
827 column_mapping_test_field("p", "col_p", 1),
828 column_mapping_test_field("a", "col_a", 2),
829 column_mapping_test_field("b", "col_b", 3),
830 ])
831 .unwrap();
832 let table_configuration = build_test_table_configuration(
833 logical_schema,
834 vec!["p".to_string()],
835 HashMap::from([
836 (
837 "delta.columnMapping.mode".to_string(),
838 column_mapping_mode.to_string(),
839 ),
840 (
841 "delta.dataSkippingStatsColumns".to_string(),
842 "a".to_string(),
843 ),
844 ]),
845 );
846
847 let stats_schema = table_configuration.stats_schema().unwrap();
848
849 let min_values = match stats_schema.field("minValues").unwrap().data_type() {
850 DataType::Struct(fields) => fields,
851 other => panic!("expected minValues struct, got {other:?}"),
852 };
853 assert!(min_values.field("col_a").is_some());
854 assert!(min_values.field("a").is_none());
855 assert!(min_values.field("col_p").is_none());
856 assert!(min_values.field("col_b").is_none());
857 }
858 }
859
860 #[test]
861 fn test_table_configuration_stats_schema_translates_nested_stats_columns_to_physical_names() {
862 for column_mapping_mode in ["name", "id"] {
864 let nested_schema = StructType::try_new([
865 column_mapping_test_field("a", "col_a", 3),
866 column_mapping_test_field("b", "col_b", 4),
867 ])
868 .unwrap();
869 let logical_schema = StructType::try_new([
870 column_mapping_test_field("p", "col_p", 1),
871 column_mapping_test_field_with_type(
872 "s",
873 "col_s",
874 2,
875 DataType::Struct(Box::new(nested_schema)),
876 ),
877 ])
878 .unwrap();
879 let table_configuration = build_test_table_configuration(
880 logical_schema,
881 vec!["p".to_string()],
882 HashMap::from([
883 (
884 "delta.columnMapping.mode".to_string(),
885 column_mapping_mode.to_string(),
886 ),
887 (
888 "delta.dataSkippingStatsColumns".to_string(),
889 "s.a".to_string(),
890 ),
891 ]),
892 );
893
894 let stats_schema = table_configuration.stats_schema().unwrap();
895
896 let min_values = match stats_schema.field("minValues").unwrap().data_type() {
897 DataType::Struct(fields) => fields,
898 other => panic!("expected minValues struct, got {other:?}"),
899 };
900 let nested_min_values = match min_values.field("col_s").unwrap().data_type() {
901 DataType::Struct(fields) => fields,
902 other => panic!("expected nested minValues struct, got {other:?}"),
903 };
904 assert!(nested_min_values.field("col_a").is_some());
905 assert!(nested_min_values.field("a").is_none());
906 assert!(nested_min_values.field("col_b").is_none());
907 assert!(min_values.field("s").is_none());
908 assert!(min_values.field("col_p").is_none());
909 }
910 }
911
912 #[test]
913 fn test_table_configuration_stats_schema_drops_stats_columns_with_non_struct_intermediate() {
914 for column_mapping_mode in ["name", "id"] {
916 let logical_schema = StructType::try_new([
917 column_mapping_test_field("p", "col_p", 1),
918 column_mapping_test_field("a", "col_a", 2),
919 ])
920 .unwrap();
921 let table_configuration = build_test_table_configuration(
922 logical_schema,
923 vec!["p".to_string()],
924 HashMap::from([
925 (
926 "delta.columnMapping.mode".to_string(),
927 column_mapping_mode.to_string(),
928 ),
929 (
930 "delta.dataSkippingStatsColumns".to_string(),
931 "a.b".to_string(),
932 ),
933 ]),
934 );
935
936 let stats_schema = table_configuration.stats_schema().unwrap();
937
938 assert!(stats_schema.field("nullCount").is_none());
939 assert!(stats_schema.field("minValues").is_none());
940 assert!(stats_schema.field("maxValues").is_none());
941 }
942 }
943
944 #[test]
945 fn test_table_configuration_stats_schema_num_indexed_cols_ignores_partition_columns() {
946 let logical_schema = StructType::try_new([
947 StructField::nullable("p", DataType::INTEGER),
948 StructField::nullable("a", DataType::INTEGER),
949 ])
950 .unwrap();
951 let table_configuration = build_test_table_configuration(
952 logical_schema,
953 vec!["p".to_string()],
954 HashMap::from([(
955 "delta.dataSkippingNumIndexedCols".to_string(),
956 "1".to_string(),
957 )]),
958 );
959
960 let stats_schema = table_configuration.stats_schema().unwrap();
961
962 let min_values = match stats_schema.field("minValues").unwrap().data_type() {
963 DataType::Struct(fields) => fields,
964 other => panic!("expected minValues struct, got {other:?}"),
965 };
966 assert!(min_values.field("a").is_some());
967 assert!(min_values.field("p").is_none());
968 assert_eq!(1, min_values.fields().count());
969 }
970
971 #[test]
972 fn test_stats_schema_n_cols() {
973 let properties: TableProperties = [(
974 "delta.dataSkippingNumIndexedCols".to_string(),
975 "1".to_string(),
976 )]
977 .into();
978
979 let logical_schema = StructType::try_new([
980 StructField::nullable("name", DataType::STRING),
981 StructField::nullable("age", DataType::INTEGER),
982 ])
983 .unwrap();
984
985 let stats_schema = stats_schema(&logical_schema, &properties);
986
987 let expected_fields =
988 StructType::try_new([StructField::nullable("name", DataType::STRING)]).unwrap();
989 let null_count = NullCountStatsTransform
990 .transform_struct(&expected_fields)
991 .unwrap()
992 .into_owned();
993
994 let expected = StructType::try_new([
995 StructField::nullable("numRecords", DataType::LONG),
996 StructField::nullable("nullCount", null_count),
997 StructField::nullable("minValues", expected_fields.clone()),
998 StructField::nullable("maxValues", expected_fields.clone()),
999 ])
1000 .unwrap();
1001
1002 assert_eq!(&expected, &stats_schema);
1003 }
1004
1005 #[test]
1012 fn test_stats_schema_different_fields_in_null_vs_minmax() {
1013 let properties: TableProperties = [("key", "value")].into();
1014
1015 let file_schema = StructType::try_new([
1020 StructField::nullable("id", DataType::LONG),
1021 StructField::nullable("is_active", DataType::BOOLEAN),
1022 StructField::nullable("metadata", DataType::BINARY),
1023 ])
1024 .unwrap();
1025
1026 let stats_schema = stats_schema(&file_schema, &properties);
1027
1028 let expected_null_count = StructType::try_new([
1030 StructField::nullable("id", DataType::LONG),
1031 StructField::nullable("is_active", DataType::LONG),
1032 StructField::nullable("metadata", DataType::LONG),
1033 ])
1034 .unwrap();
1035
1036 let expected_min_max = StructType::try_new([
1038 StructField::nullable("id", DataType::LONG),
1039 StructField::nullable("is_active", DataType::BOOLEAN),
1040 ])
1041 .unwrap();
1042
1043 let expected = StructType::try_new([
1044 StructField::nullable("numRecords", DataType::LONG),
1045 StructField::nullable("nullCount", expected_null_count),
1046 StructField::nullable("minValues", expected_min_max.clone()),
1047 StructField::nullable("maxValues", expected_min_max),
1048 ])
1049 .unwrap();
1050
1051 assert_eq!(&expected, &stats_schema);
1052 }
1053
1054 #[test]
1055 fn test_stats_schema_nested_different_fields_in_null_vs_minmax() {
1056 let properties: TableProperties = [("key", "value")].into();
1057
1058 let user_struct = StructType::try_new([
1060 StructField::nullable("name", DataType::STRING), StructField::nullable("is_admin", DataType::BOOLEAN), StructField::nullable("age", DataType::INTEGER), StructField::nullable("profile_pic", DataType::BINARY), ])
1065 .unwrap();
1066
1067 let file_schema = StructType::try_new([
1068 StructField::nullable("id", DataType::LONG),
1069 StructField::nullable("user", DataType::Struct(Box::new(user_struct.clone()))),
1070 StructField::nullable("is_deleted", DataType::BOOLEAN), ])
1072 .unwrap();
1073
1074 let stats_schema = stats_schema(&file_schema, &properties);
1075
1076 let expected_null_user = StructType::try_new([
1078 StructField::nullable("name", DataType::LONG),
1079 StructField::nullable("is_admin", DataType::LONG),
1080 StructField::nullable("age", DataType::LONG),
1081 StructField::nullable("profile_pic", DataType::LONG),
1082 ])
1083 .unwrap();
1084 let expected_null_count = StructType::try_new([
1085 StructField::nullable("id", DataType::LONG),
1086 StructField::nullable("user", DataType::Struct(Box::new(expected_null_user))),
1087 StructField::nullable("is_deleted", DataType::LONG),
1088 ])
1089 .unwrap();
1090
1091 let expected_minmax_user = StructType::try_new([
1093 StructField::nullable("name", DataType::STRING),
1094 StructField::nullable("is_admin", DataType::BOOLEAN),
1095 StructField::nullable("age", DataType::INTEGER),
1096 ])
1097 .unwrap();
1098 let expected_min_max = StructType::try_new([
1099 StructField::nullable("id", DataType::LONG),
1100 StructField::nullable("user", DataType::Struct(Box::new(expected_minmax_user))),
1101 StructField::nullable("is_deleted", DataType::BOOLEAN),
1102 ])
1103 .unwrap();
1104
1105 let expected = StructType::try_new([
1106 StructField::nullable("numRecords", DataType::LONG),
1107 StructField::nullable("nullCount", expected_null_count),
1108 StructField::nullable("minValues", expected_min_max.clone()),
1109 StructField::nullable("maxValues", expected_min_max),
1110 ])
1111 .unwrap();
1112
1113 assert_eq!(&expected, &stats_schema);
1114 }
1115
1116 #[test]
1117 fn test_stats_schema_only_non_eligible_fields() {
1118 let properties: TableProperties = [("key", "value")].into();
1119
1120 let file_schema = StructType::try_new([
1122 StructField::nullable("metadata", DataType::BINARY),
1123 StructField::nullable(
1124 "tags",
1125 DataType::Array(Box::new(ArrayType::new(DataType::STRING, false))),
1126 ),
1127 ])
1128 .unwrap();
1129
1130 let stats_schema = stats_schema(&file_schema, &properties);
1131
1132 let expected_null_count = StructType::try_new([
1134 StructField::nullable("metadata", DataType::LONG),
1135 StructField::nullable("tags", DataType::LONG),
1136 ])
1137 .unwrap();
1138
1139 let expected = StructType::try_new([
1142 StructField::nullable("numRecords", DataType::LONG),
1143 StructField::nullable("nullCount", expected_null_count),
1144 ])
1146 .unwrap();
1147
1148 assert_eq!(&expected, &stats_schema);
1149 }
1150
1151 #[test]
1155 fn test_stats_schema_drops_nested_struct_without_eligible_fields() {
1156 let properties: TableProperties = [("key", "value")].into();
1157 let payload =
1159 StructType::try_new([StructField::nullable("data", DataType::BINARY)]).unwrap();
1160 let file_schema = StructType::try_new([
1161 StructField::nullable("id", DataType::LONG),
1162 StructField::nullable("payload", DataType::Struct(Box::new(payload))),
1163 ])
1164 .unwrap();
1165
1166 let null_count = StructType::try_new([
1168 StructField::nullable("id", DataType::LONG),
1169 StructField::nullable(
1170 "payload",
1171 DataType::Struct(Box::new(
1172 StructType::try_new([StructField::nullable("data", DataType::LONG)]).unwrap(),
1173 )),
1174 ),
1175 ])
1176 .unwrap();
1177 let min_max_without_payload =
1179 StructType::try_new([StructField::nullable("id", DataType::LONG)]).unwrap();
1180
1181 let expected = StructType::try_new([
1182 StructField::nullable("numRecords", DataType::LONG),
1183 StructField::nullable("nullCount", null_count),
1184 StructField::nullable("minValues", min_max_without_payload.clone()),
1185 StructField::nullable("maxValues", min_max_without_payload),
1186 ])
1187 .unwrap();
1188
1189 assert_eq!(&expected, &stats_schema(&file_schema, &properties));
1190 }
1191
1192 #[test]
1193 fn test_partitions_schema() -> DeltaResultLocal<()> {
1194 let logical_schema = StructType::try_new([
1195 StructField::nullable("name", DataType::STRING),
1196 StructField::nullable("age", DataType::INTEGER),
1197 ])
1198 .unwrap();
1199
1200 let result = partitions_schema(&logical_schema, &[])?;
1201 assert_eq!(None, result);
1202 Ok(())
1203 }
1204
1205 #[tokio::test]
1206 async fn test_partition_schema2() {
1207 let schema = StructType::try_new(vec![
1208 StructField::new("id", DataType::LONG, true),
1209 StructField::new("name", DataType::STRING, true),
1210 StructField::new("date", DataType::DATE, true),
1211 ])
1212 .unwrap();
1213
1214 let partition_columns = vec!["date".to_string()];
1215 let expected =
1216 StructType::try_new(vec![StructField::new("date", DataType::DATE, true)]).unwrap();
1217 assert_eq!(
1218 partitions_schema(&schema, &partition_columns).unwrap(),
1219 Some(expected)
1220 );
1221
1222 assert_eq!(partitions_schema(&schema, &[]).unwrap(), None);
1223 }
1224}