Skip to main content

deltalake_core/kernel/arrow/
engine_ext.rs

1//! Utilities for interacting with Kernel APIs using Arrow data structures.
2//!
3use std::borrow::Cow;
4use std::sync::Arc;
5
6use arrow_schema::{
7    DataType as ArrowDataType, Field, Schema as ArrowSchema, SchemaRef as ArrowSchemaRef,
8};
9use delta_kernel::arrow::array::BooleanArray;
10use delta_kernel::arrow::compute::filter_record_batch;
11use delta_kernel::arrow::record_batch::RecordBatch;
12use delta_kernel::engine::arrow_conversion::TryIntoArrow;
13use delta_kernel::engine::arrow_data::ArrowEngineData;
14use delta_kernel::expressions::{ColumnName, Scalar, StructData};
15use delta_kernel::scan::ScanMetadata;
16use delta_kernel::schema::{
17    ArrayType, DataType, MapType, PrimitiveType, Schema, SchemaRef, StructField, StructType,
18};
19use delta_kernel::snapshot::Snapshot;
20use delta_kernel::table_configuration::TableConfiguration;
21use delta_kernel::table_features::ColumnMappingMode;
22use delta_kernel::table_properties::{DataSkippingNumIndexedCols, TableProperties};
23use delta_kernel::transforms::{SchemaTransform, transform_output_type};
24use delta_kernel::{DeltaResult, ExpressionEvaluator};
25
26use crate::errors::{DeltaResult as DeltaResultLocal, DeltaTableError};
27use crate::kernel::{SCAN_ROW_ARROW_SCHEMA, StatsProjection};
28
29/// Internal extension traits to the Kernel Snapshot.
30///
31/// These traits provide additional convenience functionality for working with Kernel snapshots.
32/// Some of this may eventually be upstreamed as the kernel implementation matures.
33pub(crate) trait SnapshotExt {
34    /// Returns the expected file statistics schema for the snapshot.
35    fn stats_schema(&self) -> DeltaResult<SchemaRef>;
36
37    /// The expected schema for partition values
38    fn partitions_schema(&self) -> DeltaResultLocal<Option<SchemaRef>>;
39
40    /// The scheme expected for the data returned from a scan.
41    ///
42    /// This extends the raw schema. It keeps the raw `stats` JSON column and adds
43    /// `stats_parsed` and `partitionValues_parsed`.
44    fn scan_row_parsed_schema_arrow(&self) -> DeltaResultLocal<ArrowSchemaRef>;
45}
46
47impl SnapshotExt for TableConfiguration {
48    fn stats_schema(&self) -> DeltaResult<SchemaRef> {
49        let (source_schema, table_properties) = stats_inputs(self)?;
50        Ok(Arc::new(stats_schema(
51            &source_schema,
52            table_properties.as_ref(),
53        )))
54    }
55
56    fn partitions_schema(&self) -> DeltaResultLocal<Option<SchemaRef>> {
57        Ok(partitions_schema(
58            self.logical_schema().as_ref(),
59            self.metadata().partition_columns(),
60        )?
61        .map(Arc::new))
62    }
63
64    fn scan_row_parsed_schema_arrow(&self) -> DeltaResultLocal<ArrowSchemaRef> {
65        build_scan_row_parsed_schema_arrow(self.stats_schema()?, self.partitions_schema()?)
66    }
67}
68
69impl SnapshotExt for Snapshot {
70    fn stats_schema(&self) -> DeltaResult<SchemaRef> {
71        self.table_configuration().stats_schema()
72    }
73
74    fn partitions_schema(&self) -> DeltaResultLocal<Option<SchemaRef>> {
75        self.table_configuration().partitions_schema()
76    }
77
78    fn scan_row_parsed_schema_arrow(&self) -> DeltaResultLocal<ArrowSchemaRef> {
79        StatsProjection::Full.parsed_scan_row_schema_arrow(self)
80    }
81}
82
83fn build_scan_row_parsed_schema_arrow(
84    stats_schema: SchemaRef,
85    partition_schema: Option<SchemaRef>,
86) -> DeltaResultLocal<ArrowSchemaRef> {
87    let mut fields = SCAN_ROW_ARROW_SCHEMA.fields().to_vec();
88
89    let stats_schema: ArrowSchema = stats_schema.as_ref().try_into_arrow()?;
90    fields.push(Arc::new(Field::new(
91        "stats_parsed",
92        ArrowDataType::Struct(stats_schema.fields().to_owned()),
93        true,
94    )));
95
96    if let Some(partition_schema) = partition_schema {
97        let partition_schema: ArrowSchema = partition_schema.as_ref().try_into_arrow()?;
98        fields.push(Arc::new(Field::new(
99            "partitionValues_parsed",
100            ArrowDataType::Struct(partition_schema.fields().to_owned()),
101            false,
102        )));
103    }
104
105    Ok(Arc::new(ArrowSchema::new(fields)))
106}
107
108fn partitions_schema(
109    schema: &StructType,
110    partition_columns: &[String],
111) -> DeltaResultLocal<Option<StructType>> {
112    if partition_columns.is_empty() {
113        return Ok(None);
114    }
115    Ok(Some(StructType::try_new(
116        partition_columns
117            .iter()
118            .map(|col| {
119                schema.field(col).cloned().ok_or_else(|| {
120                    DeltaTableError::Generic(format!("Partition column {col} not found in schema"))
121                })
122            })
123            .collect::<Result<Vec<_>, _>>()?,
124    )?))
125}
126
127/// Returns the schema and table properties used to parse `Add.stats`.
128///
129/// Partition columns are excluded from the returned schema; see [`stats_source_schema`].
130pub(crate) fn stats_inputs(
131    table_configuration: &TableConfiguration,
132) -> DeltaResult<(StructType, Cow<'_, TableProperties>)> {
133    let source_schema = stats_source_schema(
134        table_configuration.logical_schema().as_ref(),
135        table_configuration.metadata().partition_columns(),
136        table_configuration.column_mapping_mode(),
137    )?;
138    let table_properties = stats_table_properties(
139        table_configuration.logical_schema().as_ref(),
140        table_configuration.table_properties(),
141        table_configuration.column_mapping_mode(),
142    );
143    Ok((source_schema, table_properties))
144}
145
146/// Returns the schema of columns that may legally appear inside `Add.stats`.
147///
148/// Partition columns are excluded intentionally: their stats belong in
149/// `Add.partitionValues`, not `Add.stats`, even when table features materialize them in
150/// parquet files. When column mapping is disabled, fields retain their logical names.
151/// When column mapping is enabled (`Name` or `Id`), fields use their physical names.
152fn stats_source_schema(
153    logical_schema: &StructType,
154    partition_columns: &[String],
155    column_mapping_mode: ColumnMappingMode,
156) -> DeltaResult<StructType> {
157    let fields = logical_schema
158        .fields()
159        .filter(|field| !partition_columns.contains(field.name()))
160        .map(|field| field.make_physical(column_mapping_mode))
161        .collect::<DeltaResult<Vec<_>>>()?;
162    StructType::try_new(fields)
163}
164
165/// Translates `dataSkippingStatsColumns` from logical to physical names.
166///
167/// Columns that cannot be resolved against `logical_schema` are dropped.
168pub(crate) fn stats_table_properties<'a>(
169    logical_schema: &StructType,
170    table_properties: &'a TableProperties,
171    column_mapping_mode: ColumnMappingMode,
172) -> Cow<'a, TableProperties> {
173    if column_mapping_mode == ColumnMappingMode::None
174        || table_properties.data_skipping_stats_columns.is_none()
175    {
176        return Cow::Borrowed(table_properties);
177    }
178
179    let mut table_properties = table_properties.clone();
180    table_properties.data_skipping_stats_columns =
181        table_properties.data_skipping_stats_columns.map(|columns| {
182            columns
183                .iter()
184                .filter_map(|column| {
185                    physical_column_name(logical_schema, column, column_mapping_mode)
186                })
187                .collect()
188        });
189    Cow::Owned(table_properties)
190}
191
192/// Returns a physical column path, or `None` if the logical path cannot be resolved.
193fn physical_column_name(
194    logical_schema: &StructType,
195    column: &ColumnName,
196    column_mapping_mode: ColumnMappingMode,
197) -> Option<ColumnName> {
198    let mut path = column.path().iter().peekable();
199    let mut physical_path = Vec::with_capacity(column.path().len());
200    let mut current_struct = logical_schema;
201
202    while let Some(field_name) = path.next() {
203        let field = current_struct.field(field_name)?;
204        physical_path.push(field.physical_name(column_mapping_mode).to_string());
205
206        if path.peek().is_some() {
207            let DataType::Struct(inner) = field.data_type() else {
208                return None;
209            };
210            current_struct = inner;
211        }
212    }
213
214    Some(ColumnName::new(physical_path))
215}
216
217/// Generates the expected schema for parsing `Add.stats`.
218///
219/// The input schema must already represent only columns that may legally appear
220/// in `Add.stats`: use physical names when column mapping is enabled, and
221/// exclude partition columns even if data files materialize them.
222///
223/// The base stats schema follows the current table configuration.
224/// With `dataSkippingStatsColumns`, include only those columns. Column names may
225/// refer to struct fields, which includes all child fields.
226/// Without that setting, include the first `dataSkippingNumIndexedCols` leaf
227/// fields, defaulting to 32. All fields are made nullable.
228///
229/// For the `nullCount` schema, we consider the whole base schema and convert all leaf fields
230/// to data type LONG. Maps, arrays, and variant are considered leaf fields in this case.
231///
232/// For the min / max schemas, remove ineligible leaf fields from the base schema.
233/// The field data type determines eligibility via
234/// [`is_public_min_max_stats_eligible_primitive`].
235///
236/// The overall schema is then:
237/// ```ignored
238/// {
239///    numRecords: long,
240///    nullCount: <derived null count schema>,
241///    minValues: <derived min/max schema>,
242///    maxValues: <derived min/max schema>,
243/// }
244/// ```
245pub(crate) fn stats_schema(
246    stats_source_schema: &Schema,
247    table_properties: &TableProperties,
248) -> Schema {
249    let mut fields = Vec::with_capacity(4);
250    fields.push(StructField::nullable("numRecords", DataType::LONG));
251
252    // Build the base stats schema by making fields nullable and applying table properties.
253    let mut base_transform = BaseStatsTransform::new(table_properties);
254    if let Some(base_schema) = base_transform.transform_struct(stats_source_schema) {
255        let base_schema = base_schema.into_owned();
256
257        // convert all leaf fields to data type LONG for null count
258        let mut null_count_transform = NullCountStatsTransform;
259        if let Some(null_count_schema) = null_count_transform.transform_struct(&base_schema) {
260            fields.push(StructField::nullable(
261                "nullCount",
262                null_count_schema.into_owned(),
263            ));
264        };
265
266        // include only min/max skipping eligible fields (data types)
267        let mut min_max_transform = MinMaxStatsTransform;
268        if let Some(min_max_schema) = min_max_transform.transform_struct(&base_schema) {
269            let min_max_schema = min_max_schema.into_owned();
270            fields.push(StructField::nullable("minValues", min_max_schema.clone()));
271            fields.push(StructField::nullable("maxValues", min_max_schema));
272        }
273    }
274
275    StructType::try_new(fields).expect("Failed to construct StructType for stats_schema")
276}
277
278// Convert a min/max stats schema into a nullcount schema (all leaf fields are LONG)
279pub(crate) struct NullCountStatsTransform;
280impl<'a> SchemaTransform<'a> for NullCountStatsTransform {
281    transform_output_type!(|'a, T| Option<Cow<'a, T>>);
282
283    fn transform_primitive(&mut self, _ptype: &'a PrimitiveType) -> Option<Cow<'a, PrimitiveType>> {
284        Some(Cow::Owned(PrimitiveType::Long))
285    }
286    fn transform_struct_field(&mut self, field: &'a StructField) -> Option<Cow<'a, StructField>> {
287        use Cow::*;
288
289        if matches!(
290            &field.data_type,
291            DataType::Array(_) | DataType::Map(_) | DataType::Variant(_)
292        ) {
293            return Some(Cow::Owned(StructField {
294                name: field.name.clone(),
295                data_type: DataType::LONG,
296                nullable: true,
297                metadata: Default::default(),
298            }));
299        }
300
301        match self.transform(&field.data_type)? {
302            Borrowed(_) => Some(Borrowed(field)),
303            dt => Some(Owned(StructField {
304                name: field.name.clone(),
305                data_type: dt.into_owned(),
306                nullable: true,
307                metadata: Default::default(),
308            })),
309        }
310    }
311}
312
313/// Transforms a table schema into a base stats schema.
314///
315/// Base stats schema in this case refers the subsets of fields in the table schema
316/// that may be considered for stats collection. Depending on the type of stats - min/max/nullcount/... -
317/// additional transformations may be applied.
318///
319/// The concrete shape of the schema depends on the table configuration.
320/// * `dataSkippingStatsColumns` - used to explicitly specify the columns
321///   to be used for data skipping statistics. (takes precedence)
322/// * `dataSkippingNumIndexedCols` - used to specify the number of columns
323///   to be used for data skipping statistics. Defaults to 32.
324///
325/// All fields are nullable.
326struct BaseStatsTransform {
327    n_columns: Option<DataSkippingNumIndexedCols>,
328    added_columns: u64,
329    column_names: Option<Vec<ColumnName>>,
330    path: Vec<String>,
331}
332
333impl BaseStatsTransform {
334    fn new(props: &TableProperties) -> Self {
335        // if data_skipping_stats_columns is specified, it takes precedence
336        // over data_skipping_num_indexed_cols, even if that is also specified
337        if let Some(columns_names) = &props.data_skipping_stats_columns {
338            Self {
339                n_columns: None,
340                added_columns: 0,
341                column_names: Some(columns_names.clone()),
342                path: Vec::new(),
343            }
344        } else {
345            Self {
346                n_columns: Some(
347                    props
348                        .data_skipping_num_indexed_cols
349                        .unwrap_or(DataSkippingNumIndexedCols::NumColumns(32)),
350                ),
351                added_columns: 0,
352                column_names: None,
353                path: Vec::new(),
354            }
355        }
356    }
357}
358
359impl<'a> SchemaTransform<'a> for BaseStatsTransform {
360    transform_output_type!(|'a, T| Option<Cow<'a, T>>);
361
362    // Arrays, maps, and variants are opaque leaves for data skipping purposes:
363    // we keep the field so it can carry a null count, but their inner schema
364    // does not get traversed and its leaves do not count against
365    // `dataSkippingNumIndexedCols`. This matches Delta/Spark, which excludes
366    // nested fields inside arrays and maps from stats collection.
367    // See delta-io/delta-rs#3172.
368    fn transform_array(&mut self, atype: &'a ArrayType) -> Option<Cow<'a, ArrayType>> {
369        Some(Cow::Borrowed(atype))
370    }
371    fn transform_map(&mut self, mtype: &'a MapType) -> Option<Cow<'a, MapType>> {
372        Some(Cow::Borrowed(mtype))
373    }
374    fn transform_variant(&mut self, stype: &'a StructType) -> Option<Cow<'a, StructType>> {
375        Some(Cow::Borrowed(stype))
376    }
377
378    fn transform_struct_field(&mut self, field: &'a StructField) -> Option<Cow<'a, StructField>> {
379        use Cow::*;
380
381        // Check if the number of columns is set and if the added columns exceed the limit.
382        // In the constructor we assert this will always be None if column_names are specified.
383        if let Some(DataSkippingNumIndexedCols::NumColumns(n_cols)) = self.n_columns
384            && self.added_columns >= n_cols
385        {
386            return None;
387        }
388
389        self.path.push(field.name.clone());
390        let data_type = field.data_type();
391
392        // keep the field if it:
393        // - is a struct field and we need to traverse its children
394        // - OR it is referenced by the column names
395        // - OR it is a primitive type / leaf field
396        let should_include = matches!(data_type, DataType::Struct(_))
397            || self
398                .column_names
399                .as_ref()
400                .map(|ns| should_include_column(&ColumnName::new(&self.path), ns))
401                .unwrap_or(true);
402
403        if !should_include {
404            self.path.pop();
405            return None;
406        }
407
408        // increment count only for leaf columns.
409        if !matches!(data_type, DataType::Struct(_)) {
410            self.added_columns += 1;
411        }
412
413        let transformed = self.transform(&field.data_type);
414        // Pop the path before potentially propagating a `None` result.
415        self.path.pop();
416
417        let field = match transformed? {
418            Borrowed(_) if field.is_nullable() => Borrowed(field),
419            data_type => Owned(StructField {
420                name: field.name.clone(),
421                data_type: data_type.into_owned(),
422                nullable: true,
423                metadata: Default::default(),
424            }),
425        };
426
427        // exclude struct fields with no children
428        if matches!(field.data_type(), DataType::Struct(dt) if dt.num_fields() == 0) {
429            None
430        } else {
431            Some(field)
432        }
433    }
434}
435
436// Remove fields with data types that cannot provide min/max stats.
437//
438// Apply this only to schemas processed by `BaseStatsTransform`.
439struct MinMaxStatsTransform;
440
441impl<'a> SchemaTransform<'a> for MinMaxStatsTransform {
442    transform_output_type!(|'a, T| Option<Cow<'a, T>>);
443
444    // array and map fields are not eligible for data skipping, so filter them out.
445    fn transform_array(&mut self, _: &'a ArrayType) -> Option<Cow<'a, ArrayType>> {
446        None
447    }
448    fn transform_map(&mut self, _: &'a MapType) -> Option<Cow<'a, MapType>> {
449        None
450    }
451    fn transform_variant(&mut self, _: &'a StructType) -> Option<Cow<'a, StructType>> {
452        None
453    }
454
455    fn transform_primitive(&mut self, ptype: &'a PrimitiveType) -> Option<Cow<'a, PrimitiveType>> {
456        if is_public_min_max_stats_eligible_primitive(ptype) {
457            Some(Cow::Borrowed(ptype))
458        } else {
459            None
460        }
461    }
462}
463
464// Checks if a column should be included or traversed into.
465//
466// Returns true if the column name is included in the list of column names
467// or if the column name is a prefix of any column name in the list
468// or if the column name is a child of any column name in the list
469fn should_include_column(column_name: &ColumnName, column_names: &[ColumnName]) -> bool {
470    column_names.iter().any(|name| {
471        name.as_ref().starts_with(column_name) || column_name.as_ref().starts_with(name)
472    })
473}
474
475/// Returns true when a primitive belongs in the public min/max stats schema.
476///
477/// The public `Add.stats` min/max schema includes booleans. [`StatsProjection`]
478/// applies a stricter scan projection before asking Delta Kernel to parse stats.
479/// https://github.com/delta-io/delta/blob/143ab3337121248d2ca6a7d5bc31deae7c8fe4be/kernel/kernel-api/src/main/java/io/delta/kernel/internal/skipping/StatsSchemaHelper.java#L61
480///
481/// [`StatsProjection`]: crate::kernel::snapshot::StatsProjection
482pub(crate) fn is_public_min_max_stats_eligible_primitive(data_type: &PrimitiveType) -> bool {
483    #[cfg(not(feature = "nanosecond-timestamps"))]
484    let matches_nanos = false;
485    #[cfg(feature = "nanosecond-timestamps")]
486    let matches_nanos = matches!(
487        data_type,
488        &PrimitiveType::TimestampNanos | &PrimitiveType::TimestampNanosNtz
489    );
490    matches!(
491        data_type,
492        &PrimitiveType::Byte
493            | &PrimitiveType::Short
494            | &PrimitiveType::Integer
495            | &PrimitiveType::Long
496            | &PrimitiveType::Float
497            | &PrimitiveType::Double
498            | &PrimitiveType::Boolean
499            | &PrimitiveType::Date
500            | &PrimitiveType::Timestamp
501            | &PrimitiveType::TimestampNtz
502            | &PrimitiveType::String
503            | PrimitiveType::Decimal(_)
504    ) || matches_nanos
505}
506
507pub(crate) fn rb_from_scan_meta(metadata: ScanMetadata) -> DeltaResult<RecordBatch> {
508    let (underlying_data, selection_vector) = metadata.scan_files.into_parts();
509    let batch = ArrowEngineData::try_from_engine_data(underlying_data)?.into();
510    Ok(filter_record_batch(
511        &batch,
512        &BooleanArray::from(selection_vector),
513    )?)
514}
515
516/// Internal extension trait for expression evaluators.
517///
518/// Provides conversion between Arrow [`RecordBatch`]es and
519/// Kernel's [`ArrowEngineData`].
520pub(crate) trait ExpressionEvaluatorExt {
521    fn evaluate_arrow(&self, batch: RecordBatch) -> DeltaResult<RecordBatch>;
522}
523
524impl<T: ExpressionEvaluator + ?Sized> ExpressionEvaluatorExt for T {
525    fn evaluate_arrow(&self, batch: RecordBatch) -> DeltaResult<RecordBatch> {
526        let engine_data = ArrowEngineData::new(batch);
527        Ok(ArrowEngineData::try_from_engine_data(T::evaluate(self, &engine_data)?)?.into())
528    }
529}
530
531/// Extension trait for Kernel's [`StructData`].
532///
533/// StructData is the data structure contained in a Struct scalar.
534/// The exposed API on kernels struct data is very minimal and does not allow
535/// for conveniently probing the fields / values contained within [`StructData`].
536///
537/// This trait therefore adds convenience methods for accessing fields and values.
538pub trait StructDataExt {
539    /// Returns a reference to the field with the given name, if it exists.
540    fn field(&self, name: &str) -> Option<&StructField>;
541
542    /// Returns a reference to the value with the given index, if it exists.
543    fn value(&self, index: usize) -> Option<&Scalar>;
544
545    /// Returns the index of the field with the given name, if it exists.
546    fn index_of(&self, name: &str) -> Option<usize>;
547}
548
549impl StructDataExt for StructData {
550    fn field(&self, name: &str) -> Option<&StructField> {
551        self.fields().iter().find(|f| f.name() == name)
552    }
553
554    fn index_of(&self, name: &str) -> Option<usize> {
555        self.fields().iter().position(|f| f.name() == name)
556    }
557
558    fn value(&self, index: usize) -> Option<&Scalar> {
559        self.values().get(index)
560    }
561}
562
563#[cfg(test)]
564mod tests {
565    use std::collections::HashMap;
566    use std::sync::Arc;
567
568    use super::*;
569
570    use crate::test_utils::{
571        build_test_table_configuration, column_mapping_test_field,
572        column_mapping_test_field_with_type,
573    };
574
575    use delta_kernel::EvaluationHandler;
576    use delta_kernel::arrow::array::Int32Array;
577    use delta_kernel::arrow::datatypes::{DataType as ArrowDataType, Field, Schema};
578    use delta_kernel::arrow::record_batch::RecordBatch;
579    use delta_kernel::engine::arrow_conversion::TryIntoKernel;
580    use delta_kernel::engine::arrow_expression::ArrowEvaluationHandler;
581    use delta_kernel::expressions::*;
582    use delta_kernel::schema::{ArrayType, DataType};
583    use pretty_assertions::assert_eq;
584
585    #[test]
586    fn test_evaluate_arrow() {
587        let handler = ArrowEvaluationHandler;
588
589        let schema = Schema::new(vec![Field::new("a", ArrowDataType::Int32, false)]);
590        let values = Int32Array::from(vec![1, 2, 3]);
591        let batch = RecordBatch::try_new(Arc::new(schema.clone()), vec![Arc::new(values)]).unwrap();
592
593        let expression = column_expr!("a");
594        let expr = handler
595            .new_expression_evaluator(
596                Arc::new((&schema).try_into_kernel().unwrap()),
597                expression.into(),
598                DataType::INTEGER,
599            )
600            .unwrap();
601
602        let result = expr.evaluate_arrow(batch);
603        assert!(result.is_ok());
604    }
605
606    #[test]
607    fn test_should_include_column() {
608        let full_name = vec![ColumnName::new(["lvl1", "lvl2", "lvl3", "lvl4"])];
609        let parent = ColumnName::new(["lvl1", "lvl2", "lvl3"]);
610        assert!(should_include_column(&parent, &full_name));
611        assert!(should_include_column(&full_name[0], &full_name));
612        // child fields should also be included
613        assert!(should_include_column(&full_name[0], &[parent]));
614
615        let not_parent = ColumnName::new(["lvl1", "lvl2", "lvl3", "lvl5"]);
616        assert!(!should_include_column(&not_parent, &full_name));
617        let not_parent = ColumnName::new(["lvl1", "lvl3", "lvl4"]);
618        assert!(!should_include_column(&not_parent, &full_name));
619
620        let not_parent = ColumnName::new(["lvl1", "lvl2", "lvl4"]);
621        assert!(!should_include_column(&not_parent, &full_name));
622    }
623
624    #[test]
625    fn test_stats_schema_simple() {
626        let properties: TableProperties = [("key", "value")].into();
627        let file_schema =
628            StructType::try_new([StructField::nullable("id", DataType::LONG)]).unwrap();
629
630        let stats_schema = stats_schema(&file_schema, &properties);
631        let expected = StructType::try_new([
632            StructField::nullable("numRecords", DataType::LONG),
633            StructField::nullable("nullCount", file_schema.clone()),
634            StructField::nullable("minValues", file_schema.clone()),
635            StructField::nullable("maxValues", file_schema),
636        ])
637        .unwrap();
638
639        assert_eq!(&expected, &stats_schema);
640    }
641
642    #[test]
643    fn test_stats_schema_with_non_eligible_field() {
644        let properties: TableProperties = [("key", "value")].into();
645
646        // Create a nested logical schema with:
647        // - top-level field "id" (LONG) - eligible for data skipping
648        // - nested struct "metadata" containing:
649        //   - "name" (STRING) - eligible for data skipping
650        //   - "tags" (ARRAY) - NOT eligible for data skipping
651        //   - "score" (DOUBLE) - eligible for data skipping
652
653        // Create array type for a field that's not eligible for data skipping
654        let array_type = DataType::Array(Box::new(ArrayType::new(DataType::STRING, false)));
655        let metadata_struct = StructType::try_new([
656            StructField::nullable("name", DataType::STRING),
657            StructField::nullable("tags", array_type),
658            StructField::nullable("score", DataType::DOUBLE),
659        ])
660        .unwrap();
661        let file_schema = StructType::try_new([
662            StructField::nullable("id", DataType::LONG),
663            StructField::nullable(
664                "metadata",
665                DataType::Struct(Box::new(metadata_struct.clone())),
666            ),
667        ])
668        .unwrap();
669
670        let stats_schema = stats_schema(&file_schema, &properties);
671
672        let expected_null_nested = StructType::try_new([
673            StructField::nullable("name", DataType::LONG),
674            StructField::nullable("tags", DataType::LONG),
675            StructField::nullable("score", DataType::LONG),
676        ])
677        .unwrap();
678        let expected_null = StructType::try_new([
679            StructField::nullable("id", DataType::LONG),
680            StructField::nullable("metadata", DataType::Struct(Box::new(expected_null_nested))),
681        ])
682        .unwrap();
683
684        let expected_nested = StructType::try_new([
685            StructField::nullable("name", DataType::STRING),
686            StructField::nullable("score", DataType::DOUBLE),
687        ])
688        .unwrap();
689        let expected_fields = StructType::try_new([
690            StructField::nullable("id", DataType::LONG),
691            StructField::nullable("metadata", DataType::Struct(Box::new(expected_nested))),
692        ])
693        .unwrap();
694
695        let expected = StructType::try_new([
696            StructField::nullable("numRecords", DataType::LONG),
697            StructField::nullable("nullCount", expected_null),
698            StructField::nullable("minValues", expected_fields.clone()),
699            StructField::nullable("maxValues", expected_fields.clone()),
700        ])
701        .unwrap();
702
703        assert_eq!(&expected, &stats_schema);
704    }
705
706    #[test]
707    fn test_stats_schema_col_names() {
708        let properties: TableProperties = [(
709            "delta.dataSkippingStatsColumns".to_string(),
710            "`user.info`.name".to_string(),
711        )]
712        .into();
713
714        let user_struct = StructType::try_new([
715            StructField::nullable("name", DataType::STRING),
716            StructField::nullable("age", DataType::INTEGER),
717        ])
718        .unwrap();
719        let file_schema = StructType::try_new([
720            StructField::nullable("id", DataType::LONG),
721            StructField::nullable("user.info", DataType::Struct(Box::new(user_struct.clone()))),
722        ])
723        .unwrap();
724
725        let stats_schema = stats_schema(&file_schema, &properties);
726
727        let expected_nested =
728            StructType::try_new([StructField::nullable("name", DataType::STRING)]).unwrap();
729        let expected_fields = StructType::try_new([StructField::nullable(
730            "user.info",
731            DataType::Struct(Box::new(expected_nested)),
732        )])
733        .unwrap();
734        let null_count = NullCountStatsTransform
735            .transform_struct(&expected_fields)
736            .unwrap()
737            .into_owned();
738
739        let expected = StructType::try_new([
740            StructField::nullable("numRecords", DataType::LONG),
741            StructField::nullable("nullCount", null_count),
742            StructField::nullable("minValues", expected_fields.clone()),
743            StructField::nullable("maxValues", expected_fields.clone()),
744        ])
745        .unwrap();
746
747        assert_eq!(&expected, &stats_schema);
748    }
749
750    /// Fields listed in `dataSkippingStatsColumns` that follow a struct with no listed
751    /// leaves must still be included in the stats schema.
752    #[test]
753    fn test_stats_schema_col_names_after_excluded_struct() {
754        let properties: TableProperties = [(
755            "delta.dataSkippingStatsColumns".to_string(),
756            "a,b".to_string(),
757        )]
758        .into();
759
760        let st = StructType::try_new([StructField::nullable("x", DataType::LONG)]).unwrap();
761        let file_schema = StructType::try_new([
762            StructField::nullable("a", DataType::LONG),
763            StructField::nullable("st", DataType::Struct(Box::new(st))),
764            StructField::nullable("b", DataType::LONG),
765        ])
766        .unwrap();
767
768        let stats_schema = stats_schema(&file_schema, &properties);
769
770        let expected_fields = StructType::try_new([
771            StructField::nullable("a", DataType::LONG),
772            StructField::nullable("b", DataType::LONG),
773        ])
774        .unwrap();
775        let expected = StructType::try_new([
776            StructField::nullable("numRecords", DataType::LONG),
777            StructField::nullable("nullCount", expected_fields.clone()),
778            StructField::nullable("minValues", expected_fields.clone()),
779            StructField::nullable("maxValues", expected_fields),
780        ])
781        .unwrap();
782
783        assert_eq!(&expected, &stats_schema);
784    }
785
786    #[test]
787    fn test_stats_source_schema_excludes_partition_columns() {
788        let logical_schema = StructType::try_new([
789            StructField::nullable("p", DataType::INTEGER),
790            StructField::nullable("a", DataType::INTEGER),
791        ])
792        .unwrap();
793
794        let stats_schema =
795            stats_source_schema(&logical_schema, &["p".to_string()], ColumnMappingMode::None)
796                .unwrap();
797
798        assert!(stats_schema.field("a").is_some());
799        assert!(stats_schema.field("p").is_none());
800    }
801
802    #[test]
803    fn test_stats_source_schema_applies_column_mapping_physical_names() {
804        let logical_schema = StructType::try_new([
805            column_mapping_test_field("p", "col_p", 1),
806            column_mapping_test_field("a", "col_a", 2),
807        ])
808        .unwrap();
809
810        for column_mapping_mode in [ColumnMappingMode::Name, ColumnMappingMode::Id] {
811            let stats_schema =
812                stats_source_schema(&logical_schema, &["p".to_string()], column_mapping_mode)
813                    .unwrap();
814
815            assert!(stats_schema.field("col_a").is_some());
816            assert!(stats_schema.field("a").is_none());
817            assert!(stats_schema.field("col_p").is_none());
818            assert!(stats_schema.field("p").is_none());
819        }
820    }
821
822    #[test]
823    fn test_table_configuration_stats_schema_translates_stats_columns_to_physical_names() {
824        // Use table property strings to exercise TableConfiguration parsing.
825        for column_mapping_mode in ["name", "id"] {
826            let logical_schema = StructType::try_new([
827                column_mapping_test_field("p", "col_p", 1),
828                column_mapping_test_field("a", "col_a", 2),
829                column_mapping_test_field("b", "col_b", 3),
830            ])
831            .unwrap();
832            let table_configuration = build_test_table_configuration(
833                logical_schema,
834                vec!["p".to_string()],
835                HashMap::from([
836                    (
837                        "delta.columnMapping.mode".to_string(),
838                        column_mapping_mode.to_string(),
839                    ),
840                    (
841                        "delta.dataSkippingStatsColumns".to_string(),
842                        "a".to_string(),
843                    ),
844                ]),
845            );
846
847            let stats_schema = table_configuration.stats_schema().unwrap();
848
849            let min_values = match stats_schema.field("minValues").unwrap().data_type() {
850                DataType::Struct(fields) => fields,
851                other => panic!("expected minValues struct, got {other:?}"),
852            };
853            assert!(min_values.field("col_a").is_some());
854            assert!(min_values.field("a").is_none());
855            assert!(min_values.field("col_p").is_none());
856            assert!(min_values.field("col_b").is_none());
857        }
858    }
859
860    #[test]
861    fn test_table_configuration_stats_schema_translates_nested_stats_columns_to_physical_names() {
862        // Use table property strings to exercise TableConfiguration parsing.
863        for column_mapping_mode in ["name", "id"] {
864            let nested_schema = StructType::try_new([
865                column_mapping_test_field("a", "col_a", 3),
866                column_mapping_test_field("b", "col_b", 4),
867            ])
868            .unwrap();
869            let logical_schema = StructType::try_new([
870                column_mapping_test_field("p", "col_p", 1),
871                column_mapping_test_field_with_type(
872                    "s",
873                    "col_s",
874                    2,
875                    DataType::Struct(Box::new(nested_schema)),
876                ),
877            ])
878            .unwrap();
879            let table_configuration = build_test_table_configuration(
880                logical_schema,
881                vec!["p".to_string()],
882                HashMap::from([
883                    (
884                        "delta.columnMapping.mode".to_string(),
885                        column_mapping_mode.to_string(),
886                    ),
887                    (
888                        "delta.dataSkippingStatsColumns".to_string(),
889                        "s.a".to_string(),
890                    ),
891                ]),
892            );
893
894            let stats_schema = table_configuration.stats_schema().unwrap();
895
896            let min_values = match stats_schema.field("minValues").unwrap().data_type() {
897                DataType::Struct(fields) => fields,
898                other => panic!("expected minValues struct, got {other:?}"),
899            };
900            let nested_min_values = match min_values.field("col_s").unwrap().data_type() {
901                DataType::Struct(fields) => fields,
902                other => panic!("expected nested minValues struct, got {other:?}"),
903            };
904            assert!(nested_min_values.field("col_a").is_some());
905            assert!(nested_min_values.field("a").is_none());
906            assert!(nested_min_values.field("col_b").is_none());
907            assert!(min_values.field("s").is_none());
908            assert!(min_values.field("col_p").is_none());
909        }
910    }
911
912    #[test]
913    fn test_table_configuration_stats_schema_drops_stats_columns_with_non_struct_intermediate() {
914        // Use table property strings to exercise TableConfiguration parsing.
915        for column_mapping_mode in ["name", "id"] {
916            let logical_schema = StructType::try_new([
917                column_mapping_test_field("p", "col_p", 1),
918                column_mapping_test_field("a", "col_a", 2),
919            ])
920            .unwrap();
921            let table_configuration = build_test_table_configuration(
922                logical_schema,
923                vec!["p".to_string()],
924                HashMap::from([
925                    (
926                        "delta.columnMapping.mode".to_string(),
927                        column_mapping_mode.to_string(),
928                    ),
929                    (
930                        "delta.dataSkippingStatsColumns".to_string(),
931                        "a.b".to_string(),
932                    ),
933                ]),
934            );
935
936            let stats_schema = table_configuration.stats_schema().unwrap();
937
938            assert!(stats_schema.field("nullCount").is_none());
939            assert!(stats_schema.field("minValues").is_none());
940            assert!(stats_schema.field("maxValues").is_none());
941        }
942    }
943
944    #[test]
945    fn test_table_configuration_stats_schema_num_indexed_cols_ignores_partition_columns() {
946        let logical_schema = StructType::try_new([
947            StructField::nullable("p", DataType::INTEGER),
948            StructField::nullable("a", DataType::INTEGER),
949        ])
950        .unwrap();
951        let table_configuration = build_test_table_configuration(
952            logical_schema,
953            vec!["p".to_string()],
954            HashMap::from([(
955                "delta.dataSkippingNumIndexedCols".to_string(),
956                "1".to_string(),
957            )]),
958        );
959
960        let stats_schema = table_configuration.stats_schema().unwrap();
961
962        let min_values = match stats_schema.field("minValues").unwrap().data_type() {
963            DataType::Struct(fields) => fields,
964            other => panic!("expected minValues struct, got {other:?}"),
965        };
966        assert!(min_values.field("a").is_some());
967        assert!(min_values.field("p").is_none());
968        assert_eq!(1, min_values.fields().count());
969    }
970
971    #[test]
972    fn test_stats_schema_n_cols() {
973        let properties: TableProperties = [(
974            "delta.dataSkippingNumIndexedCols".to_string(),
975            "1".to_string(),
976        )]
977        .into();
978
979        let logical_schema = StructType::try_new([
980            StructField::nullable("name", DataType::STRING),
981            StructField::nullable("age", DataType::INTEGER),
982        ])
983        .unwrap();
984
985        let stats_schema = stats_schema(&logical_schema, &properties);
986
987        let expected_fields =
988            StructType::try_new([StructField::nullable("name", DataType::STRING)]).unwrap();
989        let null_count = NullCountStatsTransform
990            .transform_struct(&expected_fields)
991            .unwrap()
992            .into_owned();
993
994        let expected = StructType::try_new([
995            StructField::nullable("numRecords", DataType::LONG),
996            StructField::nullable("nullCount", null_count),
997            StructField::nullable("minValues", expected_fields.clone()),
998            StructField::nullable("maxValues", expected_fields.clone()),
999        ])
1000        .unwrap();
1001
1002        assert_eq!(&expected, &stats_schema);
1003    }
1004
1005    /// Validate that the stats schema shows up with the fields that are needed
1006    ///
1007    /// The Delta Lake protocol does not specify that min/max fields should be dropped for any data
1008    /// types but instead that timestamps and strings may be truncated
1009    ///
1010    /// [related issue](https://github.com/delta-io/delta-rs/issues/4224)
1011    #[test]
1012    fn test_stats_schema_different_fields_in_null_vs_minmax() {
1013        let properties: TableProperties = [("key", "value")].into();
1014
1015        // Create a schema with fields that have different eligibility for min/max vs null count
1016        // - "id" (LONG) - eligible for both null count and min/max
1017        // - "is_active" (BOOLEAN) - eligible for null count and for min/max
1018        // - "metadata" (BINARY) - eligible for null count but NOT for min/max
1019        let file_schema = StructType::try_new([
1020            StructField::nullable("id", DataType::LONG),
1021            StructField::nullable("is_active", DataType::BOOLEAN),
1022            StructField::nullable("metadata", DataType::BINARY),
1023        ])
1024        .unwrap();
1025
1026        let stats_schema = stats_schema(&file_schema, &properties);
1027
1028        // Expected nullCount schema: all fields converted to LONG
1029        let expected_null_count = StructType::try_new([
1030            StructField::nullable("id", DataType::LONG),
1031            StructField::nullable("is_active", DataType::LONG),
1032            StructField::nullable("metadata", DataType::LONG),
1033        ])
1034        .unwrap();
1035
1036        // Expected minValues/maxValues schema: only eligible fields (no binary)
1037        let expected_min_max = StructType::try_new([
1038            StructField::nullable("id", DataType::LONG),
1039            StructField::nullable("is_active", DataType::BOOLEAN),
1040        ])
1041        .unwrap();
1042
1043        let expected = StructType::try_new([
1044            StructField::nullable("numRecords", DataType::LONG),
1045            StructField::nullable("nullCount", expected_null_count),
1046            StructField::nullable("minValues", expected_min_max.clone()),
1047            StructField::nullable("maxValues", expected_min_max),
1048        ])
1049        .unwrap();
1050
1051        assert_eq!(&expected, &stats_schema);
1052    }
1053
1054    #[test]
1055    fn test_stats_schema_nested_different_fields_in_null_vs_minmax() {
1056        let properties: TableProperties = [("key", "value")].into();
1057
1058        // Create a nested schema where some nested fields are eligible for min/max and others aren't
1059        let user_struct = StructType::try_new([
1060            StructField::nullable("name", DataType::STRING), // eligible for min/max
1061            StructField::nullable("is_admin", DataType::BOOLEAN), // eligible for min/max
1062            StructField::nullable("age", DataType::INTEGER), // eligible for min/max
1063            StructField::nullable("profile_pic", DataType::BINARY), // NOT eligible for min/max
1064        ])
1065        .unwrap();
1066
1067        let file_schema = StructType::try_new([
1068            StructField::nullable("id", DataType::LONG),
1069            StructField::nullable("user", DataType::Struct(Box::new(user_struct.clone()))),
1070            StructField::nullable("is_deleted", DataType::BOOLEAN), // eligible for min/max
1071        ])
1072        .unwrap();
1073
1074        let stats_schema = stats_schema(&file_schema, &properties);
1075
1076        // Expected nullCount schema: all fields converted to LONG, maintaining structure
1077        let expected_null_user = StructType::try_new([
1078            StructField::nullable("name", DataType::LONG),
1079            StructField::nullable("is_admin", DataType::LONG),
1080            StructField::nullable("age", DataType::LONG),
1081            StructField::nullable("profile_pic", DataType::LONG),
1082        ])
1083        .unwrap();
1084        let expected_null_count = StructType::try_new([
1085            StructField::nullable("id", DataType::LONG),
1086            StructField::nullable("user", DataType::Struct(Box::new(expected_null_user))),
1087            StructField::nullable("is_deleted", DataType::LONG),
1088        ])
1089        .unwrap();
1090
1091        // Expected minValues/maxValues schema: only eligible fields
1092        let expected_minmax_user = StructType::try_new([
1093            StructField::nullable("name", DataType::STRING),
1094            StructField::nullable("is_admin", DataType::BOOLEAN),
1095            StructField::nullable("age", DataType::INTEGER),
1096        ])
1097        .unwrap();
1098        let expected_min_max = StructType::try_new([
1099            StructField::nullable("id", DataType::LONG),
1100            StructField::nullable("user", DataType::Struct(Box::new(expected_minmax_user))),
1101            StructField::nullable("is_deleted", DataType::BOOLEAN),
1102        ])
1103        .unwrap();
1104
1105        let expected = StructType::try_new([
1106            StructField::nullable("numRecords", DataType::LONG),
1107            StructField::nullable("nullCount", expected_null_count),
1108            StructField::nullable("minValues", expected_min_max.clone()),
1109            StructField::nullable("maxValues", expected_min_max),
1110        ])
1111        .unwrap();
1112
1113        assert_eq!(&expected, &stats_schema);
1114    }
1115
1116    #[test]
1117    fn test_stats_schema_only_non_eligible_fields() {
1118        let properties: TableProperties = [("key", "value")].into();
1119
1120        // Create a schema with only fields that are NOT eligible for min/max skipping
1121        let file_schema = StructType::try_new([
1122            StructField::nullable("metadata", DataType::BINARY),
1123            StructField::nullable(
1124                "tags",
1125                DataType::Array(Box::new(ArrayType::new(DataType::STRING, false))),
1126            ),
1127        ])
1128        .unwrap();
1129
1130        let stats_schema = stats_schema(&file_schema, &properties);
1131
1132        // Expected nullCount schema: all fields converted to LONG
1133        let expected_null_count = StructType::try_new([
1134            StructField::nullable("metadata", DataType::LONG),
1135            StructField::nullable("tags", DataType::LONG),
1136        ])
1137        .unwrap();
1138
1139        // Expected minValues/maxValues schema: empty since no fields are eligible
1140        // Since there are no eligible fields, minValues and maxValues should not be present
1141        let expected = StructType::try_new([
1142            StructField::nullable("numRecords", DataType::LONG),
1143            StructField::nullable("nullCount", expected_null_count),
1144            // No minValues or maxValues fields since no primitive fields are eligible
1145        ])
1146        .unwrap();
1147
1148        assert_eq!(&expected, &stats_schema);
1149    }
1150
1151    /// A struct whose leaves are all ineligible for min/max stats is dropped from
1152    /// `minValues`/`maxValues`. Keeping it as an empty struct produced zero-length struct
1153    /// arrays in `stats_parsed`. See delta-io/delta-rs#3237.
1154    #[test]
1155    fn test_stats_schema_drops_nested_struct_without_eligible_fields() {
1156        let properties: TableProperties = [("key", "value")].into();
1157        // `payload` holds a single binary leaf, so it has no min/max eligible leaf at all.
1158        let payload =
1159            StructType::try_new([StructField::nullable("data", DataType::BINARY)]).unwrap();
1160        let file_schema = StructType::try_new([
1161            StructField::nullable("id", DataType::LONG),
1162            StructField::nullable("payload", DataType::Struct(Box::new(payload))),
1163        ])
1164        .unwrap();
1165
1166        // `nullCount` counts every leaf, so it still carries `payload`.
1167        let null_count = StructType::try_new([
1168            StructField::nullable("id", DataType::LONG),
1169            StructField::nullable(
1170                "payload",
1171                DataType::Struct(Box::new(
1172                    StructType::try_new([StructField::nullable("data", DataType::LONG)]).unwrap(),
1173                )),
1174            ),
1175        ])
1176        .unwrap();
1177        // `payload` is absent here: it is dropped, not kept as an empty struct.
1178        let min_max_without_payload =
1179            StructType::try_new([StructField::nullable("id", DataType::LONG)]).unwrap();
1180
1181        let expected = StructType::try_new([
1182            StructField::nullable("numRecords", DataType::LONG),
1183            StructField::nullable("nullCount", null_count),
1184            StructField::nullable("minValues", min_max_without_payload.clone()),
1185            StructField::nullable("maxValues", min_max_without_payload),
1186        ])
1187        .unwrap();
1188
1189        assert_eq!(&expected, &stats_schema(&file_schema, &properties));
1190    }
1191
1192    #[test]
1193    fn test_partitions_schema() -> DeltaResultLocal<()> {
1194        let logical_schema = StructType::try_new([
1195            StructField::nullable("name", DataType::STRING),
1196            StructField::nullable("age", DataType::INTEGER),
1197        ])
1198        .unwrap();
1199
1200        let result = partitions_schema(&logical_schema, &[])?;
1201        assert_eq!(None, result);
1202        Ok(())
1203    }
1204
1205    #[tokio::test]
1206    async fn test_partition_schema2() {
1207        let schema = StructType::try_new(vec![
1208            StructField::new("id", DataType::LONG, true),
1209            StructField::new("name", DataType::STRING, true),
1210            StructField::new("date", DataType::DATE, true),
1211        ])
1212        .unwrap();
1213
1214        let partition_columns = vec!["date".to_string()];
1215        let expected =
1216            StructType::try_new(vec![StructField::new("date", DataType::DATE, true)]).unwrap();
1217        assert_eq!(
1218            partitions_schema(&schema, &partition_columns).unwrap(),
1219            Some(expected)
1220        );
1221
1222        assert_eq!(partitions_schema(&schema, &[]).unwrap(), None);
1223    }
1224}