Skip to main content

deltalake_core/kernel/snapshot/
log_data.rs

1use std::sync::Arc;
2
3use arrow_array::RecordBatch;
4use delta_kernel::actions::{Metadata, Protocol};
5use delta_kernel::expressions::{Scalar, StructData};
6use delta_kernel::table_configuration::TableConfiguration;
7use delta_kernel::table_properties::TableProperties;
8use indexmap::IndexMap;
9
10use super::super::scalars::ScalarExt;
11use super::iterators::LogicalFileView;
12
13pub(crate) trait PartitionsExt {
14    fn hive_partition_path(&self) -> String;
15}
16
17impl PartitionsExt for IndexMap<String, Scalar> {
18    fn hive_partition_path(&self) -> String {
19        let sep = String::from('/');
20        itertools::Itertools::intersperse(
21            self.iter().map(|(k, v)| {
22                let encoded = v.serialize_encoded();
23                format!("{k}={encoded}")
24            }),
25            sep,
26        )
27        .collect()
28    }
29}
30
31impl PartitionsExt for StructData {
32    fn hive_partition_path(&self) -> String {
33        let sep = String::from('/');
34        itertools::Itertools::intersperse(
35            self.fields()
36                .iter()
37                .zip(self.values().iter())
38                .map(|(k, v)| {
39                    let encoded = v.serialize_encoded();
40                    format!("{}={encoded}", k.name())
41                }),
42            sep,
43        )
44        .collect()
45    }
46}
47
48impl<T: PartitionsExt> PartitionsExt for Arc<T> {
49    fn hive_partition_path(&self) -> String {
50        self.as_ref().hive_partition_path()
51    }
52}
53
54/// Provides semanitc access to the log data.
55///
56/// This is a helper struct that provides access to the log data in a more semantic way
57/// to avid the necessiity of knowing the exact layout of the underlying log data.
58#[derive(Clone)]
59pub struct LogDataHandler<'a> {
60    data: &'a [RecordBatch],
61    config: &'a TableConfiguration,
62}
63
64impl<'a> LogDataHandler<'a> {
65    pub(crate) fn new(data: &'a [RecordBatch], config: &'a TableConfiguration) -> Self {
66        Self { data, config }
67    }
68
69    pub(crate) fn table_configuration(&self) -> &TableConfiguration {
70        self.config
71    }
72
73    pub(crate) fn table_properties(&self) -> &TableProperties {
74        self.config.table_properties()
75    }
76
77    pub(crate) fn protocol(&self) -> &Protocol {
78        self.config.protocol()
79    }
80
81    pub(crate) fn metadata(&self) -> &Metadata {
82        self.config.metadata()
83    }
84
85    /// The number of files in the log data.
86    /// Returns the total number of files represented across all underlying record batches.
87    pub fn num_files(&self) -> usize {
88        self.data.iter().map(|batch| batch.num_rows()).sum()
89    }
90
91    /// Iterate over each file as a [`LogicalFileView`] without materializing all of them at once.
92    pub fn iter(&self) -> impl Iterator<Item = LogicalFileView> + '_ {
93        self.data.iter().flat_map(|batch| {
94            (0..batch.num_rows()).map(move |idx| LogicalFileView::new(batch.clone(), idx))
95        })
96    }
97}
98
99impl IntoIterator for LogDataHandler<'_> {
100    type Item = LogicalFileView;
101    type IntoIter = Box<dyn Iterator<Item = Self::Item>>;
102
103    fn into_iter(self) -> Self::IntoIter {
104        let owned = self.data.to_vec();
105        Box::new(owned.into_iter().flat_map(|batch| {
106            (0..batch.num_rows()).map(move |idx| LogicalFileView::new(batch.clone(), idx))
107        }))
108    }
109}
110
111#[cfg(feature = "datafusion")]
112mod datafusion {
113    use std::collections::HashSet;
114    use std::sync::{Arc, LazyLock};
115
116    use ::datafusion::common::Column;
117    use ::datafusion::common::scalar::ScalarValue;
118    use ::datafusion::physical_optimizer::pruning::PruningStatistics;
119    use arrow::compute::concat;
120    use arrow_array::{Array, StringArray};
121    use arrow_array::{ArrayRef, BooleanArray, UInt64Array};
122    use arrow_schema::DataType as ArrowDataType;
123    use delta_kernel::expressions::Expression;
124    use delta_kernel::schema::{DataType, PrimitiveType};
125    use delta_kernel::{EvaluationHandler, ExpressionEvaluator};
126    use itertools::Itertools;
127
128    use super::*;
129    use crate::kernel::arrow::engine_ext::ExpressionEvaluatorExt as _;
130    use crate::{DeltaResult, DeltaTableError};
131
132    use crate::kernel::ARROW_HANDLER;
133    use crate::kernel::snapshot::ceil_timestamps;
134
135    impl LogDataHandler<'_> {
136        fn pick_stats(&self, column: &Column, stats_field: &'static str) -> Option<ArrayRef> {
137            let schema = self.config.logical_schema();
138            let field = schema.field(&column.name)?;
139            // See issue #1214. Binary type does not support natural order which is required for Datafusion to prune
140            if field.data_type() == &DataType::Primitive(PrimitiveType::Binary) {
141                return None;
142            }
143            let expression = if self
144                .config
145                .metadata()
146                .partition_columns()
147                .contains(&column.name)
148            {
149                Expression::column(["partitionValues_parsed", &column.name])
150            } else {
151                Expression::column(["stats_parsed", stats_field, &column.name])
152            };
153            let evaluator = ARROW_HANDLER
154                .new_expression_evaluator(
155                    crate::kernel::models::fields::log_schema_ref().clone(),
156                    expression.into(),
157                    field.data_type().clone(),
158                )
159                .ok()?;
160
161            let results: Vec<_> = self
162                .data
163                .iter()
164                .map(|data| -> DeltaResult<_> {
165                    Ok(evaluator.evaluate_arrow(data.clone())?.column(0).clone())
166                })
167                .try_collect()
168                .ok()?;
169            concat(
170                &results
171                    .iter()
172                    .map(|result| result.as_ref())
173                    .collect::<Vec<_>>(),
174            )
175            .ok()
176        }
177
178        /// Evaluate the raw partition values for a `Binary`-typed partition column.
179        ///
180        /// Unlike [`pick_stats`], this does NOT bail out for binary types: binary values do
181        /// not support natural ordering (so `min_values`/`max_values` must stay `None`) but
182        /// they *do* support exact-match comparison via [`PruningStatistics::contained`].
183        fn pick_binary_partition_values(&self, column: &Column) -> Option<ArrayRef> {
184            let schema = self.config.logical_schema();
185            let field = schema.field(&column.name)?;
186            if field.data_type() != &DataType::Primitive(PrimitiveType::Binary) {
187                return None;
188            }
189            if !self
190                .config
191                .metadata()
192                .partition_columns()
193                .contains(&column.name)
194            {
195                return None;
196            }
197            let expression = Expression::column(["partitionValues_parsed", &column.name]);
198            let evaluator = ARROW_HANDLER
199                .new_expression_evaluator(
200                    crate::kernel::models::fields::log_schema_ref().clone(),
201                    expression.into(),
202                    field.data_type().clone(),
203                )
204                .ok()?;
205            let results: Vec<_> = self
206                .data
207                .iter()
208                .map(|data| -> DeltaResult<_> {
209                    Ok(evaluator.evaluate_arrow(data.clone())?.column(0).clone())
210                })
211                .try_collect()
212                .ok()?;
213            concat(
214                &results
215                    .iter()
216                    .map(|result| result.as_ref())
217                    .collect::<Vec<_>>(),
218            )
219            .ok()
220        }
221
222        /// Evaluate whether each file's binary partition value is a member of `values`.
223        ///
224        /// Returns `false` for a file when its partition value is definitely NOT in `values`
225        /// (safe to prune).  Returns `true` when it matches any value in the set, or when
226        /// the partition value is null (conservative: keep the file).
227        fn contained_binary(
228            &self,
229            column: &Column,
230            values: &HashSet<ScalarValue>,
231        ) -> Option<BooleanArray> {
232            use arrow_array::BinaryArray;
233
234            let partition_values = self.pick_binary_partition_values(column)?;
235            let binary_arr = partition_values.as_any().downcast_ref::<BinaryArray>()?;
236
237            // Binary partition values are serialized into the Delta log as unicode-escape
238            // hex strings (e.g. b"aaa" → "\\u0061\\u0061\\u0061") by
239            // `create_escaped_binary_string`, then parsed back into `Scalar::Binary` by
240            // `PrimitiveType::parse_scalar` via `raw.to_string().into_bytes()`.
241            // This means the stored bytes are the UTF-8 encoding of that escape string,
242            // NOT the original raw bytes.  We must apply the same encoding to the predicate
243            // scalars before comparing so that equality holds.
244            fn encode_binary_bytes(bytes: &[u8]) -> Vec<u8> {
245                bytes
246                    .iter()
247                    .flat_map(|&b| format!("\\u{b:04X}").into_bytes())
248                    .collect()
249            }
250
251            // Unwrap the dictionary wrapper that `_arrow_schema` adds to binary
252            // partition columns.
253            fn unwrap_binary_scalar(scalar: &ScalarValue) -> Option<Vec<u8>> {
254                match scalar {
255                    ScalarValue::Binary(Some(v)) | ScalarValue::LargeBinary(Some(v)) => {
256                        Some(encode_binary_bytes(v))
257                    }
258                    ScalarValue::Dictionary(_, inner) => unwrap_binary_scalar(inner),
259                    _ => None,
260                }
261            }
262
263            let mut contains = Vec::with_capacity(binary_arr.len());
264            for i in 0..binary_arr.len() {
265                if binary_arr.is_null(i) {
266                    // Null partition value: conservatively keep the file.
267                    contains.push(true);
268                } else {
269                    let pv: &[u8] = binary_arr.value(i);
270                    contains.push(
271                        values
272                            .iter()
273                            .filter_map(unwrap_binary_scalar)
274                            .any(|encoded| encoded.as_slice() == pv),
275                    );
276                }
277            }
278            Some(BooleanArray::from(contains))
279        }
280    }
281
282    impl PruningStatistics for LogDataHandler<'_> {
283        /// return the minimum values for the named column, if known.
284        /// Note: the returned array must contain `num_containers()` rows
285        fn min_values(&self, column: &Column) -> Option<ArrayRef> {
286            self.pick_stats(column, "minValues")
287        }
288
289        /// return the maximum values for the named column, if known.
290        /// Note: the returned array must contain `num_containers()` rows.
291        fn max_values(&self, column: &Column) -> Option<ArrayRef> {
292            let max = self.pick_stats(column, "maxValues")?;
293            if self
294                .config
295                .metadata()
296                .partition_columns()
297                .contains(&column.name)
298            {
299                return Some(max);
300            }
301            Some(ceil_timestamps(max))
302        }
303
304        /// return the number of containers (e.g. row groups) being
305        /// pruned with these statistics
306        fn num_containers(&self) -> usize {
307            self.data.iter().map(|b| b.num_rows()).sum()
308        }
309
310        /// return the number of null values for the named column as an
311        /// `Option<UInt64Array>`.
312        ///
313        /// Note: the returned array must contain `num_containers()` rows.
314        fn null_counts(&self, column: &Column) -> Option<ArrayRef> {
315            if !self
316                .config
317                .metadata()
318                .partition_columns()
319                .contains(&column.name)
320            {
321                let counts = self.pick_stats(column, "nullCount")?;
322                return arrow_cast::cast(counts.as_ref(), &ArrowDataType::UInt64).ok();
323            }
324            let partition_values = self.pick_stats(column, "__dummy__")?;
325            let row_counts = self.row_counts()?;
326            let row_counts = row_counts.as_any().downcast_ref::<UInt64Array>()?;
327            let mut null_counts = Vec::with_capacity(partition_values.len());
328            for i in 0..partition_values.len() {
329                let null_count = if partition_values.is_null(i) {
330                    row_counts.value(i)
331                } else {
332                    0
333                };
334                null_counts.push(null_count);
335            }
336            Some(Arc::new(UInt64Array::from(null_counts)))
337        }
338
339        /// return the number of rows in each container as an `Option<UInt64Array>`.
340        ///
341        /// Note: the returned array must contain `num_containers()` rows
342        fn row_counts(&self) -> Option<ArrayRef> {
343            static ROW_COUNTS_EVAL: LazyLock<Arc<dyn ExpressionEvaluator>> = LazyLock::new(|| {
344                ARROW_HANDLER
345                    .new_expression_evaluator(
346                        crate::kernel::models::fields::log_schema_ref().clone(),
347                        Expression::column(["add", "stats_parsed", "numRecords"]).into(),
348                        DataType::Primitive(PrimitiveType::Long),
349                    )
350                    .expect("Failed to create row counts evaluator")
351            });
352
353            let results: Vec<_> = self
354                .data
355                .iter()
356                .map(|data| -> DeltaResult<_> {
357                    let batch = ROW_COUNTS_EVAL.evaluate_arrow(data.clone())?;
358                    Ok(arrow_cast::cast(batch.column(0), &ArrowDataType::UInt64)?)
359                })
360                .try_collect()
361                .ok()?;
362
363            concat(
364                &results
365                    .iter()
366                    .map(|result| result.as_ref())
367                    .collect::<Vec<_>>(),
368            )
369            .ok()
370        }
371
372        // This function is optional but will optimize partition column pruning
373        fn contained(&self, column: &Column, value: &HashSet<ScalarValue>) -> Option<BooleanArray> {
374            if value.is_empty()
375                || !self
376                    .config
377                    .metadata()
378                    .partition_columns()
379                    .contains(&column.name)
380            {
381                return None;
382            }
383
384            // Binary partition columns support exact-match pruning but not range pruning
385            // (no natural ordering).  Delegate to a dedicated path that bypasses the
386            // ordering guard in `pick_stats`.  See issue #1214.
387            let schema = self.config.logical_schema();
388            if schema
389                .field(&column.name)
390                .map(|f| f.data_type() == &DataType::Primitive(PrimitiveType::Binary))
391                .unwrap_or(false)
392            {
393                return self.contained_binary(column, value);
394            }
395
396            // Retrieve the partition values for the column
397            let partition_values = self.pick_stats(column, "__dummy__")?;
398
399            let partition_values = partition_values
400                .as_any()
401                .downcast_ref::<StringArray>()
402                .ok_or(DeltaTableError::generic(
403                    "failed to downcast string result to StringArray.",
404                ))
405                .ok()?;
406
407            let mut contains = Vec::with_capacity(partition_values.len());
408
409            // TODO: this was inspired by parquet's BloomFilter pruning, decide if we should
410            //  just convert to Vec<String> for a subset of column types and use .contains
411            fn check_scalar(pv: &str, value: &ScalarValue) -> bool {
412                match value {
413                    ScalarValue::Utf8(Some(v))
414                    | ScalarValue::Utf8View(Some(v))
415                    | ScalarValue::LargeUtf8(Some(v)) => pv == v,
416
417                    ScalarValue::Dictionary(_, inner) => check_scalar(pv, inner),
418                    // FIXME: is this a good enough default or should we sync this with
419                    //  expr_applicable_for_cols and bail out with None
420                    _ => value.to_string() == pv,
421                }
422            }
423
424            for i in 0..partition_values.len() {
425                if partition_values.is_null(i) {
426                    contains.push(false);
427                } else {
428                    contains.push(
429                        value
430                            .iter()
431                            .any(|scalar| check_scalar(partition_values.value(i), scalar)),
432                    );
433                }
434            }
435
436            Some(BooleanArray::from(contains))
437        }
438    }
439}
440
441#[cfg(test)]
442mod tests {
443    use super::*;
444    use delta_kernel::schema::DataType;
445    use delta_kernel::schema::PrimitiveType;
446    use delta_kernel::schema::StructField;
447    use pretty_assertions::assert_eq;
448
449    #[test]
450    fn test_partitionsext_structdata() {
451        let partitions = StructData::try_new(
452            vec![
453                StructField::new("year", DataType::LONG, false),
454                StructField::new("month", DataType::LONG, false),
455                StructField::new("day", DataType::LONG, false),
456            ],
457            vec![Scalar::Long(2025), Scalar::Long(1), Scalar::Long(1)],
458        )
459        .expect("Failed to make StructData");
460        assert_eq!("year=2025/month=1/day=1", partitions.hive_partition_path());
461
462        let partitions = StructData::try_new(
463            vec![StructField::new("year", DataType::LONG, true)],
464            vec![Scalar::Null(DataType::Primitive(PrimitiveType::Long))],
465        )
466        .expect("Failed to make StructData");
467        assert_eq!(
468            "year=__HIVE_DEFAULT_PARTITION__",
469            partitions.hive_partition_path()
470        );
471    }
472
473    #[test]
474    fn test_partitionsext_indexmap() {
475        let partitions: IndexMap<String, Scalar> = IndexMap::from([
476            ("year".to_string(), Scalar::Long(2025)),
477            ("month".to_string(), Scalar::Long(1)),
478            ("day".to_string(), Scalar::Long(1)),
479        ]);
480        assert_eq!("year=2025/month=1/day=1", partitions.hive_partition_path());
481
482        let partitions: IndexMap<String, Scalar> = IndexMap::from([(
483            "year".to_string(),
484            Scalar::Null(DataType::Primitive(PrimitiveType::String)),
485        )]);
486        assert_eq!(
487            "year=__HIVE_DEFAULT_PARTITION__",
488            partitions.hive_partition_path()
489        );
490    }
491}
492
493#[cfg(all(test, feature = "datafusion"))]
494mod df_tests {
495    use futures::TryStreamExt;
496
497    #[tokio::test]
498    async fn read_delta_1_2_1_struct_stats_table() {
499        let table_path = std::path::Path::new("../test/tests/data/delta-1.2.1-only-struct-stats");
500        let table_uri =
501            url::Url::from_directory_path(std::fs::canonicalize(table_path).unwrap()).unwrap();
502        let table_from_struct_stats = crate::open_table(table_uri.clone()).await.unwrap();
503        let table_from_json_stats = crate::open_table_with_version(table_uri, 1).await.unwrap();
504        let log_store = table_from_struct_stats.log_store();
505
506        let json_adds: Vec<_> = table_from_json_stats
507            .snapshot()
508            .unwrap()
509            .snapshot()
510            .file_views(&log_store, None)
511            .try_collect()
512            .await
513            .unwrap();
514        let json_action = json_adds
515            .iter()
516            .find(|f| {
517                f.path().ends_with(
518                    "part-00000-7a509247-4f58-4453-9202-51d75dee59af-c000.snappy.parquet",
519                )
520            })
521            .unwrap();
522
523        let struct_adds: Vec<_> = table_from_struct_stats
524            .snapshot()
525            .unwrap()
526            .snapshot()
527            .file_views(&log_store, None)
528            .try_collect()
529            .await
530            .unwrap();
531        let struct_action = struct_adds
532            .iter()
533            .find(|f| {
534                f.path().ends_with(
535                    "part-00000-7a509247-4f58-4453-9202-51d75dee59af-c000.snappy.parquet",
536                )
537            })
538            .unwrap();
539
540        assert_eq!(json_action.path(), struct_action.path());
541        assert_eq!(
542            json_action.partition_values(),
543            struct_action.partition_values()
544        );
545        // assert_eq!(
546        //     json_action.max_values().unwrap(),
547        //     struct_action.max_values().unwrap()
548        // );
549        // assert_eq!(
550        //     json_action.min_values().unwrap(),
551        //     struct_action.min_values().unwrap()
552        // );
553    }
554}