deltalake_core/kernel/snapshot/
log_data.rs1use std::sync::Arc;
2
3use arrow_array::RecordBatch;
4use delta_kernel::actions::{Metadata, Protocol};
5use delta_kernel::expressions::{Scalar, StructData};
6use delta_kernel::table_configuration::TableConfiguration;
7use delta_kernel::table_properties::TableProperties;
8use indexmap::IndexMap;
9
10use super::super::scalars::ScalarExt;
11use super::iterators::LogicalFileView;
12
13pub(crate) trait PartitionsExt {
14 fn hive_partition_path(&self) -> String;
15}
16
17impl PartitionsExt for IndexMap<String, Scalar> {
18 fn hive_partition_path(&self) -> String {
19 let sep = String::from('/');
20 itertools::Itertools::intersperse(
21 self.iter().map(|(k, v)| {
22 let encoded = v.serialize_encoded();
23 format!("{k}={encoded}")
24 }),
25 sep,
26 )
27 .collect()
28 }
29}
30
31impl PartitionsExt for StructData {
32 fn hive_partition_path(&self) -> String {
33 let sep = String::from('/');
34 itertools::Itertools::intersperse(
35 self.fields()
36 .iter()
37 .zip(self.values().iter())
38 .map(|(k, v)| {
39 let encoded = v.serialize_encoded();
40 format!("{}={encoded}", k.name())
41 }),
42 sep,
43 )
44 .collect()
45 }
46}
47
48impl<T: PartitionsExt> PartitionsExt for Arc<T> {
49 fn hive_partition_path(&self) -> String {
50 self.as_ref().hive_partition_path()
51 }
52}
53
54#[derive(Clone)]
59pub struct LogDataHandler<'a> {
60 data: &'a [RecordBatch],
61 config: &'a TableConfiguration,
62}
63
64impl<'a> LogDataHandler<'a> {
65 pub(crate) fn new(data: &'a [RecordBatch], config: &'a TableConfiguration) -> Self {
66 Self { data, config }
67 }
68
69 pub(crate) fn table_configuration(&self) -> &TableConfiguration {
70 self.config
71 }
72
73 pub(crate) fn table_properties(&self) -> &TableProperties {
74 self.config.table_properties()
75 }
76
77 pub(crate) fn protocol(&self) -> &Protocol {
78 self.config.protocol()
79 }
80
81 pub(crate) fn metadata(&self) -> &Metadata {
82 self.config.metadata()
83 }
84
85 pub fn num_files(&self) -> usize {
88 self.data.iter().map(|batch| batch.num_rows()).sum()
89 }
90
91 pub fn iter(&self) -> impl Iterator<Item = LogicalFileView> + '_ {
93 self.data.iter().flat_map(|batch| {
94 (0..batch.num_rows()).map(move |idx| LogicalFileView::new(batch.clone(), idx))
95 })
96 }
97}
98
99impl IntoIterator for LogDataHandler<'_> {
100 type Item = LogicalFileView;
101 type IntoIter = Box<dyn Iterator<Item = Self::Item>>;
102
103 fn into_iter(self) -> Self::IntoIter {
104 let owned = self.data.to_vec();
105 Box::new(owned.into_iter().flat_map(|batch| {
106 (0..batch.num_rows()).map(move |idx| LogicalFileView::new(batch.clone(), idx))
107 }))
108 }
109}
110
111#[cfg(feature = "datafusion")]
112mod datafusion {
113 use std::collections::HashSet;
114 use std::sync::{Arc, LazyLock};
115
116 use ::datafusion::common::Column;
117 use ::datafusion::common::scalar::ScalarValue;
118 use ::datafusion::physical_optimizer::pruning::PruningStatistics;
119 use arrow::compute::concat;
120 use arrow_array::{Array, StringArray};
121 use arrow_array::{ArrayRef, BooleanArray, UInt64Array};
122 use arrow_schema::DataType as ArrowDataType;
123 use delta_kernel::expressions::Expression;
124 use delta_kernel::schema::{DataType, PrimitiveType};
125 use delta_kernel::{EvaluationHandler, ExpressionEvaluator};
126 use itertools::Itertools;
127
128 use super::*;
129 use crate::kernel::arrow::engine_ext::ExpressionEvaluatorExt as _;
130 use crate::{DeltaResult, DeltaTableError};
131
132 use crate::kernel::ARROW_HANDLER;
133 use crate::kernel::snapshot::ceil_timestamps;
134
135 impl LogDataHandler<'_> {
136 fn pick_stats(&self, column: &Column, stats_field: &'static str) -> Option<ArrayRef> {
137 let schema = self.config.logical_schema();
138 let field = schema.field(&column.name)?;
139 if field.data_type() == &DataType::Primitive(PrimitiveType::Binary) {
141 return None;
142 }
143 let expression = if self
144 .config
145 .metadata()
146 .partition_columns()
147 .contains(&column.name)
148 {
149 Expression::column(["partitionValues_parsed", &column.name])
150 } else {
151 Expression::column(["stats_parsed", stats_field, &column.name])
152 };
153 let evaluator = ARROW_HANDLER
154 .new_expression_evaluator(
155 crate::kernel::models::fields::log_schema_ref().clone(),
156 expression.into(),
157 field.data_type().clone(),
158 )
159 .ok()?;
160
161 let results: Vec<_> = self
162 .data
163 .iter()
164 .map(|data| -> DeltaResult<_> {
165 Ok(evaluator.evaluate_arrow(data.clone())?.column(0).clone())
166 })
167 .try_collect()
168 .ok()?;
169 concat(
170 &results
171 .iter()
172 .map(|result| result.as_ref())
173 .collect::<Vec<_>>(),
174 )
175 .ok()
176 }
177
178 fn pick_binary_partition_values(&self, column: &Column) -> Option<ArrayRef> {
184 let schema = self.config.logical_schema();
185 let field = schema.field(&column.name)?;
186 if field.data_type() != &DataType::Primitive(PrimitiveType::Binary) {
187 return None;
188 }
189 if !self
190 .config
191 .metadata()
192 .partition_columns()
193 .contains(&column.name)
194 {
195 return None;
196 }
197 let expression = Expression::column(["partitionValues_parsed", &column.name]);
198 let evaluator = ARROW_HANDLER
199 .new_expression_evaluator(
200 crate::kernel::models::fields::log_schema_ref().clone(),
201 expression.into(),
202 field.data_type().clone(),
203 )
204 .ok()?;
205 let results: Vec<_> = self
206 .data
207 .iter()
208 .map(|data| -> DeltaResult<_> {
209 Ok(evaluator.evaluate_arrow(data.clone())?.column(0).clone())
210 })
211 .try_collect()
212 .ok()?;
213 concat(
214 &results
215 .iter()
216 .map(|result| result.as_ref())
217 .collect::<Vec<_>>(),
218 )
219 .ok()
220 }
221
222 fn contained_binary(
228 &self,
229 column: &Column,
230 values: &HashSet<ScalarValue>,
231 ) -> Option<BooleanArray> {
232 use arrow_array::BinaryArray;
233
234 let partition_values = self.pick_binary_partition_values(column)?;
235 let binary_arr = partition_values.as_any().downcast_ref::<BinaryArray>()?;
236
237 fn encode_binary_bytes(bytes: &[u8]) -> Vec<u8> {
245 bytes
246 .iter()
247 .flat_map(|&b| format!("\\u{b:04X}").into_bytes())
248 .collect()
249 }
250
251 fn unwrap_binary_scalar(scalar: &ScalarValue) -> Option<Vec<u8>> {
254 match scalar {
255 ScalarValue::Binary(Some(v)) | ScalarValue::LargeBinary(Some(v)) => {
256 Some(encode_binary_bytes(v))
257 }
258 ScalarValue::Dictionary(_, inner) => unwrap_binary_scalar(inner),
259 _ => None,
260 }
261 }
262
263 let mut contains = Vec::with_capacity(binary_arr.len());
264 for i in 0..binary_arr.len() {
265 if binary_arr.is_null(i) {
266 contains.push(true);
268 } else {
269 let pv: &[u8] = binary_arr.value(i);
270 contains.push(
271 values
272 .iter()
273 .filter_map(unwrap_binary_scalar)
274 .any(|encoded| encoded.as_slice() == pv),
275 );
276 }
277 }
278 Some(BooleanArray::from(contains))
279 }
280 }
281
282 impl PruningStatistics for LogDataHandler<'_> {
283 fn min_values(&self, column: &Column) -> Option<ArrayRef> {
286 self.pick_stats(column, "minValues")
287 }
288
289 fn max_values(&self, column: &Column) -> Option<ArrayRef> {
292 let max = self.pick_stats(column, "maxValues")?;
293 if self
294 .config
295 .metadata()
296 .partition_columns()
297 .contains(&column.name)
298 {
299 return Some(max);
300 }
301 Some(ceil_timestamps(max))
302 }
303
304 fn num_containers(&self) -> usize {
307 self.data.iter().map(|b| b.num_rows()).sum()
308 }
309
310 fn null_counts(&self, column: &Column) -> Option<ArrayRef> {
315 if !self
316 .config
317 .metadata()
318 .partition_columns()
319 .contains(&column.name)
320 {
321 let counts = self.pick_stats(column, "nullCount")?;
322 return arrow_cast::cast(counts.as_ref(), &ArrowDataType::UInt64).ok();
323 }
324 let partition_values = self.pick_stats(column, "__dummy__")?;
325 let row_counts = self.row_counts()?;
326 let row_counts = row_counts.as_any().downcast_ref::<UInt64Array>()?;
327 let mut null_counts = Vec::with_capacity(partition_values.len());
328 for i in 0..partition_values.len() {
329 let null_count = if partition_values.is_null(i) {
330 row_counts.value(i)
331 } else {
332 0
333 };
334 null_counts.push(null_count);
335 }
336 Some(Arc::new(UInt64Array::from(null_counts)))
337 }
338
339 fn row_counts(&self) -> Option<ArrayRef> {
343 static ROW_COUNTS_EVAL: LazyLock<Arc<dyn ExpressionEvaluator>> = LazyLock::new(|| {
344 ARROW_HANDLER
345 .new_expression_evaluator(
346 crate::kernel::models::fields::log_schema_ref().clone(),
347 Expression::column(["add", "stats_parsed", "numRecords"]).into(),
348 DataType::Primitive(PrimitiveType::Long),
349 )
350 .expect("Failed to create row counts evaluator")
351 });
352
353 let results: Vec<_> = self
354 .data
355 .iter()
356 .map(|data| -> DeltaResult<_> {
357 let batch = ROW_COUNTS_EVAL.evaluate_arrow(data.clone())?;
358 Ok(arrow_cast::cast(batch.column(0), &ArrowDataType::UInt64)?)
359 })
360 .try_collect()
361 .ok()?;
362
363 concat(
364 &results
365 .iter()
366 .map(|result| result.as_ref())
367 .collect::<Vec<_>>(),
368 )
369 .ok()
370 }
371
372 fn contained(&self, column: &Column, value: &HashSet<ScalarValue>) -> Option<BooleanArray> {
374 if value.is_empty()
375 || !self
376 .config
377 .metadata()
378 .partition_columns()
379 .contains(&column.name)
380 {
381 return None;
382 }
383
384 let schema = self.config.logical_schema();
388 if schema
389 .field(&column.name)
390 .map(|f| f.data_type() == &DataType::Primitive(PrimitiveType::Binary))
391 .unwrap_or(false)
392 {
393 return self.contained_binary(column, value);
394 }
395
396 let partition_values = self.pick_stats(column, "__dummy__")?;
398
399 let partition_values = partition_values
400 .as_any()
401 .downcast_ref::<StringArray>()
402 .ok_or(DeltaTableError::generic(
403 "failed to downcast string result to StringArray.",
404 ))
405 .ok()?;
406
407 let mut contains = Vec::with_capacity(partition_values.len());
408
409 fn check_scalar(pv: &str, value: &ScalarValue) -> bool {
412 match value {
413 ScalarValue::Utf8(Some(v))
414 | ScalarValue::Utf8View(Some(v))
415 | ScalarValue::LargeUtf8(Some(v)) => pv == v,
416
417 ScalarValue::Dictionary(_, inner) => check_scalar(pv, inner),
418 _ => value.to_string() == pv,
421 }
422 }
423
424 for i in 0..partition_values.len() {
425 if partition_values.is_null(i) {
426 contains.push(false);
427 } else {
428 contains.push(
429 value
430 .iter()
431 .any(|scalar| check_scalar(partition_values.value(i), scalar)),
432 );
433 }
434 }
435
436 Some(BooleanArray::from(contains))
437 }
438 }
439}
440
441#[cfg(test)]
442mod tests {
443 use super::*;
444 use delta_kernel::schema::DataType;
445 use delta_kernel::schema::PrimitiveType;
446 use delta_kernel::schema::StructField;
447 use pretty_assertions::assert_eq;
448
449 #[test]
450 fn test_partitionsext_structdata() {
451 let partitions = StructData::try_new(
452 vec![
453 StructField::new("year", DataType::LONG, false),
454 StructField::new("month", DataType::LONG, false),
455 StructField::new("day", DataType::LONG, false),
456 ],
457 vec![Scalar::Long(2025), Scalar::Long(1), Scalar::Long(1)],
458 )
459 .expect("Failed to make StructData");
460 assert_eq!("year=2025/month=1/day=1", partitions.hive_partition_path());
461
462 let partitions = StructData::try_new(
463 vec![StructField::new("year", DataType::LONG, true)],
464 vec![Scalar::Null(DataType::Primitive(PrimitiveType::Long))],
465 )
466 .expect("Failed to make StructData");
467 assert_eq!(
468 "year=__HIVE_DEFAULT_PARTITION__",
469 partitions.hive_partition_path()
470 );
471 }
472
473 #[test]
474 fn test_partitionsext_indexmap() {
475 let partitions: IndexMap<String, Scalar> = IndexMap::from([
476 ("year".to_string(), Scalar::Long(2025)),
477 ("month".to_string(), Scalar::Long(1)),
478 ("day".to_string(), Scalar::Long(1)),
479 ]);
480 assert_eq!("year=2025/month=1/day=1", partitions.hive_partition_path());
481
482 let partitions: IndexMap<String, Scalar> = IndexMap::from([(
483 "year".to_string(),
484 Scalar::Null(DataType::Primitive(PrimitiveType::String)),
485 )]);
486 assert_eq!(
487 "year=__HIVE_DEFAULT_PARTITION__",
488 partitions.hive_partition_path()
489 );
490 }
491}
492
493#[cfg(all(test, feature = "datafusion"))]
494mod df_tests {
495 use futures::TryStreamExt;
496
497 #[tokio::test]
498 async fn read_delta_1_2_1_struct_stats_table() {
499 let table_path = std::path::Path::new("../test/tests/data/delta-1.2.1-only-struct-stats");
500 let table_uri =
501 url::Url::from_directory_path(std::fs::canonicalize(table_path).unwrap()).unwrap();
502 let table_from_struct_stats = crate::open_table(table_uri.clone()).await.unwrap();
503 let table_from_json_stats = crate::open_table_with_version(table_uri, 1).await.unwrap();
504 let log_store = table_from_struct_stats.log_store();
505
506 let json_adds: Vec<_> = table_from_json_stats
507 .snapshot()
508 .unwrap()
509 .snapshot()
510 .file_views(&log_store, None)
511 .try_collect()
512 .await
513 .unwrap();
514 let json_action = json_adds
515 .iter()
516 .find(|f| {
517 f.path().ends_with(
518 "part-00000-7a509247-4f58-4453-9202-51d75dee59af-c000.snappy.parquet",
519 )
520 })
521 .unwrap();
522
523 let struct_adds: Vec<_> = table_from_struct_stats
524 .snapshot()
525 .unwrap()
526 .snapshot()
527 .file_views(&log_store, None)
528 .try_collect()
529 .await
530 .unwrap();
531 let struct_action = struct_adds
532 .iter()
533 .find(|f| {
534 f.path().ends_with(
535 "part-00000-7a509247-4f58-4453-9202-51d75dee59af-c000.snappy.parquet",
536 )
537 })
538 .unwrap();
539
540 assert_eq!(json_action.path(), struct_action.path());
541 assert_eq!(
542 json_action.partition_values(),
543 struct_action.partition_values()
544 );
545 }
554}