1use std::borrow::Cow;
2use std::collections::HashMap;
3use std::sync::{Arc, LazyLock};
4
5use arrow::datatypes::Int32Type;
6use arrow_array::cast::AsArray;
7use arrow_array::types::{Int64Type, TimestampMicrosecondType, TimestampNanosecondType};
8use arrow_array::{Array, ArrayRef, MapArray, RecordBatch, StringArray, StructArray};
9use arrow_schema::{DataType as ArrowDataType, TimeUnit};
10use chrono::{DateTime, Utc};
11use delta_kernel::engine::arrow_expression::evaluate_expression::to_json;
12use delta_kernel::expressions::{Scalar, StructData};
13use delta_kernel::scan::scan_row_schema;
14use delta_kernel::schema::DataType;
15#[cfg(any(test, feature = "datafusion"))]
16use indexmap::IndexMap;
17use object_store::ObjectMeta;
18use object_store::path::Path;
19use percent_encoding::percent_decode_str;
20
21#[cfg(feature = "datafusion")]
22pub(crate) use self::scan_row::parse_stats_column_with_schema;
23pub use self::tombstones::TombstoneView;
24#[cfg(any(test, feature = "datafusion"))]
25use crate::kernel::StructType;
26use crate::kernel::scalars::ScalarExt;
27use crate::kernel::{Add, DeletionVectorDescriptor, Remove};
28use crate::{DeltaResult, DeltaTableError};
29
30pub(crate) use self::scan_row::{ScanRowOutStream, scan_row_in_eval};
31
32mod scan_row;
33mod tombstones;
34
35const FIELD_NAME_PATH: &str = "path";
36const FIELD_NAME_SIZE: &str = "size";
37const FIELD_NAME_MODIFICATION_TIME: &str = "modificationTime";
38const FIELD_NAME_FILE_CONSTANT_VALUES: &str = "fileConstantValues";
39const FIELD_NAME_RAW_PARTITION_VALUES: &str = "partitionValues";
40const FIELD_NAME_TAGS: &str = "tags";
41const FIELD_NAME_BASE_ROW_ID: &str = "baseRowId";
42const FIELD_NAME_DEFAULT_ROW_COMMIT_VERSION: &str = "defaultRowCommitVersion";
43const FIELD_NAME_CLUSTERING_PROVIDER: &str = "clusteringProvider";
44const FIELD_NAME_STATS: &str = "stats";
45const FIELD_NAME_STATS_PARSED: &str = "stats_parsed";
46const FIELD_NAME_PARTITION_VALUES_PARSED: &str = "partitionValues_parsed";
47const FIELD_NAME_DELETION_VECTOR: &str = "deletionVector";
48
49const STATS_FIELD_NUM_RECORDS: &str = "numRecords";
50const STATS_FIELD_MIN_VALUES: &str = "minValues";
51const STATS_FIELD_MAX_VALUES: &str = "maxValues";
52const STATS_FIELD_NULL_COUNT: &str = "nullCount";
53
54const DV_FIELD_STORAGE_TYPE: &str = "storageType";
55const DV_FIELD_PATH_OR_INLINE_DV: &str = "pathOrInlineDv";
56const DV_FIELD_SIZE_IN_BYTES: &str = "sizeInBytes";
57const DV_FIELD_CARDINALITY: &str = "cardinality";
58const DV_FIELD_OFFSET: &str = "offset";
59
60static FIELD_INDICES: LazyLock<HashMap<&'static str, usize>> = LazyLock::new(|| {
61 let schema = scan_row_schema();
62 let mut indices = HashMap::new();
63
64 let path_idx = schema.index_of(FIELD_NAME_PATH).unwrap();
65 indices.insert(FIELD_NAME_PATH, path_idx);
66
67 let size_idx = schema.index_of(FIELD_NAME_SIZE).unwrap();
68 indices.insert(FIELD_NAME_SIZE, size_idx);
69
70 let modification_time_idx = schema.index_of(FIELD_NAME_MODIFICATION_TIME).unwrap();
71 indices.insert(FIELD_NAME_MODIFICATION_TIME, modification_time_idx);
72
73 indices
74});
75
76static DV_FIELD_INDICES: LazyLock<HashMap<&'static str, usize>> = LazyLock::new(|| {
77 let schema = scan_row_schema();
78 let dv_field = schema.field(FIELD_NAME_DELETION_VECTOR).unwrap();
79 let DataType::Struct(dv_type) = dv_field.data_type() else {
80 panic!("Expected DataType::Struct for deletion vector field");
81 };
82
83 let mut indices = HashMap::new();
84
85 let storage_type_idx = dv_type.index_of(DV_FIELD_STORAGE_TYPE).unwrap();
86 indices.insert(DV_FIELD_STORAGE_TYPE, storage_type_idx);
87
88 let path_or_inline_dv_idx = dv_type.index_of(DV_FIELD_PATH_OR_INLINE_DV).unwrap();
89 indices.insert(DV_FIELD_PATH_OR_INLINE_DV, path_or_inline_dv_idx);
90
91 let size_in_bytes_idx = dv_type.index_of(DV_FIELD_SIZE_IN_BYTES).unwrap();
92 indices.insert(DV_FIELD_SIZE_IN_BYTES, size_in_bytes_idx);
93
94 let cardinality_idx = dv_type.index_of(DV_FIELD_CARDINALITY).unwrap();
95 indices.insert(DV_FIELD_CARDINALITY, cardinality_idx);
96
97 indices
98});
99
100#[derive(Clone)]
107pub struct LogicalFileView {
108 files: RecordBatch,
109 index: usize,
110}
111
112impl LogicalFileView {
113 pub(crate) fn new(files: RecordBatch, index: usize) -> Self {
115 Self { files, index }
116 }
117
118 pub fn path(&self) -> Cow<'_, str> {
120 let raw = get_string_value(
121 self.files
122 .column(*FIELD_INDICES.get(FIELD_NAME_PATH).unwrap()),
123 self.index,
124 )
125 .unwrap();
126 percent_decode_str(raw).decode_utf8_lossy()
127 }
128
129 #[cfg(any(test, feature = "datafusion"))]
131 pub(crate) fn path_raw(&self) -> &str {
132 get_string_value(
133 self.files
134 .column(*FIELD_INDICES.get(FIELD_NAME_PATH).unwrap()),
135 self.index,
136 )
137 .unwrap()
138 }
139
140 pub fn object_store_path(&self) -> Path {
145 let path = self.path();
146 match Path::parse(path.as_ref()) {
148 Ok(path) => path,
149 Err(_) => Path::from(path.as_ref()),
150 }
151 }
152
153 pub fn size(&self) -> i64 {
155 self.files
156 .column(*FIELD_INDICES.get(FIELD_NAME_SIZE).unwrap())
157 .as_primitive::<Int64Type>()
158 .value(self.index)
159 }
160
161 pub fn modification_time(&self) -> i64 {
163 self.files
164 .column(*FIELD_INDICES.get(FIELD_NAME_MODIFICATION_TIME).unwrap())
165 .as_primitive::<Int64Type>()
166 .value(self.index)
167 }
168
169 pub fn modification_datetime(&self) -> DeltaResult<chrono::DateTime<Utc>> {
171 DateTime::from_timestamp_millis(self.modification_time()).ok_or(
172 DeltaTableError::MetadataError(format!(
173 "invalid modification_time: {:?}",
174 self.modification_time()
175 )),
176 )
177 }
178
179 pub fn stats(&self) -> Option<String> {
181 self.files
182 .column_by_name(FIELD_NAME_STATS)
183 .and_then(|col| get_string_value(col, self.index))
184 .map(ToString::to_string)
185 .or_else(|| {
186 let stats = self.stats_parsed()?.slice(self.index, 1);
187 let value = to_json(&stats)
188 .ok()
189 .map(|arr| arr.as_string::<i32>().value(0).to_string());
190 value.and_then(|v| (!v.is_empty()).then_some(v))
191 })
192 }
193
194 pub fn partition_values(&self) -> Option<StructData> {
196 self.files
197 .column_by_name(FIELD_NAME_PARTITION_VALUES_PARSED)
198 .and_then(|col| col.as_struct_opt())
199 .and_then(|arr| {
200 arr.is_valid(self.index)
201 .then(|| match Scalar::from_array(arr, self.index) {
202 Some(Scalar::Struct(s)) => Some(s),
203 _ => None,
204 })
205 .flatten()
206 })
207 }
208
209 fn file_constant_values(&self) -> Option<&StructArray> {
210 self.files
211 .column_by_name(FIELD_NAME_FILE_CONSTANT_VALUES)
212 .and_then(|col| col.as_struct_opt())
213 .filter(|file_constants| file_constants.is_valid(self.index))
214 }
215
216 fn file_constant_map(&self, field_name: &str) -> Option<HashMap<String, Option<String>>> {
217 let values = self
218 .file_constant_values()?
219 .column_by_name(field_name)?
220 .as_map_opt()?;
221 values
222 .is_valid(self.index)
223 .then(|| collect_string_map(&values.value(self.index)))
224 .flatten()
225 }
226
227 fn file_constant_i64(&self, field_name: &str) -> Option<i64> {
228 let values = self
229 .file_constant_values()?
230 .column_by_name(field_name)?
231 .as_primitive_opt::<Int64Type>()?;
232 values
233 .is_valid(self.index)
234 .then(|| values.value(self.index))
235 }
236
237 fn file_constant_string(&self, field_name: &str) -> Option<String> {
238 self.file_constant_values()?
239 .column_by_name(field_name)
240 .and_then(|values| get_string_value(values, self.index))
241 .map(ToString::to_string)
242 }
243
244 fn raw_partition_values(&self) -> Option<&MapArray> {
245 self.file_constant_values()?
246 .column_by_name(FIELD_NAME_RAW_PARTITION_VALUES)
247 .and_then(|col| col.as_map_opt())
248 }
249
250 pub fn partition_values_map(&self) -> HashMap<String, Option<String>> {
255 self.raw_partition_values()
256 .filter(|partitions| partitions.is_valid(self.index))
257 .and_then(|partitions| collect_string_map(&partitions.value(self.index)))
258 .unwrap_or_default()
259 }
260
261 #[cfg(feature = "datafusion")]
263 pub(crate) fn full_partition_values(
264 &self,
265 partition_columns: &[String],
266 table_schema: &StructType,
267 ) -> DeltaResult<IndexMap<String, Scalar>> {
268 typed_partition_values_from_raw_map(
269 &self.partition_values_map(),
270 partition_columns,
271 table_schema,
272 )
273 }
274
275 fn stats_parsed(&self) -> Option<&StructArray> {
277 self.files
278 .column_by_name(FIELD_NAME_STATS_PARSED)
279 .and_then(|col| col.as_struct_opt())
280 }
281
282 pub fn num_records(&self) -> Option<usize> {
286 self.stats_parsed()
287 .and_then(|stats| stats.column_by_name(STATS_FIELD_NUM_RECORDS))
288 .and_then(|col| col.as_primitive_opt::<Int64Type>())
289 .and_then(|a| {
290 a.is_valid(self.index)
291 .then(|| usize::try_from(a.value(self.index)).ok())
292 .flatten()
293 })
294 }
295
296 pub fn null_counts(&self) -> Option<Scalar> {
298 self.stats_parsed()
299 .and_then(|stats| stats.column_by_name(STATS_FIELD_NULL_COUNT))
300 .and_then(|c| Scalar::from_array(c.as_ref(), self.index))
301 }
302
303 pub fn min_values(&self) -> Option<Scalar> {
305 self.stats_parsed()
306 .and_then(|stats| stats.column_by_name(STATS_FIELD_MIN_VALUES))
307 .and_then(|c| Scalar::from_array(c.as_ref(), self.index))
308 }
309
310 pub fn max_values(&self) -> Option<Scalar> {
315 self.stats_parsed()
316 .and_then(|stats| stats.column_by_name(STATS_FIELD_MAX_VALUES))
317 .and_then(|c| Scalar::from_array(c.as_ref(), self.index))
318 .map(|s| round_ms_datetimes(s, &ceil_datetime))
319 }
320
321 pub fn deletion_vector_descriptor(&self) -> Option<DeletionVectorDescriptor> {
326 self.deletion_vector().map(|dv| dv.descriptor())
327 }
328
329 fn deletion_vector(&self) -> Option<DeletionVectorView<'_>> {
331 let dv_col = self
332 .files
333 .column_by_name(FIELD_NAME_DELETION_VECTOR)
334 .and_then(|col| col.as_struct_opt())?;
335 if dv_col.null_count() == dv_col.len() {
336 return None;
337 }
338 dv_col
339 .is_valid(self.index)
340 .then(|| {
341 let storage_col =
342 dv_col.column(*DV_FIELD_INDICES.get(DV_FIELD_STORAGE_TYPE).unwrap());
343 storage_col
344 .is_valid(self.index)
345 .then_some(DeletionVectorView {
346 data: dv_col,
347 index: self.index,
348 })
349 })
350 .flatten()
351 }
352
353 pub(crate) fn to_add(&self) -> Add {
355 Add {
356 path: self.path().to_string(),
357 partition_values: self.partition_values_map(),
358 size: self.size(),
359 modification_time: self.modification_time(),
360 data_change: true,
361 stats: self.stats(),
362 tags: self.file_constant_map(FIELD_NAME_TAGS),
363 deletion_vector: self.deletion_vector().map(|dv| dv.descriptor()),
364 base_row_id: self.file_constant_i64(FIELD_NAME_BASE_ROW_ID),
365 default_row_commit_version: self
366 .file_constant_i64(FIELD_NAME_DEFAULT_ROW_COMMIT_VERSION),
367 clustering_provider: self.file_constant_string(FIELD_NAME_CLUSTERING_PROVIDER),
368 }
369 }
370
371 #[deprecated(
373 since = "0.31.0",
374 note = "Use Arrow arrays directly instead of converting to Add actions."
375 )]
376 pub fn add_action(&self) -> Add {
377 self.to_add()
378 }
379
380 pub fn remove_action(&self, data_change: bool) -> Remove {
382 Remove {
383 path: self.path().to_string(),
385 data_change,
386 deletion_timestamp: Some(Utc::now().timestamp_millis()),
387 extended_file_metadata: Some(true),
388 size: Some(self.size()),
389 partition_values: Some(self.partition_values_map()),
390 deletion_vector: self.deletion_vector().map(|dv| dv.descriptor()),
391 tags: None,
392 base_row_id: None,
393 default_row_commit_version: None,
394 }
395 }
396}
397
398#[cfg(any(test, feature = "datafusion"))]
399fn typed_partition_values_from_raw_map(
400 raw_partition_values: &HashMap<String, Option<String>>,
401 partition_columns: &[String],
402 table_schema: &StructType,
403) -> DeltaResult<IndexMap<String, Scalar>> {
404 let mut partition_values = IndexMap::with_capacity(partition_columns.len());
405
406 for column in partition_columns {
407 let field = table_schema
408 .field(column)
409 .ok_or_else(|| DeltaTableError::SchemaMismatch {
410 msg: format!("Partition column '{column}' is not present in table schema"),
411 })?;
412 let primitive_type = field.data_type().as_primitive_opt().ok_or_else(|| {
413 DeltaTableError::SchemaMismatch {
414 msg: format!("Partition column '{column}' is not a primitive type"),
415 }
416 })?;
417 let scalar = match raw_partition_values.get(column) {
418 Some(Some(raw)) => primitive_type.parse_scalar(raw.as_str())?,
419 Some(None) => Scalar::Null(field.data_type().clone()),
420 None => {
421 return Err(DeltaTableError::SchemaMismatch {
422 msg: format!(
423 "Add action for optimized file is missing partition value for '{column}'"
424 ),
425 });
426 }
427 };
428
429 partition_values.insert(column.clone(), scalar);
430 }
431
432 Ok(partition_values)
433}
434
435fn ceil_datetime(v: i64, ratio: i64) -> i64 {
441 let remainder = v % ratio;
442 if remainder == 0 {
443 v.saturating_add(ratio)
446 } else {
447 v
448 }
449}
450
451pub(crate) fn ceil_timestamps(max: ArrayRef) -> ArrayRef {
453 match max.data_type() {
454 ArrowDataType::Timestamp(TimeUnit::Microsecond, tz) => Arc::new(
455 max.as_primitive::<TimestampMicrosecondType>()
456 .unary::<_, TimestampMicrosecondType>(|v| ceil_datetime(v, 1_000))
457 .with_timezone_opt(tz.clone()),
458 ),
459 ArrowDataType::Timestamp(TimeUnit::Nanosecond, tz) => Arc::new(
460 max.as_primitive::<TimestampNanosecondType>()
461 .unary::<_, TimestampNanosecondType>(|v| ceil_datetime(v, 1_000_000))
462 .with_timezone_opt(tz.clone()),
463 ),
464 _ => max,
465 }
466}
467
468fn round_ms_datetimes<F>(value: Scalar, func: &F) -> Scalar
470where
471 F: Fn(i64, i64) -> i64,
472{
473 match value {
474 #[cfg(feature = "nanosecond-timestamps")]
475 Scalar::TimestampNanos(v) => Scalar::TimestampNanos(func(v, 1_000_000)),
476 #[cfg(feature = "nanosecond-timestamps")]
477 Scalar::TimestampNanosNtz(v) => Scalar::TimestampNanosNtz(func(v, 1_000_000)),
478 Scalar::Timestamp(v) => Scalar::Timestamp(func(v, 1_000)),
479 Scalar::TimestampNtz(v) => Scalar::TimestampNtz(func(v, 1_000)),
480 Scalar::Struct(struct_data) => {
481 let mut fields = Vec::with_capacity(struct_data.fields().len());
482 let mut scalars = Vec::with_capacity(struct_data.values().len());
483
484 for (field, value) in struct_data.fields().iter().zip(struct_data.values().iter()) {
485 fields.push(field.clone());
486 scalars.push(round_ms_datetimes(value.clone(), func));
487 }
488 let data = StructData::try_new(fields, scalars).unwrap();
489 Scalar::Struct(data)
490 }
491 value => value,
492 }
493}
494
495#[derive(Debug)]
500struct DeletionVectorView<'a> {
501 data: &'a StructArray,
502 index: usize,
504}
505
506impl DeletionVectorView<'_> {
507 fn descriptor(&self) -> DeletionVectorDescriptor {
509 DeletionVectorDescriptor {
510 storage_type: self.storage_type().parse().unwrap(),
511 path_or_inline_dv: self.path_or_inline_dv().to_string(),
512 size_in_bytes: self.size_in_bytes(),
513 cardinality: self.cardinality(),
514 offset: self.offset(),
515 }
516 }
517
518 fn storage_type(&self) -> &str {
520 get_string_value(
521 self.data
522 .column(*DV_FIELD_INDICES.get(DV_FIELD_STORAGE_TYPE).unwrap()),
523 self.index,
524 )
525 .unwrap()
526 }
527
528 fn path_or_inline_dv(&self) -> &str {
530 get_string_value(
531 self.data
532 .column(*DV_FIELD_INDICES.get(DV_FIELD_PATH_OR_INLINE_DV).unwrap()),
533 self.index,
534 )
535 .unwrap()
536 }
537
538 fn size_in_bytes(&self) -> i32 {
540 self.data
541 .column(*DV_FIELD_INDICES.get(DV_FIELD_SIZE_IN_BYTES).unwrap())
542 .as_primitive::<Int32Type>()
543 .value(self.index)
544 }
545
546 fn cardinality(&self) -> i64 {
548 self.data
549 .column(*DV_FIELD_INDICES.get(DV_FIELD_CARDINALITY).unwrap())
550 .as_primitive::<Int64Type>()
551 .value(self.index)
552 }
553
554 fn offset(&self) -> Option<i32> {
556 let col = self
557 .data
558 .column_by_name(DV_FIELD_OFFSET)
559 .map(|c| c.as_primitive::<Int32Type>())?;
560 col.is_valid(self.index).then(|| col.value(self.index))
561 }
562}
563
564fn get_string_value(data: &dyn Array, index: usize) -> Option<&str> {
569 match data.data_type() {
570 ArrowDataType::Utf8 => {
571 let arr = data.as_string::<i32>();
572 arr.is_valid(index).then(|| arr.value(index))
573 }
574 ArrowDataType::LargeUtf8 => {
575 let arr = data.as_string::<i64>();
576 arr.is_valid(index).then(|| arr.value(index))
577 }
578 ArrowDataType::Utf8View => {
579 let arr = data.as_string_view();
580 arr.is_valid(index).then(|| arr.value(index))
581 }
582 _ => None,
583 }
584}
585
586fn collect_string_map(data: &dyn Array) -> Option<HashMap<String, Option<String>>> {
587 let entries = data.as_any().downcast_ref::<StructArray>()?;
588 let keys = entries.column(0).as_any().downcast_ref::<StringArray>()?;
589 let values = entries.column(1).as_any().downcast_ref::<StringArray>()?;
590 Some(
591 keys.iter()
592 .zip(values.iter())
593 .filter_map(|(key, value)| key.map(|k| (k.to_string(), value.map(str::to_string))))
594 .collect(),
595 )
596}
597
598impl TryFrom<&LogicalFileView> for ObjectMeta {
599 type Error = DeltaTableError;
600
601 fn try_from(file_stats: &LogicalFileView) -> Result<Self, Self::Error> {
602 Ok(ObjectMeta {
603 location: file_stats.object_store_path(),
604 size: file_stats.size() as u64,
605 last_modified: file_stats.modification_datetime()?,
606 version: None,
607 e_tag: None,
608 })
609 }
610}
611
612#[cfg(test)]
613mod tests {
614 use super::*;
615 use crate::test_utils::TestTables;
616 use arrow::array::{ArrayRef, Int64Array, new_null_array};
617 use chrono::DateTime;
618 use delta_kernel::engine::arrow_conversion::TryIntoArrow as _;
619 use delta_kernel::scan::scan_row_schema;
620 use futures::TryStreamExt;
621 use std::sync::Arc;
622
623 fn logical_file_view_with_stats(
624 raw_stats_json: Option<&str>,
625 stats_parsed: StructArray,
626 ) -> LogicalFileView {
627 let base_schema: arrow_schema::Schema =
628 scan_row_schema().as_ref().try_into_arrow().unwrap();
629 let mut columns: Vec<ArrayRef> = base_schema
630 .fields()
631 .iter()
632 .map(|field| new_null_array(field.data_type(), 1))
633 .collect();
634 columns[base_schema.index_of("path").unwrap()] =
635 Arc::new(StringArray::from(vec![Some("part-000.parquet")]));
636 columns[base_schema.index_of("size").unwrap()] = Arc::new(Int64Array::from(vec![1]));
637 columns[base_schema.index_of("modificationTime").unwrap()] =
638 Arc::new(Int64Array::from(vec![1]));
639 columns[base_schema.index_of("stats").unwrap()] =
640 Arc::new(StringArray::from(vec![raw_stats_json]));
641
642 let mut fields = base_schema.fields().to_vec();
643 fields.push(Arc::new(arrow_schema::Field::new(
644 "stats_parsed",
645 stats_parsed.data_type().to_owned(),
646 true,
647 )));
648 columns.push(Arc::new(stats_parsed));
649
650 let batch =
651 RecordBatch::try_new(Arc::new(arrow_schema::Schema::new(fields)), columns).unwrap();
652 LogicalFileView::new(batch, 0)
653 }
654
655 fn logical_file_view_with_partial_stats(full_stats_json: &str) -> LogicalFileView {
656 let partial_stats = StructArray::from(vec![(
657 Arc::new(arrow_schema::Field::new(
658 "numRecords",
659 ArrowDataType::Int64,
660 true,
661 )),
662 Arc::new(Int64Array::from(vec![Some(11)])) as ArrayRef,
663 )]);
664
665 logical_file_view_with_stats(Some(full_stats_json), partial_stats)
666 }
667
668 fn logical_file_view_without_raw_stats() -> LogicalFileView {
669 let min_values = StructArray::from(vec![(
670 Arc::new(arrow_schema::Field::new(
671 "value",
672 ArrowDataType::Int64,
673 true,
674 )),
675 Arc::new(Int64Array::from(vec![Some(1)])) as ArrayRef,
676 )]);
677 let max_values = StructArray::from(vec![(
678 Arc::new(arrow_schema::Field::new(
679 "value",
680 ArrowDataType::Int64,
681 true,
682 )),
683 Arc::new(Int64Array::from(vec![Some(9)])) as ArrayRef,
684 )]);
685 let null_count = StructArray::from(vec![(
686 Arc::new(arrow_schema::Field::new(
687 "value",
688 ArrowDataType::Int64,
689 true,
690 )),
691 Arc::new(Int64Array::from(vec![Some(0)])) as ArrayRef,
692 )]);
693 let full_stats = StructArray::from(vec![
694 (
695 Arc::new(arrow_schema::Field::new(
696 "numRecords",
697 ArrowDataType::Int64,
698 true,
699 )),
700 Arc::new(Int64Array::from(vec![Some(11)])) as ArrayRef,
701 ),
702 (
703 Arc::new(arrow_schema::Field::new(
704 "minValues",
705 min_values.data_type().to_owned(),
706 true,
707 )),
708 Arc::new(min_values) as ArrayRef,
709 ),
710 (
711 Arc::new(arrow_schema::Field::new(
712 "maxValues",
713 max_values.data_type().to_owned(),
714 true,
715 )),
716 Arc::new(max_values) as ArrayRef,
717 ),
718 (
719 Arc::new(arrow_schema::Field::new(
720 "nullCount",
721 null_count.data_type().to_owned(),
722 true,
723 )),
724 Arc::new(null_count) as ArrayRef,
725 ),
726 ]);
727
728 logical_file_view_with_stats(None, full_stats)
729 }
730
731 fn partition_identity_schema() -> StructType {
732 StructType::try_new([
733 crate::kernel::StructField::nullable("part_b", DataType::STRING),
734 crate::kernel::StructField::nullable("part_a", DataType::LONG),
735 crate::kernel::StructField::nullable("part_c", DataType::STRING),
736 ])
737 .unwrap()
738 }
739
740 #[test]
741 fn typed_partition_values_from_raw_map_preserves_metadata_order_and_parses_values() {
742 let raw_partition_values = HashMap::from([
743 ("part_a".to_string(), Some("42".to_string())),
744 ("part_b".to_string(), Some("x".to_string())),
745 ]);
746 let partition_columns = vec!["part_b".to_string(), "part_a".to_string()];
747
748 let partition_values = typed_partition_values_from_raw_map(
749 &raw_partition_values,
750 &partition_columns,
751 &partition_identity_schema(),
752 )
753 .unwrap();
754
755 assert_eq!(
756 partition_values.keys().cloned().collect::<Vec<_>>(),
757 partition_columns
758 );
759 assert_eq!(
760 partition_values.get("part_b"),
761 Some(&Scalar::String("x".to_string()))
762 );
763 assert_eq!(partition_values.get("part_a"), Some(&Scalar::Long(42)));
764 }
765
766 #[test]
767 fn typed_partition_values_from_raw_map_preserves_typed_nulls() {
768 let raw_partition_values = HashMap::from([
769 ("part_b".to_string(), Some("x".to_string())),
770 ("part_a".to_string(), None),
771 ]);
772 let partition_columns = vec!["part_b".to_string(), "part_a".to_string()];
773
774 let partition_values = typed_partition_values_from_raw_map(
775 &raw_partition_values,
776 &partition_columns,
777 &partition_identity_schema(),
778 )
779 .unwrap();
780
781 assert_eq!(
782 partition_values.get("part_a"),
783 Some(&Scalar::Null(DataType::LONG))
784 );
785 }
786
787 #[test]
788 fn typed_partition_values_from_raw_map_errors_on_missing_partition_column() {
789 let raw_partition_values = HashMap::from([("part_b".to_string(), Some("x".to_string()))]);
790 let partition_columns = vec!["part_b".to_string(), "part_a".to_string()];
791
792 let error = typed_partition_values_from_raw_map(
793 &raw_partition_values,
794 &partition_columns,
795 &partition_identity_schema(),
796 )
797 .unwrap_err();
798
799 assert!(
800 error
801 .to_string()
802 .contains("missing partition value for 'part_a'"),
803 "{error}"
804 );
805 }
806
807 #[test]
808 fn typed_partition_values_from_raw_map_ignores_extra_raw_partition_keys() {
809 let raw_partition_values = HashMap::from([
810 ("part_a".to_string(), Some("42".to_string())),
811 ("part_b".to_string(), Some("x".to_string())),
812 ("part_c".to_string(), Some("extra".to_string())),
813 ]);
814 let partition_columns = vec!["part_b".to_string(), "part_a".to_string()];
815
816 let partition_values = typed_partition_values_from_raw_map(
817 &raw_partition_values,
818 &partition_columns,
819 &partition_identity_schema(),
820 )
821 .unwrap();
822
823 assert_eq!(partition_values.len(), 2);
824 assert!(!partition_values.contains_key("part_c"));
825 }
826
827 #[tokio::test]
828 async fn test_logical_file_view_with_real_data() {
829 let log_store = TestTables::Simple
831 .table_builder()
832 .expect("Failed to create table builder")
833 .build_storage()
834 .expect("Failed to build storage");
835 let snapshot = crate::kernel::snapshot::Snapshot::try_new(&log_store, None)
836 .await
837 .unwrap();
838
839 let files: Vec<_> = snapshot
840 .files(&log_store, None)
841 .try_collect()
842 .await
843 .unwrap();
844 assert!(!files.is_empty(), "Should have test files");
845
846 let first_batch = &files[0];
847 let view = LogicalFileView::new(first_batch.clone(), 0);
848
849 assert!(!view.path().is_empty());
851 assert!(view.size() > 0);
852 assert!(view.modification_time() > 0);
853
854 let datetime = view.modification_datetime().unwrap();
856 assert!(datetime.timestamp_millis() > 0);
857
858 let add_action = view.to_add();
860 assert_eq!(add_action.path, view.path());
861 assert_eq!(add_action.size, view.size());
862 assert!(add_action.data_change);
863
864 let remove_action = view.remove_action(true);
865 assert_eq!(remove_action.path, view.path());
866 assert!(remove_action.data_change);
867 assert!(remove_action.deletion_timestamp.is_some());
868
869 let object_meta: ObjectMeta = (&view).try_into().unwrap();
871 assert_eq!(object_meta.size as i64, view.size());
872 assert_eq!(
873 object_meta.last_modified.timestamp_millis(),
874 view.modification_time()
875 );
876 }
877
878 #[test]
879 fn test_path_url_decoding() {
880 use percent_encoding::percent_decode_str;
882 use std::borrow::Cow;
883
884 let encoded_path = "path/to/file%20with%20spaces.parquet";
885 let decoded: Cow<str> = percent_decode_str(encoded_path).decode_utf8_lossy();
886 assert_eq!(decoded, "path/to/file with spaces.parquet");
887 }
888
889 #[test]
890 fn test_get_string_value_different_types() {
891 use arrow_array::{Int32Array, StringArray};
892
893 let utf8_array = StringArray::from(vec![Some("test"), None]);
895 assert_eq!(get_string_value(&utf8_array, 0), Some("test"));
896 assert_eq!(get_string_value(&utf8_array, 1), None);
897
898 let large_utf8_array = arrow_array::LargeStringArray::from(vec![Some("large_test"), None]);
900 assert_eq!(get_string_value(&large_utf8_array, 0), Some("large_test"));
901 assert_eq!(get_string_value(&large_utf8_array, 1), None);
902
903 let utf8_view_array = arrow_array::StringViewArray::from(vec![Some("view_test"), None]);
905 assert_eq!(get_string_value(&utf8_view_array, 0), Some("view_test"));
906 assert_eq!(get_string_value(&utf8_view_array, 1), None);
907
908 let int_array = Int32Array::from(vec![123]);
910 assert_eq!(get_string_value(&int_array, 0), None);
911 }
912
913 #[test]
914 fn test_ceil_datetime() {
915 assert_eq!(ceil_datetime(1609459200000, 1000), 1609459201000);
918
919 assert_eq!(ceil_datetime(1609459200123, 1000), 1609459200123);
921
922 assert_eq!(ceil_datetime(0, 1000), 1000);
924
925 assert_eq!(ceil_datetime(1609459200000000, 1000000), 1609459201000000);
927 assert_eq!(ceil_datetime(1609459200000123, 1000000), 1609459200000123);
928 assert_eq!(ceil_datetime(0, 1000000), 1000000);
929 assert_eq!(
931 ceil_datetime(1700611178002000000, 1000000),
932 1700611178003000000
933 );
934 }
935
936 #[test]
937 fn test_round_ms_datetimes() {
938 use delta_kernel::expressions::{Scalar, StructData};
939 use delta_kernel::schema::{DataType, PrimitiveType, StructField};
940
941 let ceil_fn = |v: i64, ratio: i64| v + ratio;
942
943 let timestamp = Scalar::Timestamp(1609459200000);
945 let rounded = round_ms_datetimes(timestamp, &ceil_fn);
946 assert_eq!(rounded, Scalar::Timestamp(1609459201000));
947
948 let timestamp_ntz = Scalar::TimestampNtz(1609459200000);
950 let rounded = round_ms_datetimes(timestamp_ntz, &ceil_fn);
951 assert_eq!(rounded, Scalar::TimestampNtz(1609459201000));
952
953 #[cfg(feature = "nanosecond-timestamps")]
954 {
955 let timestamp = Scalar::TimestampNanos(1609459200000000);
956 let rounded = round_ms_datetimes(timestamp, &ceil_fn);
957 assert_eq!(rounded, Scalar::TimestampNanos(1609459201000000));
958
959 let timestamp_ntz = Scalar::TimestampNanosNtz(1609459200000000);
960 let rounded = round_ms_datetimes(timestamp_ntz, &ceil_fn);
961 assert_eq!(rounded, Scalar::TimestampNanosNtz(1609459201000000));
962 }
963
964 let string_scalar = Scalar::String("test".into());
966 let rounded = round_ms_datetimes(string_scalar.clone(), &ceil_fn);
967 assert_eq!(rounded, string_scalar);
968
969 let fields = vec![StructField::new(
971 "ts",
972 DataType::Primitive(PrimitiveType::Timestamp),
973 true,
974 )];
975 let values = vec![Scalar::Timestamp(1609459200000)];
976 let struct_data = StructData::try_new(fields.clone(), values).unwrap();
977 let struct_scalar = Scalar::Struct(struct_data);
978
979 let rounded = round_ms_datetimes(struct_scalar, &ceil_fn);
980 if let Scalar::Struct(rounded_struct) = rounded {
981 assert_eq!(rounded_struct.values()[0], Scalar::Timestamp(1609459201000));
982 } else {
983 panic!("Expected struct scalar");
984 }
985 }
986
987 #[test]
988 fn test_invalid_modification_time() {
989 let invalid_timestamp = i64::MAX;
991 let result = DateTime::from_timestamp_millis(invalid_timestamp);
992 assert!(result.is_none());
993
994 let valid_timestamp = 1609459200000; let result = DateTime::from_timestamp_millis(valid_timestamp);
997 assert!(result.is_some());
998 }
999
1000 #[test]
1001 fn logical_file_view_stats_prefers_raw_stats_json() {
1002 let full_stats_json = r#"{"maxValues":{"value":9},"numRecords":11,"nullCount":{"value":0},"minValues":{"value":1}}"#;
1003 let view = logical_file_view_with_partial_stats(full_stats_json);
1004
1005 assert_eq!(view.stats().as_deref(), Some(full_stats_json));
1006 }
1007
1008 #[test]
1009 #[allow(deprecated)]
1010 fn logical_file_view_add_action_preserves_full_stats_when_stats_parsed_is_partial() {
1011 let full_stats_json = r#"{"maxValues":{"value":9},"numRecords":11,"nullCount":{"value":0},"minValues":{"value":1}}"#;
1012 let view = logical_file_view_with_partial_stats(full_stats_json);
1013
1014 assert_eq!(view.add_action().stats.as_deref(), Some(full_stats_json));
1015 assert_eq!(view.num_records(), Some(11));
1016 assert!(view.min_values().is_none());
1017 }
1018
1019 #[test]
1020 fn logical_file_view_stats_falls_back_to_parsed_stats_when_raw_is_missing() {
1021 let view = logical_file_view_without_raw_stats();
1022 let stats = view
1023 .stats()
1024 .expect("stats fallback should rebuild JSON from stats_parsed");
1025 let actual: serde_json::Value = serde_json::from_str(&stats).unwrap();
1026
1027 assert_eq!(
1028 actual,
1029 serde_json::json!({
1030 "numRecords": 11,
1031 "minValues": {"value": 1},
1032 "maxValues": {"value": 9},
1033 "nullCount": {"value": 0}
1034 })
1035 );
1036 }
1037}