Skip to main content

deltalake_core/kernel/snapshot/
iterators.rs

1use std::borrow::Cow;
2use std::collections::HashMap;
3use std::sync::{Arc, LazyLock};
4
5use arrow::datatypes::Int32Type;
6use arrow_array::cast::AsArray;
7use arrow_array::types::{Int64Type, TimestampMicrosecondType, TimestampNanosecondType};
8use arrow_array::{Array, ArrayRef, MapArray, RecordBatch, StringArray, StructArray};
9use arrow_schema::{DataType as ArrowDataType, TimeUnit};
10use chrono::{DateTime, Utc};
11use delta_kernel::engine::arrow_expression::evaluate_expression::to_json;
12use delta_kernel::expressions::{Scalar, StructData};
13use delta_kernel::scan::scan_row_schema;
14use delta_kernel::schema::DataType;
15#[cfg(any(test, feature = "datafusion"))]
16use indexmap::IndexMap;
17use object_store::ObjectMeta;
18use object_store::path::Path;
19use percent_encoding::percent_decode_str;
20
21#[cfg(feature = "datafusion")]
22pub(crate) use self::scan_row::parse_stats_column_with_schema;
23pub use self::tombstones::TombstoneView;
24#[cfg(any(test, feature = "datafusion"))]
25use crate::kernel::StructType;
26use crate::kernel::scalars::ScalarExt;
27use crate::kernel::{Add, DeletionVectorDescriptor, Remove};
28use crate::{DeltaResult, DeltaTableError};
29
30pub(crate) use self::scan_row::{ScanRowOutStream, scan_row_in_eval};
31
32mod scan_row;
33mod tombstones;
34
35const FIELD_NAME_PATH: &str = "path";
36const FIELD_NAME_SIZE: &str = "size";
37const FIELD_NAME_MODIFICATION_TIME: &str = "modificationTime";
38const FIELD_NAME_FILE_CONSTANT_VALUES: &str = "fileConstantValues";
39const FIELD_NAME_RAW_PARTITION_VALUES: &str = "partitionValues";
40const FIELD_NAME_TAGS: &str = "tags";
41const FIELD_NAME_BASE_ROW_ID: &str = "baseRowId";
42const FIELD_NAME_DEFAULT_ROW_COMMIT_VERSION: &str = "defaultRowCommitVersion";
43const FIELD_NAME_CLUSTERING_PROVIDER: &str = "clusteringProvider";
44const FIELD_NAME_STATS: &str = "stats";
45const FIELD_NAME_STATS_PARSED: &str = "stats_parsed";
46const FIELD_NAME_PARTITION_VALUES_PARSED: &str = "partitionValues_parsed";
47const FIELD_NAME_DELETION_VECTOR: &str = "deletionVector";
48
49const STATS_FIELD_NUM_RECORDS: &str = "numRecords";
50const STATS_FIELD_MIN_VALUES: &str = "minValues";
51const STATS_FIELD_MAX_VALUES: &str = "maxValues";
52const STATS_FIELD_NULL_COUNT: &str = "nullCount";
53
54const DV_FIELD_STORAGE_TYPE: &str = "storageType";
55const DV_FIELD_PATH_OR_INLINE_DV: &str = "pathOrInlineDv";
56const DV_FIELD_SIZE_IN_BYTES: &str = "sizeInBytes";
57const DV_FIELD_CARDINALITY: &str = "cardinality";
58const DV_FIELD_OFFSET: &str = "offset";
59
60static FIELD_INDICES: LazyLock<HashMap<&'static str, usize>> = LazyLock::new(|| {
61    let schema = scan_row_schema();
62    let mut indices = HashMap::new();
63
64    let path_idx = schema.index_of(FIELD_NAME_PATH).unwrap();
65    indices.insert(FIELD_NAME_PATH, path_idx);
66
67    let size_idx = schema.index_of(FIELD_NAME_SIZE).unwrap();
68    indices.insert(FIELD_NAME_SIZE, size_idx);
69
70    let modification_time_idx = schema.index_of(FIELD_NAME_MODIFICATION_TIME).unwrap();
71    indices.insert(FIELD_NAME_MODIFICATION_TIME, modification_time_idx);
72
73    indices
74});
75
76static DV_FIELD_INDICES: LazyLock<HashMap<&'static str, usize>> = LazyLock::new(|| {
77    let schema = scan_row_schema();
78    let dv_field = schema.field(FIELD_NAME_DELETION_VECTOR).unwrap();
79    let DataType::Struct(dv_type) = dv_field.data_type() else {
80        panic!("Expected DataType::Struct for deletion vector field");
81    };
82
83    let mut indices = HashMap::new();
84
85    let storage_type_idx = dv_type.index_of(DV_FIELD_STORAGE_TYPE).unwrap();
86    indices.insert(DV_FIELD_STORAGE_TYPE, storage_type_idx);
87
88    let path_or_inline_dv_idx = dv_type.index_of(DV_FIELD_PATH_OR_INLINE_DV).unwrap();
89    indices.insert(DV_FIELD_PATH_OR_INLINE_DV, path_or_inline_dv_idx);
90
91    let size_in_bytes_idx = dv_type.index_of(DV_FIELD_SIZE_IN_BYTES).unwrap();
92    indices.insert(DV_FIELD_SIZE_IN_BYTES, size_in_bytes_idx);
93
94    let cardinality_idx = dv_type.index_of(DV_FIELD_CARDINALITY).unwrap();
95    indices.insert(DV_FIELD_CARDINALITY, cardinality_idx);
96
97    indices
98});
99
100/// Provides semantic, typed access to file metadata from Delta log replay.
101///
102/// This struct wraps a RecordBatch containing file data and provides zero-copy
103/// access to individual file entries through an index. It serves as a view into
104/// the kernel's log replay results, offering convenient methods to extract
105/// file properties without unnecessary data copies.
106#[derive(Clone)]
107pub struct LogicalFileView {
108    files: RecordBatch,
109    index: usize,
110}
111
112impl LogicalFileView {
113    /// Creates a new view into the specified file entry.
114    pub(crate) fn new(files: RecordBatch, index: usize) -> Self {
115        Self { files, index }
116    }
117
118    /// Returns the file path with URL decoding applied.
119    pub fn path(&self) -> Cow<'_, str> {
120        let raw = get_string_value(
121            self.files
122                .column(*FIELD_INDICES.get(FIELD_NAME_PATH).unwrap()),
123            self.index,
124        )
125        .unwrap();
126        percent_decode_str(raw).decode_utf8_lossy()
127    }
128
129    /// Returns the raw file path as stored in the log, without URL decoding.
130    #[cfg(any(test, feature = "datafusion"))]
131    pub(crate) fn path_raw(&self) -> &str {
132        get_string_value(
133            self.files
134                .column(*FIELD_INDICES.get(FIELD_NAME_PATH).unwrap()),
135            self.index,
136        )
137        .unwrap()
138    }
139
140    /// An object store [`Path`] to the file.
141    ///
142    /// this tries to parse the file string and if that fails, it will return the string as is.
143    // TODO assert consistent handling of the paths encoding when reading log data so this logic can be removed.
144    pub fn object_store_path(&self) -> Path {
145        let path = self.path();
146        // Try to preserve percent encoding if possible
147        match Path::parse(path.as_ref()) {
148            Ok(path) => path,
149            Err(_) => Path::from(path.as_ref()),
150        }
151    }
152
153    /// Returns the file size in bytes.
154    pub fn size(&self) -> i64 {
155        self.files
156            .column(*FIELD_INDICES.get(FIELD_NAME_SIZE).unwrap())
157            .as_primitive::<Int64Type>()
158            .value(self.index)
159    }
160
161    /// Returns the file modification time in milliseconds since Unix epoch.
162    pub fn modification_time(&self) -> i64 {
163        self.files
164            .column(*FIELD_INDICES.get(FIELD_NAME_MODIFICATION_TIME).unwrap())
165            .as_primitive::<Int64Type>()
166            .value(self.index)
167    }
168
169    /// Returns the file modification time as a UTC DateTime.
170    pub fn modification_datetime(&self) -> DeltaResult<chrono::DateTime<Utc>> {
171        DateTime::from_timestamp_millis(self.modification_time()).ok_or(
172            DeltaTableError::MetadataError(format!(
173                "invalid modification_time: {:?}",
174                self.modification_time()
175            )),
176        )
177    }
178
179    /// Returns the raw JSON statistics string for this file, if available.
180    pub fn stats(&self) -> Option<String> {
181        self.files
182            .column_by_name(FIELD_NAME_STATS)
183            .and_then(|col| get_string_value(col, self.index))
184            .map(ToString::to_string)
185            .or_else(|| {
186                let stats = self.stats_parsed()?.slice(self.index, 1);
187                let value = to_json(&stats)
188                    .ok()
189                    .map(|arr| arr.as_string::<i32>().value(0).to_string());
190                value.and_then(|v| (!v.is_empty()).then_some(v))
191            })
192    }
193
194    /// Returns the parsed partition values as structured data.
195    pub fn partition_values(&self) -> Option<StructData> {
196        self.files
197            .column_by_name(FIELD_NAME_PARTITION_VALUES_PARSED)
198            .and_then(|col| col.as_struct_opt())
199            .and_then(|arr| {
200                arr.is_valid(self.index)
201                    .then(|| match Scalar::from_array(arr, self.index) {
202                        Some(Scalar::Struct(s)) => Some(s),
203                        _ => None,
204                    })
205                    .flatten()
206            })
207    }
208
209    fn file_constant_values(&self) -> Option<&StructArray> {
210        self.files
211            .column_by_name(FIELD_NAME_FILE_CONSTANT_VALUES)
212            .and_then(|col| col.as_struct_opt())
213            .filter(|file_constants| file_constants.is_valid(self.index))
214    }
215
216    fn file_constant_map(&self, field_name: &str) -> Option<HashMap<String, Option<String>>> {
217        let values = self
218            .file_constant_values()?
219            .column_by_name(field_name)?
220            .as_map_opt()?;
221        values
222            .is_valid(self.index)
223            .then(|| collect_string_map(&values.value(self.index)))
224            .flatten()
225    }
226
227    fn file_constant_i64(&self, field_name: &str) -> Option<i64> {
228        let values = self
229            .file_constant_values()?
230            .column_by_name(field_name)?
231            .as_primitive_opt::<Int64Type>()?;
232        values
233            .is_valid(self.index)
234            .then(|| values.value(self.index))
235    }
236
237    fn file_constant_string(&self, field_name: &str) -> Option<String> {
238        self.file_constant_values()?
239            .column_by_name(field_name)
240            .and_then(|values| get_string_value(values, self.index))
241            .map(ToString::to_string)
242    }
243
244    fn raw_partition_values(&self) -> Option<&MapArray> {
245        self.file_constant_values()?
246            .column_by_name(FIELD_NAME_RAW_PARTITION_VALUES)
247            .and_then(|col| col.as_map_opt())
248    }
249
250    /// Returns the raw partition value map stored in the log for this file.
251    ///
252    /// This preserves all partition columns even when `partitionValues_parsed` was narrowed to the
253    /// predicate-referenced subset for data skipping.
254    pub fn partition_values_map(&self) -> HashMap<String, Option<String>> {
255        self.raw_partition_values()
256            .filter(|partitions| partitions.is_valid(self.index))
257            .and_then(|partitions| collect_string_map(&partitions.value(self.index)))
258            .unwrap_or_default()
259    }
260
261    /// Builds the complete typed partition tuple in table metadata order.
262    #[cfg(feature = "datafusion")]
263    pub(crate) fn full_partition_values(
264        &self,
265        partition_columns: &[String],
266        table_schema: &StructType,
267    ) -> DeltaResult<IndexMap<String, Scalar>> {
268        typed_partition_values_from_raw_map(
269            &self.partition_values_map(),
270            partition_columns,
271            table_schema,
272        )
273    }
274
275    /// Returns the parsed statistics as a StructArray, if available.
276    fn stats_parsed(&self) -> Option<&StructArray> {
277        self.files
278            .column_by_name(FIELD_NAME_STATS_PARSED)
279            .and_then(|col| col.as_struct_opt())
280    }
281
282    /// Returns the raw file row count from Add-action statistics before deletion-vector filtering.
283    ///
284    /// Returns `None` when the stat is missing or invalid for the current platform.
285    pub fn num_records(&self) -> Option<usize> {
286        self.stats_parsed()
287            .and_then(|stats| stats.column_by_name(STATS_FIELD_NUM_RECORDS))
288            .and_then(|col| col.as_primitive_opt::<Int64Type>())
289            .and_then(|a| {
290                a.is_valid(self.index)
291                    .then(|| usize::try_from(a.value(self.index)).ok())
292                    .flatten()
293            })
294    }
295
296    /// Returns null counts for all columns in this file as structured data.
297    pub fn null_counts(&self) -> Option<Scalar> {
298        self.stats_parsed()
299            .and_then(|stats| stats.column_by_name(STATS_FIELD_NULL_COUNT))
300            .and_then(|c| Scalar::from_array(c.as_ref(), self.index))
301    }
302
303    /// Returns minimum values for all columns with statics in this file as structured data.
304    pub fn min_values(&self) -> Option<Scalar> {
305        self.stats_parsed()
306            .and_then(|stats| stats.column_by_name(STATS_FIELD_MIN_VALUES))
307            .and_then(|c| Scalar::from_array(c.as_ref(), self.index))
308    }
309
310    /// Returns maximum values for all columns in this file as structured data.
311    ///
312    /// For timestamp columns, values are rounded up to handle microsecond truncation
313    /// in checkpoint statistics.
314    pub fn max_values(&self) -> Option<Scalar> {
315        self.stats_parsed()
316            .and_then(|stats| stats.column_by_name(STATS_FIELD_MAX_VALUES))
317            .and_then(|c| Scalar::from_array(c.as_ref(), self.index))
318            .map(|s| round_ms_datetimes(s, &ceil_datetime))
319    }
320
321    /// Return the underlying [DeletionVectorDescriptor] if it exists.
322    ///
323    /// **NOTE**: THis API may be removed in the future without deprecation warnings as the
324    /// utilization of deletion vectors inside of delta-rs becomes more sophisticated.
325    pub fn deletion_vector_descriptor(&self) -> Option<DeletionVectorDescriptor> {
326        self.deletion_vector().map(|dv| dv.descriptor())
327    }
328
329    /// Returns a view into the deletion vector for this file, if present.
330    fn deletion_vector(&self) -> Option<DeletionVectorView<'_>> {
331        let dv_col = self
332            .files
333            .column_by_name(FIELD_NAME_DELETION_VECTOR)
334            .and_then(|col| col.as_struct_opt())?;
335        if dv_col.null_count() == dv_col.len() {
336            return None;
337        }
338        dv_col
339            .is_valid(self.index)
340            .then(|| {
341                let storage_col =
342                    dv_col.column(*DV_FIELD_INDICES.get(DV_FIELD_STORAGE_TYPE).unwrap());
343                storage_col
344                    .is_valid(self.index)
345                    .then_some(DeletionVectorView {
346                        data: dv_col,
347                        index: self.index,
348                    })
349            })
350            .flatten()
351    }
352
353    /// Internal API
354    pub(crate) fn to_add(&self) -> Add {
355        Add {
356            path: self.path().to_string(),
357            partition_values: self.partition_values_map(),
358            size: self.size(),
359            modification_time: self.modification_time(),
360            data_change: true,
361            stats: self.stats(),
362            tags: self.file_constant_map(FIELD_NAME_TAGS),
363            deletion_vector: self.deletion_vector().map(|dv| dv.descriptor()),
364            base_row_id: self.file_constant_i64(FIELD_NAME_BASE_ROW_ID),
365            default_row_commit_version: self
366                .file_constant_i64(FIELD_NAME_DEFAULT_ROW_COMMIT_VERSION),
367            clustering_provider: self.file_constant_string(FIELD_NAME_CLUSTERING_PROVIDER),
368        }
369    }
370
371    /// Converts this file view into an Add action for log operations.
372    #[deprecated(
373        since = "0.31.0",
374        note = "Use Arrow arrays directly instead of converting to Add actions."
375    )]
376    pub fn add_action(&self) -> Add {
377        self.to_add()
378    }
379
380    /// Converts this file view into a Remove action for log operations.
381    pub fn remove_action(&self, data_change: bool) -> Remove {
382        Remove {
383            // TODO use the raw (still encoded) path here once we reconciled serde ...
384            path: self.path().to_string(),
385            data_change,
386            deletion_timestamp: Some(Utc::now().timestamp_millis()),
387            extended_file_metadata: Some(true),
388            size: Some(self.size()),
389            partition_values: Some(self.partition_values_map()),
390            deletion_vector: self.deletion_vector().map(|dv| dv.descriptor()),
391            tags: None,
392            base_row_id: None,
393            default_row_commit_version: None,
394        }
395    }
396}
397
398#[cfg(any(test, feature = "datafusion"))]
399fn typed_partition_values_from_raw_map(
400    raw_partition_values: &HashMap<String, Option<String>>,
401    partition_columns: &[String],
402    table_schema: &StructType,
403) -> DeltaResult<IndexMap<String, Scalar>> {
404    let mut partition_values = IndexMap::with_capacity(partition_columns.len());
405
406    for column in partition_columns {
407        let field = table_schema
408            .field(column)
409            .ok_or_else(|| DeltaTableError::SchemaMismatch {
410                msg: format!("Partition column '{column}' is not present in table schema"),
411            })?;
412        let primitive_type = field.data_type().as_primitive_opt().ok_or_else(|| {
413            DeltaTableError::SchemaMismatch {
414                msg: format!("Partition column '{column}' is not a primitive type"),
415            }
416        })?;
417        let scalar = match raw_partition_values.get(column) {
418            Some(Some(raw)) => primitive_type.parse_scalar(raw.as_str())?,
419            Some(None) => Scalar::Null(field.data_type().clone()),
420            None => {
421                return Err(DeltaTableError::SchemaMismatch {
422                    msg: format!(
423                        "Add action for optimized file is missing partition value for '{column}'"
424                    ),
425                });
426            }
427        };
428
429        partition_values.insert(column.clone(), scalar);
430    }
431
432    Ok(partition_values)
433}
434
435/// Rounds up timestamp values to handle microsecond truncation in checkpoint statistics.
436///
437/// When delta.checkpoint.writeStatsAsStruct is enabled, microsecond timestamps are
438/// truncated to milliseconds. This function rounds up by 1ms to ensure correct
439/// range queries when stats are parsed on-the-fly.
440fn ceil_datetime(v: i64, ratio: i64) -> i64 {
441    let remainder = v % ratio;
442    if remainder == 0 {
443        // if nanoseconds precision remainder is 0, we assume it was truncated
444        // else we use the exact stats
445        v.saturating_add(ratio)
446    } else {
447        v
448    }
449}
450
451/// Applies [`ceil_datetime`] to an array of timestamp max values.
452pub(crate) fn ceil_timestamps(max: ArrayRef) -> ArrayRef {
453    match max.data_type() {
454        ArrowDataType::Timestamp(TimeUnit::Microsecond, tz) => Arc::new(
455            max.as_primitive::<TimestampMicrosecondType>()
456                .unary::<_, TimestampMicrosecondType>(|v| ceil_datetime(v, 1_000))
457                .with_timezone_opt(tz.clone()),
458        ),
459        ArrowDataType::Timestamp(TimeUnit::Nanosecond, tz) => Arc::new(
460            max.as_primitive::<TimestampNanosecondType>()
461                .unary::<_, TimestampNanosecondType>(|v| ceil_datetime(v, 1_000_000))
462                .with_timezone_opt(tz.clone()),
463        ),
464        _ => max,
465    }
466}
467
468/// Recursively applies a rounding function to timestamp values in scalar data.
469fn round_ms_datetimes<F>(value: Scalar, func: &F) -> Scalar
470where
471    F: Fn(i64, i64) -> i64,
472{
473    match value {
474        #[cfg(feature = "nanosecond-timestamps")]
475        Scalar::TimestampNanos(v) => Scalar::TimestampNanos(func(v, 1_000_000)),
476        #[cfg(feature = "nanosecond-timestamps")]
477        Scalar::TimestampNanosNtz(v) => Scalar::TimestampNanosNtz(func(v, 1_000_000)),
478        Scalar::Timestamp(v) => Scalar::Timestamp(func(v, 1_000)),
479        Scalar::TimestampNtz(v) => Scalar::TimestampNtz(func(v, 1_000)),
480        Scalar::Struct(struct_data) => {
481            let mut fields = Vec::with_capacity(struct_data.fields().len());
482            let mut scalars = Vec::with_capacity(struct_data.values().len());
483
484            for (field, value) in struct_data.fields().iter().zip(struct_data.values().iter()) {
485                fields.push(field.clone());
486                scalars.push(round_ms_datetimes(value.clone(), func));
487            }
488            let data = StructData::try_new(fields, scalars).unwrap();
489            Scalar::Struct(data)
490        }
491        value => value,
492    }
493}
494
495/// Provides typed access to deletion vector metadata from log data.
496///
497/// This struct wraps a StructArray containing deletion vector information
498/// and provides zero-copy access to individual fields through an index.
499#[derive(Debug)]
500struct DeletionVectorView<'a> {
501    data: &'a StructArray,
502    /// Index into the deletion vector data array.
503    index: usize,
504}
505
506impl DeletionVectorView<'_> {
507    /// Converts this view into a DeletionVectorDescriptor.
508    fn descriptor(&self) -> DeletionVectorDescriptor {
509        DeletionVectorDescriptor {
510            storage_type: self.storage_type().parse().unwrap(),
511            path_or_inline_dv: self.path_or_inline_dv().to_string(),
512            size_in_bytes: self.size_in_bytes(),
513            cardinality: self.cardinality(),
514            offset: self.offset(),
515        }
516    }
517
518    /// Returns the storage type of the deletion vector.
519    fn storage_type(&self) -> &str {
520        get_string_value(
521            self.data
522                .column(*DV_FIELD_INDICES.get(DV_FIELD_STORAGE_TYPE).unwrap()),
523            self.index,
524        )
525        .unwrap()
526    }
527
528    /// Returns the path or inline data for the deletion vector.
529    fn path_or_inline_dv(&self) -> &str {
530        get_string_value(
531            self.data
532                .column(*DV_FIELD_INDICES.get(DV_FIELD_PATH_OR_INLINE_DV).unwrap()),
533            self.index,
534        )
535        .unwrap()
536    }
537
538    /// Returns the size of the deletion vector in bytes.
539    fn size_in_bytes(&self) -> i32 {
540        self.data
541            .column(*DV_FIELD_INDICES.get(DV_FIELD_SIZE_IN_BYTES).unwrap())
542            .as_primitive::<Int32Type>()
543            .value(self.index)
544    }
545
546    /// Returns the number of deleted rows represented by this deletion vector.
547    fn cardinality(&self) -> i64 {
548        self.data
549            .column(*DV_FIELD_INDICES.get(DV_FIELD_CARDINALITY).unwrap())
550            .as_primitive::<Int64Type>()
551            .value(self.index)
552    }
553
554    /// Returns the offset within the deletion vector file, if applicable.
555    fn offset(&self) -> Option<i32> {
556        let col = self
557            .data
558            .column_by_name(DV_FIELD_OFFSET)
559            .map(|c| c.as_primitive::<Int32Type>())?;
560        col.is_valid(self.index).then(|| col.value(self.index))
561    }
562}
563
564/// Extracts a string value from an Arrow array at the specified index.
565///
566/// Handles different string array types (Utf8, LargeUtf8, Utf8View) and
567/// returns None for null values or unsupported types.
568fn get_string_value(data: &dyn Array, index: usize) -> Option<&str> {
569    match data.data_type() {
570        ArrowDataType::Utf8 => {
571            let arr = data.as_string::<i32>();
572            arr.is_valid(index).then(|| arr.value(index))
573        }
574        ArrowDataType::LargeUtf8 => {
575            let arr = data.as_string::<i64>();
576            arr.is_valid(index).then(|| arr.value(index))
577        }
578        ArrowDataType::Utf8View => {
579            let arr = data.as_string_view();
580            arr.is_valid(index).then(|| arr.value(index))
581        }
582        _ => None,
583    }
584}
585
586fn collect_string_map(data: &dyn Array) -> Option<HashMap<String, Option<String>>> {
587    let entries = data.as_any().downcast_ref::<StructArray>()?;
588    let keys = entries.column(0).as_any().downcast_ref::<StringArray>()?;
589    let values = entries.column(1).as_any().downcast_ref::<StringArray>()?;
590    Some(
591        keys.iter()
592            .zip(values.iter())
593            .filter_map(|(key, value)| key.map(|k| (k.to_string(), value.map(str::to_string))))
594            .collect(),
595    )
596}
597
598impl TryFrom<&LogicalFileView> for ObjectMeta {
599    type Error = DeltaTableError;
600
601    fn try_from(file_stats: &LogicalFileView) -> Result<Self, Self::Error> {
602        Ok(ObjectMeta {
603            location: file_stats.object_store_path(),
604            size: file_stats.size() as u64,
605            last_modified: file_stats.modification_datetime()?,
606            version: None,
607            e_tag: None,
608        })
609    }
610}
611
612#[cfg(test)]
613mod tests {
614    use super::*;
615    use crate::test_utils::TestTables;
616    use arrow::array::{ArrayRef, Int64Array, new_null_array};
617    use chrono::DateTime;
618    use delta_kernel::engine::arrow_conversion::TryIntoArrow as _;
619    use delta_kernel::scan::scan_row_schema;
620    use futures::TryStreamExt;
621    use std::sync::Arc;
622
623    fn logical_file_view_with_stats(
624        raw_stats_json: Option<&str>,
625        stats_parsed: StructArray,
626    ) -> LogicalFileView {
627        let base_schema: arrow_schema::Schema =
628            scan_row_schema().as_ref().try_into_arrow().unwrap();
629        let mut columns: Vec<ArrayRef> = base_schema
630            .fields()
631            .iter()
632            .map(|field| new_null_array(field.data_type(), 1))
633            .collect();
634        columns[base_schema.index_of("path").unwrap()] =
635            Arc::new(StringArray::from(vec![Some("part-000.parquet")]));
636        columns[base_schema.index_of("size").unwrap()] = Arc::new(Int64Array::from(vec![1]));
637        columns[base_schema.index_of("modificationTime").unwrap()] =
638            Arc::new(Int64Array::from(vec![1]));
639        columns[base_schema.index_of("stats").unwrap()] =
640            Arc::new(StringArray::from(vec![raw_stats_json]));
641
642        let mut fields = base_schema.fields().to_vec();
643        fields.push(Arc::new(arrow_schema::Field::new(
644            "stats_parsed",
645            stats_parsed.data_type().to_owned(),
646            true,
647        )));
648        columns.push(Arc::new(stats_parsed));
649
650        let batch =
651            RecordBatch::try_new(Arc::new(arrow_schema::Schema::new(fields)), columns).unwrap();
652        LogicalFileView::new(batch, 0)
653    }
654
655    fn logical_file_view_with_partial_stats(full_stats_json: &str) -> LogicalFileView {
656        let partial_stats = StructArray::from(vec![(
657            Arc::new(arrow_schema::Field::new(
658                "numRecords",
659                ArrowDataType::Int64,
660                true,
661            )),
662            Arc::new(Int64Array::from(vec![Some(11)])) as ArrayRef,
663        )]);
664
665        logical_file_view_with_stats(Some(full_stats_json), partial_stats)
666    }
667
668    fn logical_file_view_without_raw_stats() -> LogicalFileView {
669        let min_values = StructArray::from(vec![(
670            Arc::new(arrow_schema::Field::new(
671                "value",
672                ArrowDataType::Int64,
673                true,
674            )),
675            Arc::new(Int64Array::from(vec![Some(1)])) as ArrayRef,
676        )]);
677        let max_values = StructArray::from(vec![(
678            Arc::new(arrow_schema::Field::new(
679                "value",
680                ArrowDataType::Int64,
681                true,
682            )),
683            Arc::new(Int64Array::from(vec![Some(9)])) as ArrayRef,
684        )]);
685        let null_count = StructArray::from(vec![(
686            Arc::new(arrow_schema::Field::new(
687                "value",
688                ArrowDataType::Int64,
689                true,
690            )),
691            Arc::new(Int64Array::from(vec![Some(0)])) as ArrayRef,
692        )]);
693        let full_stats = StructArray::from(vec![
694            (
695                Arc::new(arrow_schema::Field::new(
696                    "numRecords",
697                    ArrowDataType::Int64,
698                    true,
699                )),
700                Arc::new(Int64Array::from(vec![Some(11)])) as ArrayRef,
701            ),
702            (
703                Arc::new(arrow_schema::Field::new(
704                    "minValues",
705                    min_values.data_type().to_owned(),
706                    true,
707                )),
708                Arc::new(min_values) as ArrayRef,
709            ),
710            (
711                Arc::new(arrow_schema::Field::new(
712                    "maxValues",
713                    max_values.data_type().to_owned(),
714                    true,
715                )),
716                Arc::new(max_values) as ArrayRef,
717            ),
718            (
719                Arc::new(arrow_schema::Field::new(
720                    "nullCount",
721                    null_count.data_type().to_owned(),
722                    true,
723                )),
724                Arc::new(null_count) as ArrayRef,
725            ),
726        ]);
727
728        logical_file_view_with_stats(None, full_stats)
729    }
730
731    fn partition_identity_schema() -> StructType {
732        StructType::try_new([
733            crate::kernel::StructField::nullable("part_b", DataType::STRING),
734            crate::kernel::StructField::nullable("part_a", DataType::LONG),
735            crate::kernel::StructField::nullable("part_c", DataType::STRING),
736        ])
737        .unwrap()
738    }
739
740    #[test]
741    fn typed_partition_values_from_raw_map_preserves_metadata_order_and_parses_values() {
742        let raw_partition_values = HashMap::from([
743            ("part_a".to_string(), Some("42".to_string())),
744            ("part_b".to_string(), Some("x".to_string())),
745        ]);
746        let partition_columns = vec!["part_b".to_string(), "part_a".to_string()];
747
748        let partition_values = typed_partition_values_from_raw_map(
749            &raw_partition_values,
750            &partition_columns,
751            &partition_identity_schema(),
752        )
753        .unwrap();
754
755        assert_eq!(
756            partition_values.keys().cloned().collect::<Vec<_>>(),
757            partition_columns
758        );
759        assert_eq!(
760            partition_values.get("part_b"),
761            Some(&Scalar::String("x".to_string()))
762        );
763        assert_eq!(partition_values.get("part_a"), Some(&Scalar::Long(42)));
764    }
765
766    #[test]
767    fn typed_partition_values_from_raw_map_preserves_typed_nulls() {
768        let raw_partition_values = HashMap::from([
769            ("part_b".to_string(), Some("x".to_string())),
770            ("part_a".to_string(), None),
771        ]);
772        let partition_columns = vec!["part_b".to_string(), "part_a".to_string()];
773
774        let partition_values = typed_partition_values_from_raw_map(
775            &raw_partition_values,
776            &partition_columns,
777            &partition_identity_schema(),
778        )
779        .unwrap();
780
781        assert_eq!(
782            partition_values.get("part_a"),
783            Some(&Scalar::Null(DataType::LONG))
784        );
785    }
786
787    #[test]
788    fn typed_partition_values_from_raw_map_errors_on_missing_partition_column() {
789        let raw_partition_values = HashMap::from([("part_b".to_string(), Some("x".to_string()))]);
790        let partition_columns = vec!["part_b".to_string(), "part_a".to_string()];
791
792        let error = typed_partition_values_from_raw_map(
793            &raw_partition_values,
794            &partition_columns,
795            &partition_identity_schema(),
796        )
797        .unwrap_err();
798
799        assert!(
800            error
801                .to_string()
802                .contains("missing partition value for 'part_a'"),
803            "{error}"
804        );
805    }
806
807    #[test]
808    fn typed_partition_values_from_raw_map_ignores_extra_raw_partition_keys() {
809        let raw_partition_values = HashMap::from([
810            ("part_a".to_string(), Some("42".to_string())),
811            ("part_b".to_string(), Some("x".to_string())),
812            ("part_c".to_string(), Some("extra".to_string())),
813        ]);
814        let partition_columns = vec!["part_b".to_string(), "part_a".to_string()];
815
816        let partition_values = typed_partition_values_from_raw_map(
817            &raw_partition_values,
818            &partition_columns,
819            &partition_identity_schema(),
820        )
821        .unwrap();
822
823        assert_eq!(partition_values.len(), 2);
824        assert!(!partition_values.contains_key("part_c"));
825    }
826
827    #[tokio::test]
828    async fn test_logical_file_view_with_real_data() {
829        // Use existing test table with real Delta log data
830        let log_store = TestTables::Simple
831            .table_builder()
832            .expect("Failed to create table builder")
833            .build_storage()
834            .expect("Failed to build storage");
835        let snapshot = crate::kernel::snapshot::Snapshot::try_new(&log_store, None)
836            .await
837            .unwrap();
838
839        let files: Vec<_> = snapshot
840            .files(&log_store, None)
841            .try_collect()
842            .await
843            .unwrap();
844        assert!(!files.is_empty(), "Should have test files");
845
846        let first_batch = &files[0];
847        let view = LogicalFileView::new(first_batch.clone(), 0);
848
849        // Test basic properties
850        assert!(!view.path().is_empty());
851        assert!(view.size() > 0);
852        assert!(view.modification_time() > 0);
853
854        // Test datetime conversion
855        let datetime = view.modification_datetime().unwrap();
856        assert!(datetime.timestamp_millis() > 0);
857
858        // Test action conversions
859        let add_action = view.to_add();
860        assert_eq!(add_action.path, view.path());
861        assert_eq!(add_action.size, view.size());
862        assert!(add_action.data_change);
863
864        let remove_action = view.remove_action(true);
865        assert_eq!(remove_action.path, view.path());
866        assert!(remove_action.data_change);
867        assert!(remove_action.deletion_timestamp.is_some());
868
869        // Test ObjectMeta conversion
870        let object_meta: ObjectMeta = (&view).try_into().unwrap();
871        assert_eq!(object_meta.size as i64, view.size());
872        assert_eq!(
873            object_meta.last_modified.timestamp_millis(),
874            view.modification_time()
875        );
876    }
877
878    #[test]
879    fn test_path_url_decoding() {
880        // Test URL decoding using the percent_decode_str functionality
881        use percent_encoding::percent_decode_str;
882        use std::borrow::Cow;
883
884        let encoded_path = "path/to/file%20with%20spaces.parquet";
885        let decoded: Cow<str> = percent_decode_str(encoded_path).decode_utf8_lossy();
886        assert_eq!(decoded, "path/to/file with spaces.parquet");
887    }
888
889    #[test]
890    fn test_get_string_value_different_types() {
891        use arrow_array::{Int32Array, StringArray};
892
893        // Test Utf8
894        let utf8_array = StringArray::from(vec![Some("test"), None]);
895        assert_eq!(get_string_value(&utf8_array, 0), Some("test"));
896        assert_eq!(get_string_value(&utf8_array, 1), None);
897
898        // Test LargeUtf8
899        let large_utf8_array = arrow_array::LargeStringArray::from(vec![Some("large_test"), None]);
900        assert_eq!(get_string_value(&large_utf8_array, 0), Some("large_test"));
901        assert_eq!(get_string_value(&large_utf8_array, 1), None);
902
903        // Test Utf8View
904        let utf8_view_array = arrow_array::StringViewArray::from(vec![Some("view_test"), None]);
905        assert_eq!(get_string_value(&utf8_view_array, 0), Some("view_test"));
906        assert_eq!(get_string_value(&utf8_view_array, 1), None);
907
908        // Test unsupported type
909        let int_array = Int32Array::from(vec![123]);
910        assert_eq!(get_string_value(&int_array, 0), None);
911    }
912
913    #[test]
914    fn test_ceil_datetime() {
915        // Microseconds
916        // Test exact millisecond (should be rounded up)
917        assert_eq!(ceil_datetime(1609459200000, 1000), 1609459201000);
918
919        // Test with microsecond remainder (should stay the same)
920        assert_eq!(ceil_datetime(1609459200123, 1000), 1609459200123);
921
922        // Test zero
923        assert_eq!(ceil_datetime(0, 1000), 1000);
924
925        // Nanoseconds
926        assert_eq!(ceil_datetime(1609459200000000, 1000000), 1609459201000000);
927        assert_eq!(ceil_datetime(1609459200000123, 1000000), 1609459200000123);
928        assert_eq!(ceil_datetime(0, 1000000), 1000000);
929        // Too large for an f64 to hold exactly
930        assert_eq!(
931            ceil_datetime(1700611178002000000, 1000000),
932            1700611178003000000
933        );
934    }
935
936    #[test]
937    fn test_round_ms_datetimes() {
938        use delta_kernel::expressions::{Scalar, StructData};
939        use delta_kernel::schema::{DataType, PrimitiveType, StructField};
940
941        let ceil_fn = |v: i64, ratio: i64| v + ratio;
942
943        // Test timestamp scalar
944        let timestamp = Scalar::Timestamp(1609459200000);
945        let rounded = round_ms_datetimes(timestamp, &ceil_fn);
946        assert_eq!(rounded, Scalar::Timestamp(1609459201000));
947
948        // Test timestamp ntz scalar
949        let timestamp_ntz = Scalar::TimestampNtz(1609459200000);
950        let rounded = round_ms_datetimes(timestamp_ntz, &ceil_fn);
951        assert_eq!(rounded, Scalar::TimestampNtz(1609459201000));
952
953        #[cfg(feature = "nanosecond-timestamps")]
954        {
955            let timestamp = Scalar::TimestampNanos(1609459200000000);
956            let rounded = round_ms_datetimes(timestamp, &ceil_fn);
957            assert_eq!(rounded, Scalar::TimestampNanos(1609459201000000));
958
959            let timestamp_ntz = Scalar::TimestampNanosNtz(1609459200000000);
960            let rounded = round_ms_datetimes(timestamp_ntz, &ceil_fn);
961            assert_eq!(rounded, Scalar::TimestampNanosNtz(1609459201000000));
962        }
963
964        // Test non-timestamp scalar (should be unchanged)
965        let string_scalar = Scalar::String("test".into());
966        let rounded = round_ms_datetimes(string_scalar.clone(), &ceil_fn);
967        assert_eq!(rounded, string_scalar);
968
969        // Test struct with timestamp
970        let fields = vec![StructField::new(
971            "ts",
972            DataType::Primitive(PrimitiveType::Timestamp),
973            true,
974        )];
975        let values = vec![Scalar::Timestamp(1609459200000)];
976        let struct_data = StructData::try_new(fields.clone(), values).unwrap();
977        let struct_scalar = Scalar::Struct(struct_data);
978
979        let rounded = round_ms_datetimes(struct_scalar, &ceil_fn);
980        if let Scalar::Struct(rounded_struct) = rounded {
981            assert_eq!(rounded_struct.values()[0], Scalar::Timestamp(1609459201000));
982        } else {
983            panic!("Expected struct scalar");
984        }
985    }
986
987    #[test]
988    fn test_invalid_modification_time() {
989        // Test that invalid timestamps return errors
990        let invalid_timestamp = i64::MAX;
991        let result = DateTime::from_timestamp_millis(invalid_timestamp);
992        assert!(result.is_none());
993
994        // Test a valid timestamp for comparison
995        let valid_timestamp = 1609459200000; // 2021-01-01
996        let result = DateTime::from_timestamp_millis(valid_timestamp);
997        assert!(result.is_some());
998    }
999
1000    #[test]
1001    fn logical_file_view_stats_prefers_raw_stats_json() {
1002        let full_stats_json = r#"{"maxValues":{"value":9},"numRecords":11,"nullCount":{"value":0},"minValues":{"value":1}}"#;
1003        let view = logical_file_view_with_partial_stats(full_stats_json);
1004
1005        assert_eq!(view.stats().as_deref(), Some(full_stats_json));
1006    }
1007
1008    #[test]
1009    #[allow(deprecated)]
1010    fn logical_file_view_add_action_preserves_full_stats_when_stats_parsed_is_partial() {
1011        let full_stats_json = r#"{"maxValues":{"value":9},"numRecords":11,"nullCount":{"value":0},"minValues":{"value":1}}"#;
1012        let view = logical_file_view_with_partial_stats(full_stats_json);
1013
1014        assert_eq!(view.add_action().stats.as_deref(), Some(full_stats_json));
1015        assert_eq!(view.num_records(), Some(11));
1016        assert!(view.min_values().is_none());
1017    }
1018
1019    #[test]
1020    fn logical_file_view_stats_falls_back_to_parsed_stats_when_raw_is_missing() {
1021        let view = logical_file_view_without_raw_stats();
1022        let stats = view
1023            .stats()
1024            .expect("stats fallback should rebuild JSON from stats_parsed");
1025        let actual: serde_json::Value = serde_json::from_str(&stats).unwrap();
1026
1027        assert_eq!(
1028            actual,
1029            serde_json::json!({
1030                "numRecords": 11,
1031                "minValues": {"value": 1},
1032                "maxValues": {"value": 9},
1033                "nullCount": {"value": 0}
1034            })
1035        );
1036    }
1037}