Skip to main content

deltalake_core/protocol/
mod.rs

1//! Actions included in Delta table transaction logs
2
3#![allow(non_camel_case_types)]
4
5use crate::table::Constraint;
6use std::borrow::Borrow;
7use std::collections::HashMap;
8use std::hash::{Hash, Hasher};
9use std::mem::take;
10use std::str::FromStr;
11
12use arrow::compute::filter_record_batch;
13use arrow_array::{BooleanArray, RecordBatch};
14use delta_kernel::FilteredEngineData;
15use delta_kernel::engine::arrow_data::ArrowEngineData;
16use serde::{Deserialize, Serialize};
17use serde_json::Value;
18use url::Url;
19
20use crate::crate_version;
21use crate::errors::{DeltaResult, DeltaTableError};
22use crate::kernel::{
23    Add, CommitInfo, Metadata, Protocol, Remove, StructField, TableFeatures, Version,
24};
25
26pub mod checkpoints;
27pub mod log_compaction;
28
29pub(crate) use checkpoints::{cleanup_expired_logs_for, create_checkpoint_for};
30
31/// Struct used to represent minValues and maxValues in add action statistics.
32#[derive(Serialize, Deserialize, Debug, PartialEq, Eq)]
33#[serde(untagged)]
34pub enum ColumnValueStat {
35    /// Composite HashMap representation of statistics.
36    Column(HashMap<String, ColumnValueStat>),
37    /// Json representation of statistics.
38    Value(Value),
39}
40
41impl ColumnValueStat {
42    /// Returns the HashMap representation of the ColumnValueStat.
43    pub fn as_column(&self) -> Option<&HashMap<String, ColumnValueStat>> {
44        match self {
45            ColumnValueStat::Column(m) => Some(m),
46            _ => None,
47        }
48    }
49
50    /// Returns the serde_json representation of the ColumnValueStat.
51    pub fn as_value(&self) -> Option<&Value> {
52        match self {
53            ColumnValueStat::Value(v) => Some(v),
54            _ => None,
55        }
56    }
57}
58
59/// Struct used to represent nullCount in add action statistics.
60#[derive(Serialize, Deserialize, Debug, PartialEq, Eq)]
61#[serde(untagged)]
62pub enum ColumnCountStat {
63    /// Composite HashMap representation of statistics.
64    Column(HashMap<String, ColumnCountStat>),
65    /// Json representation of statistics.
66    Value(i64),
67}
68
69impl ColumnCountStat {
70    /// Returns the HashMap representation of the ColumnCountStat.
71    pub fn as_column(&self) -> Option<&HashMap<String, ColumnCountStat>> {
72        match self {
73            ColumnCountStat::Column(m) => Some(m),
74            _ => None,
75        }
76    }
77
78    /// Returns the serde_json representation of the ColumnCountStat.
79    pub fn as_value(&self) -> Option<i64> {
80        match self {
81            ColumnCountStat::Value(v) => Some(*v),
82            _ => None,
83        }
84    }
85}
86
87/// Statistics associated with Add actions contained in the Delta log.
88#[derive(Serialize, Deserialize, Debug, Default, PartialEq, Eq)]
89#[serde(rename_all = "camelCase")]
90pub struct Stats {
91    /// Number of records in the file associated with the log action.
92    pub num_records: i64,
93
94    // start of per column stats
95    /// Contains a value smaller than all values present in the file for all columns.
96    pub min_values: HashMap<String, ColumnValueStat>,
97    /// Contains a value larger than all values present in the file for all columns.
98    pub max_values: HashMap<String, ColumnValueStat>,
99    /// The number of null values for all columns.
100    pub null_count: HashMap<String, ColumnCountStat>,
101}
102
103/// Statistics associated with Add actions contained in the Delta log.
104/// min_values, max_values and null_count are optional to allow them to be missing
105#[derive(Serialize, Deserialize, Debug, Default, PartialEq, Eq)]
106#[serde(rename_all = "camelCase")]
107struct PartialStats {
108    /// Number of records in the file associated with the log action.
109    pub num_records: i64,
110
111    // start of per column stats
112    /// Contains a value smaller than all values present in the file for all columns.
113    pub min_values: Option<HashMap<String, ColumnValueStat>>,
114    /// Contains a value larger than all values present in the file for all columns.
115    pub max_values: Option<HashMap<String, ColumnValueStat>>,
116    /// The number of null values for all columns.
117    pub null_count: Option<HashMap<String, ColumnCountStat>>,
118}
119
120impl PartialStats {
121    /// Fills in missing HashMaps
122    pub fn as_stats(&mut self) -> Stats {
123        let min_values = take(&mut self.min_values);
124        let max_values = take(&mut self.max_values);
125        let null_count = take(&mut self.null_count);
126        Stats {
127            num_records: self.num_records,
128            min_values: min_values.unwrap_or_default(),
129            max_values: max_values.unwrap_or_default(),
130            null_count: null_count.unwrap_or_default(),
131        }
132    }
133}
134
135/// File stats parsed from raw parquet format.
136#[derive(Debug, Default)]
137pub struct StatsParsed {
138    /// Number of records in the file associated with the log action.
139    pub num_records: i64,
140
141    // start of per column stats
142    /// Contains a value smaller than all values present in the file for all columns.
143    pub min_values: HashMap<String, parquet::record::Field>,
144    /// Contains a value larger than all values present in the file for all columns.
145    /// Contains a value larger than all values present in the file for all columns.
146    pub max_values: HashMap<String, parquet::record::Field>,
147    /// The number of null values for all columns.
148    pub null_count: HashMap<String, i64>,
149}
150
151impl Hash for Add {
152    fn hash<H: Hasher>(&self, state: &mut H) {
153        self.path.hash(state);
154    }
155}
156
157impl PartialEq for Add {
158    fn eq(&self, other: &Self) -> bool {
159        self.path == other.path
160            && self.size == other.size
161            && self.partition_values == other.partition_values
162            && self.modification_time == other.modification_time
163            && self.data_change == other.data_change
164            && self.stats == other.stats
165            && self.tags == other.tags
166            && self.deletion_vector == other.deletion_vector
167    }
168}
169
170impl Eq for Add {}
171
172impl Add {
173    /// Get whatever stats are available. Uses (parquet struct) parsed_stats if present falling back to json stats.
174    pub fn get_stats(&self) -> Result<Option<Stats>, serde_json::error::Error> {
175        self.get_json_stats()
176    }
177
178    /// Returns the serde_json representation of stats contained in the action if present.
179    /// Since stats are defined as optional in the protocol, this may be None.
180    fn get_json_stats(&self) -> Result<Option<Stats>, serde_json::error::Error> {
181        self.stats
182            .as_ref()
183            .map(|stats| serde_json::from_str(stats).map(|mut ps: PartialStats| ps.as_stats()))
184            .transpose()
185    }
186}
187
188impl Hash for Remove {
189    fn hash<H: Hasher>(&self, state: &mut H) {
190        self.path.hash(state);
191    }
192}
193
194/// Borrow `Remove` as str so we can look at tombstones hashset in `DeltaTableState` by using
195/// a path from action `Add`.
196impl Borrow<str> for Remove {
197    fn borrow(&self) -> &str {
198        self.path.as_ref()
199    }
200}
201
202impl PartialEq for Remove {
203    fn eq(&self, other: &Self) -> bool {
204        self.path == other.path
205            && self.deletion_timestamp == other.deletion_timestamp
206            && self.data_change == other.data_change
207            && self.extended_file_metadata == other.extended_file_metadata
208            && self.partition_values == other.partition_values
209            && self.size == other.size
210            && self.tags == other.tags
211            && self.deletion_vector == other.deletion_vector
212    }
213}
214
215#[allow(clippy::large_enum_variant)]
216#[derive(Serialize, Deserialize, Debug, Clone)]
217#[serde(rename_all = "camelCase")]
218/// Used to record the operations performed to the Delta Log
219pub struct MergePredicate {
220    /// The type of merge operation performed
221    pub action_type: String,
222    /// The predicate used for the merge operation
223    #[serde(skip_serializing_if = "Option::is_none")]
224    pub predicate: Option<String>,
225}
226
227/// Operation performed when creating a new log entry with one or more actions.
228/// This is a key element of the `CommitInfo` action.
229#[allow(clippy::large_enum_variant)]
230#[derive(Serialize, Deserialize, Debug, Clone)]
231#[serde(rename_all = "camelCase")]
232pub enum DeltaOperation {
233    /// Represents a Delta `Add Column` operation.
234    /// Used to add new columns or field in a struct
235    AddColumn {
236        /// Fields added to existing schema
237        fields: Vec<StructField>,
238    },
239
240    /// Represents a Delta `Create` operation.
241    /// Would usually only create the table, if also data is written,
242    /// a `Write` operations is more appropriate
243    Create {
244        /// The save mode used during the create.
245        mode: SaveMode,
246        /// The storage location of the new table
247        location: Url,
248        /// The min reader and writer protocol versions of the table
249        protocol: Protocol,
250        /// Metadata associated with the new table
251        metadata: Metadata,
252    },
253
254    /// Represents a Delta `Write` operation.
255    /// Write operations will typically only include `Add` actions.
256    #[serde(rename_all = "camelCase")]
257    Write {
258        /// The save mode used during the write.
259        mode: SaveMode,
260        /// The columns the write is partitioned by.
261        partition_by: Option<Vec<String>>,
262        /// The predicate used during the write.
263        predicate: Option<String>,
264    },
265
266    /// Delete data matching predicate from delta table
267    Delete {
268        /// The condition the to be deleted data must match
269        predicate: Option<String>,
270    },
271
272    /// Update data matching predicate from delta table
273    Update {
274        /// The update predicate
275        predicate: Option<String>,
276    },
277    /// Add constraints to a table
278    AddConstraint {
279        /// Constraints with Name and Expression
280        constraints: Vec<Constraint>,
281    },
282
283    /// Add table features to a table
284    AddFeature {
285        /// Name of the feature
286        name: Vec<TableFeatures>,
287    },
288
289    /// Drops constraints from a table
290    DropConstraint {
291        /// Constraints name
292        name: String,
293    },
294
295    /// Merge data with a source data with the following predicate
296    #[serde(rename_all = "camelCase")]
297    Merge {
298        /// Cleaned merge predicate for conflict checks
299        predicate: Option<String>,
300
301        /// The original merge predicate
302        merge_predicate: Option<String>,
303
304        /// Match operations performed
305        matched_predicates: Vec<MergePredicate>,
306
307        /// Not Match operations performed
308        not_matched_predicates: Vec<MergePredicate>,
309
310        /// Not Match by Source operations performed
311        not_matched_by_source_predicates: Vec<MergePredicate>,
312    },
313
314    /// Represents a Delta `StreamingUpdate` operation.
315    #[serde(rename_all = "camelCase")]
316    StreamingUpdate {
317        /// The output mode the streaming writer is using.
318        output_mode: OutputMode,
319        /// The query id of the streaming writer.
320        query_id: String,
321        /// The epoch id of the written micro-batch.
322        epoch_id: i64,
323    },
324
325    /// Set table properties operations
326    #[serde(rename_all = "camelCase")]
327    SetTableProperties {
328        /// Table properties that were added
329        properties: HashMap<String, String>,
330    },
331
332    #[serde(rename_all = "camelCase")]
333    /// Represents a `Optimize` operation
334    Optimize {
335        // TODO: Create a string representation of the filter passed to optimize
336        /// The filter used to determine which partitions to filter
337        predicate: Option<String>,
338        /// Target optimize size
339        target_size: i64,
340    },
341    #[serde(rename_all = "camelCase")]
342    /// Represents a `FileSystemCheck` operation
343    FileSystemCheck {},
344
345    /// Represents a `Restore` operation
346    Restore {
347        /// Version to restore
348        version: Option<Version>,
349        ///Datetime to restore
350        datetime: Option<i64>,
351    },
352
353    #[serde(rename_all = "camelCase")]
354    /// Represents the start of `Vacuum` operation
355    VacuumStart {
356        /// Whether the retention check is enforced
357        retention_check_enabled: bool,
358        /// The specified retetion period in milliseconds
359        specified_retention_millis: Option<i64>,
360        /// The default delta table retention milliseconds policy
361        default_retention_millis: i64,
362    },
363
364    /// Represents the end of `Vacuum` operation
365    VacuumEnd {
366        /// The status of the operation
367        status: String,
368    },
369    /// Set table field metadata operations
370    #[serde(rename_all = "camelCase")]
371    UpdateFieldMetadata {
372        /// Fields added to existing schema
373        fields: Vec<StructField>,
374    },
375    /// Update table metadata operations
376    #[serde(rename_all = "camelCase")]
377    UpdateTableMetadata {
378        /// The metadata update to apply
379        metadata_update: crate::operations::update_table_metadata::TableMetadataUpdate,
380    },
381    /// Drop the `NOT NULL` constraint on a column, making it nullable
382    #[serde(rename_all = "camelCase")]
383    DropColumnNotNull {
384        /// The name of the column whose `NOT NULL` constraint was dropped
385        column: StructField,
386    },
387}
388
389impl DeltaOperation {
390    /// A human readable name for the operation
391    pub fn name(&self) -> &str {
392        // operation names taken from https://learn.microsoft.com/en-us/azure/databricks/delta/history#--operation-metrics-keys
393        match &self {
394            DeltaOperation::AddColumn { .. } => "ADD COLUMN",
395            DeltaOperation::Create {
396                mode: SaveMode::Overwrite,
397                ..
398            } => "CREATE OR REPLACE TABLE",
399            DeltaOperation::Create { .. } => "CREATE TABLE",
400            DeltaOperation::Write { .. } => "WRITE",
401            DeltaOperation::Delete { .. } => "DELETE",
402            DeltaOperation::Update { .. } => "UPDATE",
403            DeltaOperation::Merge { .. } => "MERGE",
404            DeltaOperation::StreamingUpdate { .. } => "STREAMING UPDATE",
405            DeltaOperation::SetTableProperties { .. } => "SET TBLPROPERTIES",
406            DeltaOperation::Optimize { .. } => "OPTIMIZE",
407            DeltaOperation::FileSystemCheck { .. } => "FSCK",
408            DeltaOperation::Restore { .. } => "RESTORE",
409            DeltaOperation::VacuumStart { .. } => "VACUUM START",
410            DeltaOperation::VacuumEnd { .. } => "VACUUM END",
411            DeltaOperation::AddConstraint { .. } => "ADD CONSTRAINT",
412            DeltaOperation::DropConstraint { .. } => "DROP CONSTRAINT",
413            DeltaOperation::AddFeature { .. } => "ADD FEATURE",
414            DeltaOperation::UpdateFieldMetadata { .. } => "UPDATE FIELD METADATA",
415            DeltaOperation::UpdateTableMetadata { .. } => "UPDATE TABLE METADATA",
416            DeltaOperation::DropColumnNotNull { .. } => "CHANGE COLUMN",
417        }
418    }
419
420    /// Parameters configured for operation.
421    pub fn operation_parameters(&self) -> DeltaResult<HashMap<String, Value>> {
422        let value = serde_json::to_value(self)?;
423        if let Value::Object(mut operation) = value {
424            if let Some(Value::Object(parameters)) = operation.values_mut().next() {
425                return Ok(take(parameters)
426                    .into_iter()
427                    .filter(|item| !item.1.is_null())
428                    .map(|(key, value)| (key, operation_parameter_value(value)))
429                    .collect());
430            }
431        }
432
433        Err(DeltaTableError::Generic(
434            "Operation parameters serialized into unexpected shape".into(),
435        ))
436    }
437
438    /// Denotes if the operation changes the data contained in the table
439    pub fn changes_data(&self) -> bool {
440        match self {
441            Self::Optimize { .. }
442            | Self::UpdateFieldMetadata { .. }
443            | Self::UpdateTableMetadata { .. }
444            | Self::DropColumnNotNull { .. }
445            | Self::SetTableProperties { .. }
446            | Self::AddColumn { .. }
447            | Self::AddFeature { .. }
448            | Self::VacuumStart { .. }
449            | Self::VacuumEnd { .. }
450            | Self::AddConstraint { .. }
451            | Self::DropConstraint { .. } => false,
452            Self::Create { .. }
453            | Self::FileSystemCheck {}
454            | Self::StreamingUpdate { .. }
455            | Self::Write { .. }
456            | Self::Delete { .. }
457            | Self::Merge { .. }
458            | Self::Update { .. }
459            | Self::Restore { .. } => true,
460        }
461    }
462
463    /// Retrieve basic commit information to be added to Delta commits
464    pub fn get_commit_info(&self) -> CommitInfo {
465        // TODO infer additional info from operation parameters ...
466        CommitInfo {
467            operation: Some(self.name().into()),
468            operation_parameters: self.operation_parameters().ok(),
469            engine_info: Some(format!("delta-rs:{}", crate_version())),
470            ..Default::default()
471        }
472    }
473
474    /// Get predicate expression applied when the operation reads data from the table.
475    pub fn read_predicate(&self) -> Option<String> {
476        match self {
477            // TODO add more operations
478            Self::Write { predicate, .. } => predicate.clone(),
479            Self::Delete { predicate, .. } => predicate.clone(),
480            Self::Update { predicate, .. } => predicate.clone(),
481            Self::Merge { predicate, .. } => predicate.clone(),
482            _ => None,
483        }
484    }
485
486    /// Denotes if the operation reads the entire table
487    pub fn read_whole_table(&self) -> bool {
488        match self {
489            // Predicate is none -> Merge operation had to join full source and target
490            Self::Merge { predicate, .. } if predicate.is_none() => true,
491            // Validates every existing row against the new constraint
492            Self::AddConstraint { .. } => true,
493            _ => false,
494        }
495    }
496}
497
498pub(crate) fn operation_parameter_value(value: Value) -> Value {
499    Value::String(match value {
500        Value::String(value) => value,
501        value => value.to_string(),
502    })
503}
504
505/// The SaveMode used when performing a DeltaOperation
506#[derive(Serialize, Deserialize, Debug, Copy, Clone, PartialEq, Eq)]
507pub enum SaveMode {
508    /// Files will be appended to the target location.
509    Append,
510    /// The target location will be overwritten.
511    Overwrite,
512    /// If files exist for the target, the operation must fail.
513    ErrorIfExists,
514    /// If files exist for the target, the operation must not proceed or change any data.
515    Ignore,
516}
517
518impl FromStr for SaveMode {
519    type Err = DeltaTableError;
520
521    fn from_str(s: &str) -> DeltaResult<Self> {
522        match s.to_ascii_lowercase().as_str() {
523            "append" => Ok(SaveMode::Append),
524            "overwrite" => Ok(SaveMode::Overwrite),
525            "error" => Ok(SaveMode::ErrorIfExists),
526            "ignore" => Ok(SaveMode::Ignore),
527            _ => Err(DeltaTableError::Generic(format!(
528                "Invalid save mode provided: {s}, only these are supported: ['append', 'overwrite', 'error', 'ignore']"
529            ))),
530        }
531    }
532}
533
534/// The OutputMode used in streaming operations.
535#[derive(Serialize, Deserialize, Debug, Copy, Clone)]
536pub enum OutputMode {
537    /// Only new rows will be written when new data is available.
538    Append,
539    /// The full output (all rows) will be written whenever new data is available.
540    Complete,
541    /// Only rows with updates will be written when new or changed data is available.
542    Update,
543}
544
545pub(crate) fn to_rb(data: FilteredEngineData) -> DeltaResult<RecordBatch> {
546    let (underlying_data, selection_vector) = data.into_parts();
547    let engine_data = ArrowEngineData::try_from_engine_data(underlying_data)?;
548    let predicate = BooleanArray::from(selection_vector);
549    let batch = filter_record_batch(engine_data.record_batch(), &predicate)?;
550    Ok(batch)
551}
552
553#[cfg(test)]
554mod tests {
555    use pretty_assertions::assert_eq;
556
557    use super::*;
558    use crate::kernel::Action;
559
560    #[test]
561    fn test_load_table_stats() {
562        let action = Add {
563            stats: Some(
564                serde_json::json!({
565                    "numRecords": 22,
566                    "minValues": {"a": 1, "nested": {"b": 2, "c": "a"}},
567                    "maxValues": {"a": 10, "nested": {"b": 20, "c": "z"}},
568                    "nullCount": {"a": 1, "nested": {"b": 0, "c": 1}},
569                })
570                .to_string(),
571            ),
572            path: Default::default(),
573            data_change: true,
574            deletion_vector: None,
575            partition_values: Default::default(),
576            tags: None,
577            size: 0,
578            modification_time: 0,
579            base_row_id: None,
580            default_row_commit_version: None,
581            clustering_provider: None,
582        };
583
584        let stats = action.get_stats().unwrap().unwrap();
585
586        assert_eq!(stats.num_records, 22);
587
588        assert_eq!(
589            stats.min_values["a"].as_value().unwrap(),
590            &serde_json::json!(1)
591        );
592        assert_eq!(
593            stats.min_values["nested"].as_column().unwrap()["b"]
594                .as_value()
595                .unwrap(),
596            &serde_json::json!(2)
597        );
598        assert_eq!(
599            stats.min_values["nested"].as_column().unwrap()["c"]
600                .as_value()
601                .unwrap(),
602            &serde_json::json!("a")
603        );
604
605        assert_eq!(
606            stats.max_values["a"].as_value().unwrap(),
607            &serde_json::json!(10)
608        );
609        assert_eq!(
610            stats.max_values["nested"].as_column().unwrap()["b"]
611                .as_value()
612                .unwrap(),
613            &serde_json::json!(20)
614        );
615        assert_eq!(
616            stats.max_values["nested"].as_column().unwrap()["c"]
617                .as_value()
618                .unwrap(),
619            &serde_json::json!("z")
620        );
621
622        assert_eq!(stats.null_count["a"].as_value().unwrap(), 1);
623        assert_eq!(
624            stats.null_count["nested"].as_column().unwrap()["b"]
625                .as_value()
626                .unwrap(),
627            0
628        );
629        assert_eq!(
630            stats.null_count["nested"].as_column().unwrap()["c"]
631                .as_value()
632                .unwrap(),
633            1
634        );
635    }
636
637    #[test]
638    fn test_load_table_partial_stats() {
639        let action = Add {
640            stats: Some(
641                serde_json::json!({
642                    "numRecords": 22
643                })
644                .to_string(),
645            ),
646            path: Default::default(),
647            data_change: true,
648            deletion_vector: None,
649            partition_values: Default::default(),
650            tags: None,
651            size: 0,
652            modification_time: 0,
653            base_row_id: None,
654            default_row_commit_version: None,
655            clustering_provider: None,
656        };
657
658        let stats = action.get_stats().unwrap().unwrap();
659
660        assert_eq!(stats.num_records, 22);
661        assert_eq!(stats.min_values.len(), 0);
662        assert_eq!(stats.max_values.len(), 0);
663        assert_eq!(stats.null_count.len(), 0);
664    }
665
666    #[test]
667    fn test_read_commit_info() {
668        let raw = r#"
669        {
670            "timestamp": 1670892998177,
671            "operation": "WRITE",
672            "operationParameters": {
673                "mode": "Append",
674                "partitionBy": "[\"c1\",\"c2\"]"
675            },
676            "isolationLevel": "Serializable",
677            "isBlindAppend": true,
678            "operationMetrics": {
679                "numFiles": "3",
680                "numOutputRows": "3",
681                "numOutputBytes": "1356"
682            },
683            "engineInfo": "Apache-Spark/3.3.1 Delta-Lake/2.2.0",
684            "txnId": "046a258f-45e3-4657-b0bf-abfb0f76681c"
685        }"#;
686
687        let info = serde_json::from_str::<CommitInfo>(raw);
688        assert!(info.is_ok());
689
690        // assert that commit info has no required fields
691        let raw = "{}";
692        let info = serde_json::from_str::<CommitInfo>(raw);
693        assert!(info.is_ok());
694
695        // arbitrary field data may be added to commit
696        let raw = r#"
697        {
698            "timestamp": 1670892998177,
699            "operation": "WRITE",
700            "operationParameters": {
701                "mode": "Append",
702                "partitionBy": "[\"c1\",\"c2\"]"
703            },
704            "isolationLevel": "Serializable",
705            "isBlindAppend": true,
706            "operationMetrics": {
707                "numFiles": "3",
708                "numOutputRows": "3",
709                "numOutputBytes": "1356"
710            },
711            "engineInfo": "Apache-Spark/3.3.1 Delta-Lake/2.2.0",
712            "txnId": "046a258f-45e3-4657-b0bf-abfb0f76681c",
713            "additionalField": "more data",
714            "additionalStruct": {
715                "key": "value",
716                "otherKey": 123
717            }
718        }"#;
719
720        let info = serde_json::from_str::<CommitInfo>(raw).expect("should parse");
721        assert!(info.info.contains_key("additionalField"));
722        assert!(info.info.contains_key("additionalStruct"));
723    }
724
725    #[test]
726    fn test_read_domain_metadata() {
727        let buf = r#"{"domainMetadata":{"domain":"delta.liquid","configuration":"{\"clusteringColumns\":[{\"physicalName\":[\"id\"]}],\"domainName\":\"delta.liquid\"}","removed":false}}"#;
728        let _action: Action =
729            serde_json::from_str(buf).expect("Expected to be able to deserialize");
730    }
731
732    mod arrow_tests {
733        use crate::DeltaResult;
734        use arrow::array::{self, ArrayRef, StructArray};
735        use arrow::compute::kernels::cast_utils::Parser;
736        use arrow::compute::sort_to_indices;
737        use arrow::datatypes::{Date32Type, TimestampMicrosecondType};
738        use arrow::record_batch::RecordBatch;
739        use futures::TryStreamExt;
740        use pretty_assertions::assert_eq;
741        use std::path::Path;
742        use std::sync::Arc;
743        use url::Url;
744        fn sort_batch_by(batch: &RecordBatch, column: &str) -> arrow::error::Result<RecordBatch> {
745            let sort_column = batch.column(batch.schema().column_with_name(column).unwrap().0);
746            let sort_indices = sort_to_indices(sort_column, None, None)?;
747            let schema = batch.schema();
748            let sorted_columns: Vec<(&String, ArrayRef)> = schema
749                .fields()
750                .iter()
751                .zip(batch.columns().iter())
752                .map(|(field, column)| {
753                    Ok((
754                        field.name(),
755                        arrow::compute::take(column, &sort_indices, None)?,
756                    ))
757                })
758                .collect::<arrow::error::Result<_>>()?;
759            RecordBatch::try_from_iter(sorted_columns)
760        }
761
762        #[tokio::test]
763        async fn test_with_partitions() {
764            // test table with partitions
765            let path = "../test/tests/data/delta-0.8.0-null-partition";
766            let table_uri =
767                Url::from_directory_path(std::fs::canonicalize(Path::new(path)).unwrap()).unwrap();
768            let table = crate::open_table(table_uri).await.unwrap();
769            let actions = table.snapshot().unwrap().add_actions_table(true).unwrap();
770
771            let expected_columns: Vec<(&str, ArrayRef)> = vec![
772                (
773                    "path",
774                    Arc::new(array::StringArray::from(vec![
775                        "k=A/part-00000-b1f1dbbb-70bc-4970-893f-9bb772bf246e.c000.snappy.parquet",
776                        "k=__HIVE_DEFAULT_PARTITION__/part-00001-8474ac85-360b-4f58-b3ea-23990c71b932.c000.snappy.parquet",
777                    ])),
778                ),
779                (
780                    "size_bytes",
781                    Arc::new(array::Int64Array::from(vec![460, 460])),
782                ),
783                (
784                    "modification_time",
785                    Arc::new(arrow::array::Int64Array::from(vec![
786                        1627990384000,
787                        1627990384000,
788                    ])),
789                ),
790                (
791                    "num_records",
792                    Arc::new(array::Int64Array::from(vec![None, None])),
793                ),
794                (
795                    "null_count.v",
796                    Arc::new(array::Int64Array::from(vec![None, None])),
797                ),
798                ("min.v", Arc::new(array::Int64Array::from(vec![None, None]))),
799                ("max.v", Arc::new(array::Int64Array::from(vec![None, None]))),
800                (
801                    "partition.k",
802                    Arc::new(array::StringArray::from(vec![Some("A"), None])),
803                ),
804            ];
805            let expected = RecordBatch::try_from_iter(expected_columns.clone()).unwrap();
806
807            assert_eq!(expected, actions);
808
809            // let actions = table.snapshot().unwrap().add_actions_table(false).unwrap();
810            // let actions = sort_batch_by(&actions, "path").unwrap();
811
812            // expected_columns[4] = (
813            //     "partition_values",
814            //     Arc::new(array::StructArray::new(
815            //         Fields::from(vec![Field::new("k", DataType::Utf8, true)]),
816            //         vec![Arc::new(array::StringArray::from(vec![Some("A"), None])) as ArrayRef],
817            //         None,
818            //     )),
819            // );
820            // let expected = RecordBatch::try_from_iter(expected_columns).unwrap();
821
822            // assert_eq!(expected, actions);
823        }
824
825        #[tokio::test]
826        async fn test_with_deletion_vector() -> DeltaResult<()> {
827            // test table with partitions
828            let path = "../test/tests/data/table_with_deletion_logs";
829            let table_uri = Url::from_directory_path(
830                std::fs::canonicalize(path).expect("Failed to canonicalize"),
831            )
832            .expect("Failed to create URL");
833            let _table = crate::open_table(table_uri).await?;
834            Ok(())
835        }
836
837        #[tokio::test]
838        async fn test_without_partitions() {
839            // test table without partitions
840            let path = "../test/tests/data/simple_table";
841            let table_uri = Url::from_directory_path(std::fs::canonicalize(path).unwrap()).unwrap();
842            let table = crate::open_table(table_uri).await.unwrap();
843
844            let actions = table.snapshot().unwrap().add_actions_table(true).unwrap();
845            let actions = sort_batch_by(&actions, "path").unwrap();
846
847            let expected_columns: Vec<(&str, ArrayRef)> = vec![
848                (
849                    "path",
850                    Arc::new(array::StringArray::from(vec![
851                        "part-00000-2befed33-c358-4768-a43c-3eda0d2a499d-c000.snappy.parquet",
852                        "part-00000-c1777d7d-89d9-4790-b38a-6ee7e24456b1-c000.snappy.parquet",
853                        "part-00001-7891c33d-cedc-47c3-88a6-abcfb049d3b4-c000.snappy.parquet",
854                        "part-00004-315835fe-fb44-4562-98f6-5e6cfa3ae45d-c000.snappy.parquet",
855                        "part-00007-3a0e4727-de0d-41b6-81ef-5223cf40f025-c000.snappy.parquet",
856                    ])),
857                ),
858                (
859                    "size_bytes",
860                    Arc::new(array::Int64Array::from(vec![262, 262, 429, 429, 429])),
861                ),
862                (
863                    "modification_time",
864                    Arc::new(arrow::array::Int64Array::from(vec![
865                        1587968626000,
866                        1587968602000,
867                        1587968602000,
868                        1587968602000,
869                        1587968602000,
870                    ])),
871                ),
872                (
873                    "num_records",
874                    Arc::new(arrow::array::Int64Array::from(vec![
875                        None, None, None, None, None,
876                    ])),
877                ),
878                (
879                    "null_count.id",
880                    Arc::new(arrow::array::Int64Array::from(vec![
881                        None, None, None, None, None,
882                    ])),
883                ),
884                (
885                    "min.id",
886                    Arc::new(arrow::array::Int64Array::from(vec![
887                        None, None, None, None, None,
888                    ])),
889                ),
890                (
891                    "max.id",
892                    Arc::new(arrow::array::Int64Array::from(vec![
893                        None, None, None, None, None,
894                    ])),
895                ),
896            ];
897            let expected = RecordBatch::try_from_iter(expected_columns.clone()).unwrap();
898            assert_eq!(expected, actions);
899        }
900
901        #[tokio::test]
902        async fn test_with_column_mapping() -> DeltaResult<()> {
903            // test table with column mapping and partitions
904            let path = "../test/tests/data/table_with_column_mapping";
905            let table_uri = Url::from_directory_path(
906                std::fs::canonicalize(path).expect("Failed to canonicalize"),
907            )
908            .expect("Failed to create URL");
909            let _table = crate::open_table(table_uri).await?;
910            Ok(())
911        }
912
913        #[tokio::test]
914        async fn test_with_stats() {
915            // test table with stats
916            let path = "../test/tests/data/delta-0.8.0";
917            let table_uri =
918                Url::from_directory_path(std::fs::canonicalize(Path::new(path)).unwrap()).unwrap();
919            let table = crate::open_table(table_uri).await.unwrap();
920            let actions = table.snapshot().unwrap().add_actions_table(true).unwrap();
921            let actions = sort_batch_by(&actions, "path").unwrap();
922
923            let expected_columns: Vec<(&str, ArrayRef)> = vec![
924                (
925                    "path",
926                    Arc::new(array::StringArray::from(vec![
927                        "part-00000-04ec9591-0b73-459e-8d18-ba5711d6cbe1-c000.snappy.parquet",
928                        "part-00000-c9b90f86-73e6-46c8-93ba-ff6bfaf892a1-c000.snappy.parquet",
929                    ])),
930                ),
931                (
932                    "size_bytes",
933                    Arc::new(array::Int64Array::from(vec![440, 440])),
934                ),
935                (
936                    "modification_time",
937                    Arc::new(arrow::array::Int64Array::from(vec![
938                        1615043776000,
939                        1615043767000,
940                    ])),
941                ),
942                ("num_records", Arc::new(array::Int64Array::from(vec![2, 2]))),
943                (
944                    "null_count.value",
945                    Arc::new(array::Int64Array::from(vec![0, 0])),
946                ),
947                ("min.value", Arc::new(array::Int32Array::from(vec![2, 0]))),
948                ("max.value", Arc::new(array::Int32Array::from(vec![4, 2]))),
949            ];
950            let expected = RecordBatch::try_from_iter(expected_columns.clone()).unwrap();
951
952            assert_eq!(expected, actions);
953        }
954
955        #[tokio::test]
956        async fn test_table_not_always_with_stats() {
957            let path = "../test/tests/data/delta-stats-optional";
958            let table_uri =
959                Url::from_directory_path(std::fs::canonicalize(Path::new(path)).unwrap()).unwrap();
960            let mut table = crate::open_table(table_uri).await.unwrap();
961            table.load().await.unwrap();
962            let actions = table.snapshot().unwrap().add_actions_table(true).unwrap();
963            let actions = sort_batch_by(&actions, "path").unwrap();
964            // get column-0 path, and column-4 num_records, and column_5 null_count.integer
965            let expected_path: ArrayRef = Arc::new(array::StringArray::from(vec![
966                "part-00000-28925d3a-bdf2-411e-bca9-b067444cbcb0-c000.snappy.parquet",
967                "part-00000-7a509247-4f58-4453-9202-51d75dee59af-c000.snappy.parquet",
968            ]));
969            let expected_num_records: ArrayRef =
970                Arc::new(array::Int64Array::from(vec![None, Some(1)]));
971            let expected_null_count: ArrayRef =
972                Arc::new(array::Int64Array::from(vec![None, Some(0)]));
973
974            let path_column = actions.column_by_name("path").unwrap();
975            let num_records_column = actions.column_by_name("num_records").unwrap();
976            let null_count_column = actions.column_by_name("null_count.integer").unwrap();
977
978            assert_eq!(&expected_path, path_column);
979            assert_eq!(&expected_num_records, num_records_column);
980            assert_eq!(&expected_null_count, null_count_column);
981        }
982
983        #[tokio::test]
984        async fn test_table_checkpoint_not_always_with_stats() {
985            let path = "../test/tests/data/delta-checkpoint-stats-optional";
986            let table_uri =
987                Url::from_directory_path(std::fs::canonicalize(Path::new(path)).unwrap()).unwrap();
988            let mut table = crate::open_table(table_uri).await.unwrap();
989            table.load().await.unwrap();
990
991            let snapshot = table.snapshot().unwrap().snapshot();
992            let files: Vec<_> = snapshot
993                .file_views(&table.log_store, None)
994                .try_collect()
995                .await
996                .unwrap();
997
998            assert_eq!(2, files.len());
999        }
1000
1001        #[tokio::test]
1002        async fn test_only_struct_stats() {
1003            // test table with no json stats
1004            let path = "../test/tests/data/delta-1.2.1-only-struct-stats";
1005            let table_uri =
1006                Url::from_directory_path(std::fs::canonicalize(Path::new(path)).unwrap()).unwrap();
1007            let mut table = crate::open_table(table_uri).await.unwrap();
1008            table.load_version(1).await.unwrap();
1009
1010            let actions = table.snapshot().unwrap().add_actions_table(true).unwrap();
1011
1012            let expected_columns: Vec<(&str, ArrayRef)> = vec![
1013                (
1014                    "path",
1015                    Arc::new(array::StringArray::from(vec![
1016                        "part-00000-7a509247-4f58-4453-9202-51d75dee59af-c000.snappy.parquet",
1017                    ])),
1018                ),
1019                ("size_bytes", Arc::new(array::Int64Array::from(vec![5489]))),
1020                (
1021                    "modification_time",
1022                    Arc::new(arrow::array::Int64Array::from(vec![1666652373000])),
1023                ),
1024                ("num_records", Arc::new(array::Int64Array::from(vec![1]))),
1025                (
1026                    "null_count.integer",
1027                    Arc::new(array::Int64Array::from(vec![0])),
1028                ),
1029                (
1030                    "null_count.null",
1031                    Arc::new(array::Int64Array::from(vec![1])),
1032                ),
1033                (
1034                    "null_count.boolean",
1035                    Arc::new(array::Int64Array::from(vec![0])),
1036                ),
1037                (
1038                    "null_count.double",
1039                    Arc::new(array::Int64Array::from(vec![0])),
1040                ),
1041                (
1042                    "null_count.decimal",
1043                    Arc::new(array::Int64Array::from(vec![0])),
1044                ),
1045                (
1046                    "null_count.string",
1047                    Arc::new(array::Int64Array::from(vec![0])),
1048                ),
1049                (
1050                    "null_count.binary",
1051                    Arc::new(array::Int64Array::from(vec![0])),
1052                ),
1053                (
1054                    "null_count.date",
1055                    Arc::new(array::Int64Array::from(vec![0])),
1056                ),
1057                (
1058                    "null_count.timestamp",
1059                    Arc::new(array::Int64Array::from(vec![0])),
1060                ),
1061                (
1062                    "null_count.struct.struct_element",
1063                    Arc::new(array::Int64Array::from(vec![0])),
1064                ),
1065                ("null_count.map", Arc::new(array::Int64Array::from(vec![0]))),
1066                (
1067                    "null_count.array",
1068                    Arc::new(array::Int64Array::from(vec![0])),
1069                ),
1070                (
1071                    "null_count.nested_struct.struct_element.nested_struct_element",
1072                    Arc::new(array::Int64Array::from(vec![0])),
1073                ),
1074                (
1075                    "null_count.struct_of_array_of_map.struct_element",
1076                    Arc::new(array::Int64Array::from(vec![0])),
1077                ),
1078                ("min.integer", Arc::new(array::Int32Array::from(vec![0]))),
1079                ("min.null", Arc::new(array::BooleanArray::from(vec![None]))),
1080                (
1081                    "min.boolean",
1082                    Arc::new(array::BooleanArray::from(vec![None])),
1083                ),
1084                (
1085                    "min.double",
1086                    Arc::new(array::Float64Array::from(vec![1.234])),
1087                ),
1088                (
1089                    "min.decimal",
1090                    Arc::new(
1091                        array::Decimal128Array::from_iter_values([-567800])
1092                            .with_precision_and_scale(8, 5)
1093                            .unwrap(),
1094                    ),
1095                ),
1096                (
1097                    "min.string",
1098                    Arc::new(array::StringArray::from(vec!["string"])),
1099                ),
1100                (
1101                    "min.date",
1102                    Arc::new(array::Date32Array::from(vec![Date32Type::parse(
1103                        "2022-10-24",
1104                    )])),
1105                ),
1106                (
1107                    "min.timestamp",
1108                    Arc::new(
1109                        array::TimestampMicrosecondArray::from(vec![
1110                            TimestampMicrosecondType::parse("2022-10-24T22:59:32.846Z"),
1111                        ])
1112                        .with_timezone("UTC"),
1113                    ),
1114                ),
1115                (
1116                    "min.struct.struct_element",
1117                    Arc::new(array::StringArray::from(vec!["struct_value"])),
1118                ),
1119                (
1120                    "min.nested_struct.struct_element.nested_struct_element",
1121                    Arc::new(array::StringArray::from(vec!["nested_struct_value"])),
1122                ),
1123                ("max.integer", Arc::new(array::Int32Array::from(vec![0]))),
1124                ("max.null", Arc::new(array::BooleanArray::from(vec![None]))),
1125                (
1126                    "max.boolean",
1127                    Arc::new(array::BooleanArray::from(vec![None])),
1128                ),
1129                (
1130                    "max.double",
1131                    Arc::new(array::Float64Array::from(vec![1.234])),
1132                ),
1133                (
1134                    "max.decimal",
1135                    Arc::new(
1136                        array::Decimal128Array::from_iter_values([-567800])
1137                            .with_precision_and_scale(8, 5)
1138                            .unwrap(),
1139                    ),
1140                ),
1141                (
1142                    "max.string",
1143                    Arc::new(array::StringArray::from(vec!["string"])),
1144                ),
1145                (
1146                    "max.date",
1147                    Arc::new(array::Date32Array::from(vec![Date32Type::parse(
1148                        "2022-10-24",
1149                    )])),
1150                ),
1151                (
1152                    "max.timestamp",
1153                    Arc::new(
1154                        array::TimestampMicrosecondArray::from(vec![
1155                            TimestampMicrosecondType::parse("2022-10-24T22:59:32.846Z"),
1156                        ])
1157                        .with_timezone("UTC"),
1158                    ),
1159                ),
1160                (
1161                    "max.struct.struct_element",
1162                    Arc::new(array::StringArray::from(vec!["struct_value"])),
1163                ),
1164                (
1165                    "max.nested_struct.struct_element.nested_struct_element",
1166                    Arc::new(array::StringArray::from(vec!["nested_struct_value"])),
1167                ),
1168            ];
1169            // `path`, `size_bytes` and `modification_time` are the only non-nullable
1170            // columns of the add actions schema.
1171            let non_nullable = ["path", "size_bytes", "modification_time"];
1172            let expected = RecordBatch::try_from_iter_with_nullable(
1173                expected_columns
1174                    .iter()
1175                    .map(|(name, array)| (*name, array.clone(), !non_nullable.contains(name))),
1176            )
1177            .unwrap();
1178
1179            assert_eq!(
1180                expected
1181                    .schema()
1182                    .fields()
1183                    .iter()
1184                    .map(|field| field.name().as_str())
1185                    .collect::<Vec<&str>>(),
1186                actions
1187                    .schema()
1188                    .fields()
1189                    .iter()
1190                    .map(|field| field.name().as_str())
1191                    .collect::<Vec<&str>>()
1192            );
1193            assert_eq!(expected, actions);
1194
1195            let actions = table.snapshot().unwrap().add_actions_table(false).unwrap();
1196            // For brevity, just checking a few nested columns in stats
1197
1198            assert_eq!(
1199                actions
1200                    .get_field_at_path(&[
1201                        "null_count",
1202                        "nested_struct",
1203                        "struct_element",
1204                        "nested_struct_element"
1205                    ])
1206                    .unwrap()
1207                    .as_any()
1208                    .downcast_ref::<array::Int64Array>()
1209                    .unwrap(),
1210                &array::Int64Array::from(vec![0]),
1211            );
1212
1213            assert_eq!(
1214                actions
1215                    .get_field_at_path(&[
1216                        "min",
1217                        "nested_struct",
1218                        "struct_element",
1219                        "nested_struct_element"
1220                    ])
1221                    .unwrap()
1222                    .as_any()
1223                    .downcast_ref::<array::StringArray>()
1224                    .unwrap(),
1225                &array::StringArray::from(vec!["nested_struct_value"]),
1226            );
1227
1228            assert_eq!(
1229                actions
1230                    .get_field_at_path(&[
1231                        "max",
1232                        "nested_struct",
1233                        "struct_element",
1234                        "nested_struct_element"
1235                    ])
1236                    .unwrap()
1237                    .as_any()
1238                    .downcast_ref::<array::StringArray>()
1239                    .unwrap(),
1240                &array::StringArray::from(vec!["nested_struct_value"]),
1241            );
1242
1243            assert_eq!(
1244                actions
1245                    .get_field_at_path(&["null_count", "struct_of_array_of_map", "struct_element"])
1246                    .unwrap()
1247                    .as_any()
1248                    .downcast_ref::<array::Int64Array>()
1249                    .unwrap(),
1250                &array::Int64Array::from(vec![0])
1251            );
1252        }
1253
1254        /// Trait to make it easier to access nested fields
1255        trait NestedTabular {
1256            fn get_field_at_path(&self, path: &[&str]) -> Option<ArrayRef>;
1257        }
1258
1259        impl NestedTabular for RecordBatch {
1260            fn get_field_at_path(&self, path: &[&str]) -> Option<ArrayRef> {
1261                // First, get array in the batch
1262                let (first_key, remainder) = path.split_at(1);
1263                let mut col = self.column(self.schema().column_with_name(first_key[0])?.0);
1264
1265                if remainder.is_empty() {
1266                    return Some(Arc::clone(col));
1267                }
1268
1269                for segment in remainder {
1270                    col = col
1271                        .as_any()
1272                        .downcast_ref::<StructArray>()?
1273                        .column_by_name(segment)?;
1274                }
1275
1276                Some(Arc::clone(col))
1277            }
1278        }
1279    }
1280}