1#![allow(non_camel_case_types)]
4
5use crate::table::Constraint;
6use std::borrow::Borrow;
7use std::collections::HashMap;
8use std::hash::{Hash, Hasher};
9use std::mem::take;
10use std::str::FromStr;
11
12use arrow::compute::filter_record_batch;
13use arrow_array::{BooleanArray, RecordBatch};
14use delta_kernel::FilteredEngineData;
15use delta_kernel::engine::arrow_data::ArrowEngineData;
16use serde::{Deserialize, Serialize};
17use serde_json::Value;
18use url::Url;
19
20use crate::crate_version;
21use crate::errors::{DeltaResult, DeltaTableError};
22use crate::kernel::{
23 Add, CommitInfo, Metadata, Protocol, Remove, StructField, TableFeatures, Version,
24};
25
26pub mod checkpoints;
27pub mod log_compaction;
28
29pub(crate) use checkpoints::{cleanup_expired_logs_for, create_checkpoint_for};
30
31#[derive(Serialize, Deserialize, Debug, PartialEq, Eq)]
33#[serde(untagged)]
34pub enum ColumnValueStat {
35 Column(HashMap<String, ColumnValueStat>),
37 Value(Value),
39}
40
41impl ColumnValueStat {
42 pub fn as_column(&self) -> Option<&HashMap<String, ColumnValueStat>> {
44 match self {
45 ColumnValueStat::Column(m) => Some(m),
46 _ => None,
47 }
48 }
49
50 pub fn as_value(&self) -> Option<&Value> {
52 match self {
53 ColumnValueStat::Value(v) => Some(v),
54 _ => None,
55 }
56 }
57}
58
59#[derive(Serialize, Deserialize, Debug, PartialEq, Eq)]
61#[serde(untagged)]
62pub enum ColumnCountStat {
63 Column(HashMap<String, ColumnCountStat>),
65 Value(i64),
67}
68
69impl ColumnCountStat {
70 pub fn as_column(&self) -> Option<&HashMap<String, ColumnCountStat>> {
72 match self {
73 ColumnCountStat::Column(m) => Some(m),
74 _ => None,
75 }
76 }
77
78 pub fn as_value(&self) -> Option<i64> {
80 match self {
81 ColumnCountStat::Value(v) => Some(*v),
82 _ => None,
83 }
84 }
85}
86
87#[derive(Serialize, Deserialize, Debug, Default, PartialEq, Eq)]
89#[serde(rename_all = "camelCase")]
90pub struct Stats {
91 pub num_records: i64,
93
94 pub min_values: HashMap<String, ColumnValueStat>,
97 pub max_values: HashMap<String, ColumnValueStat>,
99 pub null_count: HashMap<String, ColumnCountStat>,
101}
102
103#[derive(Serialize, Deserialize, Debug, Default, PartialEq, Eq)]
106#[serde(rename_all = "camelCase")]
107struct PartialStats {
108 pub num_records: i64,
110
111 pub min_values: Option<HashMap<String, ColumnValueStat>>,
114 pub max_values: Option<HashMap<String, ColumnValueStat>>,
116 pub null_count: Option<HashMap<String, ColumnCountStat>>,
118}
119
120impl PartialStats {
121 pub fn as_stats(&mut self) -> Stats {
123 let min_values = take(&mut self.min_values);
124 let max_values = take(&mut self.max_values);
125 let null_count = take(&mut self.null_count);
126 Stats {
127 num_records: self.num_records,
128 min_values: min_values.unwrap_or_default(),
129 max_values: max_values.unwrap_or_default(),
130 null_count: null_count.unwrap_or_default(),
131 }
132 }
133}
134
135#[derive(Debug, Default)]
137pub struct StatsParsed {
138 pub num_records: i64,
140
141 pub min_values: HashMap<String, parquet::record::Field>,
144 pub max_values: HashMap<String, parquet::record::Field>,
147 pub null_count: HashMap<String, i64>,
149}
150
151impl Hash for Add {
152 fn hash<H: Hasher>(&self, state: &mut H) {
153 self.path.hash(state);
154 }
155}
156
157impl PartialEq for Add {
158 fn eq(&self, other: &Self) -> bool {
159 self.path == other.path
160 && self.size == other.size
161 && self.partition_values == other.partition_values
162 && self.modification_time == other.modification_time
163 && self.data_change == other.data_change
164 && self.stats == other.stats
165 && self.tags == other.tags
166 && self.deletion_vector == other.deletion_vector
167 }
168}
169
170impl Eq for Add {}
171
172impl Add {
173 pub fn get_stats(&self) -> Result<Option<Stats>, serde_json::error::Error> {
175 self.get_json_stats()
176 }
177
178 fn get_json_stats(&self) -> Result<Option<Stats>, serde_json::error::Error> {
181 self.stats
182 .as_ref()
183 .map(|stats| serde_json::from_str(stats).map(|mut ps: PartialStats| ps.as_stats()))
184 .transpose()
185 }
186}
187
188impl Hash for Remove {
189 fn hash<H: Hasher>(&self, state: &mut H) {
190 self.path.hash(state);
191 }
192}
193
194impl Borrow<str> for Remove {
197 fn borrow(&self) -> &str {
198 self.path.as_ref()
199 }
200}
201
202impl PartialEq for Remove {
203 fn eq(&self, other: &Self) -> bool {
204 self.path == other.path
205 && self.deletion_timestamp == other.deletion_timestamp
206 && self.data_change == other.data_change
207 && self.extended_file_metadata == other.extended_file_metadata
208 && self.partition_values == other.partition_values
209 && self.size == other.size
210 && self.tags == other.tags
211 && self.deletion_vector == other.deletion_vector
212 }
213}
214
215#[allow(clippy::large_enum_variant)]
216#[derive(Serialize, Deserialize, Debug, Clone)]
217#[serde(rename_all = "camelCase")]
218pub struct MergePredicate {
220 pub action_type: String,
222 #[serde(skip_serializing_if = "Option::is_none")]
224 pub predicate: Option<String>,
225}
226
227#[allow(clippy::large_enum_variant)]
230#[derive(Serialize, Deserialize, Debug, Clone)]
231#[serde(rename_all = "camelCase")]
232pub enum DeltaOperation {
233 AddColumn {
236 fields: Vec<StructField>,
238 },
239
240 Create {
244 mode: SaveMode,
246 location: Url,
248 protocol: Protocol,
250 metadata: Metadata,
252 },
253
254 #[serde(rename_all = "camelCase")]
257 Write {
258 mode: SaveMode,
260 partition_by: Option<Vec<String>>,
262 predicate: Option<String>,
264 },
265
266 Delete {
268 predicate: Option<String>,
270 },
271
272 Update {
274 predicate: Option<String>,
276 },
277 AddConstraint {
279 constraints: Vec<Constraint>,
281 },
282
283 AddFeature {
285 name: Vec<TableFeatures>,
287 },
288
289 DropConstraint {
291 name: String,
293 },
294
295 #[serde(rename_all = "camelCase")]
297 Merge {
298 predicate: Option<String>,
300
301 merge_predicate: Option<String>,
303
304 matched_predicates: Vec<MergePredicate>,
306
307 not_matched_predicates: Vec<MergePredicate>,
309
310 not_matched_by_source_predicates: Vec<MergePredicate>,
312 },
313
314 #[serde(rename_all = "camelCase")]
316 StreamingUpdate {
317 output_mode: OutputMode,
319 query_id: String,
321 epoch_id: i64,
323 },
324
325 #[serde(rename_all = "camelCase")]
327 SetTableProperties {
328 properties: HashMap<String, String>,
330 },
331
332 #[serde(rename_all = "camelCase")]
333 Optimize {
335 predicate: Option<String>,
338 target_size: i64,
340 },
341 #[serde(rename_all = "camelCase")]
342 FileSystemCheck {},
344
345 Restore {
347 version: Option<Version>,
349 datetime: Option<i64>,
351 },
352
353 #[serde(rename_all = "camelCase")]
354 VacuumStart {
356 retention_check_enabled: bool,
358 specified_retention_millis: Option<i64>,
360 default_retention_millis: i64,
362 },
363
364 VacuumEnd {
366 status: String,
368 },
369 #[serde(rename_all = "camelCase")]
371 UpdateFieldMetadata {
372 fields: Vec<StructField>,
374 },
375 #[serde(rename_all = "camelCase")]
377 UpdateTableMetadata {
378 metadata_update: crate::operations::update_table_metadata::TableMetadataUpdate,
380 },
381 #[serde(rename_all = "camelCase")]
383 DropColumnNotNull {
384 column: StructField,
386 },
387}
388
389impl DeltaOperation {
390 pub fn name(&self) -> &str {
392 match &self {
394 DeltaOperation::AddColumn { .. } => "ADD COLUMN",
395 DeltaOperation::Create {
396 mode: SaveMode::Overwrite,
397 ..
398 } => "CREATE OR REPLACE TABLE",
399 DeltaOperation::Create { .. } => "CREATE TABLE",
400 DeltaOperation::Write { .. } => "WRITE",
401 DeltaOperation::Delete { .. } => "DELETE",
402 DeltaOperation::Update { .. } => "UPDATE",
403 DeltaOperation::Merge { .. } => "MERGE",
404 DeltaOperation::StreamingUpdate { .. } => "STREAMING UPDATE",
405 DeltaOperation::SetTableProperties { .. } => "SET TBLPROPERTIES",
406 DeltaOperation::Optimize { .. } => "OPTIMIZE",
407 DeltaOperation::FileSystemCheck { .. } => "FSCK",
408 DeltaOperation::Restore { .. } => "RESTORE",
409 DeltaOperation::VacuumStart { .. } => "VACUUM START",
410 DeltaOperation::VacuumEnd { .. } => "VACUUM END",
411 DeltaOperation::AddConstraint { .. } => "ADD CONSTRAINT",
412 DeltaOperation::DropConstraint { .. } => "DROP CONSTRAINT",
413 DeltaOperation::AddFeature { .. } => "ADD FEATURE",
414 DeltaOperation::UpdateFieldMetadata { .. } => "UPDATE FIELD METADATA",
415 DeltaOperation::UpdateTableMetadata { .. } => "UPDATE TABLE METADATA",
416 DeltaOperation::DropColumnNotNull { .. } => "CHANGE COLUMN",
417 }
418 }
419
420 pub fn operation_parameters(&self) -> DeltaResult<HashMap<String, Value>> {
422 let value = serde_json::to_value(self)?;
423 if let Value::Object(mut operation) = value {
424 if let Some(Value::Object(parameters)) = operation.values_mut().next() {
425 return Ok(take(parameters)
426 .into_iter()
427 .filter(|item| !item.1.is_null())
428 .map(|(key, value)| (key, operation_parameter_value(value)))
429 .collect());
430 }
431 }
432
433 Err(DeltaTableError::Generic(
434 "Operation parameters serialized into unexpected shape".into(),
435 ))
436 }
437
438 pub fn changes_data(&self) -> bool {
440 match self {
441 Self::Optimize { .. }
442 | Self::UpdateFieldMetadata { .. }
443 | Self::UpdateTableMetadata { .. }
444 | Self::DropColumnNotNull { .. }
445 | Self::SetTableProperties { .. }
446 | Self::AddColumn { .. }
447 | Self::AddFeature { .. }
448 | Self::VacuumStart { .. }
449 | Self::VacuumEnd { .. }
450 | Self::AddConstraint { .. }
451 | Self::DropConstraint { .. } => false,
452 Self::Create { .. }
453 | Self::FileSystemCheck {}
454 | Self::StreamingUpdate { .. }
455 | Self::Write { .. }
456 | Self::Delete { .. }
457 | Self::Merge { .. }
458 | Self::Update { .. }
459 | Self::Restore { .. } => true,
460 }
461 }
462
463 pub fn get_commit_info(&self) -> CommitInfo {
465 CommitInfo {
467 operation: Some(self.name().into()),
468 operation_parameters: self.operation_parameters().ok(),
469 engine_info: Some(format!("delta-rs:{}", crate_version())),
470 ..Default::default()
471 }
472 }
473
474 pub fn read_predicate(&self) -> Option<String> {
476 match self {
477 Self::Write { predicate, .. } => predicate.clone(),
479 Self::Delete { predicate, .. } => predicate.clone(),
480 Self::Update { predicate, .. } => predicate.clone(),
481 Self::Merge { predicate, .. } => predicate.clone(),
482 _ => None,
483 }
484 }
485
486 pub fn read_whole_table(&self) -> bool {
488 match self {
489 Self::Merge { predicate, .. } if predicate.is_none() => true,
491 Self::AddConstraint { .. } => true,
493 _ => false,
494 }
495 }
496}
497
498pub(crate) fn operation_parameter_value(value: Value) -> Value {
499 Value::String(match value {
500 Value::String(value) => value,
501 value => value.to_string(),
502 })
503}
504
505#[derive(Serialize, Deserialize, Debug, Copy, Clone, PartialEq, Eq)]
507pub enum SaveMode {
508 Append,
510 Overwrite,
512 ErrorIfExists,
514 Ignore,
516}
517
518impl FromStr for SaveMode {
519 type Err = DeltaTableError;
520
521 fn from_str(s: &str) -> DeltaResult<Self> {
522 match s.to_ascii_lowercase().as_str() {
523 "append" => Ok(SaveMode::Append),
524 "overwrite" => Ok(SaveMode::Overwrite),
525 "error" => Ok(SaveMode::ErrorIfExists),
526 "ignore" => Ok(SaveMode::Ignore),
527 _ => Err(DeltaTableError::Generic(format!(
528 "Invalid save mode provided: {s}, only these are supported: ['append', 'overwrite', 'error', 'ignore']"
529 ))),
530 }
531 }
532}
533
534#[derive(Serialize, Deserialize, Debug, Copy, Clone)]
536pub enum OutputMode {
537 Append,
539 Complete,
541 Update,
543}
544
545pub(crate) fn to_rb(data: FilteredEngineData) -> DeltaResult<RecordBatch> {
546 let (underlying_data, selection_vector) = data.into_parts();
547 let engine_data = ArrowEngineData::try_from_engine_data(underlying_data)?;
548 let predicate = BooleanArray::from(selection_vector);
549 let batch = filter_record_batch(engine_data.record_batch(), &predicate)?;
550 Ok(batch)
551}
552
553#[cfg(test)]
554mod tests {
555 use pretty_assertions::assert_eq;
556
557 use super::*;
558 use crate::kernel::Action;
559
560 #[test]
561 fn test_load_table_stats() {
562 let action = Add {
563 stats: Some(
564 serde_json::json!({
565 "numRecords": 22,
566 "minValues": {"a": 1, "nested": {"b": 2, "c": "a"}},
567 "maxValues": {"a": 10, "nested": {"b": 20, "c": "z"}},
568 "nullCount": {"a": 1, "nested": {"b": 0, "c": 1}},
569 })
570 .to_string(),
571 ),
572 path: Default::default(),
573 data_change: true,
574 deletion_vector: None,
575 partition_values: Default::default(),
576 tags: None,
577 size: 0,
578 modification_time: 0,
579 base_row_id: None,
580 default_row_commit_version: None,
581 clustering_provider: None,
582 };
583
584 let stats = action.get_stats().unwrap().unwrap();
585
586 assert_eq!(stats.num_records, 22);
587
588 assert_eq!(
589 stats.min_values["a"].as_value().unwrap(),
590 &serde_json::json!(1)
591 );
592 assert_eq!(
593 stats.min_values["nested"].as_column().unwrap()["b"]
594 .as_value()
595 .unwrap(),
596 &serde_json::json!(2)
597 );
598 assert_eq!(
599 stats.min_values["nested"].as_column().unwrap()["c"]
600 .as_value()
601 .unwrap(),
602 &serde_json::json!("a")
603 );
604
605 assert_eq!(
606 stats.max_values["a"].as_value().unwrap(),
607 &serde_json::json!(10)
608 );
609 assert_eq!(
610 stats.max_values["nested"].as_column().unwrap()["b"]
611 .as_value()
612 .unwrap(),
613 &serde_json::json!(20)
614 );
615 assert_eq!(
616 stats.max_values["nested"].as_column().unwrap()["c"]
617 .as_value()
618 .unwrap(),
619 &serde_json::json!("z")
620 );
621
622 assert_eq!(stats.null_count["a"].as_value().unwrap(), 1);
623 assert_eq!(
624 stats.null_count["nested"].as_column().unwrap()["b"]
625 .as_value()
626 .unwrap(),
627 0
628 );
629 assert_eq!(
630 stats.null_count["nested"].as_column().unwrap()["c"]
631 .as_value()
632 .unwrap(),
633 1
634 );
635 }
636
637 #[test]
638 fn test_load_table_partial_stats() {
639 let action = Add {
640 stats: Some(
641 serde_json::json!({
642 "numRecords": 22
643 })
644 .to_string(),
645 ),
646 path: Default::default(),
647 data_change: true,
648 deletion_vector: None,
649 partition_values: Default::default(),
650 tags: None,
651 size: 0,
652 modification_time: 0,
653 base_row_id: None,
654 default_row_commit_version: None,
655 clustering_provider: None,
656 };
657
658 let stats = action.get_stats().unwrap().unwrap();
659
660 assert_eq!(stats.num_records, 22);
661 assert_eq!(stats.min_values.len(), 0);
662 assert_eq!(stats.max_values.len(), 0);
663 assert_eq!(stats.null_count.len(), 0);
664 }
665
666 #[test]
667 fn test_read_commit_info() {
668 let raw = r#"
669 {
670 "timestamp": 1670892998177,
671 "operation": "WRITE",
672 "operationParameters": {
673 "mode": "Append",
674 "partitionBy": "[\"c1\",\"c2\"]"
675 },
676 "isolationLevel": "Serializable",
677 "isBlindAppend": true,
678 "operationMetrics": {
679 "numFiles": "3",
680 "numOutputRows": "3",
681 "numOutputBytes": "1356"
682 },
683 "engineInfo": "Apache-Spark/3.3.1 Delta-Lake/2.2.0",
684 "txnId": "046a258f-45e3-4657-b0bf-abfb0f76681c"
685 }"#;
686
687 let info = serde_json::from_str::<CommitInfo>(raw);
688 assert!(info.is_ok());
689
690 let raw = "{}";
692 let info = serde_json::from_str::<CommitInfo>(raw);
693 assert!(info.is_ok());
694
695 let raw = r#"
697 {
698 "timestamp": 1670892998177,
699 "operation": "WRITE",
700 "operationParameters": {
701 "mode": "Append",
702 "partitionBy": "[\"c1\",\"c2\"]"
703 },
704 "isolationLevel": "Serializable",
705 "isBlindAppend": true,
706 "operationMetrics": {
707 "numFiles": "3",
708 "numOutputRows": "3",
709 "numOutputBytes": "1356"
710 },
711 "engineInfo": "Apache-Spark/3.3.1 Delta-Lake/2.2.0",
712 "txnId": "046a258f-45e3-4657-b0bf-abfb0f76681c",
713 "additionalField": "more data",
714 "additionalStruct": {
715 "key": "value",
716 "otherKey": 123
717 }
718 }"#;
719
720 let info = serde_json::from_str::<CommitInfo>(raw).expect("should parse");
721 assert!(info.info.contains_key("additionalField"));
722 assert!(info.info.contains_key("additionalStruct"));
723 }
724
725 #[test]
726 fn test_read_domain_metadata() {
727 let buf = r#"{"domainMetadata":{"domain":"delta.liquid","configuration":"{\"clusteringColumns\":[{\"physicalName\":[\"id\"]}],\"domainName\":\"delta.liquid\"}","removed":false}}"#;
728 let _action: Action =
729 serde_json::from_str(buf).expect("Expected to be able to deserialize");
730 }
731
732 mod arrow_tests {
733 use crate::DeltaResult;
734 use arrow::array::{self, ArrayRef, StructArray};
735 use arrow::compute::kernels::cast_utils::Parser;
736 use arrow::compute::sort_to_indices;
737 use arrow::datatypes::{Date32Type, TimestampMicrosecondType};
738 use arrow::record_batch::RecordBatch;
739 use futures::TryStreamExt;
740 use pretty_assertions::assert_eq;
741 use std::path::Path;
742 use std::sync::Arc;
743 use url::Url;
744 fn sort_batch_by(batch: &RecordBatch, column: &str) -> arrow::error::Result<RecordBatch> {
745 let sort_column = batch.column(batch.schema().column_with_name(column).unwrap().0);
746 let sort_indices = sort_to_indices(sort_column, None, None)?;
747 let schema = batch.schema();
748 let sorted_columns: Vec<(&String, ArrayRef)> = schema
749 .fields()
750 .iter()
751 .zip(batch.columns().iter())
752 .map(|(field, column)| {
753 Ok((
754 field.name(),
755 arrow::compute::take(column, &sort_indices, None)?,
756 ))
757 })
758 .collect::<arrow::error::Result<_>>()?;
759 RecordBatch::try_from_iter(sorted_columns)
760 }
761
762 #[tokio::test]
763 async fn test_with_partitions() {
764 let path = "../test/tests/data/delta-0.8.0-null-partition";
766 let table_uri =
767 Url::from_directory_path(std::fs::canonicalize(Path::new(path)).unwrap()).unwrap();
768 let table = crate::open_table(table_uri).await.unwrap();
769 let actions = table.snapshot().unwrap().add_actions_table(true).unwrap();
770
771 let expected_columns: Vec<(&str, ArrayRef)> = vec![
772 (
773 "path",
774 Arc::new(array::StringArray::from(vec![
775 "k=A/part-00000-b1f1dbbb-70bc-4970-893f-9bb772bf246e.c000.snappy.parquet",
776 "k=__HIVE_DEFAULT_PARTITION__/part-00001-8474ac85-360b-4f58-b3ea-23990c71b932.c000.snappy.parquet",
777 ])),
778 ),
779 (
780 "size_bytes",
781 Arc::new(array::Int64Array::from(vec![460, 460])),
782 ),
783 (
784 "modification_time",
785 Arc::new(arrow::array::Int64Array::from(vec![
786 1627990384000,
787 1627990384000,
788 ])),
789 ),
790 (
791 "num_records",
792 Arc::new(array::Int64Array::from(vec![None, None])),
793 ),
794 (
795 "null_count.v",
796 Arc::new(array::Int64Array::from(vec![None, None])),
797 ),
798 ("min.v", Arc::new(array::Int64Array::from(vec![None, None]))),
799 ("max.v", Arc::new(array::Int64Array::from(vec![None, None]))),
800 (
801 "partition.k",
802 Arc::new(array::StringArray::from(vec![Some("A"), None])),
803 ),
804 ];
805 let expected = RecordBatch::try_from_iter(expected_columns.clone()).unwrap();
806
807 assert_eq!(expected, actions);
808
809 }
824
825 #[tokio::test]
826 async fn test_with_deletion_vector() -> DeltaResult<()> {
827 let path = "../test/tests/data/table_with_deletion_logs";
829 let table_uri = Url::from_directory_path(
830 std::fs::canonicalize(path).expect("Failed to canonicalize"),
831 )
832 .expect("Failed to create URL");
833 let _table = crate::open_table(table_uri).await?;
834 Ok(())
835 }
836
837 #[tokio::test]
838 async fn test_without_partitions() {
839 let path = "../test/tests/data/simple_table";
841 let table_uri = Url::from_directory_path(std::fs::canonicalize(path).unwrap()).unwrap();
842 let table = crate::open_table(table_uri).await.unwrap();
843
844 let actions = table.snapshot().unwrap().add_actions_table(true).unwrap();
845 let actions = sort_batch_by(&actions, "path").unwrap();
846
847 let expected_columns: Vec<(&str, ArrayRef)> = vec![
848 (
849 "path",
850 Arc::new(array::StringArray::from(vec![
851 "part-00000-2befed33-c358-4768-a43c-3eda0d2a499d-c000.snappy.parquet",
852 "part-00000-c1777d7d-89d9-4790-b38a-6ee7e24456b1-c000.snappy.parquet",
853 "part-00001-7891c33d-cedc-47c3-88a6-abcfb049d3b4-c000.snappy.parquet",
854 "part-00004-315835fe-fb44-4562-98f6-5e6cfa3ae45d-c000.snappy.parquet",
855 "part-00007-3a0e4727-de0d-41b6-81ef-5223cf40f025-c000.snappy.parquet",
856 ])),
857 ),
858 (
859 "size_bytes",
860 Arc::new(array::Int64Array::from(vec![262, 262, 429, 429, 429])),
861 ),
862 (
863 "modification_time",
864 Arc::new(arrow::array::Int64Array::from(vec![
865 1587968626000,
866 1587968602000,
867 1587968602000,
868 1587968602000,
869 1587968602000,
870 ])),
871 ),
872 (
873 "num_records",
874 Arc::new(arrow::array::Int64Array::from(vec![
875 None, None, None, None, None,
876 ])),
877 ),
878 (
879 "null_count.id",
880 Arc::new(arrow::array::Int64Array::from(vec![
881 None, None, None, None, None,
882 ])),
883 ),
884 (
885 "min.id",
886 Arc::new(arrow::array::Int64Array::from(vec![
887 None, None, None, None, None,
888 ])),
889 ),
890 (
891 "max.id",
892 Arc::new(arrow::array::Int64Array::from(vec![
893 None, None, None, None, None,
894 ])),
895 ),
896 ];
897 let expected = RecordBatch::try_from_iter(expected_columns.clone()).unwrap();
898 assert_eq!(expected, actions);
899 }
900
901 #[tokio::test]
902 async fn test_with_column_mapping() -> DeltaResult<()> {
903 let path = "../test/tests/data/table_with_column_mapping";
905 let table_uri = Url::from_directory_path(
906 std::fs::canonicalize(path).expect("Failed to canonicalize"),
907 )
908 .expect("Failed to create URL");
909 let _table = crate::open_table(table_uri).await?;
910 Ok(())
911 }
912
913 #[tokio::test]
914 async fn test_with_stats() {
915 let path = "../test/tests/data/delta-0.8.0";
917 let table_uri =
918 Url::from_directory_path(std::fs::canonicalize(Path::new(path)).unwrap()).unwrap();
919 let table = crate::open_table(table_uri).await.unwrap();
920 let actions = table.snapshot().unwrap().add_actions_table(true).unwrap();
921 let actions = sort_batch_by(&actions, "path").unwrap();
922
923 let expected_columns: Vec<(&str, ArrayRef)> = vec![
924 (
925 "path",
926 Arc::new(array::StringArray::from(vec![
927 "part-00000-04ec9591-0b73-459e-8d18-ba5711d6cbe1-c000.snappy.parquet",
928 "part-00000-c9b90f86-73e6-46c8-93ba-ff6bfaf892a1-c000.snappy.parquet",
929 ])),
930 ),
931 (
932 "size_bytes",
933 Arc::new(array::Int64Array::from(vec![440, 440])),
934 ),
935 (
936 "modification_time",
937 Arc::new(arrow::array::Int64Array::from(vec![
938 1615043776000,
939 1615043767000,
940 ])),
941 ),
942 ("num_records", Arc::new(array::Int64Array::from(vec![2, 2]))),
943 (
944 "null_count.value",
945 Arc::new(array::Int64Array::from(vec![0, 0])),
946 ),
947 ("min.value", Arc::new(array::Int32Array::from(vec![2, 0]))),
948 ("max.value", Arc::new(array::Int32Array::from(vec![4, 2]))),
949 ];
950 let expected = RecordBatch::try_from_iter(expected_columns.clone()).unwrap();
951
952 assert_eq!(expected, actions);
953 }
954
955 #[tokio::test]
956 async fn test_table_not_always_with_stats() {
957 let path = "../test/tests/data/delta-stats-optional";
958 let table_uri =
959 Url::from_directory_path(std::fs::canonicalize(Path::new(path)).unwrap()).unwrap();
960 let mut table = crate::open_table(table_uri).await.unwrap();
961 table.load().await.unwrap();
962 let actions = table.snapshot().unwrap().add_actions_table(true).unwrap();
963 let actions = sort_batch_by(&actions, "path").unwrap();
964 let expected_path: ArrayRef = Arc::new(array::StringArray::from(vec![
966 "part-00000-28925d3a-bdf2-411e-bca9-b067444cbcb0-c000.snappy.parquet",
967 "part-00000-7a509247-4f58-4453-9202-51d75dee59af-c000.snappy.parquet",
968 ]));
969 let expected_num_records: ArrayRef =
970 Arc::new(array::Int64Array::from(vec![None, Some(1)]));
971 let expected_null_count: ArrayRef =
972 Arc::new(array::Int64Array::from(vec![None, Some(0)]));
973
974 let path_column = actions.column_by_name("path").unwrap();
975 let num_records_column = actions.column_by_name("num_records").unwrap();
976 let null_count_column = actions.column_by_name("null_count.integer").unwrap();
977
978 assert_eq!(&expected_path, path_column);
979 assert_eq!(&expected_num_records, num_records_column);
980 assert_eq!(&expected_null_count, null_count_column);
981 }
982
983 #[tokio::test]
984 async fn test_table_checkpoint_not_always_with_stats() {
985 let path = "../test/tests/data/delta-checkpoint-stats-optional";
986 let table_uri =
987 Url::from_directory_path(std::fs::canonicalize(Path::new(path)).unwrap()).unwrap();
988 let mut table = crate::open_table(table_uri).await.unwrap();
989 table.load().await.unwrap();
990
991 let snapshot = table.snapshot().unwrap().snapshot();
992 let files: Vec<_> = snapshot
993 .file_views(&table.log_store, None)
994 .try_collect()
995 .await
996 .unwrap();
997
998 assert_eq!(2, files.len());
999 }
1000
1001 #[tokio::test]
1002 async fn test_only_struct_stats() {
1003 let path = "../test/tests/data/delta-1.2.1-only-struct-stats";
1005 let table_uri =
1006 Url::from_directory_path(std::fs::canonicalize(Path::new(path)).unwrap()).unwrap();
1007 let mut table = crate::open_table(table_uri).await.unwrap();
1008 table.load_version(1).await.unwrap();
1009
1010 let actions = table.snapshot().unwrap().add_actions_table(true).unwrap();
1011
1012 let expected_columns: Vec<(&str, ArrayRef)> = vec![
1013 (
1014 "path",
1015 Arc::new(array::StringArray::from(vec![
1016 "part-00000-7a509247-4f58-4453-9202-51d75dee59af-c000.snappy.parquet",
1017 ])),
1018 ),
1019 ("size_bytes", Arc::new(array::Int64Array::from(vec![5489]))),
1020 (
1021 "modification_time",
1022 Arc::new(arrow::array::Int64Array::from(vec![1666652373000])),
1023 ),
1024 ("num_records", Arc::new(array::Int64Array::from(vec![1]))),
1025 (
1026 "null_count.integer",
1027 Arc::new(array::Int64Array::from(vec![0])),
1028 ),
1029 (
1030 "null_count.null",
1031 Arc::new(array::Int64Array::from(vec![1])),
1032 ),
1033 (
1034 "null_count.boolean",
1035 Arc::new(array::Int64Array::from(vec![0])),
1036 ),
1037 (
1038 "null_count.double",
1039 Arc::new(array::Int64Array::from(vec![0])),
1040 ),
1041 (
1042 "null_count.decimal",
1043 Arc::new(array::Int64Array::from(vec![0])),
1044 ),
1045 (
1046 "null_count.string",
1047 Arc::new(array::Int64Array::from(vec![0])),
1048 ),
1049 (
1050 "null_count.binary",
1051 Arc::new(array::Int64Array::from(vec![0])),
1052 ),
1053 (
1054 "null_count.date",
1055 Arc::new(array::Int64Array::from(vec![0])),
1056 ),
1057 (
1058 "null_count.timestamp",
1059 Arc::new(array::Int64Array::from(vec![0])),
1060 ),
1061 (
1062 "null_count.struct.struct_element",
1063 Arc::new(array::Int64Array::from(vec![0])),
1064 ),
1065 ("null_count.map", Arc::new(array::Int64Array::from(vec![0]))),
1066 (
1067 "null_count.array",
1068 Arc::new(array::Int64Array::from(vec![0])),
1069 ),
1070 (
1071 "null_count.nested_struct.struct_element.nested_struct_element",
1072 Arc::new(array::Int64Array::from(vec![0])),
1073 ),
1074 (
1075 "null_count.struct_of_array_of_map.struct_element",
1076 Arc::new(array::Int64Array::from(vec![0])),
1077 ),
1078 ("min.integer", Arc::new(array::Int32Array::from(vec![0]))),
1079 ("min.null", Arc::new(array::BooleanArray::from(vec![None]))),
1080 (
1081 "min.boolean",
1082 Arc::new(array::BooleanArray::from(vec![None])),
1083 ),
1084 (
1085 "min.double",
1086 Arc::new(array::Float64Array::from(vec![1.234])),
1087 ),
1088 (
1089 "min.decimal",
1090 Arc::new(
1091 array::Decimal128Array::from_iter_values([-567800])
1092 .with_precision_and_scale(8, 5)
1093 .unwrap(),
1094 ),
1095 ),
1096 (
1097 "min.string",
1098 Arc::new(array::StringArray::from(vec!["string"])),
1099 ),
1100 (
1101 "min.date",
1102 Arc::new(array::Date32Array::from(vec![Date32Type::parse(
1103 "2022-10-24",
1104 )])),
1105 ),
1106 (
1107 "min.timestamp",
1108 Arc::new(
1109 array::TimestampMicrosecondArray::from(vec![
1110 TimestampMicrosecondType::parse("2022-10-24T22:59:32.846Z"),
1111 ])
1112 .with_timezone("UTC"),
1113 ),
1114 ),
1115 (
1116 "min.struct.struct_element",
1117 Arc::new(array::StringArray::from(vec!["struct_value"])),
1118 ),
1119 (
1120 "min.nested_struct.struct_element.nested_struct_element",
1121 Arc::new(array::StringArray::from(vec!["nested_struct_value"])),
1122 ),
1123 ("max.integer", Arc::new(array::Int32Array::from(vec![0]))),
1124 ("max.null", Arc::new(array::BooleanArray::from(vec![None]))),
1125 (
1126 "max.boolean",
1127 Arc::new(array::BooleanArray::from(vec![None])),
1128 ),
1129 (
1130 "max.double",
1131 Arc::new(array::Float64Array::from(vec![1.234])),
1132 ),
1133 (
1134 "max.decimal",
1135 Arc::new(
1136 array::Decimal128Array::from_iter_values([-567800])
1137 .with_precision_and_scale(8, 5)
1138 .unwrap(),
1139 ),
1140 ),
1141 (
1142 "max.string",
1143 Arc::new(array::StringArray::from(vec!["string"])),
1144 ),
1145 (
1146 "max.date",
1147 Arc::new(array::Date32Array::from(vec![Date32Type::parse(
1148 "2022-10-24",
1149 )])),
1150 ),
1151 (
1152 "max.timestamp",
1153 Arc::new(
1154 array::TimestampMicrosecondArray::from(vec![
1155 TimestampMicrosecondType::parse("2022-10-24T22:59:32.846Z"),
1156 ])
1157 .with_timezone("UTC"),
1158 ),
1159 ),
1160 (
1161 "max.struct.struct_element",
1162 Arc::new(array::StringArray::from(vec!["struct_value"])),
1163 ),
1164 (
1165 "max.nested_struct.struct_element.nested_struct_element",
1166 Arc::new(array::StringArray::from(vec!["nested_struct_value"])),
1167 ),
1168 ];
1169 let non_nullable = ["path", "size_bytes", "modification_time"];
1172 let expected = RecordBatch::try_from_iter_with_nullable(
1173 expected_columns
1174 .iter()
1175 .map(|(name, array)| (*name, array.clone(), !non_nullable.contains(name))),
1176 )
1177 .unwrap();
1178
1179 assert_eq!(
1180 expected
1181 .schema()
1182 .fields()
1183 .iter()
1184 .map(|field| field.name().as_str())
1185 .collect::<Vec<&str>>(),
1186 actions
1187 .schema()
1188 .fields()
1189 .iter()
1190 .map(|field| field.name().as_str())
1191 .collect::<Vec<&str>>()
1192 );
1193 assert_eq!(expected, actions);
1194
1195 let actions = table.snapshot().unwrap().add_actions_table(false).unwrap();
1196 assert_eq!(
1199 actions
1200 .get_field_at_path(&[
1201 "null_count",
1202 "nested_struct",
1203 "struct_element",
1204 "nested_struct_element"
1205 ])
1206 .unwrap()
1207 .as_any()
1208 .downcast_ref::<array::Int64Array>()
1209 .unwrap(),
1210 &array::Int64Array::from(vec![0]),
1211 );
1212
1213 assert_eq!(
1214 actions
1215 .get_field_at_path(&[
1216 "min",
1217 "nested_struct",
1218 "struct_element",
1219 "nested_struct_element"
1220 ])
1221 .unwrap()
1222 .as_any()
1223 .downcast_ref::<array::StringArray>()
1224 .unwrap(),
1225 &array::StringArray::from(vec!["nested_struct_value"]),
1226 );
1227
1228 assert_eq!(
1229 actions
1230 .get_field_at_path(&[
1231 "max",
1232 "nested_struct",
1233 "struct_element",
1234 "nested_struct_element"
1235 ])
1236 .unwrap()
1237 .as_any()
1238 .downcast_ref::<array::StringArray>()
1239 .unwrap(),
1240 &array::StringArray::from(vec!["nested_struct_value"]),
1241 );
1242
1243 assert_eq!(
1244 actions
1245 .get_field_at_path(&["null_count", "struct_of_array_of_map", "struct_element"])
1246 .unwrap()
1247 .as_any()
1248 .downcast_ref::<array::Int64Array>()
1249 .unwrap(),
1250 &array::Int64Array::from(vec![0])
1251 );
1252 }
1253
1254 trait NestedTabular {
1256 fn get_field_at_path(&self, path: &[&str]) -> Option<ArrayRef>;
1257 }
1258
1259 impl NestedTabular for RecordBatch {
1260 fn get_field_at_path(&self, path: &[&str]) -> Option<ArrayRef> {
1261 let (first_key, remainder) = path.split_at(1);
1263 let mut col = self.column(self.schema().column_with_name(first_key[0])?.0);
1264
1265 if remainder.is_empty() {
1266 return Some(Arc::clone(col));
1267 }
1268
1269 for segment in remainder {
1270 col = col
1271 .as_any()
1272 .downcast_ref::<StructArray>()?
1273 .column_by_name(segment)?;
1274 }
1275
1276 Some(Arc::clone(col))
1277 }
1278 }
1279 }
1280}