deltalake_core/delta_datafusion/cdf/
mod.rs1use std::collections::HashMap;
3use std::sync::LazyLock;
4
5use arrow_schema::{DataType, Field, TimeUnit};
6
7pub(crate) use self::scan_utils::*;
8use crate::DeltaResult;
9use crate::kernel::{Add, AddCDCFile, DeletionVectorDescriptor, Remove, Version};
10
11pub mod scan;
13mod scan_utils;
14
15pub const CHANGE_TYPE_COL: &str = "_change_type";
17pub const COMMIT_VERSION_COL: &str = "_commit_version";
19pub const COMMIT_TIMESTAMP_COL: &str = "_commit_timestamp";
21
22pub(crate) static CDC_PARTITION_SCHEMA: LazyLock<Vec<Field>> = LazyLock::new(|| {
23 vec![
24 Field::new(COMMIT_VERSION_COL, DataType::UInt64, true),
25 Field::new(
26 COMMIT_TIMESTAMP_COL,
27 DataType::Timestamp(TimeUnit::Millisecond, None),
28 true,
29 ),
30 ]
31});
32pub(crate) static ADD_PARTITION_SCHEMA: LazyLock<Vec<Field>> = LazyLock::new(|| {
33 vec![
34 Field::new(CHANGE_TYPE_COL, DataType::Utf8, true),
35 Field::new(COMMIT_VERSION_COL, DataType::UInt64, true),
36 Field::new(
37 COMMIT_TIMESTAMP_COL,
38 DataType::Timestamp(TimeUnit::Millisecond, None),
39 true,
40 ),
41 ]
42});
43
44#[derive(Debug)]
49pub(crate) struct ResolvedPair {
50 pub version: Version,
51 pub timestamp: i64,
52 pub add: Add,
53 pub rm_dv: Option<DeletionVectorDescriptor>,
54}
55
56#[derive(Debug)]
57pub(crate) struct CdcDataSpec<F: FileAction> {
58 pub version: Version,
59 pub timestamp: i64,
60 pub actions: Vec<F>,
61}
62
63impl<F: FileAction> CdcDataSpec<F> {
64 pub fn new(version: Version, timestamp: i64, actions: Vec<F>) -> Self {
65 Self {
66 version,
67 timestamp,
68 actions,
69 }
70 }
71
72 pub fn into_parts(self) -> (Version, i64, Vec<F>) {
73 (self.version, self.timestamp, self.actions)
74 }
75}
76
77pub trait FileAction {
79 fn partition_values(&self) -> DeltaResult<&HashMap<String, Option<String>>>;
81 fn path(&self) -> String;
83 fn size(&self) -> DeltaResult<usize>;
85 fn deletion_vector(&self) -> Option<DeletionVectorDescriptor>;
87 fn has_deletion_vector(&self) -> bool {
89 false
90 }
91}
92
93impl FileAction for Add {
94 fn partition_values(&self) -> DeltaResult<&HashMap<String, Option<String>>> {
95 Ok(&self.partition_values)
96 }
97
98 fn path(&self) -> String {
99 self.path.clone()
100 }
101
102 fn size(&self) -> DeltaResult<usize> {
103 Ok(self.size as usize)
104 }
105
106 fn deletion_vector(&self) -> Option<DeletionVectorDescriptor> {
107 self.deletion_vector.clone()
108 }
109
110 fn has_deletion_vector(&self) -> bool {
111 self.deletion_vector.is_some()
112 }
113}
114
115impl FileAction for AddCDCFile {
116 fn partition_values(&self) -> DeltaResult<&HashMap<String, Option<String>>> {
117 Ok(&self.partition_values)
118 }
119
120 fn path(&self) -> String {
121 self.path.clone()
122 }
123
124 fn size(&self) -> DeltaResult<usize> {
125 Ok(self.size as usize)
126 }
127
128 fn deletion_vector(&self) -> Option<DeletionVectorDescriptor> {
129 None
130 }
131}
132
133impl FileAction for Remove {
134 fn partition_values(&self) -> DeltaResult<&HashMap<String, Option<String>>> {
135 if self.extended_file_metadata.unwrap_or_default() {
137 Ok(self.partition_values.as_ref().unwrap())
138 } else {
139 match self.partition_values {
140 Some(ref part_map) => Ok(part_map),
141 _ => Err(crate::DeltaTableError::MetadataError(
142 "Remove action is missing required field: 'partition_values'".to_string(),
143 )),
144 }
145 }
146 }
147
148 fn path(&self) -> String {
149 self.path.clone()
150 }
151
152 fn size(&self) -> DeltaResult<usize> {
153 if self.extended_file_metadata.unwrap_or_default() {
155 Ok(self.size.unwrap() as usize)
156 } else {
157 match self.size {
158 Some(size) => Ok(size as usize),
159 _ => Err(crate::DeltaTableError::MetadataError(
160 "Remove action is missing required field: 'size'".to_string(),
161 )),
162 }
163 }
164 }
165
166 fn deletion_vector(&self) -> Option<DeletionVectorDescriptor> {
167 self.deletion_vector.clone()
168 }
169
170 fn has_deletion_vector(&self) -> bool {
171 self.deletion_vector.is_some()
172 }
173}