Skip to main content

deltalake_core/delta_datafusion/cdf/
mod.rs

1//! Logical operators and physical executions for CDF
2use std::collections::HashMap;
3use std::sync::LazyLock;
4
5use arrow_schema::{DataType, Field, TimeUnit};
6
7pub(crate) use self::scan_utils::*;
8use crate::DeltaResult;
9use crate::kernel::{Add, AddCDCFile, DeletionVectorDescriptor, Remove, Version};
10
11/// Scan-related types and helpers for reading Change Data Feed (CDF) batches.
12pub mod scan;
13mod scan_utils;
14
15/// Change type column name
16pub const CHANGE_TYPE_COL: &str = "_change_type";
17/// Commit version column name
18pub const COMMIT_VERSION_COL: &str = "_commit_version";
19/// Commit Timestamp column name
20pub const COMMIT_TIMESTAMP_COL: &str = "_commit_timestamp";
21
22pub(crate) static CDC_PARTITION_SCHEMA: LazyLock<Vec<Field>> = LazyLock::new(|| {
23    vec![
24        Field::new(COMMIT_VERSION_COL, DataType::UInt64, true),
25        Field::new(
26            COMMIT_TIMESTAMP_COL,
27            DataType::Timestamp(TimeUnit::Millisecond, None),
28            true,
29        ),
30    ]
31});
32pub(crate) static ADD_PARTITION_SCHEMA: LazyLock<Vec<Field>> = LazyLock::new(|| {
33    vec![
34        Field::new(CHANGE_TYPE_COL, DataType::Utf8, true),
35        Field::new(COMMIT_VERSION_COL, DataType::UInt64, true),
36        Field::new(
37            COMMIT_TIMESTAMP_COL,
38            DataType::Timestamp(TimeUnit::Millisecond, None),
39            true,
40        ),
41    ]
42});
43
44/// Sometimes when a data file is updated after a delete commit the data file is not re-written, but
45/// the deletion vector is updated. This manifests itself in the commit log as the data file being
46/// removed (with a DV) and then added back with a new DV. In this case, the difference of the two
47/// bitmaps is what is both deleted from this file and rows that haven't been touched.
48#[derive(Debug)]
49pub(crate) struct ResolvedPair {
50    pub version: Version,
51    pub timestamp: i64,
52    pub add: Add,
53    pub rm_dv: Option<DeletionVectorDescriptor>,
54}
55
56#[derive(Debug)]
57pub(crate) struct CdcDataSpec<F: FileAction> {
58    pub version: Version,
59    pub timestamp: i64,
60    pub actions: Vec<F>,
61}
62
63impl<F: FileAction> CdcDataSpec<F> {
64    pub fn new(version: Version, timestamp: i64, actions: Vec<F>) -> Self {
65        Self {
66            version,
67            timestamp,
68            actions,
69        }
70    }
71
72    pub fn into_parts(self) -> (Version, i64, Vec<F>) {
73        (self.version, self.timestamp, self.actions)
74    }
75}
76
77/// This trait defines a generic set of operations used by CDF Reader
78pub trait FileAction {
79    /// Adds partition values
80    fn partition_values(&self) -> DeltaResult<&HashMap<String, Option<String>>>;
81    /// Physical Path to the data
82    fn path(&self) -> String;
83    /// Byte size of the physical file
84    fn size(&self) -> DeltaResult<usize>;
85    /// Possibly provide the deletion vector for the action
86    fn deletion_vector(&self) -> Option<DeletionVectorDescriptor>;
87    /// Whether this file action contains a deletion vector
88    fn has_deletion_vector(&self) -> bool {
89        false
90    }
91}
92
93impl FileAction for Add {
94    fn partition_values(&self) -> DeltaResult<&HashMap<String, Option<String>>> {
95        Ok(&self.partition_values)
96    }
97
98    fn path(&self) -> String {
99        self.path.clone()
100    }
101
102    fn size(&self) -> DeltaResult<usize> {
103        Ok(self.size as usize)
104    }
105
106    fn deletion_vector(&self) -> Option<DeletionVectorDescriptor> {
107        self.deletion_vector.clone()
108    }
109
110    fn has_deletion_vector(&self) -> bool {
111        self.deletion_vector.is_some()
112    }
113}
114
115impl FileAction for AddCDCFile {
116    fn partition_values(&self) -> DeltaResult<&HashMap<String, Option<String>>> {
117        Ok(&self.partition_values)
118    }
119
120    fn path(&self) -> String {
121        self.path.clone()
122    }
123
124    fn size(&self) -> DeltaResult<usize> {
125        Ok(self.size as usize)
126    }
127
128    fn deletion_vector(&self) -> Option<DeletionVectorDescriptor> {
129        None
130    }
131}
132
133impl FileAction for Remove {
134    fn partition_values(&self) -> DeltaResult<&HashMap<String, Option<String>>> {
135        // If extended_file_metadata is true, it should be required to have this filled in
136        if self.extended_file_metadata.unwrap_or_default() {
137            Ok(self.partition_values.as_ref().unwrap())
138        } else {
139            match self.partition_values {
140                Some(ref part_map) => Ok(part_map),
141                _ => Err(crate::DeltaTableError::MetadataError(
142                    "Remove action is missing required field: 'partition_values'".to_string(),
143                )),
144            }
145        }
146    }
147
148    fn path(&self) -> String {
149        self.path.clone()
150    }
151
152    fn size(&self) -> DeltaResult<usize> {
153        // If extended_file_metadata is true, it should be required to have this filled in
154        if self.extended_file_metadata.unwrap_or_default() {
155            Ok(self.size.unwrap() as usize)
156        } else {
157            match self.size {
158                Some(size) => Ok(size as usize),
159                _ => Err(crate::DeltaTableError::MetadataError(
160                    "Remove action is missing required field: 'size'".to_string(),
161                )),
162            }
163        }
164    }
165
166    fn deletion_vector(&self) -> Option<DeletionVectorDescriptor> {
167        self.deletion_vector.clone()
168    }
169
170    fn has_deletion_vector(&self) -> bool {
171        self.deletion_vector.is_some()
172    }
173}