Skip to main content

deltalake_core/kernel/models/
actions.rs

1use std::collections::{HashMap, HashSet};
2use std::fmt::{self, Display};
3use std::str::FromStr;
4
5use delta_kernel::schema::{DataType, StructField};
6use delta_kernel::table_features::TableFeature;
7use serde::{Deserialize, Serialize};
8
9use crate::TableProperty;
10use crate::kernel::{DeltaResult, error::Error};
11use crate::kernel::{StructType, StructTypeExt, Version};
12
13pub use delta_kernel::actions::{Metadata, Protocol};
14
15/// Please don't use, this API will be leaving shortly!
16///
17/// Since the adoption of delta-kernel-rs we lost the direct ability to create [Metadata] actions
18/// which is required for some use-cases.
19///
20/// Upstream tracked here: <https://github.com/delta-io/delta-kernel-rs/issues/1055>
21pub fn new_metadata(
22    schema: &StructType,
23    partition_columns: impl IntoIterator<Item = impl ToString>,
24    configuration: impl IntoIterator<Item = (impl ToString, impl ToString)>,
25) -> DeltaResult<Metadata> {
26    // this ugliness is a stop-gap until we resolve: https://github.com/delta-io/delta-kernel-rs/issues/1055
27    let value = serde_json::json!({
28        "id": uuid::Uuid::new_v4().to_string(),
29        "name": None::<String>,
30        "description": None::<String>,
31        "format": { "provider": "parquet", "options": {} },
32        "schemaString": serde_json::to_string(schema)?,
33        "partitionColumns": partition_columns.into_iter().map(|c| c.to_string()).collect::<Vec<_>>(),
34        "configuration": configuration.into_iter().map(|(k, v)| (k.to_string(), v.to_string())).collect::<HashMap<_, _>>(),
35        "createdTime": chrono::Utc::now().timestamp_millis(),
36    });
37    Ok(serde_json::from_value(value)?)
38}
39
40/// Extension trait for Metadata action
41///
42/// This trait is a stop-gap to adopt the Metadata action from delta-kernel-rs
43/// while the update / mutation APIs are being implemented. It allows us to implement
44/// additional APIs on the Metadata action and hide specifics of how we do the updates.
45pub trait MetadataExt {
46    /// Return a copy of the metadata with its unique table identifier replaced.
47    fn with_table_id(self, table_id: String) -> DeltaResult<Metadata>;
48
49    /// Return a copy of the metadata with the user-facing table name set.
50    fn with_name(self, name: String) -> DeltaResult<Metadata>;
51
52    /// Return a copy of the metadata with the table description set.
53    fn with_description(self, description: String) -> DeltaResult<Metadata>;
54
55    /// Return a copy of the metadata whose schema string is replaced with `schema`.
56    fn with_schema(self, schema: &StructType) -> DeltaResult<Metadata>;
57
58    /// Return a copy of the metadata with a single configuration key set to `value`.
59    fn add_config_key(self, key: String, value: String) -> DeltaResult<Metadata>;
60
61    /// Return a copy of the metadata with the given configuration key removed.
62    fn remove_config_key(self, key: &str) -> DeltaResult<Metadata>;
63}
64
65impl MetadataExt for Metadata {
66    fn with_table_id(self, table_id: String) -> DeltaResult<Metadata> {
67        let value = serde_json::json!({
68            "id": table_id,
69            "name": self.name(),
70            "description": self.description(),
71            "format": { "provider": "parquet", "options": {} },
72            "schemaString": serde_json::to_string(&self.parse_schema().unwrap())?,
73            "partitionColumns": self.partition_columns(),
74            "configuration": self.configuration(),
75            "createdTime": self.created_time(),
76        });
77        Ok(serde_json::from_value(value)?)
78    }
79
80    fn with_name(self, name: String) -> DeltaResult<Metadata> {
81        let value = serde_json::json!({
82            "id": self.id(),
83            "name": name,
84            "description": self.description(),
85            "format": { "provider": "parquet", "options": {} },
86            "schemaString": serde_json::to_string(&self.parse_schema().unwrap())?,
87            "partitionColumns": self.partition_columns(),
88            "configuration": self.configuration(),
89            "createdTime": self.created_time(),
90        });
91        Ok(serde_json::from_value(value)?)
92    }
93
94    fn with_description(self, description: String) -> DeltaResult<Metadata> {
95        let value = serde_json::json!({
96            "id": self.id(),
97            "name": self.name(),
98            "description": description,
99            "format": { "provider": "parquet", "options": {} },
100            "schemaString": serde_json::to_string(&self.parse_schema().unwrap())?,
101            "partitionColumns": self.partition_columns(),
102            "configuration": self.configuration(),
103            "createdTime": self.created_time(),
104        });
105        Ok(serde_json::from_value(value)?)
106    }
107
108    fn with_schema(self, schema: &StructType) -> DeltaResult<Metadata> {
109        let value = serde_json::json!({
110            "id": self.id(),
111            "name": self.name(),
112            "description": self.description(),
113            "format": { "provider": "parquet", "options": {} },
114            "schemaString": serde_json::to_string(schema)?,
115            "partitionColumns": self.partition_columns(),
116            "configuration": self.configuration(),
117            "createdTime": self.created_time(),
118        });
119        Ok(serde_json::from_value(value)?)
120    }
121
122    fn add_config_key(self, key: String, value: String) -> DeltaResult<Metadata> {
123        let mut config = self.configuration().clone();
124        config.insert(key, value);
125        let value = serde_json::json!({
126            "id": self.id(),
127            "name": self.name(),
128            "description": self.description(),
129            "format": { "provider": "parquet", "options": {} },
130            "schemaString": serde_json::to_string(&self.parse_schema().unwrap())?,
131            "partitionColumns": self.partition_columns(),
132            "configuration": config,
133            "createdTime": self.created_time(),
134        });
135        Ok(serde_json::from_value(value)?)
136    }
137
138    fn remove_config_key(self, key: &str) -> DeltaResult<Metadata> {
139        let mut config = self.configuration().clone();
140        config.remove(key);
141        let value = serde_json::json!({
142            "id": self.id(),
143            "name": self.name(),
144            "description": self.description(),
145            "format": { "provider": "parquet", "options": {} },
146            "schemaString": serde_json::to_string(&self.parse_schema().unwrap())?,
147            "partitionColumns": self.partition_columns(),
148            "configuration": config,
149            "createdTime": self.created_time(),
150        });
151        Ok(serde_json::from_value(value)?)
152    }
153}
154
155/// Checks if a datatype matches another type, or any of its inner fields match.
156fn matches_datatype(dtype_to_check: &DataType, dtype: &DataType) -> bool {
157    match dtype_to_check {
158        to_check if dtype == to_check => true,
159        DataType::Array(inner) => matches_datatype(inner.element_type(), dtype),
160        DataType::Struct(inner) => inner
161            .fields()
162            .any(|f| matches_datatype(f.data_type(), dtype)),
163        DataType::Map(inner) => {
164            matches_datatype(inner.key_type(), dtype) || matches_datatype(inner.value_type(), dtype)
165        }
166        _ => false,
167    }
168}
169
170/// checks if table contains timestamp_ntz in any field including nested fields.
171pub fn contains_timestampntz<'a>(mut fields: impl Iterator<Item = &'a StructField>) -> bool {
172    fields.any(|f| matches_datatype(f.data_type(), &DataType::TIMESTAMP_NTZ))
173}
174
175#[cfg(feature = "nanosecond-timestamps")]
176/// checks if table contains timestamp_nanos or timestamp_nanos_ntz in any
177/// field including nested fields. Both primitive types require the same
178/// `timestampNanos` table feature.
179pub fn contains_timestamp_nanos<'a>(mut fields: impl Iterator<Item = &'a StructField>) -> bool {
180    fields.any(|f| {
181        matches_datatype(f.data_type(), &DataType::TIMESTAMP_NANOS)
182            || matches_datatype(f.data_type(), &DataType::TIMESTAMP_NANOS_NTZ)
183    })
184}
185
186/// checks if table contains variant in any field including nested fields.
187pub(crate) fn contains_variant<'a>(mut fields: impl Iterator<Item = &'a StructField>) -> bool {
188    fn _check_type(dtype: &DataType) -> bool {
189        match dtype {
190            DataType::Variant(_) => true,
191            DataType::Array(inner) => _check_type(inner.element_type()),
192            DataType::Struct(inner) => inner.fields().any(|f| _check_type(f.data_type())),
193            DataType::Map(inner) => {
194                _check_type(inner.key_type()) || _check_type(inner.value_type())
195            }
196            _ => false,
197        }
198    }
199
200    fields.any(|f| _check_type(f.data_type()))
201}
202
203/// Extension trait for delta-kernel Protocol action.
204///
205/// Allows us to extend the Protocol struct with additional methods
206/// to update the protocol actions.
207pub(crate) trait ProtocolExt {
208    fn reader_features_set(&self) -> Option<HashSet<TableFeature>>;
209    fn writer_features_set(&self) -> Option<HashSet<TableFeature>>;
210    fn append_reader_features(self, reader_features: &[TableFeature]) -> Protocol;
211    fn append_writer_features(self, writer_features: &[TableFeature]) -> Protocol;
212    fn move_table_properties_into_features(
213        self,
214        configuration: &HashMap<String, String>,
215    ) -> Protocol;
216    fn apply_column_metadata_to_protocol(self, schema: &StructType) -> DeltaResult<Protocol>;
217    fn apply_properties_to_protocol(
218        self,
219        new_properties: &HashMap<String, String>,
220        raise_if_not_exists: bool,
221    ) -> DeltaResult<Protocol>;
222}
223
224impl ProtocolExt for Protocol {
225    fn reader_features_set(&self) -> Option<HashSet<TableFeature>> {
226        self.reader_features()
227            .map(|features| features.iter().cloned().collect())
228    }
229
230    fn writer_features_set(&self) -> Option<HashSet<TableFeature>> {
231        self.writer_features()
232            .map(|features| features.iter().cloned().collect())
233    }
234
235    fn append_reader_features(self, reader_features: &[TableFeature]) -> Protocol {
236        let mut inner = ProtocolInner::from_kernel(&self);
237        inner = inner.append_reader_features(reader_features.iter().cloned());
238        inner.as_kernel()
239    }
240
241    fn append_writer_features(self, writer_features: &[TableFeature]) -> Protocol {
242        let mut inner = ProtocolInner::from_kernel(&self);
243        inner = inner.append_writer_features(writer_features.iter().cloned());
244        inner.as_kernel()
245    }
246
247    fn move_table_properties_into_features(
248        self,
249        configuration: &HashMap<String, String>,
250    ) -> Protocol {
251        let mut inner = ProtocolInner::from_kernel(&self);
252        inner = inner.move_table_properties_into_features(configuration);
253        inner.as_kernel()
254    }
255
256    fn apply_column_metadata_to_protocol(self, schema: &StructType) -> DeltaResult<Protocol> {
257        let mut inner = ProtocolInner::from_kernel(&self);
258        inner = inner.apply_column_metadata_to_protocol(schema)?;
259        Ok(inner.as_kernel())
260    }
261
262    fn apply_properties_to_protocol(
263        self,
264        new_properties: &HashMap<String, String>,
265        raise_if_not_exists: bool,
266    ) -> DeltaResult<Protocol> {
267        let mut inner = ProtocolInner::from_kernel(&self);
268        inner = inner.apply_properties_to_protocol(new_properties, raise_if_not_exists)?;
269        Ok(inner.as_kernel())
270    }
271}
272
273/// Features that legacy protocol versions above delta-rs's default `(1, 2)` imply, with the lowest
274/// reader and writer versions that imply each one, as in the "Reader Version Requirements" and
275/// "Writer Version Requirements" tables of PROTOCOL.md. `appendOnly` and `invariants` (writer
276/// version 2) are left out so that tables upgraded from the default keep their current feature
277/// lists; [`writer_features_for_version`] adds both when properties move a table to writer 7.
278const LEGACY_IMPLIED_FEATURES: [(i32, i32, TableFeature); 5] = [
279    (1, 3, TableFeature::CheckConstraints),
280    (1, 4, TableFeature::ChangeDataFeed),
281    (1, 4, TableFeature::GeneratedColumns),
282    (2, 5, TableFeature::ColumnMapping),
283    (1, 6, TableFeature::IdentityColumns),
284];
285
286#[derive(Serialize, Deserialize, Debug, Clone, PartialEq, Eq)]
287#[serde(rename_all = "camelCase")]
288/// Temporary Shim to facilitate adoption of kernel protocol.
289///
290/// This is more or less our old local implementation of protocol. We keep it around
291/// to use the various update and translation methods defined in this struct and
292/// use it to proxy updates to the kernel protocol action.
293///
294// TODO: Remove once we can use kernel protocol update APIs.
295pub(crate) struct ProtocolInner {
296    /// The minimum version of the Delta read protocol that a client must implement
297    /// in order to correctly read this table
298    pub min_reader_version: i32,
299    /// The minimum version of the Delta write protocol that a client must implement
300    /// in order to correctly write this table
301    pub min_writer_version: i32,
302    /// A collection of features that a client must implement in order to correctly
303    /// read this table (exist only when minReaderVersion is set to 3)
304    #[serde(skip_serializing_if = "Option::is_none")]
305    pub reader_features: Option<HashSet<TableFeature>>,
306    /// A collection of features that a client must implement in order to correctly
307    /// write this table (exist only when minWriterVersion is set to 7)
308    #[serde(skip_serializing_if = "Option::is_none")]
309    pub writer_features: Option<HashSet<TableFeature>>,
310}
311
312impl Default for ProtocolInner {
313    fn default() -> Self {
314        Self {
315            min_reader_version: 1,
316            min_writer_version: 2,
317            reader_features: None,
318            writer_features: None,
319        }
320    }
321}
322
323impl ProtocolInner {
324    /// Create a new protocol action
325    #[cfg(test)]
326    pub(crate) fn new(min_reader_version: i32, min_writer_version: i32) -> Self {
327        Self {
328            min_reader_version,
329            min_writer_version,
330            reader_features: None,
331            writer_features: None,
332        }
333    }
334
335    pub(crate) fn from_kernel(value: &Protocol) -> ProtocolInner {
336        // this ugliness is a stop-gap until we resolve: https://github.com/delta-io/delta-kernel-rs/issues/1055
337        serde_json::from_value(serde_json::to_value(value).unwrap()).unwrap()
338    }
339
340    pub(crate) fn as_kernel(&self) -> Protocol {
341        // this ugliness is a stop-gap until we resolve: https://github.com/delta-io/delta-kernel-rs/issues/1055
342        serde_json::from_value(serde_json::to_value(self).unwrap())
343            .expect("Failed to convert Protocol to a kernel implementation")
344    }
345
346    /// Moves a legacy reader protocol to reader version 3, listing the reader features that its
347    /// version implied, so that moving to table features keeps them supported
348    fn upgrade_to_reader_features(&mut self) {
349        if self.min_reader_version >= 3 {
350            return;
351        }
352        let supports_column_mapping = if self.min_reader_version < 2 {
353            false
354        } else if self.min_writer_version < 7 {
355            self.min_writer_version >= 5
356        } else {
357            self.writer_features
358                .as_ref()
359                .is_some_and(|features| features.contains(&TableFeature::ColumnMapping))
360        };
361        let reader_features = self.reader_features.get_or_insert_with(HashSet::new);
362        if supports_column_mapping {
363            reader_features.insert(TableFeature::ColumnMapping);
364        }
365        self.min_reader_version = 3;
366    }
367
368    /// Moves a legacy writer protocol to writer version 7, listing the writer features that its
369    /// versions implied, so that moving to table features keeps them supported
370    fn upgrade_to_writer_features(&mut self) {
371        if self.min_writer_version >= 7 {
372            return;
373        }
374        let writer_features = self.writer_features.get_or_insert_with(HashSet::new);
375        for (min_reader_version, min_writer_version, feature) in &LEGACY_IMPLIED_FEATURES {
376            if self.min_writer_version < *min_writer_version {
377                continue;
378            }
379            let reader_supports = if self.min_reader_version < 3 {
380                self.min_reader_version >= *min_reader_version
381            } else if *min_reader_version == 1 {
382                true
383            } else {
384                self.reader_features
385                    .as_ref()
386                    .is_some_and(|features| features.contains(feature))
387            };
388            if reader_supports {
389                writer_features.insert(feature.clone());
390            }
391        }
392        self.min_writer_version = 7;
393    }
394
395    /// Append the reader features in the protocol action, automatically bumps min_reader_version
396    pub fn append_reader_features(
397        mut self,
398        reader_features: impl IntoIterator<Item = impl Into<TableFeature>>,
399    ) -> Self {
400        let all_reader_features = reader_features
401            .into_iter()
402            .map(Into::into)
403            .collect::<HashSet<_>>();
404        if !all_reader_features.is_empty() {
405            self.upgrade_to_reader_features();
406            match self.reader_features {
407                Some(mut features) => {
408                    features.extend(all_reader_features);
409                    self.reader_features = Some(features);
410                }
411                None => self.reader_features = Some(all_reader_features),
412            };
413        }
414        self
415    }
416
417    /// Append the writer features in the protocol action, automatically bumps min_writer_version
418    pub fn append_writer_features(
419        mut self,
420        writer_features: impl IntoIterator<Item = impl Into<TableFeature>>,
421    ) -> Self {
422        let all_writer_features = writer_features
423            .into_iter()
424            .map(|c| c.into())
425            .collect::<HashSet<_>>();
426        if !all_writer_features.is_empty() {
427            self.upgrade_to_writer_features();
428            match self.writer_features {
429                Some(mut features) => {
430                    features.extend(all_writer_features);
431                    self.writer_features = Some(features);
432                }
433                None => self.writer_features = Some(all_writer_features),
434            };
435        }
436        self
437    }
438
439    /// Converts existing properties into features if the reader_version is >=3 or writer_version >=7
440    /// only converts features that are "true"
441    pub fn move_table_properties_into_features(
442        mut self,
443        configuration: &HashMap<String, String>,
444    ) -> Self {
445        fn parse_bool(value: &str) -> bool {
446            value.to_ascii_lowercase().parse::<bool>().is_ok_and(|v| v)
447        }
448
449        if self.min_writer_version >= 7 {
450            // TODO: move this is in future to use delta_kernel::table_properties
451            let mut converted_writer_features = configuration
452                .iter()
453                .filter(|(_, value)| value.to_ascii_lowercase().parse::<bool>().is_ok_and(|v| v))
454                .filter_map(|(key, value)| match key.as_str() {
455                    "delta.enableChangeDataFeed" if parse_bool(value) => {
456                        Some(TableFeature::ChangeDataFeed)
457                    }
458                    "delta.appendOnly" if parse_bool(value) => Some(TableFeature::AppendOnly),
459                    "delta.enableDeletionVectors" if parse_bool(value) => {
460                        Some(TableFeature::DeletionVectors)
461                    }
462                    "delta.enableRowTracking" if parse_bool(value) => {
463                        Some(TableFeature::RowTracking)
464                    }
465                    "delta.checkpointPolicy" if value == "v2" => Some(TableFeature::V2Checkpoint),
466                    _ => None,
467                })
468                .collect::<HashSet<TableFeature>>();
469
470            if configuration
471                .keys()
472                .any(|v| v.starts_with("delta.constraints."))
473            {
474                converted_writer_features.insert(TableFeature::CheckConstraints);
475            }
476
477            match self.writer_features {
478                Some(mut features) => {
479                    features.extend(converted_writer_features);
480                    self.writer_features = Some(features);
481                }
482                None => self.writer_features = Some(converted_writer_features),
483            }
484        }
485        if self.min_reader_version >= 3 {
486            let converted_reader_features = configuration
487                .iter()
488                .filter_map(|(key, value)| match key.as_str() {
489                    "delta.enableDeletionVectors" if parse_bool(value) => {
490                        Some(TableFeature::DeletionVectors)
491                    }
492                    "delta.checkpointPolicy" if value == "v2" => Some(TableFeature::V2Checkpoint),
493                    _ => None,
494                })
495                .collect::<HashSet<TableFeature>>();
496            match self.reader_features {
497                Some(mut features) => {
498                    features.extend(converted_reader_features);
499                    self.reader_features = Some(features);
500                }
501                None => self.reader_features = Some(converted_reader_features),
502            }
503        }
504        self
505    }
506
507    /// Will apply the column metadata to the protocol by either bumping the version or setting
508    /// features
509    pub fn apply_column_metadata_to_protocol(mut self, schema: &StructType) -> DeltaResult<Self> {
510        let generated_cols = schema.get_generated_columns()?;
511        let invariants = schema.get_invariants()?;
512        let contains_timestamp_ntz = self.contains_timestampntz(schema.fields());
513        #[cfg(feature = "nanosecond-timestamps")]
514        let contains_timestamp_nanos = self.contains_timestamp_nanos(schema.fields());
515        let contains_variant = self.contains_variant(schema.fields());
516
517        if contains_timestamp_ntz {
518            self = self.enable_timestamp_ntz()
519        }
520
521        #[cfg(feature = "nanosecond-timestamps")]
522        if contains_timestamp_nanos {
523            // Per the protocol RFC, non-timezone timestamps need to be required
524            // too, since there will eventually be a
525            // nanoseconds-without-timezones primitive type too.
526            self = self.enable_timestamp_nanos().enable_timestamp_ntz()
527        }
528        if contains_variant {
529            self = self.enable_variant_type()
530        }
531
532        if !generated_cols.is_empty() {
533            self = self.enable_generated_columns()
534        }
535
536        if !invariants.is_empty() {
537            self = self.enable_invariants()
538        }
539
540        Ok(self)
541    }
542
543    /// Will apply the properties to the protocol by either bumping the version or setting
544    /// features
545    pub fn apply_properties_to_protocol(
546        mut self,
547        new_properties: &HashMap<String, String>,
548        raise_if_not_exists: bool,
549    ) -> DeltaResult<Self> {
550        let mut parsed_properties: HashMap<TableProperty, String> = HashMap::new();
551
552        for (key, value) in new_properties {
553            if let Ok(parsed_key) = key.parse::<TableProperty>() {
554                parsed_properties.insert(parsed_key, value.to_string());
555            } else if raise_if_not_exists {
556                return Err(Error::Generic(format!(
557                    "Error parsing property '{key}':'{value}'",
558                )));
559            }
560        }
561
562        // Check and update delta.minWriterVersion.
563        if let Some(min_writer_version) = parsed_properties.get(&TableProperty::MinWriterVersion) {
564            let new_min_writer_version = min_writer_version.parse::<i32>();
565            match new_min_writer_version {
566                Ok(version) => match version {
567                    7 => self.upgrade_to_writer_features(),
568                    2..=6 => {
569                        if version > self.min_writer_version {
570                            self.min_writer_version = version
571                        }
572                    }
573                    _ => {
574                        return Err(Error::Generic(format!(
575                            "delta.minWriterVersion = '{min_writer_version}' is invalid, valid values are ['2','3','4','5','6','7']"
576                        )));
577                    }
578                },
579                Err(_) => {
580                    return Err(Error::Generic(format!(
581                        "delta.minWriterVersion = '{min_writer_version}' is invalid, valid values are ['2','3','4','5','6','7']"
582                    )));
583                }
584            }
585        }
586
587        // Check and update delta.minReaderVersion
588        if let Some(min_reader_version) = parsed_properties.get(&TableProperty::MinReaderVersion) {
589            let new_min_reader_version = min_reader_version.parse::<i32>();
590            match new_min_reader_version {
591                Ok(version) => match version {
592                    3 => self.upgrade_to_reader_features(),
593                    1..=2 => {
594                        if version > self.min_reader_version {
595                            self.min_reader_version = version
596                        }
597                    }
598                    _ => {
599                        return Err(Error::Generic(format!(
600                            "delta.minReaderVersion = '{min_reader_version}' is invalid, valid values are ['1','2','3']"
601                        )));
602                    }
603                },
604                Err(_) => {
605                    return Err(Error::Generic(format!(
606                        "delta.minReaderVersion = '{min_reader_version}' is invalid, valid values are ['1','2','3']"
607                    )));
608                }
609            }
610        }
611
612        // Check columnMappingMode and bump protocol or add reader/writerFeatures
613        if let Some(mode) = parsed_properties.get(&TableProperty::ColumnMappingMode) {
614            if mode.as_str() != "none" {
615                if self.min_reader_version >= 3 {
616                    self.reader_features
617                        .get_or_insert_with(HashSet::new)
618                        .insert(TableFeature::ColumnMapping);
619                } else {
620                    self.min_reader_version = self.min_reader_version.max(2);
621                }
622                if self.min_writer_version >= 7 {
623                    self.writer_features
624                        .get_or_insert_with(HashSet::new)
625                        .insert(TableFeature::ColumnMapping);
626                } else {
627                    self.min_writer_version = self.min_writer_version.max(5);
628                }
629            }
630        }
631
632        // Check enableChangeDataFeed and bump protocol or add writerFeature if writer versions is >=7
633        if let Some(enable_cdf) = parsed_properties.get(&TableProperty::EnableChangeDataFeed) {
634            let if_enable_cdf = enable_cdf.to_ascii_lowercase().parse::<bool>();
635            match if_enable_cdf {
636                Ok(true) => {
637                    if self.min_writer_version >= 7 {
638                        match self.writer_features {
639                            Some(mut features) => {
640                                features.insert(TableFeature::ChangeDataFeed);
641                                self.writer_features = Some(features);
642                            }
643                            None => {
644                                self.writer_features =
645                                    Some(HashSet::from([TableFeature::ChangeDataFeed]))
646                            }
647                        }
648                    } else if self.min_writer_version <= 3 {
649                        self.min_writer_version = 4
650                    }
651                }
652                Ok(false) => {}
653                _ => {
654                    return Err(Error::Generic(format!(
655                        "delta.enableChangeDataFeed = '{enable_cdf}' is invalid, valid values are ['true']"
656                    )));
657                }
658            }
659        }
660
661        if let Some(enable_dv) = parsed_properties.get(&TableProperty::EnableDeletionVectors) {
662            let if_enable_dv = enable_dv.to_ascii_lowercase().parse::<bool>();
663            match if_enable_dv {
664                Ok(true) => {
665                    self.upgrade_to_reader_features();
666                    self.upgrade_to_writer_features();
667                    self.reader_features
668                        .get_or_insert_with(HashSet::new)
669                        .insert(TableFeature::DeletionVectors);
670                    self.writer_features
671                        .get_or_insert_with(HashSet::new)
672                        .insert(TableFeature::DeletionVectors);
673                }
674                Ok(false) => {}
675                _ => {
676                    return Err(Error::Generic(format!(
677                        "delta.enableDeletionVectors = '{enable_dv}' is invalid, valid values are ['true']"
678                    )));
679                }
680            }
681        }
682
683        // If the minWriterVersion has been set to 7 then there are additional writer features
684        // which must be applied
685        if self.min_writer_version >= 7
686            && let Some(features) = writer_features_for_version(self.min_writer_version)
687        {
688            self = self.append_writer_features(features);
689            // When setting the minWriterVersion the minReaderVersion must necessarily come up to
690            // version 2 if it's not already there
691            if self.min_reader_version < 2 {
692                self.min_reader_version = 2;
693            }
694        }
695        // minReaderVersion 3 needs a readerFeatures list, but it lists only the features the table
696        // uses, so it can be empty
697        if self.min_reader_version >= 3 && self.reader_features.is_none() {
698            self.reader_features = Some(HashSet::new());
699        }
700        Ok(self)
701    }
702
703    /// checks if table contains timestamp_ntz in any field including nested fields.
704    fn contains_timestampntz<'a>(&self, fields: impl Iterator<Item = &'a StructField>) -> bool {
705        contains_timestampntz(fields)
706    }
707
708    /// checks if table contains variant in any field including nested fields.
709    fn contains_variant<'a>(&self, fields: impl Iterator<Item = &'a StructField>) -> bool {
710        contains_variant(fields)
711    }
712
713    /// Enable timestamp_ntz in the protocol
714    fn enable_timestamp_ntz(mut self) -> Self {
715        self = self.append_reader_features([TableFeature::TimestampWithoutTimezone]);
716        self = self.append_writer_features([TableFeature::TimestampWithoutTimezone]);
717        self
718    }
719
720    #[cfg(feature = "nanosecond-timestamps")]
721    /// checks if table contains timestamp_nanos in any field including nested fields.
722    fn contains_timestamp_nanos<'a>(&self, fields: impl Iterator<Item = &'a StructField>) -> bool {
723        contains_timestamp_nanos(fields)
724    }
725
726    #[cfg(feature = "nanosecond-timestamps")]
727    /// Enable timestamp_nanos in the protocol
728    fn enable_timestamp_nanos(mut self) -> Self {
729        self = self.append_reader_features([TableFeature::TimestampNanos]);
730        self = self.append_writer_features([TableFeature::TimestampNanos]);
731        self
732    }
733
734    /// Enable variantType in the protocol
735    fn enable_variant_type(mut self) -> Self {
736        self = self.append_reader_features([TableFeature::VariantType]);
737        self = self.append_writer_features([TableFeature::VariantType]);
738        self
739    }
740
741    /// Enabled generated columns
742    fn enable_generated_columns(mut self) -> Self {
743        if self.min_writer_version < 4 {
744            self.min_writer_version = 4;
745        }
746        if self.min_writer_version >= 7 {
747            self = self.append_writer_features([TableFeature::GeneratedColumns]);
748        }
749        self
750    }
751
752    /// Enabled generated columns
753    fn enable_invariants(mut self) -> Self {
754        if self.min_writer_version >= 7 {
755            self = self.append_writer_features([TableFeature::Invariants]);
756        }
757        self
758    }
759}
760
761/// High level table features
762#[derive(Serialize, Deserialize, Debug, Clone, Eq, PartialEq, Hash)]
763#[serde(rename_all = "camelCase")]
764pub enum TableFeatures {
765    /// Mapping of one column to another
766    ColumnMapping,
767    /// Deletion vectors for merge, update, delete
768    DeletionVectors,
769    /// timestamps without timezone support
770    #[serde(rename = "timestampNtz")]
771    TimestampWithoutTimezone,
772    #[cfg(feature = "nanosecond-timestamps")]
773    #[serde(rename = "timestampNanos")]
774    /// Timestamps that are nanosecond resolution
775    TimestampNanos,
776    /// version 2 of checkpointing
777    V2Checkpoint,
778    /// Append Only Tables
779    AppendOnly,
780    /// Table invariants
781    Invariants,
782    /// Check constraints on columns
783    CheckConstraints,
784    /// CDF on a table
785    ChangeDataFeed,
786    /// Columns with generated values
787    GeneratedColumns,
788    /// ID Columns
789    IdentityColumns,
790    /// Row tracking on tables
791    RowTracking,
792    /// domain specific metadata
793    DomainMetadata,
794    /// Iceberg compatibility support
795    IcebergCompatV1,
796    /// Variant type support
797    VariantType,
798    /// Preview variant type support
799    VariantTypePreview,
800    /// Preview shredded variant support
801    VariantShreddingPreview,
802    /// Support for materializing partition column values into data files.
803    MaterializePartitionColumns,
804}
805
806impl FromStr for TableFeatures {
807    type Err = ();
808
809    fn from_str(value: &str) -> Result<Self, Self::Err> {
810        match value {
811            "columnMapping" => Ok(TableFeatures::ColumnMapping),
812            "deletionVectors" => Ok(TableFeatures::DeletionVectors),
813            "timestampNtz" => Ok(TableFeatures::TimestampWithoutTimezone),
814            #[cfg(feature = "nanosecond-timestamps")]
815            "timestampNanos" => Ok(TableFeatures::TimestampNanos),
816            "v2Checkpoint" => Ok(TableFeatures::V2Checkpoint),
817            "appendOnly" => Ok(TableFeatures::AppendOnly),
818            "invariants" => Ok(TableFeatures::Invariants),
819            "checkConstraints" => Ok(TableFeatures::CheckConstraints),
820            "changeDataFeed" => Ok(TableFeatures::ChangeDataFeed),
821            "generatedColumns" => Ok(TableFeatures::GeneratedColumns),
822            "identityColumns" => Ok(TableFeatures::IdentityColumns),
823            "rowTracking" => Ok(TableFeatures::RowTracking),
824            "domainMetadata" => Ok(TableFeatures::DomainMetadata),
825            "icebergCompatV1" => Ok(TableFeatures::IcebergCompatV1),
826            "variantType" => Ok(TableFeatures::VariantType),
827            "variantType-preview" => Ok(TableFeatures::VariantTypePreview),
828            "variantShredding-preview" => Ok(TableFeatures::VariantShreddingPreview),
829            "materializePartitionColumns" => Ok(TableFeatures::MaterializePartitionColumns),
830            _ => Err(()),
831        }
832    }
833}
834
835impl AsRef<str> for TableFeatures {
836    fn as_ref(&self) -> &str {
837        match self {
838            TableFeatures::ColumnMapping => "columnMapping",
839            TableFeatures::DeletionVectors => "deletionVectors",
840            TableFeatures::TimestampWithoutTimezone => "timestampNtz",
841            #[cfg(feature = "nanosecond-timestamps")]
842            TableFeatures::TimestampNanos => "timestampNanos",
843            TableFeatures::V2Checkpoint => "v2Checkpoint",
844            TableFeatures::AppendOnly => "appendOnly",
845            TableFeatures::Invariants => "invariants",
846            TableFeatures::CheckConstraints => "checkConstraints",
847            TableFeatures::ChangeDataFeed => "changeDataFeed",
848            TableFeatures::GeneratedColumns => "generatedColumns",
849            TableFeatures::IdentityColumns => "identityColumns",
850            TableFeatures::RowTracking => "rowTracking",
851            TableFeatures::DomainMetadata => "domainMetadata",
852            TableFeatures::IcebergCompatV1 => "icebergCompatV1",
853            TableFeatures::VariantType => "variantType",
854            TableFeatures::VariantTypePreview => "variantType-preview",
855            TableFeatures::VariantShreddingPreview => "variantShredding-preview",
856            TableFeatures::MaterializePartitionColumns => "materializePartitionColumns",
857        }
858    }
859}
860
861impl fmt::Display for TableFeatures {
862    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
863        write!(f, "{}", self.as_ref())
864    }
865}
866
867impl TryFrom<&TableFeatures> for TableFeature {
868    type Error = std::convert::Infallible;
869
870    fn try_from(value: &TableFeatures) -> Result<Self, Self::Error> {
871        TableFeature::try_from(value.as_ref())
872    }
873}
874
875impl TableFeatures {
876    /// Convert table feature to respective reader or/and write feature
877    pub fn to_reader_writer_features(&self) -> (Option<TableFeature>, Option<TableFeature>) {
878        let feature = TableFeature::try_from(self).ok();
879        match feature {
880            Some(feature) => {
881                // Classify features based on their type
882                // Writer-only features
883                match feature {
884                    TableFeature::AppendOnly
885                    | TableFeature::Invariants
886                    | TableFeature::CheckConstraints
887                    | TableFeature::ChangeDataFeed
888                    | TableFeature::GeneratedColumns
889                    | TableFeature::IdentityColumns
890                    | TableFeature::InCommitTimestamp
891                    | TableFeature::RowTracking
892                    | TableFeature::DomainMetadata
893                    | TableFeature::IcebergCompatV1
894                    | TableFeature::IcebergCompatV2
895                    | TableFeature::ClusteredTable
896                    | TableFeature::MaterializePartitionColumns => (None, Some(feature)),
897
898                    // ReaderWriter features
899                    TableFeature::CatalogManaged
900                    | TableFeature::CatalogOwnedPreview
901                    | TableFeature::ColumnMapping
902                    | TableFeature::DeletionVectors
903                    | TableFeature::TimestampWithoutTimezone
904                    | TableFeature::TypeWidening
905                    | TableFeature::TypeWideningPreview
906                    | TableFeature::V2Checkpoint
907                    | TableFeature::VacuumProtocolCheck
908                    | TableFeature::VariantType
909                    | TableFeature::VariantTypePreview
910                    | TableFeature::VariantShreddingPreview => {
911                        (Some(feature.clone()), Some(feature))
912                    }
913
914                    // Optional ReaderWriter features
915                    #[cfg(feature = "nanosecond-timestamps")]
916                    TableFeature::TimestampNanos => (Some(feature.clone()), Some(feature)),
917
918                    // Unknown features
919                    TableFeature::Unknown(_) => (None, None),
920                    others => {
921                        panic!("This table has unsupported table features: {others:?}");
922                    }
923                }
924            }
925            None => (None, None),
926        }
927    }
928}
929
930///Storage type of deletion vector
931#[derive(Serialize, Deserialize, Copy, Clone, Debug, PartialEq, Eq, Default)]
932pub enum StorageType {
933    /// Stored at relative path derived from a UUID.
934    #[serde(rename = "u")]
935    #[default]
936    UuidRelativePath,
937    /// Stored as inline string.
938    #[serde(rename = "i")]
939    Inline,
940    /// Stored at an absolute path.
941    #[serde(rename = "p")]
942    AbsolutePath,
943}
944
945impl FromStr for StorageType {
946    type Err = Error;
947
948    fn from_str(s: &str) -> Result<Self, Self::Err> {
949        match s {
950            "u" => Ok(Self::UuidRelativePath),
951            "i" => Ok(Self::Inline),
952            "p" => Ok(Self::AbsolutePath),
953            _ => Err(Error::DeletionVector(format!(
954                "Unknown storage format: '{s}'."
955            ))),
956        }
957    }
958}
959
960impl AsRef<str> for StorageType {
961    fn as_ref(&self) -> &str {
962        match self {
963            Self::UuidRelativePath => "u",
964            Self::Inline => "i",
965            Self::AbsolutePath => "p",
966        }
967    }
968}
969
970impl Display for StorageType {
971    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
972        write!(f, "{}", self.as_ref())
973    }
974}
975
976#[derive(Serialize, Deserialize, Debug, Clone, PartialEq, Eq)]
977#[serde(rename_all = "camelCase")]
978/// Defines a deletion vector
979pub struct DeletionVectorDescriptor {
980    /// A single character to indicate how to access the DV. Legal options are: ['u', 'i', 'p'].
981    pub storage_type: StorageType,
982
983    /// Three format options are currently proposed:
984    /// - If `storageType = 'u'` then `<random prefix - optional><base85 encoded uuid>`:
985    ///   The deletion vector is stored in a file with a path relative to the data
986    ///   directory of this Delta table, and the file name can be reconstructed from
987    ///   the UUID. See Derived Fields for how to reconstruct the file name. The random
988    ///   prefix is recovered as the extra characters before the (20 characters fixed length) uuid.
989    /// - If `storageType = 'i'` then `<base85 encoded bytes>`: The deletion vector
990    ///   is stored inline in the log. The format used is the `RoaringBitmapArray`
991    ///   format also used when the DV is stored on disk and described in [Deletion Vector Format].
992    /// - If `storageType = 'p'` then `<absolute path>`: The DV is stored in a file with an
993    ///   absolute path given by this path, which has the same format as the `path` field
994    ///   in the `add`/`remove` actions.
995    ///
996    /// [Deletion Vector Format]: https://github.com/delta-io/delta/blob/master/PROTOCOL.md#Deletion-Vector-Format
997    pub path_or_inline_dv: String,
998
999    /// Start of the data for this DV in number of bytes from the beginning of the file it is stored in.
1000    /// Always None (absent in JSON) when `storageType = 'i'`.
1001    #[serde(skip_serializing_if = "Option::is_none")]
1002    pub offset: Option<i32>,
1003
1004    /// Size of the serialized DV in bytes (raw data size, i.e. before base85 encoding, if inline).
1005    pub size_in_bytes: i32,
1006
1007    /// Number of rows the given DV logically removes from the file.
1008    pub cardinality: i64,
1009}
1010
1011impl TryFrom<DeletionVectorDescriptor>
1012    for delta_kernel::actions::deletion_vector::DeletionVectorDescriptor
1013{
1014    type Error = delta_kernel::Error;
1015
1016    fn try_from(dv: DeletionVectorDescriptor) -> Result<Self, Self::Error> {
1017        use delta_kernel::actions::deletion_vector::DeletionVectorStorageType;
1018
1019        let storage_type = match dv.storage_type {
1020            StorageType::UuidRelativePath => DeletionVectorStorageType::PersistedRelative,
1021            StorageType::Inline => DeletionVectorStorageType::Inline,
1022            StorageType::AbsolutePath => DeletionVectorStorageType::PersistedAbsolute,
1023        };
1024
1025        Self::try_new(
1026            storage_type,
1027            dv.path_or_inline_dv,
1028            dv.offset,
1029            dv.size_in_bytes,
1030            dv.cardinality,
1031        )
1032    }
1033}
1034
1035#[derive(Serialize, Deserialize, Debug, Clone, Default)]
1036#[serde(rename_all = "camelCase")]
1037/// Defines an add action
1038pub struct Add {
1039    /// A relative path to a data file from the root of the table or an absolute path to a file
1040    /// that should be added to the table. The path is a URI as specified by
1041    /// [RFC 2396 URI Generic Syntax], which needs to be decoded to get the data file path.
1042    ///
1043    /// [RFC 2396 URI Generic Syntax]: https://www.ietf.org/rfc/rfc2396.txt
1044    #[serde(with = "serde_path")]
1045    pub path: String,
1046
1047    /// A map from partition column to value for this logical file.
1048    pub partition_values: HashMap<String, Option<String>>,
1049
1050    /// The size of this data file in bytes
1051    pub size: i64,
1052
1053    /// The time this logical file was created, as milliseconds since the epoch.
1054    pub modification_time: i64,
1055
1056    /// When `false` the logical file must already be present in the table or the records
1057    /// in the added file must be contained in one or more remove actions in the same version.
1058    pub data_change: bool,
1059
1060    /// Contains [statistics] (e.g., count, min/max values for columns) about the data in this logical file.
1061    ///
1062    /// [statistics]: https://github.com/delta-io/delta/blob/master/PROTOCOL.md#Per-file-Statistics
1063    pub stats: Option<String>,
1064
1065    /// Map containing metadata about this logical file.
1066    pub tags: Option<HashMap<String, Option<String>>>,
1067
1068    #[serde(skip_serializing_if = "Option::is_none")]
1069    /// Information about deletion vector (DV) associated with this add action
1070    pub deletion_vector: Option<DeletionVectorDescriptor>,
1071
1072    /// Default generated Row ID of the first row in the file. The default generated Row IDs
1073    /// of the other rows in the file can be reconstructed by adding the physical index of the
1074    /// row within the file to the base Row ID
1075    pub base_row_id: Option<i64>,
1076
1077    /// First commit version in which an add action with the same path was committed to the table.
1078    pub default_row_commit_version: Option<i64>,
1079
1080    /// The name of the clustering implementation
1081    pub clustering_provider: Option<String>,
1082}
1083
1084/// Represents a tombstone (deleted file) in the Delta log.
1085#[derive(Serialize, Deserialize, Debug, Clone, Eq, Default)]
1086#[serde(rename_all = "camelCase")]
1087pub struct Remove {
1088    /// A relative path to a data file from the root of the table or an absolute path to a file
1089    /// that should be removed from the table. The path is a URI as specified by
1090    /// [RFC 2396 URI Generic Syntax], which needs to be decoded to get the data file path.
1091    ///
1092    /// [RFC 2396 URI Generic Syntax]: https://www.ietf.org/rfc/rfc2396.txt
1093    #[serde(with = "serde_path")]
1094    pub path: String,
1095
1096    /// When `false` the records in the removed file must be contained
1097    /// in one or more add file actions in the same version
1098    pub data_change: bool,
1099
1100    /// The time the deletion occurred, represented as milliseconds since the epoch
1101    #[serde(skip_serializing_if = "Option::is_none")]
1102    pub deletion_timestamp: Option<i64>,
1103
1104    /// When true the fields `partition_values`, `size`, and `tags` are present
1105    #[serde(skip_serializing_if = "Option::is_none")]
1106    pub extended_file_metadata: Option<bool>,
1107
1108    /// A map from partition column to value for this logical file.
1109    #[serde(skip_serializing_if = "Option::is_none")]
1110    pub partition_values: Option<HashMap<String, Option<String>>>,
1111
1112    /// The size of this data file in bytes
1113    #[serde(skip_serializing_if = "Option::is_none")]
1114    pub size: Option<i64>,
1115
1116    /// Map containing metadata about this logical file.
1117    #[serde(skip_serializing_if = "Option::is_none")]
1118    pub tags: Option<HashMap<String, Option<String>>>,
1119
1120    /// Information about deletion vector (DV) associated with this remove action
1121    #[serde(skip_serializing_if = "Option::is_none")]
1122    pub deletion_vector: Option<DeletionVectorDescriptor>,
1123
1124    /// Default generated Row ID of the first row in the file. The default generated Row IDs
1125    /// of the other rows in the file can be reconstructed by adding the physical index of the
1126    /// row within the file to the base Row ID
1127    #[serde(skip_serializing_if = "Option::is_none")]
1128    pub base_row_id: Option<i64>,
1129
1130    /// First commit version in which an add action with the same path was committed to the table.
1131    #[serde(skip_serializing_if = "Option::is_none")]
1132    pub default_row_commit_version: Option<i64>,
1133}
1134
1135/// Delta AddCDCFile action that describes a parquet CDC data file.
1136#[derive(Serialize, Deserialize, Clone, Debug, Default, PartialEq, Eq)]
1137#[serde(rename_all = "camelCase")]
1138pub struct AddCDCFile {
1139    /// A relative path, from the root of the table, or an
1140    /// absolute path to a CDC file
1141    #[serde(with = "serde_path")]
1142    pub path: String,
1143
1144    /// The size of this file in bytes
1145    pub size: i64,
1146
1147    /// A map from partition column to value for this file
1148    pub partition_values: HashMap<String, Option<String>>,
1149
1150    /// Should always be set to false because they do not change the underlying data of the table
1151    pub data_change: bool,
1152
1153    /// Map containing metadata about this file
1154    #[serde(skip_serializing_if = "Option::is_none")]
1155    pub tags: Option<HashMap<String, Option<String>>>,
1156}
1157
1158/// Action used by streaming systems to track progress using application-specific versions to
1159/// enable idempotency.
1160#[derive(Serialize, Deserialize, Debug, Default, Clone, PartialEq, Eq)]
1161#[serde(rename_all = "camelCase")]
1162pub struct Transaction {
1163    /// A unique identifier for the application performing the transaction.
1164    pub app_id: String,
1165
1166    /// An application-specific numeric identifier for this transaction.
1167    pub version: i64,
1168
1169    /// The time when this transaction action was created in milliseconds since the Unix epoch.
1170    #[serde(skip_serializing_if = "Option::is_none")]
1171    pub last_updated: Option<i64>,
1172}
1173
1174impl Transaction {
1175    /// Create a new application transactions. See [`Txn`] for details.
1176    pub fn new(app_id: impl ToString, version: i64) -> Self {
1177        Self::new_with_last_update(app_id, version, None)
1178    }
1179
1180    /// Create a new application transactions. See [`Txn`] for details.
1181    pub fn new_with_last_update(
1182        app_id: impl ToString,
1183        version: i64,
1184        last_updated: Option<i64>,
1185    ) -> Self {
1186        Transaction {
1187            app_id: app_id.to_string(),
1188            version,
1189            last_updated,
1190        }
1191    }
1192}
1193
1194/// The commitInfo is a fairly flexible action within the delta specification, where arbitrary data can be stored.
1195/// However, the reference implementation as well as delta-rs store useful information that may for instance
1196/// allow us to be more permissive in commit conflict resolution.
1197#[derive(Serialize, Deserialize, Debug, Clone, Default, PartialEq, Eq)]
1198#[serde(rename_all = "camelCase")]
1199pub struct CommitInfo {
1200    /// Timestamp in millis when the commit was created
1201    #[serde(skip_serializing_if = "Option::is_none")]
1202    pub timestamp: Option<i64>,
1203
1204    /// Same as timestamp above, but cooler
1205    #[serde(skip_serializing_if = "Option::is_none")]
1206    pub in_commit_timestamp: Option<i64>,
1207
1208    /// Id of the user invoking the commit
1209    #[serde(skip_serializing_if = "Option::is_none")]
1210    pub user_id: Option<String>,
1211
1212    /// Name of the user invoking the commit
1213    #[serde(skip_serializing_if = "Option::is_none")]
1214    pub user_name: Option<String>,
1215
1216    /// The operation performed during the
1217    #[serde(skip_serializing_if = "Option::is_none")]
1218    pub operation: Option<String>,
1219
1220    /// Parameters used for table operation
1221    #[serde(skip_serializing_if = "Option::is_none")]
1222    pub operation_parameters: Option<HashMap<String, serde_json::Value>>,
1223
1224    /// Version of the table when the operation was started
1225    #[serde(skip_serializing_if = "Option::is_none")]
1226    pub read_version: Option<Version>,
1227
1228    /// The isolation level of the commit
1229    #[serde(skip_serializing_if = "Option::is_none")]
1230    pub isolation_level: Option<IsolationLevel>,
1231
1232    /// TODO
1233    #[serde(skip_serializing_if = "Option::is_none")]
1234    pub is_blind_append: Option<bool>,
1235
1236    /// Delta engine which created the commit.
1237    #[serde(skip_serializing_if = "Option::is_none")]
1238    pub engine_info: Option<String>,
1239
1240    /// Additional provenance information for the commit
1241    #[serde(flatten, default)]
1242    pub info: HashMap<String, serde_json::Value>,
1243
1244    /// User defined metadata
1245    #[serde(skip_serializing_if = "Option::is_none")]
1246    pub user_metadata: Option<String>,
1247}
1248
1249/// The domain metadata action contains a configuration (string) for a named metadata domain
1250#[derive(Serialize, Deserialize, Debug, Clone, Default, PartialEq, Eq)]
1251#[serde(rename_all = "camelCase")]
1252pub struct DomainMetadata {
1253    /// Identifier for this domain (system or user-provided)
1254    pub domain: String,
1255
1256    /// String containing configuration for the metadata domain
1257    pub configuration: String,
1258
1259    /// When `true` the action serves as a tombstone
1260    pub removed: bool,
1261}
1262
1263#[derive(Serialize, Deserialize, Debug, Clone, Default, PartialEq, Eq)]
1264/// This action is only allowed in checkpoints following V2 spec. It describes the details about the checkpoint.
1265pub struct CheckpointMetadata {
1266    /// The flavor of the V2 checkpoint. Allowed values: "flat".
1267    pub flavor: String,
1268
1269    /// Map containing any additional metadata about the v2 spec checkpoint.
1270    #[serde(skip_serializing_if = "Option::is_none")]
1271    pub tags: Option<HashMap<String, Option<String>>>,
1272}
1273
1274/// The sidecar action references a sidecar file which provides some of the checkpoint's file actions.
1275/// This action is only allowed in checkpoints following V2 spec.
1276#[derive(Serialize, Deserialize, Debug, Clone, Default, PartialEq, Eq)]
1277#[serde(rename_all = "camelCase")]
1278pub struct Sidecar {
1279    /// The name of the sidecar file (not a path).
1280    /// The file must reside in the _delta_log/_sidecars directory.
1281    pub file_name: String,
1282
1283    /// The size of the sidecar file in bytes
1284    pub size_in_bytes: i64,
1285
1286    /// The time this sidecar file was created, as milliseconds since the epoch.
1287    pub modification_time: i64,
1288
1289    /// Type of sidecar. Valid values are: "fileaction".
1290    /// This could be extended in future to allow different kinds of sidecars.
1291    #[serde(rename = "type")]
1292    pub sidecar_type: String,
1293
1294    /// Map containing any additional metadata about the checkpoint sidecar file.
1295    #[serde(skip_serializing_if = "Option::is_none")]
1296    pub tags: Option<HashMap<String, Option<String>>>,
1297}
1298
1299#[derive(Serialize, Deserialize, Debug, Copy, Clone, PartialEq, Eq)]
1300/// The isolation level applied during transaction
1301#[derive(Default)]
1302pub enum IsolationLevel {
1303    /// The strongest isolation level. It ensures that committed write operations
1304    /// and all reads are Serializable. Operations are allowed as long as there
1305    /// exists a serial sequence of executing them one-at-a-time that generates
1306    /// the same outcome as that seen in the table. For the write operations,
1307    /// the serial sequence is exactly the same as that seen in the table’s history.
1308    #[default]
1309    Serializable,
1310
1311    /// A weaker isolation level than Serializable. It ensures only that the write
1312    /// operations (that is, not reads) are serializable. However, this is still stronger
1313    /// than Snapshot isolation. WriteSerializable is the default isolation level because
1314    /// it provides great balance of data consistency and availability for most common operations.
1315    WriteSerializable,
1316
1317    /// SnapshotIsolation is a guarantee that all reads made in a transaction will see a consistent
1318    /// snapshot of the database (in practice it reads the last committed values that existed at the
1319    /// time it started), and the transaction itself will successfully commit only if no updates
1320    /// it has made conflict with any concurrent updates made since that snapshot.
1321    SnapshotIsolation,
1322}
1323
1324// Spark assumes Serializable as default isolation level
1325// https://github.com/delta-io/delta/blob/abb171c8401200e7772b27e3be6ea8682528ac72/core/src/main/scala/org/apache/spark/sql/delta/OptimisticTransaction.scala#L1023
1326
1327impl AsRef<str> for IsolationLevel {
1328    fn as_ref(&self) -> &str {
1329        match self {
1330            Self::Serializable => "Serializable",
1331            Self::WriteSerializable => "WriteSerializable",
1332            Self::SnapshotIsolation => "SnapshotIsolation",
1333        }
1334    }
1335}
1336
1337impl FromStr for IsolationLevel {
1338    type Err = Error;
1339
1340    fn from_str(s: &str) -> Result<Self, Self::Err> {
1341        match s.to_ascii_lowercase().as_str() {
1342            "serializable" => Ok(Self::Serializable),
1343            "writeserializable" | "write_serializable" => Ok(Self::WriteSerializable),
1344            "snapshotisolation" | "snapshot_isolation" => Ok(Self::SnapshotIsolation),
1345            _ => Err(Error::Generic("Invalid string for IsolationLevel".into())),
1346        }
1347    }
1348}
1349
1350pub(crate) mod serde_path {
1351    use std::str::Utf8Error;
1352
1353    use percent_encoding::percent_decode_str;
1354    use percent_encoding_rfc3986::{AsciiSet, CONTROLS, utf8_percent_encode};
1355    use serde::{self, Deserialize, Deserializer, Serialize, Serializer};
1356
1357    pub fn deserialize<'de, D>(deserializer: D) -> Result<String, D::Error>
1358    where
1359        D: Deserializer<'de>,
1360    {
1361        let s = String::deserialize(deserializer)?;
1362        decode_path(&s).map_err(serde::de::Error::custom)
1363    }
1364
1365    pub fn serialize<S>(value: &str, serializer: S) -> Result<S::Ok, S::Error>
1366    where
1367        S: Serializer,
1368    {
1369        let encoded = encode_path(value);
1370        String::serialize(&encoded, serializer)
1371    }
1372
1373    pub const _DELIMITER: &str = "/";
1374    /// The path delimiter as a single byte
1375    pub const _DELIMITER_BYTE: u8 = _DELIMITER.as_bytes()[0];
1376
1377    /// Characters we want to encode.
1378    ///
1379    /// Paths are URIs, so the hierarchy delimiter stays literal. Partition values are single
1380    /// strings and go through the stricter set in [`crate::kernel::scalars`] instead.
1381    const INVALID: &AsciiSet = &CONTROLS
1382        // The delimiter we are reserving for internal hierarchy
1383        // .add(DELIMITER_BYTE)
1384        // RFC 2396 excludes the space from URIs and CONTROLS does not cover it, see #4304
1385        .add(b' ')
1386        // Characters AWS recommends avoiding for object keys
1387        // https://docs.aws.amazon.com/AmazonS3/latest/dev/UsingMetadata.html
1388        .add(b'\\')
1389        .add(b'{')
1390        .add(b'^')
1391        .add(b'}')
1392        .add(b'%')
1393        .add(b'`')
1394        .add(b']')
1395        .add(b'"')
1396        .add(b'>')
1397        .add(b'[')
1398        // .add(b'~')
1399        .add(b'<')
1400        .add(b'#')
1401        .add(b'|')
1402        // Characters Google Cloud Storage recommends avoiding for object names
1403        // https://cloud.google.com/storage/docs/naming-objects
1404        .add(b'\r')
1405        .add(b'\n')
1406        .add(b'*')
1407        .add(b'?');
1408
1409    pub(crate) fn encode_path(path: &str) -> String {
1410        utf8_percent_encode(path, INVALID).to_string()
1411    }
1412
1413    // rfc3986's percent_decode_str is fallible, so decoding stays on the url spec crate.
1414    pub fn decode_path(path: &str) -> Result<String, Utf8Error> {
1415        Ok(percent_decode_str(path).decode_utf8()?.to_string())
1416    }
1417}
1418
1419/// Determine required writer features for a given writer version
1420pub(crate) fn writer_features_for_version(
1421    writer_version: i32,
1422) -> Option<Vec<delta_kernel::table_features::TableFeature>> {
1423    if writer_version >= 7 {
1424        // For writer version 7+, include common features
1425        Some(vec![
1426            delta_kernel::table_features::TableFeature::Invariants,
1427            delta_kernel::table_features::TableFeature::AppendOnly,
1428        ])
1429    } else {
1430        // No special features needed for older versions
1431        None
1432    }
1433}
1434
1435#[cfg(test)]
1436mod tests {
1437    use super::*;
1438    use crate::kernel::PrimitiveType;
1439
1440    #[test]
1441    fn test_primitive() {
1442        let types: PrimitiveType = serde_json::from_str("\"string\"").unwrap();
1443        println!("{types:?}");
1444    }
1445
1446    fn nested_in_maps(dtype: DataType) -> Vec<DataType> {
1447        use delta_kernel::schema::{ArrayType, MapType};
1448        let map_key = DataType::Map(Box::new(MapType::new(
1449            dtype.clone(),
1450            DataType::STRING,
1451            true,
1452        )));
1453        let map_value = DataType::Map(Box::new(MapType::new(DataType::STRING, dtype, true)));
1454        let array_of_map = DataType::Array(Box::new(ArrayType::new(map_value.clone(), true)));
1455        vec![map_key, map_value, array_of_map]
1456    }
1457
1458    #[test]
1459    fn test_contains_timestampntz_in_map() {
1460        for dtype in nested_in_maps(DataType::TIMESTAMP_NTZ) {
1461            let field = StructField::nullable("c", dtype.clone());
1462            assert!(contains_timestampntz(std::iter::once(&field)), "{dtype:?}");
1463        }
1464        let field = StructField::nullable("c", nested_in_maps(DataType::TIMESTAMP).remove(0));
1465        assert!(!contains_timestampntz(std::iter::once(&field)));
1466    }
1467
1468    #[cfg(feature = "nanosecond-timestamps")]
1469    #[test]
1470    fn test_contains_timestamp_nanos_in_map() {
1471        for dtype in nested_in_maps(DataType::TIMESTAMP_NANOS_NTZ) {
1472            let field = StructField::nullable("c", dtype.clone());
1473            assert!(
1474                contains_timestamp_nanos(std::iter::once(&field)),
1475                "{dtype:?}"
1476            );
1477        }
1478    }
1479
1480    #[test]
1481    fn test_deserialize_protocol() {
1482        // protocol json data
1483        let raw = serde_json::json!(
1484            {
1485              "minReaderVersion": 3,
1486              "minWriterVersion": 7,
1487              "readerFeatures": ["catalogOwned"],
1488              "writerFeatures": ["catalogOwned", "invariants", "appendOnly"]
1489            }
1490        );
1491        let protocol: Protocol = serde_json::from_value(raw).unwrap();
1492        assert_eq!(protocol.min_reader_version(), 3);
1493        assert_eq!(protocol.min_writer_version(), 7);
1494        assert_eq!(
1495            protocol.reader_features(),
1496            Some(vec![TableFeature::Unknown("catalogOwned".to_owned())].as_slice())
1497        );
1498        assert_eq!(
1499            protocol.writer_features(),
1500            Some(
1501                vec![
1502                    TableFeature::Unknown("catalogOwned".to_owned()),
1503                    TableFeature::Invariants,
1504                    TableFeature::AppendOnly
1505                ]
1506                .as_slice()
1507            )
1508        );
1509    }
1510
1511    #[test]
1512    fn test_serialize_add_path_encodes_space() {
1513        let add = Add {
1514            path: "year=2021/part 0.parquet".to_string(),
1515            ..Default::default()
1516        };
1517
1518        let value = serde_json::to_value(&add).unwrap();
1519        assert_eq!(value["path"], "year=2021/part%200.parquet");
1520    }
1521
1522    #[test]
1523    fn test_serialize_remove_and_cdc_path_encodes_space() {
1524        let remove = Remove {
1525            path: "part 1.parquet".to_string(),
1526            ..Default::default()
1527        };
1528        let cdc = AddCDCFile {
1529            path: "_change_data/part 2.parquet".to_string(),
1530            ..Default::default()
1531        };
1532
1533        assert_eq!(
1534            serde_json::to_value(&remove).unwrap()["path"],
1535            "part%201.parquet"
1536        );
1537        assert_eq!(
1538            serde_json::to_value(&cdc).unwrap()["path"],
1539            "_change_data/part%202.parquet"
1540        );
1541    }
1542
1543    #[test]
1544    fn test_serialize_path_preserves_hive_partition_delimiters() {
1545        // A path is a multi segment URI, so encoding the delimiter or the `=` of a hive style
1546        // directory would point the action at a file that does not exist.
1547        let add = Add {
1548            path: "year=2020/month=2/day=3/part-00000-abc.snappy.parquet".to_string(),
1549            ..Default::default()
1550        };
1551
1552        let value = serde_json::to_value(&add).unwrap();
1553        assert_eq!(
1554            value["path"],
1555            "year=2020/month=2/day=3/part-00000-abc.snappy.parquet"
1556        );
1557    }
1558
1559    #[test]
1560    fn test_serialize_round_trips_spark_written_path() {
1561        // Spark writes this path, see dat/v0.0.3/reader_tests/generated/multi_partitioned_2.
1562        // Re-serializing it has to reproduce the same string, because kernel matches removes
1563        // against adds on the raw path rather than the decoded one.
1564        let path = "bool=false/time=1970-01-02%2008%253A45%253A00/amount=12.000000000000000000/part-00000-12004196-1e98-4a42-a622-a12f05a4578e.c000.snappy.parquet";
1565        let raw = serde_json::json!({
1566            "path": path,
1567            "partitionValues": {},
1568            "size": 0,
1569            "modificationTime": 0,
1570            "dataChange": true
1571        });
1572
1573        let add: Add = serde_json::from_value(raw).unwrap();
1574        assert_eq!(serde_json::to_value(&add).unwrap()["path"], path);
1575    }
1576
1577    #[test]
1578    fn test_deserialize_path_accepts_unencoded_space() {
1579        // Commits written before spaces were encoded have to keep decoding to the same path.
1580        let raw = serde_json::json!({
1581            "path": "city=New York/part-00000-abc.parquet",
1582            "partitionValues": {},
1583            "size": 0,
1584            "modificationTime": 0,
1585            "dataChange": true
1586        });
1587
1588        let add: Add = serde_json::from_value(raw).unwrap();
1589        assert_eq!(add.path, "city=New York/part-00000-abc.parquet");
1590    }
1591
1592    #[test]
1593    fn test_apply_properties_column_mapping_legacy_versions() {
1594        let protocol: Protocol = serde_json::from_value(serde_json::json!({
1595            "minReaderVersion": 1,
1596            "minWriterVersion": 2,
1597        }))
1598        .unwrap();
1599        let config = HashMap::from([("delta.columnMapping.mode".to_string(), "name".to_string())]);
1600
1601        let protocol = protocol
1602            .apply_properties_to_protocol(&config, true)
1603            .unwrap();
1604
1605        assert_eq!(protocol.min_reader_version(), 2);
1606        assert_eq!(protocol.min_writer_version(), 5);
1607    }
1608
1609    #[test]
1610    fn test_apply_properties_column_mapping_table_features() {
1611        // On a table-features protocol (reader v3 / writer v7), column mapping is a ReaderWriter
1612        // feature and must be declared in both the reader and writer feature lists.
1613        let protocol: Protocol = serde_json::from_value(serde_json::json!({
1614            "minReaderVersion": 3,
1615            "minWriterVersion": 7,
1616            "readerFeatures": [],
1617            "writerFeatures": [],
1618        }))
1619        .expect("Failed to initialize a new Protocol");
1620        let config = HashMap::from([("delta.columnMapping.mode".to_string(), "name".to_string())]);
1621
1622        let protocol = protocol
1623            .apply_properties_to_protocol(&config, true)
1624            .expect("Failed to apply properties to protocol");
1625
1626        assert_eq!(protocol.min_reader_version(), 3);
1627        assert_eq!(protocol.min_writer_version(), 7);
1628        assert!(
1629            protocol
1630                .reader_features()
1631                .unwrap()
1632                .contains(&TableFeature::ColumnMapping),
1633            "readerFeatures should contain ColumnMapping, got: {:?}",
1634            protocol.reader_features()
1635        );
1636        assert!(
1637            protocol
1638                .writer_features()
1639                .unwrap()
1640                .contains(&TableFeature::ColumnMapping),
1641            "writerFeatures should contain ColumnMapping, got: {:?}",
1642            protocol.writer_features()
1643        );
1644    }
1645
1646    #[test]
1647    fn test_apply_properties_does_not_add_unused_variant_type() {
1648        // Reader v3 / writer v7 alone must not declare variantType: readers without Variant
1649        // support refuse any table that lists it (#4835). Variant columns enable it from the schema.
1650        let timestamp_ntz: Protocol = serde_json::from_value(serde_json::json!({
1651            "minReaderVersion": 3,
1652            "minWriterVersion": 7,
1653            "readerFeatures": ["timestampNtz"],
1654            "writerFeatures": ["timestampNtz"],
1655        }))
1656        .unwrap();
1657        let legacy: Protocol = serde_json::from_value(serde_json::json!({
1658            "minReaderVersion": 1,
1659            "minWriterVersion": 2,
1660        }))
1661        .unwrap();
1662        let cases = [
1663            (timestamp_ntz, HashMap::new()),
1664            (
1665                legacy,
1666                HashMap::from([
1667                    ("delta.minReaderVersion".to_string(), "3".to_string()),
1668                    ("delta.minWriterVersion".to_string(), "7".to_string()),
1669                ]),
1670            ),
1671        ];
1672
1673        for (protocol, config) in cases {
1674            let protocol = protocol
1675                .apply_properties_to_protocol(&config, true)
1676                .unwrap();
1677
1678            assert_eq!(protocol.min_reader_version(), 3);
1679            assert_eq!(protocol.min_writer_version(), 7);
1680            for features in [protocol.reader_features(), protocol.writer_features()] {
1681                assert!(
1682                    !features.is_some_and(|f| f.contains(&TableFeature::VariantType)),
1683                    "variantType should not be added without a variant column, got: {features:?}"
1684                );
1685            }
1686        }
1687    }
1688
1689    #[test]
1690    fn test_apply_properties_column_mapping_independent_reader_writer() {
1691        // Reader and writer are bumped independently: a feature-based writer (v7) declares the
1692        // ColumnMapping writer feature, while the legacy reader is bumped to v2.
1693        let protocol: Protocol = serde_json::from_value(serde_json::json!({
1694            "minReaderVersion": 1,
1695            "minWriterVersion": 7,
1696            "writerFeatures": [],
1697        }))
1698        .unwrap();
1699        let config = HashMap::from([("delta.columnMapping.mode".to_string(), "name".to_string())]);
1700
1701        let protocol = protocol
1702            .apply_properties_to_protocol(&config, true)
1703            .unwrap();
1704
1705        assert_eq!(protocol.min_reader_version(), 2);
1706        assert_eq!(protocol.min_writer_version(), 7);
1707        assert!(
1708            protocol
1709                .writer_features()
1710                .unwrap()
1711                .contains(&TableFeature::ColumnMapping),
1712            "writerFeatures should contain ColumnMapping, got: {:?}",
1713            protocol.writer_features()
1714        );
1715
1716        assert!(
1717            protocol
1718                .writer_features()
1719                .unwrap()
1720                .contains(&TableFeature::Invariants),
1721            "writerFeatures should be upgraded include Invariants, got: {:?}",
1722            protocol.writer_features()
1723        );
1724
1725        // Reader stayed legacy (v2), so it carries no ColumnMapping reader feature.
1726        assert!(
1727            !protocol
1728                .reader_features()
1729                .map(|f| f.contains(&TableFeature::ColumnMapping))
1730                .unwrap_or(false),
1731            "reader side should remain legacy, got: {:?}",
1732            protocol.reader_features()
1733        );
1734    }
1735
1736    /// Builds the reader 3 / writer 7 protocol that lists exactly the given features, to spell out
1737    /// the protocol an upgrade to table features is expected to produce
1738    fn table_features_protocol(
1739        reader_features: impl IntoIterator<Item = TableFeature>,
1740        writer_features: impl IntoIterator<Item = TableFeature>,
1741    ) -> ProtocolInner {
1742        ProtocolInner {
1743            min_reader_version: 3,
1744            min_writer_version: 7,
1745            reader_features: Some(reader_features.into_iter().collect()),
1746            writer_features: Some(writer_features.into_iter().collect()),
1747        }
1748    }
1749
1750    /// Table properties that enable deletion vectors, the most common way a legacy protocol is
1751    /// moved to table features, for passing to `apply_properties_to_protocol`
1752    fn enable_deletion_vectors() -> HashMap<String, String> {
1753        HashMap::from([(
1754            "delta.enableDeletionVectors".to_string(),
1755            "true".to_string(),
1756        )])
1757    }
1758
1759    #[test]
1760    fn test_enable_deletion_vectors_keeps_legacy_implied_features() {
1761        use TableFeature::*;
1762        let writer_4_features = [
1763            AppendOnly,
1764            Invariants,
1765            CheckConstraints,
1766            ChangeDataFeed,
1767            GeneratedColumns,
1768            DeletionVectors,
1769        ];
1770        let table_features =
1771            table_features_protocol([DeletionVectors], [AppendOnly, Invariants, DeletionVectors]);
1772        let cases = [
1773            ((1, 2), table_features.clone()),
1774            (
1775                (1, 3),
1776                table_features_protocol(
1777                    [DeletionVectors],
1778                    [AppendOnly, Invariants, CheckConstraints, DeletionVectors],
1779                ),
1780            ),
1781            (
1782                (1, 4),
1783                table_features_protocol([DeletionVectors], writer_4_features.clone()),
1784            ),
1785            (
1786                (1, 5),
1787                table_features_protocol([DeletionVectors], writer_4_features.clone()),
1788            ),
1789            (
1790                (2, 5),
1791                table_features_protocol(
1792                    [ColumnMapping, DeletionVectors],
1793                    writer_4_features.clone().into_iter().chain([ColumnMapping]),
1794                ),
1795            ),
1796            (
1797                (2, 6),
1798                table_features_protocol(
1799                    [ColumnMapping, DeletionVectors],
1800                    writer_4_features
1801                        .clone()
1802                        .into_iter()
1803                        .chain([ColumnMapping, IdentityColumns]),
1804                ),
1805            ),
1806        ];
1807        let cases = cases
1808            .into_iter()
1809            .map(|((reader, writer), expected)| (ProtocolInner::new(reader, writer), expected))
1810            .chain([(table_features.clone(), table_features)]);
1811
1812        for (protocol, expected) in cases {
1813            let upgraded = protocol
1814                .clone()
1815                .apply_properties_to_protocol(&enable_deletion_vectors(), true)
1816                .unwrap();
1817            assert_eq!(upgraded, expected, "upgrading {protocol:?}");
1818            let upgraded_again = upgraded
1819                .apply_properties_to_protocol(&enable_deletion_vectors(), true)
1820                .unwrap();
1821            assert_eq!(upgraded_again, expected, "upgrading {protocol:?} twice");
1822        }
1823    }
1824
1825    #[test]
1826    fn test_every_table_features_upgrade_keeps_column_mapping() {
1827        let column_mapped = || ProtocolInner::new(2, 5);
1828        let min_versions = HashMap::from([
1829            ("delta.minReaderVersion".to_string(), "3".to_string()),
1830            ("delta.minWriterVersion".to_string(), "7".to_string()),
1831        ]);
1832        let timestamp_ntz_schema =
1833            StructType::try_new([StructField::new("ts", DataType::TIMESTAMP_NTZ, true)]).unwrap();
1834        let upgrades = [
1835            column_mapped()
1836                .append_reader_features([TableFeature::DeletionVectors])
1837                .append_writer_features([TableFeature::DeletionVectors]),
1838            column_mapped()
1839                .apply_column_metadata_to_protocol(&timestamp_ntz_schema)
1840                .unwrap(),
1841            column_mapped()
1842                .apply_properties_to_protocol(&min_versions, true)
1843                .unwrap(),
1844            ProtocolInner {
1845                min_reader_version: 2,
1846                min_writer_version: 7,
1847                reader_features: None,
1848                writer_features: Some(HashSet::from([TableFeature::ColumnMapping])),
1849            }
1850            .apply_properties_to_protocol(&enable_deletion_vectors(), true)
1851            .unwrap(),
1852        ];
1853
1854        for upgraded in upgrades {
1855            assert_eq!(
1856                (upgraded.min_reader_version, upgraded.min_writer_version),
1857                (3, 7)
1858            );
1859            for features in [&upgraded.reader_features, &upgraded.writer_features] {
1860                assert!(
1861                    features
1862                        .as_ref()
1863                        .is_some_and(|f| f.contains(&TableFeature::ColumnMapping)),
1864                    "columnMapping missing from {upgraded:?}"
1865                );
1866            }
1867        }
1868
1869        let writer_only = column_mapped().append_writer_features([TableFeature::AppendOnly]);
1870        assert_eq!(writer_only.min_reader_version, 2);
1871        assert_eq!(writer_only.reader_features, None);
1872        assert!(
1873            writer_only
1874                .writer_features
1875                .as_ref()
1876                .is_some_and(|f| f.contains(&TableFeature::ColumnMapping)),
1877            "columnMapping missing from {writer_only:?}"
1878        );
1879    }
1880
1881    #[test]
1882    fn test_legacy_protocol_without_table_features_upgrade_is_unchanged() {
1883        let properties = HashMap::from([
1884            ("delta.enableChangeDataFeed".to_string(), "true".to_string()),
1885            ("delta.appendOnly".to_string(), "true".to_string()),
1886        ]);
1887        let unchanged = ProtocolInner::new(2, 5)
1888            .apply_properties_to_protocol(&properties, true)
1889            .unwrap()
1890            .move_table_properties_into_features(&properties);
1891        assert_eq!(unchanged, ProtocolInner::new(2, 5));
1892    }
1893
1894    // #[test]
1895    // fn test_deletion_vector_read() {
1896    //     let store = Arc::new(LocalFileSystem::new());
1897    //     let path =
1898    //         std::fs::canonicalize(PathBuf::from("../deltalake-test/tests/data/table-with-dv-small/")).unwrap();
1899    //     let parent = url::Url::from_directory_path(path).unwrap();
1900    //     let root = object_store::path::Path::from(parent.path());
1901    //     let fs_client = Arc::new(ObjectStoreFileSystemClient::new(
1902    //         store,
1903    //         root,
1904    //         Arc::new(TokioBackgroundExecutor::new()),
1905    //     ));
1906    //
1907    //     let example = dv_example();
1908    //     let tree_map = example.read(fs_client, parent).unwrap();
1909    //
1910    //     let expected: Vec<u64> = vec![0, 9];
1911    //     let found = tree_map.iter().collect::<Vec<_>>();
1912    //     assert_eq!(found, expected)
1913    // }
1914}