1use std::collections::{HashMap, HashSet};
2use std::fmt::{self, Display};
3use std::str::FromStr;
4
5use delta_kernel::schema::{DataType, StructField};
6use delta_kernel::table_features::TableFeature;
7use serde::{Deserialize, Serialize};
8
9use crate::TableProperty;
10use crate::kernel::{DeltaResult, error::Error};
11use crate::kernel::{StructType, StructTypeExt, Version};
12
13pub use delta_kernel::actions::{Metadata, Protocol};
14
15pub fn new_metadata(
22 schema: &StructType,
23 partition_columns: impl IntoIterator<Item = impl ToString>,
24 configuration: impl IntoIterator<Item = (impl ToString, impl ToString)>,
25) -> DeltaResult<Metadata> {
26 let value = serde_json::json!({
28 "id": uuid::Uuid::new_v4().to_string(),
29 "name": None::<String>,
30 "description": None::<String>,
31 "format": { "provider": "parquet", "options": {} },
32 "schemaString": serde_json::to_string(schema)?,
33 "partitionColumns": partition_columns.into_iter().map(|c| c.to_string()).collect::<Vec<_>>(),
34 "configuration": configuration.into_iter().map(|(k, v)| (k.to_string(), v.to_string())).collect::<HashMap<_, _>>(),
35 "createdTime": chrono::Utc::now().timestamp_millis(),
36 });
37 Ok(serde_json::from_value(value)?)
38}
39
40pub trait MetadataExt {
46 fn with_table_id(self, table_id: String) -> DeltaResult<Metadata>;
48
49 fn with_name(self, name: String) -> DeltaResult<Metadata>;
51
52 fn with_description(self, description: String) -> DeltaResult<Metadata>;
54
55 fn with_schema(self, schema: &StructType) -> DeltaResult<Metadata>;
57
58 fn add_config_key(self, key: String, value: String) -> DeltaResult<Metadata>;
60
61 fn remove_config_key(self, key: &str) -> DeltaResult<Metadata>;
63}
64
65impl MetadataExt for Metadata {
66 fn with_table_id(self, table_id: String) -> DeltaResult<Metadata> {
67 let value = serde_json::json!({
68 "id": table_id,
69 "name": self.name(),
70 "description": self.description(),
71 "format": { "provider": "parquet", "options": {} },
72 "schemaString": serde_json::to_string(&self.parse_schema().unwrap())?,
73 "partitionColumns": self.partition_columns(),
74 "configuration": self.configuration(),
75 "createdTime": self.created_time(),
76 });
77 Ok(serde_json::from_value(value)?)
78 }
79
80 fn with_name(self, name: String) -> DeltaResult<Metadata> {
81 let value = serde_json::json!({
82 "id": self.id(),
83 "name": name,
84 "description": self.description(),
85 "format": { "provider": "parquet", "options": {} },
86 "schemaString": serde_json::to_string(&self.parse_schema().unwrap())?,
87 "partitionColumns": self.partition_columns(),
88 "configuration": self.configuration(),
89 "createdTime": self.created_time(),
90 });
91 Ok(serde_json::from_value(value)?)
92 }
93
94 fn with_description(self, description: String) -> DeltaResult<Metadata> {
95 let value = serde_json::json!({
96 "id": self.id(),
97 "name": self.name(),
98 "description": description,
99 "format": { "provider": "parquet", "options": {} },
100 "schemaString": serde_json::to_string(&self.parse_schema().unwrap())?,
101 "partitionColumns": self.partition_columns(),
102 "configuration": self.configuration(),
103 "createdTime": self.created_time(),
104 });
105 Ok(serde_json::from_value(value)?)
106 }
107
108 fn with_schema(self, schema: &StructType) -> DeltaResult<Metadata> {
109 let value = serde_json::json!({
110 "id": self.id(),
111 "name": self.name(),
112 "description": self.description(),
113 "format": { "provider": "parquet", "options": {} },
114 "schemaString": serde_json::to_string(schema)?,
115 "partitionColumns": self.partition_columns(),
116 "configuration": self.configuration(),
117 "createdTime": self.created_time(),
118 });
119 Ok(serde_json::from_value(value)?)
120 }
121
122 fn add_config_key(self, key: String, value: String) -> DeltaResult<Metadata> {
123 let mut config = self.configuration().clone();
124 config.insert(key, value);
125 let value = serde_json::json!({
126 "id": self.id(),
127 "name": self.name(),
128 "description": self.description(),
129 "format": { "provider": "parquet", "options": {} },
130 "schemaString": serde_json::to_string(&self.parse_schema().unwrap())?,
131 "partitionColumns": self.partition_columns(),
132 "configuration": config,
133 "createdTime": self.created_time(),
134 });
135 Ok(serde_json::from_value(value)?)
136 }
137
138 fn remove_config_key(self, key: &str) -> DeltaResult<Metadata> {
139 let mut config = self.configuration().clone();
140 config.remove(key);
141 let value = serde_json::json!({
142 "id": self.id(),
143 "name": self.name(),
144 "description": self.description(),
145 "format": { "provider": "parquet", "options": {} },
146 "schemaString": serde_json::to_string(&self.parse_schema().unwrap())?,
147 "partitionColumns": self.partition_columns(),
148 "configuration": config,
149 "createdTime": self.created_time(),
150 });
151 Ok(serde_json::from_value(value)?)
152 }
153}
154
155fn matches_datatype(dtype_to_check: &DataType, dtype: &DataType) -> bool {
157 match dtype_to_check {
158 to_check if dtype == to_check => true,
159 DataType::Array(inner) => matches_datatype(inner.element_type(), dtype),
160 DataType::Struct(inner) => inner
161 .fields()
162 .any(|f| matches_datatype(f.data_type(), dtype)),
163 DataType::Map(inner) => {
164 matches_datatype(inner.key_type(), dtype) || matches_datatype(inner.value_type(), dtype)
165 }
166 _ => false,
167 }
168}
169
170pub fn contains_timestampntz<'a>(mut fields: impl Iterator<Item = &'a StructField>) -> bool {
172 fields.any(|f| matches_datatype(f.data_type(), &DataType::TIMESTAMP_NTZ))
173}
174
175#[cfg(feature = "nanosecond-timestamps")]
176pub fn contains_timestamp_nanos<'a>(mut fields: impl Iterator<Item = &'a StructField>) -> bool {
180 fields.any(|f| {
181 matches_datatype(f.data_type(), &DataType::TIMESTAMP_NANOS)
182 || matches_datatype(f.data_type(), &DataType::TIMESTAMP_NANOS_NTZ)
183 })
184}
185
186pub(crate) fn contains_variant<'a>(mut fields: impl Iterator<Item = &'a StructField>) -> bool {
188 fn _check_type(dtype: &DataType) -> bool {
189 match dtype {
190 DataType::Variant(_) => true,
191 DataType::Array(inner) => _check_type(inner.element_type()),
192 DataType::Struct(inner) => inner.fields().any(|f| _check_type(f.data_type())),
193 DataType::Map(inner) => {
194 _check_type(inner.key_type()) || _check_type(inner.value_type())
195 }
196 _ => false,
197 }
198 }
199
200 fields.any(|f| _check_type(f.data_type()))
201}
202
203pub(crate) trait ProtocolExt {
208 fn reader_features_set(&self) -> Option<HashSet<TableFeature>>;
209 fn writer_features_set(&self) -> Option<HashSet<TableFeature>>;
210 fn append_reader_features(self, reader_features: &[TableFeature]) -> Protocol;
211 fn append_writer_features(self, writer_features: &[TableFeature]) -> Protocol;
212 fn move_table_properties_into_features(
213 self,
214 configuration: &HashMap<String, String>,
215 ) -> Protocol;
216 fn apply_column_metadata_to_protocol(self, schema: &StructType) -> DeltaResult<Protocol>;
217 fn apply_properties_to_protocol(
218 self,
219 new_properties: &HashMap<String, String>,
220 raise_if_not_exists: bool,
221 ) -> DeltaResult<Protocol>;
222}
223
224impl ProtocolExt for Protocol {
225 fn reader_features_set(&self) -> Option<HashSet<TableFeature>> {
226 self.reader_features()
227 .map(|features| features.iter().cloned().collect())
228 }
229
230 fn writer_features_set(&self) -> Option<HashSet<TableFeature>> {
231 self.writer_features()
232 .map(|features| features.iter().cloned().collect())
233 }
234
235 fn append_reader_features(self, reader_features: &[TableFeature]) -> Protocol {
236 let mut inner = ProtocolInner::from_kernel(&self);
237 inner = inner.append_reader_features(reader_features.iter().cloned());
238 inner.as_kernel()
239 }
240
241 fn append_writer_features(self, writer_features: &[TableFeature]) -> Protocol {
242 let mut inner = ProtocolInner::from_kernel(&self);
243 inner = inner.append_writer_features(writer_features.iter().cloned());
244 inner.as_kernel()
245 }
246
247 fn move_table_properties_into_features(
248 self,
249 configuration: &HashMap<String, String>,
250 ) -> Protocol {
251 let mut inner = ProtocolInner::from_kernel(&self);
252 inner = inner.move_table_properties_into_features(configuration);
253 inner.as_kernel()
254 }
255
256 fn apply_column_metadata_to_protocol(self, schema: &StructType) -> DeltaResult<Protocol> {
257 let mut inner = ProtocolInner::from_kernel(&self);
258 inner = inner.apply_column_metadata_to_protocol(schema)?;
259 Ok(inner.as_kernel())
260 }
261
262 fn apply_properties_to_protocol(
263 self,
264 new_properties: &HashMap<String, String>,
265 raise_if_not_exists: bool,
266 ) -> DeltaResult<Protocol> {
267 let mut inner = ProtocolInner::from_kernel(&self);
268 inner = inner.apply_properties_to_protocol(new_properties, raise_if_not_exists)?;
269 Ok(inner.as_kernel())
270 }
271}
272
273const LEGACY_IMPLIED_FEATURES: [(i32, i32, TableFeature); 5] = [
279 (1, 3, TableFeature::CheckConstraints),
280 (1, 4, TableFeature::ChangeDataFeed),
281 (1, 4, TableFeature::GeneratedColumns),
282 (2, 5, TableFeature::ColumnMapping),
283 (1, 6, TableFeature::IdentityColumns),
284];
285
286#[derive(Serialize, Deserialize, Debug, Clone, PartialEq, Eq)]
287#[serde(rename_all = "camelCase")]
288pub(crate) struct ProtocolInner {
296 pub min_reader_version: i32,
299 pub min_writer_version: i32,
302 #[serde(skip_serializing_if = "Option::is_none")]
305 pub reader_features: Option<HashSet<TableFeature>>,
306 #[serde(skip_serializing_if = "Option::is_none")]
309 pub writer_features: Option<HashSet<TableFeature>>,
310}
311
312impl Default for ProtocolInner {
313 fn default() -> Self {
314 Self {
315 min_reader_version: 1,
316 min_writer_version: 2,
317 reader_features: None,
318 writer_features: None,
319 }
320 }
321}
322
323impl ProtocolInner {
324 #[cfg(test)]
326 pub(crate) fn new(min_reader_version: i32, min_writer_version: i32) -> Self {
327 Self {
328 min_reader_version,
329 min_writer_version,
330 reader_features: None,
331 writer_features: None,
332 }
333 }
334
335 pub(crate) fn from_kernel(value: &Protocol) -> ProtocolInner {
336 serde_json::from_value(serde_json::to_value(value).unwrap()).unwrap()
338 }
339
340 pub(crate) fn as_kernel(&self) -> Protocol {
341 serde_json::from_value(serde_json::to_value(self).unwrap())
343 .expect("Failed to convert Protocol to a kernel implementation")
344 }
345
346 fn upgrade_to_reader_features(&mut self) {
349 if self.min_reader_version >= 3 {
350 return;
351 }
352 let supports_column_mapping = if self.min_reader_version < 2 {
353 false
354 } else if self.min_writer_version < 7 {
355 self.min_writer_version >= 5
356 } else {
357 self.writer_features
358 .as_ref()
359 .is_some_and(|features| features.contains(&TableFeature::ColumnMapping))
360 };
361 let reader_features = self.reader_features.get_or_insert_with(HashSet::new);
362 if supports_column_mapping {
363 reader_features.insert(TableFeature::ColumnMapping);
364 }
365 self.min_reader_version = 3;
366 }
367
368 fn upgrade_to_writer_features(&mut self) {
371 if self.min_writer_version >= 7 {
372 return;
373 }
374 let writer_features = self.writer_features.get_or_insert_with(HashSet::new);
375 for (min_reader_version, min_writer_version, feature) in &LEGACY_IMPLIED_FEATURES {
376 if self.min_writer_version < *min_writer_version {
377 continue;
378 }
379 let reader_supports = if self.min_reader_version < 3 {
380 self.min_reader_version >= *min_reader_version
381 } else if *min_reader_version == 1 {
382 true
383 } else {
384 self.reader_features
385 .as_ref()
386 .is_some_and(|features| features.contains(feature))
387 };
388 if reader_supports {
389 writer_features.insert(feature.clone());
390 }
391 }
392 self.min_writer_version = 7;
393 }
394
395 pub fn append_reader_features(
397 mut self,
398 reader_features: impl IntoIterator<Item = impl Into<TableFeature>>,
399 ) -> Self {
400 let all_reader_features = reader_features
401 .into_iter()
402 .map(Into::into)
403 .collect::<HashSet<_>>();
404 if !all_reader_features.is_empty() {
405 self.upgrade_to_reader_features();
406 match self.reader_features {
407 Some(mut features) => {
408 features.extend(all_reader_features);
409 self.reader_features = Some(features);
410 }
411 None => self.reader_features = Some(all_reader_features),
412 };
413 }
414 self
415 }
416
417 pub fn append_writer_features(
419 mut self,
420 writer_features: impl IntoIterator<Item = impl Into<TableFeature>>,
421 ) -> Self {
422 let all_writer_features = writer_features
423 .into_iter()
424 .map(|c| c.into())
425 .collect::<HashSet<_>>();
426 if !all_writer_features.is_empty() {
427 self.upgrade_to_writer_features();
428 match self.writer_features {
429 Some(mut features) => {
430 features.extend(all_writer_features);
431 self.writer_features = Some(features);
432 }
433 None => self.writer_features = Some(all_writer_features),
434 };
435 }
436 self
437 }
438
439 pub fn move_table_properties_into_features(
442 mut self,
443 configuration: &HashMap<String, String>,
444 ) -> Self {
445 fn parse_bool(value: &str) -> bool {
446 value.to_ascii_lowercase().parse::<bool>().is_ok_and(|v| v)
447 }
448
449 if self.min_writer_version >= 7 {
450 let mut converted_writer_features = configuration
452 .iter()
453 .filter(|(_, value)| value.to_ascii_lowercase().parse::<bool>().is_ok_and(|v| v))
454 .filter_map(|(key, value)| match key.as_str() {
455 "delta.enableChangeDataFeed" if parse_bool(value) => {
456 Some(TableFeature::ChangeDataFeed)
457 }
458 "delta.appendOnly" if parse_bool(value) => Some(TableFeature::AppendOnly),
459 "delta.enableDeletionVectors" if parse_bool(value) => {
460 Some(TableFeature::DeletionVectors)
461 }
462 "delta.enableRowTracking" if parse_bool(value) => {
463 Some(TableFeature::RowTracking)
464 }
465 "delta.checkpointPolicy" if value == "v2" => Some(TableFeature::V2Checkpoint),
466 _ => None,
467 })
468 .collect::<HashSet<TableFeature>>();
469
470 if configuration
471 .keys()
472 .any(|v| v.starts_with("delta.constraints."))
473 {
474 converted_writer_features.insert(TableFeature::CheckConstraints);
475 }
476
477 match self.writer_features {
478 Some(mut features) => {
479 features.extend(converted_writer_features);
480 self.writer_features = Some(features);
481 }
482 None => self.writer_features = Some(converted_writer_features),
483 }
484 }
485 if self.min_reader_version >= 3 {
486 let converted_reader_features = configuration
487 .iter()
488 .filter_map(|(key, value)| match key.as_str() {
489 "delta.enableDeletionVectors" if parse_bool(value) => {
490 Some(TableFeature::DeletionVectors)
491 }
492 "delta.checkpointPolicy" if value == "v2" => Some(TableFeature::V2Checkpoint),
493 _ => None,
494 })
495 .collect::<HashSet<TableFeature>>();
496 match self.reader_features {
497 Some(mut features) => {
498 features.extend(converted_reader_features);
499 self.reader_features = Some(features);
500 }
501 None => self.reader_features = Some(converted_reader_features),
502 }
503 }
504 self
505 }
506
507 pub fn apply_column_metadata_to_protocol(mut self, schema: &StructType) -> DeltaResult<Self> {
510 let generated_cols = schema.get_generated_columns()?;
511 let invariants = schema.get_invariants()?;
512 let contains_timestamp_ntz = self.contains_timestampntz(schema.fields());
513 #[cfg(feature = "nanosecond-timestamps")]
514 let contains_timestamp_nanos = self.contains_timestamp_nanos(schema.fields());
515 let contains_variant = self.contains_variant(schema.fields());
516
517 if contains_timestamp_ntz {
518 self = self.enable_timestamp_ntz()
519 }
520
521 #[cfg(feature = "nanosecond-timestamps")]
522 if contains_timestamp_nanos {
523 self = self.enable_timestamp_nanos().enable_timestamp_ntz()
527 }
528 if contains_variant {
529 self = self.enable_variant_type()
530 }
531
532 if !generated_cols.is_empty() {
533 self = self.enable_generated_columns()
534 }
535
536 if !invariants.is_empty() {
537 self = self.enable_invariants()
538 }
539
540 Ok(self)
541 }
542
543 pub fn apply_properties_to_protocol(
546 mut self,
547 new_properties: &HashMap<String, String>,
548 raise_if_not_exists: bool,
549 ) -> DeltaResult<Self> {
550 let mut parsed_properties: HashMap<TableProperty, String> = HashMap::new();
551
552 for (key, value) in new_properties {
553 if let Ok(parsed_key) = key.parse::<TableProperty>() {
554 parsed_properties.insert(parsed_key, value.to_string());
555 } else if raise_if_not_exists {
556 return Err(Error::Generic(format!(
557 "Error parsing property '{key}':'{value}'",
558 )));
559 }
560 }
561
562 if let Some(min_writer_version) = parsed_properties.get(&TableProperty::MinWriterVersion) {
564 let new_min_writer_version = min_writer_version.parse::<i32>();
565 match new_min_writer_version {
566 Ok(version) => match version {
567 7 => self.upgrade_to_writer_features(),
568 2..=6 => {
569 if version > self.min_writer_version {
570 self.min_writer_version = version
571 }
572 }
573 _ => {
574 return Err(Error::Generic(format!(
575 "delta.minWriterVersion = '{min_writer_version}' is invalid, valid values are ['2','3','4','5','6','7']"
576 )));
577 }
578 },
579 Err(_) => {
580 return Err(Error::Generic(format!(
581 "delta.minWriterVersion = '{min_writer_version}' is invalid, valid values are ['2','3','4','5','6','7']"
582 )));
583 }
584 }
585 }
586
587 if let Some(min_reader_version) = parsed_properties.get(&TableProperty::MinReaderVersion) {
589 let new_min_reader_version = min_reader_version.parse::<i32>();
590 match new_min_reader_version {
591 Ok(version) => match version {
592 3 => self.upgrade_to_reader_features(),
593 1..=2 => {
594 if version > self.min_reader_version {
595 self.min_reader_version = version
596 }
597 }
598 _ => {
599 return Err(Error::Generic(format!(
600 "delta.minReaderVersion = '{min_reader_version}' is invalid, valid values are ['1','2','3']"
601 )));
602 }
603 },
604 Err(_) => {
605 return Err(Error::Generic(format!(
606 "delta.minReaderVersion = '{min_reader_version}' is invalid, valid values are ['1','2','3']"
607 )));
608 }
609 }
610 }
611
612 if let Some(mode) = parsed_properties.get(&TableProperty::ColumnMappingMode) {
614 if mode.as_str() != "none" {
615 if self.min_reader_version >= 3 {
616 self.reader_features
617 .get_or_insert_with(HashSet::new)
618 .insert(TableFeature::ColumnMapping);
619 } else {
620 self.min_reader_version = self.min_reader_version.max(2);
621 }
622 if self.min_writer_version >= 7 {
623 self.writer_features
624 .get_or_insert_with(HashSet::new)
625 .insert(TableFeature::ColumnMapping);
626 } else {
627 self.min_writer_version = self.min_writer_version.max(5);
628 }
629 }
630 }
631
632 if let Some(enable_cdf) = parsed_properties.get(&TableProperty::EnableChangeDataFeed) {
634 let if_enable_cdf = enable_cdf.to_ascii_lowercase().parse::<bool>();
635 match if_enable_cdf {
636 Ok(true) => {
637 if self.min_writer_version >= 7 {
638 match self.writer_features {
639 Some(mut features) => {
640 features.insert(TableFeature::ChangeDataFeed);
641 self.writer_features = Some(features);
642 }
643 None => {
644 self.writer_features =
645 Some(HashSet::from([TableFeature::ChangeDataFeed]))
646 }
647 }
648 } else if self.min_writer_version <= 3 {
649 self.min_writer_version = 4
650 }
651 }
652 Ok(false) => {}
653 _ => {
654 return Err(Error::Generic(format!(
655 "delta.enableChangeDataFeed = '{enable_cdf}' is invalid, valid values are ['true']"
656 )));
657 }
658 }
659 }
660
661 if let Some(enable_dv) = parsed_properties.get(&TableProperty::EnableDeletionVectors) {
662 let if_enable_dv = enable_dv.to_ascii_lowercase().parse::<bool>();
663 match if_enable_dv {
664 Ok(true) => {
665 self.upgrade_to_reader_features();
666 self.upgrade_to_writer_features();
667 self.reader_features
668 .get_or_insert_with(HashSet::new)
669 .insert(TableFeature::DeletionVectors);
670 self.writer_features
671 .get_or_insert_with(HashSet::new)
672 .insert(TableFeature::DeletionVectors);
673 }
674 Ok(false) => {}
675 _ => {
676 return Err(Error::Generic(format!(
677 "delta.enableDeletionVectors = '{enable_dv}' is invalid, valid values are ['true']"
678 )));
679 }
680 }
681 }
682
683 if self.min_writer_version >= 7
686 && let Some(features) = writer_features_for_version(self.min_writer_version)
687 {
688 self = self.append_writer_features(features);
689 if self.min_reader_version < 2 {
692 self.min_reader_version = 2;
693 }
694 }
695 if self.min_reader_version >= 3 && self.reader_features.is_none() {
698 self.reader_features = Some(HashSet::new());
699 }
700 Ok(self)
701 }
702
703 fn contains_timestampntz<'a>(&self, fields: impl Iterator<Item = &'a StructField>) -> bool {
705 contains_timestampntz(fields)
706 }
707
708 fn contains_variant<'a>(&self, fields: impl Iterator<Item = &'a StructField>) -> bool {
710 contains_variant(fields)
711 }
712
713 fn enable_timestamp_ntz(mut self) -> Self {
715 self = self.append_reader_features([TableFeature::TimestampWithoutTimezone]);
716 self = self.append_writer_features([TableFeature::TimestampWithoutTimezone]);
717 self
718 }
719
720 #[cfg(feature = "nanosecond-timestamps")]
721 fn contains_timestamp_nanos<'a>(&self, fields: impl Iterator<Item = &'a StructField>) -> bool {
723 contains_timestamp_nanos(fields)
724 }
725
726 #[cfg(feature = "nanosecond-timestamps")]
727 fn enable_timestamp_nanos(mut self) -> Self {
729 self = self.append_reader_features([TableFeature::TimestampNanos]);
730 self = self.append_writer_features([TableFeature::TimestampNanos]);
731 self
732 }
733
734 fn enable_variant_type(mut self) -> Self {
736 self = self.append_reader_features([TableFeature::VariantType]);
737 self = self.append_writer_features([TableFeature::VariantType]);
738 self
739 }
740
741 fn enable_generated_columns(mut self) -> Self {
743 if self.min_writer_version < 4 {
744 self.min_writer_version = 4;
745 }
746 if self.min_writer_version >= 7 {
747 self = self.append_writer_features([TableFeature::GeneratedColumns]);
748 }
749 self
750 }
751
752 fn enable_invariants(mut self) -> Self {
754 if self.min_writer_version >= 7 {
755 self = self.append_writer_features([TableFeature::Invariants]);
756 }
757 self
758 }
759}
760
761#[derive(Serialize, Deserialize, Debug, Clone, Eq, PartialEq, Hash)]
763#[serde(rename_all = "camelCase")]
764pub enum TableFeatures {
765 ColumnMapping,
767 DeletionVectors,
769 #[serde(rename = "timestampNtz")]
771 TimestampWithoutTimezone,
772 #[cfg(feature = "nanosecond-timestamps")]
773 #[serde(rename = "timestampNanos")]
774 TimestampNanos,
776 V2Checkpoint,
778 AppendOnly,
780 Invariants,
782 CheckConstraints,
784 ChangeDataFeed,
786 GeneratedColumns,
788 IdentityColumns,
790 RowTracking,
792 DomainMetadata,
794 IcebergCompatV1,
796 VariantType,
798 VariantTypePreview,
800 VariantShreddingPreview,
802 MaterializePartitionColumns,
804}
805
806impl FromStr for TableFeatures {
807 type Err = ();
808
809 fn from_str(value: &str) -> Result<Self, Self::Err> {
810 match value {
811 "columnMapping" => Ok(TableFeatures::ColumnMapping),
812 "deletionVectors" => Ok(TableFeatures::DeletionVectors),
813 "timestampNtz" => Ok(TableFeatures::TimestampWithoutTimezone),
814 #[cfg(feature = "nanosecond-timestamps")]
815 "timestampNanos" => Ok(TableFeatures::TimestampNanos),
816 "v2Checkpoint" => Ok(TableFeatures::V2Checkpoint),
817 "appendOnly" => Ok(TableFeatures::AppendOnly),
818 "invariants" => Ok(TableFeatures::Invariants),
819 "checkConstraints" => Ok(TableFeatures::CheckConstraints),
820 "changeDataFeed" => Ok(TableFeatures::ChangeDataFeed),
821 "generatedColumns" => Ok(TableFeatures::GeneratedColumns),
822 "identityColumns" => Ok(TableFeatures::IdentityColumns),
823 "rowTracking" => Ok(TableFeatures::RowTracking),
824 "domainMetadata" => Ok(TableFeatures::DomainMetadata),
825 "icebergCompatV1" => Ok(TableFeatures::IcebergCompatV1),
826 "variantType" => Ok(TableFeatures::VariantType),
827 "variantType-preview" => Ok(TableFeatures::VariantTypePreview),
828 "variantShredding-preview" => Ok(TableFeatures::VariantShreddingPreview),
829 "materializePartitionColumns" => Ok(TableFeatures::MaterializePartitionColumns),
830 _ => Err(()),
831 }
832 }
833}
834
835impl AsRef<str> for TableFeatures {
836 fn as_ref(&self) -> &str {
837 match self {
838 TableFeatures::ColumnMapping => "columnMapping",
839 TableFeatures::DeletionVectors => "deletionVectors",
840 TableFeatures::TimestampWithoutTimezone => "timestampNtz",
841 #[cfg(feature = "nanosecond-timestamps")]
842 TableFeatures::TimestampNanos => "timestampNanos",
843 TableFeatures::V2Checkpoint => "v2Checkpoint",
844 TableFeatures::AppendOnly => "appendOnly",
845 TableFeatures::Invariants => "invariants",
846 TableFeatures::CheckConstraints => "checkConstraints",
847 TableFeatures::ChangeDataFeed => "changeDataFeed",
848 TableFeatures::GeneratedColumns => "generatedColumns",
849 TableFeatures::IdentityColumns => "identityColumns",
850 TableFeatures::RowTracking => "rowTracking",
851 TableFeatures::DomainMetadata => "domainMetadata",
852 TableFeatures::IcebergCompatV1 => "icebergCompatV1",
853 TableFeatures::VariantType => "variantType",
854 TableFeatures::VariantTypePreview => "variantType-preview",
855 TableFeatures::VariantShreddingPreview => "variantShredding-preview",
856 TableFeatures::MaterializePartitionColumns => "materializePartitionColumns",
857 }
858 }
859}
860
861impl fmt::Display for TableFeatures {
862 fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
863 write!(f, "{}", self.as_ref())
864 }
865}
866
867impl TryFrom<&TableFeatures> for TableFeature {
868 type Error = std::convert::Infallible;
869
870 fn try_from(value: &TableFeatures) -> Result<Self, Self::Error> {
871 TableFeature::try_from(value.as_ref())
872 }
873}
874
875impl TableFeatures {
876 pub fn to_reader_writer_features(&self) -> (Option<TableFeature>, Option<TableFeature>) {
878 let feature = TableFeature::try_from(self).ok();
879 match feature {
880 Some(feature) => {
881 match feature {
884 TableFeature::AppendOnly
885 | TableFeature::Invariants
886 | TableFeature::CheckConstraints
887 | TableFeature::ChangeDataFeed
888 | TableFeature::GeneratedColumns
889 | TableFeature::IdentityColumns
890 | TableFeature::InCommitTimestamp
891 | TableFeature::RowTracking
892 | TableFeature::DomainMetadata
893 | TableFeature::IcebergCompatV1
894 | TableFeature::IcebergCompatV2
895 | TableFeature::ClusteredTable
896 | TableFeature::MaterializePartitionColumns => (None, Some(feature)),
897
898 TableFeature::CatalogManaged
900 | TableFeature::CatalogOwnedPreview
901 | TableFeature::ColumnMapping
902 | TableFeature::DeletionVectors
903 | TableFeature::TimestampWithoutTimezone
904 | TableFeature::TypeWidening
905 | TableFeature::TypeWideningPreview
906 | TableFeature::V2Checkpoint
907 | TableFeature::VacuumProtocolCheck
908 | TableFeature::VariantType
909 | TableFeature::VariantTypePreview
910 | TableFeature::VariantShreddingPreview => {
911 (Some(feature.clone()), Some(feature))
912 }
913
914 #[cfg(feature = "nanosecond-timestamps")]
916 TableFeature::TimestampNanos => (Some(feature.clone()), Some(feature)),
917
918 TableFeature::Unknown(_) => (None, None),
920 others => {
921 panic!("This table has unsupported table features: {others:?}");
922 }
923 }
924 }
925 None => (None, None),
926 }
927 }
928}
929
930#[derive(Serialize, Deserialize, Copy, Clone, Debug, PartialEq, Eq, Default)]
932pub enum StorageType {
933 #[serde(rename = "u")]
935 #[default]
936 UuidRelativePath,
937 #[serde(rename = "i")]
939 Inline,
940 #[serde(rename = "p")]
942 AbsolutePath,
943}
944
945impl FromStr for StorageType {
946 type Err = Error;
947
948 fn from_str(s: &str) -> Result<Self, Self::Err> {
949 match s {
950 "u" => Ok(Self::UuidRelativePath),
951 "i" => Ok(Self::Inline),
952 "p" => Ok(Self::AbsolutePath),
953 _ => Err(Error::DeletionVector(format!(
954 "Unknown storage format: '{s}'."
955 ))),
956 }
957 }
958}
959
960impl AsRef<str> for StorageType {
961 fn as_ref(&self) -> &str {
962 match self {
963 Self::UuidRelativePath => "u",
964 Self::Inline => "i",
965 Self::AbsolutePath => "p",
966 }
967 }
968}
969
970impl Display for StorageType {
971 fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
972 write!(f, "{}", self.as_ref())
973 }
974}
975
976#[derive(Serialize, Deserialize, Debug, Clone, PartialEq, Eq)]
977#[serde(rename_all = "camelCase")]
978pub struct DeletionVectorDescriptor {
980 pub storage_type: StorageType,
982
983 pub path_or_inline_dv: String,
998
999 #[serde(skip_serializing_if = "Option::is_none")]
1002 pub offset: Option<i32>,
1003
1004 pub size_in_bytes: i32,
1006
1007 pub cardinality: i64,
1009}
1010
1011impl TryFrom<DeletionVectorDescriptor>
1012 for delta_kernel::actions::deletion_vector::DeletionVectorDescriptor
1013{
1014 type Error = delta_kernel::Error;
1015
1016 fn try_from(dv: DeletionVectorDescriptor) -> Result<Self, Self::Error> {
1017 use delta_kernel::actions::deletion_vector::DeletionVectorStorageType;
1018
1019 let storage_type = match dv.storage_type {
1020 StorageType::UuidRelativePath => DeletionVectorStorageType::PersistedRelative,
1021 StorageType::Inline => DeletionVectorStorageType::Inline,
1022 StorageType::AbsolutePath => DeletionVectorStorageType::PersistedAbsolute,
1023 };
1024
1025 Self::try_new(
1026 storage_type,
1027 dv.path_or_inline_dv,
1028 dv.offset,
1029 dv.size_in_bytes,
1030 dv.cardinality,
1031 )
1032 }
1033}
1034
1035#[derive(Serialize, Deserialize, Debug, Clone, Default)]
1036#[serde(rename_all = "camelCase")]
1037pub struct Add {
1039 #[serde(with = "serde_path")]
1045 pub path: String,
1046
1047 pub partition_values: HashMap<String, Option<String>>,
1049
1050 pub size: i64,
1052
1053 pub modification_time: i64,
1055
1056 pub data_change: bool,
1059
1060 pub stats: Option<String>,
1064
1065 pub tags: Option<HashMap<String, Option<String>>>,
1067
1068 #[serde(skip_serializing_if = "Option::is_none")]
1069 pub deletion_vector: Option<DeletionVectorDescriptor>,
1071
1072 pub base_row_id: Option<i64>,
1076
1077 pub default_row_commit_version: Option<i64>,
1079
1080 pub clustering_provider: Option<String>,
1082}
1083
1084#[derive(Serialize, Deserialize, Debug, Clone, Eq, Default)]
1086#[serde(rename_all = "camelCase")]
1087pub struct Remove {
1088 #[serde(with = "serde_path")]
1094 pub path: String,
1095
1096 pub data_change: bool,
1099
1100 #[serde(skip_serializing_if = "Option::is_none")]
1102 pub deletion_timestamp: Option<i64>,
1103
1104 #[serde(skip_serializing_if = "Option::is_none")]
1106 pub extended_file_metadata: Option<bool>,
1107
1108 #[serde(skip_serializing_if = "Option::is_none")]
1110 pub partition_values: Option<HashMap<String, Option<String>>>,
1111
1112 #[serde(skip_serializing_if = "Option::is_none")]
1114 pub size: Option<i64>,
1115
1116 #[serde(skip_serializing_if = "Option::is_none")]
1118 pub tags: Option<HashMap<String, Option<String>>>,
1119
1120 #[serde(skip_serializing_if = "Option::is_none")]
1122 pub deletion_vector: Option<DeletionVectorDescriptor>,
1123
1124 #[serde(skip_serializing_if = "Option::is_none")]
1128 pub base_row_id: Option<i64>,
1129
1130 #[serde(skip_serializing_if = "Option::is_none")]
1132 pub default_row_commit_version: Option<i64>,
1133}
1134
1135#[derive(Serialize, Deserialize, Clone, Debug, Default, PartialEq, Eq)]
1137#[serde(rename_all = "camelCase")]
1138pub struct AddCDCFile {
1139 #[serde(with = "serde_path")]
1142 pub path: String,
1143
1144 pub size: i64,
1146
1147 pub partition_values: HashMap<String, Option<String>>,
1149
1150 pub data_change: bool,
1152
1153 #[serde(skip_serializing_if = "Option::is_none")]
1155 pub tags: Option<HashMap<String, Option<String>>>,
1156}
1157
1158#[derive(Serialize, Deserialize, Debug, Default, Clone, PartialEq, Eq)]
1161#[serde(rename_all = "camelCase")]
1162pub struct Transaction {
1163 pub app_id: String,
1165
1166 pub version: i64,
1168
1169 #[serde(skip_serializing_if = "Option::is_none")]
1171 pub last_updated: Option<i64>,
1172}
1173
1174impl Transaction {
1175 pub fn new(app_id: impl ToString, version: i64) -> Self {
1177 Self::new_with_last_update(app_id, version, None)
1178 }
1179
1180 pub fn new_with_last_update(
1182 app_id: impl ToString,
1183 version: i64,
1184 last_updated: Option<i64>,
1185 ) -> Self {
1186 Transaction {
1187 app_id: app_id.to_string(),
1188 version,
1189 last_updated,
1190 }
1191 }
1192}
1193
1194#[derive(Serialize, Deserialize, Debug, Clone, Default, PartialEq, Eq)]
1198#[serde(rename_all = "camelCase")]
1199pub struct CommitInfo {
1200 #[serde(skip_serializing_if = "Option::is_none")]
1202 pub timestamp: Option<i64>,
1203
1204 #[serde(skip_serializing_if = "Option::is_none")]
1206 pub in_commit_timestamp: Option<i64>,
1207
1208 #[serde(skip_serializing_if = "Option::is_none")]
1210 pub user_id: Option<String>,
1211
1212 #[serde(skip_serializing_if = "Option::is_none")]
1214 pub user_name: Option<String>,
1215
1216 #[serde(skip_serializing_if = "Option::is_none")]
1218 pub operation: Option<String>,
1219
1220 #[serde(skip_serializing_if = "Option::is_none")]
1222 pub operation_parameters: Option<HashMap<String, serde_json::Value>>,
1223
1224 #[serde(skip_serializing_if = "Option::is_none")]
1226 pub read_version: Option<Version>,
1227
1228 #[serde(skip_serializing_if = "Option::is_none")]
1230 pub isolation_level: Option<IsolationLevel>,
1231
1232 #[serde(skip_serializing_if = "Option::is_none")]
1234 pub is_blind_append: Option<bool>,
1235
1236 #[serde(skip_serializing_if = "Option::is_none")]
1238 pub engine_info: Option<String>,
1239
1240 #[serde(flatten, default)]
1242 pub info: HashMap<String, serde_json::Value>,
1243
1244 #[serde(skip_serializing_if = "Option::is_none")]
1246 pub user_metadata: Option<String>,
1247}
1248
1249#[derive(Serialize, Deserialize, Debug, Clone, Default, PartialEq, Eq)]
1251#[serde(rename_all = "camelCase")]
1252pub struct DomainMetadata {
1253 pub domain: String,
1255
1256 pub configuration: String,
1258
1259 pub removed: bool,
1261}
1262
1263#[derive(Serialize, Deserialize, Debug, Clone, Default, PartialEq, Eq)]
1264pub struct CheckpointMetadata {
1266 pub flavor: String,
1268
1269 #[serde(skip_serializing_if = "Option::is_none")]
1271 pub tags: Option<HashMap<String, Option<String>>>,
1272}
1273
1274#[derive(Serialize, Deserialize, Debug, Clone, Default, PartialEq, Eq)]
1277#[serde(rename_all = "camelCase")]
1278pub struct Sidecar {
1279 pub file_name: String,
1282
1283 pub size_in_bytes: i64,
1285
1286 pub modification_time: i64,
1288
1289 #[serde(rename = "type")]
1292 pub sidecar_type: String,
1293
1294 #[serde(skip_serializing_if = "Option::is_none")]
1296 pub tags: Option<HashMap<String, Option<String>>>,
1297}
1298
1299#[derive(Serialize, Deserialize, Debug, Copy, Clone, PartialEq, Eq)]
1300#[derive(Default)]
1302pub enum IsolationLevel {
1303 #[default]
1309 Serializable,
1310
1311 WriteSerializable,
1316
1317 SnapshotIsolation,
1322}
1323
1324impl AsRef<str> for IsolationLevel {
1328 fn as_ref(&self) -> &str {
1329 match self {
1330 Self::Serializable => "Serializable",
1331 Self::WriteSerializable => "WriteSerializable",
1332 Self::SnapshotIsolation => "SnapshotIsolation",
1333 }
1334 }
1335}
1336
1337impl FromStr for IsolationLevel {
1338 type Err = Error;
1339
1340 fn from_str(s: &str) -> Result<Self, Self::Err> {
1341 match s.to_ascii_lowercase().as_str() {
1342 "serializable" => Ok(Self::Serializable),
1343 "writeserializable" | "write_serializable" => Ok(Self::WriteSerializable),
1344 "snapshotisolation" | "snapshot_isolation" => Ok(Self::SnapshotIsolation),
1345 _ => Err(Error::Generic("Invalid string for IsolationLevel".into())),
1346 }
1347 }
1348}
1349
1350pub(crate) mod serde_path {
1351 use std::str::Utf8Error;
1352
1353 use percent_encoding::percent_decode_str;
1354 use percent_encoding_rfc3986::{AsciiSet, CONTROLS, utf8_percent_encode};
1355 use serde::{self, Deserialize, Deserializer, Serialize, Serializer};
1356
1357 pub fn deserialize<'de, D>(deserializer: D) -> Result<String, D::Error>
1358 where
1359 D: Deserializer<'de>,
1360 {
1361 let s = String::deserialize(deserializer)?;
1362 decode_path(&s).map_err(serde::de::Error::custom)
1363 }
1364
1365 pub fn serialize<S>(value: &str, serializer: S) -> Result<S::Ok, S::Error>
1366 where
1367 S: Serializer,
1368 {
1369 let encoded = encode_path(value);
1370 String::serialize(&encoded, serializer)
1371 }
1372
1373 pub const _DELIMITER: &str = "/";
1374 pub const _DELIMITER_BYTE: u8 = _DELIMITER.as_bytes()[0];
1376
1377 const INVALID: &AsciiSet = &CONTROLS
1382 .add(b' ')
1386 .add(b'\\')
1389 .add(b'{')
1390 .add(b'^')
1391 .add(b'}')
1392 .add(b'%')
1393 .add(b'`')
1394 .add(b']')
1395 .add(b'"')
1396 .add(b'>')
1397 .add(b'[')
1398 .add(b'<')
1400 .add(b'#')
1401 .add(b'|')
1402 .add(b'\r')
1405 .add(b'\n')
1406 .add(b'*')
1407 .add(b'?');
1408
1409 pub(crate) fn encode_path(path: &str) -> String {
1410 utf8_percent_encode(path, INVALID).to_string()
1411 }
1412
1413 pub fn decode_path(path: &str) -> Result<String, Utf8Error> {
1415 Ok(percent_decode_str(path).decode_utf8()?.to_string())
1416 }
1417}
1418
1419pub(crate) fn writer_features_for_version(
1421 writer_version: i32,
1422) -> Option<Vec<delta_kernel::table_features::TableFeature>> {
1423 if writer_version >= 7 {
1424 Some(vec![
1426 delta_kernel::table_features::TableFeature::Invariants,
1427 delta_kernel::table_features::TableFeature::AppendOnly,
1428 ])
1429 } else {
1430 None
1432 }
1433}
1434
1435#[cfg(test)]
1436mod tests {
1437 use super::*;
1438 use crate::kernel::PrimitiveType;
1439
1440 #[test]
1441 fn test_primitive() {
1442 let types: PrimitiveType = serde_json::from_str("\"string\"").unwrap();
1443 println!("{types:?}");
1444 }
1445
1446 fn nested_in_maps(dtype: DataType) -> Vec<DataType> {
1447 use delta_kernel::schema::{ArrayType, MapType};
1448 let map_key = DataType::Map(Box::new(MapType::new(
1449 dtype.clone(),
1450 DataType::STRING,
1451 true,
1452 )));
1453 let map_value = DataType::Map(Box::new(MapType::new(DataType::STRING, dtype, true)));
1454 let array_of_map = DataType::Array(Box::new(ArrayType::new(map_value.clone(), true)));
1455 vec![map_key, map_value, array_of_map]
1456 }
1457
1458 #[test]
1459 fn test_contains_timestampntz_in_map() {
1460 for dtype in nested_in_maps(DataType::TIMESTAMP_NTZ) {
1461 let field = StructField::nullable("c", dtype.clone());
1462 assert!(contains_timestampntz(std::iter::once(&field)), "{dtype:?}");
1463 }
1464 let field = StructField::nullable("c", nested_in_maps(DataType::TIMESTAMP).remove(0));
1465 assert!(!contains_timestampntz(std::iter::once(&field)));
1466 }
1467
1468 #[cfg(feature = "nanosecond-timestamps")]
1469 #[test]
1470 fn test_contains_timestamp_nanos_in_map() {
1471 for dtype in nested_in_maps(DataType::TIMESTAMP_NANOS_NTZ) {
1472 let field = StructField::nullable("c", dtype.clone());
1473 assert!(
1474 contains_timestamp_nanos(std::iter::once(&field)),
1475 "{dtype:?}"
1476 );
1477 }
1478 }
1479
1480 #[test]
1481 fn test_deserialize_protocol() {
1482 let raw = serde_json::json!(
1484 {
1485 "minReaderVersion": 3,
1486 "minWriterVersion": 7,
1487 "readerFeatures": ["catalogOwned"],
1488 "writerFeatures": ["catalogOwned", "invariants", "appendOnly"]
1489 }
1490 );
1491 let protocol: Protocol = serde_json::from_value(raw).unwrap();
1492 assert_eq!(protocol.min_reader_version(), 3);
1493 assert_eq!(protocol.min_writer_version(), 7);
1494 assert_eq!(
1495 protocol.reader_features(),
1496 Some(vec![TableFeature::Unknown("catalogOwned".to_owned())].as_slice())
1497 );
1498 assert_eq!(
1499 protocol.writer_features(),
1500 Some(
1501 vec![
1502 TableFeature::Unknown("catalogOwned".to_owned()),
1503 TableFeature::Invariants,
1504 TableFeature::AppendOnly
1505 ]
1506 .as_slice()
1507 )
1508 );
1509 }
1510
1511 #[test]
1512 fn test_serialize_add_path_encodes_space() {
1513 let add = Add {
1514 path: "year=2021/part 0.parquet".to_string(),
1515 ..Default::default()
1516 };
1517
1518 let value = serde_json::to_value(&add).unwrap();
1519 assert_eq!(value["path"], "year=2021/part%200.parquet");
1520 }
1521
1522 #[test]
1523 fn test_serialize_remove_and_cdc_path_encodes_space() {
1524 let remove = Remove {
1525 path: "part 1.parquet".to_string(),
1526 ..Default::default()
1527 };
1528 let cdc = AddCDCFile {
1529 path: "_change_data/part 2.parquet".to_string(),
1530 ..Default::default()
1531 };
1532
1533 assert_eq!(
1534 serde_json::to_value(&remove).unwrap()["path"],
1535 "part%201.parquet"
1536 );
1537 assert_eq!(
1538 serde_json::to_value(&cdc).unwrap()["path"],
1539 "_change_data/part%202.parquet"
1540 );
1541 }
1542
1543 #[test]
1544 fn test_serialize_path_preserves_hive_partition_delimiters() {
1545 let add = Add {
1548 path: "year=2020/month=2/day=3/part-00000-abc.snappy.parquet".to_string(),
1549 ..Default::default()
1550 };
1551
1552 let value = serde_json::to_value(&add).unwrap();
1553 assert_eq!(
1554 value["path"],
1555 "year=2020/month=2/day=3/part-00000-abc.snappy.parquet"
1556 );
1557 }
1558
1559 #[test]
1560 fn test_serialize_round_trips_spark_written_path() {
1561 let path = "bool=false/time=1970-01-02%2008%253A45%253A00/amount=12.000000000000000000/part-00000-12004196-1e98-4a42-a622-a12f05a4578e.c000.snappy.parquet";
1565 let raw = serde_json::json!({
1566 "path": path,
1567 "partitionValues": {},
1568 "size": 0,
1569 "modificationTime": 0,
1570 "dataChange": true
1571 });
1572
1573 let add: Add = serde_json::from_value(raw).unwrap();
1574 assert_eq!(serde_json::to_value(&add).unwrap()["path"], path);
1575 }
1576
1577 #[test]
1578 fn test_deserialize_path_accepts_unencoded_space() {
1579 let raw = serde_json::json!({
1581 "path": "city=New York/part-00000-abc.parquet",
1582 "partitionValues": {},
1583 "size": 0,
1584 "modificationTime": 0,
1585 "dataChange": true
1586 });
1587
1588 let add: Add = serde_json::from_value(raw).unwrap();
1589 assert_eq!(add.path, "city=New York/part-00000-abc.parquet");
1590 }
1591
1592 #[test]
1593 fn test_apply_properties_column_mapping_legacy_versions() {
1594 let protocol: Protocol = serde_json::from_value(serde_json::json!({
1595 "minReaderVersion": 1,
1596 "minWriterVersion": 2,
1597 }))
1598 .unwrap();
1599 let config = HashMap::from([("delta.columnMapping.mode".to_string(), "name".to_string())]);
1600
1601 let protocol = protocol
1602 .apply_properties_to_protocol(&config, true)
1603 .unwrap();
1604
1605 assert_eq!(protocol.min_reader_version(), 2);
1606 assert_eq!(protocol.min_writer_version(), 5);
1607 }
1608
1609 #[test]
1610 fn test_apply_properties_column_mapping_table_features() {
1611 let protocol: Protocol = serde_json::from_value(serde_json::json!({
1614 "minReaderVersion": 3,
1615 "minWriterVersion": 7,
1616 "readerFeatures": [],
1617 "writerFeatures": [],
1618 }))
1619 .expect("Failed to initialize a new Protocol");
1620 let config = HashMap::from([("delta.columnMapping.mode".to_string(), "name".to_string())]);
1621
1622 let protocol = protocol
1623 .apply_properties_to_protocol(&config, true)
1624 .expect("Failed to apply properties to protocol");
1625
1626 assert_eq!(protocol.min_reader_version(), 3);
1627 assert_eq!(protocol.min_writer_version(), 7);
1628 assert!(
1629 protocol
1630 .reader_features()
1631 .unwrap()
1632 .contains(&TableFeature::ColumnMapping),
1633 "readerFeatures should contain ColumnMapping, got: {:?}",
1634 protocol.reader_features()
1635 );
1636 assert!(
1637 protocol
1638 .writer_features()
1639 .unwrap()
1640 .contains(&TableFeature::ColumnMapping),
1641 "writerFeatures should contain ColumnMapping, got: {:?}",
1642 protocol.writer_features()
1643 );
1644 }
1645
1646 #[test]
1647 fn test_apply_properties_does_not_add_unused_variant_type() {
1648 let timestamp_ntz: Protocol = serde_json::from_value(serde_json::json!({
1651 "minReaderVersion": 3,
1652 "minWriterVersion": 7,
1653 "readerFeatures": ["timestampNtz"],
1654 "writerFeatures": ["timestampNtz"],
1655 }))
1656 .unwrap();
1657 let legacy: Protocol = serde_json::from_value(serde_json::json!({
1658 "minReaderVersion": 1,
1659 "minWriterVersion": 2,
1660 }))
1661 .unwrap();
1662 let cases = [
1663 (timestamp_ntz, HashMap::new()),
1664 (
1665 legacy,
1666 HashMap::from([
1667 ("delta.minReaderVersion".to_string(), "3".to_string()),
1668 ("delta.minWriterVersion".to_string(), "7".to_string()),
1669 ]),
1670 ),
1671 ];
1672
1673 for (protocol, config) in cases {
1674 let protocol = protocol
1675 .apply_properties_to_protocol(&config, true)
1676 .unwrap();
1677
1678 assert_eq!(protocol.min_reader_version(), 3);
1679 assert_eq!(protocol.min_writer_version(), 7);
1680 for features in [protocol.reader_features(), protocol.writer_features()] {
1681 assert!(
1682 !features.is_some_and(|f| f.contains(&TableFeature::VariantType)),
1683 "variantType should not be added without a variant column, got: {features:?}"
1684 );
1685 }
1686 }
1687 }
1688
1689 #[test]
1690 fn test_apply_properties_column_mapping_independent_reader_writer() {
1691 let protocol: Protocol = serde_json::from_value(serde_json::json!({
1694 "minReaderVersion": 1,
1695 "minWriterVersion": 7,
1696 "writerFeatures": [],
1697 }))
1698 .unwrap();
1699 let config = HashMap::from([("delta.columnMapping.mode".to_string(), "name".to_string())]);
1700
1701 let protocol = protocol
1702 .apply_properties_to_protocol(&config, true)
1703 .unwrap();
1704
1705 assert_eq!(protocol.min_reader_version(), 2);
1706 assert_eq!(protocol.min_writer_version(), 7);
1707 assert!(
1708 protocol
1709 .writer_features()
1710 .unwrap()
1711 .contains(&TableFeature::ColumnMapping),
1712 "writerFeatures should contain ColumnMapping, got: {:?}",
1713 protocol.writer_features()
1714 );
1715
1716 assert!(
1717 protocol
1718 .writer_features()
1719 .unwrap()
1720 .contains(&TableFeature::Invariants),
1721 "writerFeatures should be upgraded include Invariants, got: {:?}",
1722 protocol.writer_features()
1723 );
1724
1725 assert!(
1727 !protocol
1728 .reader_features()
1729 .map(|f| f.contains(&TableFeature::ColumnMapping))
1730 .unwrap_or(false),
1731 "reader side should remain legacy, got: {:?}",
1732 protocol.reader_features()
1733 );
1734 }
1735
1736 fn table_features_protocol(
1739 reader_features: impl IntoIterator<Item = TableFeature>,
1740 writer_features: impl IntoIterator<Item = TableFeature>,
1741 ) -> ProtocolInner {
1742 ProtocolInner {
1743 min_reader_version: 3,
1744 min_writer_version: 7,
1745 reader_features: Some(reader_features.into_iter().collect()),
1746 writer_features: Some(writer_features.into_iter().collect()),
1747 }
1748 }
1749
1750 fn enable_deletion_vectors() -> HashMap<String, String> {
1753 HashMap::from([(
1754 "delta.enableDeletionVectors".to_string(),
1755 "true".to_string(),
1756 )])
1757 }
1758
1759 #[test]
1760 fn test_enable_deletion_vectors_keeps_legacy_implied_features() {
1761 use TableFeature::*;
1762 let writer_4_features = [
1763 AppendOnly,
1764 Invariants,
1765 CheckConstraints,
1766 ChangeDataFeed,
1767 GeneratedColumns,
1768 DeletionVectors,
1769 ];
1770 let table_features =
1771 table_features_protocol([DeletionVectors], [AppendOnly, Invariants, DeletionVectors]);
1772 let cases = [
1773 ((1, 2), table_features.clone()),
1774 (
1775 (1, 3),
1776 table_features_protocol(
1777 [DeletionVectors],
1778 [AppendOnly, Invariants, CheckConstraints, DeletionVectors],
1779 ),
1780 ),
1781 (
1782 (1, 4),
1783 table_features_protocol([DeletionVectors], writer_4_features.clone()),
1784 ),
1785 (
1786 (1, 5),
1787 table_features_protocol([DeletionVectors], writer_4_features.clone()),
1788 ),
1789 (
1790 (2, 5),
1791 table_features_protocol(
1792 [ColumnMapping, DeletionVectors],
1793 writer_4_features.clone().into_iter().chain([ColumnMapping]),
1794 ),
1795 ),
1796 (
1797 (2, 6),
1798 table_features_protocol(
1799 [ColumnMapping, DeletionVectors],
1800 writer_4_features
1801 .clone()
1802 .into_iter()
1803 .chain([ColumnMapping, IdentityColumns]),
1804 ),
1805 ),
1806 ];
1807 let cases = cases
1808 .into_iter()
1809 .map(|((reader, writer), expected)| (ProtocolInner::new(reader, writer), expected))
1810 .chain([(table_features.clone(), table_features)]);
1811
1812 for (protocol, expected) in cases {
1813 let upgraded = protocol
1814 .clone()
1815 .apply_properties_to_protocol(&enable_deletion_vectors(), true)
1816 .unwrap();
1817 assert_eq!(upgraded, expected, "upgrading {protocol:?}");
1818 let upgraded_again = upgraded
1819 .apply_properties_to_protocol(&enable_deletion_vectors(), true)
1820 .unwrap();
1821 assert_eq!(upgraded_again, expected, "upgrading {protocol:?} twice");
1822 }
1823 }
1824
1825 #[test]
1826 fn test_every_table_features_upgrade_keeps_column_mapping() {
1827 let column_mapped = || ProtocolInner::new(2, 5);
1828 let min_versions = HashMap::from([
1829 ("delta.minReaderVersion".to_string(), "3".to_string()),
1830 ("delta.minWriterVersion".to_string(), "7".to_string()),
1831 ]);
1832 let timestamp_ntz_schema =
1833 StructType::try_new([StructField::new("ts", DataType::TIMESTAMP_NTZ, true)]).unwrap();
1834 let upgrades = [
1835 column_mapped()
1836 .append_reader_features([TableFeature::DeletionVectors])
1837 .append_writer_features([TableFeature::DeletionVectors]),
1838 column_mapped()
1839 .apply_column_metadata_to_protocol(×tamp_ntz_schema)
1840 .unwrap(),
1841 column_mapped()
1842 .apply_properties_to_protocol(&min_versions, true)
1843 .unwrap(),
1844 ProtocolInner {
1845 min_reader_version: 2,
1846 min_writer_version: 7,
1847 reader_features: None,
1848 writer_features: Some(HashSet::from([TableFeature::ColumnMapping])),
1849 }
1850 .apply_properties_to_protocol(&enable_deletion_vectors(), true)
1851 .unwrap(),
1852 ];
1853
1854 for upgraded in upgrades {
1855 assert_eq!(
1856 (upgraded.min_reader_version, upgraded.min_writer_version),
1857 (3, 7)
1858 );
1859 for features in [&upgraded.reader_features, &upgraded.writer_features] {
1860 assert!(
1861 features
1862 .as_ref()
1863 .is_some_and(|f| f.contains(&TableFeature::ColumnMapping)),
1864 "columnMapping missing from {upgraded:?}"
1865 );
1866 }
1867 }
1868
1869 let writer_only = column_mapped().append_writer_features([TableFeature::AppendOnly]);
1870 assert_eq!(writer_only.min_reader_version, 2);
1871 assert_eq!(writer_only.reader_features, None);
1872 assert!(
1873 writer_only
1874 .writer_features
1875 .as_ref()
1876 .is_some_and(|f| f.contains(&TableFeature::ColumnMapping)),
1877 "columnMapping missing from {writer_only:?}"
1878 );
1879 }
1880
1881 #[test]
1882 fn test_legacy_protocol_without_table_features_upgrade_is_unchanged() {
1883 let properties = HashMap::from([
1884 ("delta.enableChangeDataFeed".to_string(), "true".to_string()),
1885 ("delta.appendOnly".to_string(), "true".to_string()),
1886 ]);
1887 let unchanged = ProtocolInner::new(2, 5)
1888 .apply_properties_to_protocol(&properties, true)
1889 .unwrap()
1890 .move_table_properties_into_features(&properties);
1891 assert_eq!(unchanged, ProtocolInner::new(2, 5));
1892 }
1893
1894 }