delta_kernel/schema/
derive_macro_utils.rs

1//! Utility traits that support the [`delta_kernel_derive::ToSchema`] macro.
2///
3/// Not intended for use by normal code.
4use std::collections::{HashMap, HashSet};
5
6use bytes::Bytes;
7use delta_kernel_derive::internal_api;
8
9use crate::error::add_scalar_path_context;
10use crate::expressions::{Scalar, StructData};
11use crate::schema::{ArrayType, DataType, MapType, StructField, StructType, ToSchema};
12use crate::utils::require;
13use crate::{DeltaResult, Error};
14
15/// Converts a type to a [`DataType`]. Implemented for the primitive types and automatically derived
16/// for all types that implement [`ToSchema`].
17///
18/// # Warning
19///
20/// If a type implementing this trait also implements `Into<Scalar>`, then for every value `v`, the
21/// scalar `s: Scalar = v.into()` **must** satisfy:
22/// - `!s.is_null()`, and
23/// - `s.data_type() == Self::to_data_type()`.
24///
25/// `IntoScalar` automatically marks every type with both impls, and infallible conversions like
26/// `impl<T: IntoScalar> From<Vec<T>> for Scalar` rely on this contract without runtime validation.
27#[internal_api]
28pub(crate) trait ToDataType {
29    fn to_data_type() -> DataType;
30}
31
32// Blanket impl for all types that implement `ToSchema`
33impl<T: ToSchema> ToDataType for T {
34    fn to_data_type() -> DataType {
35        T::to_schema().into()
36    }
37}
38
39// Helper macro to implement `ToDataType` for primitive types
40macro_rules! impl_to_data_type {
41    ( $(($rust_type: ty, $data_type: expr)), * ) => {
42        $(
43            impl ToDataType for $rust_type {
44                fn to_data_type() -> DataType {
45                    $data_type
46                }
47            }
48        )*
49    };
50}
51
52impl_to_data_type!(
53    (String, DataType::STRING),
54    (Bytes, DataType::BINARY),
55    (i64, DataType::LONG),
56    (i32, DataType::INTEGER),
57    (i16, DataType::SHORT),
58    (i8, DataType::BYTE),
59    (f32, DataType::FLOAT),
60    (f64, DataType::DOUBLE),
61    (bool, DataType::BOOLEAN)
62);
63
64// ToDataType impl for non-nullable array types
65impl<T: ToDataType> ToDataType for Vec<T> {
66    fn to_data_type() -> DataType {
67        ArrayType::new(T::to_data_type(), false).into()
68    }
69}
70
71// ToDataType impl for arrays that may contain null elements
72impl<T: ToDataType> ToDataType for Vec<Option<T>> {
73    fn to_data_type() -> DataType {
74        ArrayType::new(T::to_data_type(), true).into()
75    }
76}
77
78// ToDataType impl for non-nullable set types
79impl<T: ToDataType> ToDataType for HashSet<T> {
80    fn to_data_type() -> DataType {
81        ArrayType::new(T::to_data_type(), false).into()
82    }
83}
84
85// ToDataType impl for non-nullable map types
86impl<K: ToDataType, V: ToDataType> ToDataType for HashMap<K, V> {
87    fn to_data_type() -> DataType {
88        MapType::new(K::to_data_type(), V::to_data_type(), false).into()
89    }
90}
91
92// ToDataType impl for maps with nullable values
93impl<K: ToDataType, V: ToDataType> ToDataType for HashMap<K, Option<V>> {
94    fn to_data_type() -> DataType {
95        MapType::new(K::to_data_type(), V::to_data_type(), true).into()
96    }
97}
98
99/// The [`delta_kernel_derive::ToSchema`] macro uses this to convert a struct field's name + type
100/// into a `StructField` definition. A blanket impl for `Option<T: ToDataType>` supports nullable
101/// struct fields, which otherwise default to non-nullable.
102#[internal_api]
103pub(crate) trait GetStructField {
104    fn get_struct_field(name: impl Into<String>) -> StructField;
105}
106
107// Normal types produce non-nullable fields
108impl<T: ToDataType> GetStructField for T {
109    fn get_struct_field(name: impl Into<String>) -> StructField {
110        StructField::not_null(name, T::to_data_type())
111    }
112}
113
114// Option types produce nullable fields
115impl<T: ToDataType> GetStructField for Option<T> {
116    fn get_struct_field(name: impl Into<String>) -> StructField {
117        StructField::nullable(name, T::to_data_type())
118    }
119}
120
121/// The [`delta_kernel_derive::ToSchema`] macro uses this trait to implement the
122/// `allow_null_container_values` attribute. It is similar to [`ToDataType`], except the containers
123/// it produces have nullable elements, e.g. [`MapType::value_contains_null`] is true.
124pub(crate) trait ToNullableContainerType {
125    fn to_nullable_container_type() -> DataType;
126}
127
128// Blanket impl for maps with nullable values
129impl<K: ToDataType, V: ToDataType> ToNullableContainerType for HashMap<K, V> {
130    fn to_nullable_container_type() -> DataType {
131        MapType::new(K::to_data_type(), V::to_data_type(), true).into()
132    }
133}
134
135// The [`delta_kernel_derive::ToSchema`] macro uses this to convert a struct field's name + type
136// into a `StructField` definition for a container with nullable values, when the struct field was
137// annotated with the `allow_null_container_values` attribute.
138#[internal_api]
139pub(crate) trait GetNullableContainerStructField {
140    fn get_nullable_container_struct_field(name: impl Into<String>) -> StructField;
141}
142
143// Blanket impl for all container types with nullable values
144impl<T: ToNullableContainerType> GetNullableContainerStructField for T {
145    fn get_nullable_container_struct_field(name: impl Into<String>) -> StructField {
146        StructField::not_null(name, T::to_nullable_container_type())
147    }
148}
149
150// Optional container types produce nullable fields with nullable values.
151impl<T: ToNullableContainerType> GetNullableContainerStructField for Option<T> {
152    fn get_nullable_container_struct_field(name: impl Into<String>) -> StructField {
153        StructField::nullable(name, T::to_nullable_container_type())
154    }
155}
156
157/// Named fields consumed by the [`delta_kernel_derive::TryFromStructData`] macro.
158///
159/// Field conversion errors acquire their path element as they unwind. Successful conversion does
160/// not allocate or maintain path state.
161#[internal_api]
162pub(crate) struct StructDataFields {
163    expected: StructType,
164    fields: HashMap<String, (StructField, Scalar)>,
165}
166
167impl StructDataFields {
168    pub(crate) fn try_new(data: StructData, expected: StructType) -> DeltaResult<Self> {
169        let (actual_fields, values) = data.into_parts();
170        require!(
171            actual_fields.len() == values.len(),
172            Error::scalar_conversion(
173                format!("{} struct values", actual_fields.len()),
174                format!("{} struct values", values.len()),
175            )
176        );
177        require!(
178            actual_fields.len() == expected.num_fields(),
179            Error::scalar_conversion(
180                format!("struct with {} fields", expected.num_fields()),
181                format!("struct with {} fields", actual_fields.len()),
182            )
183        );
184        let mut fields = HashMap::with_capacity(actual_fields.len());
185        for (field, value) in actual_fields.into_iter().zip(values) {
186            let name = field.name().clone();
187            match fields.entry(name) {
188                std::collections::hash_map::Entry::Vacant(entry) => {
189                    entry.insert((field, value));
190                }
191                std::collections::hash_map::Entry::Occupied(entry) => {
192                    return Err(add_scalar_path_context(
193                        Error::scalar_conversion("one field", "duplicate fields"),
194                        entry.key().clone(),
195                    ));
196                }
197            }
198        }
199        Ok(Self { expected, fields })
200    }
201
202    pub(crate) fn take_field<T: TryFrom<Scalar, Error = Error>>(
203        &mut self,
204        field_name: &str,
205    ) -> DeltaResult<T> {
206        let expected = self.expected.field(field_name).ok_or_else(|| {
207            Error::InternalError(format!(
208                "Derived schema does not contain generated field {field_name:?}"
209            ))
210        })?;
211        let (actual_field, value) = self.fields.remove(field_name).ok_or_else(|| {
212            add_scalar_path_context(
213                Error::scalar_conversion("present field", "missing field"),
214                field_name,
215            )
216        })?;
217        require!(
218            actual_field.is_nullable() == expected.is_nullable(),
219            add_scalar_path_context(
220                Error::scalar_conversion(
221                    if expected.is_nullable() {
222                        "nullable field"
223                    } else {
224                        "non-nullable field"
225                    },
226                    if actual_field.is_nullable() {
227                        "nullable field"
228                    } else {
229                        "non-nullable field"
230                    },
231                ),
232                field_name,
233            )
234        );
235
236        T::try_from(value).map_err(|error| add_scalar_path_context(error, field_name))
237    }
238
239    /// Verifies that every named field was consumed.
240    pub(crate) fn finish(self) -> DeltaResult<()> {
241        if self.fields.is_empty() {
242            return Ok(());
243        }
244        let mut extra: Vec<_> = self.fields.keys().collect();
245        extra.sort_unstable();
246        Err(Error::scalar_conversion(
247            "no additional fields",
248            format!("fields {extra:?}"),
249        ))
250    }
251}
delta_kernel/schema/derive_macro_utils.rs

delta_kernel/schema/
derive_macro_utils.rs
