delta_kernel/engine/arrow_conversion/
mod.rs

1//! Conversions between kernel schema types and arrow schema types.
2//!
3//! Two directions, used at the engine <-> kernel boundary:
4//!
5//! - **arrow -> kernel** ([`TryFromArrow`] / [`TryIntoKernel`]): an example usage is converting a
6//!   checkpoint Parquet file's Arrow footer schema into a kernel `StructType` during log replay.
7//!
8//! - **kernel -> arrow** ([`TryFromKernel`] / [`TryIntoArrow`]): an example usage is materializing
9//!   a kernel `StructType` as an `ArrowSchema` to build a `RecordBatch` on the write path.
10//!
11//! These conversions can be used on both logical and physical schemas. E.g. when writing a parquet
12//! file, the default engine converts the physical kernel schema to an Arrow schema and writes the
13//! data. When transforming physical data to logical data via an
14//! [`Expression`][crate::expressions::Expression],
15//! [`DefaultExpressionEvaluator`][crate::engine::arrow_expression::DefaultExpressionEvaluator]
16//! converts the logical schema to an Arrow schema as the output schema.
17
18pub mod scalar;
19
20use std::collections::HashMap;
21use std::sync::Arc;
22
23use itertools::Itertools;
24
25use crate::arrow::datatypes::{
26    DataType as ArrowDataType, Field as ArrowField, Schema as ArrowSchema,
27    SchemaRef as ArrowSchemaRef, TimeUnit,
28};
29use crate::arrow::error::ArrowError;
30use crate::error::KernelError;
31use crate::parquet::arrow::PARQUET_FIELD_ID_META_KEY;
32use crate::schema::{
33    ArrayType, ColumnMetadataKey, DataType, MapType, MetadataValue, PrimitiveType, StructField,
34    StructType,
35};
36
37pub(crate) const LIST_ARRAY_ROOT: &str = "element";
38pub(crate) const MAP_ROOT_DEFAULT: &str = "key_value";
39pub(crate) const MAP_KEY_DEFAULT: &str = "key";
40pub(crate) const MAP_VALUE_DEFAULT: &str = "value";
41
42/// Translate a kernel [`StructField`]'s flat (non-nested) parquet field id metadata into Arrow
43/// field metadata: rewrites kernel-side `"parquet.field.id"` to arrow-side `"PARQUET:field_id"`
44/// (see [`PARQUET_FIELD_ID_META_KEY`]). All other entries pass through unchanged.
45pub(crate) fn kernel_flat_parquet_id_to_arrow_metadata(
46    field: &StructField,
47) -> Result<HashMap<String, String>, ArrowError> {
48    field
49        .metadata()
50        .iter()
51        .map(|(key, val)| {
52            let transformed_key = if key == ColumnMetadataKey::ParquetFieldId.as_ref() {
53                PARQUET_FIELD_ID_META_KEY.to_string()
54            } else {
55                key.clone()
56            };
57            match val {
58                MetadataValue::String(s) => Ok((transformed_key, s.clone())),
59                _ => Ok((
60                    transformed_key,
61                    serde_json::to_string(val).map_err(|e| ArrowError::JsonError(e.to_string()))?,
62                )),
63            }
64        })
65        .collect()
66}
67
68/// Looks up a nested-field id for the given dot-path on a StructField's nested-ids metadata.
69///
70/// Parquet field IDs for synthesized list/map element/key/value fields are stored under
71/// `delta.columnMapping.nested.ids`. The path is dot-chained from the nearest ancestor
72/// StructField's name.
73///
74/// # Example
75///
76/// Given `col2: map[int, array[int]]` whose StructField's metadata has
77/// `{ "col2.key": 102, "col2.value": 103, "col2.value.element": 104 }`:
78///
79/// - `lookup_nested_field_id(col2, "col2.key")` -> `Some(102)`
80/// - `lookup_nested_field_id(col2, "col2.value")` -> `Some(103)`
81/// - `lookup_nested_field_id(col2, "col2.value.element")` -> `Some(104)`
82///
83/// Returns `None` when the metadata key is not present, or when the path is not in the JSON
84/// map. The latter is expected when the caller passes a logical schema, because nested.ids
85/// JSON keys are rooted at the physical field name.
86///
87/// Returns an error when the metadata value is not a JSON object, or the id found is not an
88/// integer.
89pub(crate) fn lookup_nested_field_id(
90    field: &StructField,
91    path: &str,
92) -> Result<Option<i64>, ArrowError> {
93    let key = ColumnMetadataKey::ColumnMappingNestedIds.as_ref();
94    let Some(value) = field.metadata().get(key) else {
95        return Ok(None);
96    };
97    let MetadataValue::Other(serde_json::Value::Object(obj)) = value else {
98        return Err(ArrowError::SchemaError(format!(
99            "'{key}' on field '{}' must be a JSON object",
100            field.name()
101        )));
102    };
103    // Missing entry returns Ok(None) rather than Err. The JSON map's keys are rooted at the
104    // *physical* field name, so when this function is invoked on a logical schema, it's
105    // expected that the lookup misses.
106    obj.get(path).map_or(Ok(None), |entry| {
107        entry.as_i64().map(Some).ok_or_else(|| {
108            ArrowError::SchemaError(format!(
109                "'{key}['{path}']' on field '{}' must be an integer, got {entry}",
110                field.name()
111            ))
112        })
113    })
114}
115
116/// Builds Arrow field metadata containing the `PARQUET:field_id` entry, or empty if `id` is
117/// `None`.
118pub(crate) fn parquet_field_id_metadata(id: Option<i64>) -> HashMap<String, String> {
119    let mut meta = HashMap::new();
120    if let Some(id) = id {
121        meta.insert(PARQUET_FIELD_ID_META_KEY.to_string(), id.to_string());
122    }
123    meta
124}
125
126/// Convert a kernel type into an arrow type (automatically implemented for all types that
127/// implement [`TryFromKernel`])
128pub trait TryIntoArrow<ArrowType> {
129    fn try_into_arrow(self) -> Result<ArrowType, ArrowError>;
130}
131
132/// Convert an arrow type into a kernel type (a similar [`TryIntoKernel`] trait is automatically
133/// implemented for all types that implement [`TryFromArrow`])
134pub trait TryFromArrow<ArrowType>: Sized {
135    fn try_from_arrow(t: ArrowType) -> Result<Self, ArrowError>;
136}
137
138/// Convert an arrow type into a kernel type (automatically implemented for all types that
139/// implement [`TryFromArrow`])
140pub trait TryIntoKernel<KernelType> {
141    fn try_into_kernel(self) -> Result<KernelType, ArrowError>;
142}
143
144/// Convert a kernel type into an arrow type (a similar [`TryIntoArrow`] trait is automatically
145/// implemented for all types that implement [`TryFromKernel`])
146pub trait TryFromKernel<KernelType>: Sized {
147    fn try_from_kernel(t: KernelType) -> Result<Self, ArrowError>;
148}
149
150impl<KernelType, ArrowType> TryIntoArrow<ArrowType> for KernelType
151where
152    ArrowType: TryFromKernel<KernelType>,
153{
154    fn try_into_arrow(self) -> Result<ArrowType, ArrowError> {
155        ArrowType::try_from_kernel(self)
156    }
157}
158
159impl<KernelType, ArrowType> TryIntoKernel<KernelType> for ArrowType
160where
161    KernelType: TryFromArrow<ArrowType>,
162{
163    fn try_into_kernel(self) -> Result<KernelType, ArrowError> {
164        KernelType::try_from_arrow(self)
165    }
166}
167
168/// Converts a kernel [`StructType`] to a `Vec<ArrowField>`.
169fn try_kernel_struct_to_arrow_fields(s: &StructType) -> Result<Vec<ArrowField>, ArrowError> {
170    s.fields().map(|f| f.try_into_arrow()).try_collect()
171}
172
173impl TryFromKernel<&StructType> for ArrowSchema {
174    fn try_from_kernel(s: &StructType) -> Result<Self, ArrowError> {
175        Ok(ArrowSchema::new(try_kernel_struct_to_arrow_fields(s)?))
176    }
177}
178
179impl TryFromKernel<&StructField> for ArrowField {
180    fn try_from_kernel(f: &StructField) -> Result<Self, ArrowError> {
181        let mut metadata = kernel_flat_parquet_id_to_arrow_metadata(f)?;
182        // `ColumnMetadataKey::ColumnMappingNestedIds` is a kernel-side metadata key, not
183        // retained in Arrow; its content is processed by `kernel_field_into_arrow`.
184        metadata.remove(ColumnMetadataKey::ColumnMappingNestedIds.as_ref());
185        let arrow_type = kernel_field_into_arrow(f, f.name(), f.data_type())?;
186        Ok(ArrowField::new(f.name(), arrow_type, f.is_nullable()).with_metadata(metadata))
187    }
188}
189
190/// Recursive Arrow-type builder. For Array/Map types, it propagates nested field IDs from
191/// `ancestor`'s `delta.columnMapping.nested.ids` metadata onto the synthesized
192/// `element` / `key` / `value` Arrow fields. For Struct/Primitive/Variant types it delegates to
193/// the standard `DataType -> ArrowDataType` conversion.
194///
195/// # Parameters
196/// - `ancestor`: the nearest ancestor kernel [`StructField`]; its metadata holds the nested-ids
197///   JSON map. Unused for Struct/Primitive/Variant types, only consulted on Array/Map recursion.
198/// - `relative_path`: dot-chained path rooted at `ancestor.name()`. Only consulted for Array/Map
199///   recursion.
200/// - `datatype`: the kernel data type at the current position.
201///
202/// # Example
203///
204/// Kernel field: [`StructField`]:
205///
206/// ```json
207/// {
208///   "name": "array_in_map",
209///   "type": {
210///     "type": "map",
211///     "keyType":   "integer",
212///     "valueType": { "type": "array", "elementType": "integer", "containsNull": true },
213///     "valueContainsNull": true
214///   },
215///   "nullable": true,
216///   "metadata": {
217///     "delta.columnMapping.nested.ids": {
218///       "array_in_map.key":           100,
219///       "array_in_map.value":         101,
220///       "array_in_map.value.element": 102
221///     }
222///   }
223/// }
224/// ```
225///
226/// Calling `kernel_field_into_arrow(field, "array_in_map", &field.data_type())`
227/// produces this Arrow `DataType` (each synthesized `key`/`value`/`element` Arrow Field
228/// carries `PARQUET:field_id` in its `metadata`, looked up from the JSON above):
229///
230/// ```json
231/// {
232///   "type": "map",
233///   "keys_sorted": false,
234///   "entries": {
235///     "name": "key_value",
236///     "type": "struct",
237///     "nullable": false,
238///     "fields": [
239///       {
240///         "name": "key",
241///         "type": "int32",
242///         "nullable": false,
243///         "metadata": { "PARQUET:field_id": "100" }
244///       },
245///       {
246///         "name": "value",
247///         "type": "list",
248///         "nullable": true,
249///         "metadata": { "PARQUET:field_id": "101" },
250///         "element": {
251///           "name": "element",
252///           "type": "int32",
253///           "nullable": true,
254///           "metadata": { "PARQUET:field_id": "102" }
255///         }
256///       }
257///     ]
258///   }
259/// }
260/// ```
261fn kernel_field_into_arrow(
262    ancestor: &StructField,
263    relative_path: &str,
264    datatype: &DataType,
265) -> Result<ArrowDataType, ArrowError> {
266    match datatype {
267        #[cfg(feature = "udt-in-dev")]
268        DataType::UserDefined(udt) => udt.sql_type().try_into_arrow(),
269        DataType::Array(a) => {
270            let element_path = format!("{relative_path}.{LIST_ARRAY_ROOT}");
271            let element_id = lookup_nested_field_id(ancestor, &element_path)?;
272            let arrow_element_type =
273                kernel_field_into_arrow(ancestor, &element_path, a.element_type())?;
274            // Kernel's array element field is anonymous; we use `LIST_ARRAY_ROOT` as the
275            // synthesized field name by convention. Same below for map's `key`/`value` fields.
276            let arrow_element_field =
277                ArrowField::new(LIST_ARRAY_ROOT, arrow_element_type, a.contains_null())
278                    .with_metadata(parquet_field_id_metadata(element_id));
279            Ok(ArrowDataType::List(Arc::new(arrow_element_field)))
280        }
281        DataType::Map(m) => {
282            let key_path = format!("{relative_path}.{MAP_KEY_DEFAULT}");
283            let value_path = format!("{relative_path}.{MAP_VALUE_DEFAULT}");
284            let key_id = lookup_nested_field_id(ancestor, &key_path)?;
285            let value_id = lookup_nested_field_id(ancestor, &value_path)?;
286            let arrow_key_type = kernel_field_into_arrow(ancestor, &key_path, m.key_type())?;
287            let arrow_value_type = kernel_field_into_arrow(ancestor, &value_path, m.value_type())?;
288            // Map keys are never nullable.
289            let arrow_key_field = ArrowField::new(MAP_KEY_DEFAULT, arrow_key_type, false)
290                .with_metadata(parquet_field_id_metadata(key_id));
291            let arrow_value_field =
292                ArrowField::new(MAP_VALUE_DEFAULT, arrow_value_type, m.value_contains_null())
293                    .with_metadata(parquet_field_id_metadata(value_id));
294            // `MapArray::try_new` rejects a nullable entries field, so hardcode nullable to `false`
295            // here. See <https://docs.rs/arrow/latest/arrow/array/struct.MapArray.html#method.try_new>.
296            let arrow_entries_field = ArrowField::new(
297                MAP_ROOT_DEFAULT,
298                ArrowDataType::Struct(vec![arrow_key_field, arrow_value_field].into()),
299                false, /* always non-null */
300            );
301            Ok(ArrowDataType::Map(
302                Arc::new(arrow_entries_field),
303                false, /* keys_sorted */
304            ))
305        }
306        DataType::Struct(_) | DataType::Primitive(_) | DataType::Variant(_) => {
307            datatype.try_into_arrow()
308        }
309    }
310}
311
312impl TryFromKernel<&ArrayType> for ArrowField {
313    fn try_from_kernel(a: &ArrayType) -> Result<Self, ArrowError> {
314        Ok(ArrowField::new(
315            LIST_ARRAY_ROOT,
316            a.element_type().try_into_arrow()?,
317            a.contains_null(),
318        ))
319    }
320}
321
322impl TryFromKernel<&MapType> for ArrowField {
323    fn try_from_kernel(a: &MapType) -> Result<Self, ArrowError> {
324        Ok(ArrowField::new(
325            MAP_ROOT_DEFAULT,
326            ArrowDataType::Struct(
327                vec![
328                    ArrowField::new(MAP_KEY_DEFAULT, a.key_type().try_into_arrow()?, false),
329                    ArrowField::new(
330                        MAP_VALUE_DEFAULT,
331                        a.value_type().try_into_arrow()?,
332                        a.value_contains_null(),
333                    ),
334                ]
335                .into(),
336            ),
337            false, // always non-null
338        ))
339    }
340}
341
342impl TryFromKernel<&DataType> for ArrowDataType {
343    fn try_from_kernel(t: &DataType) -> Result<Self, ArrowError> {
344        match t {
345            #[cfg(feature = "udt-in-dev")]
346            DataType::UserDefined(_) => t.physical_type().try_into_arrow(),
347            DataType::Primitive(p) => {
348                match p {
349                    PrimitiveType::String => Ok(ArrowDataType::Utf8),
350                    PrimitiveType::Long => Ok(ArrowDataType::Int64), // undocumented type
351                    PrimitiveType::Integer => Ok(ArrowDataType::Int32),
352                    PrimitiveType::Short => Ok(ArrowDataType::Int16),
353                    PrimitiveType::Byte => Ok(ArrowDataType::Int8),
354                    PrimitiveType::Float => Ok(ArrowDataType::Float32),
355                    PrimitiveType::Double => Ok(ArrowDataType::Float64),
356                    PrimitiveType::Boolean => Ok(ArrowDataType::Boolean),
357                    PrimitiveType::Binary => Ok(ArrowDataType::Binary),
358                    PrimitiveType::Decimal(dtype) => Ok(ArrowDataType::Decimal128(
359                        dtype.precision(),
360                        dtype.scale() as i8, // 0..=38
361                    )),
362                    PrimitiveType::Date => {
363                        // A calendar date, represented as a year-month-day triple without a
364                        // timezone. Stored as 4 bytes integer representing days since 1970-01-01
365                        Ok(ArrowDataType::Date32)
366                    }
367                    // TODO: https://github.com/delta-io/delta/issues/643
368                    PrimitiveType::Timestamp => Ok(ArrowDataType::Timestamp(
369                        TimeUnit::Microsecond,
370                        Some("UTC".into()),
371                    )),
372                    PrimitiveType::TimestampNtz => {
373                        Ok(ArrowDataType::Timestamp(TimeUnit::Microsecond, None))
374                    }
375                    PrimitiveType::Void => Ok(ArrowDataType::Null),
376                    PrimitiveType::IntervalYearMonth => Ok(ArrowDataType::Int32),
377                    PrimitiveType::IntervalDayTime => Ok(ArrowDataType::Int64),
378                    #[cfg(feature = "geo-type-in-dev")]
379                    PrimitiveType::Geometry(_) | PrimitiveType::Geography(_) => {
380                        Err(ArrowError::SchemaError(format!(
381                            "Geo types are not yet supported in the default engine: {p}"
382                        )))
383                    }
384                }
385            }
386            DataType::Struct(s) => Ok(ArrowDataType::Struct(
387                try_kernel_struct_to_arrow_fields(s)?.into(),
388            )),
389            DataType::Array(a) => Ok(ArrowDataType::List(Arc::new(a.as_ref().try_into_arrow()?))),
390            DataType::Map(m) => Ok(ArrowDataType::Map(
391                Arc::new(m.as_ref().try_into_arrow()?),
392                false,
393            )),
394            DataType::Variant(s) => {
395                if *t == DataType::unshredded_variant() {
396                    Ok(ArrowDataType::Struct(
397                        try_kernel_struct_to_arrow_fields(s)?.into(),
398                    ))
399                } else {
400                    Err(ArrowError::SchemaError(format!(
401                        "Incorrect Variant Schema: {t}. Only the unshredded variant schema is supported right now."
402                    )))
403                }
404            }
405        }
406    }
407}
408
409impl TryFromArrow<&ArrowSchema> for StructType {
410    fn try_from_arrow(arrow_schema: &ArrowSchema) -> Result<Self, ArrowError> {
411        StructType::try_from_results(
412            arrow_schema
413                .fields()
414                .iter()
415                .map(|field| field.as_ref().try_into_kernel()),
416        )
417        .map_err(|e| ArrowError::from_external_error(e.into()))
418    }
419}
420
421impl TryFromArrow<ArrowSchemaRef> for StructType {
422    fn try_from_arrow(arrow_schema: ArrowSchemaRef) -> Result<Self, ArrowError> {
423        arrow_schema.as_ref().try_into_kernel()
424    }
425}
426
427impl TryFromArrow<&ArrowField> for StructField {
428    fn try_from_arrow(arrow_field: &ArrowField) -> Result<Self, ArrowError> {
429        let arrow_metadata = arrow_field.metadata();
430        // If both the native Arrow key (PARQUET:field_id) and the kernel key (parquet.field.id)
431        // are present with different values, the translation below would silently overwrite one
432        // with the other. Detect and reject this up front.
433        if let (Some(arrow_id), Some(kernel_id)) = (
434            arrow_metadata.get(PARQUET_FIELD_ID_META_KEY),
435            arrow_metadata.get(ColumnMetadataKey::ParquetFieldId.as_ref()),
436        ) {
437            if arrow_id != kernel_id {
438                return Err(ArrowError::SchemaError(format!(
439                    "Field '{}': conflicting parquet field IDs: '{}' ({}) vs '{}' ({})",
440                    arrow_field.name(),
441                    arrow_id,
442                    PARQUET_FIELD_ID_META_KEY,
443                    kernel_id,
444                    ColumnMetadataKey::ParquetFieldId.as_ref(),
445                )));
446            }
447        }
448        let mut kernel_metadata: HashMap<String, MetadataValue> = arrow_metadata
449            .iter()
450            .map(|(k, v)| -> Result<_, ArrowError> {
451                // Transform "PARQUET:field_id" to "parquet.field.id" when reading from Parquet.
452                // `parquet.field.id` in kernel is `MetadataValue::Number(i64)`.
453                if k == PARQUET_FIELD_ID_META_KEY || k == ColumnMetadataKey::ParquetFieldId.as_ref()
454                {
455                    let id: i64 = v.parse().map_err(|_| {
456                        ArrowError::SchemaError(format!(
457                            "'{k}' on field '{}' must be an integer, got '{v}'",
458                            arrow_field.name()
459                        ))
460                    })?;
461                    return Ok((
462                        ColumnMetadataKey::ParquetFieldId.as_ref().to_string(),
463                        MetadataValue::Number(id),
464                    ));
465                }
466                Ok((k.clone(), MetadataValue::from(v)))
467            })
468            .collect::<Result<_, _>>()?;
469        let mut nested_ids = serde_json::Map::new();
470        collect_nested_field_ids_from_arrow(
471            arrow_field.data_type(),
472            arrow_field.name(),
473            &mut nested_ids,
474        )?;
475        if !nested_ids.is_empty() {
476            kernel_metadata.insert(
477                ColumnMetadataKey::ColumnMappingNestedIds
478                    .as_ref()
479                    .to_string(),
480                MetadataValue::Other(serde_json::Value::Object(nested_ids)),
481            );
482        }
483
484        Ok(StructField::new(
485            arrow_field.name().clone(),
486            DataType::try_from_arrow(arrow_field.data_type())?,
487            arrow_field.is_nullable(),
488        )
489        .with_metadata(kernel_metadata))
490    }
491}
492
493/// Inverse of [`kernel_field_into_arrow`]: walks `arrow_type` and aggregates
494/// `PARQUET:field_id` from the element/key/value fields of any list/map, keyed by dot-path
495/// rooted at `relative_path`.
496///
497/// # Example
498///
499/// Given an Arrow [`MapArray`] field `m: map<int, list<int>>` whose inner `key`, `value`, and
500/// `value.element` fields carry `PARQUET:field_id` of `100`, `101`, `102`, calling
501/// `collect_nested_field_ids_from_arrow(m.data_type(), "m", &mut out)` populates `out` with:
502///
503/// ```text
504/// { "m.key": 100, "m.value": 101, "m.value.element": 102 }
505/// ```
506///
507/// [`MapArray`]: arrow::array::MapArray
508fn collect_nested_field_ids_from_arrow(
509    arrow_type: &ArrowDataType,
510    relative_path: &str,
511    out: &mut serde_json::Map<String, serde_json::Value>,
512) -> Result<(), ArrowError> {
513    // `delta.columnMapping.nested.ids` only stores the nested field ids element/key/values
514    // of list/map fields. So we only recurse into list/map here.
515    let inner_fields: Vec<(&ArrowField, &str)> = match arrow_type {
516        ArrowDataType::List(elem)
517        | ArrowDataType::ListView(elem)
518        | ArrowDataType::LargeList(elem)
519        | ArrowDataType::LargeListView(elem) => vec![(elem.as_ref(), LIST_ARRAY_ROOT)],
520        ArrowDataType::FixedSizeList(elem, _) => vec![(elem.as_ref(), LIST_ARRAY_ROOT)],
521        ArrowDataType::Map(entries, _) => match entries.data_type() {
522            ArrowDataType::Struct(fields) if fields.len() == 2 => vec![
523                (fields[0].as_ref(), MAP_KEY_DEFAULT),
524                (fields[1].as_ref(), MAP_VALUE_DEFAULT),
525            ],
526            other => {
527                return Err(ArrowError::SchemaError(format!(
528                    "Map entries must be a struct with exactly 2 fields (key, value), got {other}"
529                )))
530            }
531        },
532        // Dictionary is a transparent wrapper (kernel side collapses it to its value type), so
533        // recurse without consuming a path segment.
534        ArrowDataType::Dictionary(_, value_type) => {
535            return collect_nested_field_ids_from_arrow(value_type, relative_path, out);
536        }
537        _ => return Ok(()),
538    };
539
540    for (inner, segment) in inner_fields {
541        let path = format!("{relative_path}.{segment}");
542        if let Some(id_str) = inner.metadata().get(PARQUET_FIELD_ID_META_KEY) {
543            let id: i64 = id_str.parse().map_err(|_| {
544                ArrowError::SchemaError(format!(
545                    "'{PARQUET_FIELD_ID_META_KEY}' on '{}' must be an integer, got '{id_str}'",
546                    inner.name()
547                ))
548            })?;
549            out.insert(path.clone(), serde_json::Value::from(id));
550        }
551        collect_nested_field_ids_from_arrow(inner.data_type(), &path, out)?;
552    }
553    Ok(())
554}
555
556impl TryFromArrow<&ArrowDataType> for DataType {
557    fn try_from_arrow(arrow_datatype: &ArrowDataType) -> Result<Self, ArrowError> {
558        match arrow_datatype {
559            ArrowDataType::Utf8 => Ok(DataType::STRING),
560            ArrowDataType::LargeUtf8 => Ok(DataType::STRING),
561            ArrowDataType::Utf8View => Ok(DataType::STRING),
562            ArrowDataType::Int64 => Ok(DataType::LONG), // undocumented type
563            ArrowDataType::Int32 => Ok(DataType::INTEGER),
564            ArrowDataType::Int16 => Ok(DataType::SHORT),
565            ArrowDataType::Int8 => Ok(DataType::BYTE),
566            ArrowDataType::UInt64 => Ok(DataType::LONG), // undocumented type
567            ArrowDataType::UInt32 => Ok(DataType::INTEGER),
568            ArrowDataType::UInt16 => Ok(DataType::SHORT),
569            ArrowDataType::UInt8 => Ok(DataType::BYTE),
570            ArrowDataType::Null => Ok(DataType::VOID),
571            ArrowDataType::Float32 => Ok(DataType::FLOAT),
572            ArrowDataType::Float64 => Ok(DataType::DOUBLE),
573            ArrowDataType::Boolean => Ok(DataType::BOOLEAN),
574            ArrowDataType::Binary => Ok(DataType::BINARY),
575            ArrowDataType::FixedSizeBinary(_) => Ok(DataType::BINARY),
576            ArrowDataType::LargeBinary => Ok(DataType::BINARY),
577            ArrowDataType::BinaryView => Ok(DataType::BINARY),
578            ArrowDataType::Decimal128(p, s) => {
579                if *s < 0 {
580                    return Err(ArrowError::from_external_error(
581                        KernelError::invalid_decimal("Negative scales are not supported in Delta")
582                            .into(),
583                    ));
584                };
585                DataType::decimal(*p, *s as u8)
586                    .map_err(|e| ArrowError::from_external_error(e.into()))
587            }
588            ArrowDataType::Date32 => Ok(DataType::DATE),
589            ArrowDataType::Date64 => Ok(DataType::DATE),
590            ArrowDataType::Timestamp(TimeUnit::Microsecond, None) => Ok(DataType::TIMESTAMP_NTZ),
591            ArrowDataType::Timestamp(TimeUnit::Microsecond, Some(tz))
592                if tz.eq_ignore_ascii_case("utc") =>
593            {
594                Ok(DataType::TIMESTAMP)
595            }
596            ArrowDataType::Timestamp(TimeUnit::Nanosecond, None) => Ok(DataType::TIMESTAMP_NTZ),
597            ArrowDataType::Timestamp(TimeUnit::Nanosecond, Some(tz))
598                if tz.eq_ignore_ascii_case("utc") =>
599            {
600                Ok(DataType::TIMESTAMP)
601            }
602            // Millisecond is coarser than the kernel's microsecond logical timestamp, so
603            // mapping it onto the logical type is a lossless upscale (values are rescaled
604            // x1000 by the engine on read).
605            ArrowDataType::Timestamp(TimeUnit::Millisecond, None) => Ok(DataType::TIMESTAMP_NTZ),
606            ArrowDataType::Timestamp(TimeUnit::Millisecond, Some(tz))
607                if tz.eq_ignore_ascii_case("utc") =>
608            {
609                Ok(DataType::TIMESTAMP)
610            }
611            ArrowDataType::Struct(fields) => DataType::try_struct_type_from_results(
612                fields.iter().map(|field| field.as_ref().try_into_kernel()),
613            )
614            .map_err(|e| ArrowError::from_external_error(e.into())),
615            ArrowDataType::List(field) => Ok(ArrayType::new(
616                DataType::try_from_arrow((*field).data_type())?,
617                (*field).is_nullable(),
618            )
619            .into()),
620            ArrowDataType::ListView(field) => Ok(ArrayType::new(
621                DataType::try_from_arrow((*field).data_type())?,
622                (*field).is_nullable(),
623            )
624            .into()),
625            ArrowDataType::LargeList(field) => Ok(ArrayType::new(
626                DataType::try_from_arrow((*field).data_type())?,
627                (*field).is_nullable(),
628            )
629            .into()),
630            ArrowDataType::LargeListView(field) => Ok(ArrayType::new(
631                DataType::try_from_arrow((*field).data_type())?,
632                (*field).is_nullable(),
633            )
634            .into()),
635            ArrowDataType::FixedSizeList(field, _) => Ok(ArrayType::new(
636                DataType::try_from_arrow((*field).data_type())?,
637                (*field).is_nullable(),
638            )
639            .into()),
640            ArrowDataType::Map(field, _) => {
641                if let ArrowDataType::Struct(struct_fields) = field.data_type() {
642                    let key_type = DataType::try_from_arrow(struct_fields[0].data_type())?;
643                    let value_type = DataType::try_from_arrow(struct_fields[1].data_type())?;
644                    let value_type_nullable = struct_fields[1].is_nullable();
645                    Ok(MapType::new(key_type, value_type, value_type_nullable).into())
646                } else {
647                    unreachable!("DataType::Map should contain a struct field child");
648                }
649            }
650            // Dictionary types are just an optimized in-memory representation of an array.
651            // Schema-wise, they are the same as the value type.
652            ArrowDataType::Dictionary(_, value_type) => {
653                Ok(value_type.as_ref().try_into_kernel()?)
654            }
655            s => Err(ArrowError::SchemaError(format!(
656                "Invalid data type for Delta Lake: {s}"
657            ))),
658        }
659    }
660}
661
662#[cfg(test)]
663mod tests {
664    use std::collections::HashMap;
665
666    use rstest::rstest;
667
668    use super::*;
669    use crate::engine::arrow_conversion::ArrowField;
670    use crate::engine::arrow_data::unshredded_variant_arrow_type;
671    use crate::parquet::arrow::PARQUET_FIELD_ID_META_KEY;
672    #[cfg(feature = "geo-type-in-dev")]
673    use crate::schema::EdgeInterpolationAlgorithm;
674    use crate::schema::{
675        schema, ArrayType, ColumnMetadataKey, DataType, MapType, MetadataValue, PrimitiveType,
676        StructField, StructType,
677    };
678    use crate::transforms::{transform_output_type, SchemaTransform};
679    use crate::unit_test_utils::{
680        array_in_map_kernel_schema, assert_result_error_with_message, collect_arrow_field_metadata,
681        complex_nested_with_field_ids,
682    };
683    #[cfg(feature = "geo-type-in-dev")]
684    use crate::unit_test_utils::{geography_type, geometry_type};
685    use crate::KernelResult;
686
687    #[cfg(feature = "geo-type-in-dev")]
688    #[rstest]
689    #[case(geometry_type("EPSG:4326"))]
690    #[case(geography_type("EPSG:4326", EdgeInterpolationAlgorithm::Spherical))]
691    #[case(DataType::from(schema! {
692        nullable "g": (geometry_type("EPSG:4326")),
693    }))]
694    #[case(DataType::from(ArrayType::new(geometry_type("EPSG:4326"), true)))]
695    #[case(DataType::from(MapType::new(
696        DataType::STRING,
697        geography_type("EPSG:4326", EdgeInterpolationAlgorithm::Spherical),
698        true,
699    )))]
700    fn test_geo_type_arrow_conversion_unsupported(#[case] dt: DataType) {
701        let result: KernelResult<ArrowDataType, _> = (&dt).try_into_arrow();
702        let err = result.unwrap_err();
703        assert!(matches!(err, ArrowError::SchemaError(_)), "got: {err:?}");
704    }
705
706    #[test]
707    fn test_metadata_string_conversion() -> KernelResult<()> {
708        let mut metadata = HashMap::new();
709        metadata.insert("description", "hello world".to_owned());
710        let struct_field = StructField::not_null("name", DataType::STRING).with_metadata(metadata);
711
712        let arrow_field = ArrowField::try_from_kernel(&struct_field)?;
713        let new_metadata = arrow_field.metadata();
714
715        assert_eq!(
716            new_metadata.get("description").unwrap(),
717            &"hello world".to_owned()
718        );
719        Ok(())
720    }
721
722    // Delta tables can have void columns. The kernel should parse them and convert
723    // to Arrow's Null type (and back).
724    #[test]
725    fn test_void_type_roundtrip() -> KernelResult<()> {
726        let json = r#"
727        {
728            "name": "void_col",
729            "type": "void",
730            "nullable": true,
731            "metadata": {}
732        }
733        "#;
734
735        let field: crate::schema::StructField = serde_json::from_str(json).unwrap();
736        assert_eq!(field.data_type, DataType::Primitive(PrimitiveType::Void));
737
738        let arrow_type = ArrowDataType::try_from_kernel(&field.data_type)?;
739        assert_eq!(arrow_type, ArrowDataType::Null);
740
741        let kernel_type = DataType::try_from_arrow(&ArrowDataType::Null)?;
742        assert_eq!(kernel_type, DataType::Primitive(PrimitiveType::Void));
743
744        Ok(())
745    }
746
747    // Asserts forward direction of Interval column mapping (year-month/day-time to Int32/Int64)
748    #[rstest]
749    #[case(DataType::INTERVAL_YEAR_MONTH, ArrowDataType::Int32)]
750    #[case(DataType::INTERVAL_DAY_TIME, ArrowDataType::Int64)]
751    fn test_interval_type_arrow_conversion(
752        #[case] kernel_type: DataType,
753        #[case] expected: ArrowDataType,
754    ) -> KernelResult<()> {
755        assert_eq!(ArrowDataType::try_from_kernel(&kernel_type)?, expected);
756        Ok(())
757    }
758
759    // Millisecond-precision timestamps (e.g. from externally-written checkpoint stats)
760    // must convert like microsecond/nanosecond: UTC tz -> TIMESTAMP, no tz -> TIMESTAMP_NTZ.
761    #[test]
762    fn test_millisecond_timestamp_conversion() -> KernelResult<()> {
763        let utc = DataType::try_from_arrow(&ArrowDataType::Timestamp(
764            TimeUnit::Millisecond,
765            Some("UTC".into()),
766        ))?;
767        assert_eq!(utc, DataType::TIMESTAMP);
768
769        let ntz = DataType::try_from_arrow(&ArrowDataType::Timestamp(TimeUnit::Millisecond, None))?;
770        assert_eq!(ntz, DataType::TIMESTAMP_NTZ);
771
772        Ok(())
773    }
774
775    // void is inherently always-null, so nullable=false is semantically contradictory.
776    // We tolerate it on reads (be permissive), and the Arrow conversion still
777    // produces ArrowDataType::Null. The field retains nullable=false as-is — no coercion.
778    #[test]
779    fn test_void_type_not_nullable() -> KernelResult<()> {
780        let json = r#"
781        {
782            "name": "void_col",
783            "type": "void",
784            "nullable": false,
785            "metadata": {}
786        }
787        "#;
788
789        let field: crate::schema::StructField = serde_json::from_str(json).unwrap();
790        assert_eq!(field.data_type, DataType::Primitive(PrimitiveType::Void));
791        assert!(!field.is_nullable());
792
793        // Arrow conversion still works — produces Null type
794        let arrow_field = ArrowField::try_from_kernel(&field)?;
795        assert_eq!(arrow_field.data_type(), &ArrowDataType::Null);
796
797        // Void is always-null by definition, so nullable=false is semantically
798        // contradictory. We tolerate it on reads to avoid breaking existing tables.
799        assert!(!arrow_field.is_nullable());
800
801        Ok(())
802    }
803
804    #[test]
805    fn test_void_field_in_struct() -> KernelResult<()> {
806        // A struct schema with a void column should convert to Arrow with a Null field
807        let schema = schema! {
808            nullable "id": INTEGER,
809            nullable "void_col": VOID,
810        };
811
812        let arrow_schema = ArrowSchema::try_from_kernel(&schema)?;
813        assert_eq!(arrow_schema.fields().len(), 2);
814        assert_eq!(arrow_schema.field(0).data_type(), &ArrowDataType::Int32);
815        assert_eq!(arrow_schema.field(1).data_type(), &ArrowDataType::Null);
816        assert_eq!(arrow_schema.field(1).name(), "void_col");
817
818        // And back to kernel
819        let kernel_schema = StructType::try_from_arrow(&arrow_schema)?;
820        assert_eq!(
821            kernel_schema.field("void_col").unwrap().data_type,
822            DataType::VOID
823        );
824
825        Ok(())
826    }
827
828    #[test]
829    fn test_variant_shredded_type_fail() -> KernelResult<()> {
830        let unshredded_variant = DataType::unshredded_variant();
831        let unshredded_variant_arrow = ArrowDataType::try_from_kernel(&unshredded_variant)?;
832        assert!(unshredded_variant_arrow == unshredded_variant_arrow_type());
833        let shredded_variant = DataType::variant_type([
834            StructField::nullable("metadata", DataType::BINARY),
835            StructField::nullable("value", DataType::BINARY),
836            StructField::nullable("typed_value", DataType::INTEGER),
837        ])?;
838        let shredded_variant_arrow = ArrowDataType::try_from_kernel(&shredded_variant);
839        assert!(shredded_variant_arrow
840            .unwrap_err()
841            .to_string()
842            .contains("Incorrect Variant Schema"));
843        Ok(())
844    }
845
846    /// Helper visitor to collect all field IDs from a kernel StructType
847    #[derive(Default)]
848    struct FieldIdCollector {
849        field_ids: Vec<(String, String)>, // (field_name, field_id)
850    }
851
852    impl<'a> SchemaTransform<'a> for FieldIdCollector {
853        transform_output_type!(|'a, T| ());
854
855        fn transform_struct_field(&mut self, field: &'a StructField) {
856            // Collect field ID if present
857            if let Some(field_id) = field
858                .metadata()
859                .get(ColumnMetadataKey::ParquetFieldId.as_ref())
860            {
861                self.field_ids
862                    .push((field.name().to_string(), field_id.to_string()));
863            }
864            // Recurse into nested types
865            self.recurse_into_struct_field(field)
866        }
867    }
868
869    #[test]
870    fn test_try_into_arrow_threads_nested_ids_onto_arrow_schema() -> KernelResult<()> {
871        let meta_key = ColumnMetadataKey::ColumnMappingNestedIds.as_ref();
872        let fixture = complex_nested_with_field_ids(meta_key);
873        let arrow_schema: ArrowSchema = (&fixture.kernel_schema).try_into_arrow()?;
874
875        assert_eq!(
876            arrow_schema, fixture.expected_arrow_schema,
877            "try_into_arrow should attach nested field ids for metadata key {meta_key}",
878        );
879        Ok(())
880    }
881
882    #[rstest]
883    /// The nested ids JSON map is missing (i.e. `delta.columnMapping.nested.ids` is not present).
884    #[case::without_nested_id_metadata(array_in_map_kernel_schema(std::iter::empty::<(
885        String,
886        MetadataValue,
887    )>()), /* expected_field_ids */ &[])]
888    /// Only some of the nested ids are present.
889    #[case::only_partial_nested_ids_match(array_in_map_kernel_schema([(
890        ColumnMetadataKey::ColumnMappingNestedIds.as_ref().to_string(),
891        MetadataValue::Other(test_utils::nested_ids_json(&[
892            ("array_in_map.key", 100),
893            ("array_in_map.value.element", 102),
894            ("array_in_map.notTheKey", 999),
895        ])),
896    )]), &[("element", "102"), ("key", "100")])]
897    fn test_try_into_arrow_sets_field_ids_for_matched_nested_ids_only(
898        #[case] schema: StructType,
899        #[case] expected_field_ids: &[(&str, &str)],
900    ) -> KernelResult<()> {
901        let arrow_schema: ArrowSchema = (&schema).try_into_arrow()?;
902        let field_ids: HashMap<String, String> =
903            collect_arrow_field_metadata(&arrow_schema, PARQUET_FIELD_ID_META_KEY)
904                .into_iter()
905                .collect();
906
907        assert_eq!(field_ids.len(), expected_field_ids.len());
908        for (field_name, expected_id) in expected_field_ids {
909            assert_eq!(
910                field_ids.get(*field_name).map(String::as_str),
911                Some(*expected_id),
912            );
913        }
914        Ok(())
915    }
916
917    #[test]
918    fn test_try_into_arrow_invalid_nested_ids_metadata_errors() {
919        let schema = array_in_map_kernel_schema([(
920            ColumnMetadataKey::ColumnMappingNestedIds
921                .as_ref()
922                .to_string(),
923            MetadataValue::String("not a json object".to_string()),
924        )]);
925        assert_result_error_with_message(
926            ArrowSchema::try_from_kernel(&schema),
927            "must be a JSON object",
928        );
929
930        let schema = array_in_map_kernel_schema([(
931            ColumnMetadataKey::ColumnMappingNestedIds
932                .as_ref()
933                .to_string(),
934            MetadataValue::Other(serde_json::json!({ "array_in_map.key": "oops" })),
935        )]);
936        assert_result_error_with_message(
937            ArrowSchema::try_from_kernel(&schema),
938            "must be an integer",
939        );
940    }
941
942    #[test]
943    fn test_recursive_field_id_transformation() -> KernelResult<()> {
944        // Create a complex nested structure with field IDs at multiple levels:
945        // top_struct {
946        //   simple_field: int (field_id=1)
947        //   nested_struct: struct { (field_id=2)
948        //     inner_field: string (field_id=3)
949        //   }
950        //   array_field: array<struct { (field_id=4)
951        //     array_item: int (field_id=5)
952        //   }>
953        //   map_field: map<struct { (field_id=6)
954        //     map_key_field: string (field_id=7)
955        //   }, struct {
956        //     map_value_field: int (field_id=8)
957        //   }>
958        // }
959
960        // Build nested struct
961        let inner_struct_type = schema! {
962            (StructField::not_null("inner_field", DataType::STRING).with_metadata([(
963                ColumnMetadataKey::ParquetFieldId.as_ref(),
964                MetadataValue::Number(3),
965            )])),
966        };
967
968        // Build array element struct
969        let array_item_struct = schema! {
970            (StructField::not_null("array_item", DataType::INTEGER).with_metadata([(
971                ColumnMetadataKey::ParquetFieldId.as_ref(),
972                MetadataValue::Number(5),
973            )])),
974        };
975        let array_type = ArrayType::new(array_item_struct, false);
976
977        // Build map with struct key and struct value (both with field IDs)
978        let map_key_struct = schema! {
979            (StructField::not_null("map_key_field", DataType::STRING).with_metadata([(
980                ColumnMetadataKey::ParquetFieldId.as_ref(),
981                MetadataValue::Number(7),
982            )])),
983        };
984        let map_value_struct = schema! {
985            (StructField::not_null("map_value_field", DataType::INTEGER).with_metadata([(
986                ColumnMetadataKey::ParquetFieldId.as_ref(),
987                MetadataValue::Number(8),
988            )])),
989        };
990        let map_type = MapType::new(map_key_struct, map_value_struct, false);
991
992        // Build top-level struct
993        let top_struct = schema! {
994            (StructField::not_null("simple_field", DataType::INTEGER).with_metadata([(
995                ColumnMetadataKey::ParquetFieldId.as_ref(),
996                MetadataValue::Number(1),
997            )])),
998            (StructField::not_null("nested_struct", inner_struct_type).with_metadata([(
999                ColumnMetadataKey::ParquetFieldId.as_ref(),
1000                MetadataValue::Number(2),
1001            )])),
1002            (StructField::not_null("array_field", array_type).with_metadata([(
1003                ColumnMetadataKey::ParquetFieldId.as_ref(),
1004                MetadataValue::Number(4),
1005            )])),
1006            (StructField::not_null("map_field", map_type).with_metadata([(
1007                ColumnMetadataKey::ParquetFieldId.as_ref(),
1008                MetadataValue::Number(6),
1009            )])),
1010        };
1011
1012        // Convert to Arrow schema
1013        let arrow_schema = ArrowSchema::try_from_kernel(&top_struct)?;
1014
1015        let expected_ids: HashMap<String, String> = [
1016            ("simple_field", "1"),
1017            ("nested_struct", "2"),
1018            ("inner_field", "3"),
1019            ("array_field", "4"),
1020            ("array_item", "5"),
1021            ("map_field", "6"),
1022            ("map_key_field", "7"),
1023            ("map_value_field", "8"),
1024        ]
1025        .into_iter()
1026        .map(|(k, v)| (k.to_string(), v.to_string()))
1027        .collect();
1028
1029        // Verify field IDs are transformed to PARQUET:field_id at all levels
1030        let arrow_field_ids: HashMap<String, String> =
1031            collect_arrow_field_metadata(&arrow_schema, PARQUET_FIELD_ID_META_KEY)
1032                .into_iter()
1033                .collect();
1034        assert_eq!(
1035            arrow_field_ids, expected_ids,
1036            "All field IDs should be transformed to PARQUET:field_id"
1037        );
1038
1039        // Test round-trip: Arrow -> Kernel, field IDs should be preserved unchanged
1040        let kernel_struct = StructType::try_from_arrow(&arrow_schema)?;
1041        let mut collector = FieldIdCollector::default();
1042        collector.transform_struct(&kernel_struct);
1043        let kernel_field_ids: HashMap<String, String> = collector.field_ids.into_iter().collect();
1044        assert_eq!(
1045            kernel_field_ids, arrow_field_ids,
1046            "Kernel field IDs should match Arrow field IDs after round-trip"
1047        );
1048
1049        Ok(())
1050    }
1051
1052    /// When an Arrow field carries both `PARQUET:field_id` and `parquet.field.id` with the same
1053    /// value, the round-trip to kernel should succeed (one key is kept after translation).
1054    #[test]
1055    fn test_arrow_to_kernel_matching_field_ids_succeed() {
1056        let arrow_field =
1057            ArrowField::new("a", ArrowDataType::Int32, false).with_metadata(HashMap::from([
1058                (PARQUET_FIELD_ID_META_KEY.to_string(), "42".to_string()),
1059                (
1060                    ColumnMetadataKey::ParquetFieldId.as_ref().to_string(),
1061                    "42".to_string(),
1062                ),
1063            ]));
1064        let kernel = StructField::try_from_arrow(&arrow_field).unwrap();
1065        // The two arrow keys collapse to a single kernel `parquet.field.id` entry whose value is
1066        // a `Number`.
1067        assert_eq!(
1068            kernel
1069                .metadata()
1070                .get(ColumnMetadataKey::ParquetFieldId.as_ref()),
1071            Some(&MetadataValue::Number(42)),
1072        );
1073    }
1074
1075    /// When an Arrow field carries both `PARQUET:field_id` and `parquet.field.id` with *different*
1076    /// values, converting to kernel must fail rather than silently overwriting one ID with the
1077    /// other.
1078    #[test]
1079    fn test_arrow_to_kernel_conflicting_field_ids_fail() {
1080        let arrow_field =
1081            ArrowField::new("a", ArrowDataType::Int32, false).with_metadata(HashMap::from([
1082                (PARQUET_FIELD_ID_META_KEY.to_string(), "1".to_string()),
1083                (
1084                    ColumnMetadataKey::ParquetFieldId.as_ref().to_string(),
1085                    "2".to_string(),
1086                ),
1087            ]));
1088        assert_result_error_with_message(
1089            StructField::try_from_arrow(&arrow_field),
1090            "conflicting parquet field IDs",
1091        );
1092    }
1093
1094    /// Inverse of `test_try_into_arrow_threads_nested_ids_onto_arrow_schema`: feed the same
1095    /// fixture's expected Arrow schema back through `try_from_arrow` and verify it round-trips
1096    /// to the original kernel schema.
1097    #[test]
1098    fn test_try_from_arrow_aggregates_synthesized_nested_ids() {
1099        let fixture =
1100            complex_nested_with_field_ids(ColumnMetadataKey::ColumnMappingNestedIds.as_ref());
1101        let kernel_schema_from_arrow: StructType =
1102            (&fixture.expected_arrow_schema).try_into_kernel().unwrap();
1103        assert_eq!(kernel_schema_from_arrow, fixture.kernel_schema);
1104    }
1105
1106    /// When the input Arrow schema has no `PARQUET:field_id` on any inner list/map field, the
1107    /// resulting kernel [`StructField`] must not carry a `delta.columnMapping.nested.ids` entry.
1108    #[test]
1109    fn test_try_from_arrow_no_nested_ids_when_input_lacks_field_ids() {
1110        let kernel_schema_without_metadata =
1111            array_in_map_kernel_schema(std::iter::empty::<(String, MetadataValue)>());
1112        let arrow_schema_without_parquet_id: ArrowSchema =
1113            (&kernel_schema_without_metadata).try_into_arrow().unwrap();
1114
1115        let kernel_schema: StructType = (&arrow_schema_without_parquet_id)
1116            .try_into_kernel()
1117            .unwrap();
1118        let array_in_map = kernel_schema.fields().next().unwrap();
1119        assert!(!array_in_map
1120            .metadata()
1121            .contains_key(ColumnMetadataKey::ColumnMappingNestedIds.as_ref()));
1122    }
1123
1124    /// A non-integer `PARQUET:field_id` on an inner list/map field must produce a clear error
1125    /// rather than silently being dropped.
1126    #[test]
1127    fn test_try_from_arrow_invalid_inner_field_id_errors() {
1128        let element_field =
1129            ArrowField::new("element", ArrowDataType::Int32, true).with_metadata(HashMap::from([
1130                (PARQUET_FIELD_ID_META_KEY.to_string(), "oops".to_string()),
1131            ]));
1132        let list_field = ArrowField::new("arr", ArrowDataType::List(Arc::new(element_field)), true);
1133        assert_result_error_with_message(
1134            StructField::try_from_arrow(&list_field),
1135            "must be an integer",
1136        );
1137    }
1138
1139    /// Every list-flavored Arrow datatype that the converter accepts must also be walked by the
1140    /// nested-ids aggregator.
1141    #[rstest]
1142    #[case::list(ArrowDataType::List(arc_elem_with_id(42)))]
1143    #[case::list_view(ArrowDataType::ListView(arc_elem_with_id(42)))]
1144    #[case::large_list(ArrowDataType::LargeList(arc_elem_with_id(42)))]
1145    #[case::large_list_view(ArrowDataType::LargeListView(arc_elem_with_id(42)))]
1146    #[case::fixed_size_list(ArrowDataType::FixedSizeList(arc_elem_with_id(42), 3))]
1147    fn test_try_from_arrow_aggregates_nested_id_for_all_list_kinds(
1148        #[case] dt: ArrowDataType,
1149    ) -> KernelResult<()> {
1150        let arrow_field = ArrowField::new("arr", dt, true);
1151        let kernel = StructField::try_from_arrow(&arrow_field)?;
1152        assert_eq!(
1153            kernel
1154                .metadata()
1155                .get(ColumnMetadataKey::ColumnMappingNestedIds.as_ref()),
1156            Some(&MetadataValue::Other(
1157                serde_json::json!({"arr.element": 42})
1158            )),
1159        );
1160        Ok(())
1161    }
1162
1163    /// `Dictionary` is a transparent wrapper on the Arrow side (kernel collapses it to its value
1164    /// type), so a nested-ids walker must still descend through it.
1165    #[test]
1166    fn test_try_from_arrow_aggregates_nested_id_through_dictionary() -> KernelResult<()> {
1167        let arrow_dict = ArrowDataType::Dictionary(
1168            Box::new(ArrowDataType::Int32),
1169            Box::new(ArrowDataType::List(arc_elem_with_id(7))),
1170        );
1171        let arrow_field = ArrowField::new("arr", arrow_dict, true);
1172        let kernel = StructField::try_from_arrow(&arrow_field)?;
1173        assert_eq!(
1174            kernel
1175                .metadata()
1176                .get(ColumnMetadataKey::ColumnMappingNestedIds.as_ref()),
1177            Some(&MetadataValue::Other(serde_json::json!({"arr.element": 7}))),
1178        );
1179        Ok(())
1180    }
1181
1182    fn arc_elem_with_id(id: i32) -> Arc<ArrowField> {
1183        Arc::new(
1184            ArrowField::new("element", ArrowDataType::Int32, true).with_metadata(HashMap::from([
1185                (PARQUET_FIELD_ID_META_KEY.to_string(), id.to_string()),
1186            ])),
1187        )
1188    }
1189}
delta_kernel/engine/arrow_conversion/mod.rs

delta_kernel/engine/arrow_conversion/
mod.rs
