Skip to main content

arrow_schema/extension/canonical/
variant.rs

1// SPDX-License-Identifier: Apache-2.0
2// SPDX-FileCopyrightText: See the NOTICE file distributed with this work for additional information regarding copyright ownership
3// SPDX-FileContributor: Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements
4
5//! [`VariantType`] Arrow extension type
6
7use crate::extension::ExtensionType;
8use crate::{ArrowError, DataType};
9
10/// Arrow Variant [`ExtensionType`].
11///
12/// Represents the canonical Arrow Extension Type for storing variants.
13/// See [`VariantArray`] in the `parquet-variant-compute` crate for more examples
14/// of using this extension type.
15///
16/// # Storage
17///
18/// The storage type must be a struct with the following fields, in any order:
19///
20/// 1. `metadata`: binary, large_binary, or binary_view, optionally dictionary
21///    or run-end encoded
22///
23/// 2. `value`: binary, large_binary, or binary_view, required unless
24///    `typed_value` is present
25///
26/// 3. `typed_value`: optional shredded values
27///
28/// The `typed_value` types and the encoded bytes are not checked here, see
29/// [`VariantArray`].
30///
31/// [`VariantArray`]: https://docs.rs/parquet-variant-compute/latest/parquet_variant_compute/struct.VariantArray.html
32#[derive(Debug, Clone, PartialEq)]
33pub struct VariantType;
34
35impl ExtensionType for VariantType {
36    const NAME: &'static str = "arrow.parquet.variant";
37
38    // Variants extension metadata is an empty string
39    // <https://github.com/apache/arrow/blob/d803afcc43f5d132506318fd9e162d33b2c3d4cd/docs/source/format/CanonicalExtensions.rst?plain=1#L473>
40    type Metadata = &'static str;
41
42    fn metadata(&self) -> &Self::Metadata {
43        &""
44    }
45
46    fn serialize_metadata(&self) -> Option<String> {
47        Some(String::new())
48    }
49
50    fn deserialize_metadata(_metadata: Option<&str>) -> Result<Self::Metadata, ArrowError> {
51        Ok("")
52    }
53
54    fn supports_data_type(&self, data_type: &DataType) -> Result<(), ArrowError> {
55        // `DataType::is_binary` also accepts FixedSizeBinary
56        let is_binary = |data_type: &DataType| {
57            matches!(
58                data_type,
59                DataType::Binary | DataType::LargeBinary | DataType::BinaryView
60            )
61        };
62        let err = |message: String| Err(ArrowError::InvalidArgumentError(message));
63
64        let DataType::Struct(fields) = data_type else {
65            return err(format!(
66                "VariantType only supports StructArray, got {data_type}"
67            ));
68        };
69
70        let Some((_, metadata)) = fields.find("metadata") else {
71            return err(
72                "Invalid VariantType: StructArray must contain a 'metadata' field".to_string(),
73            );
74        };
75        let metadata_ok = match metadata.data_type() {
76            DataType::Dictionary(_, values) => is_binary(values),
77            DataType::RunEndEncoded(_, values) => is_binary(values.data_type()),
78            other => is_binary(other),
79        };
80        if !metadata_ok {
81            return err(format!(
82                "VariantType 'metadata' field must be Binary, LargeBinary, BinaryView, or a Dictionary or RunEndEncoded array of one of those types, got {}",
83                metadata.data_type()
84            ));
85        }
86
87        match fields.find("value") {
88            Some((_, value)) if !is_binary(value.data_type()) => err(format!(
89                "VariantType 'value' field must be Binary, LargeBinary, or BinaryView, got {}",
90                value.data_type()
91            )),
92            None if fields.find("typed_value").is_none() => err(
93                "Invalid VariantType: StructArray must contain a 'value' or 'typed_value' field"
94                    .to_string(),
95            ),
96            _ => Ok(()),
97        }
98    }
99
100    fn try_new(data_type: &DataType, _metadata: Self::Metadata) -> Result<Self, ArrowError> {
101        Self.supports_data_type(data_type)?;
102        Ok(Self)
103    }
104
105    fn validate(data_type: &DataType, _metadata: Self::Metadata) -> Result<(), ArrowError> {
106        Self.supports_data_type(data_type)
107    }
108}
109
110#[cfg(test)]
111mod tests {
112    use super::*;
113    use crate::extension::CanonicalExtensionType;
114    use crate::{Field, Fields};
115    use std::sync::Arc;
116
117    fn storage(fields: Vec<Field>) -> DataType {
118        DataType::Struct(Fields::from(fields))
119    }
120
121    fn metadata(data_type: DataType) -> Field {
122        Field::new("metadata", data_type, false)
123    }
124
125    fn value(data_type: DataType) -> Field {
126        Field::new("value", data_type, true)
127    }
128
129    fn typed_value(data_type: DataType) -> Field {
130        Field::new("typed_value", data_type, true)
131    }
132
133    #[test]
134    fn variant_type_canonical_extension() {
135        let data_type = storage(vec![
136            metadata(DataType::BinaryView),
137            value(DataType::BinaryView),
138        ]);
139        let field = Field::new("v", data_type, true).with_extension_type(VariantType);
140        assert!(field.has_valid_extension_type::<VariantType>());
141        assert_eq!(
142            field.try_canonical_extension_type().unwrap(),
143            CanonicalExtensionType::Variant(VariantType),
144        );
145    }
146
147    #[test]
148    fn variant_type_accepts_valid_storage() {
149        let dictionary = DataType::Dictionary(Box::new(DataType::Int8), Box::new(DataType::Binary));
150        let run_end_encoded = DataType::RunEndEncoded(
151            Arc::new(Field::new("run_ends", DataType::Int32, false)),
152            Arc::new(Field::new("values", DataType::LargeBinary, true)),
153        );
154        let valid = [
155            storage(vec![metadata(DataType::Binary), value(DataType::Binary)]),
156            storage(vec![
157                value(DataType::LargeBinary),
158                metadata(DataType::LargeBinary),
159            ]),
160            storage(vec![metadata(dictionary), value(DataType::BinaryView)]),
161            storage(vec![metadata(run_end_encoded), value(DataType::BinaryView)]),
162            storage(vec![
163                metadata(DataType::BinaryView),
164                value(DataType::BinaryView),
165                typed_value(DataType::Int64),
166            ]),
167            // Allowed by the Arrow spec, though the Parquet shredding spec requires `value`
168            storage(vec![
169                metadata(DataType::BinaryView),
170                typed_value(DataType::Utf8),
171            ]),
172        ];
173        for data_type in valid {
174            VariantType::validate(&data_type, "").unwrap();
175        }
176    }
177
178    #[test]
179    fn variant_type_rejects_invalid_storage() {
180        let err = Field::new("v", DataType::Binary, true)
181            .try_with_extension_type(VariantType)
182            .unwrap_err();
183        assert_eq!(
184            err.to_string(),
185            "Invalid argument error: VariantType only supports StructArray, got Binary"
186        );
187
188        let invalid = [
189            (
190                storage(vec![Field::new("unrelated", DataType::Int32, false)]),
191                "Invalid VariantType: StructArray must contain a 'metadata' field",
192            ),
193            (
194                storage(vec![
195                    metadata(DataType::FixedSizeBinary(16)),
196                    value(DataType::BinaryView),
197                ]),
198                "VariantType 'metadata' field must be Binary, LargeBinary, BinaryView, or a Dictionary or RunEndEncoded array of one of those types, got FixedSizeBinary(16)",
199            ),
200            (
201                storage(vec![
202                    metadata(DataType::Dictionary(
203                        Box::new(DataType::Int8),
204                        Box::new(DataType::Utf8),
205                    )),
206                    value(DataType::BinaryView),
207                ]),
208                "VariantType 'metadata' field must be Binary, LargeBinary, BinaryView, or a Dictionary or RunEndEncoded array of one of those types, got Dictionary(Int8, Utf8)",
209            ),
210            (
211                storage(vec![metadata(DataType::BinaryView), value(DataType::Utf8)]),
212                "VariantType 'value' field must be Binary, LargeBinary, or BinaryView, got Utf8",
213            ),
214            (
215                storage(vec![metadata(DataType::BinaryView)]),
216                "Invalid VariantType: StructArray must contain a 'value' or 'typed_value' field",
217            ),
218        ];
219        for (data_type, expected) in invalid {
220            let err = VariantType::validate(&data_type, "").unwrap_err();
221            assert_eq!(
222                err.to_string(),
223                format!("Invalid argument error: {expected}")
224            );
225        }
226    }
227}