Skip to main content

vortex_array/scalar_fn/fns/
ext_storage.rs

1// SPDX-License-Identifier: Apache-2.0
2// SPDX-FileCopyrightText: Copyright the Vortex contributors
3
4use vortex_error::VortexResult;
5use vortex_error::vortex_bail;
6use vortex_session::VortexSession;
7use vortex_session::registry::CachedId;
8
9use crate::ArrayRef;
10use crate::ExecutionCtx;
11use crate::IntoArray;
12use crate::arrays::ConstantArray;
13use crate::arrays::ExtensionArray;
14use crate::arrays::extension::ExtensionArrayExt;
15use crate::dtype::DType;
16use crate::scalar_fn::Arity;
17use crate::scalar_fn::ChildName;
18use crate::scalar_fn::EmptyOptions;
19use crate::scalar_fn::ExecutionArgs;
20use crate::scalar_fn::ReduceNode;
21use crate::scalar_fn::ScalarFnId;
22use crate::scalar_fn::ScalarFnVTable;
23
24/// Extract the storage values from an extension array.
25#[derive(Clone)]
26pub struct ExtStorage;
27
28impl ScalarFnVTable for ExtStorage {
29    type Options = EmptyOptions;
30
31    fn id(&self) -> ScalarFnId {
32        static ID: CachedId = CachedId::new("vortex.ext.storage");
33        *ID
34    }
35
36    fn serialize(&self, _options: &Self::Options) -> VortexResult<Option<Vec<u8>>> {
37        Ok(Some(vec![]))
38    }
39
40    fn deserialize(
41        &self,
42        _metadata: &[u8],
43        _session: &VortexSession,
44    ) -> VortexResult<Self::Options> {
45        Ok(EmptyOptions)
46    }
47
48    fn arity(&self, _options: &Self::Options) -> Arity {
49        Arity::Exact(1)
50    }
51
52    fn child_name(&self, _options: &Self::Options, child_idx: usize) -> ChildName {
53        match child_idx {
54            0 => ChildName::from("input"),
55            _ => unreachable!("Invalid child index {child_idx} for ext_storage()"),
56        }
57    }
58
59    fn return_dtype(&self, _options: &Self::Options, arg_dtypes: &[DType]) -> VortexResult<DType> {
60        let DType::Extension(ext_dtype) = &arg_dtypes[0] else {
61            vortex_bail!("ext_storage() requires Extension, got {}", arg_dtypes[0]);
62        };
63
64        Ok(ext_dtype.storage_dtype().clone())
65    }
66
67    fn execute(
68        &self,
69        _options: &Self::Options,
70        args: &dyn ExecutionArgs,
71        ctx: &mut ExecutionCtx,
72    ) -> VortexResult<ArrayRef> {
73        let input = args.get(0)?;
74
75        if !matches!(input.dtype(), DType::Extension(_)) {
76            vortex_bail!("ext_storage() requires Extension, got {}", input.dtype());
77        }
78
79        if let Some(scalar) = input.as_constant() {
80            let storage_scalar = scalar.as_extension().to_storage_scalar();
81            return Ok(ConstantArray::new(storage_scalar, args.row_count()).into_array());
82        }
83
84        let input = input.execute::<ExtensionArray>(ctx)?;
85        Ok(input.storage_array().clone())
86    }
87
88    fn reduce<T: ReduceNode>(&self, _options: &Self::Options, node: &T) -> VortexResult<Option<T>> {
89        Ok(node
90            .child(0)
91            .as_constant()
92            .map(|scalar| node.new_constant(scalar.as_extension().to_storage_scalar())))
93    }
94
95    fn is_strict(&self, _options: &Self::Options) -> bool {
96        true
97    }
98
99    fn is_infallible(&self, _options: &Self::Options) -> bool {
100        true
101    }
102}
103
104#[cfg(test)]
105mod tests {
106    use std::sync::LazyLock;
107
108    use rstest::rstest;
109    use vortex_buffer::buffer;
110    use vortex_error::VortexResult;
111    use vortex_session::VortexSession;
112    use vortex_session::registry::CachedId;
113
114    use crate::IntoArray;
115    use crate::VortexSessionExecute;
116    use crate::arrays::Constant;
117    use crate::arrays::ConstantArray;
118    use crate::arrays::ExtensionArray;
119    use crate::arrays::PrimitiveArray;
120    use crate::arrays::StructArray;
121    use crate::assert_arrays_eq;
122    use crate::dtype::DType;
123    use crate::dtype::Nullability;
124    use crate::dtype::PType;
125    use crate::dtype::extension::ExtDTypeRef;
126    use crate::dtype::extension::ForeignExtDType;
127    use crate::expr::ext_storage;
128    use crate::expr::get_item;
129    use crate::expr::root;
130    use crate::expr::select;
131    use crate::extension::datetime::TimeUnit;
132    use crate::extension::datetime::Timestamp;
133    use crate::scalar::Scalar;
134
135    static SESSION: LazyLock<VortexSession> = LazyLock::new(VortexSession::empty);
136
137    fn ext_dtype(nullability: Nullability) -> ExtDTypeRef {
138        Timestamp::new(TimeUnit::Nanoseconds, nullability).erased()
139    }
140
141    #[test]
142    fn extracts_extension_storage_array() -> VortexResult<()> {
143        let storage = buffer![2i64, 4, 6].into_array();
144        let array =
145            ExtensionArray::new(ext_dtype(Nullability::NonNullable), storage.clone()).into_array();
146
147        let result = array.apply(&ext_storage(root()))?;
148
149        assert_eq!(
150            result.dtype(),
151            &DType::Primitive(PType::I64, Nullability::NonNullable)
152        );
153        assert_arrays_eq!(result, storage, &mut SESSION.create_execution_ctx());
154        Ok(())
155    }
156
157    #[test]
158    fn extracts_nullable_extension_storage_array() -> VortexResult<()> {
159        let storage = PrimitiveArray::from_option_iter([Some(2i64), None, Some(6)]).into_array();
160        let array =
161            ExtensionArray::new(ext_dtype(Nullability::Nullable), storage.clone()).into_array();
162
163        let result = array.apply(&ext_storage(root()))?;
164
165        assert_eq!(
166            result.dtype(),
167            &DType::Primitive(PType::I64, Nullability::Nullable)
168        );
169        assert_arrays_eq!(result, storage, &mut SESSION.create_execution_ctx());
170        Ok(())
171    }
172
173    #[test]
174    fn extracts_constant_extension_storage_scalar() -> VortexResult<()> {
175        let storage_scalar = Scalar::primitive(4i64, Nullability::NonNullable);
176        let scalar =
177            Scalar::extension_ref(ext_dtype(Nullability::NonNullable), storage_scalar.clone());
178        let array = ConstantArray::new(scalar, 3).into_array();
179
180        let result = array.apply(&ext_storage(root()))?;
181
182        assert_eq!(
183            result.dtype(),
184            &DType::Primitive(PType::I64, Nullability::NonNullable)
185        );
186        assert_arrays_eq!(
187            result,
188            ConstantArray::new(storage_scalar, 3),
189            &mut SESSION.create_execution_ctx()
190        );
191        Ok(())
192    }
193
194    #[rstest]
195    fn constant_storage_reduces(
196        #[values(false, true)] null: bool,
197        #[values(0, 3)] len: usize,
198    ) -> VortexResult<()> {
199        let storage = if null {
200            Scalar::null(DType::Primitive(PType::I64, Nullability::Nullable))
201        } else {
202            Scalar::primitive(4i64, Nullability::Nullable)
203        };
204        let scalar = Scalar::extension_ref(ext_dtype(Nullability::Nullable), storage.clone());
205        let result = ConstantArray::new(scalar, len)
206            .into_array()
207            .apply(&ext_storage(root()))?;
208
209        assert_eq!(result.as_::<Constant>().scalar(), &storage);
210        assert_eq!(result.len(), len);
211        Ok(())
212    }
213
214    #[test]
215    fn nested_constant_field_reduces_through_select_and_storage() -> VortexResult<()> {
216        static EXT_ID: CachedId = CachedId::new("test.measurement");
217        let storage_array =
218            StructArray::from_fields(&[("value", ConstantArray::new(42i32, 1).into_array())])?;
219        let storage = Scalar::struct_(storage_array.dtype().clone(), [Scalar::from(42i32)]);
220        let dtype = ForeignExtDType::from_parts(*EXT_ID, vec![], storage.dtype().clone())?;
221        let measurement = Scalar::extension_ref(dtype, storage);
222        let array = StructArray::from_fields(&[
223            (
224                "measurement",
225                ConstantArray::new(measurement, 3).into_array(),
226            ),
227            ("other", buffer![1i32, 2, 3].into_array()),
228        ])?
229        .into_array();
230        let expr = get_item(
231            "value",
232            ext_storage(get_item("measurement", select(["measurement"], root()))),
233        );
234
235        let result = array.apply(&expr)?;
236
237        assert_eq!(result.as_::<Constant>().scalar(), &Scalar::from(42i32));
238        assert_eq!(result.len(), 3);
239        Ok(())
240    }
241
242    #[test]
243    fn rejects_non_extension_input() {
244        let dtype = DType::Primitive(PType::U64, Nullability::NonNullable);
245        let err = ext_storage(root()).return_dtype(&dtype).unwrap_err();
246        assert!(err.to_string().contains("requires Extension"));
247    }
248
249    #[test]
250    fn test_display() {
251        assert_eq!(ext_storage(root()).to_string(), "vortex.ext.storage($)");
252    }
253}