vortex_array/scalar_fn/fns/
ext_storage.rs1use vortex_error::VortexResult;
5use vortex_error::vortex_bail;
6use vortex_session::VortexSession;
7use vortex_session::registry::CachedId;
8
9use crate::ArrayRef;
10use crate::ExecutionCtx;
11use crate::IntoArray;
12use crate::arrays::ConstantArray;
13use crate::arrays::ExtensionArray;
14use crate::arrays::extension::ExtensionArrayExt;
15use crate::dtype::DType;
16use crate::scalar_fn::Arity;
17use crate::scalar_fn::ChildName;
18use crate::scalar_fn::EmptyOptions;
19use crate::scalar_fn::ExecutionArgs;
20use crate::scalar_fn::ReduceNode;
21use crate::scalar_fn::ScalarFnId;
22use crate::scalar_fn::ScalarFnVTable;
23
24#[derive(Clone)]
26pub struct ExtStorage;
27
28impl ScalarFnVTable for ExtStorage {
29 type Options = EmptyOptions;
30
31 fn id(&self) -> ScalarFnId {
32 static ID: CachedId = CachedId::new("vortex.ext.storage");
33 *ID
34 }
35
36 fn serialize(&self, _options: &Self::Options) -> VortexResult<Option<Vec<u8>>> {
37 Ok(Some(vec![]))
38 }
39
40 fn deserialize(
41 &self,
42 _metadata: &[u8],
43 _session: &VortexSession,
44 ) -> VortexResult<Self::Options> {
45 Ok(EmptyOptions)
46 }
47
48 fn arity(&self, _options: &Self::Options) -> Arity {
49 Arity::Exact(1)
50 }
51
52 fn child_name(&self, _options: &Self::Options, child_idx: usize) -> ChildName {
53 match child_idx {
54 0 => ChildName::from("input"),
55 _ => unreachable!("Invalid child index {child_idx} for ext_storage()"),
56 }
57 }
58
59 fn return_dtype(&self, _options: &Self::Options, arg_dtypes: &[DType]) -> VortexResult<DType> {
60 let DType::Extension(ext_dtype) = &arg_dtypes[0] else {
61 vortex_bail!("ext_storage() requires Extension, got {}", arg_dtypes[0]);
62 };
63
64 Ok(ext_dtype.storage_dtype().clone())
65 }
66
67 fn execute(
68 &self,
69 _options: &Self::Options,
70 args: &dyn ExecutionArgs,
71 ctx: &mut ExecutionCtx,
72 ) -> VortexResult<ArrayRef> {
73 let input = args.get(0)?;
74
75 if !matches!(input.dtype(), DType::Extension(_)) {
76 vortex_bail!("ext_storage() requires Extension, got {}", input.dtype());
77 }
78
79 if let Some(scalar) = input.as_constant() {
80 let storage_scalar = scalar.as_extension().to_storage_scalar();
81 return Ok(ConstantArray::new(storage_scalar, args.row_count()).into_array());
82 }
83
84 let input = input.execute::<ExtensionArray>(ctx)?;
85 Ok(input.storage_array().clone())
86 }
87
88 fn reduce<T: ReduceNode>(&self, _options: &Self::Options, node: &T) -> VortexResult<Option<T>> {
89 Ok(node
90 .child(0)
91 .as_constant()
92 .map(|scalar| node.new_constant(scalar.as_extension().to_storage_scalar())))
93 }
94
95 fn is_strict(&self, _options: &Self::Options) -> bool {
96 true
97 }
98
99 fn is_infallible(&self, _options: &Self::Options) -> bool {
100 true
101 }
102}
103
104#[cfg(test)]
105mod tests {
106 use std::sync::LazyLock;
107
108 use rstest::rstest;
109 use vortex_buffer::buffer;
110 use vortex_error::VortexResult;
111 use vortex_session::VortexSession;
112 use vortex_session::registry::CachedId;
113
114 use crate::IntoArray;
115 use crate::VortexSessionExecute;
116 use crate::arrays::Constant;
117 use crate::arrays::ConstantArray;
118 use crate::arrays::ExtensionArray;
119 use crate::arrays::PrimitiveArray;
120 use crate::arrays::StructArray;
121 use crate::assert_arrays_eq;
122 use crate::dtype::DType;
123 use crate::dtype::Nullability;
124 use crate::dtype::PType;
125 use crate::dtype::extension::ExtDTypeRef;
126 use crate::dtype::extension::ForeignExtDType;
127 use crate::expr::ext_storage;
128 use crate::expr::get_item;
129 use crate::expr::root;
130 use crate::expr::select;
131 use crate::extension::datetime::TimeUnit;
132 use crate::extension::datetime::Timestamp;
133 use crate::scalar::Scalar;
134
135 static SESSION: LazyLock<VortexSession> = LazyLock::new(VortexSession::empty);
136
137 fn ext_dtype(nullability: Nullability) -> ExtDTypeRef {
138 Timestamp::new(TimeUnit::Nanoseconds, nullability).erased()
139 }
140
141 #[test]
142 fn extracts_extension_storage_array() -> VortexResult<()> {
143 let storage = buffer![2i64, 4, 6].into_array();
144 let array =
145 ExtensionArray::new(ext_dtype(Nullability::NonNullable), storage.clone()).into_array();
146
147 let result = array.apply(&ext_storage(root()))?;
148
149 assert_eq!(
150 result.dtype(),
151 &DType::Primitive(PType::I64, Nullability::NonNullable)
152 );
153 assert_arrays_eq!(result, storage, &mut SESSION.create_execution_ctx());
154 Ok(())
155 }
156
157 #[test]
158 fn extracts_nullable_extension_storage_array() -> VortexResult<()> {
159 let storage = PrimitiveArray::from_option_iter([Some(2i64), None, Some(6)]).into_array();
160 let array =
161 ExtensionArray::new(ext_dtype(Nullability::Nullable), storage.clone()).into_array();
162
163 let result = array.apply(&ext_storage(root()))?;
164
165 assert_eq!(
166 result.dtype(),
167 &DType::Primitive(PType::I64, Nullability::Nullable)
168 );
169 assert_arrays_eq!(result, storage, &mut SESSION.create_execution_ctx());
170 Ok(())
171 }
172
173 #[test]
174 fn extracts_constant_extension_storage_scalar() -> VortexResult<()> {
175 let storage_scalar = Scalar::primitive(4i64, Nullability::NonNullable);
176 let scalar =
177 Scalar::extension_ref(ext_dtype(Nullability::NonNullable), storage_scalar.clone());
178 let array = ConstantArray::new(scalar, 3).into_array();
179
180 let result = array.apply(&ext_storage(root()))?;
181
182 assert_eq!(
183 result.dtype(),
184 &DType::Primitive(PType::I64, Nullability::NonNullable)
185 );
186 assert_arrays_eq!(
187 result,
188 ConstantArray::new(storage_scalar, 3),
189 &mut SESSION.create_execution_ctx()
190 );
191 Ok(())
192 }
193
194 #[rstest]
195 fn constant_storage_reduces(
196 #[values(false, true)] null: bool,
197 #[values(0, 3)] len: usize,
198 ) -> VortexResult<()> {
199 let storage = if null {
200 Scalar::null(DType::Primitive(PType::I64, Nullability::Nullable))
201 } else {
202 Scalar::primitive(4i64, Nullability::Nullable)
203 };
204 let scalar = Scalar::extension_ref(ext_dtype(Nullability::Nullable), storage.clone());
205 let result = ConstantArray::new(scalar, len)
206 .into_array()
207 .apply(&ext_storage(root()))?;
208
209 assert_eq!(result.as_::<Constant>().scalar(), &storage);
210 assert_eq!(result.len(), len);
211 Ok(())
212 }
213
214 #[test]
215 fn nested_constant_field_reduces_through_select_and_storage() -> VortexResult<()> {
216 static EXT_ID: CachedId = CachedId::new("test.measurement");
217 let storage_array =
218 StructArray::from_fields(&[("value", ConstantArray::new(42i32, 1).into_array())])?;
219 let storage = Scalar::struct_(storage_array.dtype().clone(), [Scalar::from(42i32)]);
220 let dtype = ForeignExtDType::from_parts(*EXT_ID, vec![], storage.dtype().clone())?;
221 let measurement = Scalar::extension_ref(dtype, storage);
222 let array = StructArray::from_fields(&[
223 (
224 "measurement",
225 ConstantArray::new(measurement, 3).into_array(),
226 ),
227 ("other", buffer![1i32, 2, 3].into_array()),
228 ])?
229 .into_array();
230 let expr = get_item(
231 "value",
232 ext_storage(get_item("measurement", select(["measurement"], root()))),
233 );
234
235 let result = array.apply(&expr)?;
236
237 assert_eq!(result.as_::<Constant>().scalar(), &Scalar::from(42i32));
238 assert_eq!(result.len(), 3);
239 Ok(())
240 }
241
242 #[test]
243 fn rejects_non_extension_input() {
244 let dtype = DType::Primitive(PType::U64, Nullability::NonNullable);
245 let err = ext_storage(root()).return_dtype(&dtype).unwrap_err();
246 assert!(err.to_string().contains("requires Extension"));
247 }
248
249 #[test]
250 fn test_display() {
251 assert_eq!(ext_storage(root()).to_string(), "vortex.ext.storage($)");
252 }
253}