1mod cast;
5mod compare;
6mod fill_null;
7mod filter;
8pub(crate) mod is_constant;
9pub(crate) mod is_sorted;
10mod like;
11mod mask;
12pub(crate) mod min_max;
13pub(crate) mod rules;
14mod slice;
15
16use vortex_error::VortexResult;
17
18use super::Dict;
19use super::DictArray;
20use super::TakeExecute;
21use crate::ArrayRef;
22use crate::ExecutionCtx;
23use crate::IntoArray;
24use crate::array::ArrayView;
25use crate::arrays::dict::DictArraySlotsExt;
26
27impl TakeExecute for Dict {
28 fn take(
29 array: ArrayView<'_, Dict>,
30 indices: &ArrayRef,
31 _ctx: &mut ExecutionCtx,
32 ) -> VortexResult<Option<ArrayRef>> {
33 let codes = array.codes().take(indices.clone())?;
34
35 let taken = unsafe { DictArray::new_unchecked(codes, array.values().clone()) };
39
40 Ok(Some(taken.into_array()))
41 }
42}
43
44#[cfg(test)]
45mod test {
46 use std::sync::LazyLock;
47
48 use vortex_buffer::buffer;
49 use vortex_error::VortexResult;
50 use vortex_session::VortexSession;
51
52 use crate::ArrayRef;
53 use crate::IntoArray;
54 use crate::VortexSessionExecute;
55 use crate::arrays::ConstantArray;
56 use crate::arrays::PrimitiveArray;
57 use crate::arrays::VarBinArray;
58 use crate::arrays::VarBinViewArray;
59 use crate::assert_arrays_eq;
60 use crate::builders::dict::dict_encode;
61 use crate::builtins::ArrayBuiltins;
62 use crate::compute::conformance::filter::test_filter_conformance;
63 use crate::compute::conformance::mask::test_mask_conformance;
64 use crate::compute::conformance::take::test_take_conformance;
65 use crate::dtype::DType;
66 use crate::dtype::Nullability;
67 use crate::dtype::PType::I32;
68 use crate::scalar_fn::fns::operators::Operator;
69
70 static SESSION: LazyLock<VortexSession> = LazyLock::new(VortexSession::empty);
71
72 #[test]
73 fn canonicalise_nullable_primitive() {
74 let mut ctx = SESSION.create_execution_ctx();
75 let values: Vec<Option<i32>> = (0..65)
76 .map(|i| match i % 3 {
77 0 => Some(42),
78 1 => Some(-9),
79 2 => None,
80 _ => unreachable!(),
81 })
82 .collect();
83
84 let dict = dict_encode(
85 &PrimitiveArray::from_option_iter(values.clone()).into_array(),
86 &mut SESSION.create_execution_ctx(),
87 )
88 .unwrap();
89
90 let expected = PrimitiveArray::from_option_iter(values);
91
92 assert_arrays_eq!(dict, expected, &mut ctx);
93 }
94
95 #[test]
96 fn canonicalise_non_nullable_primitive_32_unique_values() {
97 let mut ctx = SESSION.create_execution_ctx();
98 let unique_values: Vec<i32> = (0..32).collect();
99 let expected = PrimitiveArray::from_iter((0..1000).map(|i| unique_values[i % 32]));
100
101 let dict = dict_encode(
102 &expected.clone().into_array(),
103 &mut SESSION.create_execution_ctx(),
104 )
105 .unwrap()
106 .into_array();
107
108 assert_arrays_eq!(dict, expected, &mut ctx);
109 }
110
111 #[test]
112 fn canonicalise_non_nullable_primitive_100_unique_values() {
113 let mut ctx = SESSION.create_execution_ctx();
114 let unique_values: Vec<i32> = (0..100).collect();
115 let expected = PrimitiveArray::from_iter((0..1000).map(|i| unique_values[i % 100]));
116
117 let dict = dict_encode(
118 &expected.clone().into_array(),
119 &mut SESSION.create_execution_ctx(),
120 )
121 .unwrap()
122 .into_array();
123
124 assert_arrays_eq!(dict, expected, &mut ctx);
125 }
126
127 #[test]
128 fn canonicalise_nullable_varbin() -> VortexResult<()> {
129 let reference = VarBinViewArray::from_iter(
130 vec![Some("a"), Some("b"), None, Some("a"), None, Some("b")],
131 DType::Utf8(Nullability::Nullable),
132 );
133 assert_eq!(reference.len(), 6);
134 let mut ctx = SESSION.create_execution_ctx();
135 let dict = dict_encode(&reference.clone().into_array(), &mut ctx)?;
136 let flattened_dict = dict.into_array().execute::<VarBinViewArray>(&mut ctx)?;
137 let flattened_mask = flattened_dict
138 .validity()?
139 .execute_mask(flattened_dict.len(), &mut ctx)?;
140 let flattened_values = (0..flattened_dict.len())
141 .map(|i| {
142 flattened_mask
143 .value(i)
144 .then(|| flattened_dict.bytes_at(i).to_vec())
145 })
146 .collect::<Vec<_>>();
147 let reference_mask = reference
148 .validity()?
149 .execute_mask(reference.len(), &mut ctx)?;
150 let reference_values = (0..reference.len())
151 .map(|i| {
152 reference_mask
153 .value(i)
154 .then(|| reference.bytes_at(i).to_vec())
155 })
156 .collect::<Vec<_>>();
157 assert_eq!(flattened_values, reference_values);
158 Ok(())
159 }
160
161 fn sliced_dict_array() -> ArrayRef {
162 let reference = PrimitiveArray::from_option_iter([
163 Some(42),
164 Some(-9),
165 None,
166 Some(42),
167 Some(1),
168 Some(5),
169 ]);
170 let dict =
171 dict_encode(&reference.into_array(), &mut SESSION.create_execution_ctx()).unwrap();
172 dict.slice(1..4).unwrap()
173 }
174
175 #[test]
176 fn compare_sliced_dict() {
177 let mut ctx = SESSION.create_execution_ctx();
178 use crate::arrays::BoolArray;
179 let sliced = sliced_dict_array();
180 let compared = sliced
181 .binary(ConstantArray::new(42, 3).into_array(), Operator::Eq)
182 .unwrap();
183
184 let expected = BoolArray::from_iter([Some(false), None, Some(true)]);
185 assert_arrays_eq!(compared, expected.into_array(), &mut ctx);
186 }
187
188 #[test]
189 fn test_mask_dict_array() {
190 let array = dict_encode(
191 &buffer![2, 0, 2, 0, 10].into_array(),
192 &mut SESSION.create_execution_ctx(),
193 )
194 .unwrap();
195 test_mask_conformance(&array.into_array(), &mut SESSION.create_execution_ctx());
196
197 let array = dict_encode(
198 &PrimitiveArray::from_option_iter([Some(2), None, Some(2), Some(0), Some(10)])
199 .into_array(),
200 &mut SESSION.create_execution_ctx(),
201 )
202 .unwrap();
203 test_mask_conformance(&array.into_array(), &mut SESSION.create_execution_ctx());
204
205 let array = dict_encode(
206 &VarBinArray::from_iter(
207 [
208 Some("hello"),
209 None,
210 Some("hello"),
211 Some("good"),
212 Some("good"),
213 ],
214 DType::Utf8(Nullability::Nullable),
215 )
216 .into_array(),
217 &mut SESSION.create_execution_ctx(),
218 )
219 .unwrap();
220 test_mask_conformance(&array.into_array(), &mut SESSION.create_execution_ctx());
221 }
222
223 #[test]
224 fn test_filter_dict_array() {
225 let array = dict_encode(
226 &buffer![2, 0, 2, 0, 10].into_array(),
227 &mut SESSION.create_execution_ctx(),
228 )
229 .unwrap();
230 test_filter_conformance(&array.into_array(), &mut SESSION.create_execution_ctx());
231
232 let array = dict_encode(
233 &PrimitiveArray::from_option_iter([Some(2), None, Some(2), Some(0), Some(10)])
234 .into_array(),
235 &mut SESSION.create_execution_ctx(),
236 )
237 .unwrap();
238 test_filter_conformance(&array.into_array(), &mut SESSION.create_execution_ctx());
239
240 let array = dict_encode(
241 &VarBinArray::from_iter(
242 [
243 Some("hello"),
244 None,
245 Some("hello"),
246 Some("good"),
247 Some("good"),
248 ],
249 DType::Utf8(Nullability::Nullable),
250 )
251 .into_array(),
252 &mut SESSION.create_execution_ctx(),
253 )
254 .unwrap();
255 test_filter_conformance(&array.into_array(), &mut SESSION.create_execution_ctx());
256 }
257
258 #[test]
259 fn test_take_dict() {
260 let array = dict_encode(
261 &buffer![1, 2].into_array(),
262 &mut SESSION.create_execution_ctx(),
263 )
264 .unwrap();
265
266 let taken = array
267 .take(PrimitiveArray::from_option_iter([None::<i32>]).into_array())
268 .unwrap();
269 assert_eq!(taken.dtype(), &DType::Primitive(I32, Nullability::Nullable));
270 assert_arrays_eq!(
271 taken,
272 PrimitiveArray::from_option_iter([None::<i32>]),
273 &mut SESSION.create_execution_ctx()
274 );
275 }
276
277 #[test]
278 fn test_take_dict_conformance() {
279 let array = dict_encode(
280 &buffer![2, 0, 2, 0, 10].into_array(),
281 &mut SESSION.create_execution_ctx(),
282 )
283 .unwrap();
284 test_take_conformance(&array.into_array(), &mut SESSION.create_execution_ctx());
285
286 let array = dict_encode(
287 &PrimitiveArray::from_option_iter([Some(2), None, Some(2), Some(0), Some(10)])
288 .into_array(),
289 &mut SESSION.create_execution_ctx(),
290 )
291 .unwrap();
292 test_take_conformance(&array.into_array(), &mut SESSION.create_execution_ctx());
293
294 let array = dict_encode(
295 &VarBinArray::from_iter(
296 [
297 Some("hello"),
298 None,
299 Some("hello"),
300 Some("good"),
301 Some("good"),
302 ],
303 DType::Utf8(Nullability::Nullable),
304 )
305 .into_array(),
306 &mut SESSION.create_execution_ctx(),
307 )
308 .unwrap();
309 test_take_conformance(&array.into_array(), &mut SESSION.create_execution_ctx());
310 }
311}
312
313#[cfg(test)]
314mod tests {
315 use std::sync::LazyLock;
316
317 use rstest::rstest;
318 use vortex_buffer::buffer;
319 use vortex_session::VortexSession;
320
321 use crate::IntoArray;
322 use crate::VortexSessionExecute;
323 use crate::arrays::DictArray;
324 use crate::arrays::PrimitiveArray;
325 use crate::arrays::VarBinArray;
326 use crate::builders::dict::dict_encode;
327 use crate::compute::conformance::consistency::test_array_consistency;
328 use crate::dtype::DType;
329 use crate::dtype::Nullability;
330
331 static SESSION: LazyLock<VortexSession> = LazyLock::new(VortexSession::empty);
332
333 #[rstest]
334 #[case::dict_i32(dict_encode(&buffer![1i32, 2, 3, 2, 1].into_array(), &mut SESSION.create_execution_ctx()).unwrap())]
336 #[case::dict_nullable_codes(DictArray::try_new(
337 buffer![0u32, 1, 2, 2, 0].into_array(),
338 PrimitiveArray::from_option_iter([Some(10), Some(20), None]).into_array(),
339 ).unwrap())]
340 #[case::dict_nullable_values(dict_encode(
341 &PrimitiveArray::from_option_iter([Some(1i32), None, Some(2), Some(1), None]).into_array()
342 , &mut SESSION.create_execution_ctx()).unwrap())]
343 #[case::dict_u64(dict_encode(&buffer![100u64, 200, 100, 300, 200].into_array(), &mut SESSION.create_execution_ctx()).unwrap())]
344 #[case::dict_str(dict_encode(
346 &VarBinArray::from_iter(
347 ["hello", "world", "hello", "test", "world"].map(Some),
348 DType::Utf8(Nullability::NonNullable),
349 ).into_array()
350 , &mut SESSION.create_execution_ctx()).unwrap())]
351 #[case::dict_nullable_str(dict_encode(
352 &VarBinArray::from_iter(
353 [Some("hello"), None, Some("world"), Some("hello"), None],
354 DType::Utf8(Nullability::Nullable),
355 ).into_array()
356 , &mut SESSION.create_execution_ctx()).unwrap())]
357 #[case::dict_single(dict_encode(&buffer![42i32].into_array(), &mut SESSION.create_execution_ctx()).unwrap())]
359 #[case::dict_all_same(dict_encode(&buffer![5i32, 5, 5, 5, 5].into_array(), &mut SESSION.create_execution_ctx()).unwrap())]
360 #[case::dict_large(dict_encode(&PrimitiveArray::from_iter((0..1000).map(|i| i % 10)).into_array(), &mut SESSION.create_execution_ctx()).unwrap())]
361 fn test_dict_consistency(#[case] array: DictArray) {
362 test_array_consistency(&array.into_array(), &mut SESSION.create_execution_ctx());
363 }
364}