Skip to main content

vortex_array/arrays/dict/compute/
mod.rs

1// SPDX-License-Identifier: Apache-2.0
2// SPDX-FileCopyrightText: Copyright the Vortex contributors
3
4mod cast;
5mod compare;
6mod fill_null;
7mod filter;
8pub(crate) mod is_constant;
9pub(crate) mod is_sorted;
10mod like;
11mod mask;
12pub(crate) mod min_max;
13pub(crate) mod rules;
14mod slice;
15
16use vortex_error::VortexResult;
17
18use super::Dict;
19use super::DictArray;
20use super::TakeExecute;
21use crate::ArrayRef;
22use crate::ExecutionCtx;
23use crate::IntoArray;
24use crate::array::ArrayView;
25use crate::arrays::dict::DictArraySlotsExt;
26
27impl TakeExecute for Dict {
28    fn take(
29        array: ArrayView<'_, Dict>,
30        indices: &ArrayRef,
31        _ctx: &mut ExecutionCtx,
32    ) -> VortexResult<Option<ArrayRef>> {
33        let codes = array.codes().take(indices.clone())?;
34
35        // SAFETY: Selection preserves the integer code type and non-null code bounds, so every code
36        // indexes the unchanged values. `new_unchecked` resets `all_values_referenced` because
37        // selection can remove a value's last reference.
38        let taken = unsafe { DictArray::new_unchecked(codes, array.values().clone()) };
39
40        Ok(Some(taken.into_array()))
41    }
42}
43
44#[cfg(test)]
45mod test {
46    use std::sync::LazyLock;
47
48    use vortex_buffer::buffer;
49    use vortex_error::VortexResult;
50    use vortex_session::VortexSession;
51
52    use crate::ArrayRef;
53    use crate::IntoArray;
54    use crate::VortexSessionExecute;
55    use crate::arrays::ConstantArray;
56    use crate::arrays::PrimitiveArray;
57    use crate::arrays::VarBinArray;
58    use crate::arrays::VarBinViewArray;
59    use crate::assert_arrays_eq;
60    use crate::builders::dict::dict_encode;
61    use crate::builtins::ArrayBuiltins;
62    use crate::compute::conformance::filter::test_filter_conformance;
63    use crate::compute::conformance::mask::test_mask_conformance;
64    use crate::compute::conformance::take::test_take_conformance;
65    use crate::dtype::DType;
66    use crate::dtype::Nullability;
67    use crate::dtype::PType::I32;
68    use crate::scalar_fn::fns::operators::Operator;
69
70    static SESSION: LazyLock<VortexSession> = LazyLock::new(VortexSession::empty);
71
72    #[test]
73    fn canonicalise_nullable_primitive() {
74        let mut ctx = SESSION.create_execution_ctx();
75        let values: Vec<Option<i32>> = (0..65)
76            .map(|i| match i % 3 {
77                0 => Some(42),
78                1 => Some(-9),
79                2 => None,
80                _ => unreachable!(),
81            })
82            .collect();
83
84        let dict = dict_encode(
85            &PrimitiveArray::from_option_iter(values.clone()).into_array(),
86            &mut SESSION.create_execution_ctx(),
87        )
88        .unwrap();
89
90        let expected = PrimitiveArray::from_option_iter(values);
91
92        assert_arrays_eq!(dict, expected, &mut ctx);
93    }
94
95    #[test]
96    fn canonicalise_non_nullable_primitive_32_unique_values() {
97        let mut ctx = SESSION.create_execution_ctx();
98        let unique_values: Vec<i32> = (0..32).collect();
99        let expected = PrimitiveArray::from_iter((0..1000).map(|i| unique_values[i % 32]));
100
101        let dict = dict_encode(
102            &expected.clone().into_array(),
103            &mut SESSION.create_execution_ctx(),
104        )
105        .unwrap()
106        .into_array();
107
108        assert_arrays_eq!(dict, expected, &mut ctx);
109    }
110
111    #[test]
112    fn canonicalise_non_nullable_primitive_100_unique_values() {
113        let mut ctx = SESSION.create_execution_ctx();
114        let unique_values: Vec<i32> = (0..100).collect();
115        let expected = PrimitiveArray::from_iter((0..1000).map(|i| unique_values[i % 100]));
116
117        let dict = dict_encode(
118            &expected.clone().into_array(),
119            &mut SESSION.create_execution_ctx(),
120        )
121        .unwrap()
122        .into_array();
123
124        assert_arrays_eq!(dict, expected, &mut ctx);
125    }
126
127    #[test]
128    fn canonicalise_nullable_varbin() -> VortexResult<()> {
129        let reference = VarBinViewArray::from_iter(
130            vec![Some("a"), Some("b"), None, Some("a"), None, Some("b")],
131            DType::Utf8(Nullability::Nullable),
132        );
133        assert_eq!(reference.len(), 6);
134        let mut ctx = SESSION.create_execution_ctx();
135        let dict = dict_encode(&reference.clone().into_array(), &mut ctx)?;
136        let flattened_dict = dict.into_array().execute::<VarBinViewArray>(&mut ctx)?;
137        let flattened_mask = flattened_dict
138            .validity()?
139            .execute_mask(flattened_dict.len(), &mut ctx)?;
140        let flattened_values = (0..flattened_dict.len())
141            .map(|i| {
142                flattened_mask
143                    .value(i)
144                    .then(|| flattened_dict.bytes_at(i).to_vec())
145            })
146            .collect::<Vec<_>>();
147        let reference_mask = reference
148            .validity()?
149            .execute_mask(reference.len(), &mut ctx)?;
150        let reference_values = (0..reference.len())
151            .map(|i| {
152                reference_mask
153                    .value(i)
154                    .then(|| reference.bytes_at(i).to_vec())
155            })
156            .collect::<Vec<_>>();
157        assert_eq!(flattened_values, reference_values);
158        Ok(())
159    }
160
161    fn sliced_dict_array() -> ArrayRef {
162        let reference = PrimitiveArray::from_option_iter([
163            Some(42),
164            Some(-9),
165            None,
166            Some(42),
167            Some(1),
168            Some(5),
169        ]);
170        let dict =
171            dict_encode(&reference.into_array(), &mut SESSION.create_execution_ctx()).unwrap();
172        dict.slice(1..4).unwrap()
173    }
174
175    #[test]
176    fn compare_sliced_dict() {
177        let mut ctx = SESSION.create_execution_ctx();
178        use crate::arrays::BoolArray;
179        let sliced = sliced_dict_array();
180        let compared = sliced
181            .binary(ConstantArray::new(42, 3).into_array(), Operator::Eq)
182            .unwrap();
183
184        let expected = BoolArray::from_iter([Some(false), None, Some(true)]);
185        assert_arrays_eq!(compared, expected.into_array(), &mut ctx);
186    }
187
188    #[test]
189    fn test_mask_dict_array() {
190        let array = dict_encode(
191            &buffer![2, 0, 2, 0, 10].into_array(),
192            &mut SESSION.create_execution_ctx(),
193        )
194        .unwrap();
195        test_mask_conformance(&array.into_array(), &mut SESSION.create_execution_ctx());
196
197        let array = dict_encode(
198            &PrimitiveArray::from_option_iter([Some(2), None, Some(2), Some(0), Some(10)])
199                .into_array(),
200            &mut SESSION.create_execution_ctx(),
201        )
202        .unwrap();
203        test_mask_conformance(&array.into_array(), &mut SESSION.create_execution_ctx());
204
205        let array = dict_encode(
206            &VarBinArray::from_iter(
207                [
208                    Some("hello"),
209                    None,
210                    Some("hello"),
211                    Some("good"),
212                    Some("good"),
213                ],
214                DType::Utf8(Nullability::Nullable),
215            )
216            .into_array(),
217            &mut SESSION.create_execution_ctx(),
218        )
219        .unwrap();
220        test_mask_conformance(&array.into_array(), &mut SESSION.create_execution_ctx());
221    }
222
223    #[test]
224    fn test_filter_dict_array() {
225        let array = dict_encode(
226            &buffer![2, 0, 2, 0, 10].into_array(),
227            &mut SESSION.create_execution_ctx(),
228        )
229        .unwrap();
230        test_filter_conformance(&array.into_array(), &mut SESSION.create_execution_ctx());
231
232        let array = dict_encode(
233            &PrimitiveArray::from_option_iter([Some(2), None, Some(2), Some(0), Some(10)])
234                .into_array(),
235            &mut SESSION.create_execution_ctx(),
236        )
237        .unwrap();
238        test_filter_conformance(&array.into_array(), &mut SESSION.create_execution_ctx());
239
240        let array = dict_encode(
241            &VarBinArray::from_iter(
242                [
243                    Some("hello"),
244                    None,
245                    Some("hello"),
246                    Some("good"),
247                    Some("good"),
248                ],
249                DType::Utf8(Nullability::Nullable),
250            )
251            .into_array(),
252            &mut SESSION.create_execution_ctx(),
253        )
254        .unwrap();
255        test_filter_conformance(&array.into_array(), &mut SESSION.create_execution_ctx());
256    }
257
258    #[test]
259    fn test_take_dict() {
260        let array = dict_encode(
261            &buffer![1, 2].into_array(),
262            &mut SESSION.create_execution_ctx(),
263        )
264        .unwrap();
265
266        let taken = array
267            .take(PrimitiveArray::from_option_iter([None::<i32>]).into_array())
268            .unwrap();
269        assert_eq!(taken.dtype(), &DType::Primitive(I32, Nullability::Nullable));
270        assert_arrays_eq!(
271            taken,
272            PrimitiveArray::from_option_iter([None::<i32>]),
273            &mut SESSION.create_execution_ctx()
274        );
275    }
276
277    #[test]
278    fn test_take_dict_conformance() {
279        let array = dict_encode(
280            &buffer![2, 0, 2, 0, 10].into_array(),
281            &mut SESSION.create_execution_ctx(),
282        )
283        .unwrap();
284        test_take_conformance(&array.into_array(), &mut SESSION.create_execution_ctx());
285
286        let array = dict_encode(
287            &PrimitiveArray::from_option_iter([Some(2), None, Some(2), Some(0), Some(10)])
288                .into_array(),
289            &mut SESSION.create_execution_ctx(),
290        )
291        .unwrap();
292        test_take_conformance(&array.into_array(), &mut SESSION.create_execution_ctx());
293
294        let array = dict_encode(
295            &VarBinArray::from_iter(
296                [
297                    Some("hello"),
298                    None,
299                    Some("hello"),
300                    Some("good"),
301                    Some("good"),
302                ],
303                DType::Utf8(Nullability::Nullable),
304            )
305            .into_array(),
306            &mut SESSION.create_execution_ctx(),
307        )
308        .unwrap();
309        test_take_conformance(&array.into_array(), &mut SESSION.create_execution_ctx());
310    }
311}
312
313#[cfg(test)]
314mod tests {
315    use std::sync::LazyLock;
316
317    use rstest::rstest;
318    use vortex_buffer::buffer;
319    use vortex_session::VortexSession;
320
321    use crate::IntoArray;
322    use crate::VortexSessionExecute;
323    use crate::arrays::DictArray;
324    use crate::arrays::PrimitiveArray;
325    use crate::arrays::VarBinArray;
326    use crate::builders::dict::dict_encode;
327    use crate::compute::conformance::consistency::test_array_consistency;
328    use crate::dtype::DType;
329    use crate::dtype::Nullability;
330
331    static SESSION: LazyLock<VortexSession> = LazyLock::new(VortexSession::empty);
332
333    #[rstest]
334    // Primitive arrays
335    #[case::dict_i32(dict_encode(&buffer![1i32, 2, 3, 2, 1].into_array(), &mut SESSION.create_execution_ctx()).unwrap())]
336    #[case::dict_nullable_codes(DictArray::try_new(
337        buffer![0u32, 1, 2, 2, 0].into_array(),
338        PrimitiveArray::from_option_iter([Some(10), Some(20), None]).into_array(),
339    ).unwrap())]
340    #[case::dict_nullable_values(dict_encode(
341        &PrimitiveArray::from_option_iter([Some(1i32), None, Some(2), Some(1), None]).into_array()
342    , &mut SESSION.create_execution_ctx()).unwrap())]
343    #[case::dict_u64(dict_encode(&buffer![100u64, 200, 100, 300, 200].into_array(), &mut SESSION.create_execution_ctx()).unwrap())]
344    // String arrays
345    #[case::dict_str(dict_encode(
346        &VarBinArray::from_iter(
347            ["hello", "world", "hello", "test", "world"].map(Some),
348            DType::Utf8(Nullability::NonNullable),
349        ).into_array()
350    , &mut SESSION.create_execution_ctx()).unwrap())]
351    #[case::dict_nullable_str(dict_encode(
352        &VarBinArray::from_iter(
353            [Some("hello"), None, Some("world"), Some("hello"), None],
354            DType::Utf8(Nullability::Nullable),
355        ).into_array()
356    , &mut SESSION.create_execution_ctx()).unwrap())]
357    // Edge cases
358    #[case::dict_single(dict_encode(&buffer![42i32].into_array(), &mut SESSION.create_execution_ctx()).unwrap())]
359    #[case::dict_all_same(dict_encode(&buffer![5i32, 5, 5, 5, 5].into_array(), &mut SESSION.create_execution_ctx()).unwrap())]
360    #[case::dict_large(dict_encode(&PrimitiveArray::from_iter((0..1000).map(|i| i % 10)).into_array(), &mut SESSION.create_execution_ctx()).unwrap())]
361    fn test_dict_consistency(#[case] array: DictArray) {
362        test_array_consistency(&array.into_array(), &mut SESSION.create_execution_ctx());
363    }
364}