Skip to main content

vortex_array/compute/conformance/
cast.rs

1// SPDX-License-Identifier: Apache-2.0
2// SPDX-FileCopyrightText: Copyright the Vortex contributors
3
4use vortex_error::VortexExpect;
5use vortex_error::VortexResult;
6use vortex_error::vortex_panic;
7
8use crate::ArrayRef;
9use crate::ExecutionCtx;
10use crate::IntoArray;
11use crate::RecursiveCanonical;
12use crate::aggregate_fn::NumericalAggregateOpts;
13use crate::aggregate_fn::fns::min_max::MinMaxResult;
14use crate::aggregate_fn::fns::min_max::min_max;
15use crate::builtins::ArrayBuiltins;
16use crate::dtype::DType;
17use crate::dtype::Nullability;
18use crate::dtype::PType;
19use crate::scalar::Scalar;
20
21/// Cast and force execution via `to_canonical`, returning the canonical array.
22fn cast_and_execute(
23    array: &ArrayRef,
24    dtype: DType,
25    ctx: &mut ExecutionCtx,
26) -> VortexResult<ArrayRef> {
27    Ok(array
28        .cast(dtype)?
29        .execute::<RecursiveCanonical>(ctx)?
30        .0
31        .into_array())
32}
33
34/// Test conformance of the cast compute function for an array.
35///
36/// This function tests various casting scenarios including:
37/// - Casting between numeric types (widening and narrowing)
38/// - Casting between signed and unsigned types
39/// - Casting between integral and floating-point types
40/// - Casting with nullability changes
41/// - Casting between string types (Utf8/Binary)
42/// - Edge cases like overflow behavior
43///
44/// An encoding that supports nullable dtypes **must** also pass a partially valid array, which
45/// checks the nullability contract of [`CastReduce`].
46///
47/// [`CastReduce`]: crate::scalar_fn::fns::cast::CastReduce
48pub fn test_cast_conformance(array: &ArrayRef, ctx: &mut ExecutionCtx) {
49    let dtype = array.dtype();
50
51    // Always test identity cast and nullability changes
52    test_cast_identity(array, ctx);
53
54    test_cast_to_non_nullable(array, ctx);
55    test_cast_to_nullable(array, ctx);
56
57    // Test based on the specific DType
58    match dtype {
59        DType::Null => test_cast_from_null(array, ctx),
60        DType::Primitive(ptype, ..) => match ptype {
61            PType::U8
62            | PType::U16
63            | PType::U32
64            | PType::U64
65            | PType::I8
66            | PType::I16
67            | PType::I32
68            | PType::I64 => test_cast_to_integral_types(array, ctx),
69            PType::F16 | PType::F32 | PType::F64 => test_cast_from_floating_point_types(array, ctx),
70        },
71        _ => {}
72    }
73}
74
75fn test_cast_identity(array: &ArrayRef, ctx: &mut ExecutionCtx) {
76    // Casting to the same type should be a no-op
77    let result = cast_and_execute(&array.clone(), array.dtype().clone(), ctx)
78        .vortex_expect("cast should succeed in conformance test");
79    assert_eq!(result.len(), array.len());
80    assert_eq!(result.dtype(), array.dtype());
81
82    // Verify values are unchanged
83    for i in 0..array.len().min(10) {
84        assert_eq!(
85            array
86                .execute_scalar(i, ctx)
87                .vortex_expect("scalar_at should succeed in conformance test"),
88            result
89                .execute_scalar(i, ctx)
90                .vortex_expect("scalar_at should succeed in conformance test")
91        );
92    }
93}
94
95fn test_cast_from_null(array: &ArrayRef, ctx: &mut ExecutionCtx) {
96    // Null can also be cast to any nullable type
97    let nullable_types = vec![
98        DType::Bool(Nullability::Nullable),
99        DType::Primitive(PType::I32, Nullability::Nullable),
100        DType::Primitive(PType::F64, Nullability::Nullable),
101        DType::Utf8(Nullability::Nullable),
102        DType::Binary(Nullability::Nullable),
103    ];
104
105    for dtype in nullable_types {
106        let result = cast_and_execute(&array.clone(), dtype.clone(), ctx)
107            .vortex_expect("cast should succeed in conformance test");
108        assert_eq!(result.len(), array.len());
109        assert_eq!(result.dtype(), &dtype);
110
111        // Verify all values are null
112        for i in 0..array.len().min(10) {
113            assert!(
114                result
115                    .execute_scalar(i, ctx)
116                    .vortex_expect("scalar_at should succeed in conformance test")
117                    .is_null()
118            );
119        }
120    }
121
122    // Casting to non-nullable types should fail
123    let non_nullable_types = vec![
124        DType::Bool(Nullability::NonNullable),
125        DType::Primitive(PType::I32, Nullability::NonNullable),
126    ];
127
128    for dtype in non_nullable_types {
129        assert!(cast_and_execute(&array.clone(), dtype.clone(), ctx).is_err());
130    }
131}
132
133fn test_cast_to_non_nullable(array: &ArrayRef, ctx: &mut ExecutionCtx) {
134    // DType::Null has no non-nullable form
135    if &DType::Null == array.dtype() || !array.dtype().is_nullable() {
136        return;
137    }
138
139    if array
140        .invalid_count(ctx)
141        .vortex_expect("invalid_count should succeed in conformance test")
142        == 0
143    {
144        let non_nullable = cast_and_execute(&array.clone(), array.dtype().as_nonnullable(), ctx)
145            .vortex_expect("arrays without nulls can cast to non-nullable");
146        assert_eq!(non_nullable.dtype(), &array.dtype().as_nonnullable());
147        assert_eq!(non_nullable.len(), array.len());
148
149        for i in 0..array.len().min(10) {
150            assert_eq!(
151                array
152                    .execute_scalar(i, ctx)
153                    .vortex_expect("scalar_at should succeed in conformance test"),
154                non_nullable
155                    .execute_scalar(i, ctx)
156                    .vortex_expect("scalar_at should succeed in conformance test")
157            );
158        }
159
160        let back_to_nullable = cast_and_execute(&non_nullable, array.dtype().clone(), ctx)
161            .vortex_expect("non-nullable arrays can cast to nullable");
162        assert_eq!(back_to_nullable.dtype(), array.dtype());
163        assert_eq!(back_to_nullable.len(), array.len());
164
165        for i in 0..array.len().min(10) {
166            assert_eq!(
167                array
168                    .execute_scalar(i, ctx)
169                    .vortex_expect("scalar_at should succeed in conformance test"),
170                back_to_nullable
171                    .execute_scalar(i, ctx)
172                    .vortex_expect("scalar_at should succeed in conformance test")
173            );
174        }
175    } else {
176        cast_and_execute(&array.clone(), array.dtype().as_nonnullable(), ctx)
177            .err()
178            .unwrap_or_else(|| {
179                vortex_panic!(
180                    "arrays with nulls should error when casting to non-nullable {}",
181                    array,
182                )
183            });
184    }
185}
186
187fn test_cast_to_nullable(array: &ArrayRef, ctx: &mut ExecutionCtx) {
188    if array.dtype().is_nullable() {
189        return;
190    }
191
192    let nullable = cast_and_execute(&array.clone(), array.dtype().as_nullable(), ctx)
193        .vortex_expect("arrays without nulls can cast to nullable");
194    assert_eq!(nullable.dtype(), &array.dtype().as_nullable());
195    assert_eq!(nullable.len(), array.len());
196
197    for i in 0..array.len().min(10) {
198        assert_eq!(
199            array
200                .execute_scalar(i, ctx)
201                .vortex_expect("scalar_at should succeed in conformance test"),
202            nullable
203                .execute_scalar(i, ctx)
204                .vortex_expect("scalar_at should succeed in conformance test")
205        );
206    }
207
208    let back = cast_and_execute(&nullable, array.dtype().clone(), ctx)
209        .vortex_expect("casting to nullable and back should be a no-op");
210    assert_eq!(back.dtype(), array.dtype());
211    assert_eq!(back.len(), array.len());
212
213    for i in 0..array.len().min(10) {
214        assert_eq!(
215            array
216                .execute_scalar(i, ctx)
217                .vortex_expect("scalar_at should succeed in conformance test"),
218            back.execute_scalar(i, ctx)
219                .vortex_expect("scalar_at should succeed in conformance test")
220        );
221    }
222}
223
224fn test_cast_from_floating_point_types(array: &ArrayRef, ctx: &mut ExecutionCtx) {
225    let ptype = array.dtype().as_ptype();
226    test_cast_to_primitive(array, PType::I8, false, ctx);
227    test_cast_to_primitive(array, PType::U8, false, ctx);
228    test_cast_to_primitive(array, PType::I16, false, ctx);
229    test_cast_to_primitive(array, PType::U16, false, ctx);
230    test_cast_to_primitive(array, PType::I32, false, ctx);
231    test_cast_to_primitive(array, PType::U32, false, ctx);
232    test_cast_to_primitive(array, PType::I64, false, ctx);
233    test_cast_to_primitive(array, PType::U64, false, ctx);
234    test_cast_to_primitive(array, PType::F16, matches!(ptype, PType::F16), ctx);
235    test_cast_to_primitive(
236        array,
237        PType::F32,
238        matches!(ptype, PType::F16 | PType::F32),
239        ctx,
240    );
241    test_cast_to_primitive(array, PType::F64, true, ctx);
242}
243
244fn test_cast_to_integral_types(array: &ArrayRef, ctx: &mut ExecutionCtx) {
245    test_cast_to_primitive(array, PType::I8, true, ctx);
246    test_cast_to_primitive(array, PType::U8, true, ctx);
247    test_cast_to_primitive(array, PType::I16, true, ctx);
248    test_cast_to_primitive(array, PType::U16, true, ctx);
249    test_cast_to_primitive(array, PType::I32, true, ctx);
250    test_cast_to_primitive(array, PType::U32, true, ctx);
251    test_cast_to_primitive(array, PType::I64, true, ctx);
252    test_cast_to_primitive(array, PType::U64, true, ctx);
253}
254
255/// Does this scalar fit in this type?
256fn fits(value: &Scalar, ptype: PType) -> bool {
257    let dtype = DType::Primitive(ptype, value.dtype().nullability());
258    value.cast(&dtype).is_ok()
259}
260
261fn test_cast_to_primitive(
262    array: &ArrayRef,
263    target_ptype: PType,
264    test_round_trip: bool,
265    ctx: &mut ExecutionCtx,
266) {
267    let maybe_min_max = min_max(array, ctx, NumericalAggregateOpts::default())
268        .vortex_expect("cast should succeed in conformance test");
269
270    if let Some(MinMaxResult { min, max }) = maybe_min_max
271        && (!fits(&min, target_ptype) || !fits(&max, target_ptype))
272    {
273        cast_and_execute(
274            &array.clone(),
275            DType::Primitive(target_ptype, array.dtype().nullability()),
276            ctx,
277        )
278        .err()
279        .unwrap_or_else(|| {
280            vortex_panic!(
281                "Cast must fail because some values are out of bounds. {} {:?} {:?} {} {}",
282                target_ptype,
283                min,
284                max,
285                array,
286                array.display_values(ctx),
287            )
288        });
289        return;
290    }
291
292    // Otherwise, all values must fit.
293    let target_dtype = DType::Primitive(target_ptype, array.dtype().nullability());
294    let casted = cast_and_execute(array, target_dtype.clone(), ctx).unwrap_or_else(|e| {
295        vortex_panic!(
296            e,
297            "Cast must succeed because all values are within bounds. {} {}",
298            target_ptype,
299            array.display_values(ctx),
300        )
301    });
302    assert_eq!(casted.dtype(), &target_dtype);
303    assert_eq!(casted.len(), array.len());
304    assert_eq!(
305        array
306            .validity()
307            .vortex_expect("validity_mask should succeed in conformance test")
308            .execute_mask(array.len(), ctx)
309            .vortex_expect("Failed to compute validity mask"),
310        casted
311            .validity()
312            .vortex_expect("validity_mask should succeed in conformance test")
313            .execute_mask(casted.len(), ctx)
314            .vortex_expect("Failed to compute validity mask")
315    );
316    for i in 0..array.len().min(10) {
317        let original = array
318            .execute_scalar(i, ctx)
319            .vortex_expect("scalar_at should succeed in conformance test");
320        let casted = casted
321            .execute_scalar(i, ctx)
322            .vortex_expect("scalar_at should succeed in conformance test");
323        assert_eq!(
324            original
325                .cast(&target_dtype)
326                .vortex_expect("cast should succeed in conformance test"),
327            casted,
328            "{i} {original} {casted}"
329        );
330        if test_round_trip {
331            assert_eq!(
332                original,
333                casted
334                    .cast(original.dtype())
335                    .vortex_expect("cast should succeed in conformance test"),
336                "{i} {original} {casted}"
337            );
338        }
339    }
340}
341
342#[cfg(test)]
343mod tests {
344    use std::sync::LazyLock;
345
346    use vortex_buffer::buffer;
347    use vortex_session::VortexSession;
348
349    use super::*;
350    use crate::IntoArray;
351    use crate::VortexSessionExecute;
352    use crate::arrays::BoolArray;
353    use crate::arrays::ListArray;
354    use crate::arrays::NullArray;
355    use crate::arrays::PrimitiveArray;
356    use crate::arrays::StructArray;
357    use crate::arrays::VarBinArray;
358    use crate::dtype::DType;
359    use crate::dtype::FieldNames;
360    use crate::dtype::Nullability;
361
362    static SESSION: LazyLock<VortexSession> = LazyLock::new(VortexSession::empty);
363
364    #[test]
365    fn test_cast_conformance_u32() {
366        let array = buffer![0u32, 100, 200, 65535, 1000000].into_array();
367        test_cast_conformance(&array, &mut SESSION.create_execution_ctx());
368    }
369
370    #[test]
371    fn test_cast_conformance_i32() {
372        let array = buffer![-100i32, -1, 0, 1, 100].into_array();
373        test_cast_conformance(&array, &mut SESSION.create_execution_ctx());
374    }
375
376    #[test]
377    fn test_cast_conformance_f32() {
378        let array = buffer![0.0f32, 1.5, -2.5, 100.0, 1e6].into_array();
379        test_cast_conformance(&array, &mut SESSION.create_execution_ctx());
380    }
381
382    #[test]
383    fn test_cast_conformance_nullable() {
384        let array = PrimitiveArray::from_option_iter([Some(1u8), None, Some(255), Some(0), None]);
385        test_cast_conformance(&array.into_array(), &mut SESSION.create_execution_ctx());
386    }
387
388    #[test]
389    fn test_cast_conformance_bool() {
390        let array = BoolArray::from_iter(vec![true, false, true, false]);
391        test_cast_conformance(&array.into_array(), &mut SESSION.create_execution_ctx());
392    }
393
394    #[test]
395    fn test_cast_conformance_null() {
396        let array = NullArray::new(5);
397        test_cast_conformance(&array.into_array(), &mut SESSION.create_execution_ctx());
398    }
399
400    #[test]
401    fn test_cast_conformance_utf8() {
402        let array = VarBinArray::from_iter(
403            vec![Some("hello"), None, Some("world")],
404            DType::Utf8(Nullability::Nullable),
405        );
406        test_cast_conformance(&array.into_array(), &mut SESSION.create_execution_ctx());
407    }
408
409    #[test]
410    fn test_cast_conformance_binary() {
411        let array = VarBinArray::from_iter(
412            vec![Some(b"data".as_slice()), None, Some(b"bytes".as_slice())],
413            DType::Binary(Nullability::Nullable),
414        );
415        test_cast_conformance(&array.into_array(), &mut SESSION.create_execution_ctx());
416    }
417
418    #[test]
419    fn test_cast_conformance_struct() {
420        let names = FieldNames::from(["a", "b"]);
421
422        let a = buffer![1i32, 2, 3].into_array();
423        let b = VarBinArray::from_iter(
424            vec![Some("x"), None, Some("z")],
425            DType::Utf8(Nullability::Nullable),
426        )
427        .into_array();
428
429        let array =
430            StructArray::try_new(names, vec![a, b], 3, crate::validity::Validity::NonNullable)
431                .unwrap();
432        test_cast_conformance(&array.into_array(), &mut SESSION.create_execution_ctx());
433    }
434
435    #[test]
436    fn test_cast_conformance_list() {
437        let mut ctx = SESSION.create_execution_ctx();
438        let data = buffer![1i32, 2, 3, 4, 5, 6].into_array();
439        let offsets = buffer![0i64, 2, 2, 5, 6].into_array();
440
441        let array = ListArray::try_new(
442            data,
443            offsets,
444            crate::validity::Validity::NonNullable,
445            &mut ctx,
446        )
447        .unwrap();
448        test_cast_conformance(&array.into_array(), &mut SESSION.create_execution_ctx());
449    }
450}