vortex-array 0.54.0

Vortex in memory columnar data format
Documentation 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110

// SPDX-License-Identifier: Apache-2.0
// SPDX-FileCopyrightText: Copyright the Vortex contributors

use std::ops::Not;

use vortex_buffer::BufferMut;
use vortex_dtype::{Nullability, match_each_native_ptype};
use vortex_error::{VortexExpect, VortexResult};
use vortex_scalar::Scalar;

use crate::arrays::primitive::PrimitiveArray;
use crate::arrays::{ConstantArray, PrimitiveVTable};
use crate::compute::{FillNullKernel, FillNullKernelAdapter};
use crate::validity::Validity;
use crate::vtable::ValidityHelper;
use crate::{ArrayRef, IntoArray, ToCanonical, register_kernel};

impl FillNullKernel for PrimitiveVTable {
    fn fill_null(&self, array: &PrimitiveArray, fill_value: &Scalar) -> VortexResult<ArrayRef> {
        let result_validity = match fill_value.dtype().nullability() {
            Nullability::NonNullable => Validity::NonNullable,
            Nullability::Nullable => Validity::AllValid,
        };

        Ok(match array.validity() {
            Validity::NonNullable | Validity::AllValid => {
                match_each_native_ptype!(array.ptype(), |T| {
                    PrimitiveArray::new::<T>(array.buffer(), result_validity).into_array()
                })
            }
            Validity::AllInvalid => {
                ConstantArray::new(fill_value.clone(), array.len()).into_array()
            }
            Validity::Array(is_valid) => {
                // TODO(danking): when we take PrimitiveArray by value, we should mutate in-place
                let is_invalid = is_valid.to_bool().boolean_buffer().not();
                match_each_native_ptype!(array.ptype(), |T| {
                    let mut buffer = BufferMut::copy_from(array.as_slice::<T>());
                    let fill_value = fill_value
                        .as_primitive()
                        .typed_value::<T>()
                        .vortex_expect("top-level fill_null ensure non-null fill value");
                    for invalid_index in is_invalid.set_indices() {
                        buffer[invalid_index] = fill_value;
                    }
                    PrimitiveArray::new(buffer.freeze(), result_validity).into_array()
                })
            }
        })
    }
}

register_kernel!(FillNullKernelAdapter(PrimitiveVTable).lift());

#[cfg(test)]
mod test {
    use vortex_buffer::buffer;
    use vortex_scalar::Scalar;

    use crate::IntoArray;
    use crate::arrays::BoolArray;
    use crate::arrays::primitive::PrimitiveArray;
    use crate::canonical::ToCanonical;
    use crate::compute::fill_null;
    use crate::validity::Validity;

    #[test]
    fn fill_null_leading_none() {
        let arr = PrimitiveArray::from_option_iter([None, Some(8u8), None, Some(10), None]);
        let p = fill_null(arr.as_ref(), &Scalar::from(42u8))
            .unwrap()
            .to_primitive();
        assert_eq!(p.as_slice::<u8>(), vec![42, 8, 42, 10, 42]);
        assert!(p.validity_mask().all_true());
    }

    #[test]
    fn fill_null_all_none() {
        let arr = PrimitiveArray::from_option_iter([Option::<u8>::None, None, None, None, None]);

        let p = fill_null(arr.as_ref(), &Scalar::from(255u8))
            .unwrap()
            .to_primitive();
        assert_eq!(p.as_slice::<u8>(), vec![255, 255, 255, 255, 255]);
        assert!(p.validity_mask().all_true());
    }

    #[test]
    fn fill_null_nullable_non_null() {
        let arr = PrimitiveArray::new(
            buffer![8u8, 10, 12, 14, 16],
            Validity::Array(BoolArray::from_iter([true, true, true, true, true]).into_array()),
        );
        let p = fill_null(arr.as_ref(), &Scalar::from(255u8))
            .unwrap()
            .to_primitive();
        assert_eq!(p.as_slice::<u8>(), vec![8, 10, 12, 14, 16]);
        assert!(p.validity_mask().all_true());
    }

    #[test]
    fn fill_null_non_nullable() {
        let arr = buffer![8u8, 10, 12, 14, 16].into_array();
        let p = fill_null(&arr, &Scalar::from(255u8))
            .unwrap()
            .to_primitive();
        assert_eq!(p.as_slice::<u8>(), vec![8u8, 10, 12, 14, 16]);
        assert!(p.validity_mask().all_true());
    }
}