vortex-array 0.54.0

Vortex in memory columnar data format
Documentation 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97

// SPDX-License-Identifier: Apache-2.0
// SPDX-FileCopyrightText: Copyright the Vortex contributors

use arrow_buffer::BooleanBuffer;
use itertools::Itertools;
use num_traits::{CheckedAdd, Float, ToPrimitive};
use vortex_dtype::{NativePType, match_each_native_ptype};
use vortex_error::{VortexExpect, VortexResult};
use vortex_mask::AllOr;
use vortex_scalar::Scalar;

use crate::arrays::{PrimitiveArray, PrimitiveVTable};
use crate::compute::{SumKernel, SumKernelAdapter};
use crate::register_kernel;
use crate::stats::Stat;

impl SumKernel for PrimitiveVTable {
    fn sum(&self, array: &PrimitiveArray) -> VortexResult<Scalar> {
        Ok(match array.validity_mask().boolean_buffer() {
            AllOr::All => {
                // All-valid
                match_each_native_ptype!(
                    array.ptype(),
                    unsigned: |T| { sum_integer::<_, u64>(array.as_slice::<T>()).into() },
                    signed: |T| { sum_integer::<_, i64>(array.as_slice::<T>()).into() },
                    floating: |T| { Some(sum_float(array.as_slice::<T>())).into() }
                )
            }
            AllOr::None => {
                // All-invalid
                return Ok(Scalar::null(
                    Stat::Sum
                        .dtype(array.dtype())
                        .vortex_expect("Sum dtype must be defined for primitive type"),
                ));
            }
            AllOr::Some(validity_mask) => {
                // Some-valid
                match_each_native_ptype!(
                    array.ptype(),
                    unsigned: |T| {
                        sum_integer_with_validity::<_, u64>(array.as_slice::<T>(), validity_mask).into()
                    },
                    signed: |T| {
                        sum_integer_with_validity::<_, i64>(array.as_slice::<T>(), validity_mask).into()
                    },
                    floating: |T| {
                        Some(sum_float_with_validity(array.as_slice::<T>(), validity_mask)).into()
                    }
                )
            }
        })
    }
}

register_kernel!(SumKernelAdapter(PrimitiveVTable).lift());

fn sum_integer<T: NativePType + ToPrimitive, R: NativePType + CheckedAdd>(
    values: &[T],
) -> Option<R> {
    let mut sum = R::zero();
    for &x in values {
        sum = sum.checked_add(&R::from(x)?)?;
    }
    Some(sum)
}

fn sum_integer_with_validity<T: NativePType + ToPrimitive, R: NativePType + CheckedAdd>(
    values: &[T],
    validity: &BooleanBuffer,
) -> Option<R> {
    let mut sum = R::zero();
    for (&x, valid) in values.iter().zip_eq(validity.iter()) {
        if valid {
            sum = sum.checked_add(&R::from(x)?)?;
        }
    }
    Some(sum)
}

fn sum_float<T: NativePType + Float>(values: &[T]) -> f64 {
    let mut sum = 0.0;
    for &x in values {
        sum += x.to_f64().vortex_expect("Failed to cast value to f64");
    }
    sum
}

fn sum_float_with_validity<T: NativePType + Float>(array: &[T], validity: &BooleanBuffer) -> f64 {
    let mut sum = 0.0;
    for (&x, valid) in array.iter().zip_eq(validity.iter()) {
        if valid {
            sum += x.to_f64().vortex_expect("Failed to cast value to f64");
        }
    }
    sum
}