vortex-array 0.54.0

Vortex in memory columnar data format
Documentation 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98

// SPDX-License-Identifier: Apache-2.0
// SPDX-FileCopyrightText: Copyright the Vortex contributors

use std::fmt::{Debug, Display, Formatter};

use vortex_dtype::half::f16;
use vortex_dtype::{DType, NativePType, PType};
use vortex_error::vortex_panic;

use crate::arrays::binary_view::BinaryView;

/// Defines the "vector type", a physical type describing the data that's held in the vector.
///
/// See the specific vector view types like primitive views for more details.
#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash)]
pub enum VType {
    Bool,
    Primitive(PType),
    Binary,
}

impl Display for VType {
    fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result {
        match self {
            VType::Bool => write!(f, "bool"),
            VType::Primitive(ptype) => write!(f, "{}", ptype),
            VType::Binary => write!(f, "binary"),
        }
    }
}

impl VType {
    pub fn of<T: Element>() -> Self {
        T::vtype()
    }

    pub fn byte_width(&self) -> usize {
        match self {
            VType::Bool => 1,
            VType::Primitive(ptype) => ptype.byte_width(),
            VType::Binary => size_of::<BinaryView>(),
        }
    }
}

/// A trait to identify canonical vector types.
pub trait Element: 'static + Copy + Debug + Send {
    fn vtype() -> VType;
}

/// NOTE: for now, we have chosen to store boolean values as byte-sized booleans instead
///  of packed into a bit mask, this is typically more efficient for SIMD compute operations.
///  For masks, we still use bit-packed booleans.
impl Element for bool {
    fn vtype() -> VType {
        VType::Bool
    }
}

macro_rules! canonical_ptype {
    ($T:ty) => {
        impl Element for $T {
            fn vtype() -> VType {
                VType::Primitive(<$T as NativePType>::PTYPE)
            }
        }
    };
}

canonical_ptype!(u8);
canonical_ptype!(u16);
canonical_ptype!(u32);
canonical_ptype!(u64);
canonical_ptype!(i8);
canonical_ptype!(i16);
canonical_ptype!(i32);
canonical_ptype!(i64);
canonical_ptype!(f16);
canonical_ptype!(f32);
canonical_ptype!(f64);

impl Element for BinaryView {
    fn vtype() -> VType {
        VType::Binary
    }
}

impl From<&DType> for VType {
    fn from(value: &DType) -> Self {
        match value {
            DType::Bool(_) => VType::Bool,
            DType::Primitive(ptype, _) => VType::Primitive(*ptype),
            DType::Utf8(_) => VType::Binary,
            DType::Binary(_) => VType::Binary,
            _ => vortex_panic!("Unsupported dtype for VType: {}", value),
        }
    }
}