vortex-array 0.68.0

Vortex in memory columnar data format
Documentation 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138

// SPDX-License-Identifier: Apache-2.0
// SPDX-FileCopyrightText: Copyright the Vortex contributors

use vortex_error::VortexResult;
use vortex_error::vortex_bail;
use vortex_error::vortex_ensure;
use vortex_error::vortex_ensure_eq;

use crate::dtype::DType;
use crate::dtype::PType;
use crate::scalar::PValue;
use crate::scalar::Scalar;
use crate::scalar::ScalarValue;

impl Scalar {
    /// Validate that the given [`ScalarValue`] is compatible with the given [`DType`].
    pub fn validate(dtype: &DType, value: Option<&ScalarValue>) -> VortexResult<()> {
        let Some(value) = value else {
            vortex_ensure!(
                dtype.is_nullable(),
                "non-nullable dtype {dtype} cannot hold a null value",
            );
            return Ok(());
        };

        // From here onwards, we know that the value is not null.
        match dtype {
            DType::Null => {
                vortex_bail!("null dtype cannot hold a non-null value {value}");
            }
            DType::Bool(_) => {
                vortex_ensure!(
                    matches!(value, ScalarValue::Bool(_)),
                    "bool dtype expected Bool value, got {value}",
                );
            }
            DType::Primitive(ptype, _) => {
                let ScalarValue::Primitive(pvalue) = value else {
                    vortex_bail!("primitive dtype {ptype} expected Primitive value, got {value}",);
                };

                // Note that this is a backwards compatibility check for poor design in the
                // previous implementation. `f16` `ScalarValue`s used to be serialized as
                // `pb::ScalarValue::Uint64Value(v.to_bits() as u64)`, so we need to ensure
                // that we can still represent them as such.
                let f16_backcompat_still_works =
                    matches!(ptype, &PType::F16) && matches!(pvalue, PValue::U64(_));

                vortex_ensure!(
                    f16_backcompat_still_works || pvalue.ptype() == *ptype,
                    "primitive dtype {ptype} is not compatible with value {pvalue}",
                );
            }
            DType::Decimal(dec_dtype, _) => {
                let ScalarValue::Decimal(dvalue) = value else {
                    vortex_bail!("decimal dtype expected Decimal value, got {value}");
                };

                vortex_ensure!(
                    dvalue.fits_in_precision(*dec_dtype),
                    "decimal value {dvalue} does not fit in precision of {dec_dtype}",
                );
            }
            DType::Utf8(_) => {
                vortex_ensure!(
                    matches!(value, ScalarValue::Utf8(_)),
                    "utf8 dtype expected Utf8 value, got {value}",
                );
            }
            DType::Binary(_) => {
                vortex_ensure!(
                    matches!(value, ScalarValue::Binary(_)),
                    "binary dtype expected Binary value, got {value}",
                );
            }
            DType::List(elem_dtype, _) => {
                let ScalarValue::List(elements) = value else {
                    vortex_bail!("list dtype expected List value, got {value}");
                };

                for (i, element) in elements.iter().enumerate() {
                    Self::validate(elem_dtype.as_ref(), element.as_ref())
                        .map_err(|e| vortex_error::vortex_err!("list element at index {i}: {e}"))?;
                }
            }
            DType::FixedSizeList(elem_dtype, size, _) => {
                let ScalarValue::List(elements) = value else {
                    vortex_bail!("fixed-size list dtype expected List value, got {value}",);
                };

                let len = elements.len();
                vortex_ensure_eq!(
                    len,
                    *size as usize,
                    "fixed-size list dtype expected {size} elements, got {len}",
                );

                for (i, element) in elements.iter().enumerate() {
                    Self::validate(elem_dtype.as_ref(), element.as_ref()).map_err(|e| {
                        vortex_error::vortex_err!("fixed-size list element at index {i}: {e}",)
                    })?;
                }
            }
            DType::Struct(fields, _) => {
                let ScalarValue::List(values) = value else {
                    vortex_bail!("struct dtype expected List value, got {value}");
                };

                let nfields = fields.nfields();
                let nvalues = values.len();
                vortex_ensure_eq!(
                    nvalues,
                    nfields,
                    "struct dtype expected {nfields} fields, got {nvalues}",
                );

                for (field, field_value) in fields.fields().zip(values.iter()) {
                    Self::validate(&field, field_value.as_ref())?;
                }
            }
            DType::Extension(ext_dtype) => ext_dtype.validate_storage_value(value)?,
            DType::Variant(_) => {
                let ScalarValue::Variant(inner) = value else {
                    vortex_bail!("variant dtype expected Variant value, got {value}");
                };

                Self::validate(inner.dtype(), inner.value())?;
                vortex_ensure!(
                    !inner.is_null() || matches!(inner.dtype(), DType::Null),
                    "variant nulls must use a nested null scalar, got {}",
                    inner.dtype(),
                );
            }
        }

        Ok(())
    }
}