vortex_array/scalar/
validate.rs

1// SPDX-License-Identifier: Apache-2.0
2// SPDX-FileCopyrightText: Copyright the Vortex contributors
3
4use vortex_error::VortexError;
5use vortex_error::VortexExpect;
6use vortex_error::VortexResult;
7use vortex_error::vortex_bail;
8use vortex_error::vortex_ensure;
9use vortex_error::vortex_ensure_eq;
10use vortex_error::vortex_err;
11
12use crate::dtype::DType;
13use crate::dtype::PType;
14use crate::scalar::PValue;
15use crate::scalar::Scalar;
16use crate::scalar::ScalarValue;
17
18impl Scalar {
19    /// Validate that the given [`ScalarValue`] is compatible with the given [`DType`].
20    #[inline]
21    pub fn validate(dtype: &DType, value: Option<&ScalarValue>) -> VortexResult<()> {
22        if Self::is_compatible(dtype, value) {
23            return Ok(());
24        }
25
26        Err(Self::incompatible_error(dtype, value))
27    }
28
29    /// Returns `true` if the given [`ScalarValue`] is compatible with the given [`DType`].
30    ///
31    /// This is the fast path of [`Self::validate`]. It must accept exactly the values that
32    /// [`Self::validate_detailed`] accepts, but it builds no error messages.
33    ///
34    /// Null values and leaf types are checked inline, so that callers with a known dtype fold the
35    /// check away. Other types go to [`Self::is_compatible_nested`].
36    #[inline]
37    pub(super) fn is_compatible(dtype: &DType, value: Option<&ScalarValue>) -> bool {
38        let Some(value) = value else {
39            return dtype.is_nullable();
40        };
41
42        match (dtype, value) {
43            (DType::Bool(_), ScalarValue::Bool(_))
44            | (DType::Utf8(_), ScalarValue::Utf8(_))
45            | (DType::Binary(_), ScalarValue::Binary(_)) => true,
46            (DType::Primitive(ptype, _), ScalarValue::Primitive(pvalue)) => {
47                // See `validate_detailed` for the `f16` backwards compatibility case.
48                pvalue.ptype() == *ptype
49                    || (matches!(ptype, PType::F16) && matches!(pvalue, PValue::U64(_)))
50            }
51            _ => Self::is_compatible_nested(dtype, value),
52        }
53    }
54
55    /// The out-of-line part of [`Self::is_compatible`] for non-null values of all other types.
56    fn is_compatible_nested(dtype: &DType, value: &ScalarValue) -> bool {
57        match (dtype, value) {
58            (DType::Decimal(dec_dtype, _), ScalarValue::Decimal(dvalue)) => {
59                dvalue.fits_in_precision(*dec_dtype)
60            }
61            (DType::List(elem_dtype, _), ScalarValue::Tuple(elements)) => elements
62                .iter()
63                .all(|element| Self::is_compatible(elem_dtype, element.as_ref())),
64            (DType::FixedSizeList(elem_dtype, size, _), ScalarValue::Tuple(elements)) => {
65                elements.len() == *size as usize
66                    && elements
67                        .iter()
68                        .all(|element| Self::is_compatible(elem_dtype, element.as_ref()))
69            }
70            (DType::Map(map, _), ScalarValue::Tuple(entries)) => {
71                let key_dtype = map.key_dtype();
72                let value_dtype = map.value_dtype();
73
74                entries.iter().all(|entry| match entry {
75                    Some(ScalarValue::Tuple(values)) => {
76                        values.len() == 2
77                            && Self::is_compatible(&key_dtype, values[0].as_ref())
78                            && Self::is_compatible(&value_dtype, values[1].as_ref())
79                    }
80                    _ => false,
81                })
82            }
83            (DType::Struct(fields, _), ScalarValue::Tuple(values)) => {
84                values.len() == fields.nfields()
85                    && fields
86                        .field_dtypes()
87                        .zip(values.iter())
88                        .all(|(field, value)| Self::is_compatible(field, value.as_ref()))
89            }
90            (DType::Union(variants, _), ScalarValue::Union(union_value)) => variants
91                .tag_to_child_index(union_value.type_id())
92                .and_then(|child_index| variants.variant_by_index(child_index))
93                .is_some_and(|child_dtype| {
94                    Self::is_compatible(&child_dtype, union_value.child_value())
95                }),
96            (DType::Variant(_), ScalarValue::Variant(inner)) => {
97                Self::is_compatible(inner.dtype(), inner.value())
98                    && (!inner.is_null() || matches!(inner.dtype(), DType::Null))
99            }
100            (DType::Extension(ext_dtype), value) => ext_dtype.validate_storage_value(value).is_ok(),
101            _ => false,
102        }
103    }
104
105    /// Builds the error for a [`ScalarValue`] that is not compatible with the [`DType`].
106    ///
107    /// Kept out of line so that the formatting code does not count against the hot callers when
108    /// the inliner sizes them.
109    #[cold]
110    #[inline(never)]
111    pub(super) fn incompatible_error(dtype: &DType, value: Option<&ScalarValue>) -> VortexError {
112        match Self::validate_detailed(dtype, value) {
113            Err(err) => err,
114            Ok(()) => vortex_err!(
115                "scalar validation fast path rejected value {value:?} for dtype {dtype}, but the \
116                 detailed check accepted it"
117            ),
118        }
119    }
120
121    /// Validates the [`ScalarValue`] against the [`DType`], with a detailed error message.
122    fn validate_detailed(dtype: &DType, value: Option<&ScalarValue>) -> VortexResult<()> {
123        let Some(value) = value else {
124            vortex_ensure!(
125                dtype.is_nullable(),
126                "non-nullable dtype {dtype} cannot hold a null value",
127            );
128            return Ok(());
129        };
130
131        // From here onwards, we know that the value is not null.
132        match dtype {
133            DType::Null => {
134                vortex_bail!("null dtype cannot hold a non-null value {value}");
135            }
136            DType::Bool(_) => {
137                vortex_ensure!(
138                    matches!(value, ScalarValue::Bool(_)),
139                    "bool dtype expected Bool value, got {value}",
140                );
141            }
142            DType::Primitive(ptype, _) => {
143                let ScalarValue::Primitive(pvalue) = value else {
144                    vortex_bail!("primitive dtype {ptype} expected Primitive value, got {value}",);
145                };
146
147                // Note that this is a backwards compatibility check for poor design in the
148                // previous implementation. `f16` `ScalarValue`s used to be serialized as
149                // `pb::ScalarValue::Uint64Value(v.to_bits() as u64)`, so we need to ensure
150                // that we can still represent them as such.
151                let f16_backcompat_still_works =
152                    matches!(ptype, &PType::F16) && matches!(pvalue, PValue::U64(_));
153
154                vortex_ensure!(
155                    f16_backcompat_still_works || pvalue.ptype() == *ptype,
156                    "primitive dtype {ptype} is not compatible with value {pvalue}",
157                );
158            }
159            DType::Decimal(dec_dtype, _) => {
160                let ScalarValue::Decimal(dvalue) = value else {
161                    vortex_bail!("decimal dtype expected Decimal value, got {value}");
162                };
163
164                vortex_ensure!(
165                    dvalue.fits_in_precision(*dec_dtype),
166                    "decimal value {dvalue} does not fit in precision of {dec_dtype}",
167                );
168            }
169            DType::Utf8(_) => {
170                vortex_ensure!(
171                    matches!(value, ScalarValue::Utf8(_)),
172                    "utf8 dtype expected Utf8 value, got {value}",
173                );
174            }
175            DType::Binary(_) => {
176                vortex_ensure!(
177                    matches!(value, ScalarValue::Binary(_)),
178                    "binary dtype expected Binary value, got {value}",
179                );
180            }
181            DType::List(elem_dtype, _) => {
182                let ScalarValue::Tuple(elements) = value else {
183                    vortex_bail!("list dtype expected Tuple value, got {value}");
184                };
185
186                for (i, element) in elements.iter().enumerate() {
187                    Self::validate_detailed(elem_dtype.as_ref(), element.as_ref())
188                        .map_err(|e| vortex_error::vortex_err!("list element at index {i}: {e}"))?;
189                }
190            }
191            DType::FixedSizeList(elem_dtype, size, _) => {
192                let ScalarValue::Tuple(elements) = value else {
193                    vortex_bail!("fixed-size list dtype expected Tuple value, got {value}",);
194                };
195
196                let len = elements.len();
197                vortex_ensure_eq!(
198                    len,
199                    *size as usize,
200                    "fixed-size list scalar has the wrong number of elements",
201                );
202
203                for (i, element) in elements.iter().enumerate() {
204                    Self::validate_detailed(elem_dtype.as_ref(), element.as_ref()).map_err(
205                        |e| vortex_error::vortex_err!("fixed-size list element at index {i}: {e}",),
206                    )?;
207                }
208            }
209            DType::Map(map, _) => {
210                let ScalarValue::Tuple(entries) = value else {
211                    vortex_bail!("map dtype expected Tuple value, got {value}");
212                };
213                let key_dtype = map.key_dtype();
214                let value_dtype = map.value_dtype();
215
216                for (index, entry) in entries.iter().enumerate() {
217                    let entry = entry.as_ref().ok_or_else(|| {
218                        vortex_error::vortex_err!("map entry at index {index} cannot be null")
219                    })?;
220                    let ScalarValue::Tuple(values) = entry else {
221                        vortex_bail!(
222                            "map entry at index {index} expected Tuple value, got {entry}"
223                        );
224                    };
225                    vortex_ensure_eq!(
226                        values.len(),
227                        2,
228                        "map entry at index {index} has the wrong number of values",
229                    );
230
231                    Self::validate_detailed(&key_dtype, values[0].as_ref()).map_err(|error| {
232                        vortex_error::vortex_err!("map key at entry {index}: {error}")
233                    })?;
234                    Self::validate_detailed(&value_dtype, values[1].as_ref()).map_err(|error| {
235                        vortex_error::vortex_err!("map value at entry {index}: {error}")
236                    })?;
237                }
238            }
239            DType::Struct(fields, _) => {
240                let ScalarValue::Tuple(values) = value else {
241                    vortex_bail!("struct dtype expected Tuple value, got {value}");
242                };
243
244                let nfields = fields.nfields();
245                let nvalues = values.len();
246                vortex_ensure_eq!(
247                    nvalues,
248                    nfields,
249                    "struct scalar has the wrong number of fields",
250                );
251
252                for (field, field_value) in fields.field_dtypes().zip(values.iter()) {
253                    Self::validate_detailed(field, field_value.as_ref())?;
254                }
255            }
256            DType::Union(variants, _) => {
257                let ScalarValue::Union(union_value) = value else {
258                    vortex_bail!("union dtype expected Union value, got {value}");
259                };
260
261                let type_id = union_value.type_id();
262                let Some(child_index) = variants.tag_to_child_index(type_id) else {
263                    vortex_bail!(
264                        "union value has unknown type ID {type_id}; expected one of {:?}",
265                        variants.type_ids()
266                    );
267                };
268
269                let child_dtype = variants
270                    .variant_by_index(child_index)
271                    .vortex_expect("resolved union child index must be valid");
272
273                Self::validate_detailed(&child_dtype, union_value.child_value()).map_err(
274                    |error| {
275                        vortex_error::vortex_err!(
276                            "union value for type ID {type_id} is invalid for dtype {child_dtype}: \
277                         {error}"
278                        )
279                    },
280                )?;
281            }
282            DType::Variant(_) => {
283                let ScalarValue::Variant(inner) = value else {
284                    vortex_bail!("variant dtype expected Variant value, got {value}");
285                };
286
287                Self::validate_detailed(inner.dtype(), inner.value())?;
288                vortex_ensure!(
289                    !inner.is_null() || matches!(inner.dtype(), DType::Null),
290                    "variant nulls must use a nested null scalar, got {}",
291                    inner.dtype(),
292                );
293            }
294            DType::Extension(ext_dtype) => ext_dtype.validate_storage_value(value)?,
295        }
296
297        Ok(())
298    }
299}
300
301#[cfg(test)]
302mod tests {
303    use std::sync::Arc;
304
305    use vortex_error::VortexResult;
306
307    use crate::dtype::DType;
308    use crate::dtype::DecimalDType;
309    use crate::dtype::FieldNames;
310    use crate::dtype::MapDType;
311    use crate::dtype::Nullability;
312    use crate::dtype::PType;
313    use crate::dtype::StructFields;
314    use crate::dtype::UnionVariants;
315    use crate::scalar::DecimalValue;
316    use crate::scalar::PValue;
317    use crate::scalar::Scalar;
318    use crate::scalar::ScalarValue;
319    use crate::scalar::UnionValue;
320
321    /// The fast check in `is_compatible` must accept exactly what `validate_detailed` accepts.
322    #[test]
323    fn fast_and_detailed_checks_agree() -> VortexResult<()> {
324        use Nullability::NonNullable;
325        use Nullability::Nullable;
326
327        let i32_value = || Some(ScalarValue::Primitive(PValue::I32(1)));
328        let i32_dtype = DType::Primitive(PType::I32, NonNullable);
329        let struct_dtype = DType::Struct(
330            StructFields::new(
331                FieldNames::from(["a", "b"]),
332                vec![i32_dtype.clone(), DType::Utf8(Nullable)],
333            ),
334            NonNullable,
335        );
336        let map_dtype = DType::Map(
337            MapDType::try_new(i32_dtype.clone(), DType::Bool(Nullable), false)?,
338            NonNullable,
339        );
340        let union_dtype = DType::Union(
341            UnionVariants::try_new(["int"].into(), vec![i32_dtype.clone()], vec![3])?,
342            NonNullable,
343        );
344        let list_dtype = DType::List(Arc::new(i32_dtype.clone()), NonNullable);
345        let fsl_dtype = DType::FixedSizeList(Arc::new(i32_dtype.clone()), 2, NonNullable);
346        let decimal_dtype = DType::Decimal(DecimalDType::new(3, 0), NonNullable);
347        let tuple = |values: Vec<Option<ScalarValue>>| Some(ScalarValue::Tuple(values));
348
349        let cases = [
350            (DType::Null, None, true),
351            (DType::Null, Some(ScalarValue::Bool(true)), false),
352            (DType::Bool(Nullable), None, true),
353            (DType::Bool(NonNullable), None, false),
354            (
355                DType::Bool(NonNullable),
356                Some(ScalarValue::Bool(true)),
357                true,
358            ),
359            (DType::Bool(NonNullable), i32_value(), false),
360            (i32_dtype.clone(), i32_value(), true),
361            (
362                i32_dtype,
363                Some(ScalarValue::Primitive(PValue::I64(1))),
364                false,
365            ),
366            (
367                DType::Primitive(PType::F16, NonNullable),
368                Some(ScalarValue::Primitive(PValue::U64(1))),
369                true,
370            ),
371            (
372                decimal_dtype.clone(),
373                Some(ScalarValue::Decimal(DecimalValue::I32(999))),
374                true,
375            ),
376            (
377                decimal_dtype,
378                Some(ScalarValue::Decimal(DecimalValue::I32(1000))),
379                false,
380            ),
381            (
382                DType::Utf8(NonNullable),
383                Some(ScalarValue::Utf8("a".into())),
384                true,
385            ),
386            (
387                DType::Utf8(NonNullable),
388                Some(ScalarValue::Bool(true)),
389                false,
390            ),
391            (
392                DType::Binary(NonNullable),
393                Some(ScalarValue::Utf8("a".into())),
394                false,
395            ),
396            (
397                list_dtype.clone(),
398                tuple(vec![i32_value(), i32_value()]),
399                true,
400            ),
401            (list_dtype.clone(), tuple(vec![i32_value(), None]), false),
402            (list_dtype, i32_value(), false),
403            (
404                fsl_dtype.clone(),
405                tuple(vec![i32_value(), i32_value()]),
406                true,
407            ),
408            (fsl_dtype, tuple(vec![i32_value()]), false),
409            (
410                map_dtype.clone(),
411                tuple(vec![tuple(vec![i32_value(), None])]),
412                true,
413            ),
414            (map_dtype.clone(), tuple(vec![None]), false),
415            (
416                map_dtype.clone(),
417                tuple(vec![tuple(vec![None, None])]),
418                false,
419            ),
420            (map_dtype, tuple(vec![tuple(vec![i32_value()])]), false),
421            (
422                struct_dtype.clone(),
423                tuple(vec![i32_value(), Some(ScalarValue::Utf8("a".into()))]),
424                true,
425            ),
426            (struct_dtype.clone(), tuple(vec![i32_value(), None]), true),
427            (struct_dtype.clone(), tuple(vec![None, None]), false),
428            (struct_dtype, tuple(vec![i32_value()]), false),
429            (
430                union_dtype.clone(),
431                Some(ScalarValue::Union(UnionValue::new(3, i32_value()))),
432                true,
433            ),
434            (
435                union_dtype.clone(),
436                Some(ScalarValue::Union(UnionValue::new(4, i32_value()))),
437                false,
438            ),
439            (
440                union_dtype,
441                Some(ScalarValue::Union(UnionValue::new(3, None))),
442                false,
443            ),
444            (
445                DType::Variant(NonNullable),
446                Some(ScalarValue::Variant(Box::new(Scalar::null(DType::Null)))),
447                true,
448            ),
449            (
450                DType::Variant(NonNullable),
451                Some(ScalarValue::Variant(Box::new(Scalar::null(DType::Bool(
452                    Nullable,
453                ))))),
454                false,
455            ),
456            (
457                DType::Variant(NonNullable),
458                Some(ScalarValue::Variant(Box::new(Scalar::from(true)))),
459                true,
460            ),
461        ];
462
463        for (dtype, value, expected) in &cases {
464            assert_eq!(
465                Scalar::is_compatible(dtype, value.as_ref()),
466                *expected,
467                "unexpected validity for {dtype} and {value:?}"
468            );
469            assert_eq!(
470                Scalar::is_compatible(dtype, value.as_ref()),
471                Scalar::validate_detailed(dtype, value.as_ref()).is_ok(),
472                "fast and detailed checks disagree for {dtype} and {value:?}",
473            );
474        }
475
476        Ok(())
477    }
478
479    #[test]
480    fn union_rejects_unknown_tag_and_wrong_value() -> VortexResult<()> {
481        let variants = UnionVariants::try_new(
482            ["int", "string"].into(),
483            vec![
484                DType::Primitive(PType::I32, Nullability::Nullable),
485                DType::Utf8(Nullability::NonNullable),
486            ],
487            vec![5, 9],
488        )?;
489        let dtype = DType::Union(variants, Nullability::NonNullable);
490
491        assert!(
492            Scalar::try_new(
493                dtype.clone(),
494                Some(ScalarValue::Union(UnionValue::new(
495                    7,
496                    Scalar::primitive(42_i32, Nullability::Nullable).into_value(),
497                ))),
498            )
499            .is_err()
500        );
501
502        assert!(
503            Scalar::try_new(
504                dtype,
505                Some(ScalarValue::Union(UnionValue::new(
506                    5,
507                    Scalar::utf8("wrong", Nullability::NonNullable).into_value(),
508                ))),
509            )
510            .is_err()
511        );
512
513        Ok(())
514    }
515}
vortex_array/scalar/validate.rs

vortex_array/scalar/
validate.rs
