Skip to main content

vortex_array/scalar/
validate.rs

1// SPDX-License-Identifier: Apache-2.0
2// SPDX-FileCopyrightText: Copyright the Vortex contributors
3
4//! Scalar validation keeps logical values compatible with their storage representations.
5//!
6//! The fast check and detailed error path accept the same values. Extension values are checked
7//! against the storage dtype before the extension's own validator runs.
8
9use vortex_error::VortexError;
10use vortex_error::VortexExpect;
11use vortex_error::VortexResult;
12use vortex_error::vortex_bail;
13use vortex_error::vortex_ensure;
14use vortex_error::vortex_ensure_eq;
15use vortex_error::vortex_err;
16
17use crate::dtype::DType;
18use crate::dtype::MapDType;
19use crate::dtype::UnionVariants;
20use crate::scalar::Scalar;
21use crate::scalar::ScalarValue;
22
23impl Scalar {
24    /// Validate that the given [`ScalarValue`] is compatible with the given [`DType`].
25    #[inline]
26    pub fn validate(dtype: &DType, value: Option<&ScalarValue>) -> VortexResult<()> {
27        if Self::is_compatible(dtype, value) {
28            return Ok(());
29        }
30
31        Err(Self::incompatible_error(dtype, value))
32    }
33
34    /// Returns `true` if the given [`ScalarValue`] is compatible with the given [`DType`].
35    ///
36    /// This is the fast path of [`Self::validate`]. It must accept exactly the values that
37    /// [`Self::validate_detailed`] accepts, but it builds no error messages.
38    ///
39    /// Null values and leaf types are checked inline, so that callers with a known dtype fold the
40    /// check away. Other types go to [`Self::is_compatible_nested`].
41    #[inline]
42    pub(super) fn is_compatible(dtype: &DType, value: Option<&ScalarValue>) -> bool {
43        let Some(value) = value else {
44            return dtype.is_nullable();
45        };
46
47        match (dtype, value) {
48            (DType::Bool(_), ScalarValue::Bool(_))
49            | (DType::Utf8(_), ScalarValue::Utf8(_))
50            | (DType::Binary(_), ScalarValue::Binary(_)) => true,
51            (DType::Primitive(ptype, _), ScalarValue::Primitive(pvalue)) => {
52                pvalue.ptype() == *ptype
53            }
54            _ => Self::is_compatible_nested(dtype, value),
55        }
56    }
57
58    /// The out-of-line part of [`Self::is_compatible`] for non-null values of all other types.
59    fn is_compatible_nested(dtype: &DType, value: &ScalarValue) -> bool {
60        match (dtype, value) {
61            (DType::Decimal(dec_dtype, _), ScalarValue::Decimal(dvalue)) => {
62                dvalue.fits_in_precision(*dec_dtype)
63            }
64            (DType::List(elem_dtype, _), ScalarValue::Tuple(elements)) => elements
65                .iter()
66                .all(|element| Self::is_compatible(elem_dtype, element.as_ref())),
67            (DType::FixedSizeList(elem_dtype, size, _), ScalarValue::Tuple(elements)) => {
68                elements.len() == *size as usize
69                    && elements
70                        .iter()
71                        .all(|element| Self::is_compatible(elem_dtype, element.as_ref()))
72            }
73            (DType::Map(map, _), ScalarValue::Tuple(entries)) => {
74                let key_dtype = map.key_dtype();
75                let value_dtype = map.value_dtype();
76
77                entries.iter().all(|entry| match entry {
78                    Some(ScalarValue::Tuple(values)) => {
79                        values.len() == 2
80                            && Self::is_compatible(&key_dtype, values[0].as_ref())
81                            && Self::is_compatible(&value_dtype, values[1].as_ref())
82                    }
83                    _ => false,
84                })
85            }
86            (DType::Struct(fields, _), ScalarValue::Tuple(values)) => {
87                values.len() == fields.nfields()
88                    && fields
89                        .field_dtypes()
90                        .zip(values.iter())
91                        .all(|(field, value)| Self::is_compatible(field, value.as_ref()))
92            }
93            (DType::Union(variants, _), ScalarValue::Union(union_value)) => variants
94                .tag_to_child_index(union_value.type_id())
95                .and_then(|child_index| variants.variant_by_index(child_index))
96                .is_some_and(|child_dtype| {
97                    Self::is_compatible(&child_dtype, union_value.child_value())
98                }),
99            (DType::Variant(_), ScalarValue::Variant(inner)) => {
100                Self::is_compatible(inner.dtype(), inner.value())
101                    && (!inner.is_null() || matches!(inner.dtype(), DType::Null))
102            }
103            (DType::Extension(ext_dtype), value) => {
104                Self::is_compatible(ext_dtype.storage_dtype(), Some(value))
105                    && ext_dtype.validate_storage_value(value).is_ok()
106            }
107            _ => false,
108        }
109    }
110
111    /// Builds the error for a [`ScalarValue`] that is not compatible with the [`DType`].
112    ///
113    /// Kept out of line so that the formatting code does not count against the hot callers when
114    /// the inliner sizes them.
115    #[cold]
116    #[inline(never)]
117    pub(super) fn incompatible_error(dtype: &DType, value: Option<&ScalarValue>) -> VortexError {
118        match Self::validate_detailed(dtype, value) {
119            Err(err) => err,
120            Ok(()) => vortex_err!(
121                "scalar validation fast path rejected value {value:?} for dtype {dtype}, but the \
122                 detailed check accepted it"
123            ),
124        }
125    }
126
127    /// Validates the [`ScalarValue`] against the [`DType`], with a detailed error message.
128    fn validate_detailed(dtype: &DType, value: Option<&ScalarValue>) -> VortexResult<()> {
129        let Some(value) = value else {
130            vortex_ensure!(
131                dtype.is_nullable(),
132                "non-nullable dtype {dtype} cannot hold a null value",
133            );
134            return Ok(());
135        };
136
137        // From here onwards, we know that the value is not null.
138        match dtype {
139            DType::Null => {
140                vortex_bail!("null dtype cannot hold a non-null value {value}");
141            }
142            DType::Bool(_) => {
143                vortex_ensure!(
144                    matches!(value, ScalarValue::Bool(_)),
145                    "bool dtype expected Bool value, got {value}",
146                );
147            }
148            DType::Primitive(ptype, _) => {
149                let ScalarValue::Primitive(pvalue) = value else {
150                    vortex_bail!("primitive dtype {ptype} expected Primitive value, got {value}");
151                };
152
153                vortex_ensure!(
154                    pvalue.ptype() == *ptype,
155                    "primitive dtype {ptype} requires a matching value, got {pvalue}",
156                );
157            }
158            DType::Utf8(_) => {
159                vortex_ensure!(
160                    matches!(value, ScalarValue::Utf8(_)),
161                    "utf8 dtype expected Utf8 value, got {value}",
162                );
163            }
164            DType::Binary(_) => {
165                vortex_ensure!(
166                    matches!(value, ScalarValue::Binary(_)),
167                    "binary dtype expected Binary value, got {value}",
168                );
169            }
170            DType::Decimal(dec_dtype, _) => {
171                let ScalarValue::Decimal(dvalue) = value else {
172                    vortex_bail!("decimal dtype expected Decimal value, got {value}");
173                };
174
175                vortex_ensure!(
176                    dvalue.fits_in_precision(*dec_dtype),
177                    "decimal value {dvalue} does not fit in precision of {dec_dtype}",
178                );
179            }
180            DType::List(elem_dtype, _) => {
181                let ScalarValue::Tuple(elements) = value else {
182                    vortex_bail!("list dtype expected Tuple value, got {value}");
183                };
184
185                for (i, element) in elements.iter().enumerate() {
186                    Self::validate_detailed(elem_dtype.as_ref(), element.as_ref())
187                        .map_err(|e| e.with_context(format!("list element at index {i}")))?;
188                }
189            }
190            DType::FixedSizeList(elem_dtype, size, _) => {
191                let ScalarValue::Tuple(elements) = value else {
192                    vortex_bail!("fixed-size list dtype expected Tuple value, got {value}",);
193                };
194
195                let len = elements.len();
196                vortex_ensure_eq!(
197                    len,
198                    *size as usize,
199                    "fixed-size list scalar has the wrong number of elements",
200                );
201
202                for (i, element) in elements.iter().enumerate() {
203                    Self::validate_detailed(elem_dtype.as_ref(), element.as_ref()).map_err(
204                        |e| e.with_context(format!("fixed-size list element at index {i}")),
205                    )?;
206                }
207            }
208            DType::Map(map, _) => Self::validate_map(map, value)?,
209            DType::Struct(fields, _) => {
210                let ScalarValue::Tuple(values) = value else {
211                    vortex_bail!("struct dtype expected Tuple value, got {value}");
212                };
213
214                let nfields = fields.nfields();
215                let nvalues = values.len();
216                vortex_ensure_eq!(
217                    nvalues,
218                    nfields,
219                    "struct scalar has the wrong number of fields",
220                );
221
222                for (field, field_value) in fields.field_dtypes().zip(values.iter()) {
223                    Self::validate_detailed(field, field_value.as_ref())?;
224                }
225            }
226            DType::Union(variants, _) => Self::validate_union(variants, value)?,
227            DType::Variant(_) => {
228                let ScalarValue::Variant(inner) = value else {
229                    vortex_bail!("variant dtype expected Variant value, got {value}");
230                };
231
232                Self::validate_detailed(inner.dtype(), inner.value())?;
233                vortex_ensure!(
234                    !inner.is_null() || matches!(inner.dtype(), DType::Null),
235                    "variant nulls must use a nested null scalar, got {}",
236                    inner.dtype(),
237                );
238            }
239            DType::Extension(ext_dtype) => {
240                Self::validate_detailed(ext_dtype.storage_dtype(), Some(value))?;
241
242                ext_dtype.validate_storage_value(value)?;
243            }
244        }
245
246        Ok(())
247    }
248
249    /// Validates map entries and adds their positions to storage validation errors.
250    fn validate_map(map: &MapDType, value: &ScalarValue) -> VortexResult<()> {
251        let ScalarValue::Tuple(entries) = value else {
252            vortex_bail!("map dtype expected Tuple value, got {value}");
253        };
254        let key_dtype = map.key_dtype();
255        let value_dtype = map.value_dtype();
256
257        for (index, entry) in entries.iter().enumerate() {
258            let entry = entry
259                .as_ref()
260                .ok_or_else(|| vortex_err!("map entry at index {index} cannot be null"))?;
261            let ScalarValue::Tuple(values) = entry else {
262                vortex_bail!("map entry at index {index} expected Tuple value, got {entry}");
263            };
264            vortex_ensure_eq!(
265                values.len(),
266                2,
267                "map entry at index {index} has the wrong number of values",
268            );
269
270            Self::validate_detailed(&key_dtype, values[0].as_ref())
271                .map_err(|error| error.with_context(format!("map key at entry {index}")))?;
272            Self::validate_detailed(&value_dtype, values[1].as_ref())
273                .map_err(|error| error.with_context(format!("map value at entry {index}")))?;
274        }
275
276        Ok(())
277    }
278
279    /// Validates the selected union child and adds its type ID to storage validation errors.
280    fn validate_union(variants: &UnionVariants, value: &ScalarValue) -> VortexResult<()> {
281        let ScalarValue::Union(union_value) = value else {
282            vortex_bail!("union dtype expected Union value, got {value}");
283        };
284
285        let type_id = union_value.type_id();
286        let Some(child_index) = variants.tag_to_child_index(type_id) else {
287            vortex_bail!(
288                "union value has unknown type ID {type_id}; expected one of {:?}",
289                variants.type_ids()
290            );
291        };
292
293        let child_dtype = variants
294            .variant_by_index(child_index)
295            .vortex_expect("resolved union child index must be valid");
296
297        Self::validate_detailed(&child_dtype, union_value.child_value()).map_err(|error| {
298            error.with_context(format!(
299                "union value for type ID {type_id} is invalid for dtype {child_dtype}"
300            ))
301        })
302    }
303}
304
305#[cfg(test)]
306mod tests {
307    use std::sync::Arc;
308
309    use vortex_error::VortexResult;
310
311    use crate::dtype::DType;
312    use crate::dtype::DecimalDType;
313    use crate::dtype::FieldNames;
314    use crate::dtype::MapDType;
315    use crate::dtype::Nullability;
316    use crate::dtype::PType;
317    use crate::dtype::StructFields;
318    use crate::dtype::UnionVariants;
319    use crate::extension::datetime::Date;
320    use crate::extension::datetime::TimeUnit;
321    use crate::scalar::DecimalValue;
322    use crate::scalar::PValue;
323    use crate::scalar::Scalar;
324    use crate::scalar::ScalarValue;
325    use crate::scalar::UnionValue;
326
327    /// The fast check in `is_compatible` must accept exactly what `validate_detailed` accepts.
328    #[test]
329    fn fast_and_detailed_checks_agree() -> VortexResult<()> {
330        use Nullability::NonNullable;
331        use Nullability::Nullable;
332
333        let i32_value = || Some(ScalarValue::Primitive(PValue::I32(1)));
334        let i32_dtype = DType::Primitive(PType::I32, NonNullable);
335        let struct_dtype = DType::Struct(
336            StructFields::new(
337                FieldNames::from(["a", "b"]),
338                vec![i32_dtype.clone(), DType::Utf8(Nullable)],
339            ),
340            NonNullable,
341        );
342        let map_dtype = DType::Map(
343            MapDType::try_new(i32_dtype.clone(), DType::Bool(Nullable), false)?,
344            NonNullable,
345        );
346        let union_dtype = DType::Union(
347            UnionVariants::try_new(["int"].into(), vec![i32_dtype.clone()], vec![3])?,
348            NonNullable,
349        );
350        let list_dtype = DType::List(Arc::new(i32_dtype.clone()), NonNullable);
351        let fsl_dtype = DType::FixedSizeList(Arc::new(i32_dtype.clone()), 2, NonNullable);
352        let decimal_dtype = DType::Decimal(DecimalDType::new(3, 0), NonNullable);
353        let extension_dtype = DType::Extension(Date::new(TimeUnit::Days, NonNullable).erased());
354        let tuple = |values: Vec<Option<ScalarValue>>| Some(ScalarValue::Tuple(values));
355
356        let cases = [
357            (DType::Null, None, true),
358            (DType::Null, Some(ScalarValue::Bool(true)), false),
359            (DType::Bool(Nullable), None, true),
360            (DType::Bool(NonNullable), None, false),
361            (
362                DType::Bool(NonNullable),
363                Some(ScalarValue::Bool(true)),
364                true,
365            ),
366            (DType::Bool(NonNullable), i32_value(), false),
367            (i32_dtype.clone(), i32_value(), true),
368            (
369                i32_dtype,
370                Some(ScalarValue::Primitive(PValue::I64(1))),
371                false,
372            ),
373            (
374                DType::Primitive(PType::F16, NonNullable),
375                Some(ScalarValue::Primitive(PValue::U64(1))),
376                false,
377            ),
378            (extension_dtype.clone(), i32_value(), true),
379            (
380                extension_dtype,
381                Some(ScalarValue::Primitive(PValue::F64(1.0))),
382                false,
383            ),
384            (
385                decimal_dtype.clone(),
386                Some(ScalarValue::Decimal(DecimalValue::I32(999))),
387                true,
388            ),
389            (
390                decimal_dtype,
391                Some(ScalarValue::Decimal(DecimalValue::I32(1000))),
392                false,
393            ),
394            (
395                DType::Utf8(NonNullable),
396                Some(ScalarValue::Utf8("a".into())),
397                true,
398            ),
399            (
400                DType::Utf8(NonNullable),
401                Some(ScalarValue::Bool(true)),
402                false,
403            ),
404            (
405                DType::Binary(NonNullable),
406                Some(ScalarValue::Utf8("a".into())),
407                false,
408            ),
409            (
410                list_dtype.clone(),
411                tuple(vec![i32_value(), i32_value()]),
412                true,
413            ),
414            (list_dtype.clone(), tuple(vec![i32_value(), None]), false),
415            (list_dtype, i32_value(), false),
416            (
417                fsl_dtype.clone(),
418                tuple(vec![i32_value(), i32_value()]),
419                true,
420            ),
421            (fsl_dtype, tuple(vec![i32_value()]), false),
422            (
423                map_dtype.clone(),
424                tuple(vec![tuple(vec![i32_value(), None])]),
425                true,
426            ),
427            (map_dtype.clone(), tuple(vec![None]), false),
428            (
429                map_dtype.clone(),
430                tuple(vec![tuple(vec![None, None])]),
431                false,
432            ),
433            (map_dtype, tuple(vec![tuple(vec![i32_value()])]), false),
434            (
435                struct_dtype.clone(),
436                tuple(vec![i32_value(), Some(ScalarValue::Utf8("a".into()))]),
437                true,
438            ),
439            (struct_dtype.clone(), tuple(vec![i32_value(), None]), true),
440            (struct_dtype.clone(), tuple(vec![None, None]), false),
441            (struct_dtype, tuple(vec![i32_value()]), false),
442            (
443                union_dtype.clone(),
444                Some(ScalarValue::Union(UnionValue::new(3, i32_value()))),
445                true,
446            ),
447            (
448                union_dtype.clone(),
449                Some(ScalarValue::Union(UnionValue::new(4, i32_value()))),
450                false,
451            ),
452            (
453                union_dtype,
454                Some(ScalarValue::Union(UnionValue::new(3, None))),
455                false,
456            ),
457            (
458                DType::Variant(NonNullable),
459                Some(ScalarValue::Variant(Box::new(Scalar::null(DType::Null)))),
460                true,
461            ),
462            (
463                DType::Variant(NonNullable),
464                Some(ScalarValue::Variant(Box::new(Scalar::null(DType::Bool(
465                    Nullable,
466                ))))),
467                false,
468            ),
469            (
470                DType::Variant(NonNullable),
471                Some(ScalarValue::Variant(Box::new(Scalar::from(true)))),
472                true,
473            ),
474        ];
475
476        for (dtype, value, expected) in &cases {
477            assert_eq!(
478                Scalar::is_compatible(dtype, value.as_ref()),
479                *expected,
480                "unexpected validity for {dtype} and {value:?}"
481            );
482            assert_eq!(
483                Scalar::is_compatible(dtype, value.as_ref()),
484                Scalar::validate_detailed(dtype, value.as_ref()).is_ok(),
485                "fast and detailed checks disagree for {dtype} and {value:?}",
486            );
487        }
488
489        Ok(())
490    }
491
492    #[test]
493    fn union_rejects_unknown_tag_and_wrong_value() -> VortexResult<()> {
494        let variants = UnionVariants::try_new(
495            ["int", "string"].into(),
496            vec![
497                DType::Primitive(PType::I32, Nullability::Nullable),
498                DType::Utf8(Nullability::NonNullable),
499            ],
500            vec![5, 9],
501        )?;
502        let dtype = DType::Union(variants, Nullability::NonNullable);
503
504        assert!(
505            Scalar::try_new(
506                dtype.clone(),
507                Some(ScalarValue::Union(UnionValue::new(
508                    7,
509                    Scalar::primitive(42_i32, Nullability::Nullable).into_value(),
510                ))),
511            )
512            .is_err()
513        );
514
515        assert!(
516            Scalar::try_new(
517                dtype,
518                Some(ScalarValue::Union(UnionValue::new(
519                    5,
520                    Scalar::utf8("wrong", Nullability::NonNullable).into_value(),
521                ))),
522            )
523            .is_err()
524        );
525
526        Ok(())
527    }
528}