Skip to main content

vortex_array/
validity.rs

1// SPDX-License-Identifier: Apache-2.0
2// SPDX-FileCopyrightText: Copyright the Vortex contributors
3
4//! Array validity and nullability behavior, used by arrays and compute functions.
5//!
6//! [`Validity`] describes which rows are logically present without forcing every array to carry a
7//! materialized boolean bitmap. Constant states (`NonNullable`, `AllValid`, `AllInvalid`) are cheap
8//! to clone and inspect. [`Validity::Array`] may itself be encoded or lazy, so APIs that need exact
9//! per-row answers take an [`ExecutionCtx`] and execute the validity array as needed.
10
11use std::fmt::Debug;
12use std::ops::Range;
13
14use itertools::Itertools as _;
15use vortex_buffer::BitBuffer;
16use vortex_error::VortexExpect as _;
17use vortex_error::VortexResult;
18use vortex_error::vortex_bail;
19use vortex_error::vortex_err;
20use vortex_mask::Mask;
21use vortex_mask::MaskValues;
22
23use crate::ArrayRef;
24use crate::Canonical;
25use crate::ExecutionCtx;
26use crate::IntoArray;
27use crate::arrays::BoolArray;
28use crate::arrays::ChunkedArray;
29use crate::arrays::ConstantArray;
30use crate::builtins::ArrayBuiltins;
31use crate::dtype::DType;
32use crate::dtype::Nullability;
33use crate::optimizer::ArrayOptimizer;
34use crate::patches::Patches;
35use crate::scalar::Scalar;
36use crate::scalar_fn::fns::binary::Binary;
37use crate::scalar_fn::fns::operators::Operator;
38
39/// Validity information for an array.
40#[derive(Clone)]
41pub enum Validity {
42    /// Items cannot be null because the dtype is non-nullable.
43    NonNullable,
44    /// The dtype is nullable, but every item is valid.
45    AllValid,
46    /// The dtype is nullable, and every item is null.
47    AllInvalid,
48    /// The validity of each position in the array is determined by a boolean array.
49    ///
50    /// True values are valid, false values are invalid ("null").
51    Array(ArrayRef),
52}
53
54impl Debug for Validity {
55    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
56        match self {
57            Self::NonNullable => write!(f, "NonNullable"),
58            Self::AllValid => write!(f, "AllValid"),
59            Self::AllInvalid => write!(f, "AllInvalid"),
60            Self::Array(arr) => write!(f, "SomeValid({arr})"),
61        }
62    }
63}
64
65impl Validity {
66    /// Make a step towards canonicalising validity if necessary
67    pub fn execute(self, ctx: &mut ExecutionCtx) -> VortexResult<Validity> {
68        match self {
69            v @ Validity::NonNullable | v @ Validity::AllValid | v @ Validity::AllInvalid => Ok(v),
70            Validity::Array(a) => Ok(Validity::Array(a.execute::<Canonical>(ctx)?.into_array())),
71        }
72    }
73}
74
75impl Validity {
76    /// The [`DType`] of the underlying validity array (if it exists).
77    pub const DTYPE: DType = DType::Bool(Nullability::NonNullable);
78
79    /// Convert the validity to an array representation.
80    pub fn to_array(&self, len: usize) -> ArrayRef {
81        match self {
82            Self::NonNullable | Self::AllValid => ConstantArray::new(true, len).into_array(),
83            Self::AllInvalid => ConstantArray::new(false, len).into_array(),
84            Self::Array(a) => a.clone(),
85        }
86    }
87
88    /// If Validity is [`Validity::Array`], returns the array, otherwise returns `None`.
89    #[inline]
90    pub fn into_array(self) -> Option<ArrayRef> {
91        if let Self::Array(a) = self {
92            Some(a)
93        } else {
94            None
95        }
96    }
97
98    /// If Validity is [`Validity::Array`], returns a reference to the array array, otherwise returns `None`.
99    #[inline]
100    pub fn as_array(&self) -> Option<&ArrayRef> {
101        if let Self::Array(a) = self {
102            Some(a)
103        } else {
104            None
105        }
106    }
107
108    #[inline]
109    pub fn nullability(&self) -> Nullability {
110        if matches!(self, Self::NonNullable) {
111            Nullability::NonNullable
112        } else {
113            Nullability::Nullable
114        }
115    }
116
117    /// Returns `true` if this validity *definitely* contains no null values, i.e. it is either
118    /// [`Validity::NonNullable`] or [`Validity::AllValid`].
119    ///
120    /// Returning `false` does not prove the presence of nulls: a [`Validity::Array`] may still
121    /// resolve to all-valid once executed. Callers must treat `false` as "unknown without
122    /// compute" and either fall back to a null-handling path or execute the validity.
123    #[inline]
124    pub fn definitely_no_nulls(&self) -> bool {
125        matches!(self, Self::NonNullable | Self::AllValid)
126    }
127
128    /// Returns `true` if this validity is *definitely* all-null (every value is null), i.e. it
129    /// is [`Validity::AllInvalid`].
130    ///
131    /// Returning `false` does not prove that any value is valid: a [`Validity::Array`] may still
132    /// resolve to all-null once executed. Callers must treat `false` as "unknown without
133    /// compute". For a definitive answer, execute the validity with [`Self::execute_mask`] and
134    /// check whether the resulting [`Mask`] is all-false (`Mask::all_false`). This is the
135    /// all-null counterpart to [`Self::definitely_no_nulls`].
136    #[inline]
137    pub fn definitely_all_null(&self) -> bool {
138        matches!(self, Self::AllInvalid)
139    }
140
141    /// Returns whether this validity contains no null values, executing the validity array if
142    /// necessary.
143    ///
144    /// This is the exact counterpart to [`Self::definitely_no_nulls`]: use it when the caller
145    /// needs a definitive answer rather than a cheap, conservative one.
146    pub fn execute_no_nulls(&self, length: usize, ctx: &mut ExecutionCtx) -> VortexResult<bool> {
147        match self {
148            Self::NonNullable | Self::AllValid => Ok(true),
149            Self::AllInvalid => Ok(length == 0),
150            Self::Array(_) => Ok(self.execute_mask(length, ctx)?.all_true()),
151        }
152    }
153
154    /// The union nullability and validity.
155    #[inline]
156    pub fn union_nullability(self, nullability: Nullability) -> Self {
157        match nullability {
158            Nullability::NonNullable => self,
159            Nullability::Nullable => self.into_nullable(),
160        }
161    }
162
163    /// Returns whether the `index` item is valid, using `ctx` to execute the validity array.
164    // Todo(joe): deprecate this
165    #[inline]
166    pub fn execute_is_valid(&self, index: usize, ctx: &mut ExecutionCtx) -> VortexResult<bool> {
167        Ok(match self {
168            Self::NonNullable | Self::AllValid => true,
169            Self::AllInvalid => false,
170            Self::Array(a) => a
171                .execute_scalar(index, ctx)?
172                .as_bool()
173                .value()
174                .ok_or_else(|| vortex_err!("validity value at index {index} is null"))?,
175        })
176    }
177
178    /// Returns whether the `index` item is null, using `ctx` to execute the validity array.
179    // Todo(joe): deprecate this
180    #[inline]
181    pub fn execute_is_null(&self, index: usize, ctx: &mut ExecutionCtx) -> VortexResult<bool> {
182        Ok(!self.execute_is_valid(index, ctx)?)
183    }
184
185    #[inline]
186    pub fn slice(&self, range: Range<usize>) -> VortexResult<Self> {
187        match self {
188            Self::Array(a) => Ok(Self::Array(a.slice(range)?)),
189            Self::NonNullable | Self::AllValid | Self::AllInvalid => Ok(self.clone()),
190        }
191    }
192
193    pub fn take(&self, indices: &ArrayRef) -> VortexResult<Self> {
194        match self {
195            Self::NonNullable => indices.validity(),
196            Self::AllValid => Ok(match indices.validity()? {
197                Self::NonNullable => Self::AllValid,
198                v => v,
199            }),
200            Self::AllInvalid => Ok(Self::AllInvalid),
201            Self::Array(is_valid) => {
202                let maybe_is_valid = is_valid.take(indices.clone())?;
203                // Null indices invalidate that position.
204                let is_valid = maybe_is_valid.fill_null(Scalar::from(false))?;
205                Ok(Self::Array(is_valid))
206            }
207        }
208    }
209
210    // Invert the validity
211    pub fn not(&self) -> VortexResult<Self> {
212        match self {
213            Validity::NonNullable => Ok(Validity::NonNullable),
214            Validity::AllValid => Ok(Validity::AllInvalid),
215            Validity::AllInvalid => Ok(Validity::AllValid),
216            Validity::Array(arr) => Ok(Validity::Array(arr.not()?)),
217        }
218    }
219
220    /// Lazily filters a [`Validity`] with a selection mask, which keeps only the entries for which
221    /// the mask is true.
222    ///
223    /// The result has length equal to the number of true values in mask.
224    ///
225    /// If the validity is a [`Validity::Array`], then this lazily wraps it in a `FilterArray`
226    /// instead of eagerly filtering the values immediately.
227    pub fn filter(&self, mask: &Mask) -> VortexResult<Self> {
228        // NOTE(ngates): we take the mask as a reference to avoid the caller cloning unnecessarily
229        //  if we happen to be NonNullable, AllValid, or AllInvalid.
230        match self {
231            v @ (Validity::NonNullable | Validity::AllValid | Validity::AllInvalid) => {
232                Ok(v.clone())
233            }
234            Validity::Array(arr) => Ok(Validity::Array(arr.filter(mask.clone())?)),
235        }
236    }
237
238    /// Converts this validity into a [`Mask`] of the given length.
239    ///
240    /// Valid elements are `true` and invalid elements are `false`.
241    #[deprecated(note = "Use execute_mask")]
242    pub fn to_mask(&self, length: usize, ctx: &mut ExecutionCtx) -> VortexResult<Mask> {
243        match self {
244            Self::NonNullable | Self::AllValid => Ok(Mask::new_true(length)),
245            Self::AllInvalid => Ok(Mask::new_false(length)),
246            Self::Array(arr) => arr.clone().execute::<Mask>(ctx),
247        }
248    }
249
250    /// Execute this validity into a [`Mask`] of the given length.
251    ///
252    /// Resolving [`Validity::Array`] can execute and scan up to `length` values. Optimistic fast
253    /// paths that do not need the exact mask should use [`Self::definitely_no_nulls`] or
254    /// [`Self::definitely_all_null`] and delay this work until necessary.
255    #[inline]
256    pub fn execute_mask(&self, length: usize, ctx: &mut ExecutionCtx) -> VortexResult<Mask> {
257        match self {
258            Self::NonNullable | Self::AllValid => Ok(Mask::AllTrue(length)),
259            Self::AllInvalid => Ok(Mask::AllFalse(length)),
260            Self::Array(arr) => {
261                assert_eq!(
262                    arr.len(),
263                    length,
264                    "Validity::Array length must equal to_logical's argument: {}, {}.",
265                    arr.len(),
266                    length,
267                );
268                // TODO(ngates): I'm not sure execution should take arrays by ownership.
269                //  If so we should fix call sites to clone and this function takes self.
270                arr.clone().execute::<Mask>(ctx)
271            }
272        }
273    }
274
275    /// Compare the logical masks of two Validity values of the given length, executing them
276    /// into [`Mask`]s if necessary.
277    pub fn mask_eq(
278        &self,
279        other: &Validity,
280        length: usize,
281        ctx: &mut ExecutionCtx,
282    ) -> VortexResult<bool> {
283        match (self, other) {
284            // Fast paths that avoid executing: constant variants with known-equal masks.
285            (
286                Validity::NonNullable | Validity::AllValid,
287                Validity::NonNullable | Validity::AllValid,
288            )
289            | (Validity::AllInvalid, Validity::AllInvalid) => Ok(true),
290            _ => Ok(self.execute_mask(length, ctx)? == other.execute_mask(length, ctx)?),
291        }
292    }
293
294    /// Logically & two Validity values of the same length
295    #[inline]
296    pub fn and(self, rhs: Validity) -> VortexResult<Validity> {
297        Ok(match (self, rhs) {
298            // Should be pretty clear
299            (Validity::NonNullable, Validity::NonNullable) => Validity::NonNullable,
300            // Any `AllInvalid` makes the output all invalid values
301            (Validity::AllInvalid, _) | (_, Validity::AllInvalid) => Validity::AllInvalid,
302            // All truthy values on one side, which makes no effect on an `Array` variant
303            (Validity::Array(a), Validity::AllValid)
304            | (Validity::Array(a), Validity::NonNullable)
305            | (Validity::NonNullable, Validity::Array(a))
306            | (Validity::AllValid, Validity::Array(a)) => Validity::Array(a),
307            // Both sides are all valid
308            (Validity::NonNullable, Validity::AllValid)
309            | (Validity::AllValid, Validity::NonNullable)
310            | (Validity::AllValid, Validity::AllValid) => Validity::AllValid,
311            // Here we actually have to do some work
312            (Validity::Array(lhs), Validity::Array(rhs)) => Validity::Array(
313                Binary::try_new(lhs, rhs, Operator::And)?
314                    .into_array()
315                    .optimize()?,
316            ),
317        })
318    }
319
320    pub fn patch(
321        self,
322        len: usize,
323        indices_offset: usize,
324        indices: &ArrayRef,
325        patches: &Validity,
326        ctx: &mut ExecutionCtx,
327    ) -> VortexResult<Self> {
328        match (&self, patches) {
329            (Validity::NonNullable, Validity::NonNullable) => return Ok(Validity::NonNullable),
330            (Validity::NonNullable, _) => {
331                vortex_bail!("Can't patch a non-nullable validity with nullable validity")
332            }
333            (_, Validity::NonNullable) => {
334                vortex_bail!("Can't patch a nullable validity with non-nullable validity")
335            }
336            (Validity::AllValid, Validity::AllValid) => return Ok(Validity::AllValid),
337            (Validity::AllInvalid, Validity::AllInvalid) => return Ok(Validity::AllInvalid),
338            _ => {}
339        };
340
341        if matches!(self, Validity::NonNullable) {
342            return Ok(Self::NonNullable);
343        }
344
345        // From here on we know that the validity is nullable
346        let source = match self {
347            Validity::NonNullable => BoolArray::from(BitBuffer::new_set(len)),
348            Validity::AllValid => BoolArray::from(BitBuffer::new_set(len)),
349            Validity::AllInvalid => BoolArray::from(BitBuffer::new_unset(len)),
350            Validity::Array(a) => a.execute::<BoolArray>(ctx)?,
351        };
352
353        let patch_values = match patches {
354            Validity::NonNullable => BoolArray::from(BitBuffer::new_set(indices.len())),
355            Validity::AllValid => BoolArray::from(BitBuffer::new_set(indices.len())),
356            Validity::AllInvalid => BoolArray::from(BitBuffer::new_unset(indices.len())),
357            Validity::Array(a) => a.clone().execute::<BoolArray>(ctx)?,
358        };
359
360        let patches = Patches::new(
361            len,
362            indices_offset,
363            indices.clone(),
364            patch_values.into_array(),
365            // TODO(0ax1): chunk offsets
366            None,
367            ctx,
368        )?;
369
370        Ok(Self::Array(source.patch(&patches, ctx)?.into_array()))
371    }
372
373    /// Convert into a nullable variant.
374    #[inline]
375    pub fn into_nullable(self) -> Validity {
376        match self {
377            Self::NonNullable => Self::AllValid,
378            Self::AllValid | Self::AllInvalid | Self::Array(_) => self,
379        }
380    }
381
382    /// Convert into a non-nullable variant, computing statistics if necessary.
383    ///
384    /// Returns `None` when the array contains invalid values (so the cast cannot be performed),
385    /// either because it is [`Validity::AllInvalid`] or because the validity array's minimum is
386    /// `false`.
387    #[inline]
388    pub fn into_non_nullable(self, len: usize, ctx: &mut ExecutionCtx) -> Option<Validity> {
389        match self {
390            _ if len == 0 => Some(Validity::NonNullable),
391            Self::NonNullable => Some(Self::NonNullable),
392            Self::AllValid => Some(Self::NonNullable),
393            Self::AllInvalid => None,
394            Self::Array(is_valid) => {
395                is_valid
396                    .statistics()
397                    .compute_min::<bool>(ctx)
398                    .vortex_expect("validity array must support min")
399                    .then(|| {
400                        // min true => all true
401                        Self::NonNullable
402                    })
403            }
404        }
405    }
406
407    /// Convert into a non-nullable variant without running execution.
408    ///
409    /// This is the cheap counterpart to [`Self::into_non_nullable`]: it inspects already-computed
410    /// statistics rather than triggering execution.
411    ///
412    /// Return values:
413    /// - `Ok(Some(NonNullable))` — the cast is provably safe.
414    /// - `Ok(None)` — We need to perform compute to determine whether cast is valid. Callers should fall back to [`Self::into_non_nullable`], typically by
415    ///   returning `Ok(None)` from a `CastReduce` rule so the corresponding `CastKernel` runs.
416    /// - `Err(_)` — we know the cast must fail (e.g. [`Validity::AllInvalid`]).
417    #[inline]
418    pub fn trivial_into_non_nullable(self, len: usize) -> VortexResult<Option<Validity>> {
419        match self {
420            _ if len == 0 => Ok(Some(Validity::NonNullable)),
421            Self::NonNullable => Ok(Some(Self::NonNullable)),
422            Self::AllValid => Ok(Some(Self::NonNullable)),
423            Self::AllInvalid => Err(vortex_err!("Cannot cast AllInvalid to NonNullable")),
424            Self::Array(_) => Ok(None),
425        }
426    }
427
428    /// Convert into a variant compatible with the given nullability.
429    ///
430    /// Unlike [`Self::trivially_cast_nullability`], this can execute the validity array, so a
431    /// [`CastKernel`] calls it. [`CastReduce`] documents when a cast rule needs the pair.
432    ///
433    /// Returns `Err` when nullability cannot be cast (for example, casting to non-nullable while
434    /// invalid values are present).
435    ///
436    /// [`CastKernel`]: crate::scalar_fn::fns::cast::CastKernel
437    /// [`CastReduce`]: crate::scalar_fn::fns::cast::CastReduce
438    #[inline]
439    pub fn cast_nullability(
440        self,
441        nullability: Nullability,
442        len: usize,
443        ctx: &mut ExecutionCtx,
444    ) -> VortexResult<Validity> {
445        match nullability {
446            Nullability::NonNullable => self.into_non_nullable(len, ctx).ok_or_else(|| {
447                vortex_err!("Cannot cast array with invalid values to non-nullable type.")
448            }),
449            Nullability::Nullable => Ok(self.into_nullable()),
450        }
451    }
452
453    /// Best-effort, non-executing variant of [`Self::cast_nullability`].
454    ///
455    /// A [`CastReduce`] rule that builds a new [`Validity`] calls this, because a rule cannot
456    /// execute.
457    ///
458    /// Return values:
459    /// - `Ok(Some(_))` — the cast is provably safe and the new [`Validity`] is returned.
460    /// - `Ok(None)` — the cast cannot be reduced cheaply (the `CastKernel` should be tried via
461    ///   [`Self::cast_nullability`]).
462    /// - `Err(_)` — the cast is provably impossible.
463    ///
464    /// Typical usage inside a `CastReduce`:
465    ///
466    /// ```ignore
467    /// let Some(new_validity) = array
468    ///     .validity()?
469    ///     .trivially_cast_nullability(dtype.nullability(), array.len())?
470    /// else {
471    ///     return Ok(None);
472    /// };
473    /// ```
474    ///
475    /// [`CastReduce`]: crate::scalar_fn::fns::cast::CastReduce
476    #[inline]
477    pub fn trivially_cast_nullability(
478        self,
479        nullability: Nullability,
480        len: usize,
481    ) -> VortexResult<Option<Validity>> {
482        match nullability {
483            Nullability::NonNullable => self.trivial_into_non_nullable(len),
484            Nullability::Nullable => Ok(Some(self.into_nullable())),
485        }
486    }
487
488    /// Returns the length of the validity array, if it exists.
489    #[inline]
490    pub fn maybe_len(&self) -> Option<usize> {
491        match self {
492            Self::NonNullable | Self::AllValid | Self::AllInvalid => None,
493            Self::Array(a) => Some(a.len()),
494        }
495    }
496}
497
498impl From<BitBuffer> for Validity {
499    #[inline]
500    fn from(value: BitBuffer) -> Self {
501        let true_count = value.true_count();
502        if true_count == value.len() {
503            Self::AllValid
504        } else if true_count == 0 {
505            Self::AllInvalid
506        } else {
507            Self::Array(BoolArray::from(value).into_array())
508        }
509    }
510}
511
512impl FromIterator<Mask> for Validity {
513    #[inline]
514    fn from_iter<T: IntoIterator<Item = Mask>>(iter: T) -> Self {
515        Validity::from_mask(iter.into_iter().collect(), Nullability::Nullable)
516    }
517}
518
519impl FromIterator<bool> for Validity {
520    #[inline]
521    fn from_iter<T: IntoIterator<Item = bool>>(iter: T) -> Self {
522        Validity::from(BitBuffer::from_iter(iter))
523    }
524}
525
526impl From<Nullability> for Validity {
527    #[inline]
528    fn from(value: Nullability) -> Self {
529        Validity::from(&value)
530    }
531}
532
533impl From<&Nullability> for Validity {
534    #[inline]
535    fn from(value: &Nullability) -> Self {
536        match *value {
537            Nullability::NonNullable => Validity::NonNullable,
538            Nullability::Nullable => Validity::AllValid,
539        }
540    }
541}
542
543impl Validity {
544    /// Concatenate one or more validities together.
545    ///
546    /// Returns None if the vector is empty.
547    pub fn concat(validities: Vec<(Validity, usize)>) -> Option<Self> {
548        let mut validity_kinds = validities
549            .iter()
550            .map(|(v, _)| std::mem::discriminant(v))
551            .unique();
552        let validity_kind = validity_kinds.next()?;
553        if validity_kinds.next().is_none() {
554            // If there is only one kind of validity and its not Validity::Array, avoid constructing
555            // a Validity::Array.
556            if validity_kind == std::mem::discriminant(&Validity::AllValid) {
557                return Some(Validity::AllValid);
558            }
559            if validity_kind == std::mem::discriminant(&Validity::AllInvalid) {
560                return Some(Validity::AllInvalid);
561            }
562            if validity_kind == std::mem::discriminant(&Validity::NonNullable) {
563                return Some(Validity::NonNullable);
564            }
565        }
566
567        Some(Validity::Array(
568            unsafe {
569                ChunkedArray::new_unchecked(
570                    validities.into_iter().map(|(v, len)| v.to_array(len)),
571                    DType::Bool(Nullability::NonNullable),
572                )
573            }
574            .into_array(),
575        ))
576    }
577}
578
579impl Validity {
580    pub fn from_bit_buffer(buffer: BitBuffer, nullability: Nullability) -> Self {
581        if buffer.true_count() == buffer.len() {
582            nullability.into()
583        } else if buffer.true_count() == 0 {
584            Validity::AllInvalid
585        } else {
586            Validity::Array(BoolArray::new(buffer, Validity::NonNullable).into_array())
587        }
588    }
589
590    pub fn from_mask(mask: Mask, nullability: Nullability) -> Self {
591        assert!(
592            nullability == Nullability::Nullable || matches!(mask, Mask::AllTrue(_)),
593            "NonNullable validity must be AllValid",
594        );
595        match mask {
596            Mask::AllTrue(_) => match nullability {
597                Nullability::NonNullable => Validity::NonNullable,
598                Nullability::Nullable => Validity::AllValid,
599            },
600            Mask::AllFalse(_) => Validity::AllInvalid,
601            Mask::Values(values) => Validity::Array(values.into_array()),
602        }
603    }
604}
605
606impl IntoArray for Mask {
607    #[inline]
608    fn into_array(self) -> ArrayRef {
609        match self {
610            Self::AllTrue(len) => ConstantArray::new(true, len).into_array(),
611            Self::AllFalse(len) => ConstantArray::new(false, len).into_array(),
612            Self::Values(a) => a.into_array(),
613        }
614    }
615}
616
617impl IntoArray for &MaskValues {
618    #[inline]
619    fn into_array(self) -> ArrayRef {
620        BoolArray::new(self.bit_buffer().clone(), Validity::NonNullable).into_array()
621    }
622}
623
624#[cfg(test)]
625mod tests {
626    use std::sync::LazyLock;
627
628    use rstest::rstest;
629    use vortex_buffer::Buffer;
630    use vortex_buffer::buffer;
631    use vortex_mask::Mask;
632    use vortex_session::VortexSession;
633
634    use crate::ArrayRef;
635    use crate::IntoArray;
636    use crate::VortexSessionExecute;
637    use crate::arrays::PrimitiveArray;
638    use crate::dtype::Nullability;
639    use crate::validity::BoolArray;
640    use crate::validity::Validity;
641
642    static SESSION: LazyLock<VortexSession> = LazyLock::new(VortexSession::empty);
643
644    #[rstest]
645    #[case(Validity::AllValid, 5, &[2, 4], Validity::AllValid, Validity::AllValid)]
646    #[case(
647        Validity::AllValid,
648        5,
649        &[2, 4],
650        Validity::AllInvalid,
651        Validity::Array(BoolArray::from_iter([true, true, false, true, false]).into_array())
652    )]
653    #[case(
654        Validity::AllValid,
655        5,
656        &[2, 4],
657        Validity::Array(BoolArray::from_iter([true, false]).into_array()),
658        Validity::Array(BoolArray::from_iter([true, true, true, true, false]).into_array())
659    )]
660    #[case(
661        Validity::AllInvalid,
662        5,
663        &[2, 4],
664        Validity::AllValid,
665        Validity::Array(BoolArray::from_iter([false, false, true, false, true]).into_array())
666    )]
667    #[case(Validity::AllInvalid, 5, &[2, 4], Validity::AllInvalid, Validity::AllInvalid)]
668    #[case(
669        Validity::AllInvalid,
670        5,
671        &[2, 4],
672        Validity::Array(BoolArray::from_iter([true, false]).into_array()),
673        Validity::Array(BoolArray::from_iter([false, false, true, false, false]).into_array())
674    )]
675    #[case(
676        Validity::Array(BoolArray::from_iter([false, true, false, true, false]).into_array()),
677        5,
678        &[2, 4],
679        Validity::AllValid,
680        Validity::Array(BoolArray::from_iter([false, true, true, true, true]).into_array())
681    )]
682    #[case(
683        Validity::Array(BoolArray::from_iter([false, true, false, true, false]).into_array()),
684        5,
685        &[2, 4],
686        Validity::AllInvalid,
687        Validity::Array(BoolArray::from_iter([false, true, false, true, false]).into_array())
688    )]
689    #[case(
690        Validity::Array(BoolArray::from_iter([false, true, false, true, false]).into_array()),
691        5,
692        &[2, 4],
693        Validity::Array(BoolArray::from_iter([true, false]).into_array()),
694        Validity::Array(BoolArray::from_iter([false, true, true, true, false]).into_array())
695    )]
696
697    fn patch_validity(
698        #[case] validity: Validity,
699        #[case] len: usize,
700        #[case] positions: &[u64],
701        #[case] patches: Validity,
702        #[case] expected: Validity,
703    ) {
704        let indices =
705            PrimitiveArray::new(Buffer::copy_from(positions), Validity::NonNullable).into_array();
706
707        let mut ctx = SESSION.create_execution_ctx();
708
709        assert!(
710            validity
711                .patch(len, 0, &indices, &patches, &mut ctx,)
712                .unwrap()
713                .mask_eq(&expected, len, &mut ctx)
714                .unwrap()
715        );
716    }
717
718    #[rstest]
719    #[case(
720        Validity::NonNullable,
721        1,
722        "Can't patch a non-nullable validity with nullable validity"
723    )]
724    #[case(
725        Validity::AllValid,
726        4,
727        "Patch indices 4, offset 0 are longer than the array length 2"
728    )]
729    fn rejects_invalid_patch(
730        #[case] validity: Validity,
731        #[case] index: u64,
732        #[case] message: &str,
733    ) {
734        let mut ctx = SESSION.create_execution_ctx();
735        let error = validity
736            .patch(
737                2,
738                0,
739                &buffer![index].into_array(),
740                &Validity::AllInvalid,
741                &mut ctx,
742            )
743            .unwrap_err();
744        assert!(error.to_string().contains(message), "{error}");
745    }
746
747    #[test]
748    #[should_panic(expected = "NonNullable validity must be AllValid")]
749    fn into_validity_nullable() {
750        Validity::from_mask(Mask::AllFalse(10), Nullability::NonNullable);
751    }
752
753    #[test]
754    #[should_panic(expected = "NonNullable validity must be AllValid")]
755    fn into_validity_nullable_array() {
756        Validity::from_mask(Mask::from_iter(vec![true, false]), Nullability::NonNullable);
757    }
758
759    #[rstest]
760    #[case(
761        Validity::AllValid,
762        PrimitiveArray::new(buffer![0, 1], Validity::from_iter(vec![true, false])).into_array(),
763        Validity::from_iter(vec![true, false])
764    )]
765    #[case(Validity::AllValid, buffer![0, 1].into_array(), Validity::AllValid)]
766    #[case(
767        Validity::AllValid,
768        PrimitiveArray::new(buffer![0, 1], Validity::AllInvalid).into_array(),
769        Validity::AllInvalid
770    )]
771    #[case(
772        Validity::NonNullable,
773        PrimitiveArray::new(buffer![0, 1], Validity::from_iter(vec![true, false])).into_array(),
774        Validity::from_iter(vec![true, false])
775    )]
776    #[case(Validity::NonNullable, buffer![0, 1].into_array(), Validity::NonNullable)]
777    #[case(
778        Validity::NonNullable,
779        PrimitiveArray::new(buffer![0, 1], Validity::AllInvalid).into_array(),
780        Validity::AllInvalid
781    )]
782    fn validity_take(
783        #[case] validity: Validity,
784        #[case] indices: ArrayRef,
785        #[case] expected: Validity,
786    ) {
787        let mut ctx = SESSION.create_execution_ctx();
788        let taken = validity.take(&indices).unwrap();
789        assert_eq!(taken.nullability(), expected.nullability());
790        assert!(taken.mask_eq(&expected, indices.len(), &mut ctx).unwrap());
791    }
792
793    #[rstest]
794    // Mixed constant variants with equal masks.
795    #[case(Validity::NonNullable, Validity::AllValid, true)]
796    #[case(Validity::AllValid, Validity::NonNullable, true)]
797    #[case(Validity::AllValid, Validity::AllInvalid, false)]
798    #[case(Validity::NonNullable, Validity::AllInvalid, false)]
799    // An array that resolves to a constant mask must equal the constant variant.
800    #[case(
801        Validity::Array(BoolArray::from_iter([true, true, true]).into_array()),
802        Validity::AllValid,
803        true
804    )]
805    #[case(
806        Validity::NonNullable,
807        Validity::Array(BoolArray::from_iter([true, true, true]).into_array()),
808        true
809    )]
810    #[case(
811        Validity::Array(BoolArray::from_iter([false, false, false]).into_array()),
812        Validity::AllInvalid,
813        true
814    )]
815    #[case(
816        Validity::Array(BoolArray::from_iter([true, false, true]).into_array()),
817        Validity::AllValid,
818        false
819    )]
820    #[case(
821        Validity::Array(BoolArray::from_iter([true, false, true]).into_array()),
822        Validity::AllInvalid,
823        false
824    )]
825    fn mask_eq_mixed_variants(
826        #[case] lhs: Validity,
827        #[case] rhs: Validity,
828        #[case] expected: bool,
829    ) -> vortex_error::VortexResult<()> {
830        let mut ctx = SESSION.create_execution_ctx();
831        assert_eq!(lhs.mask_eq(&rhs, 3, &mut ctx)?, expected);
832        Ok(())
833    }
834}