1use std::fmt::Debug;
12use std::ops::Range;
13
14use itertools::Itertools as _;
15use vortex_buffer::BitBuffer;
16use vortex_error::VortexExpect as _;
17use vortex_error::VortexResult;
18use vortex_error::vortex_bail;
19use vortex_error::vortex_err;
20use vortex_mask::Mask;
21use vortex_mask::MaskValues;
22
23use crate::ArrayRef;
24use crate::Canonical;
25use crate::ExecutionCtx;
26use crate::IntoArray;
27use crate::arrays::BoolArray;
28use crate::arrays::ChunkedArray;
29use crate::arrays::ConstantArray;
30use crate::builtins::ArrayBuiltins;
31use crate::dtype::DType;
32use crate::dtype::Nullability;
33use crate::optimizer::ArrayOptimizer;
34use crate::patches::Patches;
35use crate::scalar::Scalar;
36use crate::scalar_fn::fns::binary::Binary;
37use crate::scalar_fn::fns::operators::Operator;
38
39#[derive(Clone)]
41pub enum Validity {
42 NonNullable,
44 AllValid,
46 AllInvalid,
48 Array(ArrayRef),
52}
53
54impl Debug for Validity {
55 fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
56 match self {
57 Self::NonNullable => write!(f, "NonNullable"),
58 Self::AllValid => write!(f, "AllValid"),
59 Self::AllInvalid => write!(f, "AllInvalid"),
60 Self::Array(arr) => write!(f, "SomeValid({arr})"),
61 }
62 }
63}
64
65impl Validity {
66 pub fn execute(self, ctx: &mut ExecutionCtx) -> VortexResult<Validity> {
68 match self {
69 v @ Validity::NonNullable | v @ Validity::AllValid | v @ Validity::AllInvalid => Ok(v),
70 Validity::Array(a) => Ok(Validity::Array(a.execute::<Canonical>(ctx)?.into_array())),
71 }
72 }
73}
74
75impl Validity {
76 pub const DTYPE: DType = DType::Bool(Nullability::NonNullable);
78
79 pub fn to_array(&self, len: usize) -> ArrayRef {
81 match self {
82 Self::NonNullable | Self::AllValid => ConstantArray::new(true, len).into_array(),
83 Self::AllInvalid => ConstantArray::new(false, len).into_array(),
84 Self::Array(a) => a.clone(),
85 }
86 }
87
88 #[inline]
90 pub fn into_array(self) -> Option<ArrayRef> {
91 if let Self::Array(a) = self {
92 Some(a)
93 } else {
94 None
95 }
96 }
97
98 #[inline]
100 pub fn as_array(&self) -> Option<&ArrayRef> {
101 if let Self::Array(a) = self {
102 Some(a)
103 } else {
104 None
105 }
106 }
107
108 #[inline]
109 pub fn nullability(&self) -> Nullability {
110 if matches!(self, Self::NonNullable) {
111 Nullability::NonNullable
112 } else {
113 Nullability::Nullable
114 }
115 }
116
117 #[inline]
124 pub fn definitely_no_nulls(&self) -> bool {
125 matches!(self, Self::NonNullable | Self::AllValid)
126 }
127
128 #[inline]
137 pub fn definitely_all_null(&self) -> bool {
138 matches!(self, Self::AllInvalid)
139 }
140
141 pub fn execute_no_nulls(&self, length: usize, ctx: &mut ExecutionCtx) -> VortexResult<bool> {
147 match self {
148 Self::NonNullable | Self::AllValid => Ok(true),
149 Self::AllInvalid => Ok(length == 0),
150 Self::Array(_) => Ok(self.execute_mask(length, ctx)?.all_true()),
151 }
152 }
153
154 #[inline]
156 pub fn union_nullability(self, nullability: Nullability) -> Self {
157 match nullability {
158 Nullability::NonNullable => self,
159 Nullability::Nullable => self.into_nullable(),
160 }
161 }
162
163 #[inline]
166 pub fn execute_is_valid(&self, index: usize, ctx: &mut ExecutionCtx) -> VortexResult<bool> {
167 Ok(match self {
168 Self::NonNullable | Self::AllValid => true,
169 Self::AllInvalid => false,
170 Self::Array(a) => a
171 .execute_scalar(index, ctx)?
172 .as_bool()
173 .value()
174 .ok_or_else(|| vortex_err!("validity value at index {index} is null"))?,
175 })
176 }
177
178 #[inline]
181 pub fn execute_is_null(&self, index: usize, ctx: &mut ExecutionCtx) -> VortexResult<bool> {
182 Ok(!self.execute_is_valid(index, ctx)?)
183 }
184
185 #[inline]
186 pub fn slice(&self, range: Range<usize>) -> VortexResult<Self> {
187 match self {
188 Self::Array(a) => Ok(Self::Array(a.slice(range)?)),
189 Self::NonNullable | Self::AllValid | Self::AllInvalid => Ok(self.clone()),
190 }
191 }
192
193 pub fn take(&self, indices: &ArrayRef) -> VortexResult<Self> {
194 match self {
195 Self::NonNullable => indices.validity(),
196 Self::AllValid => Ok(match indices.validity()? {
197 Self::NonNullable => Self::AllValid,
198 v => v,
199 }),
200 Self::AllInvalid => Ok(Self::AllInvalid),
201 Self::Array(is_valid) => {
202 let maybe_is_valid = is_valid.take(indices.clone())?;
203 let is_valid = maybe_is_valid.fill_null(Scalar::from(false))?;
205 Ok(Self::Array(is_valid))
206 }
207 }
208 }
209
210 pub fn not(&self) -> VortexResult<Self> {
212 match self {
213 Validity::NonNullable => Ok(Validity::NonNullable),
214 Validity::AllValid => Ok(Validity::AllInvalid),
215 Validity::AllInvalid => Ok(Validity::AllValid),
216 Validity::Array(arr) => Ok(Validity::Array(arr.not()?)),
217 }
218 }
219
220 pub fn filter(&self, mask: &Mask) -> VortexResult<Self> {
228 match self {
231 v @ (Validity::NonNullable | Validity::AllValid | Validity::AllInvalid) => {
232 Ok(v.clone())
233 }
234 Validity::Array(arr) => Ok(Validity::Array(arr.filter(mask.clone())?)),
235 }
236 }
237
238 #[deprecated(note = "Use execute_mask")]
242 pub fn to_mask(&self, length: usize, ctx: &mut ExecutionCtx) -> VortexResult<Mask> {
243 match self {
244 Self::NonNullable | Self::AllValid => Ok(Mask::new_true(length)),
245 Self::AllInvalid => Ok(Mask::new_false(length)),
246 Self::Array(arr) => arr.clone().execute::<Mask>(ctx),
247 }
248 }
249
250 #[inline]
256 pub fn execute_mask(&self, length: usize, ctx: &mut ExecutionCtx) -> VortexResult<Mask> {
257 match self {
258 Self::NonNullable | Self::AllValid => Ok(Mask::AllTrue(length)),
259 Self::AllInvalid => Ok(Mask::AllFalse(length)),
260 Self::Array(arr) => {
261 assert_eq!(
262 arr.len(),
263 length,
264 "Validity::Array length must equal to_logical's argument: {}, {}.",
265 arr.len(),
266 length,
267 );
268 arr.clone().execute::<Mask>(ctx)
271 }
272 }
273 }
274
275 pub fn mask_eq(
278 &self,
279 other: &Validity,
280 length: usize,
281 ctx: &mut ExecutionCtx,
282 ) -> VortexResult<bool> {
283 match (self, other) {
284 (
286 Validity::NonNullable | Validity::AllValid,
287 Validity::NonNullable | Validity::AllValid,
288 )
289 | (Validity::AllInvalid, Validity::AllInvalid) => Ok(true),
290 _ => Ok(self.execute_mask(length, ctx)? == other.execute_mask(length, ctx)?),
291 }
292 }
293
294 #[inline]
296 pub fn and(self, rhs: Validity) -> VortexResult<Validity> {
297 Ok(match (self, rhs) {
298 (Validity::NonNullable, Validity::NonNullable) => Validity::NonNullable,
300 (Validity::AllInvalid, _) | (_, Validity::AllInvalid) => Validity::AllInvalid,
302 (Validity::Array(a), Validity::AllValid)
304 | (Validity::Array(a), Validity::NonNullable)
305 | (Validity::NonNullable, Validity::Array(a))
306 | (Validity::AllValid, Validity::Array(a)) => Validity::Array(a),
307 (Validity::NonNullable, Validity::AllValid)
309 | (Validity::AllValid, Validity::NonNullable)
310 | (Validity::AllValid, Validity::AllValid) => Validity::AllValid,
311 (Validity::Array(lhs), Validity::Array(rhs)) => Validity::Array(
313 Binary::try_new(lhs, rhs, Operator::And)?
314 .into_array()
315 .optimize()?,
316 ),
317 })
318 }
319
320 pub fn patch(
321 self,
322 len: usize,
323 indices_offset: usize,
324 indices: &ArrayRef,
325 patches: &Validity,
326 ctx: &mut ExecutionCtx,
327 ) -> VortexResult<Self> {
328 match (&self, patches) {
329 (Validity::NonNullable, Validity::NonNullable) => return Ok(Validity::NonNullable),
330 (Validity::NonNullable, _) => {
331 vortex_bail!("Can't patch a non-nullable validity with nullable validity")
332 }
333 (_, Validity::NonNullable) => {
334 vortex_bail!("Can't patch a nullable validity with non-nullable validity")
335 }
336 (Validity::AllValid, Validity::AllValid) => return Ok(Validity::AllValid),
337 (Validity::AllInvalid, Validity::AllInvalid) => return Ok(Validity::AllInvalid),
338 _ => {}
339 };
340
341 if matches!(self, Validity::NonNullable) {
342 return Ok(Self::NonNullable);
343 }
344
345 let source = match self {
347 Validity::NonNullable => BoolArray::from(BitBuffer::new_set(len)),
348 Validity::AllValid => BoolArray::from(BitBuffer::new_set(len)),
349 Validity::AllInvalid => BoolArray::from(BitBuffer::new_unset(len)),
350 Validity::Array(a) => a.execute::<BoolArray>(ctx)?,
351 };
352
353 let patch_values = match patches {
354 Validity::NonNullable => BoolArray::from(BitBuffer::new_set(indices.len())),
355 Validity::AllValid => BoolArray::from(BitBuffer::new_set(indices.len())),
356 Validity::AllInvalid => BoolArray::from(BitBuffer::new_unset(indices.len())),
357 Validity::Array(a) => a.clone().execute::<BoolArray>(ctx)?,
358 };
359
360 let patches = Patches::new(
361 len,
362 indices_offset,
363 indices.clone(),
364 patch_values.into_array(),
365 None,
367 ctx,
368 )?;
369
370 Ok(Self::Array(source.patch(&patches, ctx)?.into_array()))
371 }
372
373 #[inline]
375 pub fn into_nullable(self) -> Validity {
376 match self {
377 Self::NonNullable => Self::AllValid,
378 Self::AllValid | Self::AllInvalid | Self::Array(_) => self,
379 }
380 }
381
382 #[inline]
388 pub fn into_non_nullable(self, len: usize, ctx: &mut ExecutionCtx) -> Option<Validity> {
389 match self {
390 _ if len == 0 => Some(Validity::NonNullable),
391 Self::NonNullable => Some(Self::NonNullable),
392 Self::AllValid => Some(Self::NonNullable),
393 Self::AllInvalid => None,
394 Self::Array(is_valid) => {
395 is_valid
396 .statistics()
397 .compute_min::<bool>(ctx)
398 .vortex_expect("validity array must support min")
399 .then(|| {
400 Self::NonNullable
402 })
403 }
404 }
405 }
406
407 #[inline]
418 pub fn trivial_into_non_nullable(self, len: usize) -> VortexResult<Option<Validity>> {
419 match self {
420 _ if len == 0 => Ok(Some(Validity::NonNullable)),
421 Self::NonNullable => Ok(Some(Self::NonNullable)),
422 Self::AllValid => Ok(Some(Self::NonNullable)),
423 Self::AllInvalid => Err(vortex_err!("Cannot cast AllInvalid to NonNullable")),
424 Self::Array(_) => Ok(None),
425 }
426 }
427
428 #[inline]
439 pub fn cast_nullability(
440 self,
441 nullability: Nullability,
442 len: usize,
443 ctx: &mut ExecutionCtx,
444 ) -> VortexResult<Validity> {
445 match nullability {
446 Nullability::NonNullable => self.into_non_nullable(len, ctx).ok_or_else(|| {
447 vortex_err!("Cannot cast array with invalid values to non-nullable type.")
448 }),
449 Nullability::Nullable => Ok(self.into_nullable()),
450 }
451 }
452
453 #[inline]
477 pub fn trivially_cast_nullability(
478 self,
479 nullability: Nullability,
480 len: usize,
481 ) -> VortexResult<Option<Validity>> {
482 match nullability {
483 Nullability::NonNullable => self.trivial_into_non_nullable(len),
484 Nullability::Nullable => Ok(Some(self.into_nullable())),
485 }
486 }
487
488 #[inline]
490 pub fn maybe_len(&self) -> Option<usize> {
491 match self {
492 Self::NonNullable | Self::AllValid | Self::AllInvalid => None,
493 Self::Array(a) => Some(a.len()),
494 }
495 }
496}
497
498impl From<BitBuffer> for Validity {
499 #[inline]
500 fn from(value: BitBuffer) -> Self {
501 let true_count = value.true_count();
502 if true_count == value.len() {
503 Self::AllValid
504 } else if true_count == 0 {
505 Self::AllInvalid
506 } else {
507 Self::Array(BoolArray::from(value).into_array())
508 }
509 }
510}
511
512impl FromIterator<Mask> for Validity {
513 #[inline]
514 fn from_iter<T: IntoIterator<Item = Mask>>(iter: T) -> Self {
515 Validity::from_mask(iter.into_iter().collect(), Nullability::Nullable)
516 }
517}
518
519impl FromIterator<bool> for Validity {
520 #[inline]
521 fn from_iter<T: IntoIterator<Item = bool>>(iter: T) -> Self {
522 Validity::from(BitBuffer::from_iter(iter))
523 }
524}
525
526impl From<Nullability> for Validity {
527 #[inline]
528 fn from(value: Nullability) -> Self {
529 Validity::from(&value)
530 }
531}
532
533impl From<&Nullability> for Validity {
534 #[inline]
535 fn from(value: &Nullability) -> Self {
536 match *value {
537 Nullability::NonNullable => Validity::NonNullable,
538 Nullability::Nullable => Validity::AllValid,
539 }
540 }
541}
542
543impl Validity {
544 pub fn concat(validities: Vec<(Validity, usize)>) -> Option<Self> {
548 let mut validity_kinds = validities
549 .iter()
550 .map(|(v, _)| std::mem::discriminant(v))
551 .unique();
552 let validity_kind = validity_kinds.next()?;
553 if validity_kinds.next().is_none() {
554 if validity_kind == std::mem::discriminant(&Validity::AllValid) {
557 return Some(Validity::AllValid);
558 }
559 if validity_kind == std::mem::discriminant(&Validity::AllInvalid) {
560 return Some(Validity::AllInvalid);
561 }
562 if validity_kind == std::mem::discriminant(&Validity::NonNullable) {
563 return Some(Validity::NonNullable);
564 }
565 }
566
567 Some(Validity::Array(
568 unsafe {
569 ChunkedArray::new_unchecked(
570 validities.into_iter().map(|(v, len)| v.to_array(len)),
571 DType::Bool(Nullability::NonNullable),
572 )
573 }
574 .into_array(),
575 ))
576 }
577}
578
579impl Validity {
580 pub fn from_bit_buffer(buffer: BitBuffer, nullability: Nullability) -> Self {
581 if buffer.true_count() == buffer.len() {
582 nullability.into()
583 } else if buffer.true_count() == 0 {
584 Validity::AllInvalid
585 } else {
586 Validity::Array(BoolArray::new(buffer, Validity::NonNullable).into_array())
587 }
588 }
589
590 pub fn from_mask(mask: Mask, nullability: Nullability) -> Self {
591 assert!(
592 nullability == Nullability::Nullable || matches!(mask, Mask::AllTrue(_)),
593 "NonNullable validity must be AllValid",
594 );
595 match mask {
596 Mask::AllTrue(_) => match nullability {
597 Nullability::NonNullable => Validity::NonNullable,
598 Nullability::Nullable => Validity::AllValid,
599 },
600 Mask::AllFalse(_) => Validity::AllInvalid,
601 Mask::Values(values) => Validity::Array(values.into_array()),
602 }
603 }
604}
605
606impl IntoArray for Mask {
607 #[inline]
608 fn into_array(self) -> ArrayRef {
609 match self {
610 Self::AllTrue(len) => ConstantArray::new(true, len).into_array(),
611 Self::AllFalse(len) => ConstantArray::new(false, len).into_array(),
612 Self::Values(a) => a.into_array(),
613 }
614 }
615}
616
617impl IntoArray for &MaskValues {
618 #[inline]
619 fn into_array(self) -> ArrayRef {
620 BoolArray::new(self.bit_buffer().clone(), Validity::NonNullable).into_array()
621 }
622}
623
624#[cfg(test)]
625mod tests {
626 use std::sync::LazyLock;
627
628 use rstest::rstest;
629 use vortex_buffer::Buffer;
630 use vortex_buffer::buffer;
631 use vortex_mask::Mask;
632 use vortex_session::VortexSession;
633
634 use crate::ArrayRef;
635 use crate::IntoArray;
636 use crate::VortexSessionExecute;
637 use crate::arrays::PrimitiveArray;
638 use crate::dtype::Nullability;
639 use crate::validity::BoolArray;
640 use crate::validity::Validity;
641
642 static SESSION: LazyLock<VortexSession> = LazyLock::new(VortexSession::empty);
643
644 #[rstest]
645 #[case(Validity::AllValid, 5, &[2, 4], Validity::AllValid, Validity::AllValid)]
646 #[case(
647 Validity::AllValid,
648 5,
649 &[2, 4],
650 Validity::AllInvalid,
651 Validity::Array(BoolArray::from_iter([true, true, false, true, false]).into_array())
652 )]
653 #[case(
654 Validity::AllValid,
655 5,
656 &[2, 4],
657 Validity::Array(BoolArray::from_iter([true, false]).into_array()),
658 Validity::Array(BoolArray::from_iter([true, true, true, true, false]).into_array())
659 )]
660 #[case(
661 Validity::AllInvalid,
662 5,
663 &[2, 4],
664 Validity::AllValid,
665 Validity::Array(BoolArray::from_iter([false, false, true, false, true]).into_array())
666 )]
667 #[case(Validity::AllInvalid, 5, &[2, 4], Validity::AllInvalid, Validity::AllInvalid)]
668 #[case(
669 Validity::AllInvalid,
670 5,
671 &[2, 4],
672 Validity::Array(BoolArray::from_iter([true, false]).into_array()),
673 Validity::Array(BoolArray::from_iter([false, false, true, false, false]).into_array())
674 )]
675 #[case(
676 Validity::Array(BoolArray::from_iter([false, true, false, true, false]).into_array()),
677 5,
678 &[2, 4],
679 Validity::AllValid,
680 Validity::Array(BoolArray::from_iter([false, true, true, true, true]).into_array())
681 )]
682 #[case(
683 Validity::Array(BoolArray::from_iter([false, true, false, true, false]).into_array()),
684 5,
685 &[2, 4],
686 Validity::AllInvalid,
687 Validity::Array(BoolArray::from_iter([false, true, false, true, false]).into_array())
688 )]
689 #[case(
690 Validity::Array(BoolArray::from_iter([false, true, false, true, false]).into_array()),
691 5,
692 &[2, 4],
693 Validity::Array(BoolArray::from_iter([true, false]).into_array()),
694 Validity::Array(BoolArray::from_iter([false, true, true, true, false]).into_array())
695 )]
696
697 fn patch_validity(
698 #[case] validity: Validity,
699 #[case] len: usize,
700 #[case] positions: &[u64],
701 #[case] patches: Validity,
702 #[case] expected: Validity,
703 ) {
704 let indices =
705 PrimitiveArray::new(Buffer::copy_from(positions), Validity::NonNullable).into_array();
706
707 let mut ctx = SESSION.create_execution_ctx();
708
709 assert!(
710 validity
711 .patch(len, 0, &indices, &patches, &mut ctx,)
712 .unwrap()
713 .mask_eq(&expected, len, &mut ctx)
714 .unwrap()
715 );
716 }
717
718 #[rstest]
719 #[case(
720 Validity::NonNullable,
721 1,
722 "Can't patch a non-nullable validity with nullable validity"
723 )]
724 #[case(
725 Validity::AllValid,
726 4,
727 "Patch indices 4, offset 0 are longer than the array length 2"
728 )]
729 fn rejects_invalid_patch(
730 #[case] validity: Validity,
731 #[case] index: u64,
732 #[case] message: &str,
733 ) {
734 let mut ctx = SESSION.create_execution_ctx();
735 let error = validity
736 .patch(
737 2,
738 0,
739 &buffer![index].into_array(),
740 &Validity::AllInvalid,
741 &mut ctx,
742 )
743 .unwrap_err();
744 assert!(error.to_string().contains(message), "{error}");
745 }
746
747 #[test]
748 #[should_panic(expected = "NonNullable validity must be AllValid")]
749 fn into_validity_nullable() {
750 Validity::from_mask(Mask::AllFalse(10), Nullability::NonNullable);
751 }
752
753 #[test]
754 #[should_panic(expected = "NonNullable validity must be AllValid")]
755 fn into_validity_nullable_array() {
756 Validity::from_mask(Mask::from_iter(vec![true, false]), Nullability::NonNullable);
757 }
758
759 #[rstest]
760 #[case(
761 Validity::AllValid,
762 PrimitiveArray::new(buffer![0, 1], Validity::from_iter(vec![true, false])).into_array(),
763 Validity::from_iter(vec![true, false])
764 )]
765 #[case(Validity::AllValid, buffer![0, 1].into_array(), Validity::AllValid)]
766 #[case(
767 Validity::AllValid,
768 PrimitiveArray::new(buffer![0, 1], Validity::AllInvalid).into_array(),
769 Validity::AllInvalid
770 )]
771 #[case(
772 Validity::NonNullable,
773 PrimitiveArray::new(buffer![0, 1], Validity::from_iter(vec![true, false])).into_array(),
774 Validity::from_iter(vec![true, false])
775 )]
776 #[case(Validity::NonNullable, buffer![0, 1].into_array(), Validity::NonNullable)]
777 #[case(
778 Validity::NonNullable,
779 PrimitiveArray::new(buffer![0, 1], Validity::AllInvalid).into_array(),
780 Validity::AllInvalid
781 )]
782 fn validity_take(
783 #[case] validity: Validity,
784 #[case] indices: ArrayRef,
785 #[case] expected: Validity,
786 ) {
787 let mut ctx = SESSION.create_execution_ctx();
788 let taken = validity.take(&indices).unwrap();
789 assert_eq!(taken.nullability(), expected.nullability());
790 assert!(taken.mask_eq(&expected, indices.len(), &mut ctx).unwrap());
791 }
792
793 #[rstest]
794 #[case(Validity::NonNullable, Validity::AllValid, true)]
796 #[case(Validity::AllValid, Validity::NonNullable, true)]
797 #[case(Validity::AllValid, Validity::AllInvalid, false)]
798 #[case(Validity::NonNullable, Validity::AllInvalid, false)]
799 #[case(
801 Validity::Array(BoolArray::from_iter([true, true, true]).into_array()),
802 Validity::AllValid,
803 true
804 )]
805 #[case(
806 Validity::NonNullable,
807 Validity::Array(BoolArray::from_iter([true, true, true]).into_array()),
808 true
809 )]
810 #[case(
811 Validity::Array(BoolArray::from_iter([false, false, false]).into_array()),
812 Validity::AllInvalid,
813 true
814 )]
815 #[case(
816 Validity::Array(BoolArray::from_iter([true, false, true]).into_array()),
817 Validity::AllValid,
818 false
819 )]
820 #[case(
821 Validity::Array(BoolArray::from_iter([true, false, true]).into_array()),
822 Validity::AllInvalid,
823 false
824 )]
825 fn mask_eq_mixed_variants(
826 #[case] lhs: Validity,
827 #[case] rhs: Validity,
828 #[case] expected: bool,
829 ) -> vortex_error::VortexResult<()> {
830 let mut ctx = SESSION.create_execution_ctx();
831 assert_eq!(lhs.mask_eq(&rhs, 3, &mut ctx)?, expected);
832 Ok(())
833 }
834}