1use std::any::type_name;
5use std::fmt::Debug;
6
7use enum_iterator::all;
8use num_traits::CheckedAdd;
9use smallvec::SmallVec;
10use smallvec::smallvec;
11use vortex_error::VortexError;
12use vortex_error::VortexExpect;
13use vortex_error::VortexResult;
14use vortex_error::vortex_err;
15use vortex_error::vortex_panic;
16
17use crate::dtype::DType;
18use crate::expr::stats::IsConstant;
19use crate::expr::stats::IsSorted;
20use crate::expr::stats::IsStrictSorted;
21use crate::expr::stats::Max;
22use crate::expr::stats::Min;
23use crate::expr::stats::NaNCount;
24use crate::expr::stats::NullCount;
25use crate::expr::stats::Precision;
26use crate::expr::stats::Stat;
27use crate::expr::stats::StatBound;
28use crate::expr::stats::StatType;
29use crate::expr::stats::StatsProvider;
30use crate::expr::stats::StatsProviderExt;
31use crate::expr::stats::Sum;
32use crate::expr::stats::UncompressedSizeInBytes;
33use crate::scalar::Scalar;
34use crate::scalar::ScalarValue;
35
36pub type StatsArray = [(Stat, Precision<ScalarValue>); 4];
38
39#[derive(Default, Debug, Clone)]
40pub struct StatsSet {
41 values: SmallVec<StatsArray>,
42}
43
44impl StatsSet {
45 pub unsafe fn new_unchecked(values: SmallVec<StatsArray>) -> Self {
51 Self { values }
52 }
53
54 pub fn of(stat: Stat, value: Precision<ScalarValue>) -> Self {
56 Self {
57 values: smallvec![(stat, value)],
58 }
59 }
60
61 pub fn as_mut_typed_ref<'a, 'b>(&'a mut self, dtype: &'b DType) -> MutTypedStatsSetRef<'a, 'b> {
63 MutTypedStatsSetRef {
64 values: self,
65 dtype,
66 }
67 }
68
69 pub fn as_typed_ref<'a, 'b>(&'a self, dtype: &'b DType) -> TypedStatsSetRef<'a, 'b> {
71 TypedStatsSetRef {
72 values: self,
73 dtype,
74 }
75 }
76}
77
78impl StatsSet {
80 pub fn set(&mut self, stat: Stat, value: Precision<ScalarValue>) {
82 if let Some(existing) = self.values.iter_mut().find(|(s, _)| *s == stat) {
83 *existing = (stat, value);
84 } else {
85 self.values.push((stat, value));
86 }
87 }
88
89 pub fn clear(&mut self, stat: Stat) {
91 self.values.retain(|(s, _)| *s != stat);
92 }
93
94 pub fn retain_only(&mut self, stats: &[Stat]) {
96 self.values.retain(|(s, _)| stats.contains(s));
97 }
98
99 pub fn iter(&self) -> impl Iterator<Item = &(Stat, Precision<ScalarValue>)> {
103 self.values.iter()
104 }
105
106 pub fn get(&self, stat: Stat) -> Precision<ScalarValue> {
108 self.get_ref(stat).map(ScalarValue::clone)
109 }
110
111 fn get_ref(&self, stat: Stat) -> Precision<&ScalarValue> {
113 self.values
114 .iter()
115 .find(|(s, _)| *s == stat)
116 .map(|(_, v)| v.as_ref())
117 .unwrap_or(Precision::Absent)
118 }
119
120 pub(super) fn get_usize(&self, stat: Stat) -> Precision<usize> {
122 self.get_ref(stat).map(|v| {
123 usize::try_from(v).unwrap_or_else(|err| {
124 vortex_panic!(
125 err,
126 "Failed to get stat {} as {}",
127 stat,
128 type_name::<usize>()
129 )
130 })
131 })
132 }
133
134 pub(super) fn get_bool(&self, stat: Stat) -> Precision<bool> {
137 self.get_ref(stat).map(|v| match v {
138 ScalarValue::Bool(b) => *b,
139 _ => vortex_panic!("Failed to get stat {} as bool, found {}", stat, v),
140 })
141 }
142
143 pub fn len(&self) -> usize {
145 self.values.len()
146 }
147
148 pub fn is_empty(&self) -> bool {
150 self.values.is_empty()
151 }
152
153 pub fn get_as<T: for<'a> TryFrom<&'a Scalar, Error = VortexError>>(
155 &self,
156 stat: Stat,
157 dtype: &DType,
158 ) -> Precision<T> {
159 self.get(stat).map(|v| {
160 T::try_from(
161 &Scalar::try_new(dtype.clone(), Some(v))
162 .vortex_expect("failed to construct a scalar statistic"),
163 )
164 .unwrap_or_else(|err| {
165 vortex_panic!(err, "Failed to get stat {} as {}", stat, type_name::<T>())
166 })
167 })
168 }
169}
170
171pub struct StatsSetIntoIter(smallvec::IntoIter<StatsArray>);
177
178impl Iterator for StatsSetIntoIter {
179 type Item = (Stat, Precision<ScalarValue>);
180
181 fn next(&mut self) -> Option<Self::Item> {
182 self.0.next()
183 }
184}
185
186impl IntoIterator for StatsSet {
187 type Item = (Stat, Precision<ScalarValue>);
188 type IntoIter = StatsSetIntoIter;
189
190 fn into_iter(self) -> Self::IntoIter {
191 StatsSetIntoIter(self.values.into_iter())
192 }
193}
194
195impl FromIterator<(Stat, Precision<ScalarValue>)> for StatsSet {
196 fn from_iter<T: IntoIterator<Item = (Stat, Precision<ScalarValue>)>>(iter: T) -> Self {
197 let iter = iter.into_iter();
198
199 let mut this = Self {
200 values: SmallVec::new(),
201 };
202 this.extend(iter);
203 this
204 }
205}
206
207impl Extend<(Stat, Precision<ScalarValue>)> for StatsSet {
208 #[inline]
209 fn extend<T: IntoIterator<Item = (Stat, Precision<ScalarValue>)>>(&mut self, iter: T) {
210 iter.into_iter()
211 .for_each(|(stat, value)| self.set(stat, value));
212 }
213}
214
215impl StatsSet {
217 pub fn merge_ordered(mut self, other: &Self, dtype: &DType) -> Self {
220 self.as_mut_typed_ref(dtype)
221 .merge_ordered(&other.as_typed_ref(dtype));
222 self
223 }
224
225 pub fn merge_unordered(mut self, other: &Self, dtype: &DType) -> Self {
228 self.as_mut_typed_ref(dtype)
229 .merge_unordered(&other.as_typed_ref(dtype));
230 self
231 }
232
233 pub fn combine_sets(&mut self, other: &Self, dtype: &DType) -> VortexResult<()> {
235 self.as_mut_typed_ref(dtype)
236 .combine_sets(&other.as_typed_ref(dtype))
237 }
238}
239
240fn stat_scalar(values: &StatsSet, stat: Stat, dtype: &DType) -> Precision<Scalar> {
242 values.get_ref(stat).map(|value| {
243 let stat_dtype = stat
244 .dtype(dtype)
245 .vortex_expect("Must have valid dtype if value is present");
246 Scalar::validate(&stat_dtype, Some(value))
247 .vortex_expect("failed to construct a scalar statistic");
248
249 unsafe { Scalar::new_unchecked(stat_dtype, Some(value.clone())) }
251 })
252}
253
254pub struct TypedStatsSetRef<'a, 'b> {
255 pub values: &'a StatsSet,
256 pub dtype: &'b DType,
257}
258
259impl StatsProvider for TypedStatsSetRef<'_, '_> {
260 fn get(&self, stat: Stat) -> Precision<Scalar> {
261 stat_scalar(self.values, stat, self.dtype)
262 }
263
264 fn len(&self) -> usize {
265 self.values.len()
266 }
267}
268
269pub struct MutTypedStatsSetRef<'a, 'b> {
270 pub values: &'a mut StatsSet,
271 pub dtype: &'b DType,
272}
273
274impl MutTypedStatsSetRef<'_, '_> {
275 pub fn set(&mut self, stat: Stat, value: Precision<ScalarValue>) {
277 self.values.set(stat, value);
278 }
279
280 pub fn clear(&mut self, stat: Stat) {
282 self.values.clear(stat);
283 }
284}
285
286impl StatsProvider for MutTypedStatsSetRef<'_, '_> {
287 fn get(&self, stat: Stat) -> Precision<Scalar> {
288 stat_scalar(self.values, stat, self.dtype)
289 }
290
291 fn len(&self) -> usize {
292 self.values.len()
293 }
294}
295
296impl MutTypedStatsSetRef<'_, '_> {
298 pub fn merge_ordered(mut self, other: &TypedStatsSetRef) -> Self {
301 for s in all::<Stat>() {
302 match s {
303 Stat::IsConstant => self.merge_is_constant(other),
304 Stat::IsSorted => self.merge_is_sorted(other),
305 Stat::IsStrictSorted => self.merge_is_strict_sorted(other),
306 Stat::Max => self.merge_max(other),
307 Stat::Min => self.merge_min(other),
308 Stat::Sum => self.merge_sum(other),
309 Stat::NullCount => self.merge_null_count(other),
310 Stat::UncompressedSizeInBytes => self.merge_uncompressed_size_in_bytes(other),
311 Stat::NaNCount => self.merge_nan_count(other),
312 }
313 }
314
315 self
316 }
317
318 pub fn merge_unordered(mut self, other: &TypedStatsSetRef) -> Self {
321 for s in all::<Stat>() {
322 if !s.is_commutative() {
323 self.clear(s);
324 continue;
325 }
326
327 match s {
328 Stat::IsConstant => self.merge_is_constant(other),
329 Stat::Max => self.merge_max(other),
330 Stat::Min => self.merge_min(other),
331 Stat::Sum => self.merge_sum(other),
332 Stat::NullCount => self.merge_null_count(other),
333 Stat::UncompressedSizeInBytes => self.merge_uncompressed_size_in_bytes(other),
334 Stat::IsSorted | Stat::IsStrictSorted => {
335 unreachable!("not commutative")
336 }
337 Stat::NaNCount => self.merge_nan_count(other),
338 }
339 }
340
341 self
342 }
343
344 pub fn combine_sets(&mut self, other: &TypedStatsSetRef) -> VortexResult<()> {
346 let other_stats: Vec<_> = other.values.iter().map(|(stat, _)| *stat).collect();
347 for s in other_stats {
348 match s {
349 Stat::Max => self.combine_bound::<Max>(other)?,
350 Stat::Min => self.combine_bound::<Min>(other)?,
351 Stat::UncompressedSizeInBytes => {
352 self.combine_bound::<UncompressedSizeInBytes>(other)?
353 }
354 Stat::IsConstant => self.combine_bool_stat::<IsConstant>(other)?,
355 Stat::IsSorted => self.combine_bool_stat::<IsSorted>(other)?,
356 Stat::IsStrictSorted => self.combine_bool_stat::<IsStrictSorted>(other)?,
357 Stat::NullCount => self.combine_bound::<NullCount>(other)?,
358 Stat::Sum => self.combine_bound::<Sum>(other)?,
359 Stat::NaNCount => self.combine_bound::<NaNCount>(other)?,
360 }
361 }
362 Ok(())
363 }
364
365 fn combine_bound<S: StatType<Scalar>>(&mut self, other: &TypedStatsSetRef) -> VortexResult<()>
366 where
367 S::Bound: StatBound<Scalar> + Debug + Eq + PartialEq,
368 {
369 match (self.get_scalar_bound::<S>(), other.get_scalar_bound::<S>()) {
370 (Some(m1), Some(m2)) => {
371 let meet = m1
372 .intersection(&m2)
373 .vortex_expect("can always compare scalar")
374 .ok_or_else(|| {
375 vortex_err!("{:?} bounds ({m1:?}, {m2:?}) do not overlap", S::STAT)
376 })?;
377 if meet != m1 {
378 self.set(
379 S::STAT,
380 meet.into_value().map(|s| {
381 s.into_value()
382 .vortex_expect("stat scalar value cannot be null")
383 }),
384 );
385 }
386 }
387 (None, Some(m)) => self.set(
388 S::STAT,
389 m.into_value().map(|s| {
390 s.into_value()
391 .vortex_expect("stat scalar value cannot be null")
392 }),
393 ),
394 (Some(_), _) => (),
395 (None, None) => self.clear(S::STAT),
396 }
397 Ok(())
398 }
399
400 fn combine_bool_stat<S: StatType<bool>>(&mut self, other: &TypedStatsSetRef) -> VortexResult<()>
401 where
402 S::Bound: StatBound<bool> + Debug + Eq + PartialEq,
403 {
404 match (
405 self.get_as_bound::<S, bool>(),
406 other.get_as_bound::<S, bool>(),
407 ) {
408 (Some(m1), Some(m2)) => {
409 let intersection = m1
410 .intersection(&m2)
411 .vortex_expect("can always compare boolean")
412 .ok_or_else(|| {
413 vortex_err!("{:?} bounds ({m1:?}, {m2:?}) do not overlap", S::STAT)
414 })?;
415 if intersection != m1 {
416 self.set(S::STAT, intersection.into_value().map(ScalarValue::from));
417 }
418 }
419 (None, Some(m)) => self.set(S::STAT, m.into_value().map(ScalarValue::from)),
420 (Some(_), None) => (),
421 (None, None) => self.clear(S::STAT),
422 }
423 Ok(())
424 }
425
426 fn merge_min(&mut self, other: &TypedStatsSetRef) {
427 match (
428 self.get_scalar_bound::<Min>(),
429 other.get_scalar_bound::<Min>(),
430 ) {
431 (Some(m1), Some(m2)) => {
432 let meet = m1.union(&m2).vortex_expect("can compare scalar");
433 if meet != m1 {
434 self.set(
435 Stat::Min,
436 meet.into_value().map(|s| {
437 s.into_value()
438 .vortex_expect("stat scalar value cannot be null")
439 }),
440 );
441 }
442 }
443 _ => self.clear(Stat::Min),
444 }
445 }
446
447 fn merge_max(&mut self, other: &TypedStatsSetRef) {
448 match (
449 self.get_scalar_bound::<Max>(),
450 other.get_scalar_bound::<Max>(),
451 ) {
452 (Some(m1), Some(m2)) => {
453 let meet = m1.union(&m2).vortex_expect("can compare scalar");
454 if meet != m1 {
455 self.set(
456 Stat::Max,
457 meet.into_value().map(|s| {
458 s.into_value()
459 .vortex_expect("stat scalar value cannot be null")
460 }),
461 );
462 }
463 }
464 _ => self.clear(Stat::Max),
465 }
466 }
467
468 fn merge_sum(&mut self, other: &TypedStatsSetRef) {
469 match (
470 self.get_scalar_bound::<Sum>(),
471 other.get_scalar_bound::<Sum>(),
472 ) {
473 (Some(m1), Some(m2)) => {
474 let merged = m1.zip(m2).as_exact().and_then(|(s1, s2)| match s1.dtype() {
476 DType::Primitive(..) => s1
477 .as_primitive()
478 .checked_add(&s2.as_primitive())
479 .and_then(|pscalar| pscalar.pvalue().map(ScalarValue::Primitive)),
480 DType::Decimal(decimal_dtype, _) => s1
483 .as_decimal()
484 .checked_binary_numeric(
485 &s2.as_decimal(),
486 crate::scalar::NumericOperator::Add,
487 )
488 .ok()
489 .flatten()
490 .and_then(|scalar| scalar.as_decimal().decimal_value())
491 .filter(|value| value.fits_in_precision(*decimal_dtype))
492 .map(ScalarValue::Decimal),
493 _ => None,
494 });
495
496 match merged {
497 Some(scalar_value) => self.set(Stat::Sum, Precision::Exact(scalar_value)),
498 None => self.clear(Stat::Sum),
501 }
502 }
503 _ => self.clear(Stat::Sum),
504 }
505 }
506
507 fn merge_is_constant(&mut self, other: &TypedStatsSetRef) {
508 let self_const = self.values.get_bool(Stat::IsConstant);
509 let other_const = other.values.get_bool(Stat::IsConstant);
510
511 match (self_const, other_const) {
512 (Precision::Exact(false), _) | (_, Precision::Exact(false)) => {
514 self.set(Stat::IsConstant, Precision::exact(false));
515 }
516 (Precision::Exact(true), Precision::Exact(true)) => {
517 let self_min = self.values.get_ref(Stat::Min).as_exact();
520 let other_min = other.values.get_ref(Stat::Min).as_exact();
521
522 match (self_min, other_min) {
523 (Some(self_min), Some(other_min)) => {
524 let is_constant = self_min == other_min;
525 self.set(Stat::IsConstant, Precision::exact(is_constant));
526 }
527 _ => self.clear(Stat::IsConstant),
530 }
531 }
532 _ => self.clear(Stat::IsConstant),
533 }
534 }
535
536 fn merge_is_sorted(&mut self, other: &TypedStatsSetRef) {
537 self.merge_sortedness_stat(other, Stat::IsSorted, PartialOrd::le)
538 }
539
540 fn merge_is_strict_sorted(&mut self, other: &TypedStatsSetRef) {
541 self.merge_sortedness_stat(other, Stat::IsStrictSorted, PartialOrd::lt)
542 }
543
544 fn merge_sortedness_stat<F: Fn(&Scalar, &Scalar) -> bool>(
545 &mut self,
546 other: &TypedStatsSetRef,
547 stat: Stat,
548 cmp: F,
549 ) {
550 if (Precision::Exact(true), Precision::Exact(true))
551 == (self.values.get_bool(stat), other.values.get_bool(stat))
552 {
553 if let (Some(self_max), Some(other_min)) = (
558 self.get_scalar_bound::<Max>().and_then(|v| v.max_value()),
559 other.get_scalar_bound::<Min>().and_then(|v| v.min_value()),
560 ) {
561 return if cmp(&self_max, &other_min) {
562 } else {
564 self.set(stat, Precision::inexact(false));
565 };
566 }
567 }
568 self.clear(stat);
569 }
570
571 fn merge_null_count(&mut self, other: &TypedStatsSetRef) {
572 self.merge_sum_stat(Stat::NullCount, other)
573 }
574
575 fn merge_nan_count(&mut self, other: &TypedStatsSetRef) {
576 self.merge_sum_stat(Stat::NaNCount, other)
577 }
578
579 fn merge_uncompressed_size_in_bytes(&mut self, other: &TypedStatsSetRef) {
580 self.merge_sum_stat(Stat::UncompressedSizeInBytes, other)
581 }
582
583 fn merge_sum_stat(&mut self, stat: Stat, other: &TypedStatsSetRef) {
584 let merged = self
585 .values
586 .get_usize(stat)
587 .zip(other.values.get_usize(stat))
588 .map(|(l, r)| ScalarValue::from(l + r));
589
590 if merged.is_absent() {
591 self.clear(stat);
592 } else {
593 self.set(stat, merged);
594 }
595 }
596}
597
598#[cfg(test)]
599mod test {
600 use std::sync::LazyLock;
601
602 use enum_iterator::all;
603 use itertools::Itertools;
604 use rstest::rstest;
605 use smallvec::smallvec;
606 use vortex_session::VortexSession;
607
608 use crate::VortexSessionExecute;
609 use crate::arrays::PrimitiveArray;
610 use crate::dtype::DType;
611 use crate::dtype::DecimalDType;
612 use crate::dtype::MAX_PRECISION;
613 use crate::dtype::NativeDecimalType;
614 use crate::dtype::Nullability;
615 use crate::dtype::PType;
616 use crate::dtype::i256;
617 use crate::expr::stats::IsConstant;
618 use crate::expr::stats::Precision;
619 use crate::expr::stats::Stat;
620 use crate::expr::stats::StatsProvider;
621 use crate::expr::stats::StatsProviderExt;
622 use crate::scalar::DecimalValue;
623 use crate::scalar::ScalarValue;
624 use crate::stats::StatsSet;
625 use crate::stats::stats_set::Scalar;
626
627 static SESSION: LazyLock<VortexSession> = LazyLock::new(VortexSession::empty);
628
629 #[test]
630 fn test_iter() {
631 let set = unsafe {
633 StatsSet::new_unchecked(smallvec![
634 (Stat::Max, Precision::exact(100)),
635 (Stat::Min, Precision::exact(42)),
636 ])
637 };
638 let mut iter = set.iter();
639 let first = iter.next().unwrap().clone();
640 assert_eq!(first.0, Stat::Max);
641 assert_eq!(
642 first.1.map(
643 |f| i32::try_from(&Scalar::try_new(PType::I32.into(), Some(f)).unwrap()).unwrap()
644 ),
645 Precision::exact(100)
646 );
647 let snd = iter.next().unwrap().clone();
648 assert_eq!(snd.0, Stat::Min);
649 assert_eq!(
650 snd.1.map(
651 |s| i32::try_from(&Scalar::try_new(PType::I32.into(), Some(s)).unwrap()).unwrap()
652 ),
653 Precision::exact(42)
654 );
655 assert!(iter.next().is_none());
656 }
657
658 #[test]
659 fn into_iter() {
660 let mut set = unsafe {
662 StatsSet::new_unchecked(smallvec![
663 (Stat::Max, Precision::exact(100)),
664 (Stat::Min, Precision::exact(42)),
665 ])
666 }
667 .into_iter();
668 let (stat, first) = set.next().unwrap();
669 assert_eq!(stat, Stat::Max);
670 assert_eq!(
671 first.map(
672 |f| i32::try_from(&Scalar::try_new(PType::I32.into(), Some(f)).unwrap()).unwrap()
673 ),
674 Precision::exact(100)
675 );
676 let snd = set.next().unwrap();
677 assert_eq!(snd.0, Stat::Min);
678 assert_eq!(
679 snd.1.map(
680 |s| i32::try_from(&Scalar::try_new(PType::I32.into(), Some(s)).unwrap()).unwrap()
681 ),
682 Precision::exact(42)
683 );
684 assert!(set.next().is_none());
685 }
686
687 #[test]
688 fn merge_sums_overflow_clears() {
689 let dtype = DType::Primitive(PType::I64, Nullability::NonNullable);
692 let merged = StatsSet::of(Stat::Sum, Precision::exact(i64::MAX))
693 .merge_ordered(&StatsSet::of(Stat::Sum, Precision::exact(i64::MAX)), &dtype);
694
695 assert!(merged.get(Stat::Sum).is_absent());
696 }
697
698 #[test]
699 fn merge_decimal_sums_out_of_precision_clears() {
700 let dtype = DType::Decimal(DecimalDType::new(70, 0), Nullability::NonNullable);
704 let max = ScalarValue::Decimal(DecimalValue::I256(
705 <i256 as NativeDecimalType>::MAX_BY_PRECISION[usize::from(MAX_PRECISION)],
706 ));
707 let merged = StatsSet::of(Stat::Sum, Precision::exact(max.clone()))
708 .merge_ordered(&StatsSet::of(Stat::Sum, Precision::exact(max)), &dtype);
709
710 assert!(merged.get(Stat::Sum).is_absent());
711 }
712
713 #[test]
714 fn merge_constant() {
715 let first = StatsSet::from_iter([
716 (Stat::Min, Precision::exact(42)),
717 (Stat::IsConstant, Precision::exact(true)),
718 ])
719 .merge_ordered(
720 &StatsSet::from_iter([
721 (Stat::Min, Precision::inexact(42)),
722 (Stat::IsConstant, Precision::exact(true)),
723 ]),
724 &DType::Primitive(PType::I32, Nullability::NonNullable),
725 );
726
727 let first_ref = first.as_typed_ref(&DType::Primitive(PType::I32, Nullability::NonNullable));
729 assert_eq!(
730 first_ref.get_as::<bool>(Stat::IsConstant),
731 Precision::Absent
732 );
733 assert_eq!(first_ref.get_as::<i32>(Stat::Min), Precision::exact(42));
734 }
735
736 #[rstest]
737 #[case::equal_mins(
738 Precision::exact(true),
739 42,
740 Precision::exact(true),
741 42,
742 Precision::exact(true)
743 )]
744 #[case::different_mins(
745 Precision::exact(true),
746 42,
747 Precision::exact(true),
748 7,
749 Precision::exact(false)
750 )]
751 #[case::first_not_constant(
752 Precision::exact(false),
753 42,
754 Precision::exact(true),
755 42,
756 Precision::exact(false)
757 )]
758 #[case::second_not_constant(
759 Precision::exact(true),
760 42,
761 Precision::exact(false),
762 42,
763 Precision::exact(false)
764 )]
765 #[case::inexact_constant(
766 Precision::inexact(true),
767 42,
768 Precision::exact(true),
769 42,
770 Precision::Absent
771 )]
772 fn merge_constant_parts(
773 #[case] first_const: Precision<bool>,
774 #[case] first_min: i32,
775 #[case] second_const: Precision<bool>,
776 #[case] second_min: i32,
777 #[case] expected: Precision<bool>,
778 ) {
779 let dtype = DType::Primitive(PType::I32, Nullability::NonNullable);
780 let stats = |is_constant: Precision<bool>, min: i32| {
781 StatsSet::from_iter([
782 (Stat::Min, Precision::exact(min)),
783 (Stat::IsConstant, is_constant.map(ScalarValue::from)),
784 ])
785 };
786
787 let merged =
788 stats(first_const, first_min).merge_ordered(&stats(second_const, second_min), &dtype);
789
790 assert_eq!(
791 merged.as_typed_ref(&dtype).get_as::<bool>(Stat::IsConstant),
792 expected
793 );
794 }
795
796 #[test]
797 fn merge_into_min() {
798 let first = StatsSet::of(Stat::Min, Precision::exact(42)).merge_ordered(
799 &StatsSet::default(),
800 &DType::Primitive(PType::I32, Nullability::NonNullable),
801 );
802
803 let first_ref = first.as_typed_ref(&DType::Primitive(PType::I32, Nullability::NonNullable));
804 assert!(first_ref.get(Stat::Min).is_absent());
805 }
806
807 #[test]
808 fn merge_from_min() {
809 let first = StatsSet::default().merge_ordered(
810 &StatsSet::of(Stat::Min, Precision::exact(42)),
811 &DType::Primitive(PType::I32, Nullability::NonNullable),
812 );
813
814 let first_ref = first.as_typed_ref(&DType::Primitive(PType::I32, Nullability::NonNullable));
815 assert!(first_ref.get(Stat::Min).is_absent());
816 }
817
818 #[test]
819 fn merge_mins() {
820 let first = StatsSet::of(Stat::Min, Precision::exact(37)).merge_ordered(
821 &StatsSet::of(Stat::Min, Precision::exact(42)),
822 &DType::Primitive(PType::I32, Nullability::NonNullable),
823 );
824
825 let first_ref = first.as_typed_ref(&DType::Primitive(PType::I32, Nullability::NonNullable));
826 assert_eq!(first_ref.get_as::<i32>(Stat::Min), Precision::exact(37));
827 }
828
829 #[test]
830 fn merge_into_bound_max() {
831 let first = StatsSet::of(Stat::Max, Precision::exact(42)).merge_ordered(
832 &StatsSet::default(),
833 &DType::Primitive(PType::I32, Nullability::NonNullable),
834 );
835 assert!(first.get(Stat::Max).is_absent());
836 }
837
838 #[test]
839 fn merge_from_max() {
840 let first = StatsSet::default().merge_ordered(
841 &StatsSet::of(Stat::Max, Precision::exact(42)),
842 &DType::Primitive(PType::I32, Nullability::NonNullable),
843 );
844 assert!(first.get(Stat::Max).is_absent());
845 }
846
847 #[test]
848 fn merge_maxes() {
849 let first = StatsSet::of(Stat::Max, Precision::exact(37)).merge_ordered(
850 &StatsSet::of(Stat::Max, Precision::exact(42)),
851 &DType::Primitive(PType::I32, Nullability::NonNullable),
852 );
853 let first_ref = first.as_typed_ref(&DType::Primitive(PType::I32, Nullability::NonNullable));
854 assert_eq!(first_ref.get_as::<i32>(Stat::Max), Precision::exact(42));
855 }
856
857 #[test]
858 fn merge_maxes_bound() {
859 let dtype = DType::Primitive(PType::I32, Nullability::NonNullable);
860 let first = StatsSet::of(Stat::Max, Precision::exact(42i32))
861 .merge_ordered(&StatsSet::of(Stat::Max, Precision::inexact(43i32)), &dtype);
862 let first_ref = first.as_typed_ref(&dtype);
863 assert_eq!(first_ref.get_as::<i32>(Stat::Max), Precision::inexact(43));
864 }
865
866 #[test]
867 fn merge_into_scalar() {
868 let first = StatsSet::of(Stat::Sum, Precision::exact(42i64)).merge_ordered(
871 &StatsSet::default(),
872 &DType::Primitive(PType::I32, Nullability::NonNullable),
873 );
874 let first_ref = first.as_typed_ref(&DType::Primitive(PType::I32, Nullability::NonNullable));
875 assert!(first_ref.get(Stat::Sum).is_absent());
876 }
877
878 #[test]
879 fn merge_from_scalar() {
880 let first = StatsSet::default().merge_ordered(
883 &StatsSet::of(Stat::Sum, Precision::exact(42i64)),
884 &DType::Primitive(PType::I32, Nullability::NonNullable),
885 );
886 let first_ref = first.as_typed_ref(&DType::Primitive(PType::I32, Nullability::NonNullable));
887 assert!(first_ref.get(Stat::Sum).is_absent());
888 }
889
890 #[test]
891 fn merge_scalars() {
892 let first = StatsSet::of(Stat::Sum, Precision::exact(37i64)).merge_ordered(
895 &StatsSet::of(Stat::Sum, Precision::exact(42i64)),
896 &DType::Primitive(PType::I32, Nullability::NonNullable),
897 );
898 let first_ref = first.as_typed_ref(&DType::Primitive(PType::I32, Nullability::NonNullable));
899 assert_eq!(first_ref.get_as::<i64>(Stat::Sum), Precision::exact(79i64));
900 }
901
902 #[test]
903 fn merge_into_sortedness() {
904 let first = StatsSet::of(Stat::IsStrictSorted, Precision::exact(true)).merge_ordered(
905 &StatsSet::default(),
906 &DType::Primitive(PType::I32, Nullability::NonNullable),
907 );
908 assert!(first.get(Stat::IsStrictSorted).is_absent());
909 }
910
911 #[test]
912 fn merge_from_sortedness() {
913 let first = StatsSet::default().merge_ordered(
914 &StatsSet::of(Stat::IsStrictSorted, Precision::exact(true)),
915 &DType::Primitive(PType::I32, Nullability::NonNullable),
916 );
917 assert!(first.get(Stat::IsStrictSorted).is_absent());
918 }
919
920 #[test]
921 fn merge_sortedness() {
922 let mut first = StatsSet::of(Stat::IsStrictSorted, Precision::exact(true));
923 first.set(Stat::Max, Precision::exact(1));
924 let mut second = StatsSet::of(Stat::IsStrictSorted, Precision::exact(true));
925 second.set(Stat::Min, Precision::exact(2));
926 first = first.merge_ordered(
927 &second,
928 &DType::Primitive(PType::I32, Nullability::NonNullable),
929 );
930
931 let first_ref = first.as_typed_ref(&DType::Primitive(PType::I32, Nullability::NonNullable));
932 assert_eq!(
933 first_ref.get_as::<bool>(Stat::IsStrictSorted),
934 Precision::exact(true)
935 );
936 }
937
938 #[test]
939 fn merge_sortedness_out_of_order() {
940 let mut first = StatsSet::of(Stat::IsStrictSorted, Precision::exact(true));
941 first.set(Stat::Min, Precision::exact(1));
942 let mut second = StatsSet::of(Stat::IsStrictSorted, Precision::exact(true));
943 second.set(Stat::Max, Precision::exact(2));
944 second = second.merge_ordered(
945 &first,
946 &DType::Primitive(PType::I32, Nullability::NonNullable),
947 );
948
949 let second_ref =
950 second.as_typed_ref(&DType::Primitive(PType::I32, Nullability::NonNullable));
951 assert_eq!(
952 second_ref.get_as::<bool>(Stat::IsStrictSorted),
953 Precision::inexact(false)
954 );
955 }
956
957 #[test]
958 fn merge_sortedness_only_one_sorted() {
959 let mut first = StatsSet::of(Stat::IsStrictSorted, Precision::exact(true));
960 first.set(Stat::Max, Precision::exact(1));
961 let mut second = StatsSet::of(Stat::IsStrictSorted, Precision::exact(false));
962 second.set(Stat::Min, Precision::exact(2));
963 let merged = first.merge_ordered(
964 &second,
965 &DType::Primitive(PType::I32, Nullability::NonNullable),
966 );
967
968 assert!(merged.get(Stat::IsStrictSorted).is_absent());
969 }
970
971 #[test]
972 fn merge_sortedness_missing_min() {
973 let mut first = StatsSet::of(Stat::IsStrictSorted, Precision::exact(true));
974 first.set(Stat::Max, Precision::exact(1));
975 let second = StatsSet::of(Stat::IsStrictSorted, Precision::exact(true));
976 first = first.merge_ordered(
977 &second,
978 &DType::Primitive(PType::I32, Nullability::NonNullable),
979 );
980 assert!(first.get(Stat::IsStrictSorted).is_absent());
981 }
982
983 #[test]
984 fn merge_sortedness_bound_min() {
985 let mut first = StatsSet::of(Stat::IsStrictSorted, Precision::exact(true));
986 first.set(Stat::Max, Precision::exact(1));
987 let mut second = StatsSet::of(Stat::IsStrictSorted, Precision::exact(true));
988 second.set(Stat::Min, Precision::inexact(2));
989 first = first.merge_ordered(
990 &second,
991 &DType::Primitive(PType::I32, Nullability::NonNullable),
992 );
993
994 let first_ref = first.as_typed_ref(&DType::Primitive(PType::I32, Nullability::NonNullable));
995 assert_eq!(
996 first_ref.get_as::<bool>(Stat::IsStrictSorted),
997 Precision::exact(true)
998 );
999 }
1000
1001 #[test]
1002 fn merge_unordered() {
1003 let array =
1004 PrimitiveArray::from_option_iter([Some(1), None, Some(2), Some(42), Some(10000), None]);
1005 let all_stats = all::<Stat>()
1006 .filter(|s| !matches!(s, Stat::Sum))
1007 .filter(|s| !matches!(s, Stat::NaNCount))
1008 .collect_vec();
1009 array
1010 .statistics()
1011 .compute_all(&all_stats, &mut SESSION.create_execution_ctx())
1012 .unwrap();
1013
1014 let stats = array.statistics().to_owned();
1015 for stat in &all_stats {
1016 assert!(!stats.get(*stat).is_absent(), "Stat {stat} is missing");
1017 }
1018
1019 let merged = stats.clone().merge_unordered(
1020 &stats,
1021 &DType::Primitive(PType::I32, Nullability::NonNullable),
1022 );
1023 for (stat, expected) in [
1024 (Stat::Min, Precision::exact(ScalarValue::from(1i32))),
1025 (Stat::Max, Precision::exact(ScalarValue::from(10000i32))),
1026 (Stat::NullCount, Precision::exact(ScalarValue::from(4u64))),
1027 (
1028 Stat::UncompressedSizeInBytes,
1029 Precision::exact(ScalarValue::from(50u64)),
1030 ),
1031 (Stat::IsConstant, Precision::exact(ScalarValue::from(false))),
1032 (Stat::IsSorted, Precision::Absent),
1033 (Stat::IsStrictSorted, Precision::Absent),
1034 ] {
1035 assert_eq!(merged.get(stat), expected, "{stat}");
1036 }
1037 }
1038
1039 #[test]
1040 fn merge_min_bound_same() {
1041 let merged = StatsSet::of(Stat::Min, Precision::inexact(5)).merge_ordered(
1044 &StatsSet::of(Stat::Min, Precision::exact(5)),
1045 &DType::Primitive(PType::I32, Nullability::NonNullable),
1046 );
1047 let merged_ref =
1048 merged.as_typed_ref(&DType::Primitive(PType::I32, Nullability::NonNullable));
1049 assert_eq!(merged_ref.get_as::<i32>(Stat::Min), Precision::exact(5));
1050 }
1051
1052 #[test]
1053 fn merge_min_bound_bound_lower() {
1054 let merged = StatsSet::of(Stat::Min, Precision::inexact(4)).merge_ordered(
1055 &StatsSet::of(Stat::Min, Precision::exact(5)),
1056 &DType::Primitive(PType::I32, Nullability::NonNullable),
1057 );
1058 let merged_ref =
1059 merged.as_typed_ref(&DType::Primitive(PType::I32, Nullability::NonNullable));
1060 assert_eq!(merged_ref.get_as::<i32>(Stat::Min), Precision::inexact(4));
1061 }
1062
1063 #[test]
1064 fn test_combine_is_constant() {
1065 {
1066 let mut stats = StatsSet::of(Stat::IsConstant, Precision::exact(true));
1067 let stats2 = StatsSet::of(Stat::IsConstant, Precision::exact(true));
1068 let mut stats_ref =
1069 stats.as_mut_typed_ref(&DType::Primitive(PType::I32, Nullability::NonNullable));
1070 stats_ref
1071 .combine_bool_stat::<IsConstant>(
1072 &stats2.as_typed_ref(&DType::Primitive(PType::I32, Nullability::NonNullable)),
1073 )
1074 .unwrap();
1075 assert_eq!(
1076 stats_ref.get_as::<bool>(Stat::IsConstant),
1077 Precision::exact(true)
1078 );
1079 }
1080
1081 {
1082 let mut stats = StatsSet::of(Stat::IsConstant, Precision::exact(true));
1083 let stats2 = StatsSet::of(Stat::IsConstant, Precision::inexact(false));
1084 let mut stats_ref =
1085 stats.as_mut_typed_ref(&DType::Primitive(PType::I32, Nullability::NonNullable));
1086 stats_ref
1087 .combine_bool_stat::<IsConstant>(
1088 &stats2.as_typed_ref(&DType::Primitive(PType::I32, Nullability::NonNullable)),
1089 )
1090 .unwrap();
1091 assert_eq!(
1092 stats_ref.get_as::<bool>(Stat::IsConstant),
1093 Precision::exact(true)
1094 );
1095 }
1096
1097 {
1098 let mut stats = StatsSet::of(Stat::IsConstant, Precision::exact(false));
1099 let stats2 = StatsSet::of(Stat::IsConstant, Precision::inexact(false));
1100 let mut stats_ref =
1101 stats.as_mut_typed_ref(&DType::Primitive(PType::I32, Nullability::NonNullable));
1102 stats_ref
1103 .combine_bool_stat::<IsConstant>(
1104 &stats2.as_typed_ref(&DType::Primitive(PType::I32, Nullability::NonNullable)),
1105 )
1106 .unwrap();
1107 assert_eq!(
1108 stats_ref.get_as::<bool>(Stat::IsConstant),
1109 Precision::exact(false)
1110 );
1111 }
1112 }
1113
1114 #[test]
1115 fn test_combine_sets_boolean_conflict() {
1116 let mut stats1 = StatsSet::from_iter([
1117 (Stat::IsConstant, Precision::exact(true)),
1118 (Stat::IsSorted, Precision::exact(true)),
1119 ]);
1120
1121 let stats2 = StatsSet::from_iter([
1122 (Stat::IsConstant, Precision::exact(false)),
1123 (Stat::IsSorted, Precision::exact(true)),
1124 ]);
1125
1126 let result = stats1.combine_sets(
1127 &stats2,
1128 &DType::Primitive(PType::I32, Nullability::NonNullable),
1129 );
1130 assert!(result.is_err());
1131 }
1132
1133 #[test]
1134 fn test_combine_sets_with_missing_stats() {
1135 let mut stats1 = StatsSet::from_iter([
1136 (Stat::Min, Precision::exact(42)),
1137 (Stat::UncompressedSizeInBytes, Precision::exact(1000u64)),
1138 ]);
1139
1140 let stats2 = StatsSet::from_iter([
1141 (Stat::Max, Precision::exact(100)),
1142 (Stat::IsStrictSorted, Precision::exact(true)),
1143 ]);
1144
1145 stats1
1146 .combine_sets(
1147 &stats2,
1148 &DType::Primitive(PType::I32, Nullability::NonNullable),
1149 )
1150 .unwrap();
1151
1152 let stats_ref =
1153 stats1.as_typed_ref(&DType::Primitive(PType::I32, Nullability::NonNullable));
1154
1155 assert_eq!(
1156 stats_ref.get_as::<u64>(Stat::UncompressedSizeInBytes),
1157 Precision::exact(1000u64)
1158 );
1159 assert_eq!(stats_ref.get_as::<i32>(Stat::Min), Precision::exact(42));
1161 assert_eq!(stats_ref.get_as::<i32>(Stat::Max), Precision::exact(100));
1163 assert_eq!(
1165 stats_ref.get_as::<bool>(Stat::IsStrictSorted),
1166 Precision::exact(true)
1167 );
1168 }
1169
1170 #[test]
1171 fn test_combine_sets_with_inexact() {
1172 let mut stats1 = StatsSet::from_iter([
1173 (Stat::Min, Precision::exact(42)),
1174 (Stat::Max, Precision::inexact(100)),
1175 (Stat::IsConstant, Precision::exact(false)),
1176 ]);
1177
1178 let stats2 = StatsSet::from_iter([
1179 (Stat::Min, Precision::inexact(40)),
1181 (Stat::Max, Precision::exact(90)),
1182 (Stat::IsSorted, Precision::exact(true)),
1183 ]);
1184
1185 stats1
1186 .combine_sets(
1187 &stats2,
1188 &DType::Primitive(PType::I32, Nullability::NonNullable),
1189 )
1190 .unwrap();
1191
1192 let stats_ref =
1193 stats1.as_typed_ref(&DType::Primitive(PType::I32, Nullability::NonNullable));
1194
1195 assert_eq!(
1196 stats_ref.get_as::<bool>(Stat::IsConstant),
1197 Precision::exact(false)
1198 );
1199 assert_eq!(stats_ref.get_as::<i32>(Stat::Min), Precision::exact(42));
1201 assert_eq!(stats_ref.get_as::<i32>(Stat::Max), Precision::exact(90));
1203 assert_eq!(
1205 stats_ref.get_as::<bool>(Stat::IsSorted),
1206 Precision::exact(true)
1207 );
1208 }
1209}