Skip to main content

vortex_array/stats/
stats_set.rs

1// SPDX-License-Identifier: Apache-2.0
2// SPDX-FileCopyrightText: Copyright the Vortex contributors
3
4use std::any::type_name;
5use std::fmt::Debug;
6
7use enum_iterator::all;
8use num_traits::CheckedAdd;
9use smallvec::SmallVec;
10use smallvec::smallvec;
11use vortex_error::VortexError;
12use vortex_error::VortexExpect;
13use vortex_error::VortexResult;
14use vortex_error::vortex_err;
15use vortex_error::vortex_panic;
16
17use crate::dtype::DType;
18use crate::expr::stats::IsConstant;
19use crate::expr::stats::IsSorted;
20use crate::expr::stats::IsStrictSorted;
21use crate::expr::stats::Max;
22use crate::expr::stats::Min;
23use crate::expr::stats::NaNCount;
24use crate::expr::stats::NullCount;
25use crate::expr::stats::Precision;
26use crate::expr::stats::Stat;
27use crate::expr::stats::StatBound;
28use crate::expr::stats::StatType;
29use crate::expr::stats::StatsProvider;
30use crate::expr::stats::StatsProviderExt;
31use crate::expr::stats::Sum;
32use crate::expr::stats::UncompressedSizeInBytes;
33use crate::scalar::Scalar;
34use crate::scalar::ScalarValue;
35
36/// Type of the SmallVec stored inside StatsSet
37pub type StatsArray = [(Stat, Precision<ScalarValue>); 4];
38
39#[derive(Default, Debug, Clone)]
40pub struct StatsSet {
41    values: SmallVec<StatsArray>,
42}
43
44impl StatsSet {
45    /// Create new StatSet without validating uniqueness of all the entries
46    ///
47    /// # Safety
48    ///
49    /// This method will not panic or trigger UB, but may lead to duplicate stats being stored.
50    pub unsafe fn new_unchecked(values: SmallVec<StatsArray>) -> Self {
51        Self { values }
52    }
53
54    /// Create StatsSet from single stat and value
55    pub fn of(stat: Stat, value: Precision<ScalarValue>) -> Self {
56        Self {
57            values: smallvec![(stat, value)],
58        }
59    }
60
61    /// Wrap stats set with a dtype for mutable typed scalar access
62    pub fn as_mut_typed_ref<'a, 'b>(&'a mut self, dtype: &'b DType) -> MutTypedStatsSetRef<'a, 'b> {
63        MutTypedStatsSetRef {
64            values: self,
65            dtype,
66        }
67    }
68
69    /// Wrap stats set with a dtype for typed scalar access
70    pub fn as_typed_ref<'a, 'b>(&'a self, dtype: &'b DType) -> TypedStatsSetRef<'a, 'b> {
71        TypedStatsSetRef {
72            values: self,
73            dtype,
74        }
75    }
76}
77
78// Getters and setters for individual stats.
79impl StatsSet {
80    /// Set the stat `stat` to `value`.
81    pub fn set(&mut self, stat: Stat, value: Precision<ScalarValue>) {
82        if let Some(existing) = self.values.iter_mut().find(|(s, _)| *s == stat) {
83            *existing = (stat, value);
84        } else {
85            self.values.push((stat, value));
86        }
87    }
88
89    /// Clear the stat `stat` from the set.
90    pub fn clear(&mut self, stat: Stat) {
91        self.values.retain(|(s, _)| *s != stat);
92    }
93
94    /// Only keep given stats
95    pub fn retain_only(&mut self, stats: &[Stat]) {
96        self.values.retain(|(s, _)| stats.contains(s));
97    }
98
99    /// Iterate over the statistic names and values in-place.
100    ///
101    /// See [Iterator].
102    pub fn iter(&self) -> impl Iterator<Item = &(Stat, Precision<ScalarValue>)> {
103        self.values.iter()
104    }
105
106    /// Get value for a given stat
107    pub fn get(&self, stat: Stat) -> Precision<ScalarValue> {
108        self.get_ref(stat).map(ScalarValue::clone)
109    }
110
111    /// Get a reference to the value for a given stat.
112    fn get_ref(&self, stat: Stat) -> Precision<&ScalarValue> {
113        self.values
114            .iter()
115            .find(|(s, _)| *s == stat)
116            .map(|(_, v)| v.as_ref())
117            .unwrap_or(Precision::Absent)
118    }
119
120    /// Get the value for a count stat, such as [`Stat::NullCount`], without building a [`Scalar`].
121    pub(super) fn get_usize(&self, stat: Stat) -> Precision<usize> {
122        self.get_ref(stat).map(|v| {
123            usize::try_from(v).unwrap_or_else(|err| {
124                vortex_panic!(
125                    err,
126                    "Failed to get stat {} as {}",
127                    stat,
128                    type_name::<usize>()
129                )
130            })
131        })
132    }
133
134    /// Get the value for a boolean stat, such as [`Stat::IsSorted`], without building a
135    /// [`Scalar`].
136    pub(super) fn get_bool(&self, stat: Stat) -> Precision<bool> {
137        self.get_ref(stat).map(|v| match v {
138            ScalarValue::Bool(b) => *b,
139            _ => vortex_panic!("Failed to get stat {} as bool, found {}", stat, v),
140        })
141    }
142
143    /// Length of the stats set
144    pub fn len(&self) -> usize {
145        self.values.len()
146    }
147
148    /// Check whether the statset is empty
149    pub fn is_empty(&self) -> bool {
150        self.values.is_empty()
151    }
152
153    /// Get scalar value of a given dtype
154    pub fn get_as<T: for<'a> TryFrom<&'a Scalar, Error = VortexError>>(
155        &self,
156        stat: Stat,
157        dtype: &DType,
158    ) -> Precision<T> {
159        self.get(stat).map(|v| {
160            T::try_from(
161                &Scalar::try_new(dtype.clone(), Some(v))
162                    .vortex_expect("failed to construct a scalar statistic"),
163            )
164            .unwrap_or_else(|err| {
165                vortex_panic!(err, "Failed to get stat {} as {}", stat, type_name::<T>())
166            })
167        })
168    }
169}
170
171// StatSetIntoIter just exists to protect current implementation from exposure on the public API.
172
173/// Owned iterator over the stats.
174///
175/// See [IntoIterator].
176pub struct StatsSetIntoIter(smallvec::IntoIter<StatsArray>);
177
178impl Iterator for StatsSetIntoIter {
179    type Item = (Stat, Precision<ScalarValue>);
180
181    fn next(&mut self) -> Option<Self::Item> {
182        self.0.next()
183    }
184}
185
186impl IntoIterator for StatsSet {
187    type Item = (Stat, Precision<ScalarValue>);
188    type IntoIter = StatsSetIntoIter;
189
190    fn into_iter(self) -> Self::IntoIter {
191        StatsSetIntoIter(self.values.into_iter())
192    }
193}
194
195impl FromIterator<(Stat, Precision<ScalarValue>)> for StatsSet {
196    fn from_iter<T: IntoIterator<Item = (Stat, Precision<ScalarValue>)>>(iter: T) -> Self {
197        let iter = iter.into_iter();
198
199        let mut this = Self {
200            values: SmallVec::new(),
201        };
202        this.extend(iter);
203        this
204    }
205}
206
207impl Extend<(Stat, Precision<ScalarValue>)> for StatsSet {
208    #[inline]
209    fn extend<T: IntoIterator<Item = (Stat, Precision<ScalarValue>)>>(&mut self, iter: T) {
210        iter.into_iter()
211            .for_each(|(stat, value)| self.set(stat, value));
212    }
213}
214
215/// Merge helpers
216impl StatsSet {
217    /// Merge stats set `other` into `self`, with the semantic assumption that `other`
218    /// contains stats from a disjoint array that is *appended* to the array represented by `self`.
219    pub fn merge_ordered(mut self, other: &Self, dtype: &DType) -> Self {
220        self.as_mut_typed_ref(dtype)
221            .merge_ordered(&other.as_typed_ref(dtype));
222        self
223    }
224
225    /// Merge stats set `other` into `self`, from a disjoint array, with no ordering assumptions.
226    /// Stats that are not commutative (e.g., is_sorted) are dropped from the result.
227    pub fn merge_unordered(mut self, other: &Self, dtype: &DType) -> Self {
228        self.as_mut_typed_ref(dtype)
229            .merge_unordered(&other.as_typed_ref(dtype));
230        self
231    }
232
233    /// Given two sets of stats (of differing precision) for the same array, combine them
234    pub fn combine_sets(&mut self, other: &Self, dtype: &DType) -> VortexResult<()> {
235        self.as_mut_typed_ref(dtype)
236            .combine_sets(&other.as_typed_ref(dtype))
237    }
238}
239
240/// Builds the typed [`Scalar`] for `stat` over an array of `dtype` from its stored value.
241fn stat_scalar(values: &StatsSet, stat: Stat, dtype: &DType) -> Precision<Scalar> {
242    values.get_ref(stat).map(|value| {
243        let stat_dtype = stat
244            .dtype(dtype)
245            .vortex_expect("Must have valid dtype if value is present");
246        Scalar::validate(&stat_dtype, Some(value))
247            .vortex_expect("failed to construct a scalar statistic");
248
249        // SAFETY: The value was validated against the dtype above.
250        unsafe { Scalar::new_unchecked(stat_dtype, Some(value.clone())) }
251    })
252}
253
254pub struct TypedStatsSetRef<'a, 'b> {
255    pub values: &'a StatsSet,
256    pub dtype: &'b DType,
257}
258
259impl StatsProvider for TypedStatsSetRef<'_, '_> {
260    fn get(&self, stat: Stat) -> Precision<Scalar> {
261        stat_scalar(self.values, stat, self.dtype)
262    }
263
264    fn len(&self) -> usize {
265        self.values.len()
266    }
267}
268
269pub struct MutTypedStatsSetRef<'a, 'b> {
270    pub values: &'a mut StatsSet,
271    pub dtype: &'b DType,
272}
273
274impl MutTypedStatsSetRef<'_, '_> {
275    /// Set the stat `stat` to `value`.
276    pub fn set(&mut self, stat: Stat, value: Precision<ScalarValue>) {
277        self.values.set(stat, value);
278    }
279
280    /// Clear the stat `stat` from the set.
281    pub fn clear(&mut self, stat: Stat) {
282        self.values.clear(stat);
283    }
284}
285
286impl StatsProvider for MutTypedStatsSetRef<'_, '_> {
287    fn get(&self, stat: Stat) -> Precision<Scalar> {
288        stat_scalar(self.values, stat, self.dtype)
289    }
290
291    fn len(&self) -> usize {
292        self.values.len()
293    }
294}
295
296// Merge helpers
297impl MutTypedStatsSetRef<'_, '_> {
298    /// Merge stats set `other` into `self`, with the semantic assumption that `other`
299    /// contains stats from a disjoint array that is *appended* to the array represented by `self`.
300    pub fn merge_ordered(mut self, other: &TypedStatsSetRef) -> Self {
301        for s in all::<Stat>() {
302            match s {
303                Stat::IsConstant => self.merge_is_constant(other),
304                Stat::IsSorted => self.merge_is_sorted(other),
305                Stat::IsStrictSorted => self.merge_is_strict_sorted(other),
306                Stat::Max => self.merge_max(other),
307                Stat::Min => self.merge_min(other),
308                Stat::Sum => self.merge_sum(other),
309                Stat::NullCount => self.merge_null_count(other),
310                Stat::UncompressedSizeInBytes => self.merge_uncompressed_size_in_bytes(other),
311                Stat::NaNCount => self.merge_nan_count(other),
312            }
313        }
314
315        self
316    }
317
318    /// Merge stats set `other` into `self`, from a disjoint array, with no ordering assumptions.
319    /// Stats that are not commutative (e.g., is_sorted) are dropped from the result.
320    pub fn merge_unordered(mut self, other: &TypedStatsSetRef) -> Self {
321        for s in all::<Stat>() {
322            if !s.is_commutative() {
323                self.clear(s);
324                continue;
325            }
326
327            match s {
328                Stat::IsConstant => self.merge_is_constant(other),
329                Stat::Max => self.merge_max(other),
330                Stat::Min => self.merge_min(other),
331                Stat::Sum => self.merge_sum(other),
332                Stat::NullCount => self.merge_null_count(other),
333                Stat::UncompressedSizeInBytes => self.merge_uncompressed_size_in_bytes(other),
334                Stat::IsSorted | Stat::IsStrictSorted => {
335                    unreachable!("not commutative")
336                }
337                Stat::NaNCount => self.merge_nan_count(other),
338            }
339        }
340
341        self
342    }
343
344    /// Given two sets of stats (of differing precision) for the same array, combine them
345    pub fn combine_sets(&mut self, other: &TypedStatsSetRef) -> VortexResult<()> {
346        let other_stats: Vec<_> = other.values.iter().map(|(stat, _)| *stat).collect();
347        for s in other_stats {
348            match s {
349                Stat::Max => self.combine_bound::<Max>(other)?,
350                Stat::Min => self.combine_bound::<Min>(other)?,
351                Stat::UncompressedSizeInBytes => {
352                    self.combine_bound::<UncompressedSizeInBytes>(other)?
353                }
354                Stat::IsConstant => self.combine_bool_stat::<IsConstant>(other)?,
355                Stat::IsSorted => self.combine_bool_stat::<IsSorted>(other)?,
356                Stat::IsStrictSorted => self.combine_bool_stat::<IsStrictSorted>(other)?,
357                Stat::NullCount => self.combine_bound::<NullCount>(other)?,
358                Stat::Sum => self.combine_bound::<Sum>(other)?,
359                Stat::NaNCount => self.combine_bound::<NaNCount>(other)?,
360            }
361        }
362        Ok(())
363    }
364
365    fn combine_bound<S: StatType<Scalar>>(&mut self, other: &TypedStatsSetRef) -> VortexResult<()>
366    where
367        S::Bound: StatBound<Scalar> + Debug + Eq + PartialEq,
368    {
369        match (self.get_scalar_bound::<S>(), other.get_scalar_bound::<S>()) {
370            (Some(m1), Some(m2)) => {
371                let meet = m1
372                    .intersection(&m2)
373                    .vortex_expect("can always compare scalar")
374                    .ok_or_else(|| {
375                        vortex_err!("{:?} bounds ({m1:?}, {m2:?}) do not overlap", S::STAT)
376                    })?;
377                if meet != m1 {
378                    self.set(
379                        S::STAT,
380                        meet.into_value().map(|s| {
381                            s.into_value()
382                                .vortex_expect("stat scalar value cannot be null")
383                        }),
384                    );
385                }
386            }
387            (None, Some(m)) => self.set(
388                S::STAT,
389                m.into_value().map(|s| {
390                    s.into_value()
391                        .vortex_expect("stat scalar value cannot be null")
392                }),
393            ),
394            (Some(_), _) => (),
395            (None, None) => self.clear(S::STAT),
396        }
397        Ok(())
398    }
399
400    fn combine_bool_stat<S: StatType<bool>>(&mut self, other: &TypedStatsSetRef) -> VortexResult<()>
401    where
402        S::Bound: StatBound<bool> + Debug + Eq + PartialEq,
403    {
404        match (
405            self.get_as_bound::<S, bool>(),
406            other.get_as_bound::<S, bool>(),
407        ) {
408            (Some(m1), Some(m2)) => {
409                let intersection = m1
410                    .intersection(&m2)
411                    .vortex_expect("can always compare boolean")
412                    .ok_or_else(|| {
413                        vortex_err!("{:?} bounds ({m1:?}, {m2:?}) do not overlap", S::STAT)
414                    })?;
415                if intersection != m1 {
416                    self.set(S::STAT, intersection.into_value().map(ScalarValue::from));
417                }
418            }
419            (None, Some(m)) => self.set(S::STAT, m.into_value().map(ScalarValue::from)),
420            (Some(_), None) => (),
421            (None, None) => self.clear(S::STAT),
422        }
423        Ok(())
424    }
425
426    fn merge_min(&mut self, other: &TypedStatsSetRef) {
427        match (
428            self.get_scalar_bound::<Min>(),
429            other.get_scalar_bound::<Min>(),
430        ) {
431            (Some(m1), Some(m2)) => {
432                let meet = m1.union(&m2).vortex_expect("can compare scalar");
433                if meet != m1 {
434                    self.set(
435                        Stat::Min,
436                        meet.into_value().map(|s| {
437                            s.into_value()
438                                .vortex_expect("stat scalar value cannot be null")
439                        }),
440                    );
441                }
442            }
443            _ => self.clear(Stat::Min),
444        }
445    }
446
447    fn merge_max(&mut self, other: &TypedStatsSetRef) {
448        match (
449            self.get_scalar_bound::<Max>(),
450            other.get_scalar_bound::<Max>(),
451        ) {
452            (Some(m1), Some(m2)) => {
453                let meet = m1.union(&m2).vortex_expect("can compare scalar");
454                if meet != m1 {
455                    self.set(
456                        Stat::Max,
457                        meet.into_value().map(|s| {
458                            s.into_value()
459                                .vortex_expect("stat scalar value cannot be null")
460                        }),
461                    );
462                }
463            }
464            _ => self.clear(Stat::Max),
465        }
466    }
467
468    fn merge_sum(&mut self, other: &TypedStatsSetRef) {
469        match (
470            self.get_scalar_bound::<Sum>(),
471            other.get_scalar_bound::<Sum>(),
472        ) {
473            (Some(m1), Some(m2)) => {
474                // If the combined sum is exact, then we can sum them.
475                let merged = m1.zip(m2).as_exact().and_then(|(s1, s2)| match s1.dtype() {
476                    DType::Primitive(..) => s1
477                        .as_primitive()
478                        .checked_add(&s2.as_primitive())
479                        .and_then(|pscalar| pscalar.pvalue().map(ScalarValue::Primitive)),
480                    // Add widens the result precision, so the merged sum is only exact as a
481                    // stat if it still fits the summands' own decimal type.
482                    DType::Decimal(decimal_dtype, _) => s1
483                        .as_decimal()
484                        .checked_binary_numeric(
485                            &s2.as_decimal(),
486                            crate::scalar::NumericOperator::Add,
487                        )
488                        .ok()
489                        .flatten()
490                        .and_then(|scalar| scalar.as_decimal().decimal_value())
491                        .filter(|value| value.fits_in_precision(*decimal_dtype))
492                        .map(ScalarValue::Decimal),
493                    _ => None,
494                });
495
496                match merged {
497                    Some(scalar_value) => self.set(Stat::Sum, Precision::Exact(scalar_value)),
498                    // An overflow, an inexact bound or a non-numeric dtype leaves the combined
499                    // sum unknown. Keeping this set's own sum would report a partial total.
500                    None => self.clear(Stat::Sum),
501                }
502            }
503            _ => self.clear(Stat::Sum),
504        }
505    }
506
507    fn merge_is_constant(&mut self, other: &TypedStatsSetRef) {
508        let self_const = self.values.get_bool(Stat::IsConstant);
509        let other_const = other.values.get_bool(Stat::IsConstant);
510
511        match (self_const, other_const) {
512            // A non-constant part makes the whole array non-constant.
513            (Precision::Exact(false), _) | (_, Precision::Exact(false)) => {
514                self.set(Stat::IsConstant, Precision::exact(false));
515            }
516            (Precision::Exact(true), Precision::Exact(true)) => {
517                // Both mins have the same dtype, so comparing the values is the same as comparing
518                // scalars.
519                let self_min = self.values.get_ref(Stat::Min).as_exact();
520                let other_min = other.values.get_ref(Stat::Min).as_exact();
521
522                match (self_min, other_min) {
523                    (Some(self_min), Some(other_min)) => {
524                        let is_constant = self_min == other_min;
525                        self.set(Stat::IsConstant, Precision::exact(is_constant));
526                    }
527                    // Without both exact mins, e.g. for an all-null part, the parts cannot be
528                    // compared.
529                    _ => self.clear(Stat::IsConstant),
530                }
531            }
532            _ => self.clear(Stat::IsConstant),
533        }
534    }
535
536    fn merge_is_sorted(&mut self, other: &TypedStatsSetRef) {
537        self.merge_sortedness_stat(other, Stat::IsSorted, PartialOrd::le)
538    }
539
540    fn merge_is_strict_sorted(&mut self, other: &TypedStatsSetRef) {
541        self.merge_sortedness_stat(other, Stat::IsStrictSorted, PartialOrd::lt)
542    }
543
544    fn merge_sortedness_stat<F: Fn(&Scalar, &Scalar) -> bool>(
545        &mut self,
546        other: &TypedStatsSetRef,
547        stat: Stat,
548        cmp: F,
549    ) {
550        if (Precision::Exact(true), Precision::Exact(true))
551            == (self.values.get_bool(stat), other.values.get_bool(stat))
552        {
553            // There might be no stat because it was dropped, or it doesn't exist
554            // (e.g. an all null array).
555            // We assume that it was the dropped case since the doesn't exist might imply sorted,
556            // but this in-precision is correct.
557            if let (Some(self_max), Some(other_min)) = (
558                self.get_scalar_bound::<Max>().and_then(|v| v.max_value()),
559                other.get_scalar_bound::<Min>().and_then(|v| v.min_value()),
560            ) {
561                return if cmp(&self_max, &other_min) {
562                    // keep value
563                } else {
564                    self.set(stat, Precision::inexact(false));
565                };
566            }
567        }
568        self.clear(stat);
569    }
570
571    fn merge_null_count(&mut self, other: &TypedStatsSetRef) {
572        self.merge_sum_stat(Stat::NullCount, other)
573    }
574
575    fn merge_nan_count(&mut self, other: &TypedStatsSetRef) {
576        self.merge_sum_stat(Stat::NaNCount, other)
577    }
578
579    fn merge_uncompressed_size_in_bytes(&mut self, other: &TypedStatsSetRef) {
580        self.merge_sum_stat(Stat::UncompressedSizeInBytes, other)
581    }
582
583    fn merge_sum_stat(&mut self, stat: Stat, other: &TypedStatsSetRef) {
584        let merged = self
585            .values
586            .get_usize(stat)
587            .zip(other.values.get_usize(stat))
588            .map(|(l, r)| ScalarValue::from(l + r));
589
590        if merged.is_absent() {
591            self.clear(stat);
592        } else {
593            self.set(stat, merged);
594        }
595    }
596}
597
598#[cfg(test)]
599mod test {
600    use std::sync::LazyLock;
601
602    use enum_iterator::all;
603    use itertools::Itertools;
604    use rstest::rstest;
605    use smallvec::smallvec;
606    use vortex_session::VortexSession;
607
608    use crate::VortexSessionExecute;
609    use crate::arrays::PrimitiveArray;
610    use crate::dtype::DType;
611    use crate::dtype::DecimalDType;
612    use crate::dtype::MAX_PRECISION;
613    use crate::dtype::NativeDecimalType;
614    use crate::dtype::Nullability;
615    use crate::dtype::PType;
616    use crate::dtype::i256;
617    use crate::expr::stats::IsConstant;
618    use crate::expr::stats::Precision;
619    use crate::expr::stats::Stat;
620    use crate::expr::stats::StatsProvider;
621    use crate::expr::stats::StatsProviderExt;
622    use crate::scalar::DecimalValue;
623    use crate::scalar::ScalarValue;
624    use crate::stats::StatsSet;
625    use crate::stats::stats_set::Scalar;
626
627    static SESSION: LazyLock<VortexSession> = LazyLock::new(VortexSession::empty);
628
629    #[test]
630    fn test_iter() {
631        // SAFETY: No duplicate stats.
632        let set = unsafe {
633            StatsSet::new_unchecked(smallvec![
634                (Stat::Max, Precision::exact(100)),
635                (Stat::Min, Precision::exact(42)),
636            ])
637        };
638        let mut iter = set.iter();
639        let first = iter.next().unwrap().clone();
640        assert_eq!(first.0, Stat::Max);
641        assert_eq!(
642            first.1.map(
643                |f| i32::try_from(&Scalar::try_new(PType::I32.into(), Some(f)).unwrap()).unwrap()
644            ),
645            Precision::exact(100)
646        );
647        let snd = iter.next().unwrap().clone();
648        assert_eq!(snd.0, Stat::Min);
649        assert_eq!(
650            snd.1.map(
651                |s| i32::try_from(&Scalar::try_new(PType::I32.into(), Some(s)).unwrap()).unwrap()
652            ),
653            Precision::exact(42)
654        );
655        assert!(iter.next().is_none());
656    }
657
658    #[test]
659    fn into_iter() {
660        // SAFETY: No duplicate stats.
661        let mut set = unsafe {
662            StatsSet::new_unchecked(smallvec![
663                (Stat::Max, Precision::exact(100)),
664                (Stat::Min, Precision::exact(42)),
665            ])
666        }
667        .into_iter();
668        let (stat, first) = set.next().unwrap();
669        assert_eq!(stat, Stat::Max);
670        assert_eq!(
671            first.map(
672                |f| i32::try_from(&Scalar::try_new(PType::I32.into(), Some(f)).unwrap()).unwrap()
673            ),
674            Precision::exact(100)
675        );
676        let snd = set.next().unwrap();
677        assert_eq!(snd.0, Stat::Min);
678        assert_eq!(
679            snd.1.map(
680                |s| i32::try_from(&Scalar::try_new(PType::I32.into(), Some(s)).unwrap()).unwrap()
681            ),
682            Precision::exact(42)
683        );
684        assert!(set.next().is_none());
685    }
686
687    #[test]
688    fn merge_sums_overflow_clears() {
689        // A sum that cannot be combined exactly leaves the merged set with no Sum at all;
690        // retaining this set's own sum would report a partial total as the combined one.
691        let dtype = DType::Primitive(PType::I64, Nullability::NonNullable);
692        let merged = StatsSet::of(Stat::Sum, Precision::exact(i64::MAX))
693            .merge_ordered(&StatsSet::of(Stat::Sum, Precision::exact(i64::MAX)), &dtype);
694
695        assert!(merged.get(Stat::Sum).is_absent());
696    }
697
698    #[test]
699    fn merge_decimal_sums_out_of_precision_clears() {
700        // A decimal(70, 0) array's Sum stat is itself decimal(76, 0): `sum_decimal_dtype` adds
701        // ten digits of headroom but saturates at MAX_PRECISION, so at this input precision there
702        // is none left and two maximal sums cannot be combined exactly.
703        let dtype = DType::Decimal(DecimalDType::new(70, 0), Nullability::NonNullable);
704        let max = ScalarValue::Decimal(DecimalValue::I256(
705            <i256 as NativeDecimalType>::MAX_BY_PRECISION[usize::from(MAX_PRECISION)],
706        ));
707        let merged = StatsSet::of(Stat::Sum, Precision::exact(max.clone()))
708            .merge_ordered(&StatsSet::of(Stat::Sum, Precision::exact(max)), &dtype);
709
710        assert!(merged.get(Stat::Sum).is_absent());
711    }
712
713    #[test]
714    fn merge_constant() {
715        let first = StatsSet::from_iter([
716            (Stat::Min, Precision::exact(42)),
717            (Stat::IsConstant, Precision::exact(true)),
718        ])
719        .merge_ordered(
720            &StatsSet::from_iter([
721                (Stat::Min, Precision::inexact(42)),
722                (Stat::IsConstant, Precision::exact(true)),
723            ]),
724            &DType::Primitive(PType::I32, Nullability::NonNullable),
725        );
726
727        // With an inexact min, the two constant parts cannot be compared.
728        let first_ref = first.as_typed_ref(&DType::Primitive(PType::I32, Nullability::NonNullable));
729        assert_eq!(
730            first_ref.get_as::<bool>(Stat::IsConstant),
731            Precision::Absent
732        );
733        assert_eq!(first_ref.get_as::<i32>(Stat::Min), Precision::exact(42));
734    }
735
736    #[rstest]
737    #[case::equal_mins(
738        Precision::exact(true),
739        42,
740        Precision::exact(true),
741        42,
742        Precision::exact(true)
743    )]
744    #[case::different_mins(
745        Precision::exact(true),
746        42,
747        Precision::exact(true),
748        7,
749        Precision::exact(false)
750    )]
751    #[case::first_not_constant(
752        Precision::exact(false),
753        42,
754        Precision::exact(true),
755        42,
756        Precision::exact(false)
757    )]
758    #[case::second_not_constant(
759        Precision::exact(true),
760        42,
761        Precision::exact(false),
762        42,
763        Precision::exact(false)
764    )]
765    #[case::inexact_constant(
766        Precision::inexact(true),
767        42,
768        Precision::exact(true),
769        42,
770        Precision::Absent
771    )]
772    fn merge_constant_parts(
773        #[case] first_const: Precision<bool>,
774        #[case] first_min: i32,
775        #[case] second_const: Precision<bool>,
776        #[case] second_min: i32,
777        #[case] expected: Precision<bool>,
778    ) {
779        let dtype = DType::Primitive(PType::I32, Nullability::NonNullable);
780        let stats = |is_constant: Precision<bool>, min: i32| {
781            StatsSet::from_iter([
782                (Stat::Min, Precision::exact(min)),
783                (Stat::IsConstant, is_constant.map(ScalarValue::from)),
784            ])
785        };
786
787        let merged =
788            stats(first_const, first_min).merge_ordered(&stats(second_const, second_min), &dtype);
789
790        assert_eq!(
791            merged.as_typed_ref(&dtype).get_as::<bool>(Stat::IsConstant),
792            expected
793        );
794    }
795
796    #[test]
797    fn merge_into_min() {
798        let first = StatsSet::of(Stat::Min, Precision::exact(42)).merge_ordered(
799            &StatsSet::default(),
800            &DType::Primitive(PType::I32, Nullability::NonNullable),
801        );
802
803        let first_ref = first.as_typed_ref(&DType::Primitive(PType::I32, Nullability::NonNullable));
804        assert!(first_ref.get(Stat::Min).is_absent());
805    }
806
807    #[test]
808    fn merge_from_min() {
809        let first = StatsSet::default().merge_ordered(
810            &StatsSet::of(Stat::Min, Precision::exact(42)),
811            &DType::Primitive(PType::I32, Nullability::NonNullable),
812        );
813
814        let first_ref = first.as_typed_ref(&DType::Primitive(PType::I32, Nullability::NonNullable));
815        assert!(first_ref.get(Stat::Min).is_absent());
816    }
817
818    #[test]
819    fn merge_mins() {
820        let first = StatsSet::of(Stat::Min, Precision::exact(37)).merge_ordered(
821            &StatsSet::of(Stat::Min, Precision::exact(42)),
822            &DType::Primitive(PType::I32, Nullability::NonNullable),
823        );
824
825        let first_ref = first.as_typed_ref(&DType::Primitive(PType::I32, Nullability::NonNullable));
826        assert_eq!(first_ref.get_as::<i32>(Stat::Min), Precision::exact(37));
827    }
828
829    #[test]
830    fn merge_into_bound_max() {
831        let first = StatsSet::of(Stat::Max, Precision::exact(42)).merge_ordered(
832            &StatsSet::default(),
833            &DType::Primitive(PType::I32, Nullability::NonNullable),
834        );
835        assert!(first.get(Stat::Max).is_absent());
836    }
837
838    #[test]
839    fn merge_from_max() {
840        let first = StatsSet::default().merge_ordered(
841            &StatsSet::of(Stat::Max, Precision::exact(42)),
842            &DType::Primitive(PType::I32, Nullability::NonNullable),
843        );
844        assert!(first.get(Stat::Max).is_absent());
845    }
846
847    #[test]
848    fn merge_maxes() {
849        let first = StatsSet::of(Stat::Max, Precision::exact(37)).merge_ordered(
850            &StatsSet::of(Stat::Max, Precision::exact(42)),
851            &DType::Primitive(PType::I32, Nullability::NonNullable),
852        );
853        let first_ref = first.as_typed_ref(&DType::Primitive(PType::I32, Nullability::NonNullable));
854        assert_eq!(first_ref.get_as::<i32>(Stat::Max), Precision::exact(42));
855    }
856
857    #[test]
858    fn merge_maxes_bound() {
859        let dtype = DType::Primitive(PType::I32, Nullability::NonNullable);
860        let first = StatsSet::of(Stat::Max, Precision::exact(42i32))
861            .merge_ordered(&StatsSet::of(Stat::Max, Precision::inexact(43i32)), &dtype);
862        let first_ref = first.as_typed_ref(&dtype);
863        assert_eq!(first_ref.get_as::<i32>(Stat::Max), Precision::inexact(43));
864    }
865
866    #[test]
867    fn merge_into_scalar() {
868        // Sum stats for primitive types are always the 64-bit version (i64 for signed, u64
869        // for unsigned, f64 for floats).
870        let first = StatsSet::of(Stat::Sum, Precision::exact(42i64)).merge_ordered(
871            &StatsSet::default(),
872            &DType::Primitive(PType::I32, Nullability::NonNullable),
873        );
874        let first_ref = first.as_typed_ref(&DType::Primitive(PType::I32, Nullability::NonNullable));
875        assert!(first_ref.get(Stat::Sum).is_absent());
876    }
877
878    #[test]
879    fn merge_from_scalar() {
880        // Sum stats for primitive types are always the 64-bit version (i64 for signed, u64
881        // for unsigned, f64 for floats).
882        let first = StatsSet::default().merge_ordered(
883            &StatsSet::of(Stat::Sum, Precision::exact(42i64)),
884            &DType::Primitive(PType::I32, Nullability::NonNullable),
885        );
886        let first_ref = first.as_typed_ref(&DType::Primitive(PType::I32, Nullability::NonNullable));
887        assert!(first_ref.get(Stat::Sum).is_absent());
888    }
889
890    #[test]
891    fn merge_scalars() {
892        // Sum stats for primitive types are always the 64-bit version (i64 for signed, u64
893        // for unsigned, f64 for floats).
894        let first = StatsSet::of(Stat::Sum, Precision::exact(37i64)).merge_ordered(
895            &StatsSet::of(Stat::Sum, Precision::exact(42i64)),
896            &DType::Primitive(PType::I32, Nullability::NonNullable),
897        );
898        let first_ref = first.as_typed_ref(&DType::Primitive(PType::I32, Nullability::NonNullable));
899        assert_eq!(first_ref.get_as::<i64>(Stat::Sum), Precision::exact(79i64));
900    }
901
902    #[test]
903    fn merge_into_sortedness() {
904        let first = StatsSet::of(Stat::IsStrictSorted, Precision::exact(true)).merge_ordered(
905            &StatsSet::default(),
906            &DType::Primitive(PType::I32, Nullability::NonNullable),
907        );
908        assert!(first.get(Stat::IsStrictSorted).is_absent());
909    }
910
911    #[test]
912    fn merge_from_sortedness() {
913        let first = StatsSet::default().merge_ordered(
914            &StatsSet::of(Stat::IsStrictSorted, Precision::exact(true)),
915            &DType::Primitive(PType::I32, Nullability::NonNullable),
916        );
917        assert!(first.get(Stat::IsStrictSorted).is_absent());
918    }
919
920    #[test]
921    fn merge_sortedness() {
922        let mut first = StatsSet::of(Stat::IsStrictSorted, Precision::exact(true));
923        first.set(Stat::Max, Precision::exact(1));
924        let mut second = StatsSet::of(Stat::IsStrictSorted, Precision::exact(true));
925        second.set(Stat::Min, Precision::exact(2));
926        first = first.merge_ordered(
927            &second,
928            &DType::Primitive(PType::I32, Nullability::NonNullable),
929        );
930
931        let first_ref = first.as_typed_ref(&DType::Primitive(PType::I32, Nullability::NonNullable));
932        assert_eq!(
933            first_ref.get_as::<bool>(Stat::IsStrictSorted),
934            Precision::exact(true)
935        );
936    }
937
938    #[test]
939    fn merge_sortedness_out_of_order() {
940        let mut first = StatsSet::of(Stat::IsStrictSorted, Precision::exact(true));
941        first.set(Stat::Min, Precision::exact(1));
942        let mut second = StatsSet::of(Stat::IsStrictSorted, Precision::exact(true));
943        second.set(Stat::Max, Precision::exact(2));
944        second = second.merge_ordered(
945            &first,
946            &DType::Primitive(PType::I32, Nullability::NonNullable),
947        );
948
949        let second_ref =
950            second.as_typed_ref(&DType::Primitive(PType::I32, Nullability::NonNullable));
951        assert_eq!(
952            second_ref.get_as::<bool>(Stat::IsStrictSorted),
953            Precision::inexact(false)
954        );
955    }
956
957    #[test]
958    fn merge_sortedness_only_one_sorted() {
959        let mut first = StatsSet::of(Stat::IsStrictSorted, Precision::exact(true));
960        first.set(Stat::Max, Precision::exact(1));
961        let mut second = StatsSet::of(Stat::IsStrictSorted, Precision::exact(false));
962        second.set(Stat::Min, Precision::exact(2));
963        let merged = first.merge_ordered(
964            &second,
965            &DType::Primitive(PType::I32, Nullability::NonNullable),
966        );
967
968        assert!(merged.get(Stat::IsStrictSorted).is_absent());
969    }
970
971    #[test]
972    fn merge_sortedness_missing_min() {
973        let mut first = StatsSet::of(Stat::IsStrictSorted, Precision::exact(true));
974        first.set(Stat::Max, Precision::exact(1));
975        let second = StatsSet::of(Stat::IsStrictSorted, Precision::exact(true));
976        first = first.merge_ordered(
977            &second,
978            &DType::Primitive(PType::I32, Nullability::NonNullable),
979        );
980        assert!(first.get(Stat::IsStrictSorted).is_absent());
981    }
982
983    #[test]
984    fn merge_sortedness_bound_min() {
985        let mut first = StatsSet::of(Stat::IsStrictSorted, Precision::exact(true));
986        first.set(Stat::Max, Precision::exact(1));
987        let mut second = StatsSet::of(Stat::IsStrictSorted, Precision::exact(true));
988        second.set(Stat::Min, Precision::inexact(2));
989        first = first.merge_ordered(
990            &second,
991            &DType::Primitive(PType::I32, Nullability::NonNullable),
992        );
993
994        let first_ref = first.as_typed_ref(&DType::Primitive(PType::I32, Nullability::NonNullable));
995        assert_eq!(
996            first_ref.get_as::<bool>(Stat::IsStrictSorted),
997            Precision::exact(true)
998        );
999    }
1000
1001    #[test]
1002    fn merge_unordered() {
1003        let array =
1004            PrimitiveArray::from_option_iter([Some(1), None, Some(2), Some(42), Some(10000), None]);
1005        let all_stats = all::<Stat>()
1006            .filter(|s| !matches!(s, Stat::Sum))
1007            .filter(|s| !matches!(s, Stat::NaNCount))
1008            .collect_vec();
1009        array
1010            .statistics()
1011            .compute_all(&all_stats, &mut SESSION.create_execution_ctx())
1012            .unwrap();
1013
1014        let stats = array.statistics().to_owned();
1015        for stat in &all_stats {
1016            assert!(!stats.get(*stat).is_absent(), "Stat {stat} is missing");
1017        }
1018
1019        let merged = stats.clone().merge_unordered(
1020            &stats,
1021            &DType::Primitive(PType::I32, Nullability::NonNullable),
1022        );
1023        for (stat, expected) in [
1024            (Stat::Min, Precision::exact(ScalarValue::from(1i32))),
1025            (Stat::Max, Precision::exact(ScalarValue::from(10000i32))),
1026            (Stat::NullCount, Precision::exact(ScalarValue::from(4u64))),
1027            (
1028                Stat::UncompressedSizeInBytes,
1029                Precision::exact(ScalarValue::from(50u64)),
1030            ),
1031            (Stat::IsConstant, Precision::exact(ScalarValue::from(false))),
1032            (Stat::IsSorted, Precision::Absent),
1033            (Stat::IsStrictSorted, Precision::Absent),
1034        ] {
1035            assert_eq!(merged.get(stat), expected, "{stat}");
1036        }
1037    }
1038
1039    #[test]
1040    fn merge_min_bound_same() {
1041        // Merging a stat with a bound and another with an exact results in exact stat.
1042        // since bound for min is a lower bound, it can in fact contain any value >= bound.
1043        let merged = StatsSet::of(Stat::Min, Precision::inexact(5)).merge_ordered(
1044            &StatsSet::of(Stat::Min, Precision::exact(5)),
1045            &DType::Primitive(PType::I32, Nullability::NonNullable),
1046        );
1047        let merged_ref =
1048            merged.as_typed_ref(&DType::Primitive(PType::I32, Nullability::NonNullable));
1049        assert_eq!(merged_ref.get_as::<i32>(Stat::Min), Precision::exact(5));
1050    }
1051
1052    #[test]
1053    fn merge_min_bound_bound_lower() {
1054        let merged = StatsSet::of(Stat::Min, Precision::inexact(4)).merge_ordered(
1055            &StatsSet::of(Stat::Min, Precision::exact(5)),
1056            &DType::Primitive(PType::I32, Nullability::NonNullable),
1057        );
1058        let merged_ref =
1059            merged.as_typed_ref(&DType::Primitive(PType::I32, Nullability::NonNullable));
1060        assert_eq!(merged_ref.get_as::<i32>(Stat::Min), Precision::inexact(4));
1061    }
1062
1063    #[test]
1064    fn test_combine_is_constant() {
1065        {
1066            let mut stats = StatsSet::of(Stat::IsConstant, Precision::exact(true));
1067            let stats2 = StatsSet::of(Stat::IsConstant, Precision::exact(true));
1068            let mut stats_ref =
1069                stats.as_mut_typed_ref(&DType::Primitive(PType::I32, Nullability::NonNullable));
1070            stats_ref
1071                .combine_bool_stat::<IsConstant>(
1072                    &stats2.as_typed_ref(&DType::Primitive(PType::I32, Nullability::NonNullable)),
1073                )
1074                .unwrap();
1075            assert_eq!(
1076                stats_ref.get_as::<bool>(Stat::IsConstant),
1077                Precision::exact(true)
1078            );
1079        }
1080
1081        {
1082            let mut stats = StatsSet::of(Stat::IsConstant, Precision::exact(true));
1083            let stats2 = StatsSet::of(Stat::IsConstant, Precision::inexact(false));
1084            let mut stats_ref =
1085                stats.as_mut_typed_ref(&DType::Primitive(PType::I32, Nullability::NonNullable));
1086            stats_ref
1087                .combine_bool_stat::<IsConstant>(
1088                    &stats2.as_typed_ref(&DType::Primitive(PType::I32, Nullability::NonNullable)),
1089                )
1090                .unwrap();
1091            assert_eq!(
1092                stats_ref.get_as::<bool>(Stat::IsConstant),
1093                Precision::exact(true)
1094            );
1095        }
1096
1097        {
1098            let mut stats = StatsSet::of(Stat::IsConstant, Precision::exact(false));
1099            let stats2 = StatsSet::of(Stat::IsConstant, Precision::inexact(false));
1100            let mut stats_ref =
1101                stats.as_mut_typed_ref(&DType::Primitive(PType::I32, Nullability::NonNullable));
1102            stats_ref
1103                .combine_bool_stat::<IsConstant>(
1104                    &stats2.as_typed_ref(&DType::Primitive(PType::I32, Nullability::NonNullable)),
1105                )
1106                .unwrap();
1107            assert_eq!(
1108                stats_ref.get_as::<bool>(Stat::IsConstant),
1109                Precision::exact(false)
1110            );
1111        }
1112    }
1113
1114    #[test]
1115    fn test_combine_sets_boolean_conflict() {
1116        let mut stats1 = StatsSet::from_iter([
1117            (Stat::IsConstant, Precision::exact(true)),
1118            (Stat::IsSorted, Precision::exact(true)),
1119        ]);
1120
1121        let stats2 = StatsSet::from_iter([
1122            (Stat::IsConstant, Precision::exact(false)),
1123            (Stat::IsSorted, Precision::exact(true)),
1124        ]);
1125
1126        let result = stats1.combine_sets(
1127            &stats2,
1128            &DType::Primitive(PType::I32, Nullability::NonNullable),
1129        );
1130        assert!(result.is_err());
1131    }
1132
1133    #[test]
1134    fn test_combine_sets_with_missing_stats() {
1135        let mut stats1 = StatsSet::from_iter([
1136            (Stat::Min, Precision::exact(42)),
1137            (Stat::UncompressedSizeInBytes, Precision::exact(1000u64)),
1138        ]);
1139
1140        let stats2 = StatsSet::from_iter([
1141            (Stat::Max, Precision::exact(100)),
1142            (Stat::IsStrictSorted, Precision::exact(true)),
1143        ]);
1144
1145        stats1
1146            .combine_sets(
1147                &stats2,
1148                &DType::Primitive(PType::I32, Nullability::NonNullable),
1149            )
1150            .unwrap();
1151
1152        let stats_ref =
1153            stats1.as_typed_ref(&DType::Primitive(PType::I32, Nullability::NonNullable));
1154
1155        assert_eq!(
1156            stats_ref.get_as::<u64>(Stat::UncompressedSizeInBytes),
1157            Precision::exact(1000u64)
1158        );
1159        // Min should remain unchanged
1160        assert_eq!(stats_ref.get_as::<i32>(Stat::Min), Precision::exact(42));
1161        // Max should be added
1162        assert_eq!(stats_ref.get_as::<i32>(Stat::Max), Precision::exact(100));
1163        // IsStrictSorted should be added
1164        assert_eq!(
1165            stats_ref.get_as::<bool>(Stat::IsStrictSorted),
1166            Precision::exact(true)
1167        );
1168    }
1169
1170    #[test]
1171    fn test_combine_sets_with_inexact() {
1172        let mut stats1 = StatsSet::from_iter([
1173            (Stat::Min, Precision::exact(42)),
1174            (Stat::Max, Precision::inexact(100)),
1175            (Stat::IsConstant, Precision::exact(false)),
1176        ]);
1177
1178        let stats2 = StatsSet::from_iter([
1179            // Must ensure Min from stats2 is <= Min from stats1
1180            (Stat::Min, Precision::inexact(40)),
1181            (Stat::Max, Precision::exact(90)),
1182            (Stat::IsSorted, Precision::exact(true)),
1183        ]);
1184
1185        stats1
1186            .combine_sets(
1187                &stats2,
1188                &DType::Primitive(PType::I32, Nullability::NonNullable),
1189            )
1190            .unwrap();
1191
1192        let stats_ref =
1193            stats1.as_typed_ref(&DType::Primitive(PType::I32, Nullability::NonNullable));
1194
1195        assert_eq!(
1196            stats_ref.get_as::<bool>(Stat::IsConstant),
1197            Precision::exact(false)
1198        );
1199        // Min should remain unchanged since it's more restrictive than the inexact value
1200        assert_eq!(stats_ref.get_as::<i32>(Stat::Min), Precision::exact(42));
1201        // Check that max was updated with the exact value
1202        assert_eq!(stats_ref.get_as::<i32>(Stat::Max), Precision::exact(90));
1203        // Check that IsSorted was added
1204        assert_eq!(
1205            stats_ref.get_as::<bool>(Stat::IsSorted),
1206            Precision::exact(true)
1207        );
1208    }
1209}