Skip to main content

vortex_array/stats/
array.rs

1// SPDX-License-Identifier: Apache-2.0
2// SPDX-FileCopyrightText: Copyright the Vortex contributors
3
4//! Stats as they are stored on arrays.
5
6use std::sync::Arc;
7use std::sync::OnceLock;
8
9use parking_lot::RwLock;
10use vortex_array::ExecutionCtx;
11use vortex_error::VortexError;
12use vortex_error::VortexResult;
13use vortex_error::vortex_panic;
14
15use super::MutTypedStatsSetRef;
16use super::StatsSet;
17use super::StatsSetIntoIter;
18use super::TypedStatsSetRef;
19use crate::ArrayRef;
20use crate::aggregate_fn::NumericalAggregateOpts;
21use crate::aggregate_fn::fns::is_constant::is_constant;
22use crate::aggregate_fn::fns::is_sorted::is_sorted;
23use crate::aggregate_fn::fns::is_sorted::is_strict_sorted;
24use crate::aggregate_fn::fns::min_max::MinMaxResult;
25use crate::aggregate_fn::fns::min_max::min_max;
26use crate::aggregate_fn::fns::nan_count::nan_count;
27use crate::aggregate_fn::fns::sum::sum;
28use crate::aggregate_fn::fns::uncompressed_size_in_bytes::uncompressed_size_in_bytes;
29use crate::expr::stats::Precision;
30use crate::expr::stats::Stat;
31use crate::expr::stats::StatsProvider;
32use crate::scalar::Scalar;
33use crate::scalar::ScalarValue;
34
35/// A shared [`StatsSet`] stored in an array. Can be shared by copies of the array and can also be mutated in place.
36/// Stats are created lazily on read or write.
37// TODO(adamg): This is a very bad name.
38#[derive(Default, Debug)]
39pub struct ArrayStats {
40    inner: OnceLock<Arc<RwLock<StatsSet>>>,
41}
42
43impl Clone for ArrayStats {
44    fn clone(&self) -> Self {
45        Self {
46            inner: OnceLock::from(Arc::clone(self.shared())),
47        }
48    }
49}
50
51/// Reference to an array's [`StatsSet`]. Can be used to get and mutate the underlying stats.
52///
53/// Constructed by calling [`ArrayStats::to_ref`].
54pub struct StatsSetRef<'a> {
55    // We need to reference back to the array
56    dyn_array_ref: &'a ArrayRef,
57    array_stats: &'a ArrayStats,
58}
59
60impl ArrayStats {
61    pub fn to_ref<'a>(&'a self, array: &'a ArrayRef) -> StatsSetRef<'a> {
62        StatsSetRef {
63            dyn_array_ref: array,
64            array_stats: self,
65        }
66    }
67
68    fn shared(&self) -> &Arc<RwLock<StatsSet>> {
69        self.inner.get_or_init(Default::default)
70    }
71
72    fn read<R>(&self, f: impl FnOnce(&StatsSet) -> R) -> R {
73        match self.inner.get() {
74            Some(shared) => f(&shared.read()),
75            None => f(&StatsSet::default()),
76        }
77    }
78
79    pub fn set(&self, stat: Stat, value: Precision<ScalarValue>) {
80        self.shared().write().set(stat, value);
81    }
82
83    pub fn clear(&self, stat: Stat) {
84        if let Some(shared) = self.inner.get() {
85            shared.write().clear(stat);
86        }
87    }
88
89    pub fn retain(&self, stats: &[Stat]) {
90        if let Some(shared) = self.inner.get() {
91            shared.write().retain_only(stats);
92        }
93    }
94}
95
96impl From<StatsSet> for ArrayStats {
97    fn from(value: StatsSet) -> Self {
98        Self {
99            inner: OnceLock::from(Arc::new(RwLock::new(value))),
100        }
101    }
102}
103
104impl From<ArrayStats> for StatsSet {
105    fn from(value: ArrayStats) -> Self {
106        value.read(|stats| stats.clone())
107    }
108}
109
110impl StatsSetRef<'_> {
111    pub(crate) fn replace(&self, stats: StatsSet) {
112        if stats.is_empty() && self.array_stats.inner.get().is_none() {
113            return;
114        }
115        *self.array_stats.shared().write() = stats;
116    }
117
118    /// Set every stat in `stats`, overwriting existing values for the same stats.
119    ///
120    /// When the array has no stats yet, the set is moved in whole rather than merged one stat at
121    /// a time.
122    pub fn set_all(&self, stats: StatsSet) {
123        if stats.is_empty() {
124            return;
125        }
126        let mut guard = self.array_stats.shared().write();
127        if guard.is_empty() {
128            *guard = stats;
129        } else {
130            for (stat, value) in stats {
131                guard.set(stat, value);
132            }
133        }
134    }
135
136    pub fn set_iter(&self, iter: StatsSetIntoIter) {
137        let mut iter = iter.peekable();
138        if iter.peek().is_none() {
139            return;
140        }
141        let mut guard = self.array_stats.shared().write();
142        for (stat, value) in iter {
143            guard.set(stat, value);
144        }
145    }
146
147    /// Copy every statistic of `stats` into this set, overwriting existing entries, without
148    /// cloning the source set. A no-op when both refer to the same stats or the source is empty.
149    pub(crate) fn transfer_from(&self, stats: &ArrayStats) {
150        let Some(source) = stats.inner.get() else {
151            return;
152        };
153
154        if self
155            .array_stats
156            .inner
157            .get()
158            .is_some_and(|shared| Arc::ptr_eq(shared, source))
159        {
160            return;
161        }
162
163        let source = source.read();
164        if source.is_empty() {
165            return;
166        }
167
168        let mut guard = self.array_stats.shared().write();
169        for (stat, value) in source.iter() {
170            guard.set(*stat, value.clone());
171        }
172    }
173
174    pub fn inherit_from(&self, stats: StatsSetRef<'_>) {
175        let Some(source) = stats.array_stats.inner.get() else {
176            return;
177        };
178        // Only inherit if the underlying stats are different
179        if self
180            .array_stats
181            .inner
182            .get()
183            .is_none_or(|shared| !Arc::ptr_eq(shared, source))
184        {
185            stats.with_iter(|iter| self.inherit(iter));
186        }
187    }
188
189    pub fn inherit<'a>(&self, iter: impl Iterator<Item = &'a (Stat, Precision<ScalarValue>)>) {
190        let mut iter = iter.peekable();
191        if iter.peek().is_none() {
192            return;
193        }
194        let mut guard = self.array_stats.shared().write();
195        for (stat, value) in iter {
196            if !value.is_exact() {
197                if !guard.get(*stat).is_exact() {
198                    guard.set(*stat, value.clone());
199                }
200            } else {
201                guard.set(*stat, value.clone());
202            }
203        }
204    }
205
206    pub fn with_typed_stats_set<U, F: FnOnce(TypedStatsSetRef) -> U>(&self, apply: F) -> U {
207        self.array_stats
208            .read(|stats| apply(stats.as_typed_ref(self.dyn_array_ref.dtype())))
209    }
210
211    pub fn with_mut_typed_stats_set<U, F: FnOnce(MutTypedStatsSetRef) -> U>(&self, apply: F) -> U {
212        apply(
213            self.array_stats
214                .shared()
215                .write()
216                .as_mut_typed_ref(self.dyn_array_ref.dtype()),
217        )
218    }
219
220    pub fn to_owned(&self) -> StatsSet {
221        self.array_stats.read(|stats| stats.clone())
222    }
223
224    /// Returns a clone of the underlying [`ArrayStats`].
225    pub fn to_array_stats(&self) -> ArrayStats {
226        self.array_stats.clone()
227    }
228
229    /// Share underlying stats if they exist or return a lazy instance
230    pub(crate) fn share_existing(&self) -> ArrayStats {
231        ArrayStats {
232            inner: match self.array_stats.inner.get() {
233                Some(shared) => OnceLock::from(Arc::clone(shared)),
234                None => OnceLock::new(),
235            },
236        }
237    }
238
239    pub fn with_iter<
240        F: for<'a> FnOnce(&mut dyn Iterator<Item = &'a (Stat, Precision<ScalarValue>)>) -> R,
241        R,
242    >(
243        &self,
244        f: F,
245    ) -> R {
246        self.array_stats.read(|stats| f(&mut stats.iter()))
247    }
248
249    /// Returns the value of `stat` by either fetching it from cache if it exists and is [`Precision::Exact`], or falling back to
250    /// computation. The underlying compute kernels will cache the computed stat in the latter case.
251    pub fn compute_stat(&self, stat: Stat, ctx: &mut ExecutionCtx) -> VortexResult<Option<Scalar>> {
252        // If it's already computed and exact, we can return it.
253        if let Precision::Exact(s) = self.get(stat) {
254            return Ok(Some(s));
255        }
256
257        Ok(match stat {
258            Stat::Min => min_max(self.dyn_array_ref, ctx, NumericalAggregateOpts::default())?
259                .map(|MinMaxResult { min, max: _ }| min),
260            Stat::Max => min_max(self.dyn_array_ref, ctx, NumericalAggregateOpts::default())?
261                .map(|MinMaxResult { min: _, max }| max),
262            Stat::Sum => {
263                Stat::Sum
264                    .dtype(self.dyn_array_ref.dtype())
265                    .is_some()
266                    .then(|| {
267                        // Sum is supported for this dtype.
268                        sum(self.dyn_array_ref, ctx)
269                    })
270                    .transpose()?
271            }
272            Stat::NullCount => self.dyn_array_ref.invalid_count(ctx).ok().map(Into::into),
273            Stat::IsConstant => {
274                if self.dyn_array_ref.is_empty() {
275                    None
276                } else {
277                    Some(is_constant(self.dyn_array_ref, ctx)?.into())
278                }
279            }
280            Stat::IsSorted => Some(is_sorted(self.dyn_array_ref, ctx)?.into()),
281            Stat::IsStrictSorted => Some(is_strict_sorted(self.dyn_array_ref, ctx)?.into()),
282            Stat::UncompressedSizeInBytes => Stat::UncompressedSizeInBytes
283                .dtype(self.dyn_array_ref.dtype())
284                .is_some()
285                .then(|| uncompressed_size_in_bytes(self.dyn_array_ref, ctx))
286                .transpose()?
287                .map(|s| s.into()),
288            Stat::NaNCount => {
289                Stat::NaNCount
290                    .dtype(self.dyn_array_ref.dtype())
291                    .is_some()
292                    .then(|| {
293                        // NaNCount is supported for this dtype.
294                        nan_count(self.dyn_array_ref, ctx)
295                    })
296                    .transpose()?
297                    .map(|s| s.into())
298            }
299        })
300    }
301
302    pub fn compute_all(&self, stats: &[Stat], ctx: &mut ExecutionCtx) -> VortexResult<StatsSet> {
303        let mut stats_set = StatsSet::default();
304        for &stat in stats {
305            if let Some(s) = self.compute_stat(stat, ctx)?
306                && let Some(value) = s.into_value()
307            {
308                stats_set.set(stat, Precision::exact(value));
309            }
310        }
311        Ok(stats_set)
312    }
313}
314
315impl StatsSetRef<'_> {
316    pub fn compute_as<U: for<'a> TryFrom<&'a Scalar, Error = VortexError>>(
317        &self,
318        stat: Stat,
319        ctx: &mut ExecutionCtx,
320    ) -> Option<U> {
321        self.compute_stat(stat, ctx)
322            .inspect_err(|e| tracing::warn!("Failed to compute stat {stat}: {e}"))
323            .ok()
324            .flatten()
325            .map(|s| U::try_from(&s))
326            .transpose()
327            .unwrap_or_else(|err| {
328                vortex_panic!(
329                    err,
330                    "Failed to compute stat {} as {}",
331                    stat,
332                    std::any::type_name::<U>()
333                )
334            })
335    }
336
337    pub fn set(&self, stat: Stat, value: Precision<ScalarValue>) {
338        self.array_stats.set(stat, value);
339    }
340
341    pub fn clear(&self, stat: Stat) {
342        self.array_stats.clear(stat);
343    }
344
345    pub fn compute_min<U: for<'a> TryFrom<&'a Scalar, Error = VortexError>>(
346        &self,
347        ctx: &mut ExecutionCtx,
348    ) -> Option<U> {
349        self.compute_as(Stat::Min, ctx)
350    }
351
352    pub fn compute_max<U: for<'a> TryFrom<&'a Scalar, Error = VortexError>>(
353        &self,
354        ctx: &mut ExecutionCtx,
355    ) -> Option<U> {
356        self.compute_as(Stat::Max, ctx)
357    }
358
359    pub fn compute_is_sorted(&self, ctx: &mut ExecutionCtx) -> Option<bool> {
360        self.compute_bool(Stat::IsSorted, ctx)
361    }
362
363    pub fn compute_is_strict_sorted(&self, ctx: &mut ExecutionCtx) -> Option<bool> {
364        self.compute_bool(Stat::IsStrictSorted, ctx)
365    }
366
367    pub fn compute_is_constant(&self, ctx: &mut ExecutionCtx) -> Option<bool> {
368        self.compute_bool(Stat::IsConstant, ctx)
369    }
370
371    pub fn compute_null_count(&self, ctx: &mut ExecutionCtx) -> Option<usize> {
372        self.compute_usize(Stat::NullCount, ctx)
373    }
374
375    pub fn compute_uncompressed_size_in_bytes(&self, ctx: &mut ExecutionCtx) -> Option<usize> {
376        self.compute_usize(Stat::UncompressedSizeInBytes, ctx)
377    }
378
379    /// Like [`Self::compute_as`], but reads a cached exact value without building a [`Scalar`].
380    fn compute_bool(&self, stat: Stat, ctx: &mut ExecutionCtx) -> Option<bool> {
381        // Bind the cached value first, so the read lock is released before any computation.
382        let cached = self.array_stats.read(|stats| stats.get_bool(stat));
383        if let Precision::Exact(value) = cached {
384            return Some(value);
385        }
386
387        self.compute_as(stat, ctx)
388    }
389
390    /// Like [`Self::compute_as`], but reads a cached exact value without building a [`Scalar`].
391    fn compute_usize(&self, stat: Stat, ctx: &mut ExecutionCtx) -> Option<usize> {
392        // Bind the cached value first, so the read lock is released before any computation.
393        let cached = self.array_stats.read(|stats| stats.get_usize(stat));
394        if let Precision::Exact(value) = cached {
395            return Some(value);
396        }
397
398        self.compute_as(stat, ctx)
399    }
400}
401
402impl StatsProvider for StatsSetRef<'_> {
403    fn get(&self, stat: Stat) -> Precision<Scalar> {
404        self.array_stats
405            .read(|stats| stats.as_typed_ref(self.dyn_array_ref.dtype()).get(stat))
406    }
407
408    fn len(&self) -> usize {
409        self.array_stats.read(|stats| stats.len())
410    }
411}
412
413#[cfg(test)]
414mod tests {
415    use vortex_buffer::buffer;
416
417    use super::ArrayStats;
418    use super::StatsSet;
419    use crate::IntoArray;
420    use crate::expr::stats::Precision;
421    use crate::expr::stats::Stat;
422    use crate::scalar::ScalarValue;
423
424    #[test]
425    fn empty_stats() {
426        let stats = ArrayStats::default();
427        assert_eq!(StatsSet::from(stats).len(), 0);
428    }
429
430    #[test]
431    fn clone_stats() {
432        let stats = ArrayStats::default();
433        let copy = stats.clone();
434
435        stats.set(Stat::NullCount, Precision::exact(ScalarValue::from(3u64)));
436        let through_copy = StatsSet::from(copy);
437        assert_eq!(
438            through_copy.get(Stat::NullCount),
439            Precision::exact(ScalarValue::from(3u64))
440        );
441    }
442
443    #[test]
444    fn clone_writes() {
445        let stats = ArrayStats::default();
446        let copy = stats.clone();
447
448        copy.set(Stat::NullCount, Precision::exact(ScalarValue::from(7u64)));
449        assert_eq!(
450            StatsSet::from(stats).get(Stat::NullCount),
451            Precision::exact(ScalarValue::from(7u64))
452        );
453    }
454
455    #[test]
456    fn transfer_from_copies_entries_without_touching_the_source() {
457        let source = buffer![1i32, 2, 3].into_array();
458        let target = buffer![4i32, 5, 6].into_array();
459        source.statistics().set(Stat::Max, Precision::exact(3i32));
460        target.statistics().set(Stat::Max, Precision::inexact(9i32));
461        target.statistics().set(Stat::Min, Precision::exact(4i32));
462
463        target
464            .statistics()
465            .transfer_from(&source.statistics().to_array_stats());
466
467        assert_eq!(
468            target.statistics().to_owned().get(Stat::Max),
469            Precision::exact(3i32)
470        );
471        assert_eq!(
472            target.statistics().to_owned().get(Stat::Min),
473            Precision::exact(4i32)
474        );
475        assert_eq!(source.statistics().to_owned().len(), 1);
476    }
477
478    #[test]
479    fn transfer_from_empty_or_self_is_a_no_op() {
480        let empty = buffer![1i32].into_array();
481        let target = buffer![2i32].into_array();
482        target.statistics().set(Stat::Max, Precision::exact(2i32));
483
484        target
485            .statistics()
486            .transfer_from(&empty.statistics().to_array_stats());
487        target
488            .statistics()
489            .transfer_from(&target.statistics().to_array_stats());
490
491        assert_eq!(target.statistics().to_owned().len(), 1);
492        assert_eq!(
493            target.statistics().to_owned().get(Stat::Max),
494            Precision::exact(2i32)
495        );
496    }
497
498    #[test]
499    fn transfer_from_uninitialised_source_keeps_target_lazy() {
500        let source = buffer![1i32].into_array();
501        let target = buffer![2i32].into_array();
502
503        target
504            .statistics()
505            .transfer_from(&source.statistics().share_existing());
506
507        assert!(source.statistics().array_stats.inner.get().is_none());
508        assert!(target.statistics().array_stats.inner.get().is_none());
509    }
510}