1use std::sync::Arc;
7use std::sync::OnceLock;
8
9use parking_lot::RwLock;
10use vortex_array::ExecutionCtx;
11use vortex_error::VortexError;
12use vortex_error::VortexResult;
13use vortex_error::vortex_panic;
14
15use super::MutTypedStatsSetRef;
16use super::StatsSet;
17use super::StatsSetIntoIter;
18use super::TypedStatsSetRef;
19use crate::ArrayRef;
20use crate::aggregate_fn::NumericalAggregateOpts;
21use crate::aggregate_fn::fns::is_constant::is_constant;
22use crate::aggregate_fn::fns::is_sorted::is_sorted;
23use crate::aggregate_fn::fns::is_sorted::is_strict_sorted;
24use crate::aggregate_fn::fns::min_max::MinMaxResult;
25use crate::aggregate_fn::fns::min_max::min_max;
26use crate::aggregate_fn::fns::nan_count::nan_count;
27use crate::aggregate_fn::fns::sum::sum;
28use crate::aggregate_fn::fns::uncompressed_size_in_bytes::uncompressed_size_in_bytes;
29use crate::expr::stats::Precision;
30use crate::expr::stats::Stat;
31use crate::expr::stats::StatsProvider;
32use crate::scalar::Scalar;
33use crate::scalar::ScalarValue;
34
35#[derive(Default, Debug)]
39pub struct ArrayStats {
40 inner: OnceLock<Arc<RwLock<StatsSet>>>,
41}
42
43impl Clone for ArrayStats {
44 fn clone(&self) -> Self {
45 Self {
46 inner: OnceLock::from(Arc::clone(self.shared())),
47 }
48 }
49}
50
51pub struct StatsSetRef<'a> {
55 dyn_array_ref: &'a ArrayRef,
57 array_stats: &'a ArrayStats,
58}
59
60impl ArrayStats {
61 pub fn to_ref<'a>(&'a self, array: &'a ArrayRef) -> StatsSetRef<'a> {
62 StatsSetRef {
63 dyn_array_ref: array,
64 array_stats: self,
65 }
66 }
67
68 fn shared(&self) -> &Arc<RwLock<StatsSet>> {
69 self.inner.get_or_init(Default::default)
70 }
71
72 fn read<R>(&self, f: impl FnOnce(&StatsSet) -> R) -> R {
73 match self.inner.get() {
74 Some(shared) => f(&shared.read()),
75 None => f(&StatsSet::default()),
76 }
77 }
78
79 pub fn set(&self, stat: Stat, value: Precision<ScalarValue>) {
80 self.shared().write().set(stat, value);
81 }
82
83 pub fn clear(&self, stat: Stat) {
84 if let Some(shared) = self.inner.get() {
85 shared.write().clear(stat);
86 }
87 }
88
89 pub fn retain(&self, stats: &[Stat]) {
90 if let Some(shared) = self.inner.get() {
91 shared.write().retain_only(stats);
92 }
93 }
94}
95
96impl From<StatsSet> for ArrayStats {
97 fn from(value: StatsSet) -> Self {
98 Self {
99 inner: OnceLock::from(Arc::new(RwLock::new(value))),
100 }
101 }
102}
103
104impl From<ArrayStats> for StatsSet {
105 fn from(value: ArrayStats) -> Self {
106 value.read(|stats| stats.clone())
107 }
108}
109
110impl StatsSetRef<'_> {
111 pub(crate) fn replace(&self, stats: StatsSet) {
112 if stats.is_empty() && self.array_stats.inner.get().is_none() {
113 return;
114 }
115 *self.array_stats.shared().write() = stats;
116 }
117
118 pub fn set_all(&self, stats: StatsSet) {
123 if stats.is_empty() {
124 return;
125 }
126 let mut guard = self.array_stats.shared().write();
127 if guard.is_empty() {
128 *guard = stats;
129 } else {
130 for (stat, value) in stats {
131 guard.set(stat, value);
132 }
133 }
134 }
135
136 pub fn set_iter(&self, iter: StatsSetIntoIter) {
137 let mut iter = iter.peekable();
138 if iter.peek().is_none() {
139 return;
140 }
141 let mut guard = self.array_stats.shared().write();
142 for (stat, value) in iter {
143 guard.set(stat, value);
144 }
145 }
146
147 pub(crate) fn transfer_from(&self, stats: &ArrayStats) {
150 let Some(source) = stats.inner.get() else {
151 return;
152 };
153
154 if self
155 .array_stats
156 .inner
157 .get()
158 .is_some_and(|shared| Arc::ptr_eq(shared, source))
159 {
160 return;
161 }
162
163 let source = source.read();
164 if source.is_empty() {
165 return;
166 }
167
168 let mut guard = self.array_stats.shared().write();
169 for (stat, value) in source.iter() {
170 guard.set(*stat, value.clone());
171 }
172 }
173
174 pub fn inherit_from(&self, stats: StatsSetRef<'_>) {
175 let Some(source) = stats.array_stats.inner.get() else {
176 return;
177 };
178 if self
180 .array_stats
181 .inner
182 .get()
183 .is_none_or(|shared| !Arc::ptr_eq(shared, source))
184 {
185 stats.with_iter(|iter| self.inherit(iter));
186 }
187 }
188
189 pub fn inherit<'a>(&self, iter: impl Iterator<Item = &'a (Stat, Precision<ScalarValue>)>) {
190 let mut iter = iter.peekable();
191 if iter.peek().is_none() {
192 return;
193 }
194 let mut guard = self.array_stats.shared().write();
195 for (stat, value) in iter {
196 if !value.is_exact() {
197 if !guard.get(*stat).is_exact() {
198 guard.set(*stat, value.clone());
199 }
200 } else {
201 guard.set(*stat, value.clone());
202 }
203 }
204 }
205
206 pub fn with_typed_stats_set<U, F: FnOnce(TypedStatsSetRef) -> U>(&self, apply: F) -> U {
207 self.array_stats
208 .read(|stats| apply(stats.as_typed_ref(self.dyn_array_ref.dtype())))
209 }
210
211 pub fn with_mut_typed_stats_set<U, F: FnOnce(MutTypedStatsSetRef) -> U>(&self, apply: F) -> U {
212 apply(
213 self.array_stats
214 .shared()
215 .write()
216 .as_mut_typed_ref(self.dyn_array_ref.dtype()),
217 )
218 }
219
220 pub fn to_owned(&self) -> StatsSet {
221 self.array_stats.read(|stats| stats.clone())
222 }
223
224 pub fn to_array_stats(&self) -> ArrayStats {
226 self.array_stats.clone()
227 }
228
229 pub(crate) fn share_existing(&self) -> ArrayStats {
231 ArrayStats {
232 inner: match self.array_stats.inner.get() {
233 Some(shared) => OnceLock::from(Arc::clone(shared)),
234 None => OnceLock::new(),
235 },
236 }
237 }
238
239 pub fn with_iter<
240 F: for<'a> FnOnce(&mut dyn Iterator<Item = &'a (Stat, Precision<ScalarValue>)>) -> R,
241 R,
242 >(
243 &self,
244 f: F,
245 ) -> R {
246 self.array_stats.read(|stats| f(&mut stats.iter()))
247 }
248
249 pub fn compute_stat(&self, stat: Stat, ctx: &mut ExecutionCtx) -> VortexResult<Option<Scalar>> {
252 if let Precision::Exact(s) = self.get(stat) {
254 return Ok(Some(s));
255 }
256
257 Ok(match stat {
258 Stat::Min => min_max(self.dyn_array_ref, ctx, NumericalAggregateOpts::default())?
259 .map(|MinMaxResult { min, max: _ }| min),
260 Stat::Max => min_max(self.dyn_array_ref, ctx, NumericalAggregateOpts::default())?
261 .map(|MinMaxResult { min: _, max }| max),
262 Stat::Sum => {
263 Stat::Sum
264 .dtype(self.dyn_array_ref.dtype())
265 .is_some()
266 .then(|| {
267 sum(self.dyn_array_ref, ctx)
269 })
270 .transpose()?
271 }
272 Stat::NullCount => self.dyn_array_ref.invalid_count(ctx).ok().map(Into::into),
273 Stat::IsConstant => {
274 if self.dyn_array_ref.is_empty() {
275 None
276 } else {
277 Some(is_constant(self.dyn_array_ref, ctx)?.into())
278 }
279 }
280 Stat::IsSorted => Some(is_sorted(self.dyn_array_ref, ctx)?.into()),
281 Stat::IsStrictSorted => Some(is_strict_sorted(self.dyn_array_ref, ctx)?.into()),
282 Stat::UncompressedSizeInBytes => Stat::UncompressedSizeInBytes
283 .dtype(self.dyn_array_ref.dtype())
284 .is_some()
285 .then(|| uncompressed_size_in_bytes(self.dyn_array_ref, ctx))
286 .transpose()?
287 .map(|s| s.into()),
288 Stat::NaNCount => {
289 Stat::NaNCount
290 .dtype(self.dyn_array_ref.dtype())
291 .is_some()
292 .then(|| {
293 nan_count(self.dyn_array_ref, ctx)
295 })
296 .transpose()?
297 .map(|s| s.into())
298 }
299 })
300 }
301
302 pub fn compute_all(&self, stats: &[Stat], ctx: &mut ExecutionCtx) -> VortexResult<StatsSet> {
303 let mut stats_set = StatsSet::default();
304 for &stat in stats {
305 if let Some(s) = self.compute_stat(stat, ctx)?
306 && let Some(value) = s.into_value()
307 {
308 stats_set.set(stat, Precision::exact(value));
309 }
310 }
311 Ok(stats_set)
312 }
313}
314
315impl StatsSetRef<'_> {
316 pub fn compute_as<U: for<'a> TryFrom<&'a Scalar, Error = VortexError>>(
317 &self,
318 stat: Stat,
319 ctx: &mut ExecutionCtx,
320 ) -> Option<U> {
321 self.compute_stat(stat, ctx)
322 .inspect_err(|e| tracing::warn!("Failed to compute stat {stat}: {e}"))
323 .ok()
324 .flatten()
325 .map(|s| U::try_from(&s))
326 .transpose()
327 .unwrap_or_else(|err| {
328 vortex_panic!(
329 err,
330 "Failed to compute stat {} as {}",
331 stat,
332 std::any::type_name::<U>()
333 )
334 })
335 }
336
337 pub fn set(&self, stat: Stat, value: Precision<ScalarValue>) {
338 self.array_stats.set(stat, value);
339 }
340
341 pub fn clear(&self, stat: Stat) {
342 self.array_stats.clear(stat);
343 }
344
345 pub fn compute_min<U: for<'a> TryFrom<&'a Scalar, Error = VortexError>>(
346 &self,
347 ctx: &mut ExecutionCtx,
348 ) -> Option<U> {
349 self.compute_as(Stat::Min, ctx)
350 }
351
352 pub fn compute_max<U: for<'a> TryFrom<&'a Scalar, Error = VortexError>>(
353 &self,
354 ctx: &mut ExecutionCtx,
355 ) -> Option<U> {
356 self.compute_as(Stat::Max, ctx)
357 }
358
359 pub fn compute_is_sorted(&self, ctx: &mut ExecutionCtx) -> Option<bool> {
360 self.compute_bool(Stat::IsSorted, ctx)
361 }
362
363 pub fn compute_is_strict_sorted(&self, ctx: &mut ExecutionCtx) -> Option<bool> {
364 self.compute_bool(Stat::IsStrictSorted, ctx)
365 }
366
367 pub fn compute_is_constant(&self, ctx: &mut ExecutionCtx) -> Option<bool> {
368 self.compute_bool(Stat::IsConstant, ctx)
369 }
370
371 pub fn compute_null_count(&self, ctx: &mut ExecutionCtx) -> Option<usize> {
372 self.compute_usize(Stat::NullCount, ctx)
373 }
374
375 pub fn compute_uncompressed_size_in_bytes(&self, ctx: &mut ExecutionCtx) -> Option<usize> {
376 self.compute_usize(Stat::UncompressedSizeInBytes, ctx)
377 }
378
379 fn compute_bool(&self, stat: Stat, ctx: &mut ExecutionCtx) -> Option<bool> {
381 let cached = self.array_stats.read(|stats| stats.get_bool(stat));
383 if let Precision::Exact(value) = cached {
384 return Some(value);
385 }
386
387 self.compute_as(stat, ctx)
388 }
389
390 fn compute_usize(&self, stat: Stat, ctx: &mut ExecutionCtx) -> Option<usize> {
392 let cached = self.array_stats.read(|stats| stats.get_usize(stat));
394 if let Precision::Exact(value) = cached {
395 return Some(value);
396 }
397
398 self.compute_as(stat, ctx)
399 }
400}
401
402impl StatsProvider for StatsSetRef<'_> {
403 fn get(&self, stat: Stat) -> Precision<Scalar> {
404 self.array_stats
405 .read(|stats| stats.as_typed_ref(self.dyn_array_ref.dtype()).get(stat))
406 }
407
408 fn len(&self) -> usize {
409 self.array_stats.read(|stats| stats.len())
410 }
411}
412
413#[cfg(test)]
414mod tests {
415 use vortex_buffer::buffer;
416
417 use super::ArrayStats;
418 use super::StatsSet;
419 use crate::IntoArray;
420 use crate::expr::stats::Precision;
421 use crate::expr::stats::Stat;
422 use crate::scalar::ScalarValue;
423
424 #[test]
425 fn empty_stats() {
426 let stats = ArrayStats::default();
427 assert_eq!(StatsSet::from(stats).len(), 0);
428 }
429
430 #[test]
431 fn clone_stats() {
432 let stats = ArrayStats::default();
433 let copy = stats.clone();
434
435 stats.set(Stat::NullCount, Precision::exact(ScalarValue::from(3u64)));
436 let through_copy = StatsSet::from(copy);
437 assert_eq!(
438 through_copy.get(Stat::NullCount),
439 Precision::exact(ScalarValue::from(3u64))
440 );
441 }
442
443 #[test]
444 fn clone_writes() {
445 let stats = ArrayStats::default();
446 let copy = stats.clone();
447
448 copy.set(Stat::NullCount, Precision::exact(ScalarValue::from(7u64)));
449 assert_eq!(
450 StatsSet::from(stats).get(Stat::NullCount),
451 Precision::exact(ScalarValue::from(7u64))
452 );
453 }
454
455 #[test]
456 fn transfer_from_copies_entries_without_touching_the_source() {
457 let source = buffer![1i32, 2, 3].into_array();
458 let target = buffer![4i32, 5, 6].into_array();
459 source.statistics().set(Stat::Max, Precision::exact(3i32));
460 target.statistics().set(Stat::Max, Precision::inexact(9i32));
461 target.statistics().set(Stat::Min, Precision::exact(4i32));
462
463 target
464 .statistics()
465 .transfer_from(&source.statistics().to_array_stats());
466
467 assert_eq!(
468 target.statistics().to_owned().get(Stat::Max),
469 Precision::exact(3i32)
470 );
471 assert_eq!(
472 target.statistics().to_owned().get(Stat::Min),
473 Precision::exact(4i32)
474 );
475 assert_eq!(source.statistics().to_owned().len(), 1);
476 }
477
478 #[test]
479 fn transfer_from_empty_or_self_is_a_no_op() {
480 let empty = buffer![1i32].into_array();
481 let target = buffer![2i32].into_array();
482 target.statistics().set(Stat::Max, Precision::exact(2i32));
483
484 target
485 .statistics()
486 .transfer_from(&empty.statistics().to_array_stats());
487 target
488 .statistics()
489 .transfer_from(&target.statistics().to_array_stats());
490
491 assert_eq!(target.statistics().to_owned().len(), 1);
492 assert_eq!(
493 target.statistics().to_owned().get(Stat::Max),
494 Precision::exact(2i32)
495 );
496 }
497
498 #[test]
499 fn transfer_from_uninitialised_source_keeps_target_lazy() {
500 let source = buffer![1i32].into_array();
501 let target = buffer![2i32].into_array();
502
503 target
504 .statistics()
505 .transfer_from(&source.statistics().share_existing());
506
507 assert!(source.statistics().array_stats.inner.get().is_none());
508 assert!(target.statistics().array_stats.inner.get().is_none());
509 }
510}