Skip to main content

vortex_array/aggregate_fn/
session.rs

1// SPDX-License-Identifier: Apache-2.0
2// SPDX-FileCopyrightText: Copyright the Vortex contributors
3
4use std::any::Any;
5use std::sync::Arc;
6
7use vortex_session::ArcSwapMap;
8use vortex_session::SessionExt;
9use vortex_session::SessionGuard;
10use vortex_session::SessionVar;
11use vortex_utils::aliases::hash_map::HashMap;
12
13use crate::aggregate_fn::AggregateFnId;
14use crate::aggregate_fn::AggregateFnPluginRef;
15use crate::aggregate_fn::AggregateFnRef;
16use crate::aggregate_fn::AggregateFnVTable;
17use crate::aggregate_fn::ExtensionStoragePushdown;
18use crate::aggregate_fn::fns::all_nan::AllNan;
19use crate::aggregate_fn::fns::all_non_distinct::AllNonDistinct;
20use crate::aggregate_fn::fns::all_non_nan::AllNonNan;
21use crate::aggregate_fn::fns::all_non_null::AllNonNull;
22use crate::aggregate_fn::fns::all_null::AllNull;
23use crate::aggregate_fn::fns::bounded_max::BoundedMax;
24use crate::aggregate_fn::fns::bounded_min::BoundedMin;
25use crate::aggregate_fn::fns::count::Count;
26use crate::aggregate_fn::fns::count::CountGroupedKernel;
27use crate::aggregate_fn::fns::first::First;
28use crate::aggregate_fn::fns::is_constant::IsConstant;
29use crate::aggregate_fn::fns::is_sorted::IsSorted;
30use crate::aggregate_fn::fns::last::Last;
31use crate::aggregate_fn::fns::max::Max;
32use crate::aggregate_fn::fns::mean::Mean;
33use crate::aggregate_fn::fns::min::Min;
34use crate::aggregate_fn::fns::min_max::MinMax;
35use crate::aggregate_fn::fns::nan_count::NanCount;
36use crate::aggregate_fn::fns::null_count::NullCount;
37use crate::aggregate_fn::fns::sum::PrimitiveGroupedSumEncodingKernel;
38use crate::aggregate_fn::fns::sum::Sum;
39use crate::aggregate_fn::fns::sum_v2::PrimitiveGroupedSumV2EncodingKernel;
40use crate::aggregate_fn::fns::sum_v2::SumV2;
41use crate::aggregate_fn::fns::uncompressed_size_in_bytes::UncompressedSizeInBytes;
42use crate::aggregate_fn::kernels::DynAggregateKernel;
43use crate::aggregate_fn::kernels::DynGroupedAggregateKernel;
44use crate::array::ArrayId;
45use crate::array::VTable;
46use crate::arrays::Chunked;
47use crate::arrays::Dict;
48use crate::arrays::Primitive;
49use crate::arrays::RunEnd;
50use crate::arrays::Widen;
51use crate::arrays::chunked::compute::aggregate::ChunkedArrayAggregate;
52use crate::arrays::dict::compute::is_constant::DictIsConstantKernel;
53use crate::arrays::dict::compute::is_sorted::DictIsSortedKernel;
54use crate::arrays::dict::compute::min_max::DictMinMaxKernel;
55use crate::arrays::runend::compute::is_constant::RunEndIsConstantKernel;
56use crate::arrays::runend::compute::is_sorted::RunEndIsSortedKernel;
57use crate::arrays::runend::compute::min_max::RunEndMinMaxKernel;
58use crate::arrays::runend::compute::sum::RunEndSumKernel;
59use crate::arrays::widen::compute::aggregate::WidenArrayAggregate;
60use crate::dtype::DType;
61use crate::dtype::extension::ExtVTable;
62use crate::extension::datetime::Date;
63use crate::extension::datetime::Time;
64use crate::extension::datetime::Timestamp;
65
66/// Session state for aggregate functions and encoding-specific aggregate kernels.
67///
68/// The default session registers the built-in aggregate functions and kernels. Additional
69/// aggregate functions and kernels may be registered by extensions when they are added to a
70/// [`VortexSession`](vortex_session::VortexSession).
71#[derive(Clone, Debug)]
72pub struct AggregateFnSession {
73    registry: AggregateFnRegistry,
74
75    kernels: AggregateKernelRegistry,
76    grouped_kernels: GroupedKernelRegistry,
77    grouped_encoding_kernels: GroupedEncodingKernelRegistry,
78}
79
80impl SessionVar for AggregateFnSession {
81    fn as_any(&self) -> &dyn Any {
82        self
83    }
84
85    fn as_any_mut(&mut self) -> &mut dyn Any {
86        self
87    }
88}
89
90type AggregateKernelKey = (ArrayId, Option<AggregateFnId>);
91type GroupedEncodingKernelKey = (ArrayId, AggregateFnId);
92
93/// Registry of aggregate function plugins, keyed by aggregate function id.
94type AggregateFnRegistry = ArcSwapMap<AggregateFnId, AggregateFnPluginRef>;
95/// Registry of aggregate kernels, keyed by encoding and optional aggregate function.
96type AggregateKernelRegistry = ArcSwapMap<AggregateKernelKey, &'static dyn DynAggregateKernel>;
97/// Registry of encoding-agnostic grouped aggregate kernels, keyed by aggregate function id.
98type GroupedKernelRegistry = ArcSwapMap<AggregateFnId, &'static dyn DynGroupedAggregateKernel>;
99/// Registry of grouped aggregate kernels, keyed by encoding and aggregate function.
100type GroupedEncodingKernelRegistry =
101    ArcSwapMap<GroupedEncodingKernelKey, &'static dyn DynGroupedAggregateKernel>;
102
103impl Default for AggregateFnSession {
104    fn default() -> Self {
105        let this = Self {
106            registry: AggregateFnRegistry::default(),
107            kernels: AggregateKernelRegistry::default(),
108            grouped_kernels: GroupedKernelRegistry::default(),
109            grouped_encoding_kernels: GroupedEncodingKernelRegistry::default(),
110        };
111
112        // Register the built-in aggregate functions
113        this.register(AllNonDistinct);
114        this.register(AllNonNan);
115        this.register(AllNonNull);
116        this.register(AllNan);
117        this.register(AllNull);
118        this.register(BoundedMax);
119        this.register(BoundedMin);
120        this.register(Count);
121        this.register(First);
122        this.register(IsConstant);
123        this.register(IsSorted);
124        this.register(Last);
125        this.register(Max);
126        this.register(Mean::combined());
127        this.register(Min);
128        this.register(MinMax);
129        this.register(NanCount);
130        this.register(NullCount);
131        this.register(Sum);
132        this.register(SumV2);
133        this.register(UncompressedSizeInBytes);
134
135        // Temporal extension types compare like their integer storage, so comparison-based
136        // aggregates are pushed down into it.
137        this.register_storage_ordered(Date);
138        this.register_storage_ordered(Time);
139        this.register_storage_ordered(Timestamp);
140
141        // Register the built-in aggregate kernels.
142        this.register_aggregate_kernel(Chunked.id(), None::<AggregateFnId>, &ChunkedArrayAggregate);
143        this.register_aggregate_kernel(Dict.id(), Some(MinMax.id()), &DictMinMaxKernel);
144        this.register_aggregate_kernel(Dict.id(), Some(IsConstant.id()), &DictIsConstantKernel);
145        this.register_aggregate_kernel(Dict.id(), Some(IsSorted.id()), &DictIsSortedKernel);
146        this.register_aggregate_kernel(RunEnd.id(), Some(MinMax.id()), &RunEndMinMaxKernel);
147        this.register_aggregate_kernel(RunEnd.id(), Some(IsConstant.id()), &RunEndIsConstantKernel);
148        this.register_aggregate_kernel(RunEnd.id(), Some(IsSorted.id()), &RunEndIsSortedKernel);
149        this.register_aggregate_kernel(RunEnd.id(), Some(Sum.id()), &RunEndSumKernel);
150        this.register_aggregate_kernel(RunEnd.id(), Some(SumV2.id()), &RunEndSumKernel);
151        this.register_aggregate_kernel(Widen.id(), Some(MinMax.id()), &WidenArrayAggregate);
152        this.register_aggregate_kernel(Widen.id(), Some(Min.id()), &WidenArrayAggregate);
153        this.register_aggregate_kernel(Widen.id(), Some(Max.id()), &WidenArrayAggregate);
154        this.register_aggregate_kernel(Widen.id(), Some(IsConstant.id()), &WidenArrayAggregate);
155        this.register_aggregate_kernel(Widen.id(), Some(IsSorted.id()), &WidenArrayAggregate);
156        this.register_aggregate_kernel(Widen.id(), Some(Sum.id()), &WidenArrayAggregate);
157
158        // Register the built-in grouped aggregate kernels.
159        this.register_grouped_kernel(Count.id(), &CountGroupedKernel);
160        this.register_grouped_encoding_kernel(
161            Primitive.id(),
162            Sum.id(),
163            &PrimitiveGroupedSumEncodingKernel,
164        );
165        this.register_grouped_encoding_kernel(
166            Primitive.id(),
167            SumV2.id(),
168            &PrimitiveGroupedSumV2EncodingKernel,
169        );
170
171        this
172    }
173}
174
175impl AggregateFnSession {
176    /// Returns the aggregate function plugin registered for `id`, if any.
177    pub fn find_plugin(&self, id: &AggregateFnId) -> Option<AggregateFnPluginRef> {
178        self.registry.get(id)
179    }
180
181    /// Register an aggregate function vtable in the session, replacing any existing vtable with
182    /// the same ID.
183    pub fn register<V: AggregateFnVTable>(&self, vtable: V) {
184        let id = vtable.id();
185        let pluginref = Arc::new(vtable) as AggregateFnPluginRef;
186        self.registry.insert(id, pluginref);
187    }
188
189    /// Push the comparison-based aggregates `is_sorted` and `min_max` down into the storage of
190    /// extension type `ext`, through [`ExtensionStoragePushdown`].
191    ///
192    /// Only call this when comparing two values of `ext` is the same as comparing their storage,
193    /// as for the temporal types and for UUIDs, which are stored big-endian. See
194    /// [`ExtensionStoragePushdown`] for how to check.
195    pub fn register_storage_ordered<E: ExtVTable>(&self, ext: E) {
196        self.register(ExtensionStoragePushdown::new(IsSorted, ext.clone()));
197        self.register(ExtensionStoragePushdown::new(MinMax, ext));
198    }
199
200    /// The default per-chunk zone statistics for a column of `input_dtype`, collected from every
201    /// registered aggregate's `zone_stat_default`.
202    ///
203    /// Each call scans the whole plugin registry, so this is intended to be called once per
204    /// column when a zoned writer is opened, not per chunk or per row.
205    pub fn zone_stat_defaults(&self, input_dtype: &DType) -> Vec<AggregateFnRef> {
206        self.registry.read(|registry| {
207            let mut fns: Vec<AggregateFnRef> = registry
208                .values()
209                .filter_map(|plugin| plugin.zone_stat_default(input_dtype))
210                .collect();
211            fns.sort_by_key(|f| f.id());
212            fns
213        })
214    }
215
216    /// Returns the aggregate kernel registered for `array_id` and `agg_fn_id`, if any.
217    ///
218    /// Lookup first checks for a kernel registered for the exact aggregate function, then falls
219    /// back to a kernel registered for all aggregate functions on the same array encoding.
220    pub fn find_aggregate_kernel(
221        &self,
222        array_id: impl Into<ArrayId>,
223        agg_fn_id: impl Into<AggregateFnId>,
224    ) -> Option<&'static dyn DynAggregateKernel> {
225        let id = array_id.into();
226        let fn_id = agg_fn_id.into();
227        self.kernels
228            .read(|kernels| lookup_aggregate_kernel(kernels, id, fn_id))
229    }
230
231    /// Registers an aggregate kernel for an array encoding.
232    ///
233    /// When `agg_fn_id` is `Some`, the kernel is used only for that aggregate function. When
234    /// `agg_fn_id` is `None`, the kernel is used as the fallback for aggregate functions on the
235    /// array encoding that do not have a more specific kernel.
236    pub fn register_aggregate_kernel(
237        &self,
238        array_id: impl Into<ArrayId>,
239        agg_fn_id: Option<impl Into<AggregateFnId>>,
240        kernel: &'static dyn DynAggregateKernel,
241    ) {
242        let id = (array_id.into(), agg_fn_id.map(|id| id.into()));
243        self.kernels.insert(id, kernel);
244    }
245
246    /// Returns the grouped aggregate kernel registered for `agg_fn_id`, if any.
247    ///
248    /// These kernels are independent of the element encoding and are checked for each element
249    /// representation, after any kernel registered for the current element encoding.
250    pub fn find_grouped_kernel(
251        &self,
252        agg_fn_id: impl Into<AggregateFnId>,
253    ) -> Option<&'static dyn DynGroupedAggregateKernel> {
254        let fn_id = agg_fn_id.into();
255        self.grouped_kernels
256            .read(|kernels| kernels.get(&fn_id).copied())
257    }
258
259    /// Registers a grouped aggregate kernel for an aggregate function.
260    pub fn register_grouped_kernel(
261        &self,
262        agg_fn_id: impl Into<AggregateFnId>,
263        kernel: &'static dyn DynGroupedAggregateKernel,
264    ) {
265        let fn_id = agg_fn_id.into();
266        self.grouped_kernels.insert(fn_id, kernel)
267    }
268
269    /// Returns the grouped aggregate kernel registered for `array_id` and `agg_fn_id`, if any.
270    ///
271    /// These kernels are matched against each intermediate element encoding while the grouped
272    /// accumulator executes the element array.
273    pub fn find_grouped_encoding_kernel(
274        &self,
275        array_id: impl Into<ArrayId>,
276        agg_fn_id: impl Into<AggregateFnId>,
277    ) -> Option<&'static dyn DynGroupedAggregateKernel> {
278        let id = array_id.into();
279        let fn_id = agg_fn_id.into();
280        self.grouped_encoding_kernels
281            .read(|kernels| kernels.get(&(id, fn_id)).copied())
282    }
283
284    /// Registers a grouped aggregate kernel for a specific aggregate function and array encoding.
285    pub fn register_grouped_encoding_kernel(
286        &self,
287        array_id: impl Into<ArrayId>,
288        agg_fn_id: impl Into<AggregateFnId>,
289        kernel: &'static dyn DynGroupedAggregateKernel,
290    ) {
291        let id = array_id.into();
292        let fn_id = agg_fn_id.into();
293        self.grouped_encoding_kernels.insert((id, fn_id), kernel)
294    }
295
296    /// Captures the currently registered aggregate kernels, so repeated lookups skip the session
297    /// and registry loads.
298    pub(crate) fn kernels_snapshot(&self) -> AggregateKernels {
299        AggregateKernels {
300            kernels: self.kernels.snapshot(),
301            grouped_kernels: self.grouped_kernels.snapshot(),
302            grouped_encoding_kernels: self.grouped_encoding_kernels.snapshot(),
303        }
304    }
305}
306
307/// A point-in-time snapshot of the aggregate kernel registries of an [`AggregateFnSession`].
308///
309/// Lookups follow the same rules as the corresponding `find_*` methods on
310/// [`AggregateFnSession`].
311#[derive(Clone, Debug)]
312pub(crate) struct AggregateKernels {
313    kernels: Arc<HashMap<AggregateKernelKey, &'static dyn DynAggregateKernel>>,
314    grouped_kernels: Arc<HashMap<AggregateFnId, &'static dyn DynGroupedAggregateKernel>>,
315    grouped_encoding_kernels:
316        Arc<HashMap<GroupedEncodingKernelKey, &'static dyn DynGroupedAggregateKernel>>,
317}
318
319impl AggregateKernels {
320    /// See [`AggregateFnSession::find_aggregate_kernel`].
321    pub(crate) fn find_aggregate_kernel(
322        &self,
323        array_id: ArrayId,
324        agg_fn_id: AggregateFnId,
325    ) -> Option<&'static dyn DynAggregateKernel> {
326        lookup_aggregate_kernel(&self.kernels, array_id, agg_fn_id)
327    }
328
329    /// See [`AggregateFnSession::find_grouped_kernel`].
330    pub(crate) fn find_grouped_kernel(
331        &self,
332        agg_fn_id: AggregateFnId,
333    ) -> Option<&'static dyn DynGroupedAggregateKernel> {
334        self.grouped_kernels.get(&agg_fn_id).copied()
335    }
336
337    /// See [`AggregateFnSession::find_grouped_encoding_kernel`].
338    pub(crate) fn find_grouped_encoding_kernel(
339        &self,
340        array_id: ArrayId,
341        agg_fn_id: AggregateFnId,
342    ) -> Option<&'static dyn DynGroupedAggregateKernel> {
343        self.grouped_encoding_kernels
344            .get(&(array_id, agg_fn_id))
345            .copied()
346    }
347}
348
349fn lookup_aggregate_kernel(
350    kernels: &HashMap<AggregateKernelKey, &'static dyn DynAggregateKernel>,
351    array_id: ArrayId,
352    agg_fn_id: AggregateFnId,
353) -> Option<&'static dyn DynAggregateKernel> {
354    kernels
355        .get(&(array_id, Some(agg_fn_id)))
356        .or_else(|| kernels.get(&(array_id, None)))
357        .copied()
358}
359
360/// Extension trait for accessing aggregate function session data.
361pub trait AggregateFnSessionExt: SessionExt {
362    /// Returns the aggregate function session data.
363    fn aggregate_fns(&self) -> SessionGuard<'_, AggregateFnSession> {
364        self.get::<AggregateFnSession>()
365    }
366}
367impl<S: SessionExt> AggregateFnSessionExt for S {}