Skip to main content

vortex_array/aggregate_fn/
proto.rs

1// SPDX-License-Identifier: Apache-2.0
2// SPDX-FileCopyrightText: Copyright the Vortex contributors
3
4use vortex_error::VortexResult;
5use vortex_error::vortex_bail;
6use vortex_error::vortex_err;
7use vortex_session::VortexSession;
8
9use crate::aggregate_fn::AggregateFnId;
10use crate::aggregate_fn::AggregateFnRef;
11use crate::aggregate_fn::new_foreign_aggregate_fn;
12use crate::aggregate_fn::session::AggregateFnSessionExt;
13use crate::proto::expr as pb;
14
15impl AggregateFnRef {
16    /// Serialize this aggregate function to its protobuf representation.
17    ///
18    /// Note: the serialization format is not stable and may change between versions.
19    pub fn serialize_proto(&self) -> VortexResult<pb::AggregateFn> {
20        let metadata = self
21            .options()
22            .serialize()?
23            .ok_or_else(|| vortex_err!("Aggregate function '{}' is not serializable", self.id()))?;
24
25        Ok(pb::AggregateFn {
26            id: self.id().to_string(),
27            metadata: Some(metadata),
28        })
29    }
30
31    /// Deserialize an aggregate function from its protobuf representation.
32    ///
33    /// Looks up the aggregate function plugin by ID in the session's registry
34    /// and delegates deserialization to it.
35    ///
36    /// Note: the serialization format is not stable and may change between versions.
37    pub fn from_proto(proto: &pb::AggregateFn, session: &VortexSession) -> VortexResult<Self> {
38        #[expect(clippy::disallowed_methods, reason = "interning a dynamic id")]
39        let agg_fn_id: AggregateFnId = AggregateFnId::new(proto.id.as_str());
40        let agg_fn = if let Some(plugin) = session.aggregate_fns().find_plugin(&agg_fn_id) {
41            plugin.deserialize(proto.metadata(), session)?
42        } else if session.allows_unknown() {
43            new_foreign_aggregate_fn(agg_fn_id, proto.metadata().to_vec())
44        } else {
45            return Err(vortex_err!("unknown aggregate function id: {}", proto.id));
46        };
47
48        if agg_fn.id() != agg_fn_id {
49            vortex_bail!(
50                "Aggregate function ID mismatch: expected {}, got {}",
51                agg_fn_id,
52                agg_fn.id()
53            );
54        }
55
56        Ok(agg_fn)
57    }
58}
59
60#[cfg(test)]
61mod tests {
62    use std::sync::LazyLock;
63
64    use prost::Message;
65    use rstest::rstest;
66    use vortex_error::VortexResult;
67    use vortex_error::vortex_panic;
68    use vortex_session::VortexSession;
69
70    use crate::ArrayRef;
71    use crate::Columnar;
72    use crate::ExecutionCtx;
73    use crate::aggregate_fn::AggregateArgs;
74    use crate::aggregate_fn::AggregateFnId;
75    use crate::aggregate_fn::AggregateFnRef;
76    use crate::aggregate_fn::AggregateFnVTable;
77    use crate::aggregate_fn::AggregateFnVTableExt;
78    use crate::aggregate_fn::EmptyOptions;
79    use crate::aggregate_fn::NumericalAggregateOpts;
80    use crate::aggregate_fn::combined::PairOptions;
81    use crate::aggregate_fn::fns::count::Count;
82    use crate::aggregate_fn::fns::first::First;
83    use crate::aggregate_fn::fns::is_constant::IsConstant;
84    use crate::aggregate_fn::fns::is_sorted::IsSorted;
85    use crate::aggregate_fn::fns::is_sorted::IsSortedOptions;
86    use crate::aggregate_fn::fns::last::Last;
87    use crate::aggregate_fn::fns::mean::Mean;
88    use crate::aggregate_fn::fns::sum::Sum;
89    use crate::aggregate_fn::session::AggregateFnSession;
90    use crate::aggregate_fn::session::AggregateFnSessionExt;
91    use crate::dtype::DType;
92    use crate::proto::expr as pb;
93    use crate::scalar::Scalar;
94
95    static SESSION: LazyLock<VortexSession> = LazyLock::new(|| {
96        let session = VortexSession::empty();
97        session.aggregate_fns().register(TestAgg);
98        session
99    });
100
101    /// A minimal serializable aggregate function used solely to exercise the serde round-trip.
102    #[derive(Clone, Debug)]
103    struct TestAgg;
104
105    impl AggregateFnVTable for TestAgg {
106        type Options = EmptyOptions;
107        type Partial = ();
108
109        #[expect(clippy::disallowed_methods, reason = "test-only id")]
110        fn id(&self) -> AggregateFnId {
111            AggregateFnId::new("vortex.test.proto")
112        }
113
114        fn serialize(&self, _options: &Self::Options) -> VortexResult<Option<Vec<u8>>> {
115            Ok(Some(vec![]))
116        }
117
118        fn deserialize(
119            &self,
120            _metadata: &[u8],
121            _session: &VortexSession,
122        ) -> VortexResult<Self::Options> {
123            Ok(EmptyOptions)
124        }
125
126        fn return_dtype(&self, _options: &Self::Options, input_dtype: &DType) -> Option<DType> {
127            Some(input_dtype.clone())
128        }
129
130        fn partial_dtype(&self, options: &Self::Options, input_dtype: &DType) -> Option<DType> {
131            self.return_dtype(options, input_dtype)
132        }
133
134        fn empty_partial(
135            &self,
136            _args: AggregateArgs<'_, Self::Options>,
137        ) -> VortexResult<Self::Partial> {
138            Ok(())
139        }
140
141        fn partial_from_scalar(
142            &self,
143            _args: AggregateArgs<'_, Self::Options>,
144            _scalar: &Scalar,
145        ) -> VortexResult<Self::Partial> {
146            Ok(())
147        }
148
149        fn merge_partials(
150            &self,
151            _args: AggregateArgs<'_, Self::Options>,
152            _first: Self::Partial,
153            _second: Self::Partial,
154        ) -> VortexResult<Self::Partial> {
155            Ok(())
156        }
157
158        fn to_scalar(
159            &self,
160            _args: AggregateArgs<'_, Self::Options>,
161            _partial: &Self::Partial,
162        ) -> VortexResult<Scalar> {
163            vortex_panic!("TestAgg is for serde tests only");
164        }
165
166        fn is_saturated(
167            &self,
168            _args: AggregateArgs<'_, Self::Options>,
169            _partial: &Self::Partial,
170        ) -> bool {
171            true
172        }
173
174        fn accumulate(
175            &self,
176            _args: AggregateArgs<'_, Self::Options>,
177            _state: &mut Self::Partial,
178            _batch: &Columnar,
179            _ctx: &mut ExecutionCtx,
180        ) -> VortexResult<()> {
181            Ok(())
182        }
183
184        fn finalize(
185            &self,
186            _args: AggregateArgs<'_, Self::Options>,
187            partials: ArrayRef,
188        ) -> VortexResult<ArrayRef> {
189            Ok(partials)
190        }
191
192        fn finalize_scalar(
193            &self,
194            _args: AggregateArgs<'_, Self::Options>,
195            _partial: &Self::Partial,
196        ) -> VortexResult<Scalar> {
197            vortex_panic!("TestAgg is for serde tests only");
198        }
199    }
200
201    #[test]
202    fn aggregate_fn_serde() {
203        let agg_fn = TestAgg.bind(EmptyOptions);
204
205        let serialized = agg_fn.serialize_proto().unwrap();
206        let buf = serialized.encode_to_vec();
207        let deserialized_proto = pb::AggregateFn::decode(buf.as_slice()).unwrap();
208        let deserialized = AggregateFnRef::from_proto(&deserialized_proto, &SESSION).unwrap();
209
210        assert_eq!(deserialized, agg_fn);
211    }
212
213    /// The `skip_nans` option must survive a protobuf serialize/deserialize round-trip for the
214    /// numeric aggregates, including the non-default NaN-including configuration.
215    #[rstest]
216    #[case(Sum.bind(NumericalAggregateOpts::skip_nans()))]
217    #[case(Sum.bind(NumericalAggregateOpts::include_nans()))]
218    #[case(Count.bind(NumericalAggregateOpts::skip_nans()))]
219    #[case(Count.bind(NumericalAggregateOpts::include_nans()))]
220    fn numeric_aggregate_options_round_trip(#[case] agg_fn: AggregateFnRef) -> VortexResult<()> {
221        assert_round_trip(agg_fn)
222    }
223
224    #[rstest]
225    #[case(First.bind(EmptyOptions))]
226    #[case(Last.bind(EmptyOptions))]
227    #[case(IsConstant.bind(EmptyOptions))]
228    #[case(IsSorted.bind(IsSortedOptions { strict: false }))]
229    #[case(IsSorted.bind(IsSortedOptions { strict: true }))]
230    fn aggregate_options_round_trip(#[case] agg_fn: AggregateFnRef) -> VortexResult<()> {
231        assert_round_trip(agg_fn)
232    }
233
234    #[rstest]
235    fn mean_options_round_trip(
236        #[values(false, true)] sum_skip_nans: bool,
237        #[values(false, true)] count_skip_nans: bool,
238    ) -> VortexResult<()> {
239        assert_round_trip(Mean::combined().bind(PairOptions(
240            NumericalAggregateOpts {
241                skip_nans: sum_skip_nans,
242            },
243            NumericalAggregateOpts {
244                skip_nans: count_skip_nans,
245            },
246        )))
247    }
248
249    #[rstest]
250    #[case(IsSorted.id(), vec![])]
251    #[case(IsSorted.id(), vec![2])]
252    #[case(IsSorted.id(), vec![0, 1])]
253    #[case(Mean::combined().id(), vec![])]
254    #[case(Mean::combined().id(), vec![0])]
255    #[case(Mean::combined().id(), vec![2, 0])]
256    #[case(Mean::combined().id(), vec![0, 2])]
257    #[case(Mean::combined().id(), vec![0, 1, 0])]
258    fn invalid_aggregate_options(#[case] id: AggregateFnId, #[case] metadata: Vec<u8>) {
259        let proto = pb::AggregateFn {
260            id: id.to_string(),
261            metadata: Some(metadata),
262        };
263        assert!(AggregateFnRef::from_proto(&proto, &VortexSession::empty()).is_err());
264    }
265
266    fn assert_round_trip(agg_fn: AggregateFnRef) -> VortexResult<()> {
267        let session = VortexSession::empty();
268
269        let proto = agg_fn.serialize_proto()?;
270        let buf = proto.encode_to_vec();
271        let decoded = pb::AggregateFn::decode(buf.as_slice())?;
272        let round_tripped = AggregateFnRef::from_proto(&decoded, &session)?;
273
274        assert_eq!(round_tripped, agg_fn);
275        Ok(())
276    }
277
278    #[test]
279    fn unknown_aggregate_fn_id_allow_unknown() {
280        let session = VortexSession::empty().with::<AggregateFnSession>();
281        session.allow_unknown();
282
283        let proto = pb::AggregateFn {
284            id: "vortex.test.foreign_aggregate".to_string(),
285            metadata: Some(vec![7, 8, 9]),
286        };
287
288        let agg_fn = AggregateFnRef::from_proto(&proto, &session).unwrap();
289        assert_eq!(agg_fn.id().as_ref(), "vortex.test.foreign_aggregate");
290
291        let roundtrip = agg_fn.serialize_proto().unwrap();
292        assert_eq!(roundtrip.id, proto.id);
293        assert_eq!(roundtrip.metadata(), proto.metadata());
294    }
295}