1use flatbuffers::FlatBufferBuilder;
5use flatbuffers::WIPOffset;
6use half::f16;
7use vortex_error::VortexResult;
8use vortex_error::vortex_bail;
9use vortex_session::VortexSession;
10
11use crate::dtype::DType;
12use crate::dtype::Nullability;
13use crate::dtype::PType;
14use crate::expr::stats::Precision;
15use crate::expr::stats::Stat;
16use crate::flatbuffers::WriteFlatBuffer;
17use crate::flatbuffers::array as fba;
18use crate::scalar::PValue;
19use crate::scalar::Scalar;
20use crate::scalar::ScalarValue;
21use crate::stats::StatsSet;
22use crate::stats::StatsSetRef;
23
24impl WriteFlatBuffer for StatsSetRef<'_> {
25 type Target<'t> = fba::ArrayStats<'t>;
26
27 fn write_flatbuffer<'fb>(
29 &self,
30 fbb: &mut FlatBufferBuilder<'fb>,
31 ) -> VortexResult<WIPOffset<Self::Target<'fb>>> {
32 self.with_typed_stats_set(|stats_set| stats_set.values.write_flatbuffer(fbb))
33 }
34}
35
36impl WriteFlatBuffer for StatsSet {
37 type Target<'t> = fba::ArrayStats<'t>;
38
39 fn write_flatbuffer<'fb>(
41 &self,
42 fbb: &mut FlatBufferBuilder<'fb>,
43 ) -> VortexResult<WIPOffset<Self::Target<'fb>>> {
44 let (min_precision, min) = match self.get(Stat::Min) {
45 Precision::Exact(min) => (
46 fba::Precision::Exact,
47 Some(fbb.create_vector(&ScalarValue::to_proto_bytes::<Vec<u8>>(Some(&min)))),
48 ),
49 Precision::Inexact(min) => (
50 fba::Precision::Inexact,
51 Some(fbb.create_vector(&ScalarValue::to_proto_bytes::<Vec<u8>>(Some(&min)))),
52 ),
53 Precision::Absent => (fba::Precision::Inexact, None),
54 };
55
56 let (max_precision, max) = match self.get(Stat::Max) {
57 Precision::Exact(max) => (
58 fba::Precision::Exact,
59 Some(fbb.create_vector(&ScalarValue::to_proto_bytes::<Vec<u8>>(Some(&max)))),
60 ),
61 Precision::Inexact(max) => (
62 fba::Precision::Inexact,
63 Some(fbb.create_vector(&ScalarValue::to_proto_bytes::<Vec<u8>>(Some(&max)))),
64 ),
65 Precision::Absent => (fba::Precision::Inexact, None),
66 };
67
68 let sum = self.get(Stat::Sum).as_exact().map(|sum| {
69 let sum = canonicalize_nan_sum(sum);
70 fbb.create_vector(&ScalarValue::to_proto_bytes::<Vec<u8>>(Some(&sum)))
71 });
72
73 let stat_args = &fba::ArrayStatsArgs {
74 min,
75 min_precision,
76 max,
77 max_precision,
78 sum,
79 is_sorted: self
80 .get_as::<bool>(Stat::IsSorted, &DType::Bool(Nullability::NonNullable))
81 .as_exact(),
82 is_strict_sorted: self
83 .get_as::<bool>(Stat::IsStrictSorted, &DType::Bool(Nullability::NonNullable))
84 .as_exact(),
85 is_constant: self
86 .get_as::<bool>(Stat::IsConstant, &DType::Bool(Nullability::NonNullable))
87 .as_exact(),
88 null_count: self
89 .get_as::<u64>(Stat::NullCount, &PType::U64.into())
90 .as_exact(),
91 uncompressed_size_in_bytes: self
92 .get_as::<u64>(Stat::UncompressedSizeInBytes, &PType::U64.into())
93 .as_exact(),
94 nan_count: self
95 .get_as::<u64>(Stat::NaNCount, &PType::U64.into())
96 .as_exact(),
97 };
98
99 Ok(fba::ArrayStats::create(fbb, stat_args))
100 }
101}
102
103fn canonicalize_nan_sum(sum: ScalarValue) -> ScalarValue {
111 match sum {
112 ScalarValue::Primitive(PValue::F16(v)) if v.is_nan() => {
113 ScalarValue::Primitive(PValue::F16(f16::from_bits(0x7e00)))
114 }
115 ScalarValue::Primitive(PValue::F32(v)) if v.is_nan() => {
116 ScalarValue::Primitive(PValue::F32(f32::from_bits(0x7fc0_0000)))
117 }
118 ScalarValue::Primitive(PValue::F64(v)) if v.is_nan() => {
119 ScalarValue::Primitive(PValue::F64(f64::from_bits(0x7ff8_0000_0000_0000)))
120 }
121 sum => sum,
122 }
123}
124
125impl StatsSet {
126 pub fn from_flatbuffer<'a>(
128 fb: &fba::ArrayStats<'a>,
129 array_dtype: &DType,
130 session: &VortexSession,
131 ) -> VortexResult<Self> {
132 let mut stats_set = StatsSet::default();
133
134 for stat in Stat::all() {
135 match stat {
136 Stat::IsConstant => {
137 if let Some(is_constant) = fb.is_constant() {
138 stats_set.set(Stat::IsConstant, Precision::Exact(is_constant.into()));
139 }
140 }
141 Stat::IsSorted => {
142 if let Some(is_sorted) = fb.is_sorted() {
143 stats_set.set(Stat::IsSorted, Precision::Exact(is_sorted.into()));
144 }
145 }
146 Stat::IsStrictSorted => {
147 if let Some(is_strict_sorted) = fb.is_strict_sorted() {
148 stats_set.set(
149 Stat::IsStrictSorted,
150 Precision::Exact(is_strict_sorted.into()),
151 );
152 }
153 }
154 Stat::Max => {
155 if let Some(max) = fb.max()
156 && let Some(stat_dtype) = stat.dtype(array_dtype)
157 {
158 let Some(value) = decode_stat_value(max.bytes(), &stat_dtype, session)?
159 else {
160 continue;
161 };
162
163 stats_set.set(
164 Stat::Max,
165 match fb.max_precision() {
166 fba::Precision::Exact => Precision::Exact(value),
167 fba::Precision::Inexact => Precision::Inexact(value),
168 other => {
169 vortex_bail!("Corrupted max_precision field: {other:?}")
170 }
171 },
172 );
173 }
174 }
175 Stat::Min => {
176 if let Some(min) = fb.min()
177 && let Some(stat_dtype) = stat.dtype(array_dtype)
178 {
179 let Some(value) = decode_stat_value(min.bytes(), &stat_dtype, session)?
180 else {
181 continue;
182 };
183
184 stats_set.set(
185 Stat::Min,
186 match fb.min_precision() {
187 fba::Precision::Exact => Precision::Exact(value),
188 fba::Precision::Inexact => Precision::Inexact(value),
189 other => {
190 vortex_bail!("Corrupted min_precision field: {other:?}")
191 }
192 },
193 );
194 }
195 }
196 Stat::NullCount => {
197 if let Some(null_count) = fb.null_count() {
198 stats_set.set(Stat::NullCount, Precision::Exact(null_count.into()));
199 }
200 }
201 Stat::UncompressedSizeInBytes => {
202 if let Some(uncompressed_size_in_bytes) = fb.uncompressed_size_in_bytes() {
203 stats_set.set(
204 Stat::UncompressedSizeInBytes,
205 Precision::Exact(uncompressed_size_in_bytes.into()),
206 );
207 }
208 }
209 Stat::Sum => {
210 if let Some(sum) = fb.sum()
211 && let Some(stat_dtype) = stat.dtype(array_dtype)
212 {
213 let Some(value) = decode_stat_value(sum.bytes(), &stat_dtype, session)?
214 else {
215 continue;
216 };
217
218 stats_set.set(Stat::Sum, Precision::Exact(value));
219 }
220 }
221 Stat::NaNCount => {
222 if let Some(nan_count) = fb.nan_count() {
223 stats_set.set(
224 Stat::NaNCount,
225 Precision::Exact(ScalarValue::from(nan_count)),
226 );
227 }
228 }
229 }
230 }
231
232 Ok(stats_set)
233 }
234}
235
236fn decode_stat_value(
244 bytes: &[u8],
245 stat_dtype: &DType,
246 session: &VortexSession,
247) -> VortexResult<Option<ScalarValue>> {
248 let Some(value) = ScalarValue::from_proto_bytes(bytes, stat_dtype, session)? else {
249 return Ok(None);
250 };
251 Scalar::validate(stat_dtype, Some(&value))?;
252
253 Ok(Some(value))
254}
255
256#[cfg(test)]
257mod tests {
258 use flatbuffers::FlatBufferBuilder;
259 use flatbuffers::root;
260 use rstest::rstest;
261 use vortex_error::VortexResult;
262 use vortex_error::vortex_err;
263 use vortex_session::VortexSession;
264
265 use crate::dtype::DType;
266 use crate::dtype::DecimalDType;
267 use crate::dtype::Nullability;
268 use crate::dtype::PType;
269 use crate::expr::stats::Precision;
270 use crate::expr::stats::Stat;
271 use crate::flatbuffers::WriteFlatBuffer;
272 use crate::flatbuffers::array as fba;
273 use crate::scalar::DecimalValue;
274 use crate::scalar::PValue;
275 use crate::scalar::ScalarValue;
276 use crate::stats::StatsSet;
277
278 fn roundtrip_sum(sum: f64, expected_bits: u64) -> VortexResult<u64> {
279 let stats = StatsSet::of(
280 Stat::Sum,
281 Precision::exact(ScalarValue::Primitive(PValue::F64(sum))),
282 );
283
284 let mut fbb = FlatBufferBuilder::new();
285 let offset = stats.write_flatbuffer(&mut fbb)?;
286 fbb.finish_minimal(offset);
287 let fb = root::<fba::ArrayStats>(fbb.finished_data())?;
288 let mut expected_bytes = vec![0x31];
290 expected_bytes.extend_from_slice(&expected_bits.to_le_bytes());
291 assert_eq!(fb.sum().expect("serialized sum").bytes(), expected_bytes);
292
293 let dtype = DType::Primitive(PType::F64, Nullability::NonNullable);
294 let read = StatsSet::from_flatbuffer(&fb, &dtype, &VortexSession::empty())?;
295 Ok(read
296 .get_as::<f64>(
297 Stat::Sum,
298 &DType::Primitive(PType::F64, Nullability::Nullable),
299 )
300 .as_exact()
301 .ok_or_else(|| vortex_err!("sum must be exact after a round trip"))?
302 .to_bits())
303 }
304
305 fn roundtrip_decimal_stat(stat: Stat, value: DecimalValue) -> VortexResult<StatsSet> {
307 let stats = StatsSet::of(stat, Precision::exact(ScalarValue::Decimal(value)));
308
309 let mut fbb = FlatBufferBuilder::new();
310 let offset = stats.write_flatbuffer(&mut fbb)?;
311 fbb.finish_minimal(offset);
312 let fb = root::<fba::ArrayStats>(fbb.finished_data())?;
313
314 let dtype = DType::Decimal(DecimalDType::new(2, 0), Nullability::NonNullable);
315 StatsSet::from_flatbuffer(&fb, &dtype, &VortexSession::empty())
316 }
317
318 #[test]
319 fn nan_sum_is_written_canonical() -> VortexResult<()> {
320 assert_eq!(
322 roundtrip_sum(f64::from_bits(0xfff8_0000_0000_0000), 0x7ff8_0000_0000_0000)?,
323 0x7ff8_0000_0000_0000
324 );
325 assert_eq!(
326 roundtrip_sum(f64::from_bits(0x7ff8_0000_dead_beef), 0x7ff8_0000_0000_0000)?,
327 0x7ff8_0000_0000_0000
328 );
329
330 assert_eq!(roundtrip_sum(3.0, 3.0f64.to_bits())?, 3.0f64.to_bits());
331 assert_eq!(
332 roundtrip_sum(f64::NEG_INFINITY, f64::NEG_INFINITY.to_bits())?,
333 f64::NEG_INFINITY.to_bits()
334 );
335
336 Ok(())
337 }
338
339 #[rstest]
341 #[case::max(Stat::Max, DecimalValue::I8(100))]
342 #[case::min(Stat::Min, DecimalValue::I8(-100))]
343 #[case::sum(Stat::Sum, DecimalValue::I64(1_000_000_000_000))]
344 fn decimal_stat_outside_precision_fails_to_read(
345 #[case] stat: Stat,
346 #[case] value: DecimalValue,
347 ) -> VortexResult<()> {
348 let err = roundtrip_decimal_stat(stat, value).unwrap_err();
349 assert!(
350 err.to_string().contains("does not fit in precision"),
351 "unexpected error: {err}"
352 );
353
354 Ok(())
355 }
356
357 #[test]
358 fn decimal_stat_inside_precision_reads() -> VortexResult<()> {
359 let read = roundtrip_decimal_stat(Stat::Max, DecimalValue::I8(99))?;
360 assert_eq!(
361 read.get(Stat::Max),
362 Precision::exact(ScalarValue::Decimal(DecimalValue::I8(99)))
363 );
364
365 Ok(())
366 }
367}