Skip to main content

vortex_array/stats/
flatbuffers.rs

1// SPDX-License-Identifier: Apache-2.0
2// SPDX-FileCopyrightText: Copyright the Vortex contributors
3
4use flatbuffers::FlatBufferBuilder;
5use flatbuffers::WIPOffset;
6use half::f16;
7use vortex_error::VortexResult;
8use vortex_error::vortex_bail;
9use vortex_session::VortexSession;
10
11use crate::dtype::DType;
12use crate::dtype::Nullability;
13use crate::dtype::PType;
14use crate::expr::stats::Precision;
15use crate::expr::stats::Stat;
16use crate::flatbuffers::WriteFlatBuffer;
17use crate::flatbuffers::array as fba;
18use crate::scalar::PValue;
19use crate::scalar::Scalar;
20use crate::scalar::ScalarValue;
21use crate::stats::StatsSet;
22use crate::stats::StatsSetRef;
23
24impl WriteFlatBuffer for StatsSetRef<'_> {
25    type Target<'t> = fba::ArrayStats<'t>;
26
27    /// All statistics written must be exact
28    fn write_flatbuffer<'fb>(
29        &self,
30        fbb: &mut FlatBufferBuilder<'fb>,
31    ) -> VortexResult<WIPOffset<Self::Target<'fb>>> {
32        self.with_typed_stats_set(|stats_set| stats_set.values.write_flatbuffer(fbb))
33    }
34}
35
36impl WriteFlatBuffer for StatsSet {
37    type Target<'t> = fba::ArrayStats<'t>;
38
39    /// All statistics written must be exact
40    fn write_flatbuffer<'fb>(
41        &self,
42        fbb: &mut FlatBufferBuilder<'fb>,
43    ) -> VortexResult<WIPOffset<Self::Target<'fb>>> {
44        let (min_precision, min) = match self.get(Stat::Min) {
45            Precision::Exact(min) => (
46                fba::Precision::Exact,
47                Some(fbb.create_vector(&ScalarValue::to_proto_bytes::<Vec<u8>>(Some(&min)))),
48            ),
49            Precision::Inexact(min) => (
50                fba::Precision::Inexact,
51                Some(fbb.create_vector(&ScalarValue::to_proto_bytes::<Vec<u8>>(Some(&min)))),
52            ),
53            Precision::Absent => (fba::Precision::Inexact, None),
54        };
55
56        let (max_precision, max) = match self.get(Stat::Max) {
57            Precision::Exact(max) => (
58                fba::Precision::Exact,
59                Some(fbb.create_vector(&ScalarValue::to_proto_bytes::<Vec<u8>>(Some(&max)))),
60            ),
61            Precision::Inexact(max) => (
62                fba::Precision::Inexact,
63                Some(fbb.create_vector(&ScalarValue::to_proto_bytes::<Vec<u8>>(Some(&max)))),
64            ),
65            Precision::Absent => (fba::Precision::Inexact, None),
66        };
67
68        let sum = self.get(Stat::Sum).as_exact().map(|sum| {
69            let sum = canonicalize_nan_sum(sum);
70            fbb.create_vector(&ScalarValue::to_proto_bytes::<Vec<u8>>(Some(&sum)))
71        });
72
73        let stat_args = &fba::ArrayStatsArgs {
74            min,
75            min_precision,
76            max,
77            max_precision,
78            sum,
79            is_sorted: self
80                .get_as::<bool>(Stat::IsSorted, &DType::Bool(Nullability::NonNullable))
81                .as_exact(),
82            is_strict_sorted: self
83                .get_as::<bool>(Stat::IsStrictSorted, &DType::Bool(Nullability::NonNullable))
84                .as_exact(),
85            is_constant: self
86                .get_as::<bool>(Stat::IsConstant, &DType::Bool(Nullability::NonNullable))
87                .as_exact(),
88            null_count: self
89                .get_as::<u64>(Stat::NullCount, &PType::U64.into())
90                .as_exact(),
91            uncompressed_size_in_bytes: self
92                .get_as::<u64>(Stat::UncompressedSizeInBytes, &PType::U64.into())
93                .as_exact(),
94            nan_count: self
95                .get_as::<u64>(Stat::NaNCount, &PType::U64.into())
96                .as_exact(),
97        };
98
99        Ok(fba::ArrayStats::create(fbb, stat_args))
100    }
101}
102
103/// Replace a NaN float sum with the canonical quiet NaN of its width.
104///
105/// A float sum is NaN when the data holds both `+inf` and `-inf`, because `inf + -inf` is an IEEE 754
106/// invalid operation. IEEE 754 does not specify the bits of that result, and targets disagree:
107/// x86_64 sets the sign bit and aarch64 does not. Without this step, the bytes of a written file
108/// depend on the architecture that wrote it. A NaN sum has no payload information, so nothing is
109/// lost. Stats that are already written are read back unchanged.
110fn canonicalize_nan_sum(sum: ScalarValue) -> ScalarValue {
111    match sum {
112        ScalarValue::Primitive(PValue::F16(v)) if v.is_nan() => {
113            ScalarValue::Primitive(PValue::F16(f16::from_bits(0x7e00)))
114        }
115        ScalarValue::Primitive(PValue::F32(v)) if v.is_nan() => {
116            ScalarValue::Primitive(PValue::F32(f32::from_bits(0x7fc0_0000)))
117        }
118        ScalarValue::Primitive(PValue::F64(v)) if v.is_nan() => {
119            ScalarValue::Primitive(PValue::F64(f64::from_bits(0x7ff8_0000_0000_0000)))
120        }
121        sum => sum,
122    }
123}
124
125impl StatsSet {
126    /// Creates a [`StatsSet`] from a flatbuffers array [`fba::ArrayStats<'a>`].
127    pub fn from_flatbuffer<'a>(
128        fb: &fba::ArrayStats<'a>,
129        array_dtype: &DType,
130        session: &VortexSession,
131    ) -> VortexResult<Self> {
132        let mut stats_set = StatsSet::default();
133
134        for stat in Stat::all() {
135            match stat {
136                Stat::IsConstant => {
137                    if let Some(is_constant) = fb.is_constant() {
138                        stats_set.set(Stat::IsConstant, Precision::Exact(is_constant.into()));
139                    }
140                }
141                Stat::IsSorted => {
142                    if let Some(is_sorted) = fb.is_sorted() {
143                        stats_set.set(Stat::IsSorted, Precision::Exact(is_sorted.into()));
144                    }
145                }
146                Stat::IsStrictSorted => {
147                    if let Some(is_strict_sorted) = fb.is_strict_sorted() {
148                        stats_set.set(
149                            Stat::IsStrictSorted,
150                            Precision::Exact(is_strict_sorted.into()),
151                        );
152                    }
153                }
154                Stat::Max => {
155                    if let Some(max) = fb.max()
156                        && let Some(stat_dtype) = stat.dtype(array_dtype)
157                    {
158                        let Some(value) = decode_stat_value(max.bytes(), &stat_dtype, session)?
159                        else {
160                            continue;
161                        };
162
163                        stats_set.set(
164                            Stat::Max,
165                            match fb.max_precision() {
166                                fba::Precision::Exact => Precision::Exact(value),
167                                fba::Precision::Inexact => Precision::Inexact(value),
168                                other => {
169                                    vortex_bail!("Corrupted max_precision field: {other:?}")
170                                }
171                            },
172                        );
173                    }
174                }
175                Stat::Min => {
176                    if let Some(min) = fb.min()
177                        && let Some(stat_dtype) = stat.dtype(array_dtype)
178                    {
179                        let Some(value) = decode_stat_value(min.bytes(), &stat_dtype, session)?
180                        else {
181                            continue;
182                        };
183
184                        stats_set.set(
185                            Stat::Min,
186                            match fb.min_precision() {
187                                fba::Precision::Exact => Precision::Exact(value),
188                                fba::Precision::Inexact => Precision::Inexact(value),
189                                other => {
190                                    vortex_bail!("Corrupted min_precision field: {other:?}")
191                                }
192                            },
193                        );
194                    }
195                }
196                Stat::NullCount => {
197                    if let Some(null_count) = fb.null_count() {
198                        stats_set.set(Stat::NullCount, Precision::Exact(null_count.into()));
199                    }
200                }
201                Stat::UncompressedSizeInBytes => {
202                    if let Some(uncompressed_size_in_bytes) = fb.uncompressed_size_in_bytes() {
203                        stats_set.set(
204                            Stat::UncompressedSizeInBytes,
205                            Precision::Exact(uncompressed_size_in_bytes.into()),
206                        );
207                    }
208                }
209                Stat::Sum => {
210                    if let Some(sum) = fb.sum()
211                        && let Some(stat_dtype) = stat.dtype(array_dtype)
212                    {
213                        let Some(value) = decode_stat_value(sum.bytes(), &stat_dtype, session)?
214                        else {
215                            continue;
216                        };
217
218                        stats_set.set(Stat::Sum, Precision::Exact(value));
219                    }
220                }
221                Stat::NaNCount => {
222                    if let Some(nan_count) = fb.nan_count() {
223                        stats_set.set(
224                            Stat::NaNCount,
225                            Precision::Exact(ScalarValue::from(nan_count)),
226                        );
227                    }
228                }
229            }
230        }
231
232        Ok(stats_set)
233    }
234}
235
236/// Decodes a stored statistic value and checks that it is a legal value of `stat_dtype`.
237///
238/// Returns `None` for a stored null. Vortex never writes an illegal statistic, for example a
239/// decimal outside its precision, so an illegal one means the file is corrupt. The check runs at
240/// decode because [`StatsProvider::get`] has no error path, so it panics on an illegal value.
241///
242/// [`StatsProvider::get`]: crate::expr::stats::StatsProvider::get
243fn decode_stat_value(
244    bytes: &[u8],
245    stat_dtype: &DType,
246    session: &VortexSession,
247) -> VortexResult<Option<ScalarValue>> {
248    let Some(value) = ScalarValue::from_proto_bytes(bytes, stat_dtype, session)? else {
249        return Ok(None);
250    };
251    Scalar::validate(stat_dtype, Some(&value))?;
252
253    Ok(Some(value))
254}
255
256#[cfg(test)]
257mod tests {
258    use flatbuffers::FlatBufferBuilder;
259    use flatbuffers::root;
260    use rstest::rstest;
261    use vortex_error::VortexResult;
262    use vortex_error::vortex_err;
263    use vortex_session::VortexSession;
264
265    use crate::dtype::DType;
266    use crate::dtype::DecimalDType;
267    use crate::dtype::Nullability;
268    use crate::dtype::PType;
269    use crate::expr::stats::Precision;
270    use crate::expr::stats::Stat;
271    use crate::flatbuffers::WriteFlatBuffer;
272    use crate::flatbuffers::array as fba;
273    use crate::scalar::DecimalValue;
274    use crate::scalar::PValue;
275    use crate::scalar::ScalarValue;
276    use crate::stats::StatsSet;
277
278    fn roundtrip_sum(sum: f64, expected_bits: u64) -> VortexResult<u64> {
279        let stats = StatsSet::of(
280            Stat::Sum,
281            Precision::exact(ScalarValue::Primitive(PValue::F64(sum))),
282        );
283
284        let mut fbb = FlatBufferBuilder::new();
285        let offset = stats.write_flatbuffer(&mut fbb)?;
286        fbb.finish_minimal(offset);
287        let fb = root::<fba::ArrayStats>(fbb.finished_data())?;
288        // ScalarValue.f64_value is protobuf field 6, wire type fixed64.
289        let mut expected_bytes = vec![0x31];
290        expected_bytes.extend_from_slice(&expected_bits.to_le_bytes());
291        assert_eq!(fb.sum().expect("serialized sum").bytes(), expected_bytes);
292
293        let dtype = DType::Primitive(PType::F64, Nullability::NonNullable);
294        let read = StatsSet::from_flatbuffer(&fb, &dtype, &VortexSession::empty())?;
295        Ok(read
296            .get_as::<f64>(
297                Stat::Sum,
298                &DType::Primitive(PType::F64, Nullability::Nullable),
299            )
300            .as_exact()
301            .ok_or_else(|| vortex_err!("sum must be exact after a round trip"))?
302            .to_bits())
303    }
304
305    /// Writes one exact decimal statistic and reads it back for a `decimal(2, 0)` column.
306    fn roundtrip_decimal_stat(stat: Stat, value: DecimalValue) -> VortexResult<StatsSet> {
307        let stats = StatsSet::of(stat, Precision::exact(ScalarValue::Decimal(value)));
308
309        let mut fbb = FlatBufferBuilder::new();
310        let offset = stats.write_flatbuffer(&mut fbb)?;
311        fbb.finish_minimal(offset);
312        let fb = root::<fba::ArrayStats>(fbb.finished_data())?;
313
314        let dtype = DType::Decimal(DecimalDType::new(2, 0), Nullability::NonNullable);
315        StatsSet::from_flatbuffer(&fb, &dtype, &VortexSession::empty())
316    }
317
318    #[test]
319    fn nan_sum_is_written_canonical() -> VortexResult<()> {
320        // x86_64 gives this NaN for `inf + -inf`.
321        assert_eq!(
322            roundtrip_sum(f64::from_bits(0xfff8_0000_0000_0000), 0x7ff8_0000_0000_0000)?,
323            0x7ff8_0000_0000_0000
324        );
325        assert_eq!(
326            roundtrip_sum(f64::from_bits(0x7ff8_0000_dead_beef), 0x7ff8_0000_0000_0000)?,
327            0x7ff8_0000_0000_0000
328        );
329
330        assert_eq!(roundtrip_sum(3.0, 3.0f64.to_bits())?, 3.0f64.to_bits());
331        assert_eq!(
332            roundtrip_sum(f64::NEG_INFINITY, f64::NEG_INFINITY.to_bits())?,
333            f64::NEG_INFINITY.to_bits()
334        );
335
336        Ok(())
337    }
338
339    // The sum of a `decimal(2, 0)` column has the dtype `decimal(12, 0)`.
340    #[rstest]
341    #[case::max(Stat::Max, DecimalValue::I8(100))]
342    #[case::min(Stat::Min, DecimalValue::I8(-100))]
343    #[case::sum(Stat::Sum, DecimalValue::I64(1_000_000_000_000))]
344    fn decimal_stat_outside_precision_fails_to_read(
345        #[case] stat: Stat,
346        #[case] value: DecimalValue,
347    ) -> VortexResult<()> {
348        let err = roundtrip_decimal_stat(stat, value).unwrap_err();
349        assert!(
350            err.to_string().contains("does not fit in precision"),
351            "unexpected error: {err}"
352        );
353
354        Ok(())
355    }
356
357    #[test]
358    fn decimal_stat_inside_precision_reads() -> VortexResult<()> {
359        let read = roundtrip_decimal_stat(Stat::Max, DecimalValue::I8(99))?;
360        assert_eq!(
361            read.get(Stat::Max),
362            Precision::exact(ScalarValue::Decimal(DecimalValue::I8(99)))
363        );
364
365        Ok(())
366    }
367}