Skip to main content

vortex_array/arrays/decimal/vtable/plugin/
mod.rs

1// SPDX-License-Identifier: Apache-2.0
2// SPDX-FileCopyrightText: Copyright the Vortex contributors
3
4//! Serde for `vortex.decimal` nodes, including legacy nodes.
5//!
6//! [`StoredDecimal::is_legacy`] defines a legacy node, which [`DecimalPlugin`] reads as the
7//! canonical array or a [`Widen`].
8//!
9//! Each legacy read scans the valid values once, because the deserializer attaches stored
10//! statistics only after the plugin returns. This cost is not limited to old files, because IPC
11//! and writes without a compressor still write a decimal `Widen` as a legacy node.
12
13mod stored;
14
15use prost::Message;
16use vortex_error::VortexResult;
17use vortex_error::vortex_bail;
18use vortex_error::vortex_ensure_eq;
19use vortex_error::vortex_err;
20use vortex_session::VortexSession;
21
22use self::stored::StoredDecimal;
23use super::Decimal;
24use crate::ArrayRef;
25use crate::IntoArray;
26use crate::VortexSessionExecute;
27use crate::array::Array;
28use crate::array::ArrayDeserialization;
29use crate::array::ArrayId;
30use crate::array::ArrayPlugin;
31use crate::array::ArraySerialization;
32use crate::array::VTable;
33use crate::arrays::Widen;
34use crate::arrays::WidenArray;
35use crate::arrays::decimal::DecimalDataParts;
36use crate::dtype::DecimalType;
37use crate::expr::stats::Precision;
38
39/// The metadata of a `vortex.decimal` node.
40#[derive(prost::Message)]
41struct DecimalMetadata {
42    /// The [`DecimalType`] discriminant of the stored width.
43    ///
44    /// The width is recorded apart from the dtype, because a legacy node can store a width other
45    /// than the canonical width of its precision.
46    #[prost(int32, tag = "1")]
47    values_type: i32,
48}
49
50impl DecimalMetadata {
51    fn get_values_type(&self) -> VortexResult<DecimalType> {
52        DecimalType::try_from(self.values_type).map_err(|_| {
53            vortex_err!(
54                "DecimalMetadata values_type must be a known DecimalType, got {}",
55                self.values_type
56            )
57        })
58    }
59}
60
61/// Writes a [`DecimalArray`] as a `vortex.decimal` node, and reads every such node as the
62/// canonical decimal array or a decimal [`Widen`].
63///
64/// A canonical node keeps its buffer without a copy or a scan, so the buffer can stay on a device,
65/// and its valid values are trusted to fit the precision. A legacy node goes through
66/// [`WidenArray::from_stored_decimal`], which scans its valid values on the host, so a legacy node
67/// in device memory fails to read.
68///
69/// The [`Decimal`] vtable does not serialize or deserialize, so the default [`ArraySession`]
70/// registers this plugin in its place.
71///
72/// [`DecimalArray`]: super::DecimalArray
73/// [`ArraySession`]: crate::session::ArraySession
74#[derive(Clone, Debug)]
75pub struct DecimalPlugin;
76
77impl ArrayPlugin for DecimalPlugin {
78    fn id(&self) -> ArrayId {
79        VTable::id(&Decimal)
80    }
81
82    fn read_ids(&self) -> Vec<ArrayId> {
83        vec![self.id()]
84    }
85
86    fn write_ids(&self) -> Vec<ArrayId> {
87        vec![self.id()]
88    }
89
90    fn serialize(
91        &self,
92        array: &ArrayRef,
93        _session: &VortexSession,
94    ) -> VortexResult<Option<ArraySerialization>> {
95        let Some(decimal) = array.as_opt::<Decimal>() else {
96            vortex_bail!(
97                "DecimalPlugin serializes only Decimal arrays, got {}",
98                array.encoding_id()
99            );
100        };
101        let metadata = DecimalMetadata {
102            values_type: decimal.values_type() as i32,
103        };
104
105        Ok(Some(ArraySerialization::from_array(
106            self.id(),
107            array,
108            metadata.encode_to_vec(),
109        )))
110    }
111
112    fn deserialize(
113        &self,
114        parts: ArrayDeserialization<'_>,
115        session: &VortexSession,
116    ) -> VortexResult<ArrayRef> {
117        vortex_ensure_eq!(
118            parts.serialized_id,
119            self.id(),
120            "DecimalPlugin reads only its own serialized ID",
121        );
122
123        let stored = StoredDecimal::deserialize(
124            parts.dtype,
125            parts.len,
126            parts.metadata,
127            parts.buffers,
128            parts.children,
129        )?;
130
131        if !stored.is_legacy() {
132            return Ok(Array::<Decimal>::try_from_parts(stored.into_array_parts()?)?.into_array());
133        }
134
135        // This is a legacy node, whose width a `Decimal` leaf no longer allows, so the valid values
136        // pick the canonical array or a `Widen`.
137        let parts = DecimalDataParts {
138            decimal_dtype: stored.decimal_dtype,
139            values: stored.values,
140            values_type: stored.values_type,
141            validity: stored.validity,
142        };
143
144        // Stored statistics are attached only after this returns, so no range is known yet.
145        WidenArray::from_stored_decimal(
146            parts,
147            Precision::Absent,
148            &mut session.create_execution_ctx(),
149        )
150    }
151
152    fn is_supported_encoding(&self, id: &ArrayId) -> bool {
153        *id == VTable::id(&Decimal) || *id == VTable::id(&Widen)
154    }
155}
156
157#[cfg(test)]
158mod tests;