Skip to main content

vortex_array/arrays/decimal/vtable/
mod.rs

1// SPDX-License-Identifier: Apache-2.0
2// SPDX-FileCopyrightText: Copyright the Vortex contributors
3
4use std::hash::Hasher;
5
6use prost::Message;
7use vortex_buffer::Alignment;
8use vortex_error::VortexResult;
9use vortex_error::vortex_bail;
10use vortex_error::vortex_ensure;
11use vortex_error::vortex_ensure_eq;
12use vortex_error::vortex_err;
13use vortex_session::VortexSession;
14
15use crate::ArrayParts;
16use crate::ArrayRef;
17use crate::ExecutionCtx;
18use crate::ExecutionResult;
19use crate::array::Array;
20use crate::array::ArrayView;
21use crate::array::VTable;
22use crate::arrays::decimal::DecimalData;
23use crate::arrays::fixed_width::vtable as fixed_width;
24use crate::buffer::BufferHandle;
25use crate::builders::ArrayBuilder;
26use crate::builders::DecimalBuilder;
27use crate::dtype::DType;
28use crate::dtype::DecimalType;
29use crate::dtype::NativeDecimalType;
30use crate::match_each_decimal_value_type;
31use crate::serde::ArrayChildren;
32mod kernel;
33mod operations;
34mod validity;
35
36use std::hash::Hash;
37
38use vortex_session::registry::CachedId;
39
40use crate::EqMode;
41use crate::array::ArrayId;
42use crate::arrays::decimal::array::DecimalSlots;
43use crate::arrays::decimal::compute::rules::RULES;
44use crate::hash::ArrayEq;
45use crate::hash::ArrayHash;
46/// A [`Decimal`]-encoded Vortex array.
47pub type DecimalArray = Array<Decimal>;
48
49pub(crate) fn initialize(session: &VortexSession) {
50    kernel::initialize(session);
51}
52
53/// Metadata for the existing `vortex.decimal` wire representation.
54#[derive(prost::Message)]
55pub struct DecimalMetadata {
56    /// Stored [`DecimalType`] discriminant, retained for wire compatibility.
57    ///
58    /// The legacy format records the actual width separately from the logical dtype. Readers
59    /// need this field even when the in-memory representation uses canonical decimal leaves.
60    #[prost(int32, tag = "1")]
61    pub(super) values_type: i32,
62}
63
64impl DecimalMetadata {
65    fn get_values_type(&self) -> VortexResult<DecimalType> {
66        DecimalType::try_from(self.values_type).map_err(|_| {
67            vortex_err!(
68                "DecimalMetadata values_type must be a known DecimalType, got {}",
69                self.values_type
70            )
71        })
72    }
73}
74
75impl ArrayHash for DecimalData {
76    fn array_hash<H: Hasher>(&self, state: &mut H, accuracy: EqMode) {
77        self.values.array_hash(state, accuracy);
78        std::mem::discriminant(&self.values_type).hash(state);
79    }
80}
81
82impl ArrayEq for DecimalData {
83    fn array_eq(&self, other: &Self, accuracy: EqMode) -> bool {
84        self.values.array_eq(&other.values, accuracy) && self.values_type == other.values_type
85    }
86}
87
88impl VTable for Decimal {
89    type TypedArrayData = DecimalData;
90
91    type OperationsVTable = Self;
92    type ValidityVTable = Self;
93
94    fn id(&self) -> ArrayId {
95        static ID: CachedId = CachedId::new("vortex.decimal");
96        *ID
97    }
98
99    fn nbuffers(_array: ArrayView<'_, Self>) -> usize {
100        1
101    }
102
103    fn buffer(array: ArrayView<'_, Self>, idx: usize) -> BufferHandle {
104        fixed_width::buffer("DecimalArray", &array.values, idx)
105    }
106
107    fn buffer_name(_array: ArrayView<'_, Self>, idx: usize) -> Option<String> {
108        fixed_width::buffer_name(idx)
109    }
110
111    fn with_buffers(
112        &self,
113        array: ArrayView<'_, Self>,
114        buffers: &[BufferHandle],
115    ) -> VortexResult<ArrayParts<Self>> {
116        let mut data = array.data().clone();
117        data.values = fixed_width::single_buffer(buffers)?;
118        Ok(ArrayParts::new(
119            self.clone(),
120            array.dtype().clone(),
121            array.len(),
122            data,
123            array.slots().iter().cloned().collect(),
124        ))
125    }
126
127    fn serialize(
128        array: ArrayView<'_, Self>,
129        _session: &VortexSession,
130    ) -> VortexResult<Option<Vec<u8>>> {
131        Ok(Some(
132            DecimalMetadata {
133                values_type: array.values_type() as i32,
134            }
135            .encode_to_vec(),
136        ))
137    }
138
139    fn validate(
140        &self,
141        data: &DecimalData,
142        dtype: &DType,
143        len: usize,
144        slots: &[Option<ArrayRef>],
145    ) -> VortexResult<()> {
146        let DType::Decimal(_, nullability) = dtype else {
147            vortex_bail!("Expected decimal dtype, got {dtype:?}");
148        };
149        vortex_ensure_eq!(
150            data.len(),
151            len,
152            "DecimalArray length does not match outer length",
153        );
154        let validity =
155            crate::array::child_to_validity(slots[DecimalSlots::VALIDITY].as_ref(), *nullability);
156        if let Some(validity_len) = validity.maybe_len() {
157            vortex_ensure_eq!(
158                validity_len,
159                len,
160                "DecimalArray validity len does not match outer length",
161            );
162        }
163
164        Ok(())
165    }
166
167    fn deserialize(
168        &self,
169        dtype: &DType,
170        len: usize,
171        metadata: &[u8],
172        buffers: &[BufferHandle],
173        children: &dyn ArrayChildren,
174        _session: &VortexSession,
175    ) -> VortexResult<ArrayParts<Self>> {
176        let metadata = DecimalMetadata::decode(metadata)?;
177        let values_type = metadata.get_values_type()?;
178        let values = fixed_width::single_buffer(buffers)?;
179
180        let validity = fixed_width::deserialize_validity(dtype.nullability(), len, children)?;
181
182        let Some(decimal_dtype) = dtype.as_decimal_opt() else {
183            vortex_bail!("Expected Decimal dtype, got {:?}", dtype)
184        };
185
186        let slots = DecimalData::make_slots(&validity, len);
187        let data = match_each_decimal_value_type!(values_type, |D| {
188            // Check and reinterpret-cast the buffer
189            vortex_ensure!(
190                values.is_aligned_to(Alignment::of::<D>()),
191                "DecimalArray buffer not aligned for values type {:?}",
192                D::DECIMAL_TYPE
193            );
194            DecimalData::try_new_handle(values, values_type, *decimal_dtype)
195        })?;
196        Ok(ArrayParts::new(
197            self.clone(),
198            dtype.clone(),
199            len,
200            data,
201            slots,
202        ))
203    }
204
205    fn slot_name(_array: ArrayView<'_, Self>, idx: usize) -> String {
206        DecimalSlots::NAMES[idx].to_string()
207    }
208
209    fn execute(array: Array<Self>, _ctx: &mut ExecutionCtx) -> VortexResult<ExecutionResult> {
210        Ok(ExecutionResult::done(array))
211    }
212
213    fn append_to_builder(
214        array: ArrayView<'_, Self>,
215        builder: &mut dyn ArrayBuilder,
216        ctx: &mut ExecutionCtx,
217    ) -> VortexResult<()> {
218        let Some(builder) = builder.as_any_mut().downcast_mut::<DecimalBuilder>() else {
219            vortex_bail!("append_to_builder for Decimal requires a DecimalBuilder");
220        };
221        builder.append_decimal_array(&array.into_owned(), ctx)
222    }
223
224    fn reduce_parent(
225        array: ArrayView<'_, Self>,
226        parent: &ArrayRef,
227        child_idx: usize,
228    ) -> VortexResult<Option<ArrayRef>> {
229        RULES.evaluate(array, parent, child_idx)
230    }
231}
232
233#[derive(Clone, Debug)]
234pub struct Decimal;
235
236#[cfg(test)]
237mod tests {
238    use std::sync::LazyLock;
239
240    use prost::Message;
241    use rstest::rstest;
242    use vortex_buffer::ByteBuffer;
243    use vortex_buffer::ByteBufferMut;
244    use vortex_buffer::buffer;
245    use vortex_error::VortexResult;
246    use vortex_session::VortexSession;
247    use vortex_session::registry::ReadContext;
248
249    use super::DecimalMetadata;
250    use crate::ArrayContext;
251    use crate::ArrayRef;
252    use crate::IntoArray;
253    use crate::VortexSessionExecute;
254    use crate::array::VTable;
255    use crate::arrays::Decimal;
256    use crate::arrays::DecimalArray;
257    use crate::assert_arrays_eq;
258    use crate::buffer::BufferHandle;
259    use crate::dtype::DType;
260    use crate::dtype::DecimalDType;
261    use crate::dtype::DecimalType;
262    use crate::dtype::Nullability;
263    use crate::dtype::i256;
264    use crate::serde::SerializeOptions;
265    use crate::serde::SerializedArray;
266    use crate::validity::Validity;
267
268    static SESSION: LazyLock<VortexSession> = LazyLock::new(VortexSession::empty);
269
270    #[test]
271    fn test_array_serde() {
272        let array = DecimalArray::new(
273            buffer![100i128, 200i128, 300i128, 400i128, 500i128],
274            DecimalDType::new(10, 2),
275            Validity::NonNullable,
276        );
277        let dtype = array.dtype().clone();
278
279        let array_ctx = ArrayContext::empty();
280        let out = array
281            .into_array()
282            .serialize(&array_ctx, &SESSION, &SerializeOptions::default())
283            .unwrap();
284        // Concat into a single buffer
285        let mut concat = ByteBufferMut::empty();
286        for buf in out {
287            concat.extend_from_slice(buf.as_ref());
288        }
289
290        let concat = concat.freeze();
291
292        let parts = SerializedArray::try_from(concat).unwrap();
293        let decoded = parts
294            .decode(&dtype, 5, &ReadContext::new(array_ctx.to_ids()), &SESSION)
295            .unwrap();
296        assert!(decoded.is::<Decimal>());
297        assert_arrays_eq!(
298            decoded,
299            DecimalArray::from_iter([100i128, 200, 300, 400, 500], DecimalDType::new(10, 2)),
300            &mut SESSION.create_execution_ctx()
301        );
302    }
303
304    #[test]
305    fn test_nullable_decimal_serde_roundtrip() {
306        let mut ctx = SESSION.create_execution_ctx();
307        let array = DecimalArray::new(
308            buffer![1234567i32, 0i32, -9999999i32],
309            DecimalDType::new(7, 3),
310            Validity::from_iter([true, false, true]),
311        );
312        let dtype = array.dtype().clone();
313        let len = array.len();
314
315        let array_ctx = ArrayContext::empty();
316        let out = array
317            .clone()
318            .into_array()
319            .serialize(&array_ctx, &SESSION, &SerializeOptions::default())
320            .unwrap();
321        let mut concat = ByteBufferMut::empty();
322        for buf in out {
323            concat.extend_from_slice(buf.as_ref());
324        }
325
326        let parts = SerializedArray::try_from(concat.freeze()).unwrap();
327        let decoded = parts
328            .decode(&dtype, len, &ReadContext::new(array_ctx.to_ids()), &SESSION)
329            .unwrap();
330
331        assert_arrays_eq!(decoded, array, &mut ctx);
332    }
333
334    #[rstest]
335    #[case::first_unused(6)]
336    #[case::negative(-1)]
337    #[case::max(i32::MAX)]
338    fn test_deserialize_rejects_unknown_values_type(#[case] raw: i32) {
339        let metadata = DecimalMetadata { values_type: raw }.encode_to_vec();
340        let buffers = [BufferHandle::new_host(ByteBuffer::zeroed(4))];
341        let children: Vec<ArrayRef> = Vec::new();
342
343        let result = Decimal.deserialize(
344            &DType::Decimal(DecimalDType::new(2, 0), Nullability::NonNullable),
345            4,
346            &metadata,
347            &buffers,
348            &children,
349            &SESSION,
350        );
351
352        let expected =
353            format!("DecimalMetadata values_type must be a known DecimalType, got {raw}");
354        assert!(result.is_err_and(|err| err.to_string().contains(&expected)));
355    }
356
357    #[rstest]
358    #[case::i8(
359        DecimalArray::from_iter([1i8, -2, 3], DecimalDType::new(2, 0)),
360        DecimalType::I8
361    )]
362    #[case::i16(
363        DecimalArray::from_iter([1i16, -2, 3], DecimalDType::new(4, 0)),
364        DecimalType::I16
365    )]
366    #[case::i32(
367        DecimalArray::from_iter([1i32, -2, 3], DecimalDType::new(9, 2)),
368        DecimalType::I32
369    )]
370    #[case::i64(
371        DecimalArray::from_iter([1i64, -2, 3], DecimalDType::new(18, 2)),
372        DecimalType::I64
373    )]
374    #[case::i128(
375        DecimalArray::from_iter([1i128, -2, 3], DecimalDType::new(38, 2)),
376        DecimalType::I128
377    )]
378    #[case::i256(
379        DecimalArray::from_iter(
380            [i256::from_i128(1), i256::from_i128(-2), i256::from_i128(3)],
381            DecimalDType::new(76, 2),
382        ),
383        DecimalType::I256
384    )]
385    fn test_serde_each_values_type(
386        #[case] array: DecimalArray,
387        #[case] values_type: DecimalType,
388    ) -> VortexResult<()> {
389        let dtype = array.dtype().clone();
390        let len = array.len();
391
392        let array_ctx = ArrayContext::empty();
393        let out = array.clone().into_array().serialize(
394            &array_ctx,
395            &SESSION,
396            &SerializeOptions::default(),
397        )?;
398        let mut concat = ByteBufferMut::empty();
399        for buf in out {
400            concat.extend_from_slice(buf.as_ref());
401        }
402
403        let parts = SerializedArray::try_from(concat.freeze())?;
404        let decoded = parts.decode(&dtype, len, &ReadContext::new(array_ctx.to_ids()), &SESSION)?;
405
406        assert_eq!(decoded.as_::<Decimal>().values_type(), values_type);
407        assert_arrays_eq!(decoded, array, &mut SESSION.create_execution_ctx());
408
409        Ok(())
410    }
411}