Skip to main content

vortex_array/arrays/decimal/
array.rs

1// SPDX-License-Identifier: Apache-2.0
2// SPDX-FileCopyrightText: Copyright the Vortex contributors
3
4use std::fmt::Display;
5use std::fmt::Formatter;
6
7use itertools::Itertools;
8use smallvec::smallvec;
9use vortex_buffer::Alignment;
10use vortex_buffer::BitBufferMut;
11use vortex_buffer::Buffer;
12use vortex_buffer::BufferMut;
13use vortex_buffer::ByteBuffer;
14use vortex_error::VortexExpect;
15use vortex_error::VortexResult;
16use vortex_error::vortex_ensure;
17use vortex_error::vortex_panic;
18
19use crate::ArrayRef;
20use crate::ArraySlots;
21use crate::ExecutionCtx;
22use crate::IntoArray;
23use crate::array::Array;
24use crate::array::ArrayParts;
25use crate::array::TypedArrayRef;
26use crate::array::child_to_validity;
27use crate::array::validity_to_child;
28use crate::array_slots;
29use crate::arrays::Decimal;
30use crate::arrays::DecimalArray;
31use crate::arrays::PrimitiveArray;
32use crate::arrays::primitive::PrimitiveArrayExt;
33use crate::buffer::BufferHandle;
34use crate::dtype::BigCast;
35use crate::dtype::DType;
36use crate::dtype::DecimalDType;
37use crate::dtype::DecimalType;
38use crate::dtype::IntegerPType;
39use crate::dtype::NativeDecimalType;
40use crate::dtype::Nullability;
41use crate::match_each_decimal_value_type;
42use crate::match_each_unsigned_integer_ptype;
43use crate::patches::Patches;
44use crate::validity::Validity;
45
46#[array_slots(Decimal)]
47pub struct DecimalSlots {
48    /// The validity bitmap indicating which elements are non-null.
49    #[slot(0)]
50    pub validity: Option<ArrayRef>,
51}
52
53/// A decimal array that stores fixed-precision decimal numbers with configurable scale.
54///
55/// This mirrors the Apache Arrow Decimal encoding and provides exact arithmetic for
56/// financial and scientific computations where floating-point precision loss is unacceptable.
57///
58/// ## Storage Format
59///
60/// Decimals are stored as scaled integers in a supported scalar value type.
61///
62/// The canonical widths for each precision are:
63/// - **i8**: precision 1-2 digits
64/// - **i16**: precision 3-4 digits
65/// - **i32**: precision 5-9 digits
66/// - **i64**: precision 10-18 digits
67/// - **i128**: precision 19-38 digits
68/// - **i256**: precision 39-76 digits
69///
70/// Storage width is still independent of precision during the canonical decimal migration.
71/// Existing `vortex.decimal` nodes and DecimalByteParts assembly can produce noncanonical storage.
72/// For example, an array with precision 39 can currently use an `i8` buffer when all values fit,
73/// while an array with precision 2 can use a `Buffer<i256>`. [`DecimalDataParts`] preserves the
74/// actual width for conversion into canonical decimal leaves, optionally under
75/// [`Widen`](crate::arrays::Widen).
76///
77/// ## Precision and Scale
78///
79/// - **Precision**: Total number of significant digits (1-76, u8 range)
80/// - **Scale**: Number of digits after the decimal point (-128 to 127, i8 range)
81/// - **Value**: `stored_integer / 10^scale`
82///
83/// For example, with precision=5 and scale=2:
84/// - Stored value 12345 represents 123.45
85/// - Range: -999.99 to 999.99
86///
87/// ## Valid Scalar Types
88///
89/// Canonical decimal leaves select one of these native types from precision:
90/// - `DecimalType::I8`, `I16`, `I32`, `I64`, `I128`, `I256`
91/// - Type selection is automatic based on the required precision
92///
93/// # Examples
94///
95/// ```
96/// use vortex_array::arrays::DecimalArray;
97/// use vortex_array::dtype::DecimalDType;
98/// use vortex_buffer::{buffer, Buffer};
99/// use vortex_array::validity::Validity;
100///
101/// // Create a decimal array with precision=5, scale=2 (e.g., 123.45)
102/// let decimal_dtype = DecimalDType::new(5, 2);
103/// let values = buffer![12345i32, 67890i32, -12300i32]; // 123.45, 678.90, -123.00
104/// let array = DecimalArray::new(values, decimal_dtype, Validity::NonNullable);
105///
106/// assert_eq!(array.precision(), 5);
107/// assert_eq!(array.scale(), 2);
108/// assert_eq!(array.len(), 3);
109/// ```
110#[derive(Clone, Debug)]
111pub struct DecimalData {
112    pub(super) decimal_dtype: DecimalDType,
113    pub(super) values: BufferHandle,
114    /// Actual storage type, including noncanonical widths still accepted during the migration.
115    pub(super) values_type: DecimalType,
116}
117
118impl Display for DecimalData {
119    fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result {
120        write!(
121            f,
122            "decimal_dtype: {}, values_type: {}",
123            self.decimal_dtype, self.values_type
124        )
125    }
126}
127
128/// Decimal storage and validity, as returned by [`DecimalArray::into_data_parts`].
129pub struct DecimalDataParts {
130    /// The precision and scale of the values.
131    pub decimal_dtype: DecimalDType,
132    /// Unscaled integers of type `values_type` in host or device memory.
133    pub values: BufferHandle,
134    /// Actual storage type, retained to interpret input before canonical conversion.
135    ///
136    /// Legacy `vortex.decimal` nodes and DecimalByteParts assembly can use a different width from
137    /// [`DecimalDType::canonical_values_type`]. Comparing the widths identifies noncanonical
138    /// storage, not the age of the data: current [`WidenPlugin`] writes use that wire form too.
139    ///
140    /// [`WidenPlugin`]: crate::arrays::widen::WidenPlugin
141    pub values_type: DecimalType,
142    /// The validity of the values, which also gives the nullability of the array.
143    pub validity: Validity,
144}
145
146pub trait DecimalArrayExt: TypedArrayRef<Decimal> {
147    fn decimal_dtype(&self) -> DecimalDType {
148        match self.dtype() {
149            DType::Decimal(decimal_dtype, _) => *decimal_dtype,
150            _ => unreachable!("DecimalArrayExt requires a decimal dtype"),
151        }
152    }
153
154    fn nullability(&self) -> Nullability {
155        match self.dtype() {
156            DType::Decimal(_, nullability) => *nullability,
157            _ => unreachable!("DecimalArrayExt requires a decimal dtype"),
158        }
159    }
160
161    fn validity_child(&self) -> Option<&ArrayRef> {
162        self.slots()[DecimalSlots::VALIDITY].as_ref()
163    }
164
165    fn validity(&self) -> Validity {
166        child_to_validity(
167            self.slots()[DecimalSlots::VALIDITY].as_ref(),
168            self.nullability(),
169        )
170    }
171
172    fn values_type(&self) -> DecimalType {
173        self.values_type
174    }
175
176    fn precision(&self) -> u8 {
177        self.decimal_dtype().precision()
178    }
179
180    fn scale(&self) -> i8 {
181        self.decimal_dtype().scale()
182    }
183
184    fn buffer_handle(&self) -> &BufferHandle {
185        &self.values
186    }
187
188    fn buffer<T: NativeDecimalType>(&self) -> Buffer<T> {
189        DecimalData::buffer::<T>(self)
190    }
191}
192impl<T: TypedArrayRef<Decimal>> DecimalArrayExt for T {}
193
194impl DecimalData {
195    /// Build the slots vector for this array.
196    pub(super) fn make_slots(validity: &Validity, len: usize) -> ArraySlots {
197        smallvec![validity_to_child(validity, len)]
198    }
199
200    /// Creates a new [`DecimalArray`] using a host-native buffer.
201    ///
202    /// # Panics
203    ///
204    /// Panics if the provided components do not satisfy the invariants documented in
205    /// [`DecimalArray::new_unchecked`].
206    pub fn new<T: NativeDecimalType>(buffer: Buffer<T>, decimal_dtype: DecimalDType) -> Self {
207        Self::try_new(buffer, decimal_dtype).vortex_expect("DecimalArray construction failed")
208    }
209
210    /// Creates a new [`DecimalArray`] from a [`BufferHandle`] of values that may live in
211    /// host or device memory.
212    ///
213    /// # Panics
214    ///
215    /// Panics if the provided components do not satisfy the invariants documented in
216    /// [`DecimalArray::new_unchecked`].
217    pub fn new_handle(
218        values: BufferHandle,
219        values_type: DecimalType,
220        decimal_dtype: DecimalDType,
221    ) -> Self {
222        Self::try_new_handle(values, values_type, decimal_dtype)
223            .vortex_expect("DecimalArray construction failed")
224    }
225
226    /// Constructs a new `DecimalArray`.
227    ///
228    /// See [`DecimalArray::new_unchecked`] for more information.
229    ///
230    /// # Errors
231    ///
232    /// Returns an error if the provided components do not satisfy the invariants documented in
233    /// [`DecimalArray::new_unchecked`].
234    pub fn try_new<T: NativeDecimalType>(
235        buffer: Buffer<T>,
236        decimal_dtype: DecimalDType,
237    ) -> VortexResult<Self> {
238        let values = BufferHandle::new_host(buffer.into_byte_buffer());
239        let values_type = T::DECIMAL_TYPE;
240
241        Self::try_new_handle(values, values_type, decimal_dtype)
242    }
243
244    /// Constructs a new `DecimalArray` with validation from a [`BufferHandle`].
245    ///
246    /// This pathway allows building new decimal arrays that may come from host or device memory.
247    ///
248    /// # Errors
249    ///
250    /// Returns an error if the buffer length is not a multiple of the width of `values_type`, or
251    /// if the buffer is not aligned to that width.
252    pub fn try_new_handle(
253        values: BufferHandle,
254        values_type: DecimalType,
255        decimal_dtype: DecimalDType,
256    ) -> VortexResult<Self> {
257        Self::validate(&values, values_type)?;
258
259        // SAFETY: `validate` checks the buffer layout. Values outside the declared precision can
260        // produce incorrect results, but do not affect memory safety.
261        Ok(unsafe { Self::new_unchecked_handle(values, values_type, decimal_dtype) })
262    }
263
264    /// Creates a new [`DecimalArray`] without validation from these components:
265    ///
266    /// * `buffer` is a typed buffer containing the decimal values.
267    /// * `decimal_dtype` specifies the decimal precision and scale.
268    ///
269    /// Values are not checked against the precision of `decimal_dtype`, either here, in
270    /// [`DecimalArray::try_new`], or when an array is read from a file. Code that turns stored values
271    /// into scalars must use a fallible path such as
272    /// [`Scalar::try_decimal`](crate::scalar::Scalar::try_decimal).
273    ///
274    /// # Safety
275    ///
276    /// `buffer` is a typed [`Buffer<T>`], so there are no further requirements on the caller.
277    pub unsafe fn new_unchecked<T: NativeDecimalType>(
278        buffer: Buffer<T>,
279        decimal_dtype: DecimalDType,
280    ) -> Self {
281        // SAFETY: new_unchecked_handle inherits the safety guarantees of new_unchecked
282        unsafe {
283            Self::new_unchecked_handle(
284                BufferHandle::new_host(buffer.into_byte_buffer()),
285                T::DECIMAL_TYPE,
286                decimal_dtype,
287            )
288        }
289    }
290
291    /// Create a new array with decimal values backed by the given buffer handle.
292    ///
293    /// As with [`DecimalArray::new_unchecked`], values are not checked against the precision of
294    /// `decimal_dtype`.
295    ///
296    /// # Safety
297    ///
298    /// - The length of `values` must be a multiple of the byte width of `values_type`.
299    /// - `values` must be aligned to the native type of `values_type`.
300    ///
301    /// Breaking either rule does not cause undefined behavior, but [`DecimalData::buffer`] panics.
302    pub unsafe fn new_unchecked_handle(
303        values: BufferHandle,
304        values_type: DecimalType,
305        decimal_dtype: DecimalDType,
306    ) -> Self {
307        Self {
308            decimal_dtype,
309            values,
310            values_type,
311        }
312    }
313
314    /// Validates the layout of a decimal values buffer.
315    ///
316    /// Checks byte length and alignment for `values_type`. Does not read values or check validity.
317    pub(crate) fn validate(buffer: &BufferHandle, values_type: DecimalType) -> VortexResult<()> {
318        let byte_width = values_type.byte_width();
319        vortex_ensure!(
320            buffer.len().is_multiple_of(byte_width),
321            "decimal buffer size {} is not divisible by element width {}",
322            buffer.len(),
323            byte_width,
324        );
325        match_each_decimal_value_type!(values_type, |D| {
326            vortex_ensure!(
327                buffer.is_aligned_to(Alignment::of::<D>()),
328                "decimal buffer alignment {:?} is invalid for values type {:?}",
329                buffer.alignment(),
330                D::DECIMAL_TYPE,
331            );
332            Ok::<(), vortex_error::VortexError>(())
333        })?;
334        Ok(())
335    }
336
337    /// Creates a new [`DecimalArray`] from a raw byte buffer without validation.
338    ///
339    /// # Safety
340    ///
341    /// The caller must ensure:
342    /// - The `byte_buffer` contains valid data for the specified `values_type`
343    /// - The buffer length is compatible with the `values_type` (i.e., divisible by the type size)
344    /// - If `validity` is [`Validity::Array`], its length must equal the number of elements
345    pub unsafe fn new_unchecked_from_byte_buffer(
346        byte_buffer: ByteBuffer,
347        values_type: DecimalType,
348        decimal_dtype: DecimalDType,
349    ) -> Self {
350        // SAFETY: inherits the same safety contract as `new_unchecked_from_byte_buffer`
351        unsafe {
352            Self::new_unchecked_handle(
353                BufferHandle::new_host(byte_buffer),
354                values_type,
355                decimal_dtype,
356            )
357        }
358    }
359
360    /// Returns the length of this array.
361    pub fn len(&self) -> usize {
362        self.values.len() / self.values_type.byte_width()
363    }
364
365    /// Returns `true` if this array is empty.
366    pub fn is_empty(&self) -> bool {
367        self.len() == 0
368    }
369
370    /// Returns the underlying [`ByteBuffer`] of the array.
371    pub fn buffer_handle(&self) -> &BufferHandle {
372        &self.values
373    }
374
375    pub fn buffer<T: NativeDecimalType>(&self) -> Buffer<T> {
376        if self.values_type != T::DECIMAL_TYPE {
377            vortex_panic!(
378                "Cannot extract Buffer<{:?}> for DecimalArray with values_type {:?}",
379                T::DECIMAL_TYPE,
380                self.values_type,
381            );
382        }
383        Buffer::<T>::from_byte_buffer(self.values.as_host().clone())
384    }
385
386    /// Return the `DecimalType` used to represent the values in the array.
387    pub fn values_type(&self) -> DecimalType {
388        self.values_type
389    }
390
391    /// Returns the decimal type information.
392    pub fn decimal_dtype(&self) -> DecimalDType {
393        self.decimal_dtype
394    }
395
396    pub fn precision(&self) -> u8 {
397        self.decimal_dtype.precision()
398    }
399
400    pub fn scale(&self) -> i8 {
401        self.decimal_dtype.scale()
402    }
403}
404
405impl Array<Decimal> {
406    pub fn into_data_parts(self) -> DecimalDataParts {
407        let validity = DecimalArrayExt::validity(&self);
408        let decimal_dtype = DecimalArrayExt::decimal_dtype(&self);
409        let data = self.into_data();
410        DecimalDataParts {
411            decimal_dtype,
412            values: data.values,
413            values_type: data.values_type,
414            validity,
415        }
416    }
417}
418
419impl Array<Decimal> {
420    /// Creates a new [`DecimalArray`] using a host-native buffer.
421    pub fn new<T: NativeDecimalType>(
422        buffer: Buffer<T>,
423        decimal_dtype: DecimalDType,
424        validity: Validity,
425    ) -> Self {
426        Self::try_new(buffer, decimal_dtype, validity)
427            .vortex_expect("DecimalArray construction failed")
428    }
429
430    /// Creates a new [`DecimalArray`] without validation.
431    ///
432    /// # Safety
433    ///
434    /// See [`DecimalData::new_unchecked`].
435    pub unsafe fn new_unchecked<T: NativeDecimalType>(
436        buffer: Buffer<T>,
437        decimal_dtype: DecimalDType,
438        validity: Validity,
439    ) -> Self {
440        let dtype = DType::Decimal(decimal_dtype, validity.nullability());
441        let len = buffer.len();
442        let slots = DecimalData::make_slots(&validity, len);
443        let data = unsafe { DecimalData::new_unchecked(buffer, decimal_dtype) };
444        unsafe {
445            Self::from_decimal_parts_unchecked(ArrayParts::new(Decimal, dtype, len, data, slots))
446        }
447    }
448
449    /// Creates a new [`DecimalArray`] from a host-native buffer with validation.
450    pub fn try_new<T: NativeDecimalType>(
451        buffer: Buffer<T>,
452        decimal_dtype: DecimalDType,
453        validity: Validity,
454    ) -> VortexResult<Self> {
455        let dtype = DType::Decimal(decimal_dtype, validity.nullability());
456        let len = buffer.len();
457        let slots = DecimalData::make_slots(&validity, len);
458        let data = DecimalData::try_new(buffer, decimal_dtype)?;
459        Self::try_from_decimal_parts(ArrayParts::new(Decimal, dtype, len, data, slots))
460    }
461
462    /// Creates the array from parts without validation.
463    ///
464    /// The generic constructors call this function. It is not generic, so crates that call those
465    /// constructors do not each compile the `Decimal` array vtable.
466    ///
467    /// # Safety
468    ///
469    /// The parts must be logically consistent.
470    unsafe fn from_decimal_parts_unchecked(parts: ArrayParts<Decimal>) -> Self {
471        unsafe { Array::from_parts_unchecked(parts) }
472    }
473
474    /// Creates the array from parts after validation.
475    ///
476    /// The generic constructors call this function. It is not generic, so crates that call those
477    /// constructors do not each compile the `Decimal` array vtable.
478    fn try_from_decimal_parts(parts: ArrayParts<Decimal>) -> VortexResult<Self> {
479        Array::try_from_parts(parts)
480    }
481
482    /// Creates a new [`DecimalArray`] from an iterator of values.
483    #[expect(
484        clippy::same_name_method,
485        reason = "intentionally named from_iter like Iterator::from_iter"
486    )]
487    pub fn from_iter<T: NativeDecimalType, I: IntoIterator<Item = T>>(
488        iter: I,
489        decimal_dtype: DecimalDType,
490    ) -> Self {
491        Self::new(
492            BufferMut::from_iter(iter).freeze(),
493            decimal_dtype,
494            Validity::NonNullable,
495        )
496    }
497
498    /// Creates a new [`DecimalArray`] from an iterator of optional values.
499    pub fn from_option_iter<T: NativeDecimalType, I: IntoIterator<Item = Option<T>>>(
500        iter: I,
501        decimal_dtype: DecimalDType,
502    ) -> Self {
503        let iter = iter.into_iter();
504        let mut values = BufferMut::with_capacity(iter.size_hint().0);
505        let mut validity = BitBufferMut::with_capacity(values.capacity());
506
507        for value in iter {
508            match value {
509                Some(value) => {
510                    values.push(value);
511                    validity.append(true);
512                }
513                None => {
514                    values.push(T::default());
515                    validity.append(false);
516                }
517            }
518        }
519
520        Self::new(
521            values.freeze(),
522            decimal_dtype,
523            Validity::from(validity.freeze()),
524        )
525    }
526
527    /// Creates a new [`DecimalArray`] from a [`BufferHandle`].
528    pub fn new_handle(
529        values: BufferHandle,
530        values_type: DecimalType,
531        decimal_dtype: DecimalDType,
532        validity: Validity,
533    ) -> Self {
534        Self::try_new_handle(values, values_type, decimal_dtype, validity)
535            .vortex_expect("DecimalArray construction failed")
536    }
537
538    /// Creates a new [`DecimalArray`] from a [`BufferHandle`] with validation.
539    pub fn try_new_handle(
540        values: BufferHandle,
541        values_type: DecimalType,
542        decimal_dtype: DecimalDType,
543        validity: Validity,
544    ) -> VortexResult<Self> {
545        let dtype = DType::Decimal(decimal_dtype, validity.nullability());
546        let len = values.len() / values_type.byte_width();
547        let slots = DecimalData::make_slots(&validity, len);
548        let data = DecimalData::try_new_handle(values, values_type, decimal_dtype)?;
549        Array::try_from_parts(ArrayParts::new(Decimal, dtype, len, data, slots))
550    }
551
552    /// Creates a new [`DecimalArray`] without validation from a [`BufferHandle`].
553    ///
554    /// # Safety
555    ///
556    /// See [`DecimalData::new_unchecked_handle`].
557    pub unsafe fn new_unchecked_handle(
558        values: BufferHandle,
559        values_type: DecimalType,
560        decimal_dtype: DecimalDType,
561        validity: Validity,
562    ) -> Self {
563        let dtype = DType::Decimal(decimal_dtype, validity.nullability());
564        let len = values.len() / values_type.byte_width();
565        let slots = DecimalData::make_slots(&validity, len);
566        let data = unsafe { DecimalData::new_unchecked_handle(values, values_type, decimal_dtype) };
567        unsafe { Array::from_parts_unchecked(ArrayParts::new(Decimal, dtype, len, data, slots)) }
568    }
569
570    #[expect(
571        clippy::cognitive_complexity,
572        reason = "patching depends on both patch and value physical types"
573    )]
574    pub fn patch(self, patches: &Patches, ctx: &mut ExecutionCtx) -> VortexResult<Self> {
575        let offset = patches.offset();
576        let dtype = self.dtype().clone();
577        let len = self.len();
578        let patch_indices = patches.indices().clone().execute::<PrimitiveArray>(ctx)?;
579        let patch_values = patches.values().clone().execute::<DecimalArray>(ctx)?;
580
581        let patch_validity = patch_values.validity()?;
582        let patched_validity = self.validity()?.patch(
583            self.len(),
584            offset,
585            &patch_indices.clone().into_array(),
586            &patch_validity,
587            ctx,
588        )?;
589        assert_eq!(self.decimal_dtype(), patch_values.decimal_dtype());
590
591        let data = self.into_data();
592        // Patch indices are non-negative; reinterpret to unsigned so this dispatches over 4 widths
593        // instead of 8 (the decimal value-type dimensions are unaffected).
594        let patch_indices_unsigned =
595            patch_indices.reinterpret_cast(patch_indices.ptype().to_unsigned());
596        let data = match_each_unsigned_integer_ptype!(patch_indices_unsigned.ptype(), |I| {
597            let patch_indices = patch_indices_unsigned.as_slice::<I>();
598            match_each_decimal_value_type!(patch_values.values_type(), |PatchDVT| {
599                let patch_values = patch_values.buffer::<PatchDVT>();
600                match_each_decimal_value_type!(data.values_type(), |ValuesDVT| {
601                    let buffer = data.buffer::<ValuesDVT>().into_mut();
602                    patch_typed(
603                        buffer,
604                        data.decimal_dtype(),
605                        patch_indices,
606                        offset,
607                        patch_values,
608                    )
609                })
610            })
611        });
612        let slots = DecimalData::make_slots(&patched_validity, len);
613        Ok(unsafe {
614            Array::from_parts_unchecked(ArrayParts::new(Decimal, dtype, len, data, slots))
615        })
616    }
617}
618
619fn patch_typed<I, ValuesDVT, PatchDVT>(
620    mut buffer: BufferMut<ValuesDVT>,
621    decimal_dtype: DecimalDType,
622    patch_indices: &[I],
623    patch_indices_offset: usize,
624    patch_values: Buffer<PatchDVT>,
625) -> DecimalData
626where
627    I: IntegerPType,
628    PatchDVT: NativeDecimalType,
629    ValuesDVT: NativeDecimalType,
630{
631    if !ValuesDVT::DECIMAL_TYPE.is_compatible_decimal_value_type(decimal_dtype) {
632        vortex_panic!(
633            "patch_typed: {:?} cannot represent every value in {}.",
634            ValuesDVT::DECIMAL_TYPE,
635            decimal_dtype
636        )
637    }
638
639    for (idx, value) in patch_indices.iter().zip_eq(patch_values) {
640        buffer[idx.as_() - patch_indices_offset] = <ValuesDVT as BigCast>::from(value).vortex_expect(
641            "values of a given DecimalDType are representable in all compatible NativeDecimalType",
642        );
643    }
644
645    DecimalData::new(buffer.freeze(), decimal_dtype)
646}