Skip to main content

vortex_array/arrays/widen/encode/
decimal.rs

1// SPDX-License-Identifier: Apache-2.0
2// SPDX-FileCopyrightText: Copyright the Vortex contributors
3
4//! Decimal encoding and conversion of legacy storage into canonical decimal leaves.
5//!
6//! The legacy `vortex.decimal` format stores integer width independently of logical precision.
7//! DecimalByteParts assembly can also produce such buffers, but we now require canonical
8//! `vortex.decimal` leaves to have a width determined and fixed by their precision.
9//!
10//! In this new world, we can retain narrow storage as a [`WidenArray`] over a canonical child.
11//! Current [`WidenPlugin`] writes still use the legacy format, so this conversion can also apply
12//! to newly written data (which will happen until `Widen` is added to the `core` edition).
13//!
14//! We unify child selection in both encoding and legacy conversion because it has non-trivial
15//! logic: width alone does not determine a valid child dtype. For example, unscaled 127 fits `i8`
16//! but needs precision 3, whose canonical width is `i16`.
17//!
18//! [`WidenPlugin`]: crate::arrays::widen::WidenPlugin
19
20use std::ops::RangeInclusive;
21
22use vortex_buffer::Buffer;
23use vortex_error::VortexExpect;
24use vortex_error::VortexResult;
25use vortex_error::vortex_bail;
26use vortex_error::vortex_ensure;
27use vortex_error::vortex_ensure_eq;
28use vortex_mask::Mask;
29
30use crate::ArrayRef;
31use crate::ExecutionCtx;
32use crate::IntoArray;
33use crate::arrays::DecimalArray;
34use crate::arrays::WidenArray;
35use crate::arrays::decimal::DecimalArrayExt;
36use crate::arrays::decimal::DecimalData;
37use crate::arrays::decimal::DecimalDataParts;
38use crate::arrays::decimal::copy_to_canonical_width;
39use crate::arrays::decimal::valid_range;
40use crate::buffer::BufferHandle;
41use crate::dtype::DType;
42use crate::dtype::DecimalDType;
43use crate::dtype::NativeDecimalType;
44use crate::dtype::i256;
45use crate::expr::stats::Precision;
46use crate::expr::stats::Stat;
47use crate::match_each_decimal_value_type;
48use crate::scalar::DecimalValue;
49use crate::scalar::ScalarValue;
50use crate::validity::Validity;
51
52impl WidenArray {
53    /// Stores a decimal array at the smallest precision that its valid values and scale allow.
54    ///
55    /// We keep the logical dtype and nullability by wrapping the narrower child in a
56    /// [`WidenArray`]. Reducing precision within the same storage width saves no space, so in that
57    /// case we return the input unchanged.
58    ///
59    /// Empty and all-null arrays have no values to constrain precision, but we still need to keep
60    /// the scale. They use precision 1 or the scale, whichever is greater. If a child is selected,
61    /// we cache any exact extrema as [`Stat::Min`] and [`Stat::Max`] for later aggregates.
62    ///
63    /// # Errors
64    ///
65    /// Returns an error if a valid value does not fit the input precision.
66    pub fn encode_decimal(array: DecimalArray, ctx: &mut ExecutionCtx) -> VortexResult<ArrayRef> {
67        let logical_decimal_dtype = array.decimal_dtype();
68        let validity = array.validity()?;
69        let validity_mask = validity.execute_mask(array.len(), ctx)?;
70        let child = match_each_decimal_value_type!(array.values_type(), |T| {
71            let values_buffer = array.buffer::<T>();
72            let valid_value_range =
73                ValidRange::resolve(Precision::Absent, values_buffer.as_slice(), &validity_mask);
74            narrowest_decimal_child(
75                &values_buffer,
76                &validity,
77                &validity_mask,
78                valid_value_range,
79                logical_decimal_dtype,
80            )?
81        });
82
83        let Some(child) = child else {
84            return Ok(array.into_array());
85        };
86
87        Ok(WidenArray::try_new(child.into_array(), array.dtype().clone())?.into_array())
88    }
89
90    /// Converts stored decimals into a canonical leaf or a [`WidenArray`] over one.
91    ///
92    /// Legacy decimal readers and DecimalByteParts assembly can give us a buffer whose width does
93    /// not match the logical precision. We need a canonical leaf without changing the logical
94    /// dtype or widening narrow storage unnecessarily.
95    ///
96    /// If the width is already canonical, we trust the declared precision and reuse the buffer
97    /// without reading its values. This also lets device buffers stay on the device. Otherwise,
98    /// child selection follows [`Self::encode_decimal`], keeping the logical dtype and validity.
99    /// We reuse buffers at the selected width and convert other widths without rescaling.
100    /// A narrowing copy writes zeros in null slots. A reused buffer keeps its null slots.
101    ///
102    /// The scan and the conversion run on the CPU, so noncanonical values and their validity
103    /// **must** be in host memory. This function does not copy device memory to the host.
104    ///
105    /// Readers and decoders may already know the range. For noncanonical storage, `known_range`
106    /// lets us reuse it:
107    ///
108    /// - [`Precision::Absent`] scans valid values to find their extrema.
109    /// - [`Precision::Exact`] **must** give the actual minimum and maximum valid unscaled values.
110    /// - [`Precision::Inexact`] **must** contain every valid unscaled value.
111    ///
112    /// We cache the range on the child as [`Stat::Min`] and [`Stat::Max`], keeping its exactness
113    /// because enclosing bounds are not necessarily actual extrema. Scanning empty or all-null
114    /// input yields no extrema to cache. Canonical-width storage ignores `known_range`.
115    /// Supplied ranges are trusted, even when a narrowing copy checks the selected precision.
116    /// Incorrect ranges or declared precision can cause incorrect results in later operations.
117    ///
118    /// # Errors
119    ///
120    /// Rejects malformed buffer layout or validity length. For noncanonical storage, also rejects
121    /// values or validity in device memory, reversed ranges, and range bounds outside the logical
122    /// precision. A narrowing copy rejects valid values outside the selected precision.
123    pub fn from_stored_decimal(
124        parts: DecimalDataParts,
125        known_range: Precision<RangeInclusive<i256>>,
126        ctx: &mut ExecutionCtx,
127    ) -> VortexResult<ArrayRef> {
128        let DecimalDataParts {
129            decimal_dtype,
130            values,
131            values_type,
132            validity,
133        } = parts;
134
135        // Validate before typed buffer access or mask execution, which can panic on malformed data.
136        DecimalData::validate(&values, values_type)?;
137        let len = values.len() / values_type.byte_width();
138        if let Some(validity_len) = validity.maybe_len() {
139            vortex_ensure_eq!(
140                validity_len,
141                len,
142                "Decimal validity must have one entry per value"
143            );
144        }
145
146        if values_type == decimal_dtype.canonical_values_type() {
147            let array = DecimalArray::try_new_handle(values, values_type, decimal_dtype, validity)?;
148
149            return Ok(array.into_array());
150        }
151
152        // The scan below reads host memory, so device memory would panic there.
153        vortex_ensure!(
154            values.is_on_host() && validity.as_array().is_none_or(ArrayRef::is_host),
155            "Decimal storage at a noncanonical width must be in host memory, got device memory"
156        );
157        let values = values.unwrap_host();
158        let mask = validity.execute_mask(len, ctx)?;
159        match_each_decimal_value_type!(values_type, |T| {
160            let values = Buffer::<T>::from_byte_buffer(values);
161            let range = ValidRange::resolve(known_range, values.as_slice(), &mask);
162            widen_or_canonical(values, validity, &mask, range, decimal_dtype)
163        })
164    }
165}
166
167/// Chooses child precision from a valid-value range and the parent scale.
168///
169/// We keep the parent scale so values do not need rescaling. The child precision must therefore
170/// be at least 1 and at least the scale. With empty or all-null input, these are the only
171/// constraints. Pass `None` for that case. Otherwise, null payloads **must** be excluded from the
172/// range.
173///
174/// # Errors
175///
176/// Returns an error if the minimum exceeds the maximum or an endpoint exceeds the parent precision.
177pub(crate) fn smallest_decimal_dtype(
178    range: Option<RangeInclusive<i256>>,
179    parent: DecimalDType,
180) -> VortexResult<DecimalDType> {
181    let minimum_precision = parent.scale().max(1) as u8;
182    let Some(range) = range else {
183        return Ok(DecimalDType::new(minimum_precision, parent.scale()));
184    };
185
186    let (min, max) = range.into_inner();
187    vortex_ensure!(
188        min <= max,
189        "Decimal range requires ordered extrema, got {min} to {max}"
190    );
191
192    for precision in minimum_precision..=parent.precision() {
193        let index = usize::from(precision);
194        if i256::MIN_BY_PRECISION[index] <= min && max <= i256::MAX_BY_PRECISION[index] {
195            return Ok(DecimalDType::new(precision, parent.scale()));
196        }
197    }
198
199    vortex_bail!("Decimal range must fit {parent}, got {min} to {max}")
200}
201
202/// Value bounds for choosing child precision and caching statistics.
203///
204/// Supplied bounds need not be actual extrema, so we keep their exactness when caching them.
205/// `Empty` means a scan found no valid values, rather than that statistics are unavailable.
206enum ValidRange {
207    /// No value is valid.
208    Empty,
209    /// The smallest and the largest valid value.
210    Exact(RangeInclusive<i256>),
211    /// Bounds that hold every valid value. They need not be valid values themselves.
212    Bounds(RangeInclusive<i256>),
213}
214
215impl ValidRange {
216    /// Uses known bounds when available, otherwise scans valid values.
217    ///
218    /// Readers and decoders may already have bounds from their input statistics. Reusing them
219    /// avoids scanning the buffer again.
220    fn resolve<F: NativeDecimalType>(
221        known_range: Precision<RangeInclusive<i256>>,
222        values: &[F],
223        mask: &Mask,
224    ) -> Self {
225        match known_range {
226            Precision::Exact(extrema) => return Self::Exact(extrema),
227            Precision::Inexact(bounds) => return Self::Bounds(bounds),
228            Precision::Absent => {}
229        }
230
231        let Some((min, max)) = valid_range(values, mask) else {
232            return Self::Empty;
233        };
234        let min = min.to_i256().vortex_expect("every decimal value fits i256");
235        let max = max.to_i256().vortex_expect("every decimal value fits i256");
236
237        Self::Exact(min..=max)
238    }
239
240    /// Returns the bounds that hold every valid value, or `None` when no value is valid.
241    fn bounds(&self) -> Option<RangeInclusive<i256>> {
242        match self {
243            Self::Empty => None,
244            Self::Exact(bounds) | Self::Bounds(bounds) => Some(bounds.clone()),
245        }
246    }
247
248    /// Caches bounds on the child so later aggregates can reuse them without another scan.
249    ///
250    /// The bounds **must** fit the child's selected precision.
251    fn cache_statistics(self, child: &DecimalArray) {
252        let extrema = match self {
253            Self::Empty => return,
254            Self::Exact(bounds) => Precision::Exact(bounds),
255            Self::Bounds(bounds) => Precision::Inexact(bounds),
256        };
257        let child_dtype = child.decimal_dtype();
258        let min = extrema
259            .clone()
260            .map(|bounds| decimal_stat(*bounds.start(), child_dtype));
261        let max = extrema.map(|bounds| decimal_stat(*bounds.end(), child_dtype));
262        child.statistics().set(Stat::Min, min);
263        child.statistics().set(Stat::Max, max);
264    }
265}
266
267/// Wraps a narrower canonical child, or builds a leaf at the logical width.
268///
269/// The result must keep the logical dtype. Returning a narrow child directly would change it.
270///
271/// `mask` **must** be the mask of `validity`.
272fn widen_or_canonical<F: NativeDecimalType>(
273    values: Buffer<F>,
274    validity: Validity,
275    mask: &Mask,
276    range: ValidRange,
277    decimal_dtype: DecimalDType,
278) -> VortexResult<ArrayRef> {
279    if let Some(child) = narrowest_decimal_child(&values, &validity, mask, range, decimal_dtype)? {
280        let dtype = DType::Decimal(decimal_dtype, validity.nullability());
281        return Ok(WidenArray::try_new(child.into_array(), dtype)?.into_array());
282    }
283
284    let array = build_canonical_decimal(&values, &validity, mask, decimal_dtype)?;
285
286    Ok(array.into_array())
287}
288
289/// Builds the narrowest canonical child, or returns `None` if no narrower width exists.
290///
291/// Reducing precision without changing width saves no space, so we return `None` in that case.
292///
293/// Every valid value **must** lie in `range`, and `mask` **must** be the mask of `validity`.
294fn narrowest_decimal_child<F: NativeDecimalType>(
295    values: &Buffer<F>,
296    validity: &Validity,
297    mask: &Mask,
298    range: ValidRange,
299    decimal_dtype: DecimalDType,
300) -> VortexResult<Option<DecimalArray>> {
301    let child_dtype = smallest_decimal_dtype(range.bounds(), decimal_dtype)?;
302    let child_type = child_dtype.canonical_values_type();
303    if child_type == decimal_dtype.canonical_values_type() {
304        return Ok(None);
305    }
306
307    let child = build_canonical_decimal(values, validity, mask, child_dtype)?;
308    range.cache_statistics(&child);
309
310    Ok(Some(child))
311}
312
313/// Reuses storage at the dtype's canonical width, copying only when conversion is needed.
314///
315/// Every valid value **must** fit `decimal_dtype`, and `mask` **must** describe `validity`.
316fn build_canonical_decimal<F: NativeDecimalType>(
317    values: &Buffer<F>,
318    validity: &Validity,
319    mask: &Mask,
320    decimal_dtype: DecimalDType,
321) -> VortexResult<DecimalArray> {
322    let values_type = decimal_dtype.canonical_values_type();
323    let buffer = if F::DECIMAL_TYPE == values_type {
324        // The copy helper allocates even at the same width, so retain the existing buffer here.
325        BufferHandle::new_host(values.clone().into_byte_buffer())
326    } else {
327        copy_to_canonical_width(values.as_slice(), mask, decimal_dtype)?
328    };
329
330    DecimalArray::try_new_handle(buffer, values_type, decimal_dtype, validity.clone())
331}
332
333/// Represents a cached bound using the child dtype.
334///
335/// These statistics belong to the child, so their values need its canonical width rather than
336/// the logical parent's width. `value` **must** fit the child's precision.
337fn decimal_stat(value: i256, decimal_dtype: DecimalDType) -> ScalarValue {
338    let value = DecimalValue::try_from_i256(value, decimal_dtype)
339        .vortex_expect("the selected child precision contains its range bounds");
340
341    ScalarValue::Decimal(value)
342}