Skip to main content

vortex_array/arrays/widen/encode/
decimal.rs

1// SPDX-License-Identifier: Apache-2.0
2// SPDX-FileCopyrightText: Copyright the Vortex contributors
3
4//! Decimal encoding and conversion of legacy storage into canonical decimal leaves.
5//!
6//! The legacy `vortex.decimal` format stores integer width independently of logical precision.
7//! DecimalByteParts assembly can also produce such buffers, but we now require canonical
8//! `vortex.decimal` leaves to have a width determined and fixed by their precision.
9//!
10//! In this new world, we can retain narrow storage as a [`WidenArray`] over a canonical child.
11//! Current [`WidenPlugin`] writes still use the legacy format, so this conversion can also apply
12//! to newly written data (which will happen until `Widen` is added to the `core` edition).
13//!
14//! We unify child selection in both encoding and legacy conversion because it has non-trivial
15//! logic: width alone does not determine a valid child dtype. For example, unscaled 127 fits `i8`
16//! but needs precision 3, whose canonical width is `i16`.
17//!
18//! [`WidenPlugin`]: crate::arrays::widen::WidenPlugin
19
20use std::ops::RangeInclusive;
21
22use vortex_buffer::Buffer;
23use vortex_error::VortexExpect;
24use vortex_error::VortexResult;
25use vortex_error::vortex_bail;
26use vortex_error::vortex_ensure;
27use vortex_error::vortex_ensure_eq;
28use vortex_mask::Mask;
29
30use crate::ArrayRef;
31use crate::ExecutionCtx;
32use crate::IntoArray;
33use crate::arrays::DecimalArray;
34use crate::arrays::WidenArray;
35use crate::arrays::decimal::DecimalArrayExt;
36use crate::arrays::decimal::DecimalData;
37use crate::arrays::decimal::DecimalDataParts;
38use crate::arrays::decimal::copy_to_canonical_width;
39use crate::arrays::decimal::valid_range;
40use crate::buffer::BufferHandle;
41use crate::dtype::DType;
42use crate::dtype::DecimalDType;
43use crate::dtype::NativeDecimalType;
44use crate::dtype::i256;
45use crate::expr::stats::Precision;
46use crate::expr::stats::Stat;
47use crate::match_each_decimal_value_type;
48use crate::scalar::DecimalValue;
49use crate::scalar::ScalarValue;
50use crate::validity::Validity;
51
52impl WidenArray {
53    /// Stores a decimal array at the smallest precision that its valid values and scale allow.
54    ///
55    /// We keep the logical dtype and nullability by wrapping the narrower child in a
56    /// [`WidenArray`]. Reducing precision within the same storage width saves no space, so in that
57    /// case we return the input unchanged.
58    ///
59    /// Empty and all-null arrays have no values to constrain precision, but we still need to keep
60    /// the scale. They use precision 1 or the scale, whichever is greater. If a child is selected,
61    /// we cache any exact extrema as [`Stat::Min`] and [`Stat::Max`] for later aggregates.
62    ///
63    /// # Errors
64    ///
65    /// Returns an error if a valid value does not fit the input precision.
66    pub fn encode_decimal(array: DecimalArray, ctx: &mut ExecutionCtx) -> VortexResult<ArrayRef> {
67        let logical_decimal_dtype = array.decimal_dtype();
68        let validity = array.validity()?;
69        let validity_mask = validity.execute_mask(array.len(), ctx)?;
70        let child = match_each_decimal_value_type!(array.values_type(), |T| {
71            let values_buffer = array.buffer::<T>();
72            let valid_value_range =
73                ValidRange::resolve(Precision::Absent, values_buffer.as_slice(), &validity_mask);
74            narrowest_decimal_child(
75                &values_buffer,
76                &validity,
77                &validity_mask,
78                valid_value_range,
79                logical_decimal_dtype,
80            )?
81        });
82
83        let Some(child) = child else {
84            return Ok(array.into_array());
85        };
86
87        Ok(WidenArray::try_new(child.into_array(), array.dtype().clone())?.into_array())
88    }
89
90    /// Converts stored decimals into a canonical leaf or a [`WidenArray`] over one.
91    ///
92    /// Legacy decimal readers and DecimalByteParts assembly can give us a buffer whose width does
93    /// not match the logical precision. We need a canonical leaf without changing the logical
94    /// dtype or widening narrow storage unnecessarily.
95    ///
96    /// If the width is already canonical, we trust the declared precision and reuse the buffer
97    /// without reading its values. This also lets device buffers stay on the device. Otherwise,
98    /// child selection follows [`Self::encode_decimal`], keeping the logical dtype and validity.
99    /// We reuse buffers at the selected width and convert other widths without rescaling.
100    /// A narrowing copy writes zeros in null slots. A reused buffer keeps its null slots.
101    ///
102    /// Scanning and conversion use CPU kernels, so noncanonical buffers are brought to the host.
103    /// Their validity **must** already be executable on the host because these kernels need a mask
104    /// to exclude null payloads.
105    ///
106    /// Readers and decoders may already know the range. For noncanonical storage, `known_range`
107    /// lets us reuse it:
108    ///
109    /// - [`Precision::Absent`] scans valid values to find their extrema.
110    /// - [`Precision::Exact`] **must** give the actual minimum and maximum valid unscaled values.
111    /// - [`Precision::Inexact`] **must** contain every valid unscaled value.
112    ///
113    /// We cache the range on the child as [`Stat::Min`] and [`Stat::Max`], keeping its exactness
114    /// because enclosing bounds are not necessarily actual extrema. Scanning empty or all-null
115    /// input yields no extrema to cache. Canonical-width storage ignores `known_range`.
116    /// Supplied ranges are trusted, even when a narrowing copy checks the selected precision.
117    /// Incorrect ranges or declared precision can cause incorrect results in later operations.
118    ///
119    /// # Errors
120    ///
121    /// Rejects malformed buffer layout or validity length. For noncanonical storage, also rejects
122    /// failed host copies, reversed ranges, and range bounds outside the logical precision. A
123    /// narrowing copy rejects valid values outside the selected precision.
124    pub fn from_stored_decimal(
125        parts: DecimalDataParts,
126        known_range: Precision<RangeInclusive<i256>>,
127        ctx: &mut ExecutionCtx,
128    ) -> VortexResult<ArrayRef> {
129        let DecimalDataParts {
130            decimal_dtype,
131            values,
132            values_type,
133            validity,
134        } = parts;
135
136        // Validate before typed buffer access or mask execution, which can panic on malformed data.
137        DecimalData::validate(&values, values_type)?;
138        let len = values.len() / values_type.byte_width();
139        if let Some(validity_len) = validity.maybe_len() {
140            vortex_ensure_eq!(
141                validity_len,
142                len,
143                "Decimal validity must have one entry per value"
144            );
145        }
146
147        if values_type == decimal_dtype.canonical_values_type() {
148            let array = DecimalArray::try_new_handle(values, values_type, decimal_dtype, validity)?;
149
150            return Ok(array.into_array());
151        }
152
153        let values = values.try_into_host_sync()?;
154        let mask = validity.execute_mask(len, ctx)?;
155        match_each_decimal_value_type!(values_type, |T| {
156            let values = Buffer::<T>::from_byte_buffer(values);
157            let range = ValidRange::resolve(known_range, values.as_slice(), &mask);
158            widen_or_canonical(values, validity, &mask, range, decimal_dtype)
159        })
160    }
161}
162
163/// Chooses child precision from a valid-value range and the parent scale.
164///
165/// We keep the parent scale so values do not need rescaling. The child precision must therefore
166/// be at least 1 and at least the scale. With empty or all-null input, these are the only
167/// constraints. Pass `None` for that case. Otherwise, null payloads **must** be excluded from the
168/// range.
169///
170/// # Errors
171///
172/// Returns an error if the minimum exceeds the maximum or an endpoint exceeds the parent precision.
173pub(crate) fn smallest_decimal_dtype(
174    range: Option<RangeInclusive<i256>>,
175    parent: DecimalDType,
176) -> VortexResult<DecimalDType> {
177    let minimum_precision = parent.scale().max(1) as u8;
178    let Some(range) = range else {
179        return Ok(DecimalDType::new(minimum_precision, parent.scale()));
180    };
181
182    let (min, max) = range.into_inner();
183    vortex_ensure!(
184        min <= max,
185        "Decimal range requires ordered extrema, got {min} to {max}"
186    );
187
188    for precision in minimum_precision..=parent.precision() {
189        let index = usize::from(precision);
190        if i256::MIN_BY_PRECISION[index] <= min && max <= i256::MAX_BY_PRECISION[index] {
191            return Ok(DecimalDType::new(precision, parent.scale()));
192        }
193    }
194
195    vortex_bail!("Decimal range must fit {parent}, got {min} to {max}")
196}
197
198/// Value bounds for choosing child precision and caching statistics.
199///
200/// Supplied bounds need not be actual extrema, so we keep their exactness when caching them.
201/// `Empty` means a scan found no valid values, rather than that statistics are unavailable.
202enum ValidRange {
203    /// No value is valid.
204    Empty,
205    /// The smallest and the largest valid value.
206    Exact(RangeInclusive<i256>),
207    /// Bounds that hold every valid value. They need not be valid values themselves.
208    Bounds(RangeInclusive<i256>),
209}
210
211impl ValidRange {
212    /// Uses known bounds when available, otherwise scans valid values.
213    ///
214    /// Readers and decoders may already have bounds from their input statistics. Reusing them
215    /// avoids scanning the buffer again.
216    fn resolve<F: NativeDecimalType>(
217        known_range: Precision<RangeInclusive<i256>>,
218        values: &[F],
219        mask: &Mask,
220    ) -> Self {
221        match known_range {
222            Precision::Exact(extrema) => return Self::Exact(extrema),
223            Precision::Inexact(bounds) => return Self::Bounds(bounds),
224            Precision::Absent => {}
225        }
226
227        let Some((min, max)) = valid_range(values, mask) else {
228            return Self::Empty;
229        };
230        let min = min.to_i256().vortex_expect("every decimal value fits i256");
231        let max = max.to_i256().vortex_expect("every decimal value fits i256");
232
233        Self::Exact(min..=max)
234    }
235
236    /// Returns the bounds that hold every valid value, or `None` when no value is valid.
237    fn bounds(&self) -> Option<RangeInclusive<i256>> {
238        match self {
239            Self::Empty => None,
240            Self::Exact(bounds) | Self::Bounds(bounds) => Some(bounds.clone()),
241        }
242    }
243
244    /// Caches bounds on the child so later aggregates can reuse them without another scan.
245    ///
246    /// The bounds **must** fit the child's selected precision.
247    fn cache_statistics(self, child: &DecimalArray) {
248        let extrema = match self {
249            Self::Empty => return,
250            Self::Exact(bounds) => Precision::Exact(bounds),
251            Self::Bounds(bounds) => Precision::Inexact(bounds),
252        };
253        let child_dtype = child.decimal_dtype();
254        let min = extrema
255            .clone()
256            .map(|bounds| decimal_stat(*bounds.start(), child_dtype));
257        let max = extrema.map(|bounds| decimal_stat(*bounds.end(), child_dtype));
258        child.statistics().set(Stat::Min, min);
259        child.statistics().set(Stat::Max, max);
260    }
261}
262
263/// Wraps a narrower canonical child, or builds a leaf at the logical width.
264///
265/// The result must keep the logical dtype. Returning a narrow child directly would change it.
266///
267/// `mask` **must** be the mask of `validity`.
268fn widen_or_canonical<F: NativeDecimalType>(
269    values: Buffer<F>,
270    validity: Validity,
271    mask: &Mask,
272    range: ValidRange,
273    decimal_dtype: DecimalDType,
274) -> VortexResult<ArrayRef> {
275    if let Some(child) = narrowest_decimal_child(&values, &validity, mask, range, decimal_dtype)? {
276        let dtype = DType::Decimal(decimal_dtype, validity.nullability());
277        return Ok(WidenArray::try_new(child.into_array(), dtype)?.into_array());
278    }
279
280    let array = build_canonical_decimal(&values, &validity, mask, decimal_dtype)?;
281
282    Ok(array.into_array())
283}
284
285/// Builds the narrowest canonical child, or returns `None` if no narrower width exists.
286///
287/// Reducing precision without changing width saves no space, so we return `None` in that case.
288///
289/// Every valid value **must** lie in `range`, and `mask` **must** be the mask of `validity`.
290fn narrowest_decimal_child<F: NativeDecimalType>(
291    values: &Buffer<F>,
292    validity: &Validity,
293    mask: &Mask,
294    range: ValidRange,
295    decimal_dtype: DecimalDType,
296) -> VortexResult<Option<DecimalArray>> {
297    let child_dtype = smallest_decimal_dtype(range.bounds(), decimal_dtype)?;
298    let child_type = child_dtype.canonical_values_type();
299    if child_type == decimal_dtype.canonical_values_type() {
300        return Ok(None);
301    }
302
303    let child = build_canonical_decimal(values, validity, mask, child_dtype)?;
304    range.cache_statistics(&child);
305
306    Ok(Some(child))
307}
308
309/// Reuses storage at the dtype's canonical width, copying only when conversion is needed.
310///
311/// Every valid value **must** fit `decimal_dtype`, and `mask` **must** describe `validity`.
312fn build_canonical_decimal<F: NativeDecimalType>(
313    values: &Buffer<F>,
314    validity: &Validity,
315    mask: &Mask,
316    decimal_dtype: DecimalDType,
317) -> VortexResult<DecimalArray> {
318    let values_type = decimal_dtype.canonical_values_type();
319    let buffer = if F::DECIMAL_TYPE == values_type {
320        // The copy helper allocates even at the same width, so retain the existing buffer here.
321        BufferHandle::new_host(values.clone().into_byte_buffer())
322    } else {
323        copy_to_canonical_width(values.as_slice(), mask, decimal_dtype)?
324    };
325
326    DecimalArray::try_new_handle(buffer, values_type, decimal_dtype, validity.clone())
327}
328
329/// Represents a cached bound using the child dtype.
330///
331/// These statistics belong to the child, so their values need its canonical width rather than
332/// the logical parent's width. `value` **must** fit the child's precision.
333fn decimal_stat(value: i256, decimal_dtype: DecimalDType) -> ScalarValue {
334    let value = DecimalValue::try_from_i256(value, decimal_dtype)
335        .vortex_expect("the selected child precision contains its range bounds");
336
337    ScalarValue::Decimal(value)
338}