vortex_array/arrays/widen/encode/decimal.rs
1// SPDX-License-Identifier: Apache-2.0
2// SPDX-FileCopyrightText: Copyright the Vortex contributors
3
4//! Decimal encoding and conversion of legacy storage into canonical decimal leaves.
5//!
6//! The legacy `vortex.decimal` format stores integer width independently of logical precision.
7//! DecimalByteParts assembly can also produce such buffers, but we now require canonical
8//! `vortex.decimal` leaves to have a width determined and fixed by their precision.
9//!
10//! In this new world, we can retain narrow storage as a [`WidenArray`] over a canonical child.
11//! Current [`WidenPlugin`] writes still use the legacy format, so this conversion can also apply
12//! to newly written data (which will happen until `Widen` is added to the `core` edition).
13//!
14//! We unify child selection in both encoding and legacy conversion because it has non-trivial
15//! logic: width alone does not determine a valid child dtype. For example, unscaled 127 fits `i8`
16//! but needs precision 3, whose canonical width is `i16`.
17//!
18//! [`WidenPlugin`]: crate::arrays::widen::WidenPlugin
19
20use std::ops::RangeInclusive;
21
22use vortex_buffer::Buffer;
23use vortex_error::VortexExpect;
24use vortex_error::VortexResult;
25use vortex_error::vortex_bail;
26use vortex_error::vortex_ensure;
27use vortex_error::vortex_ensure_eq;
28use vortex_mask::Mask;
29
30use crate::ArrayRef;
31use crate::ExecutionCtx;
32use crate::IntoArray;
33use crate::arrays::DecimalArray;
34use crate::arrays::WidenArray;
35use crate::arrays::decimal::DecimalArrayExt;
36use crate::arrays::decimal::DecimalData;
37use crate::arrays::decimal::DecimalDataParts;
38use crate::arrays::decimal::copy_to_canonical_width;
39use crate::arrays::decimal::valid_range;
40use crate::buffer::BufferHandle;
41use crate::dtype::DType;
42use crate::dtype::DecimalDType;
43use crate::dtype::NativeDecimalType;
44use crate::dtype::i256;
45use crate::expr::stats::Precision;
46use crate::expr::stats::Stat;
47use crate::match_each_decimal_value_type;
48use crate::scalar::DecimalValue;
49use crate::scalar::ScalarValue;
50use crate::validity::Validity;
51
52impl WidenArray {
53 /// Stores a decimal array at the smallest precision that its valid values and scale allow.
54 ///
55 /// We keep the logical dtype and nullability by wrapping the narrower child in a
56 /// [`WidenArray`]. Reducing precision within the same storage width saves no space, so in that
57 /// case we return the input unchanged.
58 ///
59 /// Empty and all-null arrays have no values to constrain precision, but we still need to keep
60 /// the scale. They use precision 1 or the scale, whichever is greater. If a child is selected,
61 /// we cache any exact extrema as [`Stat::Min`] and [`Stat::Max`] for later aggregates.
62 ///
63 /// # Errors
64 ///
65 /// Returns an error if a valid value does not fit the input precision.
66 pub fn encode_decimal(array: DecimalArray, ctx: &mut ExecutionCtx) -> VortexResult<ArrayRef> {
67 let logical_decimal_dtype = array.decimal_dtype();
68 let validity = array.validity()?;
69 let validity_mask = validity.execute_mask(array.len(), ctx)?;
70 let child = match_each_decimal_value_type!(array.values_type(), |T| {
71 let values_buffer = array.buffer::<T>();
72 let valid_value_range =
73 ValidRange::resolve(Precision::Absent, values_buffer.as_slice(), &validity_mask);
74 narrowest_decimal_child(
75 &values_buffer,
76 &validity,
77 &validity_mask,
78 valid_value_range,
79 logical_decimal_dtype,
80 )?
81 });
82
83 let Some(child) = child else {
84 return Ok(array.into_array());
85 };
86
87 Ok(WidenArray::try_new(child.into_array(), array.dtype().clone())?.into_array())
88 }
89
90 /// Converts stored decimals into a canonical leaf or a [`WidenArray`] over one.
91 ///
92 /// Legacy decimal readers and DecimalByteParts assembly can give us a buffer whose width does
93 /// not match the logical precision. We need a canonical leaf without changing the logical
94 /// dtype or widening narrow storage unnecessarily.
95 ///
96 /// If the width is already canonical, we trust the declared precision and reuse the buffer
97 /// without reading its values. This also lets device buffers stay on the device. Otherwise,
98 /// child selection follows [`Self::encode_decimal`], keeping the logical dtype and validity.
99 /// We reuse buffers at the selected width and convert other widths without rescaling.
100 /// A narrowing copy writes zeros in null slots. A reused buffer keeps its null slots.
101 ///
102 /// Scanning and conversion use CPU kernels, so noncanonical buffers are brought to the host.
103 /// Their validity **must** already be executable on the host because these kernels need a mask
104 /// to exclude null payloads.
105 ///
106 /// Readers and decoders may already know the range. For noncanonical storage, `known_range`
107 /// lets us reuse it:
108 ///
109 /// - [`Precision::Absent`] scans valid values to find their extrema.
110 /// - [`Precision::Exact`] **must** give the actual minimum and maximum valid unscaled values.
111 /// - [`Precision::Inexact`] **must** contain every valid unscaled value.
112 ///
113 /// We cache the range on the child as [`Stat::Min`] and [`Stat::Max`], keeping its exactness
114 /// because enclosing bounds are not necessarily actual extrema. Scanning empty or all-null
115 /// input yields no extrema to cache. Canonical-width storage ignores `known_range`.
116 /// Supplied ranges are trusted, even when a narrowing copy checks the selected precision.
117 /// Incorrect ranges or declared precision can cause incorrect results in later operations.
118 ///
119 /// # Errors
120 ///
121 /// Rejects malformed buffer layout or validity length. For noncanonical storage, also rejects
122 /// failed host copies, reversed ranges, and range bounds outside the logical precision. A
123 /// narrowing copy rejects valid values outside the selected precision.
124 pub fn from_stored_decimal(
125 parts: DecimalDataParts,
126 known_range: Precision<RangeInclusive<i256>>,
127 ctx: &mut ExecutionCtx,
128 ) -> VortexResult<ArrayRef> {
129 let DecimalDataParts {
130 decimal_dtype,
131 values,
132 values_type,
133 validity,
134 } = parts;
135
136 // Validate before typed buffer access or mask execution, which can panic on malformed data.
137 DecimalData::validate(&values, values_type)?;
138 let len = values.len() / values_type.byte_width();
139 if let Some(validity_len) = validity.maybe_len() {
140 vortex_ensure_eq!(
141 validity_len,
142 len,
143 "Decimal validity must have one entry per value"
144 );
145 }
146
147 if values_type == decimal_dtype.canonical_values_type() {
148 let array = DecimalArray::try_new_handle(values, values_type, decimal_dtype, validity)?;
149
150 return Ok(array.into_array());
151 }
152
153 let values = values.try_into_host_sync()?;
154 let mask = validity.execute_mask(len, ctx)?;
155 match_each_decimal_value_type!(values_type, |T| {
156 let values = Buffer::<T>::from_byte_buffer(values);
157 let range = ValidRange::resolve(known_range, values.as_slice(), &mask);
158 widen_or_canonical(values, validity, &mask, range, decimal_dtype)
159 })
160 }
161}
162
163/// Chooses child precision from a valid-value range and the parent scale.
164///
165/// We keep the parent scale so values do not need rescaling. The child precision must therefore
166/// be at least 1 and at least the scale. With empty or all-null input, these are the only
167/// constraints. Pass `None` for that case. Otherwise, null payloads **must** be excluded from the
168/// range.
169///
170/// # Errors
171///
172/// Returns an error if the minimum exceeds the maximum or an endpoint exceeds the parent precision.
173pub(crate) fn smallest_decimal_dtype(
174 range: Option<RangeInclusive<i256>>,
175 parent: DecimalDType,
176) -> VortexResult<DecimalDType> {
177 let minimum_precision = parent.scale().max(1) as u8;
178 let Some(range) = range else {
179 return Ok(DecimalDType::new(minimum_precision, parent.scale()));
180 };
181
182 let (min, max) = range.into_inner();
183 vortex_ensure!(
184 min <= max,
185 "Decimal range requires ordered extrema, got {min} to {max}"
186 );
187
188 for precision in minimum_precision..=parent.precision() {
189 let index = usize::from(precision);
190 if i256::MIN_BY_PRECISION[index] <= min && max <= i256::MAX_BY_PRECISION[index] {
191 return Ok(DecimalDType::new(precision, parent.scale()));
192 }
193 }
194
195 vortex_bail!("Decimal range must fit {parent}, got {min} to {max}")
196}
197
198/// Value bounds for choosing child precision and caching statistics.
199///
200/// Supplied bounds need not be actual extrema, so we keep their exactness when caching them.
201/// `Empty` means a scan found no valid values, rather than that statistics are unavailable.
202enum ValidRange {
203 /// No value is valid.
204 Empty,
205 /// The smallest and the largest valid value.
206 Exact(RangeInclusive<i256>),
207 /// Bounds that hold every valid value. They need not be valid values themselves.
208 Bounds(RangeInclusive<i256>),
209}
210
211impl ValidRange {
212 /// Uses known bounds when available, otherwise scans valid values.
213 ///
214 /// Readers and decoders may already have bounds from their input statistics. Reusing them
215 /// avoids scanning the buffer again.
216 fn resolve<F: NativeDecimalType>(
217 known_range: Precision<RangeInclusive<i256>>,
218 values: &[F],
219 mask: &Mask,
220 ) -> Self {
221 match known_range {
222 Precision::Exact(extrema) => return Self::Exact(extrema),
223 Precision::Inexact(bounds) => return Self::Bounds(bounds),
224 Precision::Absent => {}
225 }
226
227 let Some((min, max)) = valid_range(values, mask) else {
228 return Self::Empty;
229 };
230 let min = min.to_i256().vortex_expect("every decimal value fits i256");
231 let max = max.to_i256().vortex_expect("every decimal value fits i256");
232
233 Self::Exact(min..=max)
234 }
235
236 /// Returns the bounds that hold every valid value, or `None` when no value is valid.
237 fn bounds(&self) -> Option<RangeInclusive<i256>> {
238 match self {
239 Self::Empty => None,
240 Self::Exact(bounds) | Self::Bounds(bounds) => Some(bounds.clone()),
241 }
242 }
243
244 /// Caches bounds on the child so later aggregates can reuse them without another scan.
245 ///
246 /// The bounds **must** fit the child's selected precision.
247 fn cache_statistics(self, child: &DecimalArray) {
248 let extrema = match self {
249 Self::Empty => return,
250 Self::Exact(bounds) => Precision::Exact(bounds),
251 Self::Bounds(bounds) => Precision::Inexact(bounds),
252 };
253 let child_dtype = child.decimal_dtype();
254 let min = extrema
255 .clone()
256 .map(|bounds| decimal_stat(*bounds.start(), child_dtype));
257 let max = extrema.map(|bounds| decimal_stat(*bounds.end(), child_dtype));
258 child.statistics().set(Stat::Min, min);
259 child.statistics().set(Stat::Max, max);
260 }
261}
262
263/// Wraps a narrower canonical child, or builds a leaf at the logical width.
264///
265/// The result must keep the logical dtype. Returning a narrow child directly would change it.
266///
267/// `mask` **must** be the mask of `validity`.
268fn widen_or_canonical<F: NativeDecimalType>(
269 values: Buffer<F>,
270 validity: Validity,
271 mask: &Mask,
272 range: ValidRange,
273 decimal_dtype: DecimalDType,
274) -> VortexResult<ArrayRef> {
275 if let Some(child) = narrowest_decimal_child(&values, &validity, mask, range, decimal_dtype)? {
276 let dtype = DType::Decimal(decimal_dtype, validity.nullability());
277 return Ok(WidenArray::try_new(child.into_array(), dtype)?.into_array());
278 }
279
280 let array = build_canonical_decimal(&values, &validity, mask, decimal_dtype)?;
281
282 Ok(array.into_array())
283}
284
285/// Builds the narrowest canonical child, or returns `None` if no narrower width exists.
286///
287/// Reducing precision without changing width saves no space, so we return `None` in that case.
288///
289/// Every valid value **must** lie in `range`, and `mask` **must** be the mask of `validity`.
290fn narrowest_decimal_child<F: NativeDecimalType>(
291 values: &Buffer<F>,
292 validity: &Validity,
293 mask: &Mask,
294 range: ValidRange,
295 decimal_dtype: DecimalDType,
296) -> VortexResult<Option<DecimalArray>> {
297 let child_dtype = smallest_decimal_dtype(range.bounds(), decimal_dtype)?;
298 let child_type = child_dtype.canonical_values_type();
299 if child_type == decimal_dtype.canonical_values_type() {
300 return Ok(None);
301 }
302
303 let child = build_canonical_decimal(values, validity, mask, child_dtype)?;
304 range.cache_statistics(&child);
305
306 Ok(Some(child))
307}
308
309/// Reuses storage at the dtype's canonical width, copying only when conversion is needed.
310///
311/// Every valid value **must** fit `decimal_dtype`, and `mask` **must** describe `validity`.
312fn build_canonical_decimal<F: NativeDecimalType>(
313 values: &Buffer<F>,
314 validity: &Validity,
315 mask: &Mask,
316 decimal_dtype: DecimalDType,
317) -> VortexResult<DecimalArray> {
318 let values_type = decimal_dtype.canonical_values_type();
319 let buffer = if F::DECIMAL_TYPE == values_type {
320 // The copy helper allocates even at the same width, so retain the existing buffer here.
321 BufferHandle::new_host(values.clone().into_byte_buffer())
322 } else {
323 copy_to_canonical_width(values.as_slice(), mask, decimal_dtype)?
324 };
325
326 DecimalArray::try_new_handle(buffer, values_type, decimal_dtype, validity.clone())
327}
328
329/// Represents a cached bound using the child dtype.
330///
331/// These statistics belong to the child, so their values need its canonical width rather than
332/// the logical parent's width. `value` **must** fit the child's precision.
333fn decimal_stat(value: i256, decimal_dtype: DecimalDType) -> ScalarValue {
334 let value = DecimalValue::try_from_i256(value, decimal_dtype)
335 .vortex_expect("the selected child precision contains its range bounds");
336
337 ScalarValue::Decimal(value)
338}