1use std::hash::Hasher;
5
6use prost::Message;
7use vortex_buffer::Alignment;
8use vortex_error::VortexResult;
9use vortex_error::vortex_bail;
10use vortex_error::vortex_ensure;
11use vortex_error::vortex_ensure_eq;
12use vortex_error::vortex_err;
13use vortex_session::VortexSession;
14
15use crate::ArrayParts;
16use crate::ArrayRef;
17use crate::ExecutionCtx;
18use crate::ExecutionResult;
19use crate::array::Array;
20use crate::array::ArrayView;
21use crate::array::VTable;
22use crate::arrays::decimal::DecimalData;
23use crate::arrays::fixed_width::vtable as fixed_width;
24use crate::buffer::BufferHandle;
25use crate::builders::ArrayBuilder;
26use crate::builders::DecimalBuilder;
27use crate::dtype::DType;
28use crate::dtype::DecimalType;
29use crate::dtype::NativeDecimalType;
30use crate::match_each_decimal_value_type;
31use crate::serde::ArrayChildren;
32mod kernel;
33mod operations;
34mod validity;
35
36use std::hash::Hash;
37
38use vortex_session::registry::CachedId;
39
40use crate::EqMode;
41use crate::array::ArrayId;
42use crate::arrays::decimal::array::DecimalSlots;
43use crate::arrays::decimal::compute::rules::RULES;
44use crate::hash::ArrayEq;
45use crate::hash::ArrayHash;
46pub type DecimalArray = Array<Decimal>;
48
49pub(crate) fn initialize(session: &VortexSession) {
50 kernel::initialize(session);
51}
52
53#[derive(prost::Message)]
55pub struct DecimalMetadata {
56 #[prost(int32, tag = "1")]
61 pub(super) values_type: i32,
62}
63
64impl DecimalMetadata {
65 fn get_values_type(&self) -> VortexResult<DecimalType> {
66 DecimalType::try_from(self.values_type).map_err(|_| {
67 vortex_err!(
68 "DecimalMetadata values_type must be a known DecimalType, got {}",
69 self.values_type
70 )
71 })
72 }
73}
74
75impl ArrayHash for DecimalData {
76 fn array_hash<H: Hasher>(&self, state: &mut H, accuracy: EqMode) {
77 self.values.array_hash(state, accuracy);
78 std::mem::discriminant(&self.values_type).hash(state);
79 }
80}
81
82impl ArrayEq for DecimalData {
83 fn array_eq(&self, other: &Self, accuracy: EqMode) -> bool {
84 self.values.array_eq(&other.values, accuracy) && self.values_type == other.values_type
85 }
86}
87
88impl VTable for Decimal {
89 type TypedArrayData = DecimalData;
90
91 type OperationsVTable = Self;
92 type ValidityVTable = Self;
93
94 fn id(&self) -> ArrayId {
95 static ID: CachedId = CachedId::new("vortex.decimal");
96 *ID
97 }
98
99 fn nbuffers(_array: ArrayView<'_, Self>) -> usize {
100 1
101 }
102
103 fn buffer(array: ArrayView<'_, Self>, idx: usize) -> BufferHandle {
104 fixed_width::buffer("DecimalArray", &array.values, idx)
105 }
106
107 fn buffer_name(_array: ArrayView<'_, Self>, idx: usize) -> Option<String> {
108 fixed_width::buffer_name(idx)
109 }
110
111 fn with_buffers(
112 &self,
113 array: ArrayView<'_, Self>,
114 buffers: &[BufferHandle],
115 ) -> VortexResult<ArrayParts<Self>> {
116 let mut data = array.data().clone();
117 data.values = fixed_width::single_buffer(buffers)?;
118 Ok(ArrayParts::new(
119 self.clone(),
120 array.dtype().clone(),
121 array.len(),
122 data,
123 array.slots().iter().cloned().collect(),
124 ))
125 }
126
127 fn serialize(
128 array: ArrayView<'_, Self>,
129 _session: &VortexSession,
130 ) -> VortexResult<Option<Vec<u8>>> {
131 Ok(Some(
132 DecimalMetadata {
133 values_type: array.values_type() as i32,
134 }
135 .encode_to_vec(),
136 ))
137 }
138
139 fn validate(
140 &self,
141 data: &DecimalData,
142 dtype: &DType,
143 len: usize,
144 slots: &[Option<ArrayRef>],
145 ) -> VortexResult<()> {
146 let DType::Decimal(_, nullability) = dtype else {
147 vortex_bail!("Expected decimal dtype, got {dtype:?}");
148 };
149 vortex_ensure_eq!(
150 data.len(),
151 len,
152 "DecimalArray length does not match outer length",
153 );
154 let validity =
155 crate::array::child_to_validity(slots[DecimalSlots::VALIDITY].as_ref(), *nullability);
156 if let Some(validity_len) = validity.maybe_len() {
157 vortex_ensure_eq!(
158 validity_len,
159 len,
160 "DecimalArray validity len does not match outer length",
161 );
162 }
163
164 Ok(())
165 }
166
167 fn deserialize(
168 &self,
169 dtype: &DType,
170 len: usize,
171 metadata: &[u8],
172 buffers: &[BufferHandle],
173 children: &dyn ArrayChildren,
174 _session: &VortexSession,
175 ) -> VortexResult<ArrayParts<Self>> {
176 let metadata = DecimalMetadata::decode(metadata)?;
177 let values_type = metadata.get_values_type()?;
178 let values = fixed_width::single_buffer(buffers)?;
179
180 let validity = fixed_width::deserialize_validity(dtype.nullability(), len, children)?;
181
182 let Some(decimal_dtype) = dtype.as_decimal_opt() else {
183 vortex_bail!("Expected Decimal dtype, got {:?}", dtype)
184 };
185
186 let slots = DecimalData::make_slots(&validity, len);
187 let data = match_each_decimal_value_type!(values_type, |D| {
188 vortex_ensure!(
190 values.is_aligned_to(Alignment::of::<D>()),
191 "DecimalArray buffer not aligned for values type {:?}",
192 D::DECIMAL_TYPE
193 );
194 DecimalData::try_new_handle(values, values_type, *decimal_dtype)
195 })?;
196 Ok(ArrayParts::new(
197 self.clone(),
198 dtype.clone(),
199 len,
200 data,
201 slots,
202 ))
203 }
204
205 fn slot_name(_array: ArrayView<'_, Self>, idx: usize) -> String {
206 DecimalSlots::NAMES[idx].to_string()
207 }
208
209 fn execute(array: Array<Self>, _ctx: &mut ExecutionCtx) -> VortexResult<ExecutionResult> {
210 Ok(ExecutionResult::done(array))
211 }
212
213 fn append_to_builder(
214 array: ArrayView<'_, Self>,
215 builder: &mut dyn ArrayBuilder,
216 ctx: &mut ExecutionCtx,
217 ) -> VortexResult<()> {
218 let Some(builder) = builder.as_any_mut().downcast_mut::<DecimalBuilder>() else {
219 vortex_bail!("append_to_builder for Decimal requires a DecimalBuilder");
220 };
221 builder.append_decimal_array(&array.into_owned(), ctx)
222 }
223
224 fn reduce_parent(
225 array: ArrayView<'_, Self>,
226 parent: &ArrayRef,
227 child_idx: usize,
228 ) -> VortexResult<Option<ArrayRef>> {
229 RULES.evaluate(array, parent, child_idx)
230 }
231}
232
233#[derive(Clone, Debug)]
234pub struct Decimal;
235
236#[cfg(test)]
237mod tests {
238 use std::sync::LazyLock;
239
240 use prost::Message;
241 use rstest::rstest;
242 use vortex_buffer::ByteBuffer;
243 use vortex_buffer::ByteBufferMut;
244 use vortex_buffer::buffer;
245 use vortex_error::VortexResult;
246 use vortex_session::VortexSession;
247 use vortex_session::registry::ReadContext;
248
249 use super::DecimalMetadata;
250 use crate::ArrayContext;
251 use crate::ArrayRef;
252 use crate::IntoArray;
253 use crate::VortexSessionExecute;
254 use crate::array::VTable;
255 use crate::arrays::Decimal;
256 use crate::arrays::DecimalArray;
257 use crate::assert_arrays_eq;
258 use crate::buffer::BufferHandle;
259 use crate::dtype::DType;
260 use crate::dtype::DecimalDType;
261 use crate::dtype::DecimalType;
262 use crate::dtype::Nullability;
263 use crate::dtype::i256;
264 use crate::serde::SerializeOptions;
265 use crate::serde::SerializedArray;
266 use crate::validity::Validity;
267
268 static SESSION: LazyLock<VortexSession> = LazyLock::new(VortexSession::empty);
269
270 #[test]
271 fn test_array_serde() {
272 let array = DecimalArray::new(
273 buffer![100i128, 200i128, 300i128, 400i128, 500i128],
274 DecimalDType::new(10, 2),
275 Validity::NonNullable,
276 );
277 let dtype = array.dtype().clone();
278
279 let array_ctx = ArrayContext::empty();
280 let out = array
281 .into_array()
282 .serialize(&array_ctx, &SESSION, &SerializeOptions::default())
283 .unwrap();
284 let mut concat = ByteBufferMut::empty();
286 for buf in out {
287 concat.extend_from_slice(buf.as_ref());
288 }
289
290 let concat = concat.freeze();
291
292 let parts = SerializedArray::try_from(concat).unwrap();
293 let decoded = parts
294 .decode(&dtype, 5, &ReadContext::new(array_ctx.to_ids()), &SESSION)
295 .unwrap();
296 assert!(decoded.is::<Decimal>());
297 assert_arrays_eq!(
298 decoded,
299 DecimalArray::from_iter([100i128, 200, 300, 400, 500], DecimalDType::new(10, 2)),
300 &mut SESSION.create_execution_ctx()
301 );
302 }
303
304 #[test]
305 fn test_nullable_decimal_serde_roundtrip() {
306 let mut ctx = SESSION.create_execution_ctx();
307 let array = DecimalArray::new(
308 buffer![1234567i32, 0i32, -9999999i32],
309 DecimalDType::new(7, 3),
310 Validity::from_iter([true, false, true]),
311 );
312 let dtype = array.dtype().clone();
313 let len = array.len();
314
315 let array_ctx = ArrayContext::empty();
316 let out = array
317 .clone()
318 .into_array()
319 .serialize(&array_ctx, &SESSION, &SerializeOptions::default())
320 .unwrap();
321 let mut concat = ByteBufferMut::empty();
322 for buf in out {
323 concat.extend_from_slice(buf.as_ref());
324 }
325
326 let parts = SerializedArray::try_from(concat.freeze()).unwrap();
327 let decoded = parts
328 .decode(&dtype, len, &ReadContext::new(array_ctx.to_ids()), &SESSION)
329 .unwrap();
330
331 assert_arrays_eq!(decoded, array, &mut ctx);
332 }
333
334 #[rstest]
335 #[case::first_unused(6)]
336 #[case::negative(-1)]
337 #[case::max(i32::MAX)]
338 fn test_deserialize_rejects_unknown_values_type(#[case] raw: i32) {
339 let metadata = DecimalMetadata { values_type: raw }.encode_to_vec();
340 let buffers = [BufferHandle::new_host(ByteBuffer::zeroed(4))];
341 let children: Vec<ArrayRef> = Vec::new();
342
343 let result = Decimal.deserialize(
344 &DType::Decimal(DecimalDType::new(2, 0), Nullability::NonNullable),
345 4,
346 &metadata,
347 &buffers,
348 &children,
349 &SESSION,
350 );
351
352 let expected =
353 format!("DecimalMetadata values_type must be a known DecimalType, got {raw}");
354 assert!(result.is_err_and(|err| err.to_string().contains(&expected)));
355 }
356
357 #[rstest]
358 #[case::i8(
359 DecimalArray::from_iter([1i8, -2, 3], DecimalDType::new(2, 0)),
360 DecimalType::I8
361 )]
362 #[case::i16(
363 DecimalArray::from_iter([1i16, -2, 3], DecimalDType::new(4, 0)),
364 DecimalType::I16
365 )]
366 #[case::i32(
367 DecimalArray::from_iter([1i32, -2, 3], DecimalDType::new(9, 2)),
368 DecimalType::I32
369 )]
370 #[case::i64(
371 DecimalArray::from_iter([1i64, -2, 3], DecimalDType::new(18, 2)),
372 DecimalType::I64
373 )]
374 #[case::i128(
375 DecimalArray::from_iter([1i128, -2, 3], DecimalDType::new(38, 2)),
376 DecimalType::I128
377 )]
378 #[case::i256(
379 DecimalArray::from_iter(
380 [i256::from_i128(1), i256::from_i128(-2), i256::from_i128(3)],
381 DecimalDType::new(76, 2),
382 ),
383 DecimalType::I256
384 )]
385 fn test_serde_each_values_type(
386 #[case] array: DecimalArray,
387 #[case] values_type: DecimalType,
388 ) -> VortexResult<()> {
389 let dtype = array.dtype().clone();
390 let len = array.len();
391
392 let array_ctx = ArrayContext::empty();
393 let out = array.clone().into_array().serialize(
394 &array_ctx,
395 &SESSION,
396 &SerializeOptions::default(),
397 )?;
398 let mut concat = ByteBufferMut::empty();
399 for buf in out {
400 concat.extend_from_slice(buf.as_ref());
401 }
402
403 let parts = SerializedArray::try_from(concat.freeze())?;
404 let decoded = parts.decode(&dtype, len, &ReadContext::new(array_ctx.to_ids()), &SESSION)?;
405
406 assert_eq!(decoded.as_::<Decimal>().values_type(), values_type);
407 assert_arrays_eq!(decoded, array, &mut SESSION.create_execution_ctx());
408
409 Ok(())
410 }
411}