Skip to main content

vortex_alp/alp/
array.rs

1// SPDX-License-Identifier: Apache-2.0
2// SPDX-FileCopyrightText: Copyright the Vortex contributors
3
4use std::fmt::Debug;
5use std::fmt::Display;
6use std::fmt::Formatter;
7use std::hash::Hash;
8use std::hash::Hasher;
9
10use prost::Message;
11use vortex_array::Array;
12use vortex_array::ArrayEq;
13use vortex_array::ArrayHash;
14use vortex_array::ArrayId;
15use vortex_array::ArrayParts;
16use vortex_array::ArrayRef;
17use vortex_array::ArraySlots;
18use vortex_array::ArrayView;
19use vortex_array::EqMode;
20use vortex_array::ExecutionCtx;
21use vortex_array::ExecutionResult;
22use vortex_array::IntoArray;
23use vortex_array::TypedArrayRef;
24use vortex_array::array_slots;
25use vortex_array::arrays::Primitive;
26use vortex_array::buffer::BufferHandle;
27use vortex_array::dtype::DType;
28use vortex_array::dtype::NativePType;
29use vortex_array::dtype::PType;
30use vortex_array::patches::PatchSlotIndices;
31use vortex_array::patches::Patches;
32use vortex_array::patches::PatchesData;
33use vortex_array::patches::PatchesMetadata;
34use vortex_array::require_child;
35use vortex_array::require_patches;
36use vortex_array::serde::ArrayChildren;
37use vortex_array::smallvec::smallvec;
38use vortex_array::vtable::VTable;
39use vortex_array::vtable::ValidityChild;
40use vortex_array::vtable::ValidityVTableFromChild;
41use vortex_error::VortexExpect;
42use vortex_error::VortexResult;
43use vortex_error::vortex_bail;
44use vortex_error::vortex_ensure;
45use vortex_error::vortex_ensure_eq;
46use vortex_error::vortex_panic;
47use vortex_session::VortexSession;
48use vortex_session::registry::CachedId;
49
50use crate::ALPFloat;
51use crate::alp::Exponents;
52use crate::alp::decompress::execute_decompress;
53use crate::alp::rules::RULES;
54
55/// A [`ALP`]-encoded Vortex array.
56pub type ALPArray = Array<ALP>;
57
58impl ArrayHash for ALPData {
59    fn array_hash<H: Hasher>(&self, state: &mut H, _accuracy: EqMode) {
60        self.exponents.hash(state);
61        self.patches_data.hash(state);
62    }
63}
64
65impl ArrayEq for ALPData {
66    fn array_eq(&self, other: &Self, _accuracy: EqMode) -> bool {
67        self.exponents == other.exponents && self.patches_data == other.patches_data
68    }
69}
70
71impl VTable for ALP {
72    type TypedArrayData = ALPData;
73
74    type OperationsVTable = Self;
75    type ValidityVTable = ValidityVTableFromChild;
76
77    fn id(&self) -> ArrayId {
78        static ID: CachedId = CachedId::new("vortex.alp");
79        *ID
80    }
81
82    fn validate(
83        &self,
84        data: &ALPData,
85        dtype: &DType,
86        len: usize,
87        slots: &[Option<ArrayRef>],
88    ) -> VortexResult<()> {
89        let alp_slots = ALPSlotsView::from_slots(slots);
90        let patches =
91            PatchesData::patches_from_slots(data.patches_data.as_ref(), len, slots, PATCH_SLOTS);
92        validate_parts(dtype, len, data.exponents, alp_slots.encoded, patches)
93    }
94
95    fn nbuffers(_array: ArrayView<'_, Self>) -> usize {
96        0
97    }
98
99    fn buffer(_array: ArrayView<'_, Self>, idx: usize) -> BufferHandle {
100        vortex_panic!("ALPArray buffer index {idx} out of bounds")
101    }
102
103    fn buffer_name(_array: ArrayView<'_, Self>, _idx: usize) -> Option<String> {
104        None
105    }
106
107    fn with_buffers(
108        &self,
109        array: ArrayView<'_, Self>,
110        buffers: &[BufferHandle],
111    ) -> VortexResult<ArrayParts<Self>> {
112        vortex_array::vtable::with_empty_buffers(self, array, buffers)
113    }
114
115    fn serialize(
116        array: ArrayView<'_, Self>,
117        _session: &VortexSession,
118    ) -> VortexResult<Option<Vec<u8>>> {
119        let exponents = array.exponents();
120        Ok(Some(
121            ALPMetadata {
122                exp_e: exponents.e as u32,
123                exp_f: exponents.f as u32,
124                patches: array
125                    .patches()
126                    .map(|p| p.to_metadata(array.len(), array.dtype()))
127                    .transpose()?,
128            }
129            .encode_to_vec(),
130        ))
131    }
132
133    fn deserialize(
134        &self,
135        dtype: &DType,
136        len: usize,
137        metadata: &[u8],
138        _buffers: &[BufferHandle],
139        children: &dyn ArrayChildren,
140        _session: &VortexSession,
141    ) -> VortexResult<ArrayParts<Self>> {
142        let metadata = ALPMetadata::decode(metadata)?;
143        let encoded_ptype = match &dtype {
144            DType::Primitive(PType::F32, n) => DType::Primitive(PType::I32, *n),
145            DType::Primitive(PType::F64, n) => DType::Primitive(PType::I64, *n),
146            d => vortex_bail!(MismatchedTypes: "f32 or f64", d),
147        };
148        let encoded = children.get(0, &encoded_ptype, len)?;
149
150        let patches = metadata
151            .patches
152            .map(|p| {
153                let indices = children.get(1, &p.indices_dtype()?, p.len()?)?;
154                let values = children.get(2, dtype, p.len()?)?;
155                let chunk_offsets = p
156                    .chunk_offsets_dtype()?
157                    .map(|dtype| children.get(3, &dtype, usize::try_from(p.chunk_offsets_len())?))
158                    .transpose()?;
159
160                Patches::new(len, p.offset()?, indices, values, chunk_offsets)
161            })
162            .transpose()?;
163
164        let slots = ALPData::make_slots(&encoded, patches.as_ref());
165        let data = ALPData::new(
166            Exponents {
167                e: u8::try_from(metadata.exp_e)?,
168                f: u8::try_from(metadata.exp_f)?,
169            },
170            patches,
171        );
172        Ok(ArrayParts::new(
173            self.clone(),
174            dtype.clone(),
175            len,
176            data,
177            slots,
178        ))
179    }
180
181    fn slot_name(_array: ArrayView<'_, Self>, idx: usize) -> String {
182        ALPSlots::NAMES[idx].to_string()
183    }
184
185    fn execute(array: Array<Self>, ctx: &mut ExecutionCtx) -> VortexResult<ExecutionResult> {
186        let array = require_child!(array, array.encoded(), ALPSlots::ENCODED => Primitive);
187        require_patches!(
188            array,
189            ALPSlots::PATCH_INDICES,
190            ALPSlots::PATCH_VALUES,
191            ALPSlots::PATCH_CHUNK_OFFSETS
192        );
193
194        Ok(ExecutionResult::done(
195            execute_decompress(array, ctx)?.into_array(),
196        ))
197    }
198
199    fn reduce_parent(
200        array: ArrayView<'_, Self>,
201        parent: &ArrayRef,
202        child_idx: usize,
203    ) -> VortexResult<Option<ArrayRef>> {
204        RULES.evaluate(array, parent, child_idx)
205    }
206}
207
208#[array_slots(ALP)]
209pub struct ALPSlots {
210    /// The ALP-encoded values array.
211    #[slot(0)]
212    pub encoded: ArrayRef,
213    /// The indices of exception values that could not be ALP-encoded.
214    #[slot(1)]
215    pub patch_indices: Option<ArrayRef>,
216    /// The exception values that could not be ALP-encoded.
217    #[slot(2)]
218    pub patch_values: Option<ArrayRef>,
219    /// Chunk offsets for the patch indices/values.
220    #[slot(3)]
221    pub patch_chunk_offsets: Option<ArrayRef>,
222}
223
224const PATCH_SLOTS: PatchSlotIndices = PatchSlotIndices {
225    indices: ALPSlots::PATCH_INDICES,
226    values: ALPSlots::PATCH_VALUES,
227    chunk_offsets: ALPSlots::PATCH_CHUNK_OFFSETS,
228};
229
230#[derive(Clone, Debug)]
231pub struct ALPData {
232    patches_data: Option<PatchesData>,
233    exponents: Exponents,
234}
235
236impl Display for ALPData {
237    fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result {
238        write!(f, "exponents: {}", self.exponents)?;
239        if let Some(pd) = &self.patches_data {
240            write!(f, ", patch_offset: {}", pd.offset())?;
241        }
242        Ok(())
243    }
244}
245
246#[derive(Clone, Debug)]
247pub struct ALP;
248
249#[derive(Clone, prost::Message)]
250pub struct ALPMetadata {
251    #[prost(uint32, tag = "1")]
252    pub(crate) exp_e: u32,
253    #[prost(uint32, tag = "2")]
254    pub(crate) exp_f: u32,
255    #[prost(message, optional, tag = "3")]
256    pub(crate) patches: Option<PatchesMetadata>,
257}
258
259impl ALPData {
260    fn validate_components(
261        encoded: &ArrayRef,
262        exponents: Exponents,
263        patches: Option<&Patches>,
264    ) -> VortexResult<()> {
265        vortex_ensure!(
266            matches!(
267                encoded.dtype(),
268                DType::Primitive(PType::I32 | PType::I64, _)
269            ),
270            "ALP encoded ints have invalid DType {}",
271            encoded.dtype(),
272        );
273
274        // Validate exponents are in-bounds for the float, and that patches have the proper
275        // length and type.
276        let Exponents { e, f } = exponents;
277        match encoded.dtype().as_ptype() {
278            PType::I32 => {
279                vortex_ensure!(exponents.e <= f32::MAX_EXPONENT, "e out of bounds: {e}");
280                vortex_ensure!(exponents.f <= f32::MAX_EXPONENT, "f out of bounds: {f}");
281                if let Some(patches) = patches {
282                    Self::validate_patches::<f32>(patches, encoded)?;
283                }
284            }
285            PType::I64 => {
286                vortex_ensure!(e <= f64::MAX_EXPONENT, "e out of bounds: {e}");
287                vortex_ensure!(f <= f64::MAX_EXPONENT, "f out of bounds: {f}");
288
289                if let Some(patches) = patches {
290                    Self::validate_patches::<f64>(patches, encoded)?;
291                }
292            }
293            _ => unreachable!(),
294        }
295
296        // Validate patches
297        if let Some(patches) = patches {
298            vortex_ensure_eq!(patches.array_len(), encoded.len());
299
300            // Verify that the patches DType are of the proper DType.
301        }
302
303        Ok(())
304    }
305
306    fn logical_dtype(encoded: &ArrayRef) -> VortexResult<DType> {
307        match encoded.dtype() {
308            DType::Primitive(PType::I32, nullability) => {
309                Ok(DType::Primitive(PType::F32, *nullability))
310            }
311            DType::Primitive(PType::I64, nullability) => {
312                Ok(DType::Primitive(PType::F64, *nullability))
313            }
314            _ => vortex_bail!("ALP encoded ints have invalid DType {}", encoded.dtype(),),
315        }
316    }
317
318    /// Validate that any patches provided are valid for the ALPArray.
319    fn validate_patches<T: ALPFloat + NativePType>(
320        patches: &Patches,
321        encoded: &ArrayRef,
322    ) -> VortexResult<()> {
323        vortex_ensure_eq!(patches.array_len(), encoded.len());
324
325        let expected_type = DType::Primitive(T::PTYPE, encoded.dtype().nullability());
326        vortex_ensure_eq!(patches.dtype(), &expected_type);
327
328        Ok(())
329    }
330}
331
332impl ALPData {
333    /// Build a new `ALPArray` from components, panicking on validation failure.
334    ///
335    /// See [`ALP::try_new`] for reference on preconditions that must pass before
336    /// calling this method.
337    pub fn new(exponents: Exponents, patches: Option<Patches>) -> Self {
338        Self {
339            patches_data: patches.as_ref().map(PatchesData::from_patches),
340            exponents,
341        }
342    }
343
344    /// Build a new `ALPArray` from components:
345    ///
346    /// * `encoded` contains the ALP-encoded ints. Any null values are replaced with placeholders
347    /// * `exponents` are the ALP exponents, valid range depends on the data type
348    /// * `patches` are any patch values that don't cleanly encode using the ALP conversion function
349    ///
350    /// Build a new `ALPArray` from components without validation.
351    ///
352    /// See [`ALP::try_new`] for information about the preconditions that should be checked
353    /// **before** calling this method.
354    pub(crate) unsafe fn new_unchecked(exponents: Exponents, patches: Option<Patches>) -> Self {
355        Self::new(exponents, patches)
356    }
357}
358
359/// Constructors for [`ALPArray`].
360impl ALP {
361    pub fn new(encoded: ArrayRef, exponents: Exponents, patches: Option<Patches>) -> ALPArray {
362        let dtype = ALPData::logical_dtype(&encoded).vortex_expect("ALP encoded dtype");
363        let len = encoded.len();
364        let slots = ALPData::make_slots(&encoded, patches.as_ref());
365        unsafe {
366            Array::from_parts_unchecked(ArrayParts::new(
367                ALP,
368                dtype,
369                len,
370                ALPData::new(exponents, patches),
371                slots,
372            ))
373        }
374    }
375
376    pub fn try_new(
377        encoded: ArrayRef,
378        exponents: Exponents,
379        patches: Option<Patches>,
380    ) -> VortexResult<ALPArray> {
381        let dtype = ALPData::logical_dtype(&encoded)?;
382        let len = encoded.len();
383        let slots = ALPData::make_slots(&encoded, patches.as_ref());
384        let data = ALPData::new(exponents, patches);
385        Array::try_from_parts(ArrayParts::new(ALP, dtype, len, data, slots))
386    }
387
388    /// # Safety
389    /// See [`ALP::try_new`] for preconditions.
390    pub unsafe fn new_unchecked(
391        encoded: ArrayRef,
392        exponents: Exponents,
393        patches: Option<Patches>,
394    ) -> ALPArray {
395        let dtype = ALPData::logical_dtype(&encoded).vortex_expect("ALP encoded dtype");
396        let len = encoded.len();
397        let slots = ALPData::make_slots(&encoded, patches.as_ref());
398        let data = unsafe { ALPData::new_unchecked(exponents, patches) };
399        unsafe { Array::from_parts_unchecked(ArrayParts::new(ALP, dtype, len, data, slots)) }
400    }
401}
402
403impl ALPData {
404    fn make_slots(encoded: &ArrayRef, patches: Option<&Patches>) -> ArraySlots {
405        let mut slots: ArraySlots = smallvec![Some(encoded.clone())];
406        PatchesData::push_slots(&mut slots, patches);
407        slots
408    }
409
410    #[inline]
411    pub fn exponents(&self) -> Exponents {
412        self.exponents
413    }
414}
415
416pub trait ALPArrayExt: ALPArraySlotsExt {
417    fn exponents(&self) -> Exponents {
418        self.exponents
419    }
420
421    fn patches(&self) -> Option<Patches> {
422        PatchesData::patches_from_slots(
423            self.patches_data.as_ref(),
424            self.as_ref().len(),
425            self.as_ref().slots(),
426            PATCH_SLOTS,
427        )
428    }
429}
430
431fn validate_parts(
432    dtype: &DType,
433    len: usize,
434    exponents: Exponents,
435    encoded: &ArrayRef,
436    patches: Option<Patches>,
437) -> VortexResult<()> {
438    let logical_dtype = ALPData::logical_dtype(encoded)?;
439    ALPData::validate_components(encoded, exponents, patches.as_ref())?;
440    vortex_ensure_eq!(encoded.len(), len, "ALP encoded len must match outer len");
441    vortex_ensure_eq!(
442        &logical_dtype,
443        dtype,
444        "ALP dtype does not match encoded logical dtype"
445    );
446    Ok(())
447}
448
449impl<T: TypedArrayRef<ALP>> ALPArrayExt for T {}
450
451pub trait ALPArrayOwnedExt {
452    fn into_parts(self) -> (ArrayRef, Exponents, Option<Patches>);
453}
454
455impl ALPArrayOwnedExt for Array<ALP> {
456    #[inline]
457    fn into_parts(self) -> (ArrayRef, Exponents, Option<Patches>) {
458        let patches = self.patches();
459        let exponents = self.exponents();
460        let encoded = self.encoded().clone();
461        (encoded, exponents, patches)
462    }
463}
464
465impl ValidityChild<ALP> for ALP {
466    fn validity_child(array: ArrayView<'_, ALP>) -> ArrayRef {
467        array.encoded().clone()
468    }
469}
470
471#[cfg(test)]
472mod tests {
473    use std::f64::consts::PI;
474    use std::sync::LazyLock;
475
476    use rstest::rstest;
477    use vortex_array::Canonical;
478    use vortex_array::IntoArray;
479    use vortex_array::VortexSessionExecute;
480    use vortex_array::arrays::PrimitiveArray;
481    use vortex_array::assert_arrays_eq;
482    use vortex_error::VortexExpect;
483    use vortex_session::VortexSession;
484
485    use super::*;
486    use crate::alp_encode;
487    use crate::decompress_into_array;
488
489    static SESSION: LazyLock<VortexSession> = LazyLock::new(|| {
490        let session = vortex_array::array_session();
491        crate::initialize(&session);
492        session
493    });
494
495    #[rstest]
496    #[case(0)]
497    #[case(1)]
498    #[case(100)]
499    #[case(1023)]
500    #[case(1024)]
501    #[case(1025)]
502    #[case(2047)]
503    #[case(2048)]
504    #[case(2049)]
505    fn test_execute_f32(#[case] size: usize) {
506        let mut ctx = SESSION.create_execution_ctx();
507        let values = PrimitiveArray::from_iter((0..size).map(|i| i as f32));
508        let encoded = alp_encode(values.as_view(), None, &mut ctx).unwrap();
509
510        let result_canonical = {
511            encoded
512                .clone()
513                .into_array()
514                .execute::<Canonical>(&mut ctx)
515                .unwrap()
516        };
517        // Compare against the traditional array-based decompress path
518        let expected = decompress_into_array(encoded, &mut SESSION.create_execution_ctx()).unwrap();
519
520        assert_arrays_eq!(result_canonical.into_array(), expected, &mut ctx);
521    }
522
523    #[rstest]
524    #[case(0)]
525    #[case(1)]
526    #[case(100)]
527    #[case(1023)]
528    #[case(1024)]
529    #[case(1025)]
530    #[case(2047)]
531    #[case(2048)]
532    #[case(2049)]
533    fn test_execute_f64(#[case] size: usize) {
534        let values = PrimitiveArray::from_iter((0..size).map(|i| i as f64));
535        let encoded =
536            alp_encode(values.as_view(), None, &mut SESSION.create_execution_ctx()).unwrap();
537
538        let mut ctx = SESSION.create_execution_ctx();
539        let result_canonical = encoded
540            .clone()
541            .into_array()
542            .execute::<Canonical>(&mut ctx)
543            .unwrap();
544        // Compare against the traditional array-based decompress path
545        let expected = decompress_into_array(encoded, &mut SESSION.create_execution_ctx()).unwrap();
546
547        assert_arrays_eq!(result_canonical.into_array(), expected, &mut ctx);
548    }
549
550    #[rstest]
551    #[case(100)]
552    #[case(1023)]
553    #[case(1024)]
554    #[case(1025)]
555    #[case(2047)]
556    #[case(2048)]
557    #[case(2049)]
558    fn test_execute_with_patches(#[case] size: usize) {
559        let values: Vec<f64> = (0..size)
560            .map(|i| match i % 4 {
561                0..=2 => 1.0,
562                _ => PI,
563            })
564            .collect();
565
566        let array = PrimitiveArray::from_iter(values);
567        let encoded =
568            alp_encode(array.as_view(), None, &mut SESSION.create_execution_ctx()).unwrap();
569        assert!(encoded.patches().unwrap().array_len() > 0);
570
571        let mut ctx = SESSION.create_execution_ctx();
572        let result_canonical = encoded
573            .clone()
574            .into_array()
575            .execute::<Canonical>(&mut ctx)
576            .unwrap();
577        // Compare against the traditional array-based decompress path
578        let expected = decompress_into_array(encoded, &mut SESSION.create_execution_ctx()).unwrap();
579
580        assert_arrays_eq!(result_canonical.into_array(), expected, &mut ctx);
581    }
582
583    #[rstest]
584    #[case(0)]
585    #[case(1)]
586    #[case(100)]
587    #[case(1023)]
588    #[case(1024)]
589    #[case(1025)]
590    #[case(2047)]
591    #[case(2048)]
592    #[case(2049)]
593    fn test_execute_with_validity(#[case] size: usize) {
594        let values: Vec<Option<f32>> = (0..size)
595            .map(|i| if i % 2 == 1 { None } else { Some(1.0) })
596            .collect();
597
598        let array = PrimitiveArray::from_option_iter(values);
599        let encoded =
600            alp_encode(array.as_view(), None, &mut SESSION.create_execution_ctx()).unwrap();
601
602        let mut ctx = SESSION.create_execution_ctx();
603        let result_canonical = encoded
604            .clone()
605            .into_array()
606            .execute::<Canonical>(&mut ctx)
607            .unwrap();
608        // Compare against the traditional array-based decompress path
609        let expected = decompress_into_array(encoded, &mut SESSION.create_execution_ctx()).unwrap();
610
611        assert_arrays_eq!(result_canonical.into_array(), expected, &mut ctx);
612    }
613
614    #[rstest]
615    #[case(100)]
616    #[case(1023)]
617    #[case(1024)]
618    #[case(1025)]
619    #[case(2047)]
620    #[case(2048)]
621    #[case(2049)]
622    fn test_execute_with_patches_and_validity(#[case] size: usize) {
623        let values: Vec<Option<f64>> = (0..size)
624            .map(|idx| match idx % 3 {
625                0 => Some(1.0),
626                1 => None,
627                _ => Some(PI),
628            })
629            .collect();
630
631        let array = PrimitiveArray::from_option_iter(values);
632        let encoded =
633            alp_encode(array.as_view(), None, &mut SESSION.create_execution_ctx()).unwrap();
634        assert!(encoded.patches().unwrap().array_len() > 0);
635
636        let mut ctx = SESSION.create_execution_ctx();
637        let result_canonical = encoded
638            .clone()
639            .into_array()
640            .execute::<Canonical>(&mut ctx)
641            .unwrap();
642        // Compare against the traditional array-based decompress path
643        let expected = decompress_into_array(encoded, &mut SESSION.create_execution_ctx()).unwrap();
644
645        assert_arrays_eq!(result_canonical.into_array(), expected, &mut ctx);
646    }
647
648    #[rstest]
649    #[case(500, 100)]
650    #[case(1000, 200)]
651    #[case(2048, 512)]
652    fn test_execute_sliced_vector(#[case] size: usize, #[case] slice_start: usize) {
653        let values: Vec<Option<f64>> = (0..size)
654            .map(|i| {
655                if i % 5 == 0 {
656                    None
657                } else if i % 4 == 3 {
658                    Some(PI)
659                } else {
660                    Some(1.0)
661                }
662            })
663            .collect();
664
665        let mut ctx = SESSION.create_execution_ctx();
666        let array = PrimitiveArray::from_option_iter(values.clone());
667        let encoded = alp_encode(array.as_view(), None, &mut ctx).unwrap();
668
669        let slice_end = size - slice_start;
670        let slice_len = slice_end - slice_start;
671        let sliced_encoded = encoded.slice(slice_start..slice_end).unwrap();
672
673        let result_canonical = sliced_encoded.execute::<Canonical>(&mut ctx).unwrap();
674        let result_primitive = result_canonical.into_primitive();
675
676        for idx in 0..slice_len {
677            let expected_value = values[slice_start + idx];
678
679            let result_valid = result_primitive
680                .validity()
681                .vortex_expect("result validity should be derivable")
682                .execute_is_valid(idx, &mut ctx)
683                .unwrap();
684            assert_eq!(
685                result_valid,
686                expected_value.is_some(),
687                "Validity mismatch at idx={idx}",
688            );
689
690            if let Some(expected_val) = expected_value {
691                let result_val = result_primitive.as_slice::<f64>()[idx];
692                assert_eq!(result_val, expected_val, "Value mismatch at idx={idx}",);
693            }
694        }
695    }
696
697    #[rstest]
698    #[case(500, 100)]
699    #[case(1000, 200)]
700    #[case(2048, 512)]
701    fn test_sliced_to_primitive(#[case] size: usize, #[case] slice_start: usize) {
702        let mut ctx = SESSION.create_execution_ctx();
703        let values: Vec<Option<f64>> = (0..size)
704            .map(|i| {
705                if i % 5 == 0 {
706                    None
707                } else if i % 4 == 3 {
708                    Some(PI)
709                } else {
710                    Some(1.0)
711                }
712            })
713            .collect();
714
715        let array = PrimitiveArray::from_option_iter(values.clone());
716        let encoded = alp_encode(array.as_view(), None, &mut ctx).unwrap();
717
718        let slice_end = size - slice_start;
719        let slice_len = slice_end - slice_start;
720        let sliced_encoded = encoded.slice(slice_start..slice_end).unwrap();
721
722        let result_primitive = sliced_encoded.execute::<PrimitiveArray>(&mut ctx).unwrap();
723
724        for idx in 0..slice_len {
725            let expected_value = values[slice_start + idx];
726
727            let result_valid = result_primitive
728                .as_ref()
729                .validity()
730                .unwrap()
731                .execute_mask(result_primitive.as_ref().len(), &mut ctx)
732                .unwrap()
733                .value(idx);
734            assert_eq!(
735                result_valid,
736                expected_value.is_some(),
737                "Validity mismatch at idx={idx}",
738            );
739
740            if let Some(expected_val) = expected_value {
741                let buf = result_primitive.to_buffer::<f64>();
742                let result_val = buf.as_slice()[idx];
743                assert_eq!(result_val, expected_val, "Value mismatch at idx={idx}",);
744            }
745        }
746    }
747
748    /// Regression test for issue #5948: execute_decompress drops patches when chunk_offsets is
749    /// None.
750    ///
751    /// When patches exist but do NOT have chunk_offsets, the execute path incorrectly passes
752    /// `None` to `decompress_unchunked_core` instead of the actual patches.
753    ///
754    /// This can happen after file IO serialization/deserialization where chunk_offsets may not
755    /// be preserved, or when building ALPArrays manually without chunk_offsets.
756    #[test]
757    fn test_execute_decompress_with_patches_no_chunk_offsets_regression_5948() {
758        // Create an array with values that will produce patches. PI doesn't encode cleanly.
759        let values: Vec<f64> = vec![1.0, 2.0, PI, 4.0, 5.0];
760        let original = PrimitiveArray::from_iter(values);
761
762        // First encode normally to get a properly formed ALPArray with patches.
763        let normally_encoded = alp_encode(
764            original.as_view(),
765            None,
766            &mut SESSION.create_execution_ctx(),
767        )
768        .unwrap();
769        assert!(
770            normally_encoded.patches().is_some(),
771            "Test requires patches to be present"
772        );
773
774        let original_patches = normally_encoded.patches().unwrap();
775        assert!(
776            original_patches.chunk_offsets().is_some(),
777            "Normal encoding should have chunk_offsets"
778        );
779
780        // Rebuild the patches WITHOUT chunk_offsets to simulate deserialized patches.
781        let patches_without_chunk_offsets = Patches::new(
782            original_patches.array_len(),
783            original_patches.offset(),
784            original_patches.indices().clone(),
785            original_patches.values().clone(),
786            None, // NO chunk_offsets - this triggers the bug!
787        )
788        .unwrap();
789
790        // Build a new ALPArray with the same encoded data but patches without chunk_offsets.
791        let alp_without_chunk_offsets = ALP::new(
792            normally_encoded.encoded().clone(),
793            normally_encoded.exponents(),
794            Some(patches_without_chunk_offsets),
795        );
796
797        // The legacy decompress_into_array path should work correctly.
798        let result_legacy = decompress_into_array(
799            alp_without_chunk_offsets.clone(),
800            &mut SESSION.create_execution_ctx(),
801        )
802        .unwrap();
803        let legacy_slice = result_legacy.as_slice::<f64>();
804
805        // Verify the legacy path produces correct values.
806        assert!(
807            (legacy_slice[2] - PI).abs() < 1e-10,
808            "Legacy path should have PI at index 2, got {}",
809            legacy_slice[2]
810        );
811
812        // The execute path has the bug - it drops patches when chunk_offsets is None.
813        let result_execute = {
814            let mut ctx = SESSION.create_execution_ctx();
815            execute_decompress(alp_without_chunk_offsets, &mut ctx).unwrap()
816        };
817        let execute_slice = result_execute.as_slice::<f64>();
818
819        // This assertion FAILS until the bug is fixed because execute_decompress drops patches.
820        assert!(
821            (execute_slice[2] - PI).abs() < 1e-10,
822            "Execute path should have PI at index 2, but got {} (patches were dropped!)",
823            execute_slice[2]
824        );
825    }
826}