Skip to main content

vortex_array/array/
erased.rs

1// SPDX-License-Identifier: Apache-2.0
2// SPDX-FileCopyrightText: Copyright the Vortex contributors
3
4use std::any::type_name;
5use std::fmt::Debug;
6use std::fmt::Formatter;
7use std::hash::Hash;
8use std::hash::Hasher;
9use std::ops::Range;
10use std::ptr;
11use std::sync::Arc;
12
13use vortex_buffer::ByteBuffer;
14use vortex_error::VortexExpect;
15use vortex_error::VortexResult;
16use vortex_error::vortex_ensure;
17use vortex_error::vortex_ensure_eq;
18use vortex_error::vortex_err;
19use vortex_error::vortex_panic;
20use vortex_mask::Mask;
21
22use crate::AnyCanonical;
23use crate::Array;
24use crate::ArrayEq;
25use crate::ArrayHash;
26use crate::ArrayView;
27use crate::Canonical;
28use crate::ExecutionCtx;
29use crate::ExecutionResult;
30use crate::IntoArray;
31use crate::VTable;
32use crate::aggregate_fn::fns::sum::sum;
33use crate::array::ArrayData;
34use crate::array::ArrayId;
35use crate::array::ArrayInner;
36use crate::array::ArraySlots;
37use crate::array::DynArrayData;
38use crate::array::probe::ArrayProbe;
39use crate::array::probe::RepeatedArrayProbe;
40use crate::arrays::Constant;
41use crate::arrays::DictArray;
42use crate::arrays::FilterArray;
43use crate::arrays::SliceArray;
44use crate::buffer::BufferHandle;
45use crate::builders::ArrayBuilder;
46use crate::dtype::DType;
47use crate::expr::stats::Precision;
48use crate::expr::stats::Stat;
49use crate::expr::stats::StatsProviderExt;
50use crate::matcher::Matcher;
51use crate::optimizer::ArrayOptimizer;
52use crate::optimizer::OptimizedState;
53use crate::scalar::Scalar;
54use crate::scalar::ScalarValue;
55use crate::stats::StatsSetRef;
56use crate::validity::Validity;
57
58/// A depth-first pre-order iterator over an Array.
59pub struct DepthFirstArrayIterator {
60    stack: Vec<ArrayRef>,
61}
62
63impl Iterator for DepthFirstArrayIterator {
64    type Item = ArrayRef;
65
66    fn next(&mut self) -> Option<Self::Item> {
67        let next = self.stack.pop()?;
68        for child in next.children().into_iter().rev() {
69            self.stack.push(child);
70        }
71        Some(next)
72    }
73}
74
75/// A reference-counted pointer to a type-erased array.
76///
77/// Wraps `Arc<ArrayInner<dyn DynArrayData>>` — a single 16-byte fat pointer.
78/// Metadata (`len`, `dtype`, `encoding_id`) lives in `ArrayInner::meta` and is
79/// accessed as a normal struct field read — no vtable dispatch, no extra allocation.
80#[derive(Clone)]
81pub struct ArrayRef(Arc<ArrayInner<dyn DynArrayData>>);
82
83impl ArrayRef {
84    /// Create from an `Arc<ArrayInner<dyn DynArrayData>>`.
85    pub(crate) fn from_inner<D: DynArrayData>(inner: Arc<ArrayInner<D>>) -> Self {
86        Self(inner)
87    }
88
89    /// Returns a reference to the `dyn DynArrayData` inside the inner.
90    #[allow(clippy::inline_always)]
91    #[inline(always)]
92    pub(crate) fn dyn_array(&self) -> &dyn DynArrayData {
93        &self.0.data
94    }
95
96    /// Returns a mutable reference to the inner if this is the sole owner.
97    ///
98    /// The caller may change the array, so it is no longer known to be optimized.
99    #[allow(clippy::inline_always)]
100    #[inline(always)]
101    pub(crate) fn inner_mut(&mut self) -> Option<&mut ArrayInner<dyn DynArrayData>> {
102        let inner = Arc::get_mut(&mut self.0)?;
103        *inner.optimized.0.get_mut() = false;
104        Some(inner)
105    }
106
107    /// Returns the record of whether the optimizer has already taken this array to its fixpoint.
108    ///
109    /// Only the optimizer can read or set the record.
110    pub(crate) fn optimized_state(&self) -> &OptimizedState {
111        &self.0.optimized
112    }
113
114    /// Returns whether another owner holds this array too.
115    pub(crate) fn is_shared(&self) -> bool {
116        Arc::strong_count(&self.0) > 1
117    }
118
119    /// Returns the Arc::as_ptr().addr() of the underlying array.
120    /// This function is used in a couple of places, and we should migrate them to using array_eq.
121    #[doc(hidden)]
122    pub fn addr(&self) -> usize {
123        Arc::as_ptr(&self.0).addr()
124    }
125
126    /// Downcast the inner to a concrete `ArrayInner<ArrayData<V>>`.
127    ///
128    /// Uses the same raw-pointer technique as `Arc::downcast`.
129    #[allow(dead_code)]
130    pub(crate) fn downcast_inner<V: VTable>(self) -> Result<Arc<ArrayInner<ArrayData<V>>>, Self> {
131        // TODO(joe): can we use encoding id here?
132        if self.0.data.as_any().is::<ArrayData<V>>() {
133            Ok(unsafe { self.downcast_inner_unchecked() })
134        } else {
135            Err(self)
136        }
137    }
138
139    /// Downcast without a runtime type check.
140    ///
141    /// # Safety
142    /// The caller must guarantee the concrete type behind `dyn DynArrayData` is `ArrayData<V>`.
143    #[allow(clippy::inline_always)]
144    #[inline(always)]
145    pub(crate) unsafe fn downcast_inner_unchecked<V: VTable>(
146        self,
147    ) -> Arc<ArrayInner<ArrayData<V>>> {
148        debug_assert!(self.0.data.as_any().is::<ArrayData<V>>());
149        // Recover the original concrete Arc. The fat pointer's data pointer is the
150        // same allocation that was originally `Arc<ArrayInner<ArrayData<V>>>` before
151        // unsized coercion to `Arc<ArrayInner<dyn DynArrayData>>`.
152        let raw = Arc::into_raw(self.0);
153        // # Safety all arrays are constructed in this way and type aliased.
154        unsafe { Arc::from_raw(raw.cast::<ArrayInner<ArrayData<V>>>()) }
155    }
156
157    /// Returns true if the two ArrayRefs point to the same allocation.
158    pub fn ptr_eq(this: &ArrayRef, other: &ArrayRef) -> bool {
159        Arc::ptr_eq(&this.0, &other.0)
160    }
161}
162
163impl Debug for ArrayRef {
164    fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result {
165        f.debug_struct("Array")
166            .field("encoding", &self.0.encoding_id)
167            .field("dtype", &self.0.dtype)
168            .field("len", &self.0.len)
169            .field("data", &self.0.data)
170            .finish()
171    }
172}
173
174impl ArrayHash for ArrayRef {
175    fn array_hash<H: Hasher>(&self, state: &mut H, accuracy: crate::EqMode) {
176        self.0.len.hash(state);
177        self.0.dtype.hash(state);
178        self.0.encoding_id.hash(state);
179        self.0.slots.len().hash(state);
180        for slot in &self.0.slots {
181            slot.array_hash(state, accuracy);
182        }
183        self.0
184            .data
185            .dyn_array_hash(state as &mut dyn Hasher, accuracy);
186    }
187}
188
189impl ArrayEq for ArrayRef {
190    fn array_eq(&self, other: &Self, accuracy: crate::EqMode) -> bool {
191        self.0.len == other.0.len
192            && self.0.dtype == other.0.dtype
193            && self.0.encoding_id == other.0.encoding_id
194            && self.0.slots.len() == other.0.slots.len()
195            && self
196                .0
197                .slots
198                .iter()
199                .zip(other.0.slots.iter())
200                .all(|(slot, other_slot)| slot.array_eq(other_slot, accuracy))
201            && self.0.data.dyn_array_eq(other, accuracy)
202    }
203}
204impl ArrayRef {
205    /// Returns the length of the array.
206    #[inline]
207    pub fn len(&self) -> usize {
208        self.0.len
209    }
210
211    /// Returns whether the array is empty (has zero rows).
212    #[inline]
213    pub fn is_empty(&self) -> bool {
214        self.0.len == 0
215    }
216
217    /// Returns the logical Vortex [`DType`] of the array.
218    #[inline]
219    pub fn dtype(&self) -> &DType {
220        &self.0.dtype
221    }
222
223    /// Returns the encoding ID of the array.
224    #[inline]
225    pub fn encoding_id(&self) -> ArrayId {
226        self.0.encoding_id
227    }
228
229    /// Performs a constant-time slice of the array.
230    pub fn slice(&self, range: Range<usize>) -> VortexResult<ArrayRef> {
231        let len = self.len();
232        let start = range.start;
233        let stop = range.end;
234
235        if start == 0 && stop == len {
236            return Ok(self.clone());
237        }
238
239        vortex_ensure!(start <= len, "OutOfBounds: start {start} > length {}", len);
240        vortex_ensure!(stop <= len, "OutOfBounds: stop {stop} > length {}", len);
241
242        vortex_ensure!(start <= stop, "start ({start}) must be <= stop ({stop})");
243
244        if start == stop {
245            return Ok(Canonical::empty(self.dtype()).into_array());
246        }
247
248        let sliced = SliceArray::try_new(self.clone(), range)?
249            .into_array()
250            .optimize()?;
251
252        // Propagate some stats from the original array to the sliced array.
253        if !sliced.is::<Constant>() {
254            self.statistics().with_iter(|iter| {
255                sliced.statistics().inherit(iter.filter(|(stat, value)| {
256                    matches!(
257                        stat,
258                        Stat::IsConstant | Stat::IsSorted | Stat::IsStrictSorted
259                    ) && value
260                        .as_ref()
261                        .as_exact()
262                        .is_some_and(|v| matches!(v, ScalarValue::Bool(true)))
263                }));
264            });
265        }
266
267        Ok(sliced)
268    }
269
270    /// Wraps the array in a [`FilterArray`] such that it is logically filtered by the given mask.
271    pub fn filter(&self, mask: Mask) -> VortexResult<ArrayRef> {
272        FilterArray::try_new(self.clone(), mask)?
273            .into_array()
274            .optimize()
275    }
276
277    /// Wraps the array in a [`DictArray`] such that it is logically taken by the given indices.
278    pub fn take(&self, indices: ArrayRef) -> VortexResult<ArrayRef> {
279        DictArray::try_new(indices, self.clone())?
280            .into_array()
281            .optimize()
282    }
283
284    /// Execute the array to extract a scalar at the given index.
285    ///
286    /// A one-off read; the same as `self.probe().execute_scalar(index, ctx)`.
287    // TODO(joe): deprecate this in favour of `probe()`.
288    #[inline]
289    pub fn execute_scalar(&self, index: usize, ctx: &mut ExecutionCtx) -> VortexResult<Scalar> {
290        self.probe().execute_scalar(index, ctx)
291    }
292
293    /// A one-off row accessor over this array. It borrows the handle and retains nothing; for
294    /// many reads of the same array use [`Self::repeated_probe`].
295    ///
296    /// ```
297    /// use vortex_session::VortexSession;
298    /// use vortex_array::{IntoArray, VortexSessionExecute};
299    /// use vortex_array::arrays::PrimitiveArray;
300    ///
301    /// let array = PrimitiveArray::from_iter([10i32, 20, 30]).into_array();
302    /// let mut ctx = VortexSession::empty().create_execution_ctx();
303    /// assert_eq!(array.probe().execute_scalar(2, &mut ctx)?, 30i32.into());
304    /// # Ok::<(), vortex_error::VortexError>(())
305    /// ```
306    #[inline]
307    pub fn probe(&self) -> ArrayProbe<'_> {
308        ArrayProbe::Once(self)
309    }
310
311    /// A row accessor that owns a handle to this array and keeps encoding state, its validity
312    /// probe and child probes between reads.
313    ///
314    /// ```
315    /// use vortex_session::VortexSession;
316    /// use vortex_array::{IntoArray, VortexSessionExecute};
317    /// use vortex_array::arrays::PrimitiveArray;
318    ///
319    /// let array = PrimitiveArray::from_iter([10i32, 20, 30]).into_array();
320    /// let mut ctx = VortexSession::empty().create_execution_ctx();
321    /// let mut probe = array.repeated_probe();
322    /// assert_eq!(probe.execute_scalar(2, &mut ctx)?, 30i32.into());
323    /// assert_eq!(probe.execute_scalar(0, &mut ctx)?, 10i32.into());
324    /// # Ok::<(), vortex_error::VortexError>(())
325    /// ```
326    pub fn repeated_probe(&self) -> RepeatedArrayProbe {
327        RepeatedArrayProbe::new(self.clone())
328    }
329
330    /// Returns whether the item at `index` is valid.
331    ///
332    /// A one-off read; the same as `self.probe().execute_is_valid(index, ctx)`.
333    // TODO(joe): deprecate this in favour of `probe/repeated_probe()`.
334    #[inline]
335    pub fn is_valid(&self, index: usize, ctx: &mut ExecutionCtx) -> VortexResult<bool> {
336        self.probe().execute_is_valid(index, ctx)
337    }
338
339    /// Returns whether the item at `index` is invalid.
340    // TODO(joe): deprecate this.
341    #[inline]
342    pub fn is_invalid(&self, index: usize, ctx: &mut ExecutionCtx) -> VortexResult<bool> {
343        Ok(!self.is_valid(index, ctx)?)
344    }
345
346    /// Returns whether all items in the array are valid.
347    pub fn all_valid(&self, ctx: &mut ExecutionCtx) -> VortexResult<bool> {
348        if self.is_empty() {
349            return Ok(true);
350        }
351
352        match self.validity()? {
353            Validity::NonNullable | Validity::AllValid => Ok(true),
354            Validity::AllInvalid => Ok(false),
355            Validity::Array(a) => Ok(a.statistics().compute_min::<bool>(ctx).unwrap_or(false)),
356        }
357    }
358
359    /// Returns whether the array is all invalid.
360    pub fn all_invalid(&self, ctx: &mut ExecutionCtx) -> VortexResult<bool> {
361        if self.is_empty() {
362            return Ok(true);
363        }
364
365        match self.validity()? {
366            Validity::NonNullable | Validity::AllValid => Ok(false),
367            Validity::AllInvalid => Ok(true),
368            Validity::Array(a) => Ok(!a.statistics().compute_max::<bool>(ctx).unwrap_or(true)),
369        }
370    }
371
372    /// Returns the number of valid elements in the array.
373    pub fn valid_count(&self, ctx: &mut ExecutionCtx) -> VortexResult<usize> {
374        let len = self.len();
375        if let Precision::Exact(invalid_count) = self.statistics().get_as::<usize>(Stat::NullCount)
376        {
377            return Ok(len - invalid_count);
378        }
379
380        let count = match self.validity()? {
381            Validity::NonNullable | Validity::AllValid => len,
382            Validity::AllInvalid => 0,
383            Validity::Array(a) => {
384                let array_sum = sum(&a, ctx)?;
385                array_sum
386                    .as_primitive()
387                    .as_::<usize>()
388                    .ok_or_else(|| vortex_err!("sum of validity array is null"))?
389            }
390        };
391        vortex_ensure!(count <= len, "Valid count exceeds array length");
392
393        self.statistics()
394            .set(Stat::NullCount, Precision::exact(len - count));
395
396        Ok(count)
397    }
398
399    /// Returns the number of invalid elements in the array.
400    pub fn invalid_count(&self, ctx: &mut ExecutionCtx) -> VortexResult<usize> {
401        Ok(self.len() - self.valid_count(ctx)?)
402    }
403
404    /// Returns the [`Validity`] of the array.
405    pub fn validity(&self) -> VortexResult<Validity> {
406        self.0.data.validity(self)
407    }
408
409    /// Writes the array into the canonical builder.
410    pub fn append_to_builder(
411        &self,
412        builder: &mut dyn ArrayBuilder,
413        ctx: &mut ExecutionCtx,
414    ) -> VortexResult<()> {
415        self.0.data.append_to_builder(self, builder, ctx)
416    }
417
418    /// Returns the statistics of the array.
419    pub fn statistics(&self) -> StatsSetRef<'_> {
420        self.0.stats.to_ref(self)
421    }
422
423    /// Does the array match the given matcher.
424    #[inline]
425    pub fn is<M: Matcher>(&self) -> bool {
426        M::matches(self)
427    }
428
429    /// Returns the array downcast by the given matcher.
430    #[inline]
431    pub fn as_<M: Matcher>(&self) -> M::Match<'_> {
432        self.as_opt::<M>().vortex_expect("Failed to downcast")
433    }
434
435    /// Returns the array downcast by the given matcher.
436    #[inline]
437    pub fn as_opt<M: Matcher>(&self) -> Option<M::Match<'_>> {
438        M::try_match(self)
439    }
440
441    /// Returns the array downcast to the given `Array<V>` as an owned typed handle.
442    pub fn try_downcast<V: VTable>(self) -> Result<Array<V>, ArrayRef> {
443        Array::<V>::try_from_array_ref(self)
444    }
445
446    /// Returns the array downcast to the given `Array<V>` as an owned typed handle.
447    ///
448    /// # Panics
449    ///
450    /// Panics if the array is not of the given type.
451    pub fn downcast<V: VTable>(self) -> Array<V> {
452        Self::try_downcast(self)
453            .unwrap_or_else(|_| vortex_panic!("Failed to downcast to {}", type_name::<V>()))
454    }
455
456    /// Returns a reference to the typed `ArrayData<V>` if this array matches the given vtable type.
457    pub fn as_typed<V: VTable>(&self) -> Option<ArrayView<'_, V>> {
458        let inner = self.0.data.as_any().downcast_ref::<ArrayData<V>>()?;
459        Some(unsafe { ArrayView::new_unchecked(self, &inner.data) })
460    }
461
462    /// Returns a typed view without a runtime type check.
463    ///
464    /// # Safety
465    /// The caller must guarantee the concrete type behind `dyn DynArrayData` is `ArrayData<V>`.
466    #[inline]
467    pub(crate) unsafe fn as_typed_unchecked<V: VTable>(&self) -> ArrayView<'_, V> {
468        debug_assert!(self.is::<V>());
469        // SAFETY: the caller guarantees the concrete type, so the thin data pointer of the
470        // `dyn DynArrayData` points at an `ArrayData<V>`.
471        let inner = unsafe { &*ptr::from_ref(self.dyn_array()).cast::<ArrayData<V>>() };
472        // SAFETY: `inner` is the typed data stored inside `self`.
473        unsafe { ArrayView::new_unchecked(self, &inner.data) }
474    }
475
476    /// Returns the constant scalar if this is a constant array.
477    pub fn as_constant(&self) -> Option<&Scalar> {
478        self.as_opt::<Constant>().map(|a| a.data().scalar())
479    }
480
481    /// Total size of the array in bytes, including all children and buffers.
482    pub fn nbytes(&self) -> u64 {
483        let mut nbytes = 0;
484        for array in self.depth_first_traversal() {
485            for buffer in array.buffers() {
486                nbytes += buffer.len() as u64;
487            }
488        }
489        nbytes
490    }
491
492    /// Whether the array is of a canonical encoding.
493    pub fn is_canonical(&self) -> bool {
494        self.is::<AnyCanonical>()
495    }
496
497    /// Returns a new array with the slot at `slot_idx` replaced by `replacement`.
498    ///
499    /// This is only valid for physical rewrites: the replacement must have the same logical
500    /// `DType` and `len` as the existing slot.
501    ///
502    /// # Safety
503    ///
504    /// If this returns `Ok`, the caller must guarantee that the replacement slot represents the
505    /// same logical values as the original slot. Only the physical representation may change.
506    /// Existing parent statistics are preserved and must remain valid.
507    ///
508    /// Takes ownership to allow in-place mutation when the refcount is 1.
509    pub unsafe fn with_slot(
510        self,
511        slot_idx: usize,
512        replacement: ArrayRef,
513    ) -> VortexResult<ArrayRef> {
514        let mut slots: ArraySlots = self.slots().iter().cloned().collect();
515        let nslots = slots.len();
516        vortex_ensure!(
517            slot_idx < nslots,
518            "slot index {} out of bounds for array with {} slots",
519            slot_idx,
520            nslots
521        );
522        let existing = slots[slot_idx]
523            .as_ref()
524            .vortex_expect("with_slot cannot replace an absent slot");
525        vortex_ensure_eq!(
526            existing.dtype(),
527            replacement.dtype(),
528            "slot {} dtype changed during physical rewrite",
529            slot_idx
530        );
531        vortex_ensure_eq!(
532            existing.len(),
533            replacement.len(),
534            "slot {} len changed during physical rewrite",
535            slot_idx
536        );
537        slots[slot_idx] = Some(replacement);
538        // SAFETY: upheld by the caller of this unsafe API.
539        unsafe { self.with_slots(slots) }
540    }
541
542    /// Take a slot for executor-owned physical rewrites.
543    ///
544    /// On return the produced parent has the taken slot set to `None`
545    /// callers must put the slot back (typically via [`Self::put_slot_unchecked`]) before the parent is
546    /// returned from the execution loop.
547    ///
548    /// When the `Arc` was shared this allocates a fresh parent.
549    ///
550    /// # Safety
551    /// The caller must put back a slot with the same logical dtype and length before exposing the
552    /// parent array, and must only use this for physical rewrites.
553    pub(crate) unsafe fn take_slot_unchecked(
554        mut self,
555        slot_idx: usize,
556    ) -> VortexResult<(ArrayRef, ArrayRef)> {
557        if let Some(inner) = self.inner_mut() {
558            let child = inner.slots[slot_idx]
559                .take()
560                .vortex_expect("take_slot_unchecked cannot take an absent slot");
561            return Ok((self, child));
562        }
563
564        // Arc is shared: clone the child out and build a fresh parent with slot_idx = None,
565        // bypassing encoding-level validation so the absent slot does not panic `V::validate`.
566        let child = self.slots()[slot_idx]
567            .as_ref()
568            .vortex_expect("take_slot_unchecked cannot take an absent slot")
569            .clone();
570
571        let mut new_slots: ArraySlots = self.slots().iter().cloned().collect();
572        new_slots[slot_idx] = None;
573
574        // SAFETY: ensured by the caller — the None slot is either put back or driven to completion
575        // via the builder path before the parent escapes the executor.
576        let new_parent = unsafe { self.0.data.with_slots_unchecked(&self, new_slots) };
577        Ok((new_parent, child))
578    }
579
580    /// Puts an array into `slot_idx` by either, cloning the inner array if the Arc is not exclusive
581    /// or replacing the slot in this `ArrayRef`.
582    /// This is the mirror of [`Self::take_slot_unchecked`].
583    ///
584    /// # Safety
585    /// The replacement must have the same logical dtype and length as the taken slot, and this
586    /// must only be used for physical rewrites.
587    pub(crate) unsafe fn put_slot_unchecked(
588        mut self,
589        slot_idx: usize,
590        replacement: ArrayRef,
591    ) -> VortexResult<ArrayRef> {
592        if let Some(inner) = self.inner_mut() {
593            inner.slots[slot_idx] = Some(replacement);
594            return Ok(self);
595        }
596
597        let mut slots: ArraySlots = self.slots().iter().cloned().collect();
598        slots[slot_idx] = Some(replacement);
599        self.0.data.with_slots(&self, slots)
600    }
601
602    /// Returns a new array with the provided slots.
603    ///
604    /// This is only valid for physical rewrites: slot count, presence, logical `DType`, and
605    /// logical `len` must remain unchanged.
606    ///
607    /// # Safety
608    ///
609    /// If this returns `Ok`, the caller must guarantee that each replacement slot represents the
610    /// same logical values as the original slot. Only physical representation may change. Existing
611    /// parent statistics are preserved and must remain valid.
612    pub unsafe fn with_slots(self, slots: ArraySlots) -> VortexResult<ArrayRef> {
613        let old_slots = self.slots();
614        vortex_ensure_eq!(
615            old_slots.len(),
616            slots.len(),
617            "slot count changed during physical rewrite"
618        );
619        for (idx, (old_slot, new_slot)) in old_slots.iter().zip(slots.iter()).enumerate() {
620            vortex_ensure_eq!(
621                old_slot.is_some(),
622                new_slot.is_some(),
623                "slot {} presence changed during physical rewrite",
624                idx
625            );
626            if let (Some(old_slot), Some(new_slot)) = (old_slot.as_ref(), new_slot.as_ref()) {
627                vortex_ensure_eq!(
628                    old_slot.dtype(),
629                    new_slot.dtype(),
630                    "slot {} dtype changed during physical rewrite",
631                    idx
632                );
633                vortex_ensure_eq!(
634                    old_slot.len(),
635                    new_slot.len(),
636                    "slot {} len changed during physical rewrite",
637                    idx
638                );
639            }
640        }
641        self.0.data.with_slots(&self, slots)
642    }
643
644    /// Returns a new array with the provided top-level buffer handles.
645    ///
646    /// This is only valid for physical rewrites: buffer count, logical `DType`, logical `len`, and
647    /// child slots must remain unchanged. Encoding-specific validation checks buffer shape,
648    /// alignment, and metadata consistency.
649    ///
650    /// # Safety
651    ///
652    /// If this returns `Ok`, the caller must guarantee that the replacement buffers represent the
653    /// same logical values as the original buffers. Only the buffer handle implementation,
654    /// placement, or backing storage may change. Existing statistics are preserved and must remain
655    /// valid.
656    pub unsafe fn with_buffers(
657        self,
658        buffers: impl IntoIterator<Item = BufferHandle>,
659    ) -> VortexResult<ArrayRef> {
660        let buffers = buffers.into_iter().collect::<Vec<_>>();
661        let nbuffers = self.nbuffers();
662        vortex_ensure_eq!(
663            nbuffers,
664            buffers.len(),
665            "buffer count changed during physical rewrite"
666        );
667        for (idx, (old_buffer, new_buffer)) in self
668            .buffer_handles()
669            .into_iter()
670            .zip(buffers.iter())
671            .enumerate()
672        {
673            vortex_ensure_eq!(
674                old_buffer.len(),
675                new_buffer.len(),
676                "buffer {} length changed during physical rewrite",
677                idx
678            );
679        }
680        self.0.data.with_buffers(&self, buffers)
681    }
682
683    pub fn reduce(&self) -> VortexResult<Option<ArrayRef>> {
684        self.0.data.reduce(self)
685    }
686
687    pub fn reduce_parent(
688        &self,
689        parent: &ArrayRef,
690        child_idx: usize,
691    ) -> VortexResult<Option<ArrayRef>> {
692        self.0.data.reduce_parent(self, parent, child_idx)
693    }
694
695    pub(crate) fn execute_encoding(self, ctx: &mut ExecutionCtx) -> VortexResult<ExecutionResult> {
696        let inner = Arc::as_ptr(&self.0);
697        // SAFETY: the Arc outlives the DynArrayData function call
698        unsafe { (&*inner).data.execute(self, ctx) }
699    }
700
701    /// Execute a single encoding step without applying `Done`-result postconditions.
702    ///
703    /// This is for the iterative executor only. It may operate on suspended executor-private
704    /// arrays whose slots temporarily contain `None`, so the executor itself must interpret
705    /// `Done`, enforce any `len`/`dtype` invariants, and transfer statistics.
706    pub(crate) fn execute_encoding_unchecked(
707        self,
708        ctx: &mut ExecutionCtx,
709    ) -> VortexResult<ExecutionResult> {
710        let inner = Arc::as_ptr(&self.0);
711        // SAFETY: `inner` points at the allocation owned by `self.0`. `self` stays alive for the
712        // duration of the call, so the pointee remains valid. Avoiding an extra `Arc` clone here
713        // preserves uniqueness so execute-time metadata cursors can use `Arc::get_mut`.
714        unsafe { (&*inner).data.execute_unchecked(self, ctx) }
715    }
716
717    // ArrayVisitor delegation methods
718
719    /// Returns an iterator over the children of the array: its non-None slots in order.
720    pub fn children_iter(&self) -> impl Iterator<Item = &ArrayRef> {
721        self.0.slots.iter().filter_map(|s| s.as_ref())
722    }
723
724    /// Returns the children of the array.
725    pub fn children(&self) -> Vec<ArrayRef> {
726        self.children_iter().cloned().collect()
727    }
728
729    /// Returns the number of children of the array.
730    pub fn nchildren(&self) -> usize {
731        self.children_iter().count()
732    }
733
734    /// Returns the nth child of the array without allocating a Vec.
735    ///
736    /// Returns `None` if the index is out of bounds.
737    pub fn nth_child(&self, idx: usize) -> Option<&ArrayRef> {
738        self.children_iter().nth(idx)
739    }
740
741    /// Returns the names of the children of the array: the slot names of the non-None slots
742    /// in order.
743    pub fn children_names(&self) -> Vec<String> {
744        self.0
745            .slots
746            .iter()
747            .enumerate()
748            .filter(|(_, s)| s.is_some())
749            .map(|(slot_idx, _)| self.slot_name(slot_idx))
750            .collect()
751    }
752
753    /// Returns the array's children with their names.
754    pub fn named_children(&self) -> Vec<(String, ArrayRef)> {
755        self.children_names()
756            .into_iter()
757            .zip(self.children_iter().cloned())
758            .collect()
759    }
760
761    /// Returns the data buffers of the array.
762    pub fn buffers(&self) -> Vec<ByteBuffer> {
763        self.0.data.buffers(self)
764    }
765
766    /// Returns the buffer handles of the array.
767    pub fn buffer_handles(&self) -> Vec<BufferHandle> {
768        self.0.data.buffer_handles(self)
769    }
770
771    /// Returns the names of the buffers of the array.
772    pub fn buffer_names(&self) -> Vec<String> {
773        self.0.data.buffer_names(self)
774    }
775
776    /// Returns the array's buffers with their names.
777    pub fn named_buffers(&self) -> Vec<(String, BufferHandle)> {
778        self.0.data.named_buffers(self)
779    }
780
781    /// Returns the number of data buffers of the array.
782    pub fn nbuffers(&self) -> usize {
783        self.0.data.nbuffers(self)
784    }
785
786    /// Returns the slots of the array.
787    pub fn slots(&self) -> &[Option<ArrayRef>] {
788        &self.0.slots
789    }
790
791    /// Returns the name of the slot at the given index.
792    pub fn slot_name(&self, idx: usize) -> String {
793        self.0.data.slot_name(self, idx)
794    }
795
796    /// Formats a human-readable metadata description.
797    pub fn metadata_fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result {
798        self.0.data.metadata_fmt(f)
799    }
800
801    /// Returns whether all buffers are host-resident.
802    pub fn is_host(&self) -> bool {
803        for array in self.depth_first_traversal() {
804            if !array.buffer_handles().iter().all(BufferHandle::is_on_host) {
805                return false;
806            }
807        }
808        true
809    }
810
811    // ArrayVisitorExt delegation methods
812
813    /// Count the number of buffers encoded by self and all child arrays.
814    pub fn nbuffers_recursive(&self) -> usize {
815        self.children()
816            .iter()
817            .map(|c| c.nbuffers_recursive())
818            .sum::<usize>()
819            + self.nbuffers()
820    }
821
822    /// Depth-first traversal of the array and its children.
823    pub fn depth_first_traversal(&self) -> DepthFirstArrayIterator {
824        DepthFirstArrayIterator {
825            stack: vec![self.clone()],
826        }
827    }
828}
829
830impl IntoArray for ArrayRef {
831    #[allow(clippy::inline_always)]
832    #[inline(always)]
833    fn into_array(self) -> ArrayRef {
834        self
835    }
836}
837
838impl<V: VTable> Matcher for V {
839    type Match<'a> = ArrayView<'a, V>;
840
841    #[inline]
842    fn matches(array: &ArrayRef) -> bool {
843        array.0.data.as_any().is::<ArrayData<V>>()
844    }
845
846    #[inline]
847    fn try_match(array: &'_ ArrayRef) -> Option<ArrayView<'_, V>> {
848        let inner = array.0.data.as_any().downcast_ref::<ArrayData<V>>()?;
849        // # Safety checked by `downcast_ref`.
850        Some(unsafe { ArrayView::new_unchecked(array, &inner.data) })
851    }
852}