vortex-array 0.68.0

Vortex in memory columnar data format
Documentation 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56

// SPDX-License-Identifier: Apache-2.0
// SPDX-FileCopyrightText: Copyright the Vortex contributors

use vortex_error::VortexResult;

use crate::ArrayRef;
use crate::IntoArray;
use crate::array::ArrayView;
use crate::arrays::Filter;
use crate::arrays::ListView;
use crate::arrays::ListViewArray;
use crate::arrays::dict::TakeReduceAdaptor;
use crate::arrays::listview::ListViewArrayExt;
use crate::arrays::slice::SliceReduceAdaptor;
use crate::optimizer::rules::ArrayParentReduceRule;
use crate::optimizer::rules::ParentRuleSet;
use crate::scalar_fn::fns::cast::CastReduceAdaptor;
use crate::scalar_fn::fns::mask::MaskReduceAdaptor;

pub(crate) const PARENT_RULES: ParentRuleSet<ListView> = ParentRuleSet::new(&[
    ParentRuleSet::lift(&ListViewFilterPushDown),
    ParentRuleSet::lift(&CastReduceAdaptor(ListView)),
    ParentRuleSet::lift(&MaskReduceAdaptor(ListView)),
    ParentRuleSet::lift(&SliceReduceAdaptor(ListView)),
    ParentRuleSet::lift(&TakeReduceAdaptor(ListView)),
]);

#[derive(Debug)]
struct ListViewFilterPushDown;

impl ArrayParentReduceRule<ListView> for ListViewFilterPushDown {
    type Parent = Filter;

    fn reduce_parent(
        &self,
        array: ArrayView<'_, ListView>,
        parent: ArrayView<'_, Filter>,
        _child_idx: usize,
    ) -> VortexResult<Option<ArrayRef>> {
        // NOTE(ngates): if the filter is super selective, we maybe ought to consider masking
        //  the elements array too. We can create a new Vortex array that represents the explosion
        //  of the parent mask using the offsets/sizes arrays, and then that will be part of the
        //  filter plan.
        Ok(Some(
            unsafe {
                ListViewArray::new_unchecked(
                    array.elements().clone(),
                    array.offsets().filter(parent.filter_mask().clone())?,
                    array.sizes().filter(parent.filter_mask().clone())?,
                    array.validity()?.filter(parent.filter_mask())?,
                )
            }
            .into_array(),
        ))
    }
}