vortex-array 0.68.0

Vortex in memory columnar data format
Documentation 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60

// SPDX-License-Identifier: Apache-2.0
// SPDX-FileCopyrightText: Copyright the Vortex contributors

use std::iter;

use vortex_error::VortexExpect;

use crate::ToCanonical;
use crate::accessor::ArrayAccessor;
use crate::arrays::VarBinArray;
use crate::arrays::varbin::VarBinArrayExt;
use crate::match_each_integer_ptype;
use crate::validity::Validity;

impl ArrayAccessor<[u8]> for VarBinArray {
    fn with_iterator<F, R>(&self, f: F) -> R
    where
        F: for<'a> FnOnce(&mut dyn Iterator<Item = Option<&'a [u8]>>) -> R,
    {
        let offsets = self.offsets().to_primitive();
        let validity = self
            .validity()
            .vortex_expect("varbin validity should be derivable");

        let bytes = self.bytes();
        let bytes = bytes.as_slice();

        match_each_integer_ptype!(offsets.ptype(), |T| {
            let offsets = offsets.as_slice::<T>();

            #[allow(clippy::cast_possible_truncation)]
            match validity {
                Validity::NonNullable | Validity::AllValid => {
                    let mut iter = offsets
                        .windows(2)
                        .map(|w| Some(&bytes[w[0] as usize..w[1] as usize]));
                    f(&mut iter)
                }
                Validity::AllInvalid => f(&mut iter::repeat_n(None, self.len())),
                Validity::Array(v) => {
                    let validity = v.to_bool().into_bit_buffer();
                    let mut iter = offsets
                        .windows(2)
                        .zip(validity.iter())
                        .map(|(w, valid)| valid.then(|| &bytes[w[0] as usize..w[1] as usize]));
                    f(&mut iter)
                }
            }
        })
    }
}

impl ArrayAccessor<[u8]> for &VarBinArray {
    fn with_iterator<F, R>(&self, f: F) -> R
    where
        F: for<'a> FnOnce(&mut dyn Iterator<Item = Option<&'a [u8]>>) -> R,
    {
        <VarBinArray as ArrayAccessor<[u8]>>::with_iterator(*self, f)
    }
}