vortex-array 0.54.0

Vortex in memory columnar data format
Documentation 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49

// SPDX-License-Identifier: Apache-2.0
// SPDX-FileCopyrightText: Copyright the Vortex contributors

use std::iter;

use vortex_dtype::match_each_integer_ptype;
use vortex_error::VortexResult;

use crate::ToCanonical;
use crate::accessor::ArrayAccessor;
use crate::arrays::varbin::VarBinArray;
use crate::validity::Validity;
use crate::vtable::ValidityHelper;

impl ArrayAccessor<[u8]> for VarBinArray {
    fn with_iterator<F, R>(&self, f: F) -> VortexResult<R>
    where
        F: for<'a> FnOnce(&mut dyn Iterator<Item = Option<&'a [u8]>>) -> R,
    {
        let offsets = self.offsets().to_primitive();
        let validity = self.validity();

        let bytes = self.bytes();
        let bytes = bytes.as_slice();

        match_each_integer_ptype!(offsets.ptype(), |T| {
            let offsets = offsets.as_slice::<T>();

            #[allow(clippy::cast_possible_truncation)]
            match validity {
                Validity::NonNullable | Validity::AllValid => {
                    let mut iter = offsets
                        .windows(2)
                        .map(|w| Some(&bytes[w[0] as usize..w[1] as usize]));
                    Ok(f(&mut iter))
                }
                Validity::AllInvalid => Ok(f(&mut iter::repeat_n(None, self.len()))),
                Validity::Array(v) => {
                    let validity = v.to_bool();
                    let mut iter = offsets
                        .windows(2)
                        .zip(validity.boolean_buffer())
                        .map(|(w, valid)| valid.then(|| &bytes[w[0] as usize..w[1] as usize]));
                    Ok(f(&mut iter))
                }
            }
        })
    }
}