vortex-array 0.23.0

Vortex in memory columnar data format
Documentation 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51

use std::iter;

use vortex_dtype::match_each_integer_ptype;
use vortex_error::VortexResult;

use crate::accessor::ArrayAccessor;
use crate::array::varbin::VarBinArray;
use crate::validity::Validity;
use crate::variants::PrimitiveArrayTrait;
use crate::IntoArrayVariant;

impl ArrayAccessor<[u8]> for VarBinArray {
    fn with_iterator<F, R>(&self, f: F) -> VortexResult<R>
    where
        F: for<'a> FnOnce(&mut (dyn Iterator<Item = Option<&'a [u8]>>)) -> R,
    {
        let offsets = self.offsets().into_primitive()?;
        let validity = self.validity();

        let bytes = self.bytes();
        let bytes = bytes.as_slice();

        match_each_integer_ptype!(offsets.ptype(), |$T| {
            let offsets = offsets.as_slice::<$T>();

            match validity {
                Validity::NonNullable | Validity::AllValid => {
                    let mut iter = offsets
                        .windows(2)
                        .map(|w| Some(&bytes[w[0] as usize..w[1] as usize]));
                    Ok(f(&mut iter))
                }
                Validity::AllInvalid => Ok(f(&mut iter::repeat_n(None, self.len()))),
                Validity::Array(v) => {
                    let validity_buf = v.into_bool()?.boolean_buffer();
                    let mut iter = offsets
                        .windows(2)
                        .zip(validity_buf.iter())
                        .map(|(w, valid)| {
                            if valid {
                                Some(&bytes[w[0] as usize..w[1] as usize])
                            } else {
                                None
                            }
                        });
                    Ok(f(&mut iter))
                }
            }
        })
    }
}