vortex-array 0.68.0

Vortex in memory columnar data format
Documentation 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55

// SPDX-License-Identifier: Apache-2.0
// SPDX-FileCopyrightText: Copyright the Vortex contributors

use itertools::Itertools;
use vortex_error::VortexResult;

use crate::IntoArray;
use crate::array::ArrayView;
use crate::array::ValidityVTable;
use crate::arrays::Chunked;
use crate::arrays::ChunkedArray;
use crate::arrays::chunked::ChunkedArrayExt;
use crate::dtype::DType;
use crate::dtype::Nullability;
use crate::validity::Validity;

impl ValidityVTable<Chunked> for Chunked {
    fn validity(array: ArrayView<'_, Chunked>) -> VortexResult<Validity> {
        let validities: Vec<Validity> =
            array.chunks().iter().map(|c| c.validity()).try_collect()?;

        match validities.first() {
            // If there are no chunks, return the array's dtype nullability
            None => return Ok(array.dtype().nullability().into()),
            // If all chunks have the same non-array validity, return that validity directly
            // We skip Validity::Array since equality is very expensive.
            Some(first) if !matches!(first, Validity::Array(_)) => {
                let target = std::mem::discriminant(first);
                if validities
                    .iter()
                    .all(|v| std::mem::discriminant(v) == target)
                {
                    return Ok(first.clone());
                }
            }
            _ => {
                // Array validity or mixed validities, proceed to build the validity array
            }
        }

        Ok(Validity::Array(
            unsafe {
                ChunkedArray::new_unchecked(
                    validities
                        .into_iter()
                        .zip(array.iter_chunks())
                        .map(|(v, chunk)| v.to_array(chunk.len()))
                        .collect(),
                    DType::Bool(Nullability::NonNullable),
                )
            }
            .into_array(),
        ))
    }
}