vortex-array 0.62.0

Vortex in memory columnar data format
Documentation 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72

// SPDX-License-Identifier: Apache-2.0
// SPDX-FileCopyrightText: Copyright the Vortex contributors

use vortex_error::VortexResult;
use vortex_error::vortex_err;

use crate::arrays::ChunkedArray;
use crate::arrays::ChunkedVTable;
use crate::compute::MinMaxKernel;
use crate::compute::MinMaxKernelAdapter;
use crate::compute::MinMaxResult;
use crate::compute::min_max;
use crate::partial_ord::partial_max;
use crate::partial_ord::partial_min;
use crate::register_kernel;
use crate::scalar::Scalar;

impl MinMaxKernel for ChunkedVTable {
    fn min_max(&self, array: &ChunkedArray) -> VortexResult<Option<MinMaxResult>> {
        let mut min_max_all_null = true;
        let res = array
            .array_iterator()
            .map(|chunk| {
                let chunk = chunk?;
                if let Some(min_max) = min_max(&chunk)? {
                    min_max_all_null = false;
                    Ok((Some(min_max.min), Some(min_max.max)))
                } else {
                    Ok((None, None))
                }
            })
            .collect::<VortexResult<Vec<_>>>()?;

        // There are no chunks that have min/max stats, so return early
        if min_max_all_null {
            return Ok(None);
        }

        let (min_values, max_values): (Vec<Option<Scalar>>, Vec<Option<Scalar>>) =
            res.into_iter().unzip();

        Ok(Some(MinMaxResult {
            min: min_values
                .into_iter()
                .flatten()
                // This is None iff all the values `None` (refuted above) or partial_min returns None
                .fold(None, |acc, x| {
                    if let Some(acc) = acc {
                        partial_min(x, acc)
                    } else {
                        Some(x)
                    }
                })
                .ok_or_else(|| {
                    vortex_err!("Incomparable scalars (from partial_min), this is likely a bug",)
                })?,
            max: max_values
                .into_iter()
                .flatten()
                .fold(None, |acc, x| {
                    if let Some(acc) = acc {
                        partial_max(x, acc)
                    } else {
                        Some(x)
                    }
                })
                .ok_or_else(|| vortex_err!("Incomparable scalars, this is likely a bug"))?,
        }))
    }
}

register_kernel!(MinMaxKernelAdapter(ChunkedVTable).lift());