indexlake 0.6.0

An experimental table format with extensible index support.
Documentation 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49

use arrow::array::{Array, ArrayRef, BooleanArray, MutableArrayData, make_array};
use arrow::compute::{SlicesIterator, and_kleene, is_not_null};

use crate::ILResult;

/// Scatter `truthy` array by boolean mask. When the mask evaluates `true`, next
/// values of `truthy` are taken, when the mask evaluates `false` values null
/// values are filled.
///
/// # Arguments
/// * `mask` - Boolean values used to determine where to put the `truthy` values
/// * `truthy` - All values of this array are to scatter according to `mask`
///   into final result.
pub fn scatter(mask: &BooleanArray, truthy: &dyn Array) -> ILResult<ArrayRef> {
    let truthy = truthy.to_data();

    // update the mask so that any null values become false
    // (SlicesIterator doesn't respect nulls)
    let mask = and_kleene(mask, &is_not_null(mask)?)?;

    let mut mutable = MutableArrayData::new(vec![&truthy], true, mask.len());

    // the SlicesIterator slices only the true values. So the gaps left by this
    // iterator we need to fill with falsy values

    // keep track of how much is filled
    let mut filled = 0;
    // keep track of current position we have in truthy array
    let mut true_pos = 0;

    for (start, end) in SlicesIterator::new(&mask) {
        // the gap needs to be filled with nulls
        if start > filled {
            mutable.try_extend_nulls(start - filled)?;
        }
        // fill with truthy values
        let len = end - start;
        mutable.try_extend(0, true_pos, true_pos + len)?;
        true_pos += len;
        filled = end;
    }
    // the remaining part is falsy
    if filled < mask.len() {
        mutable.try_extend_nulls(mask.len() - filled)?;
    }

    let data = mutable.freeze();
    Ok(make_array(data))
}