aisle 0.2.0

Metadata-driven Parquet pruning for Rust: Skip irrelevant data before reading
Documentation 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62

use datafusion_common::ScalarValue;

use super::{cmp, context::RowGroupContext, page::PagePruning, stats};
use crate::ir::{CmpOp, TriState};

pub(super) fn eval_in_list(
    column: &str,
    values: &[ScalarValue],
    ctx: &RowGroupContext<'_>,
) -> TriState {
    let (min, max, null_count, row_count) = match stats::stats_for_column(column, ctx) {
        Some(stats) => stats,
        None => return TriState::Unknown,
    };
    let mut any_true = false;
    let mut all_false = true;
    for value in values {
        let tri = cmp::eval_cmp_stats(
            CmpOp::Eq,
            value,
            Some(&min),
            Some(&max),
            null_count,
            row_count,
        );
        match tri {
            TriState::True => {
                any_true = true;
                all_false = false;
                break;
            }
            TriState::Unknown => {
                all_false = false;
            }
            TriState::False => {}
        }
    }
    if any_true {
        TriState::True
    } else if all_false {
        TriState::False
    } else {
        TriState::Unknown
    }
}

pub(super) fn page_selection_for_in_list(
    column: &str,
    values: &[ScalarValue],
    ctx: &RowGroupContext<'_>,
) -> Option<PagePruning> {
    // IN as OR of equalities - all-or-nothing
    let mut selection: Option<PagePruning> = None;
    for value in values {
        let sel = cmp::page_selection_for_cmp(column, CmpOp::Eq, value, ctx)?; // Fail fast
        selection = match selection {
            None => Some(sel),
            Some(existing) => Some(existing.union(sel)),
        };
    }
    selection
}