Skip to main content

rypipe_core/
arrow_export.rs

1use arrow::array::{ArrayRef, AsArray, BooleanArray};
2use arrow::compute::filter_record_batch;
3use arrow::compute::kernels::boolean::and;
4use arrow::compute::kernels::cast::cast;
5use arrow::compute::kernels::cmp::{eq, gt, gt_eq, lt, lt_eq, neq};
6use arrow::datatypes::{DataType, Float64Type};
7use arrow::record_batch::RecordBatch;
8
9use crate::plan::{CompareOp, FilterPredicate};
10use crate::Result;
11
12/// Build a null array of `datatype` with `len` rows.
13pub fn null_array(datatype: &DataType, len: usize) -> ArrayRef {
14    arrow::array::new_null_array(datatype, len)
15}
16
17/// Apply a column-to-column `Compare` filter to an assembled `RecordBatch`.
18///
19/// This is evaluated entirely in Rust using `arrow::compute` comparison
20/// kernels and `filter_record_batch`.  No Python callable is invoked.
21///
22/// Per-row evaluation during parse (`FilterPredicate::check`) is
23/// authoritative: every executor path already applies the full predicate tree
24/// while assembling rows. This post-assembly pass therefore only re-applies
25/// trees made solely of `Compare` and `And` nodes (an idempotent recheck);
26/// any tree involving `Or`, `Not`, `Equal`, or `NotEqual` is returned
27/// unchanged so valid rows can never be dropped twice under different null
28/// semantics.
29pub fn apply_compare_filter(
30    batch: RecordBatch,
31    predicate: &FilterPredicate,
32) -> Result<RecordBatch> {
33    if !is_pure_compare_tree(predicate) {
34        return Ok(batch);
35    }
36
37    let mask = compare_mask(&batch, predicate)?;
38    let mask = match mask {
39        Some(m) => m,
40        None => return Ok(batch),
41    };
42    Ok(filter_record_batch(&batch, &mask)?)
43}
44
45/// True when the predicate tree contains only `Compare` and `And` nodes.
46fn is_pure_compare_tree(predicate: &FilterPredicate) -> bool {
47    match predicate {
48        FilterPredicate::Compare { .. } => true,
49        FilterPredicate::And(a, b) => is_pure_compare_tree(a) && is_pure_compare_tree(b),
50        _ => false,
51    }
52}
53
54/// Recursively evaluate a pure `Compare`/`And` tree into one boolean mask.
55/// Returns `None` for an empty conjunction (all rows pass).
56fn compare_mask(batch: &RecordBatch, predicate: &FilterPredicate) -> Result<Option<BooleanArray>> {
57    match predicate {
58        FilterPredicate::Compare {
59            field_a,
60            op,
61            field_b,
62        } => {
63            let col_a = batch.column_by_name(field_a).ok_or_else(|| {
64                crate::Error::Plan(format!(
65                    "compare filter references unknown column {field_a:?}"
66                ))
67            })?;
68            let col_b = batch.column_by_name(field_b).ok_or_else(|| {
69                crate::Error::Plan(format!(
70                    "compare filter references unknown column {field_b:?}"
71                ))
72            })?;
73            Ok(Some(compare_columns(col_a, col_b, *op)?))
74        }
75        FilterPredicate::And(a, b) => {
76            let ma = compare_mask(batch, a)?;
77            let mb = compare_mask(batch, b)?;
78            Ok(match (ma, mb) {
79                (Some(x), Some(y)) => Some(and(&x, &y)?),
80                (Some(x), None) | (None, Some(x)) => Some(x),
81                (None, None) => None,
82            })
83        }
84        _ => Ok(None),
85    }
86}
87
88fn is_numeric(dt: &DataType) -> bool {
89    matches!(
90        dt,
91        DataType::Int8
92            | DataType::Int16
93            | DataType::Int32
94            | DataType::Int64
95            | DataType::UInt8
96            | DataType::UInt16
97            | DataType::UInt32
98            | DataType::UInt64
99            | DataType::Float32
100            | DataType::Float64
101            | DataType::Boolean
102    )
103}
104
105fn compare_columns(col_a: &ArrayRef, col_b: &ArrayRef, op: CompareOp) -> Result<BooleanArray> {
106    let dt_a = col_a.data_type();
107    let dt_b = col_b.data_type();
108
109    // Choose a common concrete type so the `Datum` trait is satisfied by a
110    // concrete array reference.  Numeric columns are compared as f64;
111    // everything else is compared as UTF-8.
112    let (a, b): (ArrayRef, ArrayRef) = if is_numeric(dt_a) && is_numeric(dt_b) {
113        (
114            cast(col_a, &DataType::Float64)?,
115            cast(col_b, &DataType::Float64)?,
116        )
117    } else {
118        (cast(col_a, &DataType::Utf8)?, cast(col_b, &DataType::Utf8)?)
119    };
120
121    match a.data_type() {
122        DataType::Float64 => {
123            let a = a.as_primitive::<Float64Type>();
124            let b = b.as_primitive::<Float64Type>();
125            Ok(match op {
126                CompareOp::Gt => gt(a, b)?,
127                CompareOp::Lt => lt(a, b)?,
128                CompareOp::Ge => gt_eq(a, b)?,
129                CompareOp::Le => lt_eq(a, b)?,
130                CompareOp::Eq => eq(a, b)?,
131                CompareOp::Ne => neq(a, b)?,
132            })
133        }
134        DataType::Utf8 => {
135            let a = a.as_string::<i32>();
136            let b = b.as_string::<i32>();
137            Ok(match op {
138                CompareOp::Gt => gt(a, b)?,
139                CompareOp::Lt => lt(a, b)?,
140                CompareOp::Ge => gt_eq(a, b)?,
141                CompareOp::Le => lt_eq(a, b)?,
142                CompareOp::Eq => eq(a, b)?,
143                CompareOp::Ne => neq(a, b)?,
144            })
145        }
146        other => Err(crate::Error::Plan(format!(
147            "unsupported common type for compare filter: {other}"
148        ))),
149    }
150}