Skip to main content

rypipe_core/
arrow_export.rs

1use arrow::array::{ArrayRef, AsArray, BooleanArray};
2use arrow::compute::filter_record_batch;
3use arrow::compute::kernels::cast::cast;
4use arrow::compute::kernels::cmp::{eq, gt, gt_eq, lt, lt_eq, neq};
5use arrow::datatypes::{DataType, Float64Type};
6use arrow::record_batch::RecordBatch;
7
8use crate::plan::{CompareOp, FilterPredicate};
9use crate::Result;
10
11/// Build a null array of `datatype` with `len` rows.
12pub fn null_array(datatype: &DataType, len: usize) -> ArrayRef {
13    arrow::array::new_null_array(datatype, len)
14}
15
16/// Apply a column-to-column `Compare` filter to an assembled `RecordBatch`.
17///
18/// This is evaluated entirely in Rust using `arrow::compute` comparison
19/// kernels and `filter_record_batch`.  No Python callable is invoked.
20/// Per-row `Equal`/`NotEqual` filters are already applied during row assembly
21/// and are a no-op here.
22pub fn apply_compare_filter(
23    batch: RecordBatch,
24    predicate: &FilterPredicate,
25) -> Result<RecordBatch> {
26    let FilterPredicate::Compare {
27        field_a,
28        op,
29        field_b,
30    } = predicate
31    else {
32        return Ok(batch);
33    };
34
35    let col_a = batch.column_by_name(field_a).ok_or_else(|| {
36        crate::Error::Plan(format!(
37            "compare filter references unknown column {field_a:?}"
38        ))
39    })?;
40    let col_b = batch.column_by_name(field_b).ok_or_else(|| {
41        crate::Error::Plan(format!(
42            "compare filter references unknown column {field_b:?}"
43        ))
44    })?;
45
46    let mask = compare_columns(col_a, col_b, *op)?;
47    Ok(filter_record_batch(&batch, &mask)?)
48}
49
50fn is_numeric(dt: &DataType) -> bool {
51    matches!(
52        dt,
53        DataType::Int8
54            | DataType::Int16
55            | DataType::Int32
56            | DataType::Int64
57            | DataType::UInt8
58            | DataType::UInt16
59            | DataType::UInt32
60            | DataType::UInt64
61            | DataType::Float32
62            | DataType::Float64
63            | DataType::Boolean
64    )
65}
66
67fn compare_columns(col_a: &ArrayRef, col_b: &ArrayRef, op: CompareOp) -> Result<BooleanArray> {
68    let dt_a = col_a.data_type();
69    let dt_b = col_b.data_type();
70
71    // Choose a common concrete type so the `Datum` trait is satisfied by a
72    // concrete array reference.  Numeric columns are compared as f64;
73    // everything else is compared as UTF-8.
74    let (a, b): (ArrayRef, ArrayRef) = if is_numeric(dt_a) && is_numeric(dt_b) {
75        (
76            cast(col_a, &DataType::Float64)?,
77            cast(col_b, &DataType::Float64)?,
78        )
79    } else {
80        (cast(col_a, &DataType::Utf8)?, cast(col_b, &DataType::Utf8)?)
81    };
82
83    match a.data_type() {
84        DataType::Float64 => {
85            let a = a.as_primitive::<Float64Type>();
86            let b = b.as_primitive::<Float64Type>();
87            Ok(match op {
88                CompareOp::Gt => gt(a, b)?,
89                CompareOp::Lt => lt(a, b)?,
90                CompareOp::Ge => gt_eq(a, b)?,
91                CompareOp::Le => lt_eq(a, b)?,
92                CompareOp::Eq => eq(a, b)?,
93                CompareOp::Ne => neq(a, b)?,
94            })
95        }
96        DataType::Utf8 => {
97            let a = a.as_string::<i32>();
98            let b = b.as_string::<i32>();
99            Ok(match op {
100                CompareOp::Gt => gt(a, b)?,
101                CompareOp::Lt => lt(a, b)?,
102                CompareOp::Ge => gt_eq(a, b)?,
103                CompareOp::Le => lt_eq(a, b)?,
104                CompareOp::Eq => eq(a, b)?,
105                CompareOp::Ne => neq(a, b)?,
106            })
107        }
108        other => Err(crate::Error::Plan(format!(
109            "unsupported common type for compare filter: {other}"
110        ))),
111    }
112}