rypipe_core/
arrow_export.rs1use arrow::array::{ArrayRef, AsArray, BooleanArray};
2use arrow::compute::filter_record_batch;
3use arrow::compute::kernels::cast::cast;
4use arrow::compute::kernels::cmp::{eq, gt, gt_eq, lt, lt_eq, neq};
5use arrow::datatypes::{DataType, Float64Type};
6use arrow::record_batch::RecordBatch;
7
8use crate::plan::{CompareOp, FilterPredicate};
9use crate::Result;
10
11pub fn null_array(datatype: &DataType, len: usize) -> ArrayRef {
13 arrow::array::new_null_array(datatype, len)
14}
15
16pub fn apply_compare_filter(
23 batch: RecordBatch,
24 predicate: &FilterPredicate,
25) -> Result<RecordBatch> {
26 let FilterPredicate::Compare {
27 field_a,
28 op,
29 field_b,
30 } = predicate
31 else {
32 return Ok(batch);
33 };
34
35 let col_a = batch.column_by_name(field_a).ok_or_else(|| {
36 crate::Error::Plan(format!(
37 "compare filter references unknown column {field_a:?}"
38 ))
39 })?;
40 let col_b = batch.column_by_name(field_b).ok_or_else(|| {
41 crate::Error::Plan(format!(
42 "compare filter references unknown column {field_b:?}"
43 ))
44 })?;
45
46 let mask = compare_columns(col_a, col_b, *op)?;
47 Ok(filter_record_batch(&batch, &mask)?)
48}
49
50fn is_numeric(dt: &DataType) -> bool {
51 matches!(
52 dt,
53 DataType::Int8
54 | DataType::Int16
55 | DataType::Int32
56 | DataType::Int64
57 | DataType::UInt8
58 | DataType::UInt16
59 | DataType::UInt32
60 | DataType::UInt64
61 | DataType::Float32
62 | DataType::Float64
63 | DataType::Boolean
64 )
65}
66
67fn compare_columns(col_a: &ArrayRef, col_b: &ArrayRef, op: CompareOp) -> Result<BooleanArray> {
68 let dt_a = col_a.data_type();
69 let dt_b = col_b.data_type();
70
71 let (a, b): (ArrayRef, ArrayRef) = if is_numeric(dt_a) && is_numeric(dt_b) {
75 (
76 cast(col_a, &DataType::Float64)?,
77 cast(col_b, &DataType::Float64)?,
78 )
79 } else {
80 (cast(col_a, &DataType::Utf8)?, cast(col_b, &DataType::Utf8)?)
81 };
82
83 match a.data_type() {
84 DataType::Float64 => {
85 let a = a.as_primitive::<Float64Type>();
86 let b = b.as_primitive::<Float64Type>();
87 Ok(match op {
88 CompareOp::Gt => gt(a, b)?,
89 CompareOp::Lt => lt(a, b)?,
90 CompareOp::Ge => gt_eq(a, b)?,
91 CompareOp::Le => lt_eq(a, b)?,
92 CompareOp::Eq => eq(a, b)?,
93 CompareOp::Ne => neq(a, b)?,
94 })
95 }
96 DataType::Utf8 => {
97 let a = a.as_string::<i32>();
98 let b = b.as_string::<i32>();
99 Ok(match op {
100 CompareOp::Gt => gt(a, b)?,
101 CompareOp::Lt => lt(a, b)?,
102 CompareOp::Ge => gt_eq(a, b)?,
103 CompareOp::Le => lt_eq(a, b)?,
104 CompareOp::Eq => eq(a, b)?,
105 CompareOp::Ne => neq(a, b)?,
106 })
107 }
108 other => Err(crate::Error::Plan(format!(
109 "unsupported common type for compare filter: {other}"
110 ))),
111 }
112}