rypipe_core/
arrow_export.rs1use arrow::array::{ArrayRef, AsArray, BooleanArray};
2use arrow::compute::filter_record_batch;
3use arrow::compute::kernels::boolean::and;
4use arrow::compute::kernels::cast::cast;
5use arrow::compute::kernels::cmp::{eq, gt, gt_eq, lt, lt_eq, neq};
6use arrow::datatypes::{DataType, Float64Type};
7use arrow::record_batch::RecordBatch;
8
9use crate::plan::{CompareOp, FilterPredicate};
10use crate::Result;
11
12pub fn null_array(datatype: &DataType, len: usize) -> ArrayRef {
14 arrow::array::new_null_array(datatype, len)
15}
16
17pub fn apply_compare_filter(
30 batch: RecordBatch,
31 predicate: &FilterPredicate,
32) -> Result<RecordBatch> {
33 if !is_pure_compare_tree(predicate) {
34 return Ok(batch);
35 }
36
37 let mask = compare_mask(&batch, predicate)?;
38 let mask = match mask {
39 Some(m) => m,
40 None => return Ok(batch),
41 };
42 Ok(filter_record_batch(&batch, &mask)?)
43}
44
45fn is_pure_compare_tree(predicate: &FilterPredicate) -> bool {
47 match predicate {
48 FilterPredicate::Compare { .. } => true,
49 FilterPredicate::And(a, b) => is_pure_compare_tree(a) && is_pure_compare_tree(b),
50 _ => false,
51 }
52}
53
54fn compare_mask(batch: &RecordBatch, predicate: &FilterPredicate) -> Result<Option<BooleanArray>> {
57 match predicate {
58 FilterPredicate::Compare {
59 field_a,
60 op,
61 field_b,
62 } => {
63 let col_a = batch.column_by_name(field_a).ok_or_else(|| {
64 crate::Error::Plan(format!(
65 "compare filter references unknown column {field_a:?}"
66 ))
67 })?;
68 let col_b = batch.column_by_name(field_b).ok_or_else(|| {
69 crate::Error::Plan(format!(
70 "compare filter references unknown column {field_b:?}"
71 ))
72 })?;
73 Ok(Some(compare_columns(col_a, col_b, *op)?))
74 }
75 FilterPredicate::And(a, b) => {
76 let ma = compare_mask(batch, a)?;
77 let mb = compare_mask(batch, b)?;
78 Ok(match (ma, mb) {
79 (Some(x), Some(y)) => Some(and(&x, &y)?),
80 (Some(x), None) | (None, Some(x)) => Some(x),
81 (None, None) => None,
82 })
83 }
84 _ => Ok(None),
85 }
86}
87
88fn is_numeric(dt: &DataType) -> bool {
89 matches!(
90 dt,
91 DataType::Int8
92 | DataType::Int16
93 | DataType::Int32
94 | DataType::Int64
95 | DataType::UInt8
96 | DataType::UInt16
97 | DataType::UInt32
98 | DataType::UInt64
99 | DataType::Float32
100 | DataType::Float64
101 | DataType::Boolean
102 )
103}
104
105fn compare_columns(col_a: &ArrayRef, col_b: &ArrayRef, op: CompareOp) -> Result<BooleanArray> {
106 let dt_a = col_a.data_type();
107 let dt_b = col_b.data_type();
108
109 let (a, b): (ArrayRef, ArrayRef) = if is_numeric(dt_a) && is_numeric(dt_b) {
113 (
114 cast(col_a, &DataType::Float64)?,
115 cast(col_b, &DataType::Float64)?,
116 )
117 } else {
118 (cast(col_a, &DataType::Utf8)?, cast(col_b, &DataType::Utf8)?)
119 };
120
121 match a.data_type() {
122 DataType::Float64 => {
123 let a = a.as_primitive::<Float64Type>();
124 let b = b.as_primitive::<Float64Type>();
125 Ok(match op {
126 CompareOp::Gt => gt(a, b)?,
127 CompareOp::Lt => lt(a, b)?,
128 CompareOp::Ge => gt_eq(a, b)?,
129 CompareOp::Le => lt_eq(a, b)?,
130 CompareOp::Eq => eq(a, b)?,
131 CompareOp::Ne => neq(a, b)?,
132 })
133 }
134 DataType::Utf8 => {
135 let a = a.as_string::<i32>();
136 let b = b.as_string::<i32>();
137 Ok(match op {
138 CompareOp::Gt => gt(a, b)?,
139 CompareOp::Lt => lt(a, b)?,
140 CompareOp::Ge => gt_eq(a, b)?,
141 CompareOp::Le => lt_eq(a, b)?,
142 CompareOp::Eq => eq(a, b)?,
143 CompareOp::Ne => neq(a, b)?,
144 })
145 }
146 other => Err(crate::Error::Plan(format!(
147 "unsupported common type for compare filter: {other}"
148 ))),
149 }
150}