vortex_array/arrays/varbin/compute/
compare.rs

1// SPDX-License-Identifier: Apache-2.0
2// SPDX-FileCopyrightText: Copyright the Vortex contributors
3
4use arrow_array::BinaryArray;
5use arrow_array::StringArray;
6use arrow_ord::cmp;
7use itertools::Itertools;
8use vortex_buffer::BitBuffer;
9use vortex_dtype::DType;
10use vortex_dtype::IntegerPType;
11use vortex_dtype::match_each_integer_ptype;
12use vortex_error::VortexExpect as _;
13use vortex_error::VortexResult;
14use vortex_error::vortex_bail;
15use vortex_error::vortex_err;
16
17use crate::Array;
18use crate::ArrayRef;
19use crate::IntoArray;
20use crate::ToCanonical;
21use crate::arrays::BoolArray;
22use crate::arrays::PrimitiveArray;
23use crate::arrays::VarBinArray;
24use crate::arrays::VarBinVTable;
25use crate::arrow::Datum;
26use crate::arrow::from_arrow_array_with_len;
27use crate::compute::CompareKernel;
28use crate::compute::CompareKernelAdapter;
29use crate::compute::Operator;
30use crate::compute::compare;
31use crate::compute::compare_lengths_to_empty;
32use crate::register_kernel;
33use crate::vtable::ValidityHelper;
34
35// This implementation exists so we can have custom translation of RHS to arrow that's not the same as IntoCanonical
36impl CompareKernel for VarBinVTable {
37    fn compare(
38        &self,
39        lhs: &VarBinArray,
40        rhs: &dyn Array,
41        operator: Operator,
42    ) -> VortexResult<Option<ArrayRef>> {
43        if let Some(rhs_const) = rhs.as_constant() {
44            let nullable = lhs.dtype().is_nullable() || rhs_const.dtype().is_nullable();
45            let len = lhs.len();
46
47            let rhs_is_empty = match rhs_const.dtype() {
48                DType::Binary(_) => rhs_const
49                    .as_binary()
50                    .is_empty()
51                    .vortex_expect("RHS should not be null"),
52                DType::Utf8(_) => rhs_const
53                    .as_utf8()
54                    .is_empty()
55                    .vortex_expect("RHS should not be null"),
56                _ => vortex_bail!("VarBinArray can only have type of Binary or Utf8"),
57            };
58
59            if rhs_is_empty {
60                let buffer = match operator {
61                    Operator::Gte => BitBuffer::new_set(len), // Every possible value is >= ""
62                    Operator::Lt => BitBuffer::new_unset(len), // No value is < ""
63                    Operator::Eq | Operator::NotEq | Operator::Gt | Operator::Lte => {
64                        let lhs_offsets = lhs.offsets().to_primitive();
65                        match_each_integer_ptype!(lhs_offsets.ptype(), |P| {
66                            compare_offsets_to_empty::<P>(lhs_offsets, operator)
67                        })
68                    }
69                };
70
71                return Ok(Some(
72                    BoolArray::from_bit_buffer(
73                        buffer,
74                        lhs.validity()
75                            .clone()
76                            .union_nullability(rhs.dtype().nullability()),
77                    )
78                    .into_array(),
79                ));
80            }
81
82            let lhs = Datum::try_new(lhs.as_ref())?;
83
84            // TODO(robert): Handle LargeString/Binary arrays
85            let arrow_rhs: &dyn arrow_array::Datum = match rhs_const.dtype() {
86                DType::Utf8(_) => &rhs_const
87                    .as_utf8()
88                    .value()
89                    .map(StringArray::new_scalar)
90                    .unwrap_or_else(|| arrow_array::Scalar::new(StringArray::new_null(1))),
91                DType::Binary(_) => &rhs_const
92                    .as_binary()
93                    .value()
94                    .map(BinaryArray::new_scalar)
95                    .unwrap_or_else(|| arrow_array::Scalar::new(BinaryArray::new_null(1))),
96                _ => vortex_bail!(
97                    "VarBin array RHS can only be Utf8 or Binary, given {}",
98                    rhs_const.dtype()
99                ),
100            };
101
102            let array = match operator {
103                Operator::Eq => cmp::eq(&lhs, arrow_rhs),
104                Operator::NotEq => cmp::neq(&lhs, arrow_rhs),
105                Operator::Gt => cmp::gt(&lhs, arrow_rhs),
106                Operator::Gte => cmp::gt_eq(&lhs, arrow_rhs),
107                Operator::Lt => cmp::lt(&lhs, arrow_rhs),
108                Operator::Lte => cmp::lt_eq(&lhs, arrow_rhs),
109            }
110            .map_err(|err| vortex_err!("Failed to compare VarBin array: {}", err))?;
111
112            Ok(Some(from_arrow_array_with_len(&array, len, nullable)))
113        } else if !rhs.is::<VarBinVTable>() {
114            // NOTE: If the rhs is not a VarBin array it will be canonicalized to a VarBinView
115            // Arrow doesn't support comparing VarBin to VarBinView arrays, so we convert ourselves
116            // to VarBinView and re-invoke.
117            return Ok(Some(compare(lhs.to_varbinview().as_ref(), rhs, operator)?));
118        } else {
119            Ok(None)
120        }
121    }
122}
123
124register_kernel!(CompareKernelAdapter(VarBinVTable).lift());
125
126fn compare_offsets_to_empty<P: IntegerPType>(
127    offsets: PrimitiveArray,
128    operator: Operator,
129) -> BitBuffer {
130    let lengths_iter = offsets
131        .as_slice::<P>()
132        .iter()
133        .tuple_windows()
134        .map(|(&s, &e)| e - s);
135    compare_lengths_to_empty(lengths_iter, operator)
136}
137
138#[cfg(test)]
139mod test {
140    use vortex_buffer::BitBuffer;
141    use vortex_buffer::ByteBuffer;
142    use vortex_dtype::DType;
143    use vortex_dtype::Nullability;
144    use vortex_scalar::Scalar;
145
146    use crate::ToCanonical;
147    use crate::arrays::ConstantArray;
148    use crate::arrays::VarBinArray;
149    use crate::arrays::VarBinViewArray;
150    use crate::compute::Operator;
151    use crate::compute::compare;
152
153    #[test]
154    fn test_binary_compare() {
155        let array = VarBinArray::from_iter(
156            [Some(b"abc".to_vec()), None, Some(b"def".to_vec())],
157            DType::Binary(Nullability::Nullable),
158        );
159        let result = compare(
160            array.as_ref(),
161            ConstantArray::new(
162                Scalar::binary(ByteBuffer::copy_from(b"abc"), Nullability::Nullable),
163                3,
164            )
165            .as_ref(),
166            Operator::Eq,
167        )
168        .unwrap()
169        .to_bool();
170
171        assert_eq!(
172            &result.validity_mask().to_bit_buffer(),
173            &BitBuffer::from_iter([true, false, true])
174        );
175        assert_eq!(
176            result.bit_buffer(),
177            &BitBuffer::from_iter([true, false, false])
178        );
179    }
180
181    #[test]
182    fn varbinview_compare() {
183        let array = VarBinArray::from_iter(
184            [Some(b"abc".to_vec()), None, Some(b"def".to_vec())],
185            DType::Binary(Nullability::Nullable),
186        );
187        let vbv = VarBinViewArray::from_iter(
188            [None, None, Some(b"def".to_vec())],
189            DType::Binary(Nullability::Nullable),
190        );
191        let result = compare(array.as_ref(), vbv.as_ref(), Operator::Eq)
192            .unwrap()
193            .to_bool();
194
195        assert_eq!(
196            &result.validity_mask().to_bit_buffer(),
197            &BitBuffer::from_iter([false, false, true])
198        );
199        assert_eq!(
200            result.bit_buffer(),
201            &BitBuffer::from_iter([false, true, true])
202        );
203    }
204}
205
206#[cfg(test)]
207mod tests {
208    use vortex_dtype::DType;
209    use vortex_dtype::Nullability;
210    use vortex_scalar::Scalar;
211
212    use crate::Array;
213    use crate::arrays::ConstantArray;
214    use crate::arrays::VarBinArray;
215    use crate::compute::Operator;
216    use crate::compute::compare;
217
218    #[test]
219    fn test_null_compare() {
220        let arr = VarBinArray::from_iter([Some("h")], DType::Utf8(Nullability::NonNullable));
221
222        let const_ = ConstantArray::new(Scalar::utf8("", Nullability::Nullable), 1);
223
224        assert_eq!(
225            compare(arr.as_ref(), const_.as_ref(), Operator::Eq)
226                .unwrap()
227                .dtype(),
228            &DType::Bool(Nullability::Nullable)
229        );
230    }
231}
vortex_array/arrays/varbin/compute/compare.rs

vortex_array/arrays/varbin/compute/
compare.rs
