1mod kernel;
5
6use std::fmt::Display;
7use std::fmt::Formatter;
8
9pub use kernel::*;
10use prost::Message;
11use vortex_error::VortexResult;
12use vortex_error::vortex_bail;
13use vortex_session::VortexSession;
14use vortex_session::registry::CachedId;
15
16use crate::AnyCanonical;
17use crate::ArrayRef;
18use crate::Canonical;
19use crate::ExecutionCtx;
20use crate::IntoArray;
21use crate::arrays::ConstantArray;
22use crate::arrays::Decimal;
23use crate::arrays::Primitive;
24use crate::arrays::ScalarFnArray;
25use crate::builtins::ArrayBuiltins;
26use crate::dtype::DType;
27use crate::dtype::DType::Bool;
28use crate::dtype::NativePType;
29use crate::expr::display::ExprDisplay;
30use crate::proto::expr as pb;
31use crate::require_input;
32use crate::scalar::Scalar;
33use crate::scalar_fn::Arity;
34use crate::scalar_fn::ChildName;
35use crate::scalar_fn::ExecutionArgs;
36use crate::scalar_fn::ScalarFnExecutionResult;
37use crate::scalar_fn::ScalarFnId;
38use crate::scalar_fn::ScalarFnVTable;
39use crate::scalar_fn::ScalarFnVTableExt;
40use crate::scalar_fn::fns::operators::Operator;
41
42#[derive(Debug, Clone, PartialEq, Eq, Hash)]
43pub struct BetweenOptions {
44 pub lower_strict: StrictComparison,
45 pub upper_strict: StrictComparison,
46}
47
48impl Display for BetweenOptions {
49 fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result {
50 let lower_op = if self.lower_strict.is_strict() {
51 "<"
52 } else {
53 "<="
54 };
55 let upper_op = if self.upper_strict.is_strict() {
56 "<"
57 } else {
58 "<="
59 };
60 write!(f, "lower_strict: {}, upper_strict: {}", lower_op, upper_op)
61 }
62}
63
64#[derive(Debug, Copy, Clone, PartialEq, Eq, Hash)]
66pub enum StrictComparison {
67 Strict,
69 NonStrict,
71}
72
73impl StrictComparison {
74 pub const fn to_operator(&self) -> Operator {
75 match self {
76 StrictComparison::Strict => Operator::Lt,
77 StrictComparison::NonStrict => Operator::Lte,
78 }
79 }
80
81 pub const fn is_strict(&self) -> bool {
82 matches!(self, StrictComparison::Strict)
83 }
84
85 pub fn to_predicate<T: NativePType>(self) -> fn(T, T) -> bool {
87 match self {
88 StrictComparison::Strict => T::is_lt,
89 StrictComparison::NonStrict => T::is_le,
90 }
91 }
92}
93
94pub(super) fn short_circuit(
105 arr: &ArrayRef,
106 lower: &ArrayRef,
107 upper: &ArrayRef,
108 options: &BetweenOptions,
109) -> VortexResult<Option<ArrayRef>> {
110 let return_dtype =
111 Bool(arr.dtype().nullability() | lower.dtype().nullability() | upper.dtype().nullability());
112
113 if arr.is_empty() {
115 return Ok(Some(Canonical::empty(&return_dtype).into_array()));
116 }
117
118 if arr.as_constant().is_some_and(|v| v.is_null()) {
121 return Ok(Some(
122 ConstantArray::new(Scalar::null(return_dtype), arr.len()).into_array(),
123 ));
124 }
125
126 let lower_is_null = lower.as_constant().is_some_and(|v| v.is_null());
127 let upper_is_null = upper.as_constant().is_some_and(|v| v.is_null());
128
129 if lower_is_null && upper_is_null {
132 return Ok(Some(
133 ConstantArray::new(Scalar::null(return_dtype), arr.len()).into_array(),
134 ));
135 }
136
137 if lower_is_null || upper_is_null {
140 return as_two_compares(arr, lower, upper, options).map(Some);
141 }
142
143 Ok(None)
144}
145
146pub(crate) fn as_two_compares(
150 arr: &ArrayRef,
151 lower: &ArrayRef,
152 upper: &ArrayRef,
153 options: &BetweenOptions,
154) -> VortexResult<ArrayRef> {
155 let lower_cmp = lower.binary(arr.clone(), options.lower_strict.to_operator())?;
156 let upper_cmp = arr.binary(upper.clone(), options.upper_strict.to_operator())?;
157 lower_cmp.binary(upper_cmp, Operator::And)
158}
159
160fn between_canonical(
164 arr: &ArrayRef,
165 lower: &ArrayRef,
166 upper: &ArrayRef,
167 options: &BetweenOptions,
168 ctx: &mut ExecutionCtx,
169) -> VortexResult<ArrayRef> {
170 if let Some(result) = short_circuit(arr, lower, upper, options)? {
171 return result.execute::<ArrayRef>(ctx);
174 }
175
176 if let Some(prim) = arr.as_opt::<Primitive>()
178 && let Some(result) =
179 <Primitive as BetweenKernel>::between(prim, lower, upper, options, ctx)?
180 {
181 return Ok(result);
182 }
183 if let Some(dec) = arr.as_opt::<Decimal>()
184 && let Some(result) = <Decimal as BetweenKernel>::between(dec, lower, upper, options, ctx)?
185 {
186 return Ok(result);
187 }
188
189 as_two_compares(arr, lower, upper, options)?.execute::<ArrayRef>(ctx)
192}
193
194#[derive(Clone)]
206pub struct Between;
207
208impl Between {
209 pub fn try_new(
215 array: ArrayRef,
216 lower: ArrayRef,
217 upper: ArrayRef,
218 options: BetweenOptions,
219 ) -> VortexResult<ScalarFnArray> {
220 ScalarFnArray::try_new(Between.bind(options), [array, lower, upper])
221 }
222}
223
224impl ScalarFnVTable for Between {
225 type Options = BetweenOptions;
226
227 fn id(&self) -> ScalarFnId {
228 static ID: CachedId = CachedId::new("vortex.between");
229 *ID
230 }
231
232 fn serialize(&self, instance: &Self::Options) -> VortexResult<Option<Vec<u8>>> {
233 Ok(Some(
234 pb::BetweenOpts {
235 lower_strict: instance.lower_strict.is_strict(),
236 upper_strict: instance.upper_strict.is_strict(),
237 }
238 .encode_to_vec(),
239 ))
240 }
241
242 fn deserialize(
243 &self,
244 _metadata: &[u8],
245 _session: &VortexSession,
246 ) -> VortexResult<Self::Options> {
247 let opts = pb::BetweenOpts::decode(_metadata)?;
248 Ok(BetweenOptions {
249 lower_strict: if opts.lower_strict {
250 StrictComparison::Strict
251 } else {
252 StrictComparison::NonStrict
253 },
254 upper_strict: if opts.upper_strict {
255 StrictComparison::Strict
256 } else {
257 StrictComparison::NonStrict
258 },
259 })
260 }
261
262 fn arity(&self, _options: &Self::Options) -> Arity {
263 Arity::Exact(3)
264 }
265
266 fn child_name(&self, _instance: &Self::Options, child_idx: usize) -> ChildName {
267 match child_idx {
268 0 => ChildName::from("array"),
269 1 => ChildName::from("lower"),
270 2 => ChildName::from("upper"),
271 _ => unreachable!("Invalid child index {} for Between expression", child_idx),
272 }
273 }
274
275 fn fmt_sql(
276 &self,
277 options: &Self::Options,
278 expr: &dyn ExprDisplay,
279 f: &mut Formatter<'_>,
280 ) -> std::fmt::Result {
281 let lower_op = if options.lower_strict.is_strict() {
282 "<"
283 } else {
284 "<="
285 };
286 let upper_op = if options.upper_strict.is_strict() {
287 "<"
288 } else {
289 "<="
290 };
291 write!(
292 f,
293 "({} {} {} {} {})",
294 expr.display_child(1),
295 lower_op,
296 expr.display_child(0),
297 upper_op,
298 expr.display_child(2)
299 )
300 }
301
302 fn return_dtype(&self, _options: &Self::Options, arg_dtypes: &[DType]) -> VortexResult<DType> {
303 let arr_dt = &arg_dtypes[0];
304 let lower_dt = &arg_dtypes[1];
305 let upper_dt = &arg_dtypes[2];
306
307 if !arr_dt.eq_ignore_nullability(lower_dt) {
308 vortex_bail!(
309 "Array dtype {} does not match lower dtype {}",
310 arr_dt,
311 lower_dt
312 );
313 }
314 if !arr_dt.eq_ignore_nullability(upper_dt) {
315 vortex_bail!(
316 "Array dtype {} does not match upper dtype {}",
317 arr_dt,
318 upper_dt
319 );
320 }
321
322 Ok(Bool(
323 arr_dt.nullability() | lower_dt.nullability() | upper_dt.nullability(),
324 ))
325 }
326
327 fn execute(
328 &self,
329 options: &Self::Options,
330 args: &dyn ExecutionArgs,
331 ctx: &mut ExecutionCtx,
332 ) -> VortexResult<ArrayRef> {
333 let arr = args.get(0)?;
334 let lower = args.get(1)?;
335 let upper = args.get(2)?;
336
337 if !arr.is_canonical() {
339 return arr.execute::<Canonical>(ctx)?.into_array().between(
340 lower,
341 upper,
342 options.clone(),
343 );
344 }
345
346 between_canonical(&arr, &lower, &upper, options, ctx)
347 }
348
349 fn execute_step(
350 &self,
351 options: &Self::Options,
352 args: &dyn ExecutionArgs,
353 ctx: &mut ExecutionCtx,
354 ) -> VortexResult<ScalarFnExecutionResult> {
355 require_input!(args, 0 => AnyCanonical);
356 self.execute(options, args, ctx)
357 .map(ScalarFnExecutionResult::done)
358 }
359
360 fn is_strict(&self, _options: &Self::Options) -> bool {
361 false }
363
364 fn is_infallible(&self, _options: &Self::Options) -> bool {
365 true
366 }
367}
368
369#[cfg(test)]
370mod tests {
371 use std::sync::LazyLock;
372
373 use rstest::rstest;
374 use vortex_buffer::buffer;
375
376 use super::*;
377 use crate::IntoArray;
378 use crate::VortexSessionExecute;
379 use crate::arrays::BoolArray;
380 use crate::arrays::DecimalArray;
381 use crate::arrays::PrimitiveArray;
382 use crate::arrays::StructArray;
383 use crate::assert_arrays_eq;
384 use crate::dtype::DType;
385 use crate::dtype::DecimalDType;
386 use crate::dtype::Nullability;
387 use crate::dtype::PType;
388 use crate::expr::between;
389 use crate::expr::col;
390 use crate::expr::get_item;
391 use crate::expr::lit;
392 use crate::expr::root;
393 use crate::scalar::DecimalValue;
394 use crate::scalar::Scalar;
395 use crate::test_harness::to_int_indices;
396 use crate::validity::Validity;
397
398 static SESSION: LazyLock<VortexSession> = LazyLock::new(VortexSession::empty);
399
400 const NON_STRICT: BetweenOptions = BetweenOptions {
401 lower_strict: StrictComparison::NonStrict,
402 upper_strict: StrictComparison::NonStrict,
403 };
404
405 fn null_i32s(len: usize) -> ArrayRef {
407 let null = Scalar::null(DType::Primitive(PType::I32, Nullability::Nullable));
408 ConstantArray::new(null, len).into_array()
409 }
410
411 #[test]
412 fn lazy_validity() -> VortexResult<()> {
413 let ctx = &mut SESSION.create_execution_ctx();
414
415 let x = PrimitiveArray::from_option_iter([Some(10), Some(10), Some(1)]).into_array();
416 let lo = PrimitiveArray::from_option_iter([None, None, Some(0)]).into_array();
417 let hi = PrimitiveArray::from_option_iter([Some(5), Some(50), Some(5)]).into_array();
418
419 let data = StructArray::from_fields(&[("x", x), ("lo", lo), ("hi", hi)])?.into_array();
420 let lazy = data.apply(&between(col("x"), col("lo"), col("hi"), NON_STRICT))?;
421
422 let Validity::Array(validity) = lazy.validity()? else {
423 vortex_bail!("non-lazy validity");
424 };
425 assert_eq!(
426 validity.encoding_id().to_string(),
427 "vortex.scalar_fn_validity"
428 );
429 assert_arrays_eq!(validity, BoolArray::from_iter([true, false, true]), ctx);
430 Ok(())
431 }
432
433 #[test]
438 fn validity_agrees_with_execution() -> VortexResult<()> {
439 let ctx = &mut SESSION.create_execution_ctx();
440
441 let x = PrimitiveArray::from_option_iter([Some(10), Some(10), Some(1)]).into_array();
446 let lo = PrimitiveArray::from_option_iter([None, None, Some(0)]).into_array();
447 let hi = PrimitiveArray::from_option_iter([Some(5), Some(50), Some(5)]).into_array();
448 let data = StructArray::from_fields(&[("x", x), ("lo", lo), ("hi", hi)])?.into_array();
449
450 let expr = between(col("x"), col("lo"), col("hi"), NON_STRICT);
451
452 let executed = data
453 .clone()
454 .apply(&expr)?
455 .execute::<BoolArray>(ctx)?
456 .opt_bool_vec(ctx);
457
458 let validity_expr = expr.bind(data.dtype())?.validity()?;
459 let declared = data
460 .apply_bound(&validity_expr)?
461 .execute::<BoolArray>(ctx)?
462 .bool_vec(ctx);
463
464 assert_eq!(executed, [Some(false), None, Some(true)]);
465 assert_eq!(
466 executed.iter().map(Option::is_some).collect::<Vec<_>>(),
467 declared
468 );
469
470 Ok(())
471 }
472
473 #[test]
474 fn is_not_strict() {
475 let expr = between(
476 root(),
477 lit(0),
478 lit(100),
479 BetweenOptions {
480 lower_strict: StrictComparison::NonStrict,
481 upper_strict: StrictComparison::NonStrict,
482 },
483 );
484
485 assert!(!expr.as_scalar().is_some_and(|f| f.signature().is_strict()));
486 }
487
488 #[test]
489 fn test_display() {
490 let expr = between(
491 get_item("score", root()),
492 lit(10),
493 lit(50),
494 BetweenOptions {
495 lower_strict: StrictComparison::NonStrict,
496 upper_strict: StrictComparison::Strict,
497 },
498 );
499 assert_eq!(expr.to_string(), "(10i32 <= $.score < 50i32)");
500
501 let expr2 = between(
502 root(),
503 lit(0),
504 lit(100),
505 BetweenOptions {
506 lower_strict: StrictComparison::Strict,
507 upper_strict: StrictComparison::NonStrict,
508 },
509 );
510 assert_eq!(expr2.to_string(), "(0i32 < $ <= 100i32)");
511 }
512
513 #[rstest]
514 #[case(StrictComparison::NonStrict, StrictComparison::NonStrict, vec![0, 1, 2, 3])]
515 #[case(StrictComparison::NonStrict, StrictComparison::Strict, vec![0, 1])]
516 #[case(StrictComparison::Strict, StrictComparison::NonStrict, vec![0, 2])]
517 #[case(StrictComparison::Strict, StrictComparison::Strict, vec![0])]
518 fn test_bounds(
519 #[case] lower_strict: StrictComparison,
520 #[case] upper_strict: StrictComparison,
521 #[case] expected: Vec<u64>,
522 ) {
523 let lower = buffer![0, 0, 0, 0, 2].into_array();
524 let array = buffer![1, 0, 1, 0, 1].into_array();
525 let upper = buffer![2, 1, 1, 0, 0].into_array();
526 let ctx = &mut SESSION.create_execution_ctx();
527
528 let matches = between_canonical(
529 &array,
530 &lower,
531 &upper,
532 &BetweenOptions {
533 lower_strict,
534 upper_strict,
535 },
536 ctx,
537 )
538 .unwrap()
539 .execute::<BoolArray>(ctx)
540 .unwrap();
541
542 let indices = to_int_indices(matches, ctx).unwrap();
543 assert_eq!(indices, expected);
544 }
545
546 #[test]
547 fn test_constants() {
548 let lower = buffer![0, 0, 2, 0, 2].into_array();
549 let array = buffer![1, 0, 1, 0, 1].into_array();
550 let ctx = &mut SESSION.create_execution_ctx();
551
552 let upper = ConstantArray::new(
554 Scalar::null(DType::Primitive(PType::I32, Nullability::Nullable)),
555 5,
556 )
557 .into_array();
558
559 let matches = between_canonical(
560 &array,
561 &lower,
562 &upper,
563 &BetweenOptions {
564 lower_strict: StrictComparison::NonStrict,
565 upper_strict: StrictComparison::NonStrict,
566 },
567 ctx,
568 )
569 .unwrap()
570 .execute::<BoolArray>(ctx)
571 .unwrap();
572
573 assert_eq!(
575 matches.opt_bool_vec(ctx),
576 [None, None, Some(false), None, Some(false)]
577 );
578
579 let upper = ConstantArray::new(Scalar::from(2), 5).into_array();
581 let matches = between_canonical(
582 &array,
583 &lower,
584 &upper,
585 &BetweenOptions {
586 lower_strict: StrictComparison::NonStrict,
587 upper_strict: StrictComparison::NonStrict,
588 },
589 ctx,
590 )
591 .unwrap()
592 .execute::<BoolArray>(ctx)
593 .unwrap();
594 let indices = to_int_indices(matches, ctx).unwrap();
595 assert_eq!(indices, vec![0, 1, 3]);
596
597 let lower = ConstantArray::new(Scalar::from(0), 5).into_array();
599
600 let matches = between_canonical(
601 &array,
602 &lower,
603 &upper,
604 &BetweenOptions {
605 lower_strict: StrictComparison::NonStrict,
606 upper_strict: StrictComparison::NonStrict,
607 },
608 ctx,
609 )
610 .unwrap()
611 .execute::<BoolArray>(ctx)
612 .unwrap();
613 let indices = to_int_indices(matches, ctx).unwrap();
614 assert_eq!(indices, vec![0, 1, 2, 3, 4]);
615 }
616
617 #[rstest]
621 #[case::primitive_nulls(PrimitiveArray::from_option_iter([None::<i32>, None]).into_array())]
622 #[case::constant_null(null_i32s(2))]
623 fn null_lower_bound(#[case] lower: ArrayRef) -> VortexResult<()> {
624 let ctx = &mut SESSION.create_execution_ctx();
625 let array = buffer![10, 10].into_array();
626 let upper = buffer![5, 50].into_array();
627
628 let result = between_canonical(&array, &lower, &upper, &NON_STRICT, ctx)?
629 .execute::<BoolArray>(ctx)?;
630
631 assert_eq!(result.opt_bool_vec(ctx), [Some(false), None]);
633
634 Ok(())
635 }
636
637 #[test]
639 fn both_bounds_null() -> VortexResult<()> {
640 let ctx = &mut SESSION.create_execution_ctx();
641 let array = buffer![10, 10].into_array();
642 let bound = null_i32s(2);
643
644 let result = between_canonical(&array, &bound, &bound, &NON_STRICT, ctx)?
645 .execute::<BoolArray>(ctx)?;
646
647 assert_eq!(result.opt_bool_vec(ctx), [None, None]);
648
649 Ok(())
650 }
651
652 #[rstest]
655 #[case::f64(Scalar::null_native::<f64>(), Scalar::from(-0.0f64), Scalar::from(0.0f64))]
656 #[case::i32(Scalar::null_native::<i32>(), Scalar::from(0i32), Scalar::from(0i32))]
657 fn constant_null_array(
658 #[case] null: Scalar,
659 #[case] lower: Scalar,
660 #[case] upper: Scalar,
661 ) -> VortexResult<()> {
662 let ctx = &mut SESSION.create_execution_ctx();
663 let array = ConstantArray::new(null, 3).into_array();
664 let lower = ConstantArray::new(lower, 3).into_array();
665 let upper = ConstantArray::new(upper, 3).into_array();
666
667 let between = array.between(lower, upper, NON_STRICT)?;
668 let not_between = between.not()?;
669
670 assert_eq!(
671 between.execute::<BoolArray>(ctx)?.opt_bool_vec(ctx),
672 [None, None, None]
673 );
674 assert_eq!(
675 not_between.execute::<BoolArray>(ctx)?.opt_bool_vec(ctx),
676 [None, None, None]
677 );
678
679 Ok(())
680 }
681
682 #[test]
683 fn test_between_decimal() {
684 let ctx = &mut SESSION.create_execution_ctx();
685 let values = buffer![100i128, 200i128, 300i128, 400i128];
686 let decimal_type = DecimalDType::new(3, 2);
687 let array = DecimalArray::new(values, decimal_type, Validity::NonNullable).into_array();
688
689 let lower = ConstantArray::new(
690 Scalar::decimal(
691 DecimalValue::I128(100i128),
692 decimal_type,
693 Nullability::NonNullable,
694 ),
695 array.len(),
696 )
697 .into_array();
698 let upper = ConstantArray::new(
699 Scalar::decimal(
700 DecimalValue::I128(400i128),
701 decimal_type,
702 Nullability::NonNullable,
703 ),
704 array.len(),
705 )
706 .into_array();
707
708 let between_strict = between_canonical(
710 &array,
711 &lower,
712 &upper,
713 &BetweenOptions {
714 lower_strict: StrictComparison::Strict,
715 upper_strict: StrictComparison::NonStrict,
716 },
717 ctx,
718 )
719 .unwrap();
720 assert_arrays_eq!(
721 between_strict,
722 BoolArray::from_iter([false, true, true, true]),
723 ctx
724 );
725
726 let between_strict = between_canonical(
728 &array,
729 &lower,
730 &upper,
731 &BetweenOptions {
732 lower_strict: StrictComparison::NonStrict,
733 upper_strict: StrictComparison::Strict,
734 },
735 ctx,
736 )
737 .unwrap();
738 assert_arrays_eq!(
739 between_strict,
740 BoolArray::from_iter([true, true, true, false]),
741 ctx
742 );
743 }
744
745 #[rstest]
751 #[case(DecimalValue::I16(1), DecimalValue::I32(82246), vec![0, 1, 2, 3])]
753 #[case(DecimalValue::I32(82246), DecimalValue::I16(4), vec![])]
755 #[case(DecimalValue::I16(1), DecimalValue::I32(-82246), vec![])]
757 #[case(DecimalValue::I32(-82246), DecimalValue::I16(2), vec![0, 1])]
759 fn test_between_decimal_mismatched_storage_types(
760 #[case] lower_val: DecimalValue,
761 #[case] upper_val: DecimalValue,
762 #[case] expected_indices: Vec<u64>,
763 ) {
764 let ctx = &mut SESSION.create_execution_ctx();
765 let decimal_type = DecimalDType::new(5, -67);
768 let array = DecimalArray::new(
769 buffer![1i16, 2i16, 3i16, 4i16],
770 decimal_type,
771 Validity::NonNullable,
772 )
773 .into_array();
774
775 let lower = ConstantArray::new(
776 Scalar::decimal(lower_val, decimal_type, Nullability::NonNullable),
777 array.len(),
778 )
779 .into_array();
780 let upper = ConstantArray::new(
781 Scalar::decimal(upper_val, decimal_type, Nullability::NonNullable),
782 array.len(),
783 )
784 .into_array();
785
786 let result = between_canonical(
787 &array,
788 &lower,
789 &upper,
790 &BetweenOptions {
791 lower_strict: StrictComparison::NonStrict,
792 upper_strict: StrictComparison::NonStrict,
793 },
794 ctx,
795 )
796 .unwrap()
797 .execute::<BoolArray>(ctx)
798 .unwrap();
799
800 assert_eq!(to_int_indices(result, ctx).unwrap(), expected_indices);
801 }
802
803 #[test]
805 fn executes_non_canonical_input_iteratively() -> VortexResult<()> {
806 let mut ctx = SESSION.create_execution_ctx();
807 let chunks = vec![
808 PrimitiveArray::from_iter([1i32, 5]).into_array(),
809 PrimitiveArray::from_iter([9i32, 12]).into_array(),
810 ];
811 let dtype = chunks[0].dtype().clone();
812 let chunked = crate::arrays::ChunkedArray::try_new(chunks, dtype)?.into_array();
813 let result = chunked.between(
814 ConstantArray::new(5i32, 4).into_array(),
815 ConstantArray::new(10i32, 4).into_array(),
816 BetweenOptions {
817 lower_strict: StrictComparison::NonStrict,
818 upper_strict: StrictComparison::Strict,
819 },
820 )?;
821
822 assert_arrays_eq!(
823 result,
824 BoolArray::from_iter([false, true, true, false]),
825 &mut ctx
826 );
827 Ok(())
828 }
829}