1use std::collections::HashSet;
4use std::fmt::{Display, Formatter};
5use std::sync::Arc;
6
7use itertools::Itertools;
8use serde::{de, ser, Deserialize, Deserializer, Serialize, Serializer};
9
10#[doc(hidden)]
11pub use self::column_names::{__require_valid_simple_column_segment, column_expr};
12pub use self::column_names::{
13 col, column_expr_ref, column_name, column_pred, joined_column_expr, joined_column_name,
14 ColumnName,
15};
16pub use self::scalars::{ArrayData, DecimalData, MapData, Scalar, StructData};
17use crate::kernel_predicates::{
18 DirectDataSkippingPredicateEvaluator, DirectPredicateEvaluator,
19 IndirectDataSkippingPredicateEvaluator,
20};
21use crate::schema::SchemaRef;
22pub use crate::struct_patch::{ExpressionFieldPatch, ExpressionStructPatch};
23use crate::transforms::{transform_output_type, ExpressionTransform};
24use crate::utils::CollectInto;
25use crate::{DataType, DeltaResult, DynPartialEq, Error};
26
27mod column_names;
28pub(crate) mod literal_expression_transform;
29pub(crate) use literal_expression_transform::literal_expression_transform;
30mod scalars;
31mod sql;
32pub(crate) use self::sql::parse_sql;
33
34pub type ExpressionRef = std::sync::Arc<Expression>;
35pub type PredicateRef = std::sync::Arc<Predicate>;
36
37pub fn lit(value: impl Into<Scalar>) -> Expression {
48 Expression::literal(value)
49}
50
51pub fn null_lit(data_type: impl Into<DataType>) -> Expression {
67 Expression::Literal(Scalar::null(data_type))
68}
69
70pub type ExpressionStructPatchBuilder = crate::struct_patch::StructPatchBuilder<ExpressionRef>;
74
75#[derive(Debug, Clone, Copy, PartialEq, Serialize, Deserialize)]
81pub enum UnaryPredicateOp {
82 IsNull,
85}
86
87#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, Serialize, Deserialize)]
93pub enum BinaryPredicateOp {
94 LessThan,
96 GreaterThan,
98 Equal,
100 Distinct,
104 In,
120}
121
122#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, Serialize, Deserialize)]
124pub enum UnaryExpressionOp {
125 ToJson,
147}
148
149#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, Serialize, Deserialize)]
155pub enum BinaryExpressionOp {
156 Plus,
158 Minus,
160 Multiply,
162 Divide,
173}
174
175#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, Serialize, Deserialize)]
177pub enum VariadicExpressionOp {
178 Coalesce,
181 Array,
190}
191
192#[derive(Debug, Clone, Copy, PartialEq, Serialize, Deserialize)]
199pub enum JunctionPredicateOp {
200 And,
202 Or,
204}
205
206pub type ScalarExpressionEvaluator<'a> = dyn Fn(&Expression) -> Option<Scalar> + 'a;
214
215pub trait OpaqueExpressionOp: DynPartialEq + std::fmt::Debug {
217 fn name(&self) -> &str;
219
220 fn eval_expr_scalar(
231 &self,
232 eval_expr: &ScalarExpressionEvaluator<'_>,
233 exprs: &[Expression],
234 ) -> DeltaResult<Scalar>;
235}
236
237pub trait OpaquePredicateOp: DynPartialEq + std::fmt::Debug {
239 fn name(&self) -> &str;
241
242 fn eval_pred_scalar(
255 &self,
256 eval_expr: &ScalarExpressionEvaluator<'_>,
257 eval_pred: &DirectPredicateEvaluator<'_>,
258 exprs: &[Expression],
259 inverted: bool,
260 ) -> DeltaResult<Option<bool>>;
261
262 fn eval_as_data_skipping_predicate(
272 &self,
273 evaluator: &DirectDataSkippingPredicateEvaluator<'_>,
274 exprs: &[Expression],
275 inverted: bool,
276 ) -> Option<bool>;
277
278 fn as_data_skipping_predicate(
291 &self,
292 evaluator: &IndirectDataSkippingPredicateEvaluator<'_>,
293 exprs: &[Expression],
294 inverted: bool,
295 ) -> Option<Predicate>;
296}
297
298pub type OpaqueExpressionOpRef = Arc<dyn OpaqueExpressionOp>;
300
301pub type OpaquePredicateOpRef = Arc<dyn OpaquePredicateOp>;
303
304#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)]
309pub struct UnaryPredicate {
310 pub op: UnaryPredicateOp,
312 pub expr: Box<Expression>,
314}
315
316#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)]
317pub struct BinaryPredicate {
318 pub op: BinaryPredicateOp,
320 pub left: Box<Expression>,
322 pub right: Box<Expression>,
324}
325
326#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)]
327pub struct UnaryExpression {
328 pub op: UnaryExpressionOp,
330 pub expr: Box<Expression>,
332}
333
334#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)]
335pub struct BinaryExpression {
336 pub op: BinaryExpressionOp,
338 pub left: Box<Expression>,
340 pub right: Box<Expression>,
342}
343
344#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)]
345pub struct VariadicExpression {
346 pub op: VariadicExpressionOp,
348 pub exprs: Vec<Expression>,
350}
351
352#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)]
375pub struct ParseJsonExpression {
376 pub json_expr: Box<Expression>,
378 pub output_schema: SchemaRef,
380}
381
382#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)]
385pub struct CastExpression {
386 pub expr: Box<Expression>,
388 pub target: DataType,
390}
391
392#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)]
393pub struct JunctionPredicate {
394 pub op: JunctionPredicateOp,
396 pub preds: Vec<Predicate>,
398}
399
400#[derive(Clone, Debug)]
407pub struct OpaquePredicate {
408 pub op: OpaquePredicateOpRef,
409 pub exprs: Vec<Expression>,
410}
411fn fail_serialize_opaque_predicate<S>(
412 _value: &OpaquePredicate,
413 _serializer: S,
414) -> Result<S::Ok, S::Error>
415where
416 S: Serializer,
417{
418 Err(ser::Error::custom("Cannot serialize an Opaque Predicate"))
419}
420
421fn fail_deserialize_opaque_predicate<'de, D>(_deserializer: D) -> Result<OpaquePredicate, D::Error>
422where
423 D: Deserializer<'de>,
424{
425 Err(de::Error::custom("Cannot deserialize an Opaque Predicate"))
426}
427
428impl OpaquePredicate {
429 pub(crate) fn new(
430 op: OpaquePredicateOpRef,
431 exprs: impl IntoIterator<Item = Expression>,
432 ) -> Self {
433 let exprs = exprs.into_iter().collect();
434 Self { op, exprs }
435 }
436}
437
438#[derive(Clone, Debug)]
445pub struct OpaqueExpression {
446 pub op: OpaqueExpressionOpRef,
447 pub exprs: Vec<Expression>,
448}
449
450impl OpaqueExpression {
451 pub(crate) fn new(
452 op: OpaqueExpressionOpRef,
453 exprs: impl IntoIterator<Item = Expression>,
454 ) -> Self {
455 let exprs = exprs.into_iter().collect();
456 Self { op, exprs }
457 }
458}
459
460fn fail_serialize_opaque_expression<S>(
461 _value: &OpaqueExpression,
462 _serializer: S,
463) -> Result<S::Ok, S::Error>
464where
465 S: Serializer,
466{
467 Err(ser::Error::custom("Cannot serialize an Opaque Expression"))
468}
469
470fn fail_deserialize_opaque_expression<'de, D>(
471 _deserializer: D,
472) -> Result<OpaqueExpression, D::Error>
473where
474 D: Deserializer<'de>,
475{
476 Err(de::Error::custom("Cannot deserialize an Opaque Expression"))
477}
478
479#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
485pub enum Expression {
486 Literal(Scalar),
488 Column(ColumnName),
491 Predicate(Box<Predicate>), Struct(Vec<ExpressionRef>, Option<ExpressionRef>),
509 #[serde(alias = "Transform")]
512 StructPatch(ExpressionStructPatch),
513 Unary(UnaryExpression),
515 Binary(BinaryExpression),
517 Variadic(VariadicExpression),
519 #[serde(serialize_with = "fail_serialize_opaque_expression")]
522 #[serde(deserialize_with = "fail_deserialize_opaque_expression")]
523 Opaque(OpaqueExpression),
524 Unknown(String),
533 ParseJson(ParseJsonExpression),
537 MapToStruct(MapToStructExpression),
540 Cast(CastExpression),
542}
543
544#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
550pub enum Predicate {
551 BooleanExpression(Expression),
553 Not(Box<Predicate>),
560 Unary(UnaryPredicate),
562 Binary(BinaryPredicate),
564 Junction(JunctionPredicate),
566 #[serde(serialize_with = "fail_serialize_opaque_predicate")]
569 #[serde(deserialize_with = "fail_deserialize_opaque_predicate")]
570 Opaque(OpaquePredicate),
571 Unknown(String),
580}
581
582impl BinaryPredicateOp {
587 pub(crate) fn is_null_intolerant(&self) -> bool {
589 use BinaryPredicateOp::*;
590 match self {
591 LessThan | GreaterThan | Equal => true,
592 Distinct | In => false, }
594 }
595}
596
597impl JunctionPredicateOp {
598 pub(crate) fn invert(&self) -> JunctionPredicateOp {
599 use JunctionPredicateOp::*;
600 match self {
601 And => Or,
602 Or => And,
603 }
604 }
605}
606
607impl UnaryExpression {
608 pub(crate) fn new(op: UnaryExpressionOp, expr: impl Into<Expression>) -> Self {
609 let expr = Box::new(expr.into());
610 Self { op, expr }
611 }
612}
613
614impl UnaryPredicate {
615 pub(crate) fn new(op: UnaryPredicateOp, expr: impl Into<Expression>) -> Self {
616 let expr = Box::new(expr.into());
617 Self { op, expr }
618 }
619}
620
621impl BinaryExpression {
622 pub(crate) fn new(
623 op: BinaryExpressionOp,
624 left: impl Into<Expression>,
625 right: impl Into<Expression>,
626 ) -> Self {
627 let left = Box::new(left.into());
628 let right = Box::new(right.into());
629 Self { op, left, right }
630 }
631}
632
633impl BinaryPredicate {
634 pub(crate) fn new(
635 op: BinaryPredicateOp,
636 left: impl Into<Expression>,
637 right: impl Into<Expression>,
638 ) -> Self {
639 let left = Box::new(left.into());
640 let right = Box::new(right.into());
641 Self { op, left, right }
642 }
643}
644
645impl VariadicExpression {
646 pub(crate) fn new(
647 op: VariadicExpressionOp,
648 exprs: impl IntoIterator<Item = impl Into<Expression>>,
649 ) -> Self {
650 let exprs = exprs.into_iter().map(Into::into).collect();
651 Self { op, exprs }
652 }
653}
654
655impl ParseJsonExpression {
656 pub(crate) fn new(json_expr: impl Into<Expression>, output_schema: SchemaRef) -> Self {
657 Self {
658 json_expr: Box::new(json_expr.into()),
659 output_schema,
660 }
661 }
662}
663
664#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)]
678pub struct MapToStructExpression {
679 pub map_expr: Box<Expression>,
681}
682
683impl MapToStructExpression {
684 pub(crate) fn new(map_expr: impl Into<Expression>) -> Self {
685 Self {
686 map_expr: Box::new(map_expr.into()),
687 }
688 }
689}
690
691impl CastExpression {
692 pub(crate) fn new(expr: impl Into<Expression>, target: DataType) -> Self {
693 Self {
694 expr: Box::new(expr.into()),
695 target,
696 }
697 }
698}
699
700impl JunctionPredicate {
701 pub(crate) fn new(op: JunctionPredicateOp, preds: Vec<Predicate>) -> Self {
702 Self { op, preds }
703 }
704}
705
706impl Expression {
707 pub fn references(&self) -> HashSet<&ColumnName> {
709 let mut references = GetColumnReferences::default();
710 references.transform_expr(self);
711 references.0
712 }
713
714 pub fn column(field_names: impl CollectInto<ColumnName>) -> Expression {
716 ColumnName::new(field_names).into()
717 }
718
719 pub fn literal(value: impl Into<Scalar>) -> Self {
721 Self::Literal(value.into())
722 }
723
724 pub fn from_pred(value: Predicate) -> Self {
726 match value {
727 Predicate::BooleanExpression(expr) => expr,
728 _ => Self::Predicate(Box::new(value)),
729 }
730 }
731
732 pub fn struct_from(exprs: impl IntoIterator<Item = impl Into<Arc<Self>>>) -> Self {
739 Self::Struct(exprs.into_iter().map(Into::into).collect(), None)
740 }
741
742 pub fn struct_with_nullability_from(
747 exprs: impl IntoIterator<Item = impl Into<Arc<Self>>>,
748 nullability_predicate: impl Into<Arc<Self>>,
749 ) -> Self {
750 Self::Struct(
751 exprs.into_iter().map(Into::into).collect(),
752 Some(nullability_predicate.into()),
753 )
754 }
755
756 pub fn struct_patch<P>(patch: P) -> DeltaResult<Self>
766 where
767 P: TryInto<ExpressionStructPatch>,
768 Error: From<P::Error>,
769 {
770 Ok(Self::StructPatch(patch.try_into()?))
771 }
772
773 pub fn is_null(self) -> Predicate {
775 Predicate::is_null(self)
776 }
777
778 pub fn is_not_null(self) -> Predicate {
780 Predicate::is_not_null(self)
781 }
782
783 pub fn eq(self, other: impl Into<Self>) -> Predicate {
785 Predicate::eq(self, other)
786 }
787
788 pub fn ne(self, other: impl Into<Self>) -> Predicate {
790 Predicate::ne(self, other)
791 }
792
793 pub fn le(self, other: impl Into<Self>) -> Predicate {
795 Predicate::le(self, other)
796 }
797
798 pub fn lt(self, other: impl Into<Self>) -> Predicate {
800 Predicate::lt(self, other)
801 }
802
803 pub fn ge(self, other: impl Into<Self>) -> Predicate {
805 Predicate::ge(self, other)
806 }
807
808 pub fn gt(self, other: impl Into<Self>) -> Predicate {
810 Predicate::gt(self, other)
811 }
812
813 pub fn distinct(self, other: impl Into<Self>) -> Predicate {
815 Predicate::distinct(self, other)
816 }
817
818 pub fn unary(op: UnaryExpressionOp, expr: impl Into<Expression>) -> Self {
820 Self::Unary(UnaryExpression::new(op, expr))
821 }
822
823 pub fn binary(
825 op: BinaryExpressionOp,
826 lhs: impl Into<Expression>,
827 rhs: impl Into<Expression>,
828 ) -> Self {
829 Self::Binary(BinaryExpression::new(op, lhs, rhs))
830 }
831
832 pub fn variadic(
834 op: VariadicExpressionOp,
835 exprs: impl IntoIterator<Item = impl Into<Expression>>,
836 ) -> Self {
837 Self::Variadic(VariadicExpression::new(op, exprs))
838 }
839
840 pub fn coalesce(exprs: impl IntoIterator<Item = impl Into<Expression>>) -> Self {
845 Self::variadic(VariadicExpressionOp::Coalesce, exprs)
846 }
847
848 pub fn array(exprs: impl IntoIterator<Item = impl Into<Expression>>) -> Self {
850 Self::variadic(VariadicExpressionOp::Array, exprs)
851 }
852
853 pub fn opaque(
855 op: impl OpaqueExpressionOp,
856 exprs: impl IntoIterator<Item = Expression>,
857 ) -> Self {
858 Self::Opaque(OpaqueExpression::new(Arc::new(op), exprs))
859 }
860
861 pub fn unknown(name: impl Into<String>) -> Self {
863 Self::Unknown(name.into())
864 }
865
866 pub fn parse_json(json_expr: impl Into<Expression>, output_schema: SchemaRef) -> Self {
871 Self::ParseJson(ParseJsonExpression::new(json_expr, output_schema))
872 }
873
874 pub fn map_to_struct(map_expr: impl Into<Expression>) -> Self {
879 Self::MapToStruct(MapToStructExpression::new(map_expr))
880 }
881
882 pub fn cast(expr: impl Into<Expression>, target: DataType) -> Self {
885 Self::Cast(CastExpression::new(expr, target))
886 }
887}
888
889impl Predicate {
890 pub const TRUE: Self = Self::literal(true);
892 pub const FALSE: Self = Self::literal(false);
894 pub const NULL: Self =
896 Self::BooleanExpression(Expression::Literal(Scalar::Null(DataType::BOOLEAN)));
897
898 pub fn references(&self) -> HashSet<&ColumnName> {
900 let mut references = GetColumnReferences::default();
901 references.transform_pred(self);
902 references.0
903 }
904
905 pub fn column(field_names: impl CollectInto<ColumnName>) -> Self {
907 Self::from_expr(ColumnName::new(field_names))
908 }
909
910 pub const fn literal(value: bool) -> Self {
914 Self::BooleanExpression(Expression::Literal(Scalar::Boolean(value)))
915 }
916
917 pub fn from_expr(expr: impl Into<Expression>) -> Self {
919 match expr.into() {
920 Expression::Predicate(p) => *p,
921 expr => Predicate::BooleanExpression(expr),
922 }
923 }
924
925 pub fn not(pred: impl Into<Self>) -> Self {
927 Self::Not(Box::new(pred.into()))
928 }
929
930 pub fn is_null(expr: impl Into<Expression>) -> Self {
932 Self::unary(UnaryPredicateOp::IsNull, expr)
933 }
934
935 pub fn is_not_null(expr: impl Into<Expression>) -> Self {
937 Self::not(Self::is_null(expr))
938 }
939
940 pub fn eq(a: impl Into<Expression>, b: impl Into<Expression>) -> Self {
942 Self::binary(BinaryPredicateOp::Equal, a, b)
943 }
944
945 pub fn ne(a: impl Into<Expression>, b: impl Into<Expression>) -> Self {
947 Self::not(Self::binary(BinaryPredicateOp::Equal, a, b))
948 }
949
950 pub fn le(a: impl Into<Expression>, b: impl Into<Expression>) -> Self {
952 Self::not(Self::binary(BinaryPredicateOp::GreaterThan, a, b))
953 }
954
955 pub fn lt(a: impl Into<Expression>, b: impl Into<Expression>) -> Self {
957 Self::binary(BinaryPredicateOp::LessThan, a, b)
958 }
959
960 pub fn ge(a: impl Into<Expression>, b: impl Into<Expression>) -> Self {
962 Self::not(Self::binary(BinaryPredicateOp::LessThan, a, b))
963 }
964
965 pub fn gt(a: impl Into<Expression>, b: impl Into<Expression>) -> Self {
967 Self::binary(BinaryPredicateOp::GreaterThan, a, b)
968 }
969
970 pub fn distinct(a: impl Into<Expression>, b: impl Into<Expression>) -> Self {
972 Self::binary(BinaryPredicateOp::Distinct, a, b)
973 }
974
975 pub fn and(a: impl Into<Self>, b: impl Into<Self>) -> Self {
977 Self::and_from([a.into(), b.into()])
978 }
979
980 pub fn or(a: impl Into<Self>, b: impl Into<Self>) -> Self {
982 Self::or_from([a.into(), b.into()])
983 }
984
985 pub fn and_from(preds: impl IntoIterator<Item = Self>) -> Self {
988 Self::junction(JunctionPredicateOp::And, preds)
989 }
990
991 pub fn or_from(preds: impl IntoIterator<Item = Self>) -> Self {
994 Self::junction(JunctionPredicateOp::Or, preds)
995 }
996
997 pub fn unary(op: UnaryPredicateOp, expr: impl Into<Expression>) -> Self {
999 let expr = Box::new(expr.into());
1000 Self::Unary(UnaryPredicate { op, expr })
1001 }
1002
1003 pub fn binary(
1005 op: BinaryPredicateOp,
1006 lhs: impl Into<Expression>,
1007 rhs: impl Into<Expression>,
1008 ) -> Self {
1009 Self::Binary(BinaryPredicate {
1010 op,
1011 left: Box::new(lhs.into()),
1012 right: Box::new(rhs.into()),
1013 })
1014 }
1015
1016 pub fn junction(op: JunctionPredicateOp, preds: impl IntoIterator<Item = Self>) -> Self {
1024 let mut preds: Vec<_> = preds.into_iter().collect();
1025 match preds.len() {
1026 0 => match op {
1027 JunctionPredicateOp::And => Self::TRUE,
1028 JunctionPredicateOp::Or => Self::FALSE,
1029 },
1030 1 => preds.remove(0),
1032 _ => Self::Junction(JunctionPredicate { op, preds }),
1033 }
1034 }
1035
1036 pub fn opaque(op: impl OpaquePredicateOp, exprs: impl IntoIterator<Item = Expression>) -> Self {
1038 Self::Opaque(OpaquePredicate::new(Arc::new(op), exprs))
1039 }
1040
1041 pub fn unknown(name: impl Into<String>) -> Self {
1043 Self::Unknown(name.into())
1044 }
1045}
1046
1047impl PartialEq for OpaquePredicate {
1052 fn eq(&self, other: &Self) -> bool {
1053 self.op.dyn_eq(other.op.any_ref()) && self.exprs == other.exprs
1054 }
1055}
1056
1057impl PartialEq for OpaqueExpression {
1058 fn eq(&self, other: &Self) -> bool {
1059 self.op.dyn_eq(other.op.any_ref()) && self.exprs == other.exprs
1060 }
1061}
1062
1063impl Display for UnaryExpressionOp {
1064 fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result {
1065 use UnaryExpressionOp::*;
1066 match self {
1067 ToJson => write!(f, "TO_JSON"),
1068 }
1069 }
1070}
1071
1072impl Display for BinaryExpressionOp {
1073 fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result {
1074 use BinaryExpressionOp::*;
1075 match self {
1076 Plus => write!(f, "+"),
1077 Minus => write!(f, "-"),
1078 Multiply => write!(f, "*"),
1079 Divide => write!(f, "/"),
1080 }
1081 }
1082}
1083
1084impl Display for VariadicExpressionOp {
1085 fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result {
1086 use VariadicExpressionOp::*;
1087 match self {
1088 Coalesce => write!(f, "COALESCE"),
1089 Array => write!(f, "ARRAY"),
1090 }
1091 }
1092}
1093
1094impl Display for BinaryPredicateOp {
1095 fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result {
1096 use BinaryPredicateOp::*;
1097 match self {
1098 LessThan => write!(f, "<"),
1099 GreaterThan => write!(f, ">"),
1100 Equal => write!(f, "="),
1101 Distinct => write!(f, "DISTINCT"),
1105 In => write!(f, "IN"),
1106 }
1107 }
1108}
1109
1110fn format_child_list<T: Display>(children: &[T]) -> String {
1112 children.iter().map(|c| format!("{c}")).join(", ")
1113}
1114
1115impl Display for Expression {
1116 fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result {
1117 use Expression::*;
1118 match self {
1119 Literal(l) => write!(f, "{l}"),
1120 Column(name) => write!(f, "Column({name})"),
1121 Predicate(p) => write!(f, "{p}"),
1122 Struct(exprs, _) => write!(f, "Struct({})", format_child_list(exprs)),
1123 StructPatch(patch) => {
1124 write!(f, "StructPatch(")?;
1125 let mut sep = "";
1126 if !patch.prepended_fields.is_empty() {
1127 let prepended_fields = format_child_list(&patch.prepended_fields);
1128 write!(f, "prepend [{prepended_fields}]")?;
1129 sep = ", ";
1130 }
1131 for (field_name, field_patch) in &patch.field_patches {
1132 if !field_patch.keep_input && field_patch.insertions.is_empty() {
1133 write!(f, "{sep}drop {field_name}")?;
1134 sep = ", ";
1135 }
1136 if !field_patch.insertions.is_empty() {
1137 let insertions = format_child_list(&field_patch.insertions);
1138 let action = if field_patch.keep_input {
1139 "after"
1140 } else {
1141 "replace/after"
1142 };
1143 write!(f, "{sep}{action} {field_name} insert [{insertions}]")?;
1144 sep = ", ";
1145 }
1146 }
1147 if !patch.appended_fields.is_empty() {
1148 let appended_fields = format_child_list(&patch.appended_fields);
1149 write!(f, "{sep}append [{appended_fields}]")?;
1150 }
1151 write!(f, ")")
1152 }
1153 Unary(UnaryExpression { op, expr }) => write!(f, "{op}({expr})"),
1154 Binary(BinaryExpression { op, left, right }) => write!(f, "{left} {op} {right}"),
1155 Variadic(VariadicExpression { op, exprs }) => {
1156 write!(f, "{op}({})", format_child_list(exprs))
1157 }
1158 Opaque(OpaqueExpression { op, exprs }) => {
1159 write!(f, "{op:?}({})", format_child_list(exprs))
1160 }
1161 Unknown(name) => write!(f, "<unknown: {name}>"),
1162 ParseJson(p) => {
1163 write!(
1164 f,
1165 "PARSE_JSON({}, <schema:{} fields>)",
1166 p.json_expr,
1167 p.output_schema.fields().len()
1168 )
1169 }
1170 MapToStruct(m) => write!(f, "MAP_TO_STRUCT({})", m.map_expr),
1171 Cast(c) => write!(f, "CAST({} AS {})", c.expr, c.target),
1172 }
1173 }
1174}
1175
1176impl Display for Predicate {
1177 fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result {
1178 use Predicate::*;
1179 match self {
1180 BooleanExpression(expr) => write!(f, "{expr}"),
1181 Not(pred) => write!(f, "NOT({pred})"),
1182 Binary(BinaryPredicate {
1183 op: BinaryPredicateOp::Distinct,
1184 left,
1185 right,
1186 }) => write!(f, "DISTINCT({left}, {right})"),
1187 Binary(BinaryPredicate { op, left, right }) => write!(f, "{left} {op} {right}"),
1188 Unary(UnaryPredicate { op, expr }) => match op {
1189 UnaryPredicateOp::IsNull => write!(f, "{expr} IS NULL"),
1190 },
1191 Junction(JunctionPredicate { op, preds }) => {
1192 let op = match op {
1193 JunctionPredicateOp::And => "AND",
1194 JunctionPredicateOp::Or => "OR",
1195 };
1196 write!(f, "{op}({})", format_child_list(preds))
1197 }
1198 Opaque(OpaquePredicate { op, exprs }) => {
1199 write!(f, "{op:?}({})", format_child_list(exprs))
1200 }
1201 Unknown(name) => write!(f, "<unknown: {name}>"),
1202 }
1203 }
1204}
1205
1206impl From<Scalar> for Expression {
1207 fn from(value: Scalar) -> Self {
1208 Self::literal(value)
1209 }
1210}
1211
1212impl From<ColumnName> for Expression {
1213 fn from(value: ColumnName) -> Self {
1214 Self::Column(value)
1215 }
1216}
1217
1218impl From<Predicate> for Expression {
1219 fn from(value: Predicate) -> Self {
1220 Self::from_pred(value)
1221 }
1222}
1223
1224impl From<ColumnName> for Predicate {
1225 fn from(value: ColumnName) -> Self {
1226 Self::from_expr(value)
1227 }
1228}
1229
1230impl<R: Into<Expression>> std::ops::Add<R> for Expression {
1231 type Output = Self;
1232
1233 fn add(self, rhs: R) -> Self::Output {
1234 Self::binary(BinaryExpressionOp::Plus, self, rhs)
1235 }
1236}
1237
1238impl<R: Into<Expression>> std::ops::Sub<R> for Expression {
1239 type Output = Self;
1240
1241 fn sub(self, rhs: R) -> Self {
1242 Self::binary(BinaryExpressionOp::Minus, self, rhs)
1243 }
1244}
1245
1246impl<R: Into<Expression>> std::ops::Mul<R> for Expression {
1247 type Output = Self;
1248
1249 fn mul(self, rhs: R) -> Self {
1250 Self::binary(BinaryExpressionOp::Multiply, self, rhs)
1251 }
1252}
1253
1254impl<R: Into<Expression>> std::ops::Div<R> for Expression {
1255 type Output = Self;
1256
1257 fn div(self, rhs: R) -> Self {
1258 Self::binary(BinaryExpressionOp::Divide, self, rhs)
1259 }
1260}
1261
1262#[derive(Default)]
1264struct GetColumnReferences<'a>(HashSet<&'a ColumnName>);
1265
1266impl<'a> ExpressionTransform<'a> for GetColumnReferences<'a> {
1267 transform_output_type!(|'a, T| ());
1268
1269 fn transform_expr_column(&mut self, name: &'a ColumnName) {
1270 self.0.insert(name);
1271 }
1272}
1273
1274#[cfg(test)]
1275mod tests {
1276 use std::fmt::Debug;
1277
1278 use serde::de::DeserializeOwned;
1279 use serde::Serialize;
1280
1281 use super::{col, column_pred, lit, DataType, Expression as Expr, Predicate as Pred};
1282
1283 fn assert_roundtrip<T: Serialize + DeserializeOwned + PartialEq + Debug>(value: &T) {
1285 let json = serde_json::to_string(value).expect("serialization should succeed");
1286 let deserialized: T = serde_json::from_str(&json).expect("deserialization should succeed");
1287 assert_eq!(value, &deserialized, "roundtrip should preserve value");
1288 }
1289
1290 #[test]
1291 fn test_expression_format() {
1292 let cases = [
1293 (col!("x"), "Column(x)"),
1294 (
1295 (col!("x") + lit(4)) / lit(10) * lit(42),
1296 "Column(x) + 4 / 10 * 42",
1297 ),
1298 (
1299 Expr::struct_from([col!("x"), lit(2), lit(10)]),
1300 "Struct(Column(x), 2, 10)",
1301 ),
1302 (
1303 Expr::array([col!("x"), col!("y"), lit(0)]),
1304 "ARRAY(Column(x), Column(y), 0)",
1305 ),
1306 (
1307 Expr::cast(col!("x"), DataType::DATE),
1308 "CAST(Column(x) AS date)",
1309 ),
1310 ];
1311
1312 for (expr, expected) in cases {
1313 let result = format!("{expr}");
1314 assert_eq!(result, expected);
1315 }
1316 }
1317
1318 #[test]
1319 fn test_predicate_format() {
1320 let cases = [
1321 (column_pred!("x"), "Column(x)"),
1322 (col!("x").eq(lit(2)), "Column(x) = 2"),
1323 ((col!("x") - lit(4)).lt(lit(10)), "Column(x) - 4 < 10"),
1324 (
1325 Pred::and(col!("x").ge(lit(2)), col!("x").le(lit(10))),
1326 "AND(NOT(Column(x) < 2), NOT(Column(x) > 10))",
1327 ),
1328 (
1329 Pred::and_from([
1330 col!("x").ge(lit(2)),
1331 col!("x").le(lit(10)),
1332 col!("x").le(lit(100)),
1333 ]),
1334 "AND(NOT(Column(x) < 2), NOT(Column(x) > 10), NOT(Column(x) > 100))",
1335 ),
1336 (
1337 Pred::or(col!("x").gt(lit(2)), col!("x").lt(lit(10))),
1338 "OR(Column(x) > 2, Column(x) < 10)",
1339 ),
1340 (col!("x").eq(lit("foo")), "Column(x) = 'foo'"),
1341 ];
1342
1343 for (pred, expected) in cases {
1344 let result = format!("{pred}");
1345 assert_eq!(result, expected);
1346 }
1347 }
1348
1349 mod serde_tests {
1352 use std::sync::Arc;
1353
1354 use super::assert_roundtrip;
1355 use crate::expressions::scalars::{ArrayData, DecimalData, MapData, StructData};
1356 use crate::expressions::{
1357 col, column_name, lit, null_lit, BinaryExpressionOp, BinaryPredicateOp, ColumnName,
1358 Expression, ExpressionStructPatchBuilder, Predicate, Scalar, UnaryExpressionOp,
1359 };
1360 use crate::schema::{ArrayType, DataType, DecimalType, MapType, StructField};
1361 use crate::unit_test_utils::assert_result_error_with_message;
1362
1363 #[test]
1366 fn test_literal_scalars_roundtrip() {
1367 let cases: Vec<Expression> = vec![
1369 lit(42i32), lit(9999999999i64), lit(123i16), lit(42i8), lit(1.12345677_32), lit(1.12345667_64), lit("hello world"),
1378 lit(true),
1379 lit(false),
1380 Expression::Literal(Scalar::Timestamp(1234567890000000)),
1382 Expression::Literal(Scalar::TimestampNtz(1234567890000000)),
1383 Expression::Literal(Scalar::Date(19000)),
1384 lit(vec![1u8, 2, 3, 4, 5]),
1386 Expression::Literal(Scalar::Decimal(
1388 DecimalData::try_new(12345i128, DecimalType::try_new(10, 2).unwrap()).unwrap(),
1389 )),
1390 ];
1391
1392 for expr in &cases {
1393 assert_roundtrip(expr);
1394 }
1395 }
1396
1397 #[test]
1398 fn test_literal_complex_scalars_roundtrip() {
1399 let cases: Vec<Expression> = vec![
1401 null_lit(DataType::INTEGER),
1403 null_lit(DataType::STRING),
1404 null_lit(DataType::BOOLEAN),
1405 Expression::Literal(Scalar::Array(
1407 ArrayData::try_new(
1408 ArrayType::new(DataType::INTEGER, false),
1409 vec![Scalar::Integer(1), Scalar::Integer(2), Scalar::Integer(3)],
1410 )
1411 .unwrap(),
1412 )),
1413 Expression::Literal(Scalar::Map(
1415 MapData::try_new(
1416 MapType::new(DataType::STRING, DataType::INTEGER, false),
1417 vec![
1418 (Scalar::String("a".to_string()), Scalar::Integer(1)),
1419 (Scalar::String("b".to_string()), Scalar::Integer(2)),
1420 ],
1421 )
1422 .unwrap(),
1423 )),
1424 Expression::Literal(Scalar::Struct(
1426 StructData::try_new(
1427 vec![
1428 StructField::nullable("x", DataType::INTEGER),
1429 StructField::nullable("y", DataType::STRING),
1430 ],
1431 vec![Scalar::Integer(42), Scalar::String("hello".to_string())],
1432 )
1433 .unwrap(),
1434 )),
1435 ];
1436
1437 for expr in &cases {
1438 assert_roundtrip(expr);
1439 }
1440 }
1441
1442 #[test]
1445 fn test_column_expressions_roundtrip() {
1446 let cases: Vec<Expression> =
1447 vec![col!("my_column"), col!("parent.child"), col!("a.b.c.d")];
1448
1449 for expr in &cases {
1450 assert_roundtrip(expr);
1451 }
1452 }
1453
1454 #[test]
1455 fn test_column_names_roundtrip() {
1456 let cases: Vec<ColumnName> = vec![
1457 column_name!("simple"),
1458 column_name!("a.b.c"),
1459 ColumnName::default(),
1460 ];
1461
1462 for col in &cases {
1463 assert_roundtrip(col);
1464 }
1465 }
1466
1467 #[test]
1470 fn test_unary_expression_roundtrip() {
1471 let expr = Expression::unary(UnaryExpressionOp::ToJson, col!("data"));
1472 assert_roundtrip(&expr);
1473 }
1474
1475 #[test]
1476 fn test_binary_expressions_roundtrip() {
1477 let ops = [
1478 BinaryExpressionOp::Plus,
1479 BinaryExpressionOp::Minus,
1480 BinaryExpressionOp::Multiply,
1481 BinaryExpressionOp::Divide,
1482 ];
1483
1484 for op in ops {
1485 let expr = Expression::binary(op, col!("a"), lit(10));
1486 assert_roundtrip(&expr);
1487 }
1488 }
1489
1490 #[test]
1491 fn test_variadic_expression_roundtrip() {
1492 let expr = Expression::coalesce([col!("a"), col!("b"), lit("default")]);
1493 assert_roundtrip(&expr);
1494 }
1495
1496 #[rstest::rstest]
1497 #[case::column(Expression::cast(col!("part"), DataType::DATE))]
1498 #[case::literal(Expression::cast(lit("2025-01-01"), DataType::DATE))]
1499 #[case::nested(Expression::cast(
1500 Expression::cast(col!("part"), DataType::STRING),
1501 DataType::INTEGER,
1502 ))]
1503 fn test_cast_expression_roundtrip(#[case] expr: Expression) {
1504 assert_roundtrip(&expr);
1505 }
1506
1507 #[rstest::rstest]
1508 #[case::array_single(Expression::array([lit(7i32)]))]
1509 #[case::array_mixed(Expression::array([
1510 col!("a"),
1511 col!("b"),
1512 lit(42i64),
1513 ]))]
1514 fn test_array_expression_roundtrip(#[case] expr: Expression) {
1515 assert_roundtrip(&expr);
1516 }
1517
1518 #[test]
1519 fn test_nested_arithmetic_expression_roundtrip() {
1520 let left = Expression::binary(BinaryExpressionOp::Plus, col!("a"), col!("b"));
1522 let right = Expression::binary(BinaryExpressionOp::Minus, col!("c"), col!("d"));
1523 let mul = Expression::binary(BinaryExpressionOp::Multiply, left, right);
1524 let expr = Expression::binary(BinaryExpressionOp::Divide, mul, lit(2));
1525 assert_roundtrip(&expr);
1526 }
1527
1528 #[test]
1531 fn test_struct_expression_roundtrip() {
1532 let expr = Expression::struct_from([
1533 Arc::new(col!("x")),
1534 Arc::new(lit(42)),
1535 Arc::new(lit("hello")),
1536 ]);
1537 assert_roundtrip(&expr);
1538 }
1539
1540 #[test]
1541 fn test_transform_expressions_roundtrip() {
1542 let cases: Vec<Expression> = vec![
1543 Expression::struct_patch(ExpressionStructPatchBuilder::new()).unwrap(),
1545 Expression::struct_patch(ExpressionStructPatchBuilder::new().drop("old_column"))
1547 .unwrap(),
1548 Expression::struct_patch(
1550 ExpressionStructPatchBuilder::new().replace("original", lit(0)),
1551 )
1552 .unwrap(),
1553 Expression::struct_patch(
1555 ExpressionStructPatchBuilder::new()
1556 .insert_after("after_col", col!("new_col"))
1557 .prepend(lit("prepended"))
1558 .append(lit("appended")),
1559 )
1560 .unwrap(),
1561 Expression::struct_patch(
1563 ExpressionStructPatchBuilder::new_nested(["parent", "child"]).drop("to_drop"),
1564 )
1565 .unwrap(),
1566 ];
1567
1568 for expr in &cases {
1569 assert_roundtrip(expr);
1570 }
1571 }
1572
1573 #[test]
1574 fn test_expression_wrapping_predicate_roundtrip() {
1575 let pred = Predicate::eq(col!("x"), lit(10));
1576 let expr = Expression::from_pred(pred);
1577 assert_roundtrip(&expr);
1578 }
1579
1580 #[test]
1581 fn test_expression_unknown_roundtrip() {
1582 let expr = Expression::unknown("some_unknown_function()");
1583 assert_roundtrip(&expr);
1584 }
1585
1586 #[test]
1587 fn test_map_to_struct_expression_roundtrip() {
1588 let cases: Vec<Expression> = vec![
1589 Expression::map_to_struct(col!("pv")),
1590 Expression::map_to_struct(lit("ignored")),
1591 ];
1592
1593 for expr in &cases {
1594 assert_roundtrip(expr);
1595 }
1596 }
1597
1598 #[test]
1601 fn test_predicate_basics_roundtrip() {
1602 let cases: Vec<Predicate> = vec![
1603 Predicate::from_expr(col!("is_active")),
1605 Predicate::TRUE,
1607 Predicate::FALSE,
1608 Predicate::not(Predicate::from_expr(col!("x"))),
1610 Predicate::not(Predicate::not(Predicate::gt(col!("x"), lit(5)))),
1612 Predicate::unknown("some_unknown_predicate()"),
1614 Predicate::is_null(col!("nullable_col")),
1616 Predicate::is_not_null(col!("nullable_col")),
1617 ];
1618
1619 for pred in &cases {
1620 assert_roundtrip(pred);
1621 }
1622 }
1623
1624 #[test]
1625 fn test_predicate_null_literal_roundtrip() {
1626 assert_roundtrip(&Predicate::NULL);
1627 }
1628
1629 #[test]
1630 fn test_predicate_comparisons_roundtrip() {
1631 let cases: Vec<Predicate> = vec![
1632 Predicate::eq(col!("x"), lit(42)),
1633 Predicate::ne(col!("status"), lit("active")),
1634 Predicate::lt(col!("age"), lit(18)),
1635 Predicate::le(col!("price"), lit(100)),
1636 Predicate::gt(col!("score"), lit(90)),
1637 Predicate::ge(col!("quantity"), lit(1)),
1638 Predicate::distinct(col!("a"), col!("b")),
1639 ];
1640
1641 for pred in &cases {
1642 assert_roundtrip(pred);
1643 }
1644 }
1645
1646 #[test]
1647 fn test_predicate_in_roundtrip() {
1648 let array_data = ArrayData::try_new(
1649 ArrayType::new(DataType::INTEGER, false),
1650 vec![Scalar::Integer(1), Scalar::Integer(2), Scalar::Integer(3)],
1651 )
1652 .unwrap();
1653 let pred = Predicate::binary(
1654 BinaryPredicateOp::In,
1655 col!("x"),
1656 Expression::Literal(Scalar::Array(array_data)),
1657 );
1658 assert_roundtrip(&pred);
1659 }
1660
1661 #[test]
1662 fn test_predicate_junctions_roundtrip() {
1663 let cases: Vec<Predicate> = vec![
1664 Predicate::and(
1666 Predicate::gt(col!("x"), lit(0)),
1667 Predicate::lt(col!("x"), lit(100)),
1668 ),
1669 Predicate::or(
1671 Predicate::eq(col!("status"), lit("active")),
1672 Predicate::eq(col!("status"), lit("pending")),
1673 ),
1674 Predicate::and_from([
1676 Predicate::gt(col!("x"), lit(0)),
1677 Predicate::lt(col!("x"), lit(100)),
1678 Predicate::is_not_null(col!("x")),
1679 ]),
1680 Predicate::or_from([
1682 Predicate::eq(col!("type"), lit("A")),
1683 Predicate::eq(col!("type"), lit("B")),
1684 Predicate::eq(col!("type"), lit("C")),
1685 ]),
1686 Predicate::or(
1688 Predicate::and(
1689 Predicate::gt(col!("a"), lit(0)),
1690 Predicate::lt(col!("b"), lit(100)),
1691 ),
1692 Predicate::eq(col!("c"), lit("special")),
1693 ),
1694 ];
1695
1696 for pred in &cases {
1697 assert_roundtrip(pred);
1698 }
1699 }
1700
1701 #[test]
1704 fn test_deeply_nested_structures_roundtrip() {
1705 let add = Expression::binary(BinaryExpressionOp::Plus, col!("a"), col!("b"));
1707 let mul = Expression::binary(BinaryExpressionOp::Multiply, col!("c"), col!("d"));
1708 let coalesce = Expression::coalesce([add, mul, lit(0)]);
1709 let pred = Predicate::gt(coalesce, lit(100));
1710 assert_roundtrip(&pred);
1711
1712 let inner_pred = Predicate::and(
1714 Predicate::eq(col!("x"), lit(1)),
1715 Predicate::gt(
1716 Expression::binary(BinaryExpressionOp::Plus, col!("y"), col!("z")),
1717 lit(10),
1718 ),
1719 );
1720 let expr = Expression::from_pred(inner_pred);
1721 assert_roundtrip(&expr);
1722 }
1723
1724 #[test]
1727 fn test_opaque_expression_serialize_fails() {
1728 use crate::expressions::{OpaqueExpressionOp, ScalarExpressionEvaluator};
1729 use crate::DeltaResult;
1730
1731 #[derive(Debug, PartialEq)]
1732 struct TestOpaqueExprOp;
1733
1734 impl OpaqueExpressionOp for TestOpaqueExprOp {
1735 fn name(&self) -> &str {
1736 "test_opaque"
1737 }
1738 fn eval_expr_scalar(
1739 &self,
1740 _eval_expr: &ScalarExpressionEvaluator<'_>,
1741 _exprs: &[Expression],
1742 ) -> DeltaResult<Scalar> {
1743 Ok(Scalar::Integer(0))
1744 }
1745 }
1746
1747 let expr = Expression::opaque(TestOpaqueExprOp, [lit(1)]);
1748 let result = serde_json::to_string(&expr);
1749 assert_result_error_with_message(result, "Cannot serialize an Opaque Expression");
1750 }
1751
1752 #[test]
1753 fn test_opaque_predicate_serialize_fails() {
1754 use crate::expressions::{OpaquePredicateOp, ScalarExpressionEvaluator};
1755 use crate::kernel_predicates::{
1756 DirectDataSkippingPredicateEvaluator, DirectPredicateEvaluator,
1757 IndirectDataSkippingPredicateEvaluator,
1758 };
1759 use crate::DeltaResult;
1760
1761 #[derive(Debug, PartialEq)]
1762 struct TestOpaquePredOp;
1763
1764 impl OpaquePredicateOp for TestOpaquePredOp {
1765 fn name(&self) -> &str {
1766 "test_opaque_pred"
1767 }
1768 fn eval_pred_scalar(
1769 &self,
1770 _eval_expr: &ScalarExpressionEvaluator<'_>,
1771 _eval_pred: &DirectPredicateEvaluator<'_>,
1772 _exprs: &[Expression],
1773 _inverted: bool,
1774 ) -> DeltaResult<Option<bool>> {
1775 Ok(Some(true))
1776 }
1777 fn eval_as_data_skipping_predicate(
1778 &self,
1779 _evaluator: &DirectDataSkippingPredicateEvaluator<'_>,
1780 _exprs: &[Expression],
1781 _inverted: bool,
1782 ) -> Option<bool> {
1783 Some(true)
1784 }
1785 fn as_data_skipping_predicate(
1786 &self,
1787 _evaluator: &IndirectDataSkippingPredicateEvaluator<'_>,
1788 _exprs: &[Expression],
1789 _inverted: bool,
1790 ) -> Option<Predicate> {
1791 None
1792 }
1793 }
1794
1795 let pred = Predicate::opaque(TestOpaquePredOp, [lit(1)]);
1796 let result = serde_json::to_string(&pred);
1797 assert_result_error_with_message(result, "Cannot serialize an Opaque Predicate");
1798 }
1799 }
1800
1801 #[test]
1802 fn single_element_and_from_returns_unwrapped_predicate() {
1803 let inner = Pred::gt(col!("x"), lit(0));
1804 let result = Pred::and_from([inner.clone()]);
1805 assert_eq!(result, inner);
1806 }
1807
1808 #[test]
1809 fn single_element_or_from_returns_unwrapped_predicate() {
1810 let inner = Pred::gt(col!("x"), lit(0));
1811 let result = Pred::or_from([inner.clone()]);
1812 assert_eq!(result, inner);
1813 }
1814
1815 #[test]
1816 fn multi_element_and_from_returns_junction() {
1817 let p1 = Pred::gt(col!("x"), lit(0));
1818 let p2 = Pred::lt(col!("x"), lit(100));
1819 let result = Pred::and_from([p1.clone(), p2.clone()]);
1820 assert!(matches!(result, Pred::Junction(ref j) if j.preds.len() == 2));
1821 assert_eq!(result, Pred::and(p1, p2));
1822 }
1823
1824 #[test]
1825 fn empty_and_from_returns_identity_literal() {
1826 let result = Pred::and_from(std::iter::empty());
1827 assert_eq!(result, Pred::TRUE);
1828 }
1829
1830 #[test]
1831 fn empty_or_from_returns_identity_literal() {
1832 let result = Pred::or_from(std::iter::empty());
1833 assert_eq!(result, Pred::FALSE);
1834 }
1835}