datafusion_sql/expr/
mod.rs

1// Licensed to the Apache Software Foundation (ASF) under one
2// or more contributor license agreements.  See the NOTICE file
3// distributed with this work for additional information
4// regarding copyright ownership.  The ASF licenses this file
5// to you under the Apache License, Version 2.0 (the
6// "License"); you may not use this file except in compliance
7// with the License.  You may obtain a copy of the License at
8//
9//   http://www.apache.org/licenses/LICENSE-2.0
10//
11// Unless required by applicable law or agreed to in writing,
12// software distributed under the License is distributed on an
13// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
14// KIND, either express or implied.  See the License for the
15// specific language governing permissions and limitations
16// under the License.
17
18use std::ops::ControlFlow;
19
20use arrow::datatypes::{DataType, TimeUnit};
21use datafusion_expr::planner::{
22    PlannerResult, RawBinaryExpr, RawDictionaryExpr, RawFieldAccessExpr,
23};
24use sqlparser::ast::{
25    AccessExpr, BinaryOperator, CastFormat, CastKind, CeilFloorKind,
26    DataType as SQLDataType, DateTimeField, DictionaryField, Expr as SQLExpr,
27    ExprWithAlias as SQLExprWithAlias, JsonPath, MapEntry, Spanned, StructField,
28    Subscript, TrimWhereField, TypedString, Value, ValueWithSpan,
29};
30use sqlparser::ast::{Query, Visit, Visitor};
31
32use datafusion_common::{
33    DFSchema, Diagnostic, Result, ScalarValue, Span, internal_datafusion_err,
34    internal_err, not_impl_err, plan_err,
35};
36
37use datafusion_expr::expr::ScalarFunction;
38use datafusion_expr::expr::SetQuantifier;
39use datafusion_expr::expr::{InList, WildcardOptions};
40use datafusion_expr::{
41    Between, BinaryExpr, Cast, Expr, ExprSchemable, GetFieldAccess, Like, Literal,
42    Operator, TryCast, lit, when,
43};
44
45use crate::planner::{ContextProvider, PlannerContext, SqlToRel};
46use datafusion_functions_nested::expr_fn::{
47    array_has, array_max, array_min, array_position, cardinality,
48};
49
50mod binary_op;
51mod function;
52mod grouping_set;
53mod identifier;
54mod order_by;
55mod subquery;
56mod substring;
57mod unary_op;
58mod value;
59
60fn null_value_span(expr: &SQLExpr) -> Option<Option<Span>> {
61    if let SQLExpr::Value(ValueWithSpan {
62        value: Value::Null,
63        span,
64    }) = expr
65    {
66        Some(Span::try_from_sqlparser_span(*span))
67    } else {
68        None
69    }
70}
71
72fn null_equality_warning(expr: &SQLExpr) -> Option<Diagnostic> {
73    let SQLExpr::BinaryOp { left, op, right } = expr else {
74        return None;
75    };
76
77    let null_span = null_value_span(left).or_else(|| null_value_span(right))?;
78
79    let (message, help) = match op {
80        BinaryOperator::Eq => (
81            "comparison with NULL using `=` always evaluates to NULL",
82            "use `IS NULL` to check for NULL values",
83        ),
84        BinaryOperator::NotEq => (
85            "comparison with NULL using `<>` always evaluates to NULL",
86            "use `IS NOT NULL` to check for non-NULL values",
87        ),
88        _ => return None,
89    };
90
91    Some(
92        Diagnostic::new_warning(message, Span::try_from_sqlparser_span(expr.span()))
93            .with_help(help, null_span),
94    )
95}
96
97struct NullEqualityPredicateVisitor<'a, 'b, S: ContextProvider> {
98    sql_to_rel: &'a SqlToRel<'b, S>,
99    subquery_depth: usize,
100}
101
102impl<'a, 'b, S: ContextProvider> NullEqualityPredicateVisitor<'a, 'b, S> {
103    fn new(sql_to_rel: &'a SqlToRel<'b, S>) -> Self {
104        Self {
105            sql_to_rel,
106            subquery_depth: 0,
107        }
108    }
109}
110
111impl<S: ContextProvider> Visitor for NullEqualityPredicateVisitor<'_, '_, S> {
112    type Break = ();
113
114    fn pre_visit_query(&mut self, _query: &Query) -> ControlFlow<Self::Break> {
115        self.subquery_depth += 1;
116        ControlFlow::Continue(())
117    }
118
119    fn post_visit_query(&mut self, _query: &Query) -> ControlFlow<Self::Break> {
120        self.subquery_depth -= 1;
121        ControlFlow::Continue(())
122    }
123
124    fn pre_visit_expr(&mut self, expr: &SQLExpr) -> ControlFlow<Self::Break> {
125        if self.subquery_depth == 0
126            && let Some(warning) = null_equality_warning(expr)
127        {
128            self.sql_to_rel.add_warning(warning);
129        }
130        ControlFlow::Continue(())
131    }
132}
133
134impl<S: ContextProvider> SqlToRel<'_, S> {
135    pub(crate) fn warn_on_null_equality_predicate(&self, predicate: &SQLExpr) {
136        let mut visitor = NullEqualityPredicateVisitor::new(self);
137        let _ = predicate.visit(&mut visitor);
138    }
139
140    pub(crate) fn sql_expr_to_logical_expr_with_alias(
141        &self,
142        sql: SQLExprWithAlias,
143        schema: &DFSchema,
144        planner_context: &mut PlannerContext,
145    ) -> Result<Expr> {
146        let mut expr =
147            self.sql_expr_to_logical_expr(sql.expr, schema, planner_context)?;
148        if let Some(alias) = sql.alias {
149            expr = expr.alias(alias.value);
150        }
151        Ok(expr)
152    }
153    pub(crate) fn sql_expr_to_logical_expr(
154        &self,
155        sql: SQLExpr,
156        schema: &DFSchema,
157        planner_context: &mut PlannerContext,
158    ) -> Result<Expr> {
159        enum StackEntry {
160            SQLExpr(Box<SQLExpr>),
161            Operator(BinaryOperator),
162        }
163
164        // Virtual stack machine to convert SQLExpr to Expr
165        // This allows visiting the expr tree in a depth-first manner which
166        // produces expressions in postfix notations, i.e. `a + b` => `a b +`.
167        // See https://github.com/apache/datafusion/issues/1444
168        let mut stack = vec![StackEntry::SQLExpr(Box::new(sql))];
169        let mut eval_stack = vec![];
170
171        while let Some(entry) = stack.pop() {
172            match entry {
173                StackEntry::SQLExpr(sql_expr) => {
174                    match *sql_expr {
175                        SQLExpr::BinaryOp { left, op, right } => {
176                            // Note the order that we push the entries to the stack
177                            // is important. We want to visit the left node first.
178                            stack.push(StackEntry::Operator(op));
179                            stack.push(StackEntry::SQLExpr(right));
180                            stack.push(StackEntry::SQLExpr(left));
181                        }
182                        _ => {
183                            let expr = self.sql_expr_to_logical_expr_internal(
184                                *sql_expr,
185                                schema,
186                                planner_context,
187                            )?;
188                            eval_stack.push(expr);
189                        }
190                    }
191                }
192                StackEntry::Operator(op) => {
193                    let right = eval_stack.pop().unwrap();
194                    let left = eval_stack.pop().unwrap();
195                    let expr = self.build_logical_expr(op, left, right, schema)?;
196                    eval_stack.push(expr);
197                }
198            }
199        }
200
201        assert_eq!(1, eval_stack.len());
202        let expr = eval_stack.pop().unwrap();
203        Ok(expr)
204    }
205
206    fn build_logical_expr(
207        &self,
208        op: BinaryOperator,
209        left: Expr,
210        right: Expr,
211        schema: &DFSchema,
212    ) -> Result<Expr> {
213        // try extension planers
214        let mut binary_expr = RawBinaryExpr { op, left, right };
215        for planner in self.context_provider.get_expr_planners() {
216            match planner.plan_binary_op(binary_expr, schema)? {
217                PlannerResult::Planned(expr) => {
218                    return Ok(expr);
219                }
220                PlannerResult::Original(expr) => {
221                    binary_expr = expr;
222                }
223            }
224        }
225
226        let RawBinaryExpr { op, left, right } = binary_expr;
227        self.build_binary_expr(&op, left, right)
228    }
229
230    pub fn sql_to_expr_with_alias(
231        &self,
232        sql: SQLExprWithAlias,
233        schema: &DFSchema,
234        planner_context: &mut PlannerContext,
235    ) -> Result<Expr> {
236        let mut expr =
237            self.sql_expr_to_logical_expr_with_alias(sql, schema, planner_context)?;
238        expr = self.rewrite_partial_qualifier(expr, schema);
239        self.validate_schema_satisfies_exprs(schema, &[expr.clone()])?;
240        let (expr, _) = expr.infer_placeholder_types(schema)?;
241        Ok(expr)
242    }
243
244    /// Generate a relational expression from a SQL expression
245    pub fn sql_to_expr(
246        &self,
247        sql: SQLExpr,
248        schema: &DFSchema,
249        planner_context: &mut PlannerContext,
250    ) -> Result<Expr> {
251        // The location of the original SQL expression in the source code
252        let mut expr = self.sql_expr_to_logical_expr(sql, schema, planner_context)?;
253        expr = self.rewrite_partial_qualifier(expr, schema);
254        self.validate_schema_satisfies_exprs(schema, std::slice::from_ref(&expr))?;
255        let (expr, _) = expr.infer_placeholder_types(schema)?;
256        Ok(expr)
257    }
258
259    /// Rewrite aliases which are not-complete (e.g. ones that only include only table qualifier in a schema.table qualified relation)
260    fn rewrite_partial_qualifier(&self, expr: Expr, schema: &DFSchema) -> Expr {
261        match expr {
262            Expr::Column(col) => match &col.relation {
263                Some(q) => {
264                    match schema.iter().find(|(qualifier, field)| match qualifier {
265                        Some(field_q) => {
266                            field.name() == &col.name
267                                && field_q.to_string().ends_with(&format!(".{q}"))
268                        }
269                        _ => false,
270                    }) {
271                        Some((qualifier, df_field)) => Expr::from((qualifier, df_field)),
272                        None => Expr::Column(col),
273                    }
274                }
275                None => Expr::Column(col),
276            },
277            _ => expr,
278        }
279    }
280
281    /// Internal implementation. Use
282    /// [`Self::sql_expr_to_logical_expr`] to plan exprs.
283    #[cfg_attr(feature = "recursive_protection", recursive::recursive)]
284    fn sql_expr_to_logical_expr_internal(
285        &self,
286        sql: SQLExpr,
287        schema: &DFSchema,
288        planner_context: &mut PlannerContext,
289    ) -> Result<Expr> {
290        // NOTE: This function is called recursively, so each match arm body should be as
291        //       small as possible to decrease stack requirement.
292        //       Follow the common pattern of extracting into a separate function for
293        //       non-trivial arms. See https://github.com/apache/datafusion/pull/12384 for
294        //       more context.
295        match sql {
296            SQLExpr::Value(value) => {
297                self.parse_value(value.into(), planner_context.prepare_param_data_types())
298            }
299            SQLExpr::Extract { field, expr, .. } => {
300                let mut extract_args = vec![
301                    Expr::Literal(ScalarValue::from(format!("{field}")), None),
302                    self.sql_expr_to_logical_expr(*expr, schema, planner_context)?,
303                ];
304
305                for planner in self.context_provider.get_expr_planners() {
306                    match planner.plan_extract(extract_args)? {
307                        PlannerResult::Planned(expr) => return Ok(expr),
308                        PlannerResult::Original(args) => {
309                            extract_args = args;
310                        }
311                    }
312                }
313
314                not_impl_err!("Extract not supported by ExprPlanner: {extract_args:?}")
315            }
316
317            SQLExpr::Array(arr) => self.sql_array_literal(arr.elem, schema),
318            SQLExpr::Interval(interval) => self.sql_interval_to_expr(false, interval),
319            SQLExpr::Identifier(id) => {
320                self.sql_identifier_to_expr(id, schema, planner_context)
321            }
322
323            // <expr>["foo"], <expr>[4] or <expr>[4:5]
324            SQLExpr::CompoundFieldAccess { root, access_chain } => self
325                .sql_compound_field_access_to_expr(
326                    *root,
327                    access_chain,
328                    schema,
329                    planner_context,
330                ),
331
332            SQLExpr::CompoundIdentifier(ids) => {
333                self.sql_compound_identifier_to_expr(ids, schema, planner_context)
334            }
335
336            SQLExpr::Case {
337                operand,
338                conditions,
339                else_result,
340                case_token: _,
341                end_token: _,
342            } => self.sql_case_identifier_to_expr(
343                operand,
344                conditions,
345                else_result,
346                schema,
347                planner_context,
348            ),
349
350            SQLExpr::Cast { array: true, .. } => {
351                not_impl_err!("`CAST(... AS type ARRAY`) not supported")
352            }
353
354            SQLExpr::Cast {
355                kind: CastKind::Cast | CastKind::DoubleColon,
356                expr,
357                data_type,
358                format,
359                array: false,
360            } => {
361                self.sql_cast_to_expr(*expr, &data_type, format, schema, planner_context)
362            }
363
364            SQLExpr::Cast {
365                kind: CastKind::TryCast | CastKind::SafeCast,
366                expr,
367                data_type,
368                format,
369                array: false,
370            } => {
371                if let Some(format) = format {
372                    return not_impl_err!("CAST with format is not supported: {format}");
373                }
374
375                Ok(Expr::TryCast(TryCast::new_from_field(
376                    Box::new(self.sql_expr_to_logical_expr(
377                        *expr,
378                        schema,
379                        planner_context,
380                    )?),
381                    self.convert_data_type_to_field(&data_type)?,
382                )))
383            }
384
385            SQLExpr::TypedString(TypedString {
386                data_type,
387                value,
388                uses_odbc_syntax: _,
389            }) => {
390                let value = match value.into_string() {
391                    Some(value) => value,
392                    None => {
393                        return plan_err!("Typed literal requires a string payload");
394                    }
395                };
396
397                Ok(Expr::Cast(Cast::new_from_field(
398                    Box::new(lit(value)),
399                    self.convert_data_type_to_field(&data_type)?,
400                )))
401            }
402
403            SQLExpr::IsNull(expr) => Ok(Expr::IsNull(Box::new(
404                self.sql_expr_to_logical_expr(*expr, schema, planner_context)?,
405            ))),
406
407            SQLExpr::IsNotNull(expr) => Ok(Expr::IsNotNull(Box::new(
408                self.sql_expr_to_logical_expr(*expr, schema, planner_context)?,
409            ))),
410
411            SQLExpr::IsDistinctFrom(left, right) => {
412                Ok(Expr::BinaryExpr(BinaryExpr::new(
413                    Box::new(self.sql_expr_to_logical_expr(
414                        *left,
415                        schema,
416                        planner_context,
417                    )?),
418                    Operator::IsDistinctFrom,
419                    Box::new(self.sql_expr_to_logical_expr(
420                        *right,
421                        schema,
422                        planner_context,
423                    )?),
424                )))
425            }
426
427            SQLExpr::IsNotDistinctFrom(left, right) => {
428                Ok(Expr::BinaryExpr(BinaryExpr::new(
429                    Box::new(self.sql_expr_to_logical_expr(
430                        *left,
431                        schema,
432                        planner_context,
433                    )?),
434                    Operator::IsNotDistinctFrom,
435                    Box::new(self.sql_expr_to_logical_expr(
436                        *right,
437                        schema,
438                        planner_context,
439                    )?),
440                )))
441            }
442
443            SQLExpr::IsTrue(expr) => Ok(Expr::IsTrue(Box::new(
444                self.sql_expr_to_logical_expr(*expr, schema, planner_context)?,
445            ))),
446
447            SQLExpr::IsFalse(expr) => Ok(Expr::IsFalse(Box::new(
448                self.sql_expr_to_logical_expr(*expr, schema, planner_context)?,
449            ))),
450
451            SQLExpr::IsNotTrue(expr) => Ok(Expr::IsNotTrue(Box::new(
452                self.sql_expr_to_logical_expr(*expr, schema, planner_context)?,
453            ))),
454
455            SQLExpr::IsNotFalse(expr) => Ok(Expr::IsNotFalse(Box::new(
456                self.sql_expr_to_logical_expr(*expr, schema, planner_context)?,
457            ))),
458
459            SQLExpr::IsUnknown(expr) => Ok(Expr::IsUnknown(Box::new(
460                self.sql_expr_to_logical_expr(*expr, schema, planner_context)?,
461            ))),
462
463            SQLExpr::IsNotUnknown(expr) => Ok(Expr::IsNotUnknown(Box::new(
464                self.sql_expr_to_logical_expr(*expr, schema, planner_context)?,
465            ))),
466
467            SQLExpr::UnaryOp { op, expr } => {
468                self.parse_sql_unary_op(op, *expr, schema, planner_context)
469            }
470
471            SQLExpr::Between {
472                expr,
473                negated,
474                low,
475                high,
476            } => Ok(Expr::Between(Between::new(
477                Box::new(self.sql_expr_to_logical_expr(
478                    *expr,
479                    schema,
480                    planner_context,
481                )?),
482                negated,
483                Box::new(self.sql_expr_to_logical_expr(*low, schema, planner_context)?),
484                Box::new(self.sql_expr_to_logical_expr(
485                    *high,
486                    schema,
487                    planner_context,
488                )?),
489            ))),
490
491            SQLExpr::InList {
492                expr,
493                list,
494                negated,
495            } => self.sql_in_list_to_expr(*expr, list, negated, schema, planner_context),
496
497            SQLExpr::Like {
498                negated,
499                expr,
500                pattern,
501                escape_char,
502                any,
503            } => self.sql_like_to_expr(
504                negated,
505                *expr,
506                *pattern,
507                escape_char,
508                schema,
509                planner_context,
510                false,
511                any,
512            ),
513
514            SQLExpr::ILike {
515                negated,
516                expr,
517                pattern,
518                escape_char,
519                any,
520            } => self.sql_like_to_expr(
521                negated,
522                *expr,
523                *pattern,
524                escape_char,
525                schema,
526                planner_context,
527                true,
528                any,
529            ),
530
531            SQLExpr::SimilarTo {
532                negated,
533                expr,
534                pattern,
535                escape_char,
536            } => self.sql_similarto_to_expr(
537                negated,
538                *expr,
539                *pattern,
540                escape_char,
541                schema,
542                planner_context,
543            ),
544
545            SQLExpr::BinaryOp { .. } => {
546                internal_err!("binary_op should be handled by sql_expr_to_logical_expr.")
547            }
548
549            #[cfg(feature = "unicode_expressions")]
550            SQLExpr::Substring {
551                expr,
552                substring_from,
553                substring_for,
554                special: _,
555                shorthand: _,
556            } => self.sql_substring_to_expr(
557                expr,
558                substring_from,
559                substring_for,
560                schema,
561                planner_context,
562            ),
563
564            #[cfg(not(feature = "unicode_expressions"))]
565            SQLExpr::Substring { .. } => {
566                internal_err!(
567                    "statement substring requires compilation with feature flag: unicode_expressions."
568                )
569            }
570
571            SQLExpr::Trim {
572                expr,
573                trim_where,
574                trim_what,
575                trim_characters,
576            } => self.sql_trim_to_expr(
577                *expr,
578                trim_where,
579                trim_what,
580                trim_characters,
581                schema,
582                planner_context,
583            ),
584
585            SQLExpr::Function(function) => {
586                self.sql_function_to_expr(function, schema, planner_context)
587            }
588
589            SQLExpr::Rollup(exprs) => {
590                self.sql_rollup_to_expr(exprs, schema, planner_context)
591            }
592            SQLExpr::Cube(exprs) => self.sql_cube_to_expr(exprs, schema, planner_context),
593            SQLExpr::GroupingSets(exprs) => {
594                self.sql_grouping_sets_to_expr(exprs, schema, planner_context)
595            }
596
597            SQLExpr::Floor { expr, field } => match field {
598                CeilFloorKind::DateTimeField(DateTimeField::NoDateTime) => {
599                    self.sql_fn_name_to_expr(*expr, "floor", schema, planner_context)
600                }
601                CeilFloorKind::DateTimeField(_) => {
602                    not_impl_err!("FLOOR with datetime is not supported")
603                }
604                CeilFloorKind::Scale(_) => {
605                    not_impl_err!("FLOOR with scale is not supported")
606                }
607            },
608            SQLExpr::Ceil { expr, field } => match field {
609                CeilFloorKind::DateTimeField(DateTimeField::NoDateTime) => {
610                    self.sql_fn_name_to_expr(*expr, "ceil", schema, planner_context)
611                }
612                CeilFloorKind::DateTimeField(_) => {
613                    not_impl_err!("CEIL with datetime is not supported")
614                }
615                CeilFloorKind::Scale(_) => {
616                    not_impl_err!("CEIL with scale is not supported")
617                }
618            },
619            SQLExpr::Overlay {
620                expr,
621                overlay_what,
622                overlay_from,
623                overlay_for,
624            } => self.sql_overlay_to_expr(
625                *expr,
626                *overlay_what,
627                *overlay_from,
628                overlay_for,
629                schema,
630                planner_context,
631            ),
632            SQLExpr::Nested(e) => {
633                self.sql_expr_to_logical_expr(*e, schema, planner_context)
634            }
635
636            SQLExpr::Exists { subquery, negated } => {
637                self.parse_exists_subquery(*subquery, negated, schema, planner_context)
638            }
639            SQLExpr::InSubquery {
640                expr,
641                subquery,
642                negated,
643            } => {
644                self.parse_in_subquery(*expr, *subquery, negated, schema, planner_context)
645            }
646            SQLExpr::Subquery(subquery) => {
647                self.parse_scalar_subquery(*subquery, schema, planner_context)
648            }
649
650            SQLExpr::Struct { values, fields } => {
651                self.parse_struct(schema, planner_context, values, &fields)
652            }
653            SQLExpr::Position { expr, r#in } => {
654                self.sql_position_to_expr(*expr, *r#in, schema, planner_context)
655            }
656            SQLExpr::AtTimeZone {
657                timestamp,
658                time_zone,
659            } => Ok(Expr::Cast(Cast::new(
660                Box::new(self.sql_expr_to_logical_expr_internal(
661                    *timestamp,
662                    schema,
663                    planner_context,
664                )?),
665                match *time_zone {
666                    SQLExpr::Value(ValueWithSpan {
667                        value: Value::SingleQuotedString(s),
668                        span: _,
669                    }) => DataType::Timestamp(TimeUnit::Nanosecond, Some(s.into())),
670                    _ => {
671                        return not_impl_err!(
672                            "Unsupported ast node in sqltorel: {time_zone:?}"
673                        );
674                    }
675                },
676            ))),
677            SQLExpr::Dictionary(fields) => {
678                self.try_plan_dictionary_literal(fields, schema, planner_context)
679            }
680            SQLExpr::Map(map) => {
681                self.try_plan_map_literal(map.entries, schema, planner_context)
682            }
683            SQLExpr::AnyOp {
684                left,
685                compare_op,
686                right,
687                // ANY/SOME are equivalent, this field specifies which the user
688                // specified but it doesn't affect the plan so ignore the field
689                is_some: _,
690            } => match *right {
691                SQLExpr::Subquery(subquery) => self.parse_set_comparison_subquery(
692                    *left,
693                    *subquery,
694                    &compare_op,
695                    SetQuantifier::Any,
696                    schema,
697                    planner_context,
698                ),
699                _ => {
700                    let left_expr = self.sql_to_expr(*left, schema, planner_context)?;
701                    let right_expr = self.sql_to_expr(*right, schema, planner_context)?;
702                    plan_any_op(left_expr, right_expr, &compare_op)
703                }
704            },
705            SQLExpr::AllOp {
706                left,
707                compare_op,
708                right,
709            } => match *right {
710                SQLExpr::Subquery(subquery) => self.parse_set_comparison_subquery(
711                    *left,
712                    *subquery,
713                    &compare_op,
714                    SetQuantifier::All,
715                    schema,
716                    planner_context,
717                ),
718                _ => {
719                    let left_expr = self.sql_to_expr(*left, schema, planner_context)?;
720                    let right_expr = self.sql_to_expr(*right, schema, planner_context)?;
721                    plan_all_op(&left_expr, &right_expr, &compare_op)
722                }
723            },
724            #[expect(deprecated)]
725            SQLExpr::Wildcard(_token) => Ok(Expr::Wildcard {
726                qualifier: None,
727                options: Box::new(WildcardOptions::default()),
728            }),
729            #[expect(deprecated)]
730            SQLExpr::QualifiedWildcard(object_name, _token) => Ok(Expr::Wildcard {
731                qualifier: Some(self.object_name_to_table_reference(object_name)?),
732                options: Box::new(WildcardOptions::default()),
733            }),
734            SQLExpr::Tuple(values) => self.parse_tuple(schema, planner_context, values),
735            SQLExpr::JsonAccess { value, path } => {
736                self.parse_json_access(schema, planner_context, value, &path)
737            }
738            _ => not_impl_err!("Unsupported ast node in sqltorel: {sql:?}"),
739        }
740    }
741
742    fn parse_json_access(
743        &self,
744        schema: &DFSchema,
745        planner_context: &mut PlannerContext,
746        value: Box<SQLExpr>,
747        path: &JsonPath,
748    ) -> Result<Expr> {
749        let json_path = path.to_string();
750        let json_path = if let Some(json_path) = json_path.strip_prefix(":") {
751            // sqlparser's JsonPath display adds an extra `:` at the beginning.
752            json_path.to_owned()
753        } else {
754            json_path
755        };
756        self.build_logical_expr(
757            BinaryOperator::Custom(":".to_owned()),
758            self.sql_to_expr(*value, schema, planner_context)?,
759            // pass json path as a string literal, let the impl parse it when needed.
760            Expr::Literal(ScalarValue::Utf8(Some(json_path)), None),
761            schema,
762        )
763    }
764
765    /// Parses a struct(..) expression and plans it creation
766    fn parse_struct(
767        &self,
768        schema: &DFSchema,
769        planner_context: &mut PlannerContext,
770        values: Vec<SQLExpr>,
771        fields: &[StructField],
772    ) -> Result<Expr> {
773        if !fields.is_empty() {
774            return not_impl_err!("Struct fields are not supported yet");
775        }
776        let is_named_struct = values
777            .iter()
778            .any(|value| matches!(value, SQLExpr::Named { .. }));
779
780        let mut create_struct_args = if is_named_struct {
781            self.create_named_struct_expr(values, schema, planner_context)?
782        } else {
783            self.create_struct_expr(values, schema, planner_context)?
784        };
785
786        for planner in self.context_provider.get_expr_planners() {
787            match planner.plan_struct_literal(create_struct_args, is_named_struct)? {
788                PlannerResult::Planned(expr) => return Ok(expr),
789                PlannerResult::Original(args) => create_struct_args = args,
790            }
791        }
792        not_impl_err!("Struct not supported by ExprPlanner: {create_struct_args:?}")
793    }
794
795    fn parse_tuple(
796        &self,
797        schema: &DFSchema,
798        planner_context: &mut PlannerContext,
799        values: Vec<SQLExpr>,
800    ) -> Result<Expr> {
801        match values.first() {
802            Some(SQLExpr::Identifier(_))
803            | Some(SQLExpr::Value(_))
804            | Some(SQLExpr::CompoundIdentifier(_)) => {
805                self.parse_struct(schema, planner_context, values, &[])
806            }
807            None => not_impl_err!("Empty tuple not supported yet"),
808            _ => {
809                not_impl_err!("Only identifiers and literals are supported in tuples")
810            }
811        }
812    }
813
814    fn sql_position_to_expr(
815        &self,
816        substr_expr: SQLExpr,
817        str_expr: SQLExpr,
818        schema: &DFSchema,
819        planner_context: &mut PlannerContext,
820    ) -> Result<Expr> {
821        let substr =
822            self.sql_expr_to_logical_expr(substr_expr, schema, planner_context)?;
823        let fullstr = self.sql_expr_to_logical_expr(str_expr, schema, planner_context)?;
824        let mut position_args = vec![fullstr, substr];
825        for planner in self.context_provider.get_expr_planners() {
826            match planner.plan_position(position_args)? {
827                PlannerResult::Planned(expr) => return Ok(expr),
828                PlannerResult::Original(args) => {
829                    position_args = args;
830                }
831            }
832        }
833
834        not_impl_err!("Position not supported by ExprPlanner: {position_args:?}")
835    }
836
837    fn try_plan_dictionary_literal(
838        &self,
839        fields: Vec<DictionaryField>,
840        schema: &DFSchema,
841        planner_context: &mut PlannerContext,
842    ) -> Result<Expr> {
843        let mut keys = vec![];
844        let mut values = vec![];
845        for field in fields {
846            let key = lit(field.key.value);
847            let value =
848                self.sql_expr_to_logical_expr(*field.value, schema, planner_context)?;
849            keys.push(key);
850            values.push(value);
851        }
852
853        let mut raw_expr = RawDictionaryExpr { keys, values };
854
855        for planner in self.context_provider.get_expr_planners() {
856            match planner.plan_dictionary_literal(raw_expr, schema)? {
857                PlannerResult::Planned(expr) => {
858                    return Ok(expr);
859                }
860                PlannerResult::Original(expr) => raw_expr = expr,
861            }
862        }
863        not_impl_err!("Dictionary not supported by ExprPlanner: {raw_expr:?}")
864    }
865
866    fn try_plan_map_literal(
867        &self,
868        entries: Vec<MapEntry>,
869        schema: &DFSchema,
870        planner_context: &mut PlannerContext,
871    ) -> Result<Expr> {
872        let mut exprs: Vec<_> = entries
873            .into_iter()
874            .flat_map(|entry| vec![entry.key, entry.value].into_iter())
875            .map(|expr| self.sql_expr_to_logical_expr(*expr, schema, planner_context))
876            .collect::<Result<Vec<_>>>()?;
877        for planner in self.context_provider.get_expr_planners() {
878            match planner.plan_make_map(exprs)? {
879                PlannerResult::Planned(expr) => {
880                    return Ok(expr);
881                }
882                PlannerResult::Original(expr) => exprs = expr,
883            }
884        }
885        not_impl_err!("MAP not supported by ExprPlanner: {exprs:?}")
886    }
887
888    // Handles a call to struct(...) where the arguments are named. For example
889    // `struct (v as foo, v2 as bar)` by creating a call to the `named_struct` function
890    fn create_named_struct_expr(
891        &self,
892        values: Vec<SQLExpr>,
893        input_schema: &DFSchema,
894        planner_context: &mut PlannerContext,
895    ) -> Result<Vec<Expr>> {
896        Ok(values
897            .into_iter()
898            .enumerate()
899            .map(|(i, value)| {
900                let args = if let SQLExpr::Named { expr, name } = value {
901                    [
902                        name.value.lit(),
903                        self.sql_expr_to_logical_expr(
904                            *expr,
905                            input_schema,
906                            planner_context,
907                        )?,
908                    ]
909                } else {
910                    [
911                        format!("c{i}").lit(),
912                        self.sql_expr_to_logical_expr(
913                            value,
914                            input_schema,
915                            planner_context,
916                        )?,
917                    ]
918                };
919
920                Ok(args)
921            })
922            .collect::<Result<Vec<_>>>()?
923            .into_iter()
924            .flatten()
925            .collect())
926    }
927
928    // Handles a call to struct(...) where the arguments are not named. For example
929    // `struct (v, v2)` by creating a call to the `struct` function
930    // which will create a struct with fields named `c0`, `c1`, etc.
931    fn create_struct_expr(
932        &self,
933        values: Vec<SQLExpr>,
934        input_schema: &DFSchema,
935        planner_context: &mut PlannerContext,
936    ) -> Result<Vec<Expr>> {
937        values
938            .into_iter()
939            .map(|value| {
940                self.sql_expr_to_logical_expr(value, input_schema, planner_context)
941            })
942            .collect::<Result<Vec<_>>>()
943    }
944
945    fn sql_in_list_to_expr(
946        &self,
947        expr: SQLExpr,
948        list: Vec<SQLExpr>,
949        negated: bool,
950        schema: &DFSchema,
951        planner_context: &mut PlannerContext,
952    ) -> Result<Expr> {
953        let list_expr = list
954            .into_iter()
955            .map(|e| self.sql_expr_to_logical_expr(e, schema, planner_context))
956            .collect::<Result<Vec<_>>>()?;
957
958        Ok(Expr::InList(InList::new(
959            Box::new(self.sql_expr_to_logical_expr(expr, schema, planner_context)?),
960            list_expr,
961            negated,
962        )))
963    }
964
965    #[expect(clippy::too_many_arguments)]
966    fn sql_like_to_expr(
967        &self,
968        negated: bool,
969        expr: SQLExpr,
970        pattern: SQLExpr,
971        escape_char: Option<ValueWithSpan>,
972        schema: &DFSchema,
973        planner_context: &mut PlannerContext,
974        case_insensitive: bool,
975        any: bool,
976    ) -> Result<Expr> {
977        if any {
978            return not_impl_err!("ANY in LIKE expression");
979        }
980        let pattern = self.sql_expr_to_logical_expr(pattern, schema, planner_context)?;
981        let escape_char = match escape_char.map(|v| v.value) {
982            Some(Value::SingleQuotedString(char)) if char.len() == 1 => {
983                Some(char.chars().next().unwrap())
984            }
985            Some(value) => {
986                return plan_err!(
987                    "Invalid escape character in LIKE expression. Expected a single character wrapped with single quotes, got {value}"
988                );
989            }
990            None => None,
991        };
992        Ok(Expr::Like(Like::new(
993            negated,
994            Box::new(self.sql_expr_to_logical_expr(expr, schema, planner_context)?),
995            Box::new(pattern),
996            escape_char,
997            case_insensitive,
998        )))
999    }
1000
1001    fn sql_similarto_to_expr(
1002        &self,
1003        negated: bool,
1004        expr: SQLExpr,
1005        pattern: SQLExpr,
1006        escape_char: Option<ValueWithSpan>,
1007        schema: &DFSchema,
1008        planner_context: &mut PlannerContext,
1009    ) -> Result<Expr> {
1010        let pattern = self.sql_expr_to_logical_expr(pattern, schema, planner_context)?;
1011        let escape_char = match escape_char.map(|v| v.value) {
1012            Some(Value::SingleQuotedString(char)) if char.len() == 1 => {
1013                Some(char.chars().next().unwrap())
1014            }
1015            Some(value) => {
1016                return plan_err!(
1017                    "Invalid escape character in SIMILAR TO expression. Expected a single character wrapped with single quotes, got {value}"
1018                );
1019            }
1020            None => None,
1021        };
1022        Ok(Expr::SimilarTo(Like::new(
1023            negated,
1024            Box::new(self.sql_expr_to_logical_expr(expr, schema, planner_context)?),
1025            Box::new(pattern),
1026            escape_char,
1027            false,
1028        )))
1029    }
1030
1031    fn sql_trim_to_expr(
1032        &self,
1033        expr: SQLExpr,
1034        trim_where: Option<TrimWhereField>,
1035        trim_what: Option<Box<SQLExpr>>,
1036        trim_characters: Option<Vec<SQLExpr>>,
1037        schema: &DFSchema,
1038        planner_context: &mut PlannerContext,
1039    ) -> Result<Expr> {
1040        let arg = self.sql_expr_to_logical_expr(expr, schema, planner_context)?;
1041        let args = match (trim_what, trim_characters) {
1042            (Some(to_trim), None) => {
1043                let to_trim =
1044                    self.sql_expr_to_logical_expr(*to_trim, schema, planner_context)?;
1045                Ok(vec![arg, to_trim])
1046            }
1047            (None, Some(trim_characters)) => {
1048                if let Some(first) = trim_characters.first() {
1049                    let to_trim = self.sql_expr_to_logical_expr(
1050                        first.clone(),
1051                        schema,
1052                        planner_context,
1053                    )?;
1054                    Ok(vec![arg, to_trim])
1055                } else {
1056                    plan_err!("TRIM CHARACTERS cannot be empty")
1057                }
1058            }
1059            (Some(_), Some(_)) => {
1060                plan_err!("Both TRIM and TRIM CHARACTERS cannot be specified")
1061            }
1062            (None, None) => Ok(vec![arg]),
1063        }?;
1064
1065        let fun_name = match trim_where {
1066            Some(TrimWhereField::Leading) => "ltrim",
1067            Some(TrimWhereField::Trailing) => "rtrim",
1068            Some(TrimWhereField::Both) => "btrim",
1069            None => "trim",
1070        };
1071        let fun = self
1072            .context_provider
1073            .get_function_meta(fun_name)
1074            .ok_or_else(|| {
1075                internal_datafusion_err!("Unable to find expected '{fun_name}' function")
1076            })?;
1077
1078        Ok(Expr::ScalarFunction(ScalarFunction::new_udf(fun, args)))
1079    }
1080
1081    fn sql_overlay_to_expr(
1082        &self,
1083        expr: SQLExpr,
1084        overlay_what: SQLExpr,
1085        overlay_from: SQLExpr,
1086        overlay_for: Option<Box<SQLExpr>>,
1087        schema: &DFSchema,
1088        planner_context: &mut PlannerContext,
1089    ) -> Result<Expr> {
1090        let arg = self.sql_expr_to_logical_expr(expr, schema, planner_context)?;
1091        let what_arg =
1092            self.sql_expr_to_logical_expr(overlay_what, schema, planner_context)?;
1093        let from_arg =
1094            self.sql_expr_to_logical_expr(overlay_from, schema, planner_context)?;
1095        let mut overlay_args = match overlay_for {
1096            Some(for_expr) => {
1097                let for_expr =
1098                    self.sql_expr_to_logical_expr(*for_expr, schema, planner_context)?;
1099                vec![arg, what_arg, from_arg, for_expr]
1100            }
1101            None => vec![arg, what_arg, from_arg],
1102        };
1103        for planner in self.context_provider.get_expr_planners() {
1104            match planner.plan_overlay(overlay_args)? {
1105                PlannerResult::Planned(expr) => return Ok(expr),
1106                PlannerResult::Original(args) => overlay_args = args,
1107            }
1108        }
1109        not_impl_err!("Overlay not supported by ExprPlanner: {overlay_args:?}")
1110    }
1111
1112    fn sql_cast_to_expr(
1113        &self,
1114        expr: SQLExpr,
1115        data_type: &SQLDataType,
1116        format: Option<CastFormat>,
1117        schema: &DFSchema,
1118        planner_context: &mut PlannerContext,
1119    ) -> Result<Expr> {
1120        if let Some(format) = format {
1121            return not_impl_err!("CAST with format is not supported: {format}");
1122        }
1123
1124        let dt = self.convert_data_type_to_field(data_type)?;
1125        let expr = self.sql_expr_to_logical_expr(expr, schema, planner_context)?;
1126
1127        // numeric constants are treated as seconds (rather as nanoseconds)
1128        // to align with postgres / duckdb semantics
1129        let expr = match dt.data_type() {
1130            DataType::Timestamp(TimeUnit::Nanosecond, tz)
1131                if expr.get_type(schema)? == DataType::Int64 =>
1132            {
1133                Expr::Cast(Cast::new(
1134                    Box::new(expr),
1135                    DataType::Timestamp(TimeUnit::Second, tz.clone()),
1136                ))
1137            }
1138            _ => expr,
1139        };
1140
1141        Ok(Expr::Cast(Cast::new_from_field(Box::new(expr), dt)))
1142    }
1143
1144    /// Extracts the root expression and access chain from a compound expression.
1145    ///
1146    /// This function attempts to identify if a compound expression (like `a.b.c`) should be treated
1147    /// as a column reference with a qualifier (like `table.column`) or as a field access expression.
1148    ///
1149    /// # Arguments
1150    ///
1151    /// * `root` - The root SQL expression (e.g., the first part of `a.b.c`)
1152    /// * `access_chain` - Vector of access expressions (e.g., `.b` and `.c` parts)
1153    /// * `schema` - The schema to resolve column references against
1154    /// * `planner_context` - Context for planning expressions
1155    ///
1156    /// # Returns
1157    ///
1158    /// A tuple containing:
1159    /// * The resolved root expression
1160    /// * The remaining access chain that should be processed as field accesses
1161    fn extract_root_and_access_chain(
1162        &self,
1163        root: SQLExpr,
1164        mut access_chain: Vec<AccessExpr>,
1165        schema: &DFSchema,
1166        planner_context: &mut PlannerContext,
1167    ) -> Result<(Expr, Vec<AccessExpr>)> {
1168        let SQLExpr::Identifier(root_ident) = root else {
1169            let root = self.sql_expr_to_logical_expr(root, schema, planner_context)?;
1170            return Ok((root, access_chain));
1171        };
1172
1173        let mut compound_idents = vec![root_ident];
1174        let first_non_ident = access_chain
1175            .iter()
1176            .position(|access| !matches!(access, AccessExpr::Dot(SQLExpr::Identifier(_))))
1177            .unwrap_or(access_chain.len());
1178        for access in access_chain.drain(0..first_non_ident) {
1179            if let AccessExpr::Dot(SQLExpr::Identifier(ident)) = access {
1180                compound_idents.push(ident);
1181            } else {
1182                return internal_err!("Expected identifier in access chain");
1183            }
1184        }
1185
1186        let root = if compound_idents.len() == 1 {
1187            self.sql_identifier_to_expr(
1188                compound_idents.pop().unwrap(),
1189                schema,
1190                planner_context,
1191            )?
1192        } else {
1193            self.sql_compound_identifier_to_expr(
1194                compound_idents,
1195                schema,
1196                planner_context,
1197            )?
1198        };
1199        Ok((root, access_chain))
1200    }
1201
1202    fn sql_compound_field_access_to_expr(
1203        &self,
1204        root: SQLExpr,
1205        access_chain: Vec<AccessExpr>,
1206        schema: &DFSchema,
1207        planner_context: &mut PlannerContext,
1208    ) -> Result<Expr> {
1209        let (root, access_chain) = self.extract_root_and_access_chain(
1210            root,
1211            access_chain,
1212            schema,
1213            planner_context,
1214        )?;
1215        let fields = access_chain
1216            .into_iter()
1217            .map(|field| match field {
1218                AccessExpr::Subscript(subscript) => {
1219                    match subscript {
1220                        Subscript::Index { index } => {
1221                            // index can be a name, in which case it is a named field access
1222                            match index {
1223                                SQLExpr::Value(ValueWithSpan {
1224                                    value:
1225                                        Value::SingleQuotedString(s)
1226                                        | Value::DoubleQuotedString(s),
1227                                    span: _,
1228                                }) => Ok(Some(GetFieldAccess::NamedStructField {
1229                                    name: ScalarValue::from(s),
1230                                })),
1231                                SQLExpr::JsonAccess { .. } => {
1232                                    not_impl_err!("JsonAccess")
1233                                }
1234                                // otherwise treat like a list index
1235                                _ => Ok(Some(GetFieldAccess::ListIndex {
1236                                    key: Box::new(self.sql_expr_to_logical_expr(
1237                                        index,
1238                                        schema,
1239                                        planner_context,
1240                                    )?),
1241                                })),
1242                            }
1243                        }
1244                        Subscript::Slice {
1245                            lower_bound,
1246                            upper_bound,
1247                            stride,
1248                        } => {
1249                            // Means access like [:2]
1250                            let lower_bound = if let Some(lower_bound) = lower_bound {
1251                                self.sql_expr_to_logical_expr(
1252                                    lower_bound,
1253                                    schema,
1254                                    planner_context,
1255                                )
1256                            } else {
1257                                not_impl_err!("Slice subscript requires a lower bound")
1258                            }?;
1259
1260                            // means access like [2:]
1261                            let upper_bound = if let Some(upper_bound) = upper_bound {
1262                                self.sql_expr_to_logical_expr(
1263                                    upper_bound,
1264                                    schema,
1265                                    planner_context,
1266                                )
1267                            } else {
1268                                not_impl_err!("Slice subscript requires an upper bound")
1269                            }?;
1270
1271                            // stride, default to 1
1272                            let stride = if let Some(stride) = stride {
1273                                self.sql_expr_to_logical_expr(
1274                                    stride,
1275                                    schema,
1276                                    planner_context,
1277                                )?
1278                            } else {
1279                                lit(1i64)
1280                            };
1281
1282                            Ok(Some(GetFieldAccess::ListRange {
1283                                start: Box::new(lower_bound),
1284                                stop: Box::new(upper_bound),
1285                                stride: Box::new(stride),
1286                            }))
1287                        }
1288                    }
1289                }
1290                AccessExpr::Dot(expr) => match expr {
1291                    SQLExpr::Value(ValueWithSpan {
1292                        value: Value::SingleQuotedString(s) | Value::DoubleQuotedString(s),
1293                        span    : _
1294                    }) => Ok(Some(GetFieldAccess::NamedStructField {
1295                        name: ScalarValue::from(s),
1296                    })),
1297                    _ => {
1298                        not_impl_err!(
1299                            "Dot access not supported for non-string expr: {expr:?}"
1300                        )
1301                    }
1302                },
1303            })
1304            .collect::<Result<Vec<_>>>()?;
1305
1306        fields
1307            .into_iter()
1308            .flatten()
1309            .try_fold(root, |expr, field_access| {
1310                let mut field_access_expr = RawFieldAccessExpr { expr, field_access };
1311                for planner in self.context_provider.get_expr_planners() {
1312                    match planner.plan_field_access(field_access_expr, schema)? {
1313                        PlannerResult::Planned(expr) => return Ok(expr),
1314                        PlannerResult::Original(expr) => {
1315                            field_access_expr = expr;
1316                        }
1317                    }
1318                }
1319                not_impl_err!(
1320                    "GetFieldAccess not supported by ExprPlanner: {field_access_expr:?}"
1321                )
1322            })
1323    }
1324}
1325
1326/// Builds a CASE expression that handles NULL semantics for `x <op> ANY(arr)`:
1327///
1328/// ```text
1329/// CASE
1330///   WHEN <min_or_max>(arr) IS NOT NULL THEN <comparison>
1331///   WHEN arr IS NOT NULL THEN FALSE          -- empty or all-null array
1332///   ELSE NULL                                -- NULL array
1333/// END
1334/// ```
1335fn any_op_with_null_handling(bound: Expr, comparison: Expr, arr: Expr) -> Result<Expr> {
1336    when(bound.is_not_null(), comparison)
1337        .when(arr.is_not_null(), lit(false))
1338        .otherwise(lit(ScalarValue::Boolean(None)))
1339}
1340
1341/// Plans a `<left> <op> ANY(<right>)` expression for non-subquery operands.
1342fn plan_any_op(
1343    left_expr: Expr,
1344    right_expr: Expr,
1345    compare_op: &BinaryOperator,
1346) -> Result<Expr> {
1347    match compare_op {
1348        BinaryOperator::Eq => Ok(array_has(right_expr, left_expr)),
1349        BinaryOperator::NotEq => {
1350            let min = array_min(right_expr.clone());
1351            let max = array_max(right_expr.clone());
1352            // NOT EQ is true when either bound differs from left
1353            let comparison = min
1354                .not_eq(left_expr.clone())
1355                .or(max.clone().not_eq(left_expr));
1356            any_op_with_null_handling(max, comparison, right_expr)
1357        }
1358        BinaryOperator::Gt => {
1359            let min = array_min(right_expr.clone());
1360            any_op_with_null_handling(min.clone(), min.lt(left_expr), right_expr)
1361        }
1362        BinaryOperator::Lt => {
1363            let max = array_max(right_expr.clone());
1364            any_op_with_null_handling(max.clone(), max.gt(left_expr), right_expr)
1365        }
1366        BinaryOperator::GtEq => {
1367            let min = array_min(right_expr.clone());
1368            any_op_with_null_handling(min.clone(), min.lt_eq(left_expr), right_expr)
1369        }
1370        BinaryOperator::LtEq => {
1371            let max = array_max(right_expr.clone());
1372            any_op_with_null_handling(max.clone(), max.gt_eq(left_expr), right_expr)
1373        }
1374        _ => plan_err!(
1375            "Unsupported AnyOp: '{compare_op}', only '=', '<>', '>', '<', '>=', '<=' are supported"
1376        ),
1377    }
1378}
1379
1380/// Plans `needle <compare_op> ALL(haystack)` with proper SQL NULL semantics.
1381///
1382/// CASE/WHEN structure:
1383///   WHEN arr IS NULL        → NULL
1384///   WHEN empty              → TRUE
1385///   WHEN lhs IS NULL        → NULL
1386///   WHEN decisive_condition → FALSE
1387///   WHEN has_nulls          → NULL
1388///   ELSE                    → TRUE
1389fn plan_all_op(
1390    needle: &Expr,
1391    haystack: &Expr,
1392    compare_op: &BinaryOperator,
1393) -> Result<Expr> {
1394    let null_arr_check = haystack.clone().is_null();
1395    let empty_check = cardinality(haystack.clone()).eq(lit(0u64));
1396    let null_lhs_check = needle.clone().is_null();
1397    // DataFusion's array_position uses is_null() checks internally (not equality),
1398    // so it can locate NULL elements even though NULL = NULL is NULL in standard SQL.
1399    let has_nulls =
1400        array_position(haystack.clone(), lit(ScalarValue::Null), lit(1i64)).is_not_null();
1401
1402    let decisive_condition = match compare_op {
1403        BinaryOperator::NotEq => array_has(haystack.clone(), needle.clone()),
1404        BinaryOperator::Eq => {
1405            let all_equal = array_min(haystack.clone())
1406                .eq(needle.clone())
1407                .and(array_max(haystack.clone()).eq(needle.clone()));
1408            Expr::Not(Box::new(all_equal))
1409        }
1410        BinaryOperator::Gt => {
1411            Expr::Not(Box::new(needle.clone().gt(array_max(haystack.clone()))))
1412        }
1413        BinaryOperator::Lt => {
1414            Expr::Not(Box::new(needle.clone().lt(array_min(haystack.clone()))))
1415        }
1416        BinaryOperator::GtEq => {
1417            Expr::Not(Box::new(needle.clone().gt_eq(array_max(haystack.clone()))))
1418        }
1419        BinaryOperator::LtEq => {
1420            Expr::Not(Box::new(needle.clone().lt_eq(array_min(haystack.clone()))))
1421        }
1422        _ => {
1423            return plan_err!(
1424                "Unsupported AllOp: '{compare_op}', only '=', '<>', '>', '<', '>=', '<=' are supported"
1425            );
1426        }
1427    };
1428
1429    let null_bool = lit(ScalarValue::Boolean(None));
1430    when(null_arr_check, null_bool.clone())
1431        .when(empty_check, lit(true))
1432        .when(null_lhs_check, null_bool.clone())
1433        .when(decisive_condition, lit(false))
1434        .when(has_nulls, null_bool)
1435        .otherwise(lit(true))
1436}
1437
1438#[cfg(test)]
1439mod tests {
1440    use std::collections::HashMap;
1441    use std::sync::Arc;
1442
1443    use arrow::datatypes::{Field, Schema};
1444    use sqlparser::dialect::GenericDialect;
1445    use sqlparser::parser::Parser;
1446
1447    use datafusion_common::TableReference;
1448    use datafusion_common::config::ConfigOptions;
1449    use datafusion_expr::logical_plan::builder::LogicalTableSource;
1450    use datafusion_expr::{
1451        AggregateUDF, HigherOrderUDF, ScalarUDF, TableSource, WindowUDF,
1452    };
1453
1454    use super::*;
1455
1456    struct TestContextProvider {
1457        options: ConfigOptions,
1458        tables: HashMap<String, Arc<dyn TableSource>>,
1459    }
1460
1461    impl TestContextProvider {
1462        pub fn new() -> Self {
1463            let mut tables = HashMap::new();
1464            tables.insert(
1465                "table1".to_string(),
1466                create_table_source(vec![Field::new(
1467                    "column1".to_string(),
1468                    DataType::Utf8,
1469                    false,
1470                )]),
1471            );
1472
1473            Self {
1474                options: Default::default(),
1475                tables,
1476            }
1477        }
1478    }
1479
1480    impl ContextProvider for TestContextProvider {
1481        fn get_table_source(&self, name: TableReference) -> Result<Arc<dyn TableSource>> {
1482            match self.tables.get(name.table()) {
1483                Some(table) => Ok(Arc::clone(table)),
1484                _ => plan_err!("Table not found: {}", name.table()),
1485            }
1486        }
1487
1488        fn get_function_meta(&self, _name: &str) -> Option<Arc<ScalarUDF>> {
1489            None
1490        }
1491
1492        fn get_higher_order_meta(&self, _name: &str) -> Option<Arc<HigherOrderUDF>> {
1493            None
1494        }
1495
1496        fn get_aggregate_meta(&self, name: &str) -> Option<Arc<AggregateUDF>> {
1497            match name {
1498                "sum" => Some(datafusion_functions_aggregate::sum::sum_udaf()),
1499                _ => None,
1500            }
1501        }
1502
1503        fn get_variable_type(&self, _variable_names: &[String]) -> Option<DataType> {
1504            None
1505        }
1506
1507        fn options(&self) -> &ConfigOptions {
1508            &self.options
1509        }
1510
1511        fn get_window_meta(&self, _name: &str) -> Option<Arc<WindowUDF>> {
1512            None
1513        }
1514
1515        fn udf_names(&self) -> Vec<String> {
1516            Vec::new()
1517        }
1518
1519        fn higher_order_function_names(&self) -> Vec<String> {
1520            Vec::new()
1521        }
1522
1523        fn udaf_names(&self) -> Vec<String> {
1524            vec!["sum".to_string()]
1525        }
1526
1527        fn udwf_names(&self) -> Vec<String> {
1528            Vec::new()
1529        }
1530    }
1531
1532    fn create_table_source(fields: Vec<Field>) -> Arc<dyn TableSource> {
1533        Arc::new(LogicalTableSource::new(Arc::new(
1534            Schema::new_with_metadata(fields, HashMap::new()),
1535        )))
1536    }
1537
1538    macro_rules! test_stack_overflow {
1539        ($name:ident, $num_expr:expr) => {
1540            #[test]
1541            fn $name() {
1542                let schema = DFSchema::empty();
1543                let mut planner_context = PlannerContext::default();
1544
1545                let expr_str = (0..$num_expr)
1546                    .map(|i| format!("column1 = 'value{:?}'", i))
1547                    .collect::<Vec<String>>()
1548                    .join(" OR ");
1549
1550                let dialect = GenericDialect {};
1551                let mut parser = Parser::new(&dialect)
1552                    .try_with_sql(expr_str.as_str())
1553                    .unwrap();
1554                let sql_expr = parser.parse_expr().unwrap();
1555
1556                let context_provider = TestContextProvider::new();
1557                let sql_to_rel = SqlToRel::new(&context_provider);
1558
1559                // Should not stack overflow
1560                sql_to_rel
1561                    .sql_expr_to_logical_expr(sql_expr, &schema, &mut planner_context)
1562                    .unwrap();
1563            }
1564        };
1565    }
1566
1567    test_stack_overflow!(test_stack_overflow_64, 64);
1568    test_stack_overflow!(test_stack_overflow_128, 128);
1569    test_stack_overflow!(test_stack_overflow_256, 256);
1570    test_stack_overflow!(test_stack_overflow_512, 512);
1571    test_stack_overflow!(test_stack_overflow_1024, 1024);
1572    test_stack_overflow!(test_stack_overflow_2048, 2048);
1573    test_stack_overflow!(test_stack_overflow_4096, 4096);
1574    test_stack_overflow!(test_stack_overflow_8192, 8192);
1575    #[test]
1576    fn test_sql_to_expr_with_alias() {
1577        let schema = DFSchema::empty();
1578        let mut planner_context = PlannerContext::default();
1579
1580        let expr_str = "SUM(int_col) as sum_int_col";
1581
1582        let dialect = GenericDialect {};
1583        let mut parser = Parser::new(&dialect).try_with_sql(expr_str).unwrap();
1584        // from sqlparser
1585        let sql_expr = parser.parse_expr_with_alias().unwrap();
1586
1587        let context_provider = TestContextProvider::new();
1588        let sql_to_rel = SqlToRel::new(&context_provider);
1589
1590        let expr = sql_to_rel
1591            .sql_expr_to_logical_expr_with_alias(sql_expr, &schema, &mut planner_context)
1592            .unwrap();
1593
1594        assert!(matches!(expr, Expr::Alias(_)));
1595    }
1596
1597    #[test]
1598    fn test_parse_numbers_with_underscores() {
1599        use datafusion_common::ScalarValue::*;
1600
1601        let context_provider = TestContextProvider::new();
1602        let sql_to_rel = SqlToRel::new(&context_provider);
1603
1604        // (input, positive result, negative result)
1605        let test_cases = [
1606            ("1_000", Int64(Some(1000)), Int64(Some(-1000))),
1607            ("100_000", Int64(Some(100000)), Int64(Some(-100000))),
1608            ("1_2_3_4", Int64(Some(1234)), Int64(Some(-1234))),
1609            ("0_0", Int64(Some(0)), Int64(Some(-0))),
1610            ("1_23.4_56", Float64(Some(123.456)), Float64(Some(-123.456))),
1611        ];
1612
1613        for (literal, out_positive, out_negative) in test_cases {
1614            assert_eq!(
1615                sql_to_rel.parse_sql_number(literal, false).unwrap(),
1616                Expr::Literal(out_positive, None)
1617            );
1618            assert_eq!(
1619                sql_to_rel.parse_sql_number(literal, true).unwrap(),
1620                Expr::Literal(out_negative, None)
1621            );
1622        }
1623    }
1624}
datafusion_sql/expr/mod.rs

datafusion_sql/expr/
mod.rs
