datafusion_expr/
expr_schema.rs

1// Licensed to the Apache Software Foundation (ASF) under one
2// or more contributor license agreements.  See the NOTICE file
3// distributed with this work for additional information
4// regarding copyright ownership.  The ASF licenses this file
5// to you under the Apache License, Version 2.0 (the
6// "License"); you may not use this file except in compliance
7// with the License.  You may obtain a copy of the License at
8//
9//   http://www.apache.org/licenses/LICENSE-2.0
10//
11// Unless required by applicable law or agreed to in writing,
12// software distributed under the License is distributed on an
13// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
14// KIND, either express or implied.  See the License for the
15// specific language governing permissions and limitations
16// under the License.
17
18use super::{Between, Expr, Like, predicate_bounds};
19use crate::ValueOrLambda;
20use crate::expr::{
21    AggregateFunction, AggregateFunctionParams, Alias, BinaryExpr, Cast, InList,
22    InSubquery, Lambda, Placeholder, ScalarFunction, TryCast, Unnest, WindowFunction,
23    WindowFunctionParams,
24};
25use crate::expr::{FieldMetadata, LambdaVariable};
26use crate::higher_order_function::HigherOrderReturnFieldArgs;
27use crate::type_coercion::functions::value_fields_with_higher_order_udf_and_lambdas;
28use crate::type_coercion::functions::{UDFCoercionExt, fields_with_udf};
29use crate::udf::ReturnFieldArgs;
30use crate::{LogicalPlan, Projection, Subquery, WindowFunctionDefinition, utils};
31use arrow::compute::can_cast_types;
32use arrow::datatypes::FieldRef;
33use arrow::datatypes::{DataType, Field};
34use datafusion_common::datatype::FieldExt;
35use datafusion_common::{
36    Column, DataFusionError, ExprSchema, Result, ScalarValue, Spans, TableReference,
37    not_impl_err, plan_datafusion_err, plan_err,
38};
39use datafusion_expr_common::type_coercion::binary::BinaryTypeCoercer;
40use datafusion_functions_window_common::field::WindowUDFFieldArgs;
41use std::sync::Arc;
42
43/// Trait to allow expr to typable with respect to a schema
44pub trait ExprSchemable {
45    /// Given a schema, return the type of the expr
46    fn get_type(&self, schema: &dyn ExprSchema) -> Result<DataType>;
47
48    /// Given a schema, return the nullability of the expr
49    fn nullable(&self, input_schema: &dyn ExprSchema) -> Result<bool>;
50
51    /// Given a schema, return the expr's optional metadata
52    fn metadata(&self, schema: &dyn ExprSchema) -> Result<FieldMetadata>;
53
54    /// Convert to a field with respect to a schema
55    fn to_field(
56        &self,
57        input_schema: &dyn ExprSchema,
58    ) -> Result<(Option<TableReference>, Arc<Field>)>;
59
60    /// Cast to a type with respect to a schema
61    fn cast_to(self, cast_to_type: &DataType, schema: &dyn ExprSchema) -> Result<Expr>;
62
63    /// Given a schema, return the type and nullability of the expr
64    #[deprecated(
65        since = "51.0.0",
66        note = "Use `to_field().1.is_nullable` and `to_field().1.data_type()` directly instead"
67    )]
68    fn data_type_and_nullable(&self, schema: &dyn ExprSchema)
69    -> Result<(DataType, bool)>;
70}
71
72/// Derives the output field for a cast expression from the source field.
73/// For `TryCast`, `force_nullable` is `true` since a failed cast returns NULL.
74fn cast_output_field(
75    source_field: &FieldRef,
76    target_type: &DataType,
77    force_nullable: bool,
78) -> Arc<Field> {
79    let mut f = source_field
80        .as_ref()
81        .clone()
82        .with_data_type(target_type.clone())
83        .with_metadata(source_field.metadata().clone());
84    if force_nullable {
85        f = f.with_nullable(true);
86    }
87    Arc::new(f)
88}
89
90impl ExprSchemable for Expr {
91    /// Returns the [arrow::datatypes::DataType] of the expression
92    /// based on [ExprSchema]
93    ///
94    /// Note: [`DFSchema`] implements [ExprSchema].
95    ///
96    /// [`DFSchema`]: datafusion_common::DFSchema
97    ///
98    /// # Examples
99    ///
100    /// Get the type of an expression that adds 2 columns. Adding an Int32
101    /// and Float32 results in Float32 type
102    ///
103    /// ```
104    /// # use arrow::datatypes::{DataType, Field};
105    /// # use datafusion_common::DFSchema;
106    /// # use datafusion_expr::{col, ExprSchemable};
107    /// # use std::collections::HashMap;
108    ///
109    /// fn main() {
110    ///     let expr = col("c1") + col("c2");
111    ///     let schema = DFSchema::from_unqualified_fields(
112    ///         vec![
113    ///             Field::new("c1", DataType::Int32, true),
114    ///             Field::new("c2", DataType::Float32, true),
115    ///         ]
116    ///         .into(),
117    ///         HashMap::new(),
118    ///     )
119    ///     .unwrap();
120    ///     assert_eq!("Float32", format!("{}", expr.get_type(&schema).unwrap()));
121    /// }
122    /// ```
123    ///
124    /// # Errors
125    ///
126    /// This function errors when it is not possible to compute its
127    /// [arrow::datatypes::DataType].  This happens when e.g. the
128    /// expression refers to a column that does not exist in the
129    /// schema, or when the expression is incorrectly typed
130    /// (e.g. `[utf8] + [bool]`).
131    #[cfg_attr(feature = "recursive_protection", recursive::recursive)]
132    fn get_type(&self, schema: &dyn ExprSchema) -> Result<DataType> {
133        match self {
134            Expr::Alias(Alias { expr, name, .. }) => match &**expr {
135                Expr::Placeholder(Placeholder { field, .. }) => match &field {
136                    None => schema.data_type(&Column::from_name(name)).cloned(),
137                    Some(field) => Ok(field.data_type().clone()),
138                },
139                _ => expr.get_type(schema),
140            },
141            Expr::Negative(expr) => expr.get_type(schema),
142            Expr::Column(c) => Ok(schema.data_type(c)?.clone()),
143            Expr::OuterReferenceColumn(field, _) => Ok(field.data_type().clone()),
144            Expr::ScalarVariable(field, _) => Ok(field.data_type().clone()),
145            Expr::Literal(l, _) => Ok(l.data_type()),
146            Expr::Case(case) => {
147                for (_, then_expr) in &case.when_then_expr {
148                    let then_type = then_expr.get_type(schema)?;
149                    if !then_type.is_null() {
150                        return Ok(then_type);
151                    }
152                }
153                case.else_expr
154                    .as_ref()
155                    .map_or(Ok(DataType::Null), |e| e.get_type(schema))
156            }
157            Expr::Cast(Cast { field, .. }) | Expr::TryCast(TryCast { field, .. }) => {
158                Ok(field.data_type().clone())
159            }
160            Expr::Unnest(Unnest { expr, .. }) => {
161                let arg_data_type = expr.get_type(schema)?;
162                // Unnest's output type is the inner type of the list
163                match arg_data_type {
164                    DataType::List(field)
165                    | DataType::LargeList(field)
166                    | DataType::FixedSizeList(field, _)
167                    | DataType::ListView(field)
168                    | DataType::LargeListView(field) => Ok(field.data_type().clone()),
169                    DataType::Struct(_) => Ok(arg_data_type),
170                    DataType::Null => {
171                        not_impl_err!("unnest() does not support null yet")
172                    }
173                    _ => {
174                        plan_err!(
175                            "unnest() can only be applied to array, struct and null"
176                        )
177                    }
178                }
179            }
180            Expr::ScalarFunction(_)
181            | Expr::WindowFunction(_)
182            | Expr::AggregateFunction(_) => {
183                Ok(self.to_field(schema)?.1.data_type().clone())
184            }
185            Expr::Not(_)
186            | Expr::IsNull(_)
187            | Expr::Exists { .. }
188            | Expr::InSubquery(_)
189            | Expr::SetComparison(_)
190            | Expr::Between { .. }
191            | Expr::InList { .. }
192            | Expr::IsNotNull(_)
193            | Expr::IsTrue(_)
194            | Expr::IsFalse(_)
195            | Expr::IsUnknown(_)
196            | Expr::IsNotTrue(_)
197            | Expr::IsNotFalse(_)
198            | Expr::IsNotUnknown(_) => Ok(DataType::Boolean),
199            Expr::ScalarSubquery(subquery) => {
200                Ok(subquery.subquery.schema().field(0).data_type().clone())
201            }
202            Expr::BinaryExpr(BinaryExpr { left, right, op }) => BinaryTypeCoercer::new(
203                &left.get_type(schema)?,
204                op,
205                &right.get_type(schema)?,
206            )
207            .get_result_type(),
208            Expr::Like { .. } | Expr::SimilarTo { .. } => Ok(DataType::Boolean),
209            Expr::Placeholder(Placeholder { field, .. }) => {
210                if let Some(field) = field {
211                    Ok(field.data_type().clone())
212                } else {
213                    // If the placeholder's type hasn't been specified, treat it as
214                    // null (unspecified placeholders generate an error during planning)
215                    Ok(DataType::Null)
216                }
217            }
218            #[expect(deprecated)]
219            Expr::Wildcard { .. } => Ok(DataType::Null),
220            Expr::GroupingSet(_) => {
221                // Grouping sets do not really have a type and do not appear in projections
222                Ok(DataType::Null)
223            }
224            Expr::HigherOrderFunction(_func) => {
225                Ok(self.to_field(schema)?.1.data_type().clone())
226            }
227            Expr::Lambda(_lambda) => Ok(DataType::Null),
228            Expr::LambdaVariable(LambdaVariable { field, .. }) => match field {
229                Some(f) => Ok(f.data_type().clone()),
230                // If the lambda variable's field hasn't been specified, treat it as
231                // null (unspecified lambda variables generate an error during planning)
232                None => Ok(DataType::Null),
233            },
234        }
235    }
236
237    /// Returns the nullability of the expression based on [ExprSchema].
238    ///
239    /// Note: [`DFSchema`] implements [ExprSchema].
240    ///
241    /// [`DFSchema`]: datafusion_common::DFSchema
242    ///
243    /// # Errors
244    ///
245    /// This function errors when it is not possible to compute its
246    /// nullability.  This happens when the expression refers to a
247    /// column that does not exist in the schema.
248    fn nullable(&self, input_schema: &dyn ExprSchema) -> Result<bool> {
249        match self {
250            Expr::Alias(Alias { expr, .. }) | Expr::Not(expr) | Expr::Negative(expr) => {
251                expr.nullable(input_schema)
252            }
253
254            Expr::InList(InList { expr, list, .. }) => {
255                // Avoid inspecting too many expressions.
256                const MAX_INSPECT_LIMIT: usize = 6;
257                // Stop if a nullable expression is found or an error occurs.
258                let has_nullable = std::iter::once(expr.as_ref())
259                    .chain(list)
260                    .take(MAX_INSPECT_LIMIT)
261                    .find_map(|e| {
262                        e.nullable(input_schema)
263                            .map(|nullable| if nullable { Some(()) } else { None })
264                            .transpose()
265                    })
266                    .transpose()?;
267                Ok(match has_nullable {
268                    // If a nullable subexpression is found, the result may also be nullable.
269                    Some(_) => true,
270                    // If the list is too long, we assume it is nullable.
271                    None if list.len() + 1 > MAX_INSPECT_LIMIT => true,
272                    // All the subexpressions are non-nullable, so the result must be non-nullable.
273                    _ => false,
274                })
275            }
276
277            Expr::Between(Between {
278                expr, low, high, ..
279            }) => Ok(expr.nullable(input_schema)?
280                || low.nullable(input_schema)?
281                || high.nullable(input_schema)?),
282
283            Expr::Column(c) => input_schema.nullable(c),
284            Expr::OuterReferenceColumn(field, _) => Ok(field.is_nullable()),
285            Expr::Literal(value, _) => Ok(value.is_null()),
286            Expr::Case(case) => {
287                let nullable_then = case
288                    .when_then_expr
289                    .iter()
290                    .filter_map(|(w, t)| {
291                        let is_nullable = match t.nullable(input_schema) {
292                            Err(e) => return Some(Err(e)),
293                            Ok(n) => n,
294                        };
295
296                        // Branches with a then expression that is not nullable do not impact the
297                        // nullability of the case expression.
298                        if !is_nullable {
299                            return None;
300                        }
301
302                        // For case-with-expression assume all 'then' expressions are reachable
303                        if case.expr.is_some() {
304                            return Some(Ok(()));
305                        }
306
307                        // For branches with a nullable 'then' expression, try to determine
308                        // if the 'then' expression is ever reachable in the situation where
309                        // it would evaluate to null.
310                        let bounds = match predicate_bounds::evaluate_bounds(
311                            w,
312                            Some(unwrap_certainly_null_expr(t)),
313                            input_schema,
314                        ) {
315                            Err(e) => return Some(Err(e)),
316                            Ok(b) => b,
317                        };
318
319                        let can_be_true = match bounds
320                            .contains_value(ScalarValue::Boolean(Some(true)))
321                        {
322                            Err(e) => return Some(Err(e)),
323                            Ok(b) => b,
324                        };
325
326                        if !can_be_true {
327                            // If the derived 'when' expression can never evaluate to true, the
328                            // 'then' expression is not reachable when it would evaluate to NULL.
329                            // The most common pattern for this is `WHEN x IS NOT NULL THEN x`.
330                            None
331                        } else {
332                            // The branch might be taken
333                            Some(Ok(()))
334                        }
335                    })
336                    .next();
337
338                if let Some(nullable_then) = nullable_then {
339                    // There is at least one reachable nullable 'then' expression, so the case
340                    // expression itself is nullable.
341                    // Use `Result::map` to propagate the error from `nullable_then` if there is one.
342                    nullable_then.map(|_| true)
343                } else if let Some(e) = &case.else_expr {
344                    // There are no reachable nullable 'then' expressions, so all we still need to
345                    // check is the 'else' expression's nullability.
346                    e.nullable(input_schema)
347                } else {
348                    // CASE produces NULL if there is no `else` expr
349                    // (aka when none of the `when_then_exprs` match)
350                    Ok(true)
351                }
352            }
353            Expr::Cast(Cast { expr, .. }) => expr.nullable(input_schema),
354            Expr::ScalarFunction(_)
355            | Expr::AggregateFunction(_)
356            | Expr::WindowFunction(_) => Ok(self.to_field(input_schema)?.1.is_nullable()),
357            Expr::ScalarVariable(field, _) => Ok(field.is_nullable()),
358            Expr::TryCast { .. } | Expr::Unnest(_) | Expr::Placeholder(_) => Ok(true),
359            Expr::IsNull(_)
360            | Expr::IsNotNull(_)
361            | Expr::IsTrue(_)
362            | Expr::IsFalse(_)
363            | Expr::IsUnknown(_)
364            | Expr::IsNotTrue(_)
365            | Expr::IsNotFalse(_)
366            | Expr::IsNotUnknown(_)
367            | Expr::Exists { .. } => Ok(false),
368            Expr::SetComparison(_) => Ok(true),
369            Expr::InSubquery(InSubquery { expr, subquery, .. }) => {
370                let expr_nullable = expr.nullable(input_schema)?;
371                let subquery_nullable = subquery.subquery.schema().fields().first().ok_or_else(|| {
372                    plan_datafusion_err!("subquery must return exactly one column of data to compare against")
373                })?.is_nullable();
374
375                Ok(expr_nullable | subquery_nullable)
376            }
377            Expr::ScalarSubquery(subquery) => {
378                Ok(subquery.subquery.schema().field(0).is_nullable())
379            }
380            Expr::BinaryExpr(BinaryExpr { left, right, .. }) => {
381                Ok(left.nullable(input_schema)? || right.nullable(input_schema)?)
382            }
383            Expr::Like(Like { expr, pattern, .. })
384            | Expr::SimilarTo(Like { expr, pattern, .. }) => {
385                Ok(expr.nullable(input_schema)? || pattern.nullable(input_schema)?)
386            }
387            #[expect(deprecated)]
388            Expr::Wildcard { .. } => Ok(false),
389            Expr::GroupingSet(_) => {
390                // Grouping sets do not really have the concept of nullable and do not appear
391                // in projections
392                Ok(true)
393            }
394            Expr::HigherOrderFunction(_func) => {
395                Ok(self.to_field(input_schema)?.1.is_nullable())
396            }
397            Expr::Lambda(_lambda) => Ok(true),
398            Expr::LambdaVariable(LambdaVariable { field, .. }) => match field {
399                Some(f) => Ok(f.is_nullable()),
400                // If the lambda variable's field hasn't been specified, treat it as
401                // null (unspecified lambda variables generate an error during planning)
402                None => Ok(true),
403            },
404        }
405    }
406
407    fn metadata(&self, schema: &dyn ExprSchema) -> Result<FieldMetadata> {
408        self.to_field(schema)
409            .map(|(_, field)| FieldMetadata::from(field.metadata()))
410    }
411
412    /// Returns the datatype and nullability of the expression based on [ExprSchema].
413    ///
414    /// Note: [`DFSchema`] implements [ExprSchema].
415    ///
416    /// [`DFSchema`]: datafusion_common::DFSchema
417    ///
418    /// # Errors
419    ///
420    /// This function errors when it is not possible to compute its
421    /// datatype or nullability.
422    fn data_type_and_nullable(
423        &self,
424        schema: &dyn ExprSchema,
425    ) -> Result<(DataType, bool)> {
426        let field = self.to_field(schema)?.1;
427
428        Ok((field.data_type().clone(), field.is_nullable()))
429    }
430
431    /// Returns a [arrow::datatypes::Field] compatible with this expression.
432    ///
433    /// This function converts an expression into a field with appropriate metadata
434    /// and nullability based on the expression type and context. It is the primary
435    /// mechanism for determining field-level schemas.
436    ///
437    /// # Field Property Resolution
438    ///
439    /// For each expression, the following properties are determined:
440    ///
441    /// ## Data Type Resolution
442    /// - **Column references**: Data type from input schema field
443    /// - **Literals**: Data type inferred from literal value
444    /// - **Aliases**: Data type inherited from the underlying expression (the aliased expression)
445    /// - **Binary expressions**: Result type from type coercion rules
446    /// - **Boolean expressions**: Always a boolean type
447    /// - **Cast expressions**: Target data type from cast operation
448    /// - **Function calls**: Return type based on function signature and argument types
449    ///
450    /// ## Nullability Determination
451    /// - **Column references**: Inherit nullability from input schema field
452    /// - **Literals**: Nullable only if literal value is NULL
453    /// - **Aliases**: Inherit nullability from the underlying expression (the aliased expression)
454    /// - **Binary expressions**: Nullable if either operand is nullable
455    /// - **Boolean expressions**: Always non-nullable (IS NULL, EXISTS, etc.)
456    /// - **Cast expressions**: determined by the input expression's nullability rules
457    /// - **Function calls**: Based on function nullability rules and input nullability
458    ///
459    /// ## Metadata Handling
460    /// - **Column references**: Preserve original field metadata from input schema
461    /// - **Literals**: Use explicitly provided metadata, otherwise empty
462    /// - **Aliases**: Merge underlying expr metadata with alias-specific metadata, preferring the alias metadata
463    /// - **Binary expressions**: field metadata is empty
464    /// - **Boolean expressions**: field metadata is empty
465    /// - **Cast expressions**: determined by the input expression's field metadata handling
466    /// - **Scalar functions**: Generate metadata via function's [`return_field_from_args`] method,
467    ///   with the default implementation returning empty field metadata
468    /// - **Aggregate functions**: Generate metadata via function's [`return_field`] method,
469    ///   with the default implementation returning empty field metadata
470    /// - **Window functions**: field metadata follows the function's return field
471    ///
472    /// ## Table Reference Scoping
473    /// - Establishes proper qualified field references when columns belong to specific tables
474    /// - Maintains table context for accurate field resolution in multi-table scenarios
475    ///
476    /// So for example, a projected expression `col(c1) + col(c2)` is
477    /// placed in an output field **named** col("c1 + c2")
478    ///
479    /// [`return_field_from_args`]: crate::ScalarUDF::return_field_from_args
480    /// [`return_field`]: crate::AggregateUDF::return_field
481    fn to_field(
482        &self,
483        schema: &dyn ExprSchema,
484    ) -> Result<(Option<TableReference>, Arc<Field>)> {
485        let (relation, schema_name) = self.qualified_name();
486        #[expect(deprecated)]
487        let field = match self {
488            Expr::Alias(Alias {
489                expr,
490                name: _,
491                metadata,
492                ..
493            }) => {
494                let mut combined_metadata = expr.metadata(schema)?;
495                if let Some(metadata) = metadata {
496                    combined_metadata.extend(metadata.clone());
497                }
498
499                Ok(expr
500                    .to_field(schema)
501                    .map(|(_, f)| f)?
502                    .with_field_metadata(&combined_metadata))
503            }
504            Expr::Negative(expr) => expr.to_field(schema).map(|(_, f)| f),
505            Expr::Column(c) => schema.field_from_column(c).map(Arc::clone),
506            Expr::OuterReferenceColumn(field, _) => {
507                Ok(Arc::clone(field).renamed(&schema_name))
508            }
509            Expr::ScalarVariable(field, _) => Ok(Arc::clone(field).renamed(&schema_name)),
510            Expr::Literal(l, metadata) => Ok(Arc::new(
511                Field::new(&schema_name, l.data_type(), l.is_null())
512                    .with_field_metadata_opt(metadata.as_ref()),
513            )),
514            Expr::IsNull(_)
515            | Expr::IsNotNull(_)
516            | Expr::IsTrue(_)
517            | Expr::IsFalse(_)
518            | Expr::IsUnknown(_)
519            | Expr::IsNotTrue(_)
520            | Expr::IsNotFalse(_)
521            | Expr::IsNotUnknown(_)
522            | Expr::Exists { .. } => {
523                Ok(Arc::new(Field::new(&schema_name, DataType::Boolean, false)))
524            }
525            Expr::ScalarSubquery(subquery) => {
526                Ok(Arc::clone(&subquery.subquery.schema().fields()[0]))
527            }
528            Expr::BinaryExpr(BinaryExpr { left, right, op }) => {
529                let (left_field, right_field) =
530                    (left.to_field(schema)?.1, right.to_field(schema)?.1);
531
532                let (lhs_type, lhs_nullable) =
533                    (left_field.data_type(), left_field.is_nullable());
534                let (rhs_type, rhs_nullable) =
535                    (right_field.data_type(), right_field.is_nullable());
536                let mut coercer = BinaryTypeCoercer::new(lhs_type, op, rhs_type);
537                coercer.set_lhs_spans(left.spans().cloned().unwrap_or_default());
538                coercer.set_rhs_spans(right.spans().cloned().unwrap_or_default());
539                Ok(Arc::new(Field::new(
540                    &schema_name,
541                    coercer.get_result_type()?,
542                    lhs_nullable || rhs_nullable,
543                )))
544            }
545            Expr::WindowFunction(window_function) => {
546                let WindowFunction {
547                    fun,
548                    params: WindowFunctionParams { args, .. },
549                    ..
550                } = window_function.as_ref();
551
552                let fields = args
553                    .iter()
554                    .map(|e| e.to_field(schema).map(|(_, f)| f))
555                    .collect::<Result<Vec<_>>>()?;
556                match fun {
557                    WindowFunctionDefinition::AggregateUDF(udaf) => {
558                        let new_fields =
559                            verify_function_arguments(udaf.as_ref(), &fields)?;
560                        let return_field = udaf.return_field(&new_fields)?;
561                        Ok(return_field)
562                    }
563                    WindowFunctionDefinition::WindowUDF(udwf) => {
564                        let new_fields =
565                            verify_function_arguments(udwf.as_ref(), &fields)?;
566                        let return_field = udwf
567                            .field(WindowUDFFieldArgs::new(&new_fields, &schema_name))?;
568                        Ok(return_field)
569                    }
570                }
571            }
572            Expr::AggregateFunction(AggregateFunction {
573                func,
574                params: AggregateFunctionParams { args, .. },
575            }) => {
576                let fields = args
577                    .iter()
578                    .map(|e| e.to_field(schema).map(|(_, f)| f))
579                    .collect::<Result<Vec<_>>>()?;
580                let new_fields = verify_function_arguments(func.as_ref(), &fields)?;
581                func.return_field(&new_fields)
582            }
583            Expr::ScalarFunction(ScalarFunction { func, args }) => {
584                let fields = args
585                    .iter()
586                    .map(|e| e.to_field(schema).map(|(_, f)| f))
587                    .collect::<Result<Vec<_>>>()?;
588                let new_fields = verify_function_arguments(func.as_ref(), &fields)?;
589
590                let arguments = args
591                    .iter()
592                    .map(|e| match e {
593                        Expr::Literal(sv, _) => Some(sv),
594                        _ => None,
595                    })
596                    .collect::<Vec<_>>();
597                let args = ReturnFieldArgs {
598                    arg_fields: &new_fields,
599                    scalar_arguments: &arguments,
600                };
601
602                func.return_field_from_args(args)
603            }
604            // _ => Ok((self.get_type(schema)?, self.nullable(schema)?)),
605            Expr::Cast(Cast { expr, field }) => {
606                expr.to_field(schema).map(|(_table_ref, src)| {
607                    cast_output_field(&src, field.data_type(), false)
608                })
609            }
610            Expr::Placeholder(Placeholder {
611                id: _,
612                field: Some(field),
613            }) => Ok(Arc::clone(field).renamed(&schema_name)),
614            Expr::TryCast(TryCast { expr, field }) => {
615                expr.to_field(schema).map(|(_table_ref, src)| {
616                    cast_output_field(&src, field.data_type(), true)
617                })
618            }
619            Expr::LambdaVariable(LambdaVariable {
620                field: Some(field), ..
621            }) => Ok(Arc::clone(field).renamed(&schema_name)),
622            Expr::Like(_)
623            | Expr::SimilarTo(_)
624            | Expr::Not(_)
625            | Expr::Between(_)
626            | Expr::Case(_)
627            | Expr::InList(_)
628            | Expr::InSubquery(_)
629            | Expr::SetComparison(_)
630            | Expr::Wildcard { .. }
631            | Expr::GroupingSet(_)
632            | Expr::Placeholder(_)
633            | Expr::Unnest(_)
634            | Expr::Lambda(_)
635            | Expr::LambdaVariable(_) => Ok(Arc::new(Field::new(
636                &schema_name,
637                self.get_type(schema)?,
638                self.nullable(schema)?,
639            ))),
640            Expr::HigherOrderFunction(func) => {
641                let arg_fields = func
642                    .args
643                    .iter()
644                    .map(|arg| match arg {
645                        Expr::Lambda(Lambda { params: _, body }) => {
646                            // use the name of the lambda instead of just the body to help with debugging
647                            Ok(ValueOrLambda::Lambda(Arc::new(Field::new(
648                                arg.qualified_name().1,
649                                body.get_type(schema)?,
650                                body.nullable(schema)?,
651                            ))))
652                        }
653                        _ => Ok(ValueOrLambda::Value(arg.to_field(schema)?.1)),
654                    })
655                    .collect::<Result<Vec<_>>>()?;
656
657                let new_fields = value_fields_with_higher_order_udf_and_lambdas(
658                    &arg_fields,
659                    func.func.as_ref(),
660                )?;
661
662                let arguments = func
663                    .args
664                    .iter()
665                    .map(|e| match e {
666                        Expr::Literal(sv, _) => Some(sv),
667                        _ => None,
668                    })
669                    .collect::<Vec<_>>();
670
671                let args = HigherOrderReturnFieldArgs {
672                    arg_fields: &new_fields,
673                    scalar_arguments: &arguments,
674                };
675
676                func.func.return_field_from_args(args)
677            }
678        }?;
679
680        Ok((
681            relation,
682            // todo avoid this rename / use the name above
683            field.renamed(&schema_name),
684        ))
685    }
686
687    /// Wraps this expression in a cast to a target [arrow::datatypes::DataType].
688    ///
689    /// # Errors
690    ///
691    /// This function errors when it is impossible to cast the
692    /// expression to the target [arrow::datatypes::DataType].
693    fn cast_to(self, cast_to_type: &DataType, schema: &dyn ExprSchema) -> Result<Expr> {
694        let this_type = self.get_type(schema)?;
695        if this_type == *cast_to_type {
696            return Ok(self);
697        }
698
699        // TODO(kszucs): Most of the operations do not validate the type correctness
700        // like all of the binary expressions below. Perhaps Expr should track the
701        // type of the expression?
702
703        // Special handling for struct-to-struct casts with name-based field matching
704        let can_cast = match (&this_type, cast_to_type) {
705            (DataType::Struct(_), DataType::Struct(_)) => {
706                // Always allow struct-to-struct casts; field matching happens at runtime
707                true
708            }
709            _ => can_cast_types(&this_type, cast_to_type),
710        };
711
712        if can_cast {
713            match self {
714                Expr::ScalarSubquery(subquery) => {
715                    Ok(Expr::ScalarSubquery(cast_subquery(subquery, cast_to_type)?))
716                }
717                _ => Ok(Expr::Cast(Cast::new(Box::new(self), cast_to_type.clone()))),
718            }
719        } else {
720            plan_err!("Cannot automatically convert {this_type} to {cast_to_type}")
721        }
722    }
723}
724
725/// Verify that function is invoked with correct number and type of arguments as
726/// defined in `TypeSignature`.
727fn verify_function_arguments<F: UDFCoercionExt>(
728    function: &F,
729    input_fields: &[FieldRef],
730) -> Result<Vec<FieldRef>> {
731    fields_with_udf(input_fields, function).map_err(|err| {
732        let data_types = input_fields
733            .iter()
734            .map(|f| f.data_type())
735            .cloned()
736            .collect::<Vec<_>>();
737        plan_datafusion_err!(
738            "{}. {}",
739            match err {
740                DataFusionError::Plan(msg) => msg,
741                err => err.to_string(),
742            },
743            utils::generate_signature_error_message(
744                function.name(),
745                function.signature(),
746                &data_types
747            )
748        )
749    })
750}
751
752/// Returns the innermost [Expr] that is provably null if `expr` is null.
753fn unwrap_certainly_null_expr(expr: &Expr) -> &Expr {
754    match expr {
755        Expr::Not(e) => unwrap_certainly_null_expr(e),
756        Expr::Negative(e) => unwrap_certainly_null_expr(e),
757        Expr::Cast(e) => unwrap_certainly_null_expr(e.expr.as_ref()),
758        _ => expr,
759    }
760}
761
762/// Cast subquery in InSubquery/ScalarSubquery to a given type.
763///
764/// 1. **Projection plan**: If the subquery is a projection (i.e. a SELECT statement with specific
765///    columns), it casts the first expression in the projection to the target type and creates a
766///    new projection with the casted expression.
767/// 2. **Non-projection plan**: If the subquery isn't a projection, it adds a projection to the plan
768///    with the casted first column.
769pub fn cast_subquery(subquery: Subquery, cast_to_type: &DataType) -> Result<Subquery> {
770    if subquery.subquery.schema().field(0).data_type() == cast_to_type {
771        return Ok(subquery);
772    }
773
774    let plan = subquery.subquery.as_ref();
775    let new_plan = match plan {
776        LogicalPlan::Projection(projection) => {
777            let cast_expr = projection.expr[0]
778                .clone()
779                .cast_to(cast_to_type, projection.input.schema())?;
780            LogicalPlan::Projection(Projection::try_new(
781                vec![cast_expr],
782                Arc::clone(&projection.input),
783            )?)
784        }
785        _ => {
786            let cast_expr = Expr::Column(Column::from(plan.schema().qualified_field(0)))
787                .cast_to(cast_to_type, subquery.subquery.schema())?;
788            LogicalPlan::Projection(Projection::try_new(
789                vec![cast_expr],
790                subquery.subquery,
791            )?)
792        }
793    };
794    Ok(Subquery {
795        subquery: Arc::new(new_plan),
796        outer_ref_columns: subquery.outer_ref_columns,
797        spans: Spans::new(),
798    })
799}
800
801#[cfg(test)]
802mod tests {
803    use std::collections::HashMap;
804
805    use super::*;
806    use crate::logical_plan::builder::LogicalTableSource;
807    use crate::{
808        LogicalPlanBuilder, and, col, in_subquery, lit, not, or,
809        out_ref_col_with_metadata, when,
810    };
811
812    use arrow::datatypes::Schema;
813    use datafusion_common::{DFSchema, assert_or_internal_err};
814
815    macro_rules! test_is_expr_nullable {
816        ($EXPR_TYPE:ident) => {{
817            let expr = lit(ScalarValue::Null).$EXPR_TYPE();
818            assert!(!expr.nullable(&MockExprSchema::new()).unwrap());
819        }};
820    }
821
822    #[test]
823    fn expr_schema_nullability() {
824        let expr = col("foo").eq(lit(1));
825        assert!(!expr.nullable(&MockExprSchema::new()).unwrap());
826        assert!(
827            expr.nullable(&MockExprSchema::new().with_nullable(true))
828                .unwrap()
829        );
830
831        test_is_expr_nullable!(is_null);
832        test_is_expr_nullable!(is_not_null);
833        test_is_expr_nullable!(is_true);
834        test_is_expr_nullable!(is_not_true);
835        test_is_expr_nullable!(is_false);
836        test_is_expr_nullable!(is_not_false);
837        test_is_expr_nullable!(is_unknown);
838        test_is_expr_nullable!(is_not_unknown);
839    }
840
841    #[test]
842    fn test_between_nullability() {
843        let get_schema = |nullable| {
844            MockExprSchema::new()
845                .with_data_type(DataType::Int32)
846                .with_nullable(nullable)
847        };
848
849        let expr = col("foo").between(lit(1), lit(2));
850        assert!(!expr.nullable(&get_schema(false)).unwrap());
851        assert!(expr.nullable(&get_schema(true)).unwrap());
852
853        let null = lit(ScalarValue::Int32(None));
854
855        let expr = col("foo").between(null.clone(), lit(2));
856        assert!(expr.nullable(&get_schema(false)).unwrap());
857
858        let expr = col("foo").between(lit(1), null.clone());
859        assert!(expr.nullable(&get_schema(false)).unwrap());
860
861        let expr = col("foo").between(null.clone(), null);
862        assert!(expr.nullable(&get_schema(false)).unwrap());
863    }
864
865    fn assert_nullability(expr: &Expr, schema: &dyn ExprSchema, expected: bool) {
866        assert_eq!(
867            expr.nullable(schema).unwrap(),
868            expected,
869            "Nullability of '{expr}' should be {expected}"
870        );
871    }
872
873    fn assert_not_nullable(expr: &Expr, schema: &dyn ExprSchema) {
874        assert_nullability(expr, schema, false);
875    }
876
877    fn assert_nullable(expr: &Expr, schema: &dyn ExprSchema) {
878        assert_nullability(expr, schema, true);
879    }
880
881    #[test]
882    fn test_case_expression_nullability() -> Result<()> {
883        let nullable_schema = MockExprSchema::new()
884            .with_data_type(DataType::Int32)
885            .with_nullable(true);
886
887        let not_nullable_schema = MockExprSchema::new()
888            .with_data_type(DataType::Int32)
889            .with_nullable(false);
890
891        // CASE WHEN x IS NOT NULL THEN x ELSE 0
892        let e = when(col("x").is_not_null(), col("x")).otherwise(lit(0))?;
893        assert_not_nullable(&e, &nullable_schema);
894        assert_not_nullable(&e, &not_nullable_schema);
895
896        // CASE WHEN NOT x IS NULL THEN x ELSE 0
897        let e = when(not(col("x").is_null()), col("x")).otherwise(lit(0))?;
898        assert_not_nullable(&e, &nullable_schema);
899        assert_not_nullable(&e, &not_nullable_schema);
900
901        // CASE WHEN X = 5 THEN x ELSE 0
902        let e = when(col("x").eq(lit(5)), col("x")).otherwise(lit(0))?;
903        assert_not_nullable(&e, &nullable_schema);
904        assert_not_nullable(&e, &not_nullable_schema);
905
906        // CASE WHEN x IS NOT NULL AND x = 5 THEN x ELSE 0
907        let e = when(and(col("x").is_not_null(), col("x").eq(lit(5))), col("x"))
908            .otherwise(lit(0))?;
909        assert_not_nullable(&e, &nullable_schema);
910        assert_not_nullable(&e, &not_nullable_schema);
911
912        // CASE WHEN x = 5 AND x IS NOT NULL THEN x ELSE 0
913        let e = when(and(col("x").eq(lit(5)), col("x").is_not_null()), col("x"))
914            .otherwise(lit(0))?;
915        assert_not_nullable(&e, &nullable_schema);
916        assert_not_nullable(&e, &not_nullable_schema);
917
918        // CASE WHEN x IS NOT NULL OR x = 5 THEN x ELSE 0
919        let e = when(or(col("x").is_not_null(), col("x").eq(lit(5))), col("x"))
920            .otherwise(lit(0))?;
921        assert_not_nullable(&e, &nullable_schema);
922        assert_not_nullable(&e, &not_nullable_schema);
923
924        // CASE WHEN x = 5 OR x IS NOT NULL THEN x ELSE 0
925        let e = when(or(col("x").eq(lit(5)), col("x").is_not_null()), col("x"))
926            .otherwise(lit(0))?;
927        assert_not_nullable(&e, &nullable_schema);
928        assert_not_nullable(&e, &not_nullable_schema);
929
930        // CASE WHEN (x = 5 AND x IS NOT NULL) OR (x = bar AND x IS NOT NULL) THEN x ELSE 0
931        let e = when(
932            or(
933                and(col("x").eq(lit(5)), col("x").is_not_null()),
934                and(col("x").eq(col("bar")), col("x").is_not_null()),
935            ),
936            col("x"),
937        )
938        .otherwise(lit(0))?;
939        assert_not_nullable(&e, &nullable_schema);
940        assert_not_nullable(&e, &not_nullable_schema);
941
942        // CASE WHEN x = 5 OR x IS NULL THEN x ELSE 0
943        let e = when(or(col("x").eq(lit(5)), col("x").is_null()), col("x"))
944            .otherwise(lit(0))?;
945        assert_nullable(&e, &nullable_schema);
946        assert_not_nullable(&e, &not_nullable_schema);
947
948        // CASE WHEN x IS TRUE THEN x ELSE 0
949        let e = when(col("x").is_true(), col("x")).otherwise(lit(0))?;
950        assert_not_nullable(&e, &nullable_schema);
951        assert_not_nullable(&e, &not_nullable_schema);
952
953        // CASE WHEN x IS NOT TRUE THEN x ELSE 0
954        let e = when(col("x").is_not_true(), col("x")).otherwise(lit(0))?;
955        assert_nullable(&e, &nullable_schema);
956        assert_not_nullable(&e, &not_nullable_schema);
957
958        // CASE WHEN x IS FALSE THEN x ELSE 0
959        let e = when(col("x").is_false(), col("x")).otherwise(lit(0))?;
960        assert_not_nullable(&e, &nullable_schema);
961        assert_not_nullable(&e, &not_nullable_schema);
962
963        // CASE WHEN x IS NOT FALSE THEN x ELSE 0
964        let e = when(col("x").is_not_false(), col("x")).otherwise(lit(0))?;
965        assert_nullable(&e, &nullable_schema);
966        assert_not_nullable(&e, &not_nullable_schema);
967
968        // CASE WHEN x IS UNKNOWN THEN x ELSE 0
969        let e = when(col("x").is_unknown(), col("x")).otherwise(lit(0))?;
970        assert_nullable(&e, &nullable_schema);
971        assert_not_nullable(&e, &not_nullable_schema);
972
973        // CASE WHEN x IS NOT UNKNOWN THEN x ELSE 0
974        let e = when(col("x").is_not_unknown(), col("x")).otherwise(lit(0))?;
975        assert_not_nullable(&e, &nullable_schema);
976        assert_not_nullable(&e, &not_nullable_schema);
977
978        // CASE WHEN x LIKE 'x' THEN x ELSE 0
979        let e = when(col("x").like(lit("x")), col("x")).otherwise(lit(0))?;
980        assert_not_nullable(&e, &nullable_schema);
981        assert_not_nullable(&e, &not_nullable_schema);
982
983        // CASE WHEN 0 THEN x ELSE 0
984        let e = when(lit(0), col("x")).otherwise(lit(0))?;
985        assert_not_nullable(&e, &nullable_schema);
986        assert_not_nullable(&e, &not_nullable_schema);
987
988        // CASE WHEN 1 THEN x ELSE 0
989        let e = when(lit(1), col("x")).otherwise(lit(0))?;
990        assert_nullable(&e, &nullable_schema);
991        assert_not_nullable(&e, &not_nullable_schema);
992
993        Ok(())
994    }
995
996    #[test]
997    fn test_inlist_nullability() {
998        let get_schema = |nullable| {
999            MockExprSchema::new()
1000                .with_data_type(DataType::Int32)
1001                .with_nullable(nullable)
1002        };
1003
1004        let expr = col("foo").in_list(vec![lit(1); 5], false);
1005        assert!(!expr.nullable(&get_schema(false)).unwrap());
1006        assert!(expr.nullable(&get_schema(true)).unwrap());
1007        // Testing nullable() returns an error.
1008        assert!(
1009            expr.nullable(&get_schema(false).with_error_on_nullable(true))
1010                .is_err()
1011        );
1012
1013        let null = lit(ScalarValue::Int32(None));
1014        let expr = col("foo").in_list(vec![null, lit(1)], false);
1015        assert!(expr.nullable(&get_schema(false)).unwrap());
1016
1017        // Testing on long list
1018        let expr = col("foo").in_list(vec![lit(1); 6], false);
1019        assert!(expr.nullable(&get_schema(false)).unwrap());
1020    }
1021
1022    #[test]
1023    fn test_like_nullability() {
1024        let get_schema = |nullable| {
1025            MockExprSchema::new()
1026                .with_data_type(DataType::Utf8)
1027                .with_nullable(nullable)
1028        };
1029
1030        let expr = col("foo").like(lit("bar"));
1031        assert!(!expr.nullable(&get_schema(false)).unwrap());
1032        assert!(expr.nullable(&get_schema(true)).unwrap());
1033
1034        let expr = col("foo").like(lit(ScalarValue::Utf8(None)));
1035        assert!(expr.nullable(&get_schema(false)).unwrap());
1036    }
1037
1038    #[test]
1039    fn expr_schema_data_type() {
1040        let expr = col("foo");
1041        assert_eq!(
1042            DataType::Utf8,
1043            expr.get_type(&MockExprSchema::new().with_data_type(DataType::Utf8))
1044                .unwrap()
1045        );
1046    }
1047
1048    #[test]
1049    fn test_expr_metadata() {
1050        let mut meta = HashMap::new();
1051        meta.insert("bar".to_string(), "buzz".to_string());
1052        let meta = FieldMetadata::from(meta);
1053        let expr = col("foo");
1054        let schema = MockExprSchema::new()
1055            .with_data_type(DataType::Int32)
1056            .with_metadata(meta.clone());
1057
1058        // col, alias, and cast should be metadata-preserving
1059        assert_eq!(meta, expr.metadata(&schema).unwrap());
1060        assert_eq!(meta, expr.clone().alias("bar").metadata(&schema).unwrap());
1061        assert_eq!(
1062            meta,
1063            expr.clone()
1064                .cast_to(&DataType::Int64, &schema)
1065                .unwrap()
1066                .metadata(&schema)
1067                .unwrap()
1068        );
1069
1070        let schema = DFSchema::from_unqualified_fields(
1071            vec![meta.add_to_field(Field::new("foo", DataType::Int32, true))].into(),
1072            HashMap::new(),
1073        )
1074        .unwrap();
1075
1076        // verify to_field method populates metadata
1077        assert_eq!(meta, expr.metadata(&schema).unwrap());
1078
1079        // outer ref constructed by `out_ref_col_with_metadata` should be metadata-preserving
1080        let outer_ref = out_ref_col_with_metadata(
1081            DataType::Int32,
1082            meta.to_hashmap(),
1083            Column::from_name("foo"),
1084        );
1085        assert_eq!(meta, outer_ref.metadata(&schema).unwrap());
1086    }
1087
1088    #[test]
1089    fn test_alias_metadata_is_preserved_in_field_metadata() {
1090        let schema = MockExprSchema::new().with_data_type(DataType::Int32);
1091        let alias_metadata = FieldMetadata::from(HashMap::from([(
1092            "some_key".to_string(),
1093            "some_value".to_string(),
1094        )]));
1095
1096        let Expr::Alias(alias) = col("foo").alias("alias") else {
1097            unreachable!();
1098        };
1099        let expr = Expr::Alias(alias.with_metadata(Some(alias_metadata.clone())));
1100
1101        let field = expr.to_field(&schema).unwrap().1;
1102        assert_eq!(
1103            field.metadata().get("some_key"),
1104            Some(&"some_value".to_string())
1105        );
1106        assert_eq!(expr.metadata(&schema).unwrap(), alias_metadata);
1107    }
1108
1109    #[test]
1110    fn test_expr_placeholder() {
1111        let schema = MockExprSchema::new();
1112
1113        let mut placeholder_meta = HashMap::new();
1114        placeholder_meta.insert("bar".to_string(), "buzz".to_string());
1115        let placeholder_meta = FieldMetadata::from(placeholder_meta);
1116
1117        let expr = Expr::Placeholder(Placeholder::new_with_field(
1118            "".to_string(),
1119            Some(
1120                Field::new("", DataType::Utf8, true)
1121                    .with_metadata(placeholder_meta.to_hashmap())
1122                    .into(),
1123            ),
1124        ));
1125
1126        let field = expr.to_field(&schema).unwrap().1;
1127        assert_eq!(
1128            (field.data_type(), field.is_nullable()),
1129            (&DataType::Utf8, true)
1130        );
1131        assert_eq!(placeholder_meta, expr.metadata(&schema).unwrap());
1132
1133        let expr_alias = expr.alias("a placeholder by any other name");
1134        let expr_alias_field = expr_alias.to_field(&schema).unwrap().1;
1135        assert_eq!(
1136            (expr_alias_field.data_type(), expr_alias_field.is_nullable()),
1137            (&DataType::Utf8, true)
1138        );
1139        assert_eq!(placeholder_meta, expr_alias.metadata(&schema).unwrap());
1140
1141        // Non-nullable placeholder field should remain non-nullable
1142        let expr = Expr::Placeholder(Placeholder::new_with_field(
1143            "".to_string(),
1144            Some(Field::new("", DataType::Utf8, false).into()),
1145        ));
1146        let expr_field = expr.to_field(&schema).unwrap().1;
1147        assert_eq!(
1148            (expr_field.data_type(), expr_field.is_nullable()),
1149            (&DataType::Utf8, false)
1150        );
1151
1152        let expr_alias = expr.alias("a placeholder by any other name");
1153        let expr_alias_field = expr_alias.to_field(&schema).unwrap().1;
1154        assert_eq!(
1155            (expr_alias_field.data_type(), expr_alias_field.is_nullable()),
1156            (&DataType::Utf8, false)
1157        );
1158    }
1159
1160    #[derive(Debug)]
1161    struct MockExprSchema {
1162        field: FieldRef,
1163        error_on_nullable: bool,
1164    }
1165
1166    impl MockExprSchema {
1167        fn new() -> Self {
1168            Self {
1169                field: Arc::new(Field::new("mock_field", DataType::Null, false)),
1170                error_on_nullable: false,
1171            }
1172        }
1173
1174        fn with_nullable(mut self, nullable: bool) -> Self {
1175            Arc::make_mut(&mut self.field).set_nullable(nullable);
1176            self
1177        }
1178
1179        fn with_data_type(mut self, data_type: DataType) -> Self {
1180            Arc::make_mut(&mut self.field).set_data_type(data_type);
1181            self
1182        }
1183
1184        fn with_error_on_nullable(mut self, error_on_nullable: bool) -> Self {
1185            self.error_on_nullable = error_on_nullable;
1186            self
1187        }
1188
1189        fn with_metadata(mut self, metadata: FieldMetadata) -> Self {
1190            self.field =
1191                Arc::new(metadata.add_to_field(Arc::unwrap_or_clone(self.field)));
1192            self
1193        }
1194    }
1195
1196    impl ExprSchema for MockExprSchema {
1197        fn nullable(&self, _col: &Column) -> Result<bool> {
1198            assert_or_internal_err!(!self.error_on_nullable, "nullable error");
1199            Ok(self.field.is_nullable())
1200        }
1201
1202        fn field_from_column(&self, _col: &Column) -> Result<&FieldRef> {
1203            Ok(&self.field)
1204        }
1205    }
1206
1207    /// A scan of `t`, whose single column `a` has the given nullability.
1208    fn scan_t(a_nullable: bool) -> LogicalPlanBuilder {
1209        let schema = Schema::new(vec![Field::new("a", DataType::Int32, a_nullable)]);
1210        let source = Arc::new(LogicalTableSource::new(Arc::new(schema)));
1211        LogicalPlanBuilder::scan("t", source, None).unwrap()
1212    }
1213
1214    #[test]
1215    fn in_subquery_nullability() {
1216        // `x IN (SELECT a FROM t)` evaluates to NULL when `x` is NULL, and when `x`
1217        // matches no row while `a` contains a NULL. So it is nullable exactly when
1218        // either the compared expression or the subquery's output column is.
1219        let cases = [
1220            (false, false, false),
1221            (false, true, true),
1222            (true, false, true),
1223            (true, true, true),
1224        ];
1225
1226        for (x_nullable, a_nullable, expected) in cases {
1227            let subquery = scan_t(a_nullable)
1228                .project(vec![col("a")])
1229                .unwrap()
1230                .build()
1231                .unwrap();
1232            let expr = in_subquery(col("x"), Arc::new(subquery));
1233            let schema = MockExprSchema::new().with_nullable(x_nullable);
1234
1235            assert_eq!(expr.nullable(&schema).unwrap(), expected);
1236        }
1237    }
1238
1239    #[test]
1240    fn in_subquery_nullability_uses_subquery_output_schema() {
1241        // `DISTINCT` carries no expressions of its own, but its output column is still
1242        // nullable, so the `IN` expression must be nullable too.
1243        let subquery = scan_t(true)
1244            .project(vec![col("a")])
1245            .unwrap()
1246            .distinct()
1247            .unwrap()
1248            .build()
1249            .unwrap();
1250        let expr = in_subquery(col("x"), Arc::new(subquery));
1251        assert!(expr.nullable(&MockExprSchema::new()).unwrap());
1252
1253        // A computed projection's expressions reference `t.a`, which does not appear in
1254        // the subquery's output schema, so nullability must be read off that schema's
1255        // single column rather than by resolving the projection's expressions against it.
1256        let subquery = scan_t(false)
1257            .project(vec![col("a") + lit(1)])
1258            .unwrap()
1259            .build()
1260            .unwrap();
1261        let expr = in_subquery(col("x"), Arc::new(subquery));
1262        assert!(!expr.nullable(&MockExprSchema::new()).unwrap());
1263    }
1264
1265    #[test]
1266    fn in_subquery_nullability_errors_for_no_subquery_columns() {
1267        let subquery = LogicalPlanBuilder::empty(false).build().unwrap();
1268        let expr = in_subquery(col("x"), Arc::new(subquery));
1269
1270        let err = expr.nullable(&MockExprSchema::new()).unwrap_err();
1271        assert_eq!(
1272            err.strip_backtrace(),
1273            "Error during planning: subquery must return exactly one column of data to compare against"
1274        );
1275    }
1276
1277    #[test]
1278    fn test_scalar_variable() {
1279        let mut meta = HashMap::new();
1280        meta.insert("bar".to_string(), "buzz".to_string());
1281        let meta = FieldMetadata::from(meta);
1282
1283        let field = Field::new("foo", DataType::Int32, true);
1284        let field = meta.add_to_field(field);
1285        let field = Arc::new(field);
1286
1287        let expr = Expr::ScalarVariable(field, vec!["foo".to_string()]);
1288
1289        let schema = MockExprSchema::new();
1290
1291        assert_eq!(meta, expr.metadata(&schema).unwrap());
1292    }
1293}
datafusion_expr/expr_schema.rs

datafusion_expr/
expr_schema.rs
