datafusion-physical-expr 55.1.0

// Licensed to the Apache Software Foundation (ASF) under one
// or more contributor license agreements.  See the NOTICE file
// distributed with this work for additional information
// regarding copyright ownership.  The ASF licenses this file
// to you under the Apache License, Version 2.0 (the
// "License"); you may not use this file except in compliance
// with the License.  You may obtain a copy of the License at
//
//   http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing,
// software distributed under the License is distributed on an
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
// KIND, either express or implied.  See the License for the
// specific language governing permissions and limitations
// under the License.

use std::collections::HashMap;
use std::fmt;
use std::hash::Hash;
use std::sync::Arc;

use crate::PhysicalExpr;
use arrow::compute;
use arrow::compute::CastOptions;
use arrow::datatypes::{DataType, Field, FieldRef, Schema};
use arrow::record_batch::RecordBatch;
use arrow_schema::extension::{EXTENSION_TYPE_METADATA_KEY, EXTENSION_TYPE_NAME_KEY};
use compute::can_cast_types;
use datafusion_common::datatype::DataTypeExt;
use datafusion_common::format::DEFAULT_FORMAT_OPTIONS;
use datafusion_common::{Result, not_impl_err};
use datafusion_expr::ColumnarValue;

/// TRY_CAST expression casts an expression to a specific data type and returns NULL on invalid cast
#[derive(Debug, Clone, Eq)]
pub struct TryCastExpr {
    /// The expression to cast
    expr: Arc<dyn PhysicalExpr>,
    /// The target field.
    ///
    /// For a type-only cast (see [`TryCastExpr::new`]) this is a field
    /// synthesized from the target data type alone and only its data type is
    /// meaningful. For a cast built from an explicit field (see
    /// [`TryCastExpr::new_with_target_field`]) its metadata is applied to the
    /// output field as-is.
    target_field: FieldRef,
    /// Whether `target_field` was supplied by the caller (as opposed to being
    /// synthesized from a `DataType`), and therefore whether its metadata
    /// describes the output field exactly.
    explicit_target: bool,
}

// Manually derive PartialEq and Hash to work around https://github.com/rust-lang/rust/issues/78808
impl PartialEq for TryCastExpr {
    fn eq(&self, other: &Self) -> bool {
        // Compare the semantically meaningful parts of the target field only:
        // the field name never affects the output of this expression.
        self.expr.eq(&other.expr)
            && self.cast_type() == other.cast_type()
            && self.target_metadata() == other.target_metadata()
    }
}

impl Hash for TryCastExpr {
    fn hash<H: std::hash::Hasher>(&self, state: &mut H) {
        self.expr.hash(state);
        self.cast_type().hash(state);
        // Hash the metadata by iterating over sorted keys for deterministic ordering
        if let Some(metadata) = self.target_metadata() {
            let mut entries: Vec<_> = metadata.iter().collect();
            entries.sort_by_key(|(k, _)| *k);
            for (k, v) in entries {
                k.hash(state);
                v.hash(state);
            }
        }
    }
}

impl TryCastExpr {
    /// Create a new `TryCastExpr` using only a `DataType`.
    ///
    /// This constructor creates a type-only cast where metadata is passed through
    /// from the source expression (with extension type keys stripped).
    /// TRY_CAST results are always nullable since failed casts return NULL.
    pub fn new(expr: Arc<dyn PhysicalExpr>, cast_type: DataType) -> Self {
        Self {
            expr,
            target_field: cast_type.into_nullable_field_ref(),
            explicit_target: false,
        }
    }

    /// Create a new `TryCastExpr` with an explicit target `FieldRef`.
    ///
    /// The provided `target_field` determines the output characteristics:
    /// - The field's data type becomes the cast target type
    /// - The field's metadata is used exactly as provided
    ///
    /// TRY_CAST results are always nullable since failed casts return NULL.
    ///
    /// See [`TryCastExpr::new`] for type-only casts where source metadata should
    /// pass through.
    pub fn new_with_target_field(
        expr: Arc<dyn PhysicalExpr>,
        target_field: FieldRef,
    ) -> Self {
        Self {
            expr,
            target_field,
            explicit_target: true,
        }
    }

    /// The expression to cast
    pub fn expr(&self) -> &Arc<dyn PhysicalExpr> {
        &self.expr
    }

    /// The data type to cast to
    pub fn cast_type(&self) -> &DataType {
        self.target_field.data_type()
    }

    /// Explicit metadata for the output field, or `None` to pass through source metadata.
    pub fn target_metadata(&self) -> Option<&HashMap<String, String>> {
        self.explicit_target.then(|| self.target_field.metadata())
    }

    /// The target field this cast was constructed with.
    ///
    /// For a type-only cast this is a field synthesized from the target data
    /// type alone; only its data type is meaningful. TRY_CAST results are
    /// always nullable regardless of the target field's nullability.
    pub fn target_field(&self) -> &FieldRef {
        &self.target_field
    }
}

impl fmt::Display for TryCastExpr {
    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
        write!(f, "TRY_CAST({} AS {})", self.expr, self.cast_type())
    }
}

impl PhysicalExpr for TryCastExpr {
    fn data_type(&self, _input_schema: &Schema) -> Result<DataType> {
        Ok(self.cast_type().clone())
    }

    fn nullable(&self, _input_schema: &Schema) -> Result<bool> {
        Ok(true)
    }

    fn evaluate(&self, batch: &RecordBatch) -> Result<ColumnarValue> {
        let value = self.expr.evaluate(batch)?;
        let options = CastOptions {
            safe: true,
            format_options: DEFAULT_FORMAT_OPTIONS,
        };
        value.cast_to(self.cast_type(), Some(&options))
    }

    fn return_field(&self, input_schema: &Schema) -> Result<FieldRef> {
        // If metadata is explicit, we can build the field without source
        // (though we still try to get source for the name)
        let source_result = self.expr.return_field(input_schema);

        if let Some(metadata) = self.target_metadata() {
            // Explicit metadata: use it exactly, TRY_CAST is always nullable
            let name = source_result
                .as_ref()
                .map(|f| f.name().to_string())
                .unwrap_or_default();
            return Ok(Arc::new(
                Field::new(name, self.cast_type().clone(), true)
                    .with_metadata(metadata.clone()),
            ));
        }

        // Pass-through metadata from source (stripping extension keys)
        source_result.map(|source_field| {
            let mut metadata = source_field.metadata().clone();
            metadata.remove(EXTENSION_TYPE_NAME_KEY);
            metadata.remove(EXTENSION_TYPE_METADATA_KEY);

            Arc::new(
                source_field
                    .as_ref()
                    .clone()
                    .with_data_type(self.cast_type().clone())
                    .with_nullable(true) // TRY_CAST is always nullable
                    .with_metadata(metadata),
            )
        })
    }

    fn children(&self) -> Vec<&Arc<dyn PhysicalExpr>> {
        vec![&self.expr]
    }

    fn with_new_children(
        self: Arc<Self>,
        children: Vec<Arc<dyn PhysicalExpr>>,
    ) -> Result<Arc<dyn PhysicalExpr>> {
        Ok(Arc::new(TryCastExpr {
            expr: Arc::clone(&children[0]),
            target_field: Arc::clone(&self.target_field),
            explicit_target: self.explicit_target,
        }))
    }

    fn fmt_sql(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
        write!(f, "TRY_CAST(")?;
        self.expr.fmt_sql(f)?;
        write!(f, " AS {:?})", self.cast_type())
    }

    #[cfg(feature = "proto")]
    fn try_to_proto(
        &self,
        ctx: &datafusion_physical_expr_common::physical_expr::proto_encode::PhysicalExprEncodeCtx<'_>,
    ) -> Result<Option<datafusion_proto_models::protobuf::PhysicalExprNode>> {
        use datafusion_proto_models::protobuf;

        Ok(Some(protobuf::PhysicalExprNode {
            expr_id: None,
            expr_type: Some(protobuf::physical_expr_node::ExprType::TryCast(Box::new(
                protobuf::PhysicalTryCastNode {
                    expr: Some(Box::new(ctx.encode_child(&self.expr)?)),
                    arrow_type: Some(self.cast_type().try_into()?),
                },
            ))),
        }))
    }
}

#[cfg(feature = "proto")]
impl TryCastExpr {
    /// Reconstruct a [`TryCastExpr`] from its protobuf representation.
    pub fn try_from_proto(
        node: &datafusion_proto_models::protobuf::PhysicalExprNode,
        ctx: &datafusion_physical_expr_common::physical_expr::proto_decode::PhysicalExprDecodeCtx<'_>,
    ) -> Result<Arc<dyn PhysicalExpr>> {
        use datafusion_physical_expr_common::expect_expr_variant;
        use datafusion_physical_expr_common::physical_expr::proto_decode::require_proto_field;
        use datafusion_proto_models::protobuf;

        let try_cast = expect_expr_variant!(
            node,
            protobuf::physical_expr_node::ExprType::TryCast,
            "TryCastExpr",
        );
        let expr = ctx.decode_required_expression(
            try_cast.expr.as_deref(),
            "TryCastExpr",
            "expr",
        )?;
        let arrow_type = require_proto_field(
            try_cast.arrow_type.as_ref(),
            "TryCastExpr",
            "arrow_type",
        )?;
        let cast_type: DataType = arrow_type.try_into()?;

        Ok(Arc::new(TryCastExpr::new(expr, cast_type)))
    }
}

/// Return a PhysicalExpression representing `expr` casted to
/// `cast_type`, if any casting is needed.
///
/// Note that such casts may lose type information
pub fn try_cast(
    expr: Arc<dyn PhysicalExpr>,
    input_schema: &Schema,
    cast_type: DataType,
) -> Result<Arc<dyn PhysicalExpr>> {
    let expr_type = expr.data_type(input_schema)?;
    if expr_type == cast_type {
        Ok(Arc::clone(&expr))
    } else if can_cast_types(&expr_type, &cast_type) {
        Ok(Arc::new(TryCastExpr::new(expr, cast_type)))
    } else {
        not_impl_err!("Unsupported TRY_CAST from {expr_type} to {cast_type}")
    }
}

/// Return a PhysicalExpression representing `expr` casted to `target_field`,
/// preserving any explicit field semantics such as metadata.
///
/// TRY_CAST results are always nullable since failed casts return NULL.
///
/// If the input expression already has the same data type, the target field
/// has no explicit metadata constraints, and the source has no extension
/// metadata to strip, the original expression is returned unchanged.
pub fn try_cast_with_target_field(
    expr: Arc<dyn PhysicalExpr>,
    input_schema: &Schema,
    target_field: &FieldRef,
) -> Result<Arc<dyn PhysicalExpr>> {
    let expr_type = expr.data_type(input_schema)?;
    let cast_type = target_field.data_type();

    // Check if this is a "default" target field (type-only cast with no explicit
    // metadata constraints). This is the field created by `into_nullable_field_ref()`
    // when only a DataType is known.
    let is_type_only = target_field.name().is_empty()
        && target_field.is_nullable()
        && target_field.metadata().is_empty();

    // For same-type casts, we can skip creating a TryCastExpr only if:
    // 1. The target is type-only (no explicit metadata)
    // 2. The source has no extension metadata that needs to be stripped
    // Otherwise we need the TryCastExpr to strip extension metadata from the source.
    if expr_type == *cast_type && is_type_only {
        let source_field = expr.return_field(input_schema)?;
        let has_extension_metadata = source_field
            .metadata()
            .contains_key(EXTENSION_TYPE_NAME_KEY);
        if !has_extension_metadata {
            return Ok(Arc::clone(&expr));
        }
    }

    if !can_cast_types(&expr_type, cast_type) {
        return not_impl_err!("Unsupported TRY_CAST from {expr_type} to {cast_type}");
    }

    // For type-only casts, use TryCastExpr::new which preserves source metadata.
    // For explicit target fields, use new_with_target_field which applies the target's
    // metadata exactly.
    if is_type_only {
        Ok(Arc::new(TryCastExpr::new(expr, cast_type.clone())))
    } else {
        Ok(Arc::new(TryCastExpr::new_with_target_field(
            expr,
            Arc::clone(target_field),
        )))
    }
}

#[cfg(test)]
mod tests {
    use super::*;
    use crate::expressions::col;
    use arrow::array::{
        Decimal128Array, Decimal128Builder, StringArray, Time64NanosecondArray,
    };
    use arrow::{
        array::{
            Array, Float32Array, Float64Array, Int8Array, Int16Array, Int32Array,
            Int64Array, TimestampNanosecondArray, UInt32Array,
        },
        datatypes::*,
    };
    use datafusion_physical_expr_common::physical_expr::fmt_sql;

    // runs an end-to-end test of physical type cast
    // 1. construct a record batch with a column "a" of type A
    // 2. construct a physical expression of TRY_CAST(a AS B)
    // 3. evaluate the expression
    // 4. verify that the resulting expression is of type B
    // 5. verify that the resulting values are downcastable and correct
    macro_rules! generic_decimal_to_other_test_cast {
        ($DECIMAL_ARRAY:ident, $A_TYPE:expr, $TYPEARRAY:ident, $TYPE:expr, $VEC:expr) => {{
            let schema = Schema::new(vec![Field::new("a", $A_TYPE, true)]);
            let batch = RecordBatch::try_new(
                Arc::new(schema.clone()),
                vec![Arc::new($DECIMAL_ARRAY)],
            )?;
            // verify that we can construct the expression
            let expression = try_cast(col("a", &schema)?, &schema, $TYPE)?;

            // verify that its display is correct
            assert_eq!(
                format!("TRY_CAST(a@0 AS {})", $TYPE),
                format!("{}", expression)
            );

            // verify that the expression's type is correct
            assert_eq!(expression.data_type(&schema)?, $TYPE);

            // compute
            let result = expression
                .evaluate(&batch)?
                .into_array(batch.num_rows())
                .expect("Failed to convert to array");

            // verify that the array's data_type is correct
            assert_eq!(*result.data_type(), $TYPE);

            // verify that the data itself is downcastable
            let result = result
                .as_any()
                .downcast_ref::<$TYPEARRAY>()
                .expect("failed to downcast");

            // verify that the result itself is correct
            for (i, x) in $VEC.iter().enumerate() {
                match x {
                    Some(x) => assert_eq!(result.value(i), *x),
                    None => assert!(result.is_null(i)),
                }
            }
        }};
    }

    // runs an end-to-end test of physical type cast
    // 1. construct a record batch with a column "a" of type A
    // 2. construct a physical expression of TRY_CAST(a AS B)
    // 3. evaluate the expression
    // 4. verify that the resulting expression is of type B
    // 5. verify that the resulting values are downcastable and correct
    macro_rules! generic_test_cast {
        ($A_ARRAY:ident, $A_TYPE:expr, $A_VEC:expr, $TYPEARRAY:ident, $TYPE:expr, $VEC:expr) => {{
            let schema = Schema::new(vec![Field::new("a", $A_TYPE, true)]);
            let a_vec_len = $A_VEC.len();
            let a = $A_ARRAY::from($A_VEC);
            let batch =
                RecordBatch::try_new(Arc::new(schema.clone()), vec![Arc::new(a)])?;

            // verify that we can construct the expression
            let expression = try_cast(col("a", &schema)?, &schema, $TYPE)?;

            // verify that its display is correct
            assert_eq!(
                format!("TRY_CAST(a@0 AS {})", $TYPE),
                format!("{}", expression)
            );

            // verify that the expression's type is correct
            assert_eq!(expression.data_type(&schema)?, $TYPE);

            // compute
            let result = expression
                .evaluate(&batch)?
                .into_array(batch.num_rows())
                .expect("Failed to convert to array");

            // verify that the array's data_type is correct
            assert_eq!(*result.data_type(), $TYPE);

            // verify that the len is correct
            assert_eq!(result.len(), a_vec_len);

            // verify that the data itself is downcastable
            let result = result
                .as_any()
                .downcast_ref::<$TYPEARRAY>()
                .expect("failed to downcast");

            // verify that the result itself is correct
            for (i, x) in $VEC.iter().enumerate() {
                match x {
                    Some(x) => assert_eq!(result.value(i), *x),
                    None => assert!(result.is_null(i)),
                }
            }
        }};
    }

    #[test]
    fn test_try_cast_decimal_to_decimal() -> Result<()> {
        // try cast one decimal data type to another decimal data type
        let array: Vec<i128> = vec![1234, 2222, 3, 4000, 5000];
        let decimal_array = create_decimal_array(&array, 10, 3);
        generic_decimal_to_other_test_cast!(
            decimal_array,
            DataType::Decimal128(10, 3),
            Decimal128Array,
            DataType::Decimal128(20, 6),
            [
                Some(1_234_000),
                Some(2_222_000),
                Some(3_000),
                Some(4_000_000),
                Some(5_000_000),
                None
            ]
        );

        let decimal_array = create_decimal_array(&array, 10, 3);
        generic_decimal_to_other_test_cast!(
            decimal_array,
            DataType::Decimal128(10, 3),
            Decimal128Array,
            DataType::Decimal128(10, 2),
            [Some(123), Some(222), Some(0), Some(400), Some(500), None]
        );

        Ok(())
    }

    #[test]
    fn test_try_cast_decimal_to_numeric() -> Result<()> {
        // TODO we should add function to create Decimal128Array with value and metadata
        // https://github.com/apache/arrow-rs/issues/1009
        let array: Vec<i128> = vec![1, 2, 3, 4, 5];
        let decimal_array = create_decimal_array(&array, 10, 0);
        // decimal to i8
        generic_decimal_to_other_test_cast!(
            decimal_array,
            DataType::Decimal128(10, 0),
            Int8Array,
            DataType::Int8,
            [
                Some(1_i8),
                Some(2_i8),
                Some(3_i8),
                Some(4_i8),
                Some(5_i8),
                None
            ]
        );

        // decimal to i16
        let decimal_array = create_decimal_array(&array, 10, 0);
        generic_decimal_to_other_test_cast!(
            decimal_array,
            DataType::Decimal128(10, 0),
            Int16Array,
            DataType::Int16,
            [
                Some(1_i16),
                Some(2_i16),
                Some(3_i16),
                Some(4_i16),
                Some(5_i16),
                None
            ]
        );

        // decimal to i32
        let decimal_array = create_decimal_array(&array, 10, 0);
        generic_decimal_to_other_test_cast!(
            decimal_array,
            DataType::Decimal128(10, 0),
            Int32Array,
            DataType::Int32,
            [
                Some(1_i32),
                Some(2_i32),
                Some(3_i32),
                Some(4_i32),
                Some(5_i32),
                None
            ]
        );

        // decimal to i64
        let decimal_array = create_decimal_array(&array, 10, 0);
        generic_decimal_to_other_test_cast!(
            decimal_array,
            DataType::Decimal128(10, 0),
            Int64Array,
            DataType::Int64,
            [
                Some(1_i64),
                Some(2_i64),
                Some(3_i64),
                Some(4_i64),
                Some(5_i64),
                None
            ]
        );

        // decimal to float32
        let array: Vec<i128> = vec![1234, 2222, 3, 4000, 5000];
        let decimal_array = create_decimal_array(&array, 10, 3);
        generic_decimal_to_other_test_cast!(
            decimal_array,
            DataType::Decimal128(10, 3),
            Float32Array,
            DataType::Float32,
            [
                Some(1.234_f32),
                Some(2.222_f32),
                Some(0.003_f32),
                Some(4.0_f32),
                Some(5.0_f32),
                None
            ]
        );
        // decimal to float64
        let decimal_array = create_decimal_array(&array, 20, 6);
        generic_decimal_to_other_test_cast!(
            decimal_array,
            DataType::Decimal128(20, 6),
            Float64Array,
            DataType::Float64,
            [
                Some(0.001234_f64),
                Some(0.002222_f64),
                Some(0.000003_f64),
                Some(0.004_f64),
                Some(0.005_f64),
                None
            ]
        );

        Ok(())
    }

    #[test]
    fn test_try_cast_numeric_to_decimal() -> Result<()> {
        // int8
        generic_test_cast!(
            Int8Array,
            DataType::Int8,
            vec![1, 2, 3, 4, 5],
            Decimal128Array,
            DataType::Decimal128(3, 0),
            [Some(1), Some(2), Some(3), Some(4), Some(5)]
        );

        // int16
        generic_test_cast!(
            Int16Array,
            DataType::Int16,
            vec![1, 2, 3, 4, 5],
            Decimal128Array,
            DataType::Decimal128(5, 0),
            [Some(1), Some(2), Some(3), Some(4), Some(5)]
        );

        // int32
        generic_test_cast!(
            Int32Array,
            DataType::Int32,
            vec![1, 2, 3, 4, 5],
            Decimal128Array,
            DataType::Decimal128(10, 0),
            [Some(1), Some(2), Some(3), Some(4), Some(5)]
        );

        // int64
        generic_test_cast!(
            Int64Array,
            DataType::Int64,
            vec![1, 2, 3, 4, 5],
            Decimal128Array,
            DataType::Decimal128(20, 0),
            [Some(1), Some(2), Some(3), Some(4), Some(5)]
        );

        // int64 to different scale
        generic_test_cast!(
            Int64Array,
            DataType::Int64,
            vec![1, 2, 3, 4, 5],
            Decimal128Array,
            DataType::Decimal128(20, 2),
            [Some(100), Some(200), Some(300), Some(400), Some(500)]
        );

        // float32
        generic_test_cast!(
            Float32Array,
            DataType::Float32,
            vec![1.5, 2.5, 3.0, 1.123_456_8, 5.50],
            Decimal128Array,
            DataType::Decimal128(10, 2),
            [Some(150), Some(250), Some(300), Some(112), Some(550)]
        );

        // float64
        generic_test_cast!(
            Float64Array,
            DataType::Float64,
            vec![1.5, 2.5, 3.0, 1.123_456_8, 5.50],
            Decimal128Array,
            DataType::Decimal128(20, 4),
            [
                Some(15000),
                Some(25000),
                Some(30000),
                Some(11235),
                Some(55000)
            ]
        );
        Ok(())
    }

    #[test]
    fn test_cast_i32_u32() -> Result<()> {
        generic_test_cast!(
            Int32Array,
            DataType::Int32,
            vec![1, 2, 3, 4, 5],
            UInt32Array,
            DataType::UInt32,
            [
                Some(1_u32),
                Some(2_u32),
                Some(3_u32),
                Some(4_u32),
                Some(5_u32)
            ]
        );
        Ok(())
    }

    #[test]
    fn test_cast_i32_utf8() -> Result<()> {
        generic_test_cast!(
            Int32Array,
            DataType::Int32,
            vec![1, 2, 3, 4, 5],
            StringArray,
            DataType::Utf8,
            [Some("1"), Some("2"), Some("3"), Some("4"), Some("5")]
        );
        Ok(())
    }

    #[test]
    fn test_try_cast_utf8_i32() -> Result<()> {
        generic_test_cast!(
            StringArray,
            DataType::Utf8,
            vec!["a", "2", "3", "b", "5"],
            Int32Array,
            DataType::Int32,
            [None, Some(2), Some(3), None, Some(5)]
        );
        Ok(())
    }

    #[test]
    fn test_cast_i64_t64() -> Result<()> {
        let original = vec![1, 2, 3, 4, 5];
        let expected: Vec<Option<i64>> = original
            .iter()
            .map(|i| Some(Time64NanosecondArray::from(vec![*i]).value(0)))
            .collect();
        generic_test_cast!(
            Int64Array,
            DataType::Int64,
            original,
            TimestampNanosecondArray,
            DataType::Timestamp(TimeUnit::Nanosecond, None),
            expected
        );
        Ok(())
    }

    #[test]
    fn invalid_cast() {
        // Ensure a useful error happens at plan time if invalid casts are used
        let schema = Schema::new(vec![Field::new("a", DataType::Int32, false)]);

        let result = try_cast(
            col("a", &schema).unwrap(),
            &schema,
            DataType::Interval(IntervalUnit::MonthDayNano),
        );
        result.expect_err("expected Invalid TRY_CAST");
    }

    // create decimal array with the specified precision and scale
    fn create_decimal_array(array: &[i128], precision: u8, scale: i8) -> Decimal128Array {
        let mut decimal_builder = Decimal128Builder::with_capacity(array.len());
        for value in array {
            decimal_builder.append_value(*value);
        }
        decimal_builder.append_null();
        decimal_builder
            .finish()
            .with_precision_and_scale(precision, scale)
            .unwrap()
    }

    #[test]
    fn test_fmt_sql() -> Result<()> {
        let schema = Schema::new(vec![Field::new("a", DataType::Int32, true)]);

        // Test numeric casting
        let expr = try_cast(col("a", &schema)?, &schema, DataType::Int64)?;
        let display_string = expr.to_string();
        assert_eq!(display_string, "TRY_CAST(a@0 AS Int64)");
        let sql_string = fmt_sql(expr.as_ref()).to_string();
        assert_eq!(sql_string, "TRY_CAST(a AS Int64)");

        // Test string casting
        let schema = Schema::new(vec![Field::new("b", DataType::Utf8, true)]);
        let expr = try_cast(col("b", &schema)?, &schema, DataType::Int32)?;
        let display_string = expr.to_string();
        assert_eq!(display_string, "TRY_CAST(b@0 AS Int32)");
        let sql_string = fmt_sql(expr.as_ref()).to_string();
        assert_eq!(sql_string, "TRY_CAST(b AS Int32)");

        Ok(())
    }

    #[test]
    fn field_aware_try_cast_uses_exact_target_metadata() -> Result<()> {
        // When using field-aware cast, target's metadata should be used exactly
        let source_meta = HashMap::from([
            (
                EXTENSION_TYPE_NAME_KEY.to_string(),
                "source.type".to_string(),
            ),
            ("source_key".to_string(), "source_value".to_string()),
        ]);
        let target_meta = HashMap::from([
            (
                EXTENSION_TYPE_NAME_KEY.to_string(),
                "target.type".to_string(),
            ),
            (
                EXTENSION_TYPE_METADATA_KEY.to_string(),
                "target_ext_meta".to_string(),
            ),
            ("target_key".to_string(), "target_value".to_string()),
        ]);
        let schema = Schema::new(vec![
            Field::new("a", DataType::FixedSizeBinary(16), false)
                .with_metadata(source_meta),
        ]);

        let target_field =
            Arc::new(Field::new("b", DataType::Utf8, true).with_metadata(target_meta));
        let expr = TryCastExpr::new_with_target_field(
            col("a", &schema)?,
            Arc::clone(&target_field),
        );

        let field = expr.return_field(&schema)?;
        assert_eq!(
            field.metadata().get(EXTENSION_TYPE_NAME_KEY),
            Some(&"target.type".to_string()),
            "Field-aware try_cast should use target's extension type name"
        );
        assert_eq!(
            field.metadata().get(EXTENSION_TYPE_METADATA_KEY),
            Some(&"target_ext_meta".to_string()),
            "Field-aware try_cast should use target's extension type metadata"
        );
        assert!(
            field.metadata().get("source_key").is_none(),
            "Field-aware try_cast should NOT preserve source metadata"
        );
        assert_eq!(
            field.metadata().get("target_key"),
            Some(&"target_value".to_string()),
            "Field-aware try_cast should preserve target's non-extension metadata"
        );
        // TRY_CAST is always nullable
        assert!(field.is_nullable());

        Ok(())
    }

    #[test]
    fn field_aware_try_cast_preserves_target_field_semantics() -> Result<()> {
        // Target field metadata should be preserved exactly (no merging with source).
        // TRY_CAST is always nullable regardless of target field's nullability.
        let metadata = HashMap::from([("target_meta".to_string(), "1".to_string())]);

        for child_nullable in [true, false] {
            let schema =
                Schema::new(vec![Field::new("a", DataType::Int32, child_nullable)]);
            let target_field = Arc::new(
                Field::new("cast_target", DataType::Int64, false) // target says non-nullable
                    .with_metadata(metadata.clone()),
            );
            let expr = TryCastExpr::new_with_target_field(
                col("a", &schema)?,
                Arc::clone(&target_field),
            );

            let field = expr.return_field(&schema)?;
            // Field name comes from source
            assert_eq!(field.name(), "a");
            assert_eq!(field.data_type(), &DataType::Int64);
            // TRY_CAST is ALWAYS nullable (ignores target field's nullability)
            assert!(field.is_nullable(), "TRY_CAST should always be nullable");
            // Target metadata should be preserved exactly
            assert_eq!(
                field.metadata().get("target_meta"),
                Some(&"1".to_string()),
                "Target metadata should be preserved exactly"
            );
            assert!(
                expr.nullable(&schema)?,
                "TRY_CAST should always be nullable"
            );
        }

        Ok(())
    }

    #[test]
    fn type_only_try_cast_strips_extension_keys() -> Result<()> {
        // Type-only cast should strip extension keys but preserve other source metadata
        let source_meta = HashMap::from([
            (
                EXTENSION_TYPE_NAME_KEY.to_string(),
                "source.extension".to_string(),
            ),
            (
                EXTENSION_TYPE_METADATA_KEY.to_string(),
                "ext_meta".to_string(),
            ),
            ("custom_key".to_string(), "custom_value".to_string()),
        ]);
        let schema = Schema::new(vec![
            Field::new("a", DataType::Int32, false).with_metadata(source_meta),
        ]);

        let expr = TryCastExpr::new(col("a", &schema)?, DataType::Int64);
        let field = expr.return_field(&schema)?;

        // Extension keys should be stripped
        assert!(
            field.metadata().get(EXTENSION_TYPE_NAME_KEY).is_none(),
            "Type-only try_cast should strip extension type name"
        );
        assert!(
            field.metadata().get(EXTENSION_TYPE_METADATA_KEY).is_none(),
            "Type-only try_cast should strip extension type metadata"
        );
        // Non-extension metadata should pass through
        assert_eq!(
            field.metadata().get("custom_key"),
            Some(&"custom_value".to_string()),
            "Type-only try_cast should preserve non-extension metadata"
        );
        // Field name preserved, type changed, always nullable
        assert_eq!(field.name(), "a");
        assert_eq!(field.data_type(), &DataType::Int64);
        assert!(field.is_nullable());

        Ok(())
    }

    #[test]
    fn type_only_try_cast_is_always_nullable() -> Result<()> {
        // TRY_CAST is always nullable even when source is non-nullable
        let schema = Schema::new(vec![Field::new("a", DataType::Int32, false)]);
        let expr = TryCastExpr::new(col("a", &schema)?, DataType::Int64);

        let field = expr.return_field(&schema)?;

        assert_eq!(field.name(), "a");
        assert_eq!(field.data_type(), &DataType::Int64);
        assert!(field.is_nullable(), "TRY_CAST should always be nullable");
        assert!(
            expr.nullable(&schema)?,
            "TRY_CAST should always be nullable"
        );

        Ok(())
    }
}

#[cfg(all(test, feature = "proto"))]
mod proto_tests {
    use super::*;
    use crate::expressions::{Column, col};
    use crate::proto_test_util::{
        StubDecoder, StubEncoder, UnreachableDecoder, column_node,
    };
    use arrow::datatypes::Field;
    use datafusion_common::DataFusionError;
    use datafusion_physical_expr_common::physical_expr::proto_decode::PhysicalExprDecodeCtx;
    use datafusion_physical_expr_common::physical_expr::proto_encode::PhysicalExprEncodeCtx;
    use datafusion_proto_models::datafusion_common::ArrowType;
    use datafusion_proto_models::protobuf::{
        PhysicalExprNode, PhysicalTryCastNode, physical_expr_node,
    };

    fn try_cast_fixture() -> TryCastExpr {
        let schema = Schema::new(vec![Field::new("a", DataType::Utf8, true)]);
        TryCastExpr::new(col("a", &schema).unwrap(), DataType::Int32)
    }

    fn int32_arrow_type() -> ArrowType {
        (&DataType::Int32).try_into().unwrap()
    }

    fn try_cast_node(
        expr: Option<Box<PhysicalExprNode>>,
        arrow_type: Option<ArrowType>,
    ) -> PhysicalExprNode {
        PhysicalExprNode {
            expr_id: None,
            expr_type: Some(physical_expr_node::ExprType::TryCast(Box::new(
                PhysicalTryCastNode { expr, arrow_type },
            ))),
        }
    }

    #[test]
    fn try_to_proto_encodes_try_cast_expr() {
        let try_cast = try_cast_fixture();
        let encoder = StubEncoder::ok();
        let ctx = PhysicalExprEncodeCtx::new(&encoder);

        let node = try_cast
            .try_to_proto(&ctx)
            .unwrap()
            .expect("TryCastExpr should encode to Some(node)");

        assert!(node.expr_id.is_none());
        let try_cast_node = match node.expr_type {
            Some(physical_expr_node::ExprType::TryCast(boxed)) => *boxed,
            other => panic!("expected a TryCastExpr node, got {other:?}"),
        };
        assert!(try_cast_node.expr.is_some());

        let arrow_type = try_cast_node
            .arrow_type
            .as_ref()
            .expect("try cast type should be encoded");
        let data_type: DataType = arrow_type.try_into().unwrap();
        assert_eq!(data_type, DataType::Int32);
    }

    #[test]
    fn try_to_proto_propagates_child_encode_error() {
        let try_cast = try_cast_fixture();
        let encoder = StubEncoder::failing_on(1);
        let ctx = PhysicalExprEncodeCtx::new(&encoder);
        let err = try_cast.try_to_proto(&ctx).unwrap_err();
        assert!(matches!(err, DataFusionError::Internal(msg) if msg.contains("call 1")));
    }

    #[test]
    fn try_from_proto_decodes_try_cast_expr() {
        let node =
            try_cast_node(Some(Box::new(column_node("a"))), Some(int32_arrow_type()));
        let schema = Schema::empty();
        let decoder = StubDecoder::ok();
        let ctx = PhysicalExprDecodeCtx::new(&schema, &decoder);

        let decoded = TryCastExpr::try_from_proto(&node, &ctx).unwrap();
        let try_cast = decoded
            .downcast_ref::<TryCastExpr>()
            .expect("decoded expr should be a TryCastExpr");

        assert_eq!(try_cast.cast_type(), &DataType::Int32);
        assert!(try_cast.expr().downcast_ref::<Column>().is_some());
    }

    #[test]
    fn try_from_proto_rejects_non_try_cast_node() {
        let node = column_node("a");
        let schema = Schema::empty();
        let decoder = UnreachableDecoder;
        let ctx = PhysicalExprDecodeCtx::new(&schema, &decoder);

        let err = TryCastExpr::try_from_proto(&node, &ctx).unwrap_err();
        assert!(
            matches!(err, DataFusionError::Internal(msg) if msg.contains("PhysicalExprNode is not a TryCastExpr"))
        );
    }

    #[test]
    fn try_from_proto_rejects_missing_expr() {
        let node = try_cast_node(None, Some(int32_arrow_type()));
        let schema = Schema::empty();
        let decoder = UnreachableDecoder;
        let ctx = PhysicalExprDecodeCtx::new(&schema, &decoder);

        let err = TryCastExpr::try_from_proto(&node, &ctx).unwrap_err();
        assert!(
            matches!(err, DataFusionError::Internal(msg) if msg.contains("TryCastExpr is missing required field 'expr'"))
        );
    }

    #[test]
    fn try_from_proto_rejects_missing_arrow_type() {
        let node = try_cast_node(Some(Box::new(column_node("a"))), None);
        let schema = Schema::empty();
        let decoder = StubDecoder::ok();
        let ctx = PhysicalExprDecodeCtx::new(&schema, &decoder);

        let err = TryCastExpr::try_from_proto(&node, &ctx).unwrap_err();
        assert!(
            matches!(err, DataFusionError::Internal(msg) if msg.contains("TryCastExpr is missing required field 'arrow_type'"))
        );
    }

    #[test]
    fn try_from_proto_propagates_child_decode_error() {
        let node =
            try_cast_node(Some(Box::new(column_node("a"))), Some(int32_arrow_type()));
        let schema = Schema::empty();
        let decoder = StubDecoder::failing_on(1);
        let ctx = PhysicalExprDecodeCtx::new(&schema, &decoder);
        let err = TryCastExpr::try_from_proto(&node, &ctx).unwrap_err();
        assert!(matches!(err, DataFusionError::Internal(msg) if msg.contains("call 1")));
    }
}