datafusion_physical_expr/expressions/
try_cast.rs

1// Licensed to the Apache Software Foundation (ASF) under one
2// or more contributor license agreements.  See the NOTICE file
3// distributed with this work for additional information
4// regarding copyright ownership.  The ASF licenses this file
5// to you under the Apache License, Version 2.0 (the
6// "License"); you may not use this file except in compliance
7// with the License.  You may obtain a copy of the License at
8//
9//   http://www.apache.org/licenses/LICENSE-2.0
10//
11// Unless required by applicable law or agreed to in writing,
12// software distributed under the License is distributed on an
13// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
14// KIND, either express or implied.  See the License for the
15// specific language governing permissions and limitations
16// under the License.
17
18use std::any::Any;
19use std::fmt;
20use std::hash::Hash;
21use std::sync::Arc;
22
23use crate::PhysicalExpr;
24use arrow::compute;
25use arrow::compute::{cast_with_options, CastOptions};
26use arrow::datatypes::{DataType, FieldRef, Schema};
27use arrow::record_batch::RecordBatch;
28use compute::can_cast_types;
29use datafusion_common::format::DEFAULT_FORMAT_OPTIONS;
30use datafusion_common::{not_impl_err, Result, ScalarValue};
31use datafusion_expr::ColumnarValue;
32
33/// TRY_CAST expression casts an expression to a specific data type and returns NULL on invalid cast
34#[derive(Debug, Eq)]
35pub struct TryCastExpr {
36    /// The expression to cast
37    expr: Arc<dyn PhysicalExpr>,
38    /// The data type to cast to
39    cast_type: DataType,
40}
41
42// Manually derive PartialEq and Hash to work around https://github.com/rust-lang/rust/issues/78808
43impl PartialEq for TryCastExpr {
44    fn eq(&self, other: &Self) -> bool {
45        self.expr.eq(&other.expr) && self.cast_type == other.cast_type
46    }
47}
48
49impl Hash for TryCastExpr {
50    fn hash<H: std::hash::Hasher>(&self, state: &mut H) {
51        self.expr.hash(state);
52        self.cast_type.hash(state);
53    }
54}
55
56impl TryCastExpr {
57    /// Create a new CastExpr
58    pub fn new(expr: Arc<dyn PhysicalExpr>, cast_type: DataType) -> Self {
59        Self { expr, cast_type }
60    }
61
62    /// The expression to cast
63    pub fn expr(&self) -> &Arc<dyn PhysicalExpr> {
64        &self.expr
65    }
66
67    /// The data type to cast to
68    pub fn cast_type(&self) -> &DataType {
69        &self.cast_type
70    }
71}
72
73impl fmt::Display for TryCastExpr {
74    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
75        write!(f, "TRY_CAST({} AS {:?})", self.expr, self.cast_type)
76    }
77}
78
79impl PhysicalExpr for TryCastExpr {
80    /// Return a reference to Any that can be used for downcasting
81    fn as_any(&self) -> &dyn Any {
82        self
83    }
84
85    fn data_type(&self, _input_schema: &Schema) -> Result<DataType> {
86        Ok(self.cast_type.clone())
87    }
88
89    fn nullable(&self, _input_schema: &Schema) -> Result<bool> {
90        Ok(true)
91    }
92
93    fn evaluate(&self, batch: &RecordBatch) -> Result<ColumnarValue> {
94        let value = self.expr.evaluate(batch)?;
95        let options = CastOptions {
96            safe: true,
97            format_options: DEFAULT_FORMAT_OPTIONS,
98        };
99        match value {
100            ColumnarValue::Array(array) => {
101                let cast = cast_with_options(&array, &self.cast_type, &options)?;
102                Ok(ColumnarValue::Array(cast))
103            }
104            ColumnarValue::Scalar(scalar) => {
105                let array = scalar.to_array()?;
106                let cast_array = cast_with_options(&array, &self.cast_type, &options)?;
107                let cast_scalar = ScalarValue::try_from_array(&cast_array, 0)?;
108                Ok(ColumnarValue::Scalar(cast_scalar))
109            }
110        }
111    }
112
113    fn return_field(&self, input_schema: &Schema) -> Result<FieldRef> {
114        self.expr
115            .return_field(input_schema)
116            .map(|f| f.as_ref().clone().with_data_type(self.cast_type.clone()))
117            .map(Arc::new)
118    }
119
120    fn children(&self) -> Vec<&Arc<dyn PhysicalExpr>> {
121        vec![&self.expr]
122    }
123
124    fn with_new_children(
125        self: Arc<Self>,
126        children: Vec<Arc<dyn PhysicalExpr>>,
127    ) -> Result<Arc<dyn PhysicalExpr>> {
128        Ok(Arc::new(TryCastExpr::new(
129            Arc::clone(&children[0]),
130            self.cast_type.clone(),
131        )))
132    }
133
134    fn fmt_sql(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
135        write!(f, "TRY_CAST(")?;
136        self.expr.fmt_sql(f)?;
137        write!(f, " AS {:?})", self.cast_type)
138    }
139}
140
141/// Return a PhysicalExpression representing `expr` casted to
142/// `cast_type`, if any casting is needed.
143///
144/// Note that such casts may lose type information
145pub fn try_cast(
146    expr: Arc<dyn PhysicalExpr>,
147    input_schema: &Schema,
148    cast_type: DataType,
149) -> Result<Arc<dyn PhysicalExpr>> {
150    let expr_type = expr.data_type(input_schema)?;
151    if expr_type == cast_type {
152        Ok(Arc::clone(&expr))
153    } else if can_cast_types(&expr_type, &cast_type) {
154        Ok(Arc::new(TryCastExpr::new(expr, cast_type)))
155    } else {
156        not_impl_err!("Unsupported TRY_CAST from {expr_type:?} to {cast_type:?}")
157    }
158}
159
160#[cfg(test)]
161mod tests {
162    use super::*;
163    use crate::expressions::col;
164    use arrow::array::{
165        Decimal128Array, Decimal128Builder, StringArray, Time64NanosecondArray,
166    };
167    use arrow::{
168        array::{
169            Array, Float32Array, Float64Array, Int16Array, Int32Array, Int64Array,
170            Int8Array, TimestampNanosecondArray, UInt32Array,
171        },
172        datatypes::*,
173    };
174    use datafusion_physical_expr_common::physical_expr::fmt_sql;
175
176    // runs an end-to-end test of physical type cast
177    // 1. construct a record batch with a column "a" of type A
178    // 2. construct a physical expression of TRY_CAST(a AS B)
179    // 3. evaluate the expression
180    // 4. verify that the resulting expression is of type B
181    // 5. verify that the resulting values are downcastable and correct
182    macro_rules! generic_decimal_to_other_test_cast {
183        ($DECIMAL_ARRAY:ident, $A_TYPE:expr, $TYPEARRAY:ident, $TYPE:expr, $VEC:expr) => {{
184            let schema = Schema::new(vec![Field::new("a", $A_TYPE, true)]);
185            let batch = RecordBatch::try_new(
186                Arc::new(schema.clone()),
187                vec![Arc::new($DECIMAL_ARRAY)],
188            )?;
189            // verify that we can construct the expression
190            let expression = try_cast(col("a", &schema)?, &schema, $TYPE)?;
191
192            // verify that its display is correct
193            assert_eq!(
194                format!("TRY_CAST(a@0 AS {:?})", $TYPE),
195                format!("{}", expression)
196            );
197
198            // verify that the expression's type is correct
199            assert_eq!(expression.data_type(&schema)?, $TYPE);
200
201            // compute
202            let result = expression
203                .evaluate(&batch)?
204                .into_array(batch.num_rows())
205                .expect("Failed to convert to array");
206
207            // verify that the array's data_type is correct
208            assert_eq!(*result.data_type(), $TYPE);
209
210            // verify that the data itself is downcastable
211            let result = result
212                .as_any()
213                .downcast_ref::<$TYPEARRAY>()
214                .expect("failed to downcast");
215
216            // verify that the result itself is correct
217            for (i, x) in $VEC.iter().enumerate() {
218                match x {
219                    Some(x) => assert_eq!(result.value(i), *x),
220                    None => assert!(!result.is_valid(i)),
221                }
222            }
223        }};
224    }
225
226    // runs an end-to-end test of physical type cast
227    // 1. construct a record batch with a column "a" of type A
228    // 2. construct a physical expression of TRY_CAST(a AS B)
229    // 3. evaluate the expression
230    // 4. verify that the resulting expression is of type B
231    // 5. verify that the resulting values are downcastable and correct
232    macro_rules! generic_test_cast {
233        ($A_ARRAY:ident, $A_TYPE:expr, $A_VEC:expr, $TYPEARRAY:ident, $TYPE:expr, $VEC:expr) => {{
234            let schema = Schema::new(vec![Field::new("a", $A_TYPE, true)]);
235            let a_vec_len = $A_VEC.len();
236            let a = $A_ARRAY::from($A_VEC);
237            let batch =
238                RecordBatch::try_new(Arc::new(schema.clone()), vec![Arc::new(a)])?;
239
240            // verify that we can construct the expression
241            let expression = try_cast(col("a", &schema)?, &schema, $TYPE)?;
242
243            // verify that its display is correct
244            assert_eq!(
245                format!("TRY_CAST(a@0 AS {:?})", $TYPE),
246                format!("{}", expression)
247            );
248
249            // verify that the expression's type is correct
250            assert_eq!(expression.data_type(&schema)?, $TYPE);
251
252            // compute
253            let result = expression
254                .evaluate(&batch)?
255                .into_array(batch.num_rows())
256                .expect("Failed to convert to array");
257
258            // verify that the array's data_type is correct
259            assert_eq!(*result.data_type(), $TYPE);
260
261            // verify that the len is correct
262            assert_eq!(result.len(), a_vec_len);
263
264            // verify that the data itself is downcastable
265            let result = result
266                .as_any()
267                .downcast_ref::<$TYPEARRAY>()
268                .expect("failed to downcast");
269
270            // verify that the result itself is correct
271            for (i, x) in $VEC.iter().enumerate() {
272                match x {
273                    Some(x) => assert_eq!(result.value(i), *x),
274                    None => assert!(!result.is_valid(i)),
275                }
276            }
277        }};
278    }
279
280    #[test]
281    fn test_try_cast_decimal_to_decimal() -> Result<()> {
282        // try cast one decimal data type to another decimal data type
283        let array: Vec<i128> = vec![1234, 2222, 3, 4000, 5000];
284        let decimal_array = create_decimal_array(&array, 10, 3);
285        generic_decimal_to_other_test_cast!(
286            decimal_array,
287            DataType::Decimal128(10, 3),
288            Decimal128Array,
289            DataType::Decimal128(20, 6),
290            [
291                Some(1_234_000),
292                Some(2_222_000),
293                Some(3_000),
294                Some(4_000_000),
295                Some(5_000_000),
296                None
297            ]
298        );
299
300        let decimal_array = create_decimal_array(&array, 10, 3);
301        generic_decimal_to_other_test_cast!(
302            decimal_array,
303            DataType::Decimal128(10, 3),
304            Decimal128Array,
305            DataType::Decimal128(10, 2),
306            [Some(123), Some(222), Some(0), Some(400), Some(500), None]
307        );
308
309        Ok(())
310    }
311
312    #[test]
313    fn test_try_cast_decimal_to_numeric() -> Result<()> {
314        // TODO we should add function to create Decimal128Array with value and metadata
315        // https://github.com/apache/arrow-rs/issues/1009
316        let array: Vec<i128> = vec![1, 2, 3, 4, 5];
317        let decimal_array = create_decimal_array(&array, 10, 0);
318        // decimal to i8
319        generic_decimal_to_other_test_cast!(
320            decimal_array,
321            DataType::Decimal128(10, 0),
322            Int8Array,
323            DataType::Int8,
324            [
325                Some(1_i8),
326                Some(2_i8),
327                Some(3_i8),
328                Some(4_i8),
329                Some(5_i8),
330                None
331            ]
332        );
333
334        // decimal to i16
335        let decimal_array = create_decimal_array(&array, 10, 0);
336        generic_decimal_to_other_test_cast!(
337            decimal_array,
338            DataType::Decimal128(10, 0),
339            Int16Array,
340            DataType::Int16,
341            [
342                Some(1_i16),
343                Some(2_i16),
344                Some(3_i16),
345                Some(4_i16),
346                Some(5_i16),
347                None
348            ]
349        );
350
351        // decimal to i32
352        let decimal_array = create_decimal_array(&array, 10, 0);
353        generic_decimal_to_other_test_cast!(
354            decimal_array,
355            DataType::Decimal128(10, 0),
356            Int32Array,
357            DataType::Int32,
358            [
359                Some(1_i32),
360                Some(2_i32),
361                Some(3_i32),
362                Some(4_i32),
363                Some(5_i32),
364                None
365            ]
366        );
367
368        // decimal to i64
369        let decimal_array = create_decimal_array(&array, 10, 0);
370        generic_decimal_to_other_test_cast!(
371            decimal_array,
372            DataType::Decimal128(10, 0),
373            Int64Array,
374            DataType::Int64,
375            [
376                Some(1_i64),
377                Some(2_i64),
378                Some(3_i64),
379                Some(4_i64),
380                Some(5_i64),
381                None
382            ]
383        );
384
385        // decimal to float32
386        let array: Vec<i128> = vec![1234, 2222, 3, 4000, 5000];
387        let decimal_array = create_decimal_array(&array, 10, 3);
388        generic_decimal_to_other_test_cast!(
389            decimal_array,
390            DataType::Decimal128(10, 3),
391            Float32Array,
392            DataType::Float32,
393            [
394                Some(1.234_f32),
395                Some(2.222_f32),
396                Some(0.003_f32),
397                Some(4.0_f32),
398                Some(5.0_f32),
399                None
400            ]
401        );
402        // decimal to float64
403        let decimal_array = create_decimal_array(&array, 20, 6);
404        generic_decimal_to_other_test_cast!(
405            decimal_array,
406            DataType::Decimal128(20, 6),
407            Float64Array,
408            DataType::Float64,
409            [
410                Some(0.001234_f64),
411                Some(0.002222_f64),
412                Some(0.000003_f64),
413                Some(0.004_f64),
414                Some(0.005_f64),
415                None
416            ]
417        );
418
419        Ok(())
420    }
421
422    #[test]
423    fn test_try_cast_numeric_to_decimal() -> Result<()> {
424        // int8
425        generic_test_cast!(
426            Int8Array,
427            DataType::Int8,
428            vec![1, 2, 3, 4, 5],
429            Decimal128Array,
430            DataType::Decimal128(3, 0),
431            [Some(1), Some(2), Some(3), Some(4), Some(5)]
432        );
433
434        // int16
435        generic_test_cast!(
436            Int16Array,
437            DataType::Int16,
438            vec![1, 2, 3, 4, 5],
439            Decimal128Array,
440            DataType::Decimal128(5, 0),
441            [Some(1), Some(2), Some(3), Some(4), Some(5)]
442        );
443
444        // int32
445        generic_test_cast!(
446            Int32Array,
447            DataType::Int32,
448            vec![1, 2, 3, 4, 5],
449            Decimal128Array,
450            DataType::Decimal128(10, 0),
451            [Some(1), Some(2), Some(3), Some(4), Some(5)]
452        );
453
454        // int64
455        generic_test_cast!(
456            Int64Array,
457            DataType::Int64,
458            vec![1, 2, 3, 4, 5],
459            Decimal128Array,
460            DataType::Decimal128(20, 0),
461            [Some(1), Some(2), Some(3), Some(4), Some(5)]
462        );
463
464        // int64 to different scale
465        generic_test_cast!(
466            Int64Array,
467            DataType::Int64,
468            vec![1, 2, 3, 4, 5],
469            Decimal128Array,
470            DataType::Decimal128(20, 2),
471            [Some(100), Some(200), Some(300), Some(400), Some(500)]
472        );
473
474        // float32
475        generic_test_cast!(
476            Float32Array,
477            DataType::Float32,
478            vec![1.5, 2.5, 3.0, 1.123_456_8, 5.50],
479            Decimal128Array,
480            DataType::Decimal128(10, 2),
481            [Some(150), Some(250), Some(300), Some(112), Some(550)]
482        );
483
484        // float64
485        generic_test_cast!(
486            Float64Array,
487            DataType::Float64,
488            vec![1.5, 2.5, 3.0, 1.123_456_8, 5.50],
489            Decimal128Array,
490            DataType::Decimal128(20, 4),
491            [
492                Some(15000),
493                Some(25000),
494                Some(30000),
495                Some(11235),
496                Some(55000)
497            ]
498        );
499        Ok(())
500    }
501
502    #[test]
503    fn test_cast_i32_u32() -> Result<()> {
504        generic_test_cast!(
505            Int32Array,
506            DataType::Int32,
507            vec![1, 2, 3, 4, 5],
508            UInt32Array,
509            DataType::UInt32,
510            [
511                Some(1_u32),
512                Some(2_u32),
513                Some(3_u32),
514                Some(4_u32),
515                Some(5_u32)
516            ]
517        );
518        Ok(())
519    }
520
521    #[test]
522    fn test_cast_i32_utf8() -> Result<()> {
523        generic_test_cast!(
524            Int32Array,
525            DataType::Int32,
526            vec![1, 2, 3, 4, 5],
527            StringArray,
528            DataType::Utf8,
529            [Some("1"), Some("2"), Some("3"), Some("4"), Some("5")]
530        );
531        Ok(())
532    }
533
534    #[test]
535    fn test_try_cast_utf8_i32() -> Result<()> {
536        generic_test_cast!(
537            StringArray,
538            DataType::Utf8,
539            vec!["a", "2", "3", "b", "5"],
540            Int32Array,
541            DataType::Int32,
542            [None, Some(2), Some(3), None, Some(5)]
543        );
544        Ok(())
545    }
546
547    #[test]
548    fn test_cast_i64_t64() -> Result<()> {
549        let original = vec![1, 2, 3, 4, 5];
550        let expected: Vec<Option<i64>> = original
551            .iter()
552            .map(|i| Some(Time64NanosecondArray::from(vec![*i]).value(0)))
553            .collect();
554        generic_test_cast!(
555            Int64Array,
556            DataType::Int64,
557            original,
558            TimestampNanosecondArray,
559            DataType::Timestamp(TimeUnit::Nanosecond, None),
560            expected
561        );
562        Ok(())
563    }
564
565    #[test]
566    fn invalid_cast() {
567        // Ensure a useful error happens at plan time if invalid casts are used
568        let schema = Schema::new(vec![Field::new("a", DataType::Int32, false)]);
569
570        let result = try_cast(
571            col("a", &schema).unwrap(),
572            &schema,
573            DataType::Interval(IntervalUnit::MonthDayNano),
574        );
575        result.expect_err("expected Invalid TRY_CAST");
576    }
577
578    // create decimal array with the specified precision and scale
579    fn create_decimal_array(array: &[i128], precision: u8, scale: i8) -> Decimal128Array {
580        let mut decimal_builder = Decimal128Builder::with_capacity(array.len());
581        for value in array {
582            decimal_builder.append_value(*value);
583        }
584        decimal_builder.append_null();
585        decimal_builder
586            .finish()
587            .with_precision_and_scale(precision, scale)
588            .unwrap()
589    }
590
591    #[test]
592    fn test_fmt_sql() -> Result<()> {
593        let schema = Schema::new(vec![Field::new("a", DataType::Int32, true)]);
594
595        // Test numeric casting
596        let expr = try_cast(col("a", &schema)?, &schema, DataType::Int64)?;
597        let display_string = expr.to_string();
598        assert_eq!(display_string, "TRY_CAST(a@0 AS Int64)");
599        let sql_string = fmt_sql(expr.as_ref()).to_string();
600        assert_eq!(sql_string, "TRY_CAST(a AS Int64)");
601
602        // Test string casting
603        let schema = Schema::new(vec![Field::new("b", DataType::Utf8, true)]);
604        let expr = try_cast(col("b", &schema)?, &schema, DataType::Int32)?;
605        let display_string = expr.to_string();
606        assert_eq!(display_string, "TRY_CAST(b@0 AS Int32)");
607        let sql_string = fmt_sql(expr.as_ref()).to_string();
608        assert_eq!(sql_string, "TRY_CAST(b AS Int32)");
609
610        Ok(())
611    }
612}
datafusion_physical_expr/expressions/try_cast.rs

datafusion_physical_expr/expressions/
try_cast.rs
