1use std::fmt;
19use std::hash::Hash;
20use std::sync::Arc;
21
22use crate::PhysicalExpr;
23use arrow::compute;
24use arrow::compute::CastOptions;
25use arrow::datatypes::{DataType, FieldRef, Schema};
26use arrow::record_batch::RecordBatch;
27use compute::can_cast_types;
28use datafusion_common::format::DEFAULT_FORMAT_OPTIONS;
29use datafusion_common::{Result, not_impl_err};
30use datafusion_expr::ColumnarValue;
31
32#[derive(Debug, Eq)]
34pub struct TryCastExpr {
35 expr: Arc<dyn PhysicalExpr>,
37 cast_type: DataType,
39}
40
41impl PartialEq for TryCastExpr {
43 fn eq(&self, other: &Self) -> bool {
44 self.expr.eq(&other.expr) && self.cast_type == other.cast_type
45 }
46}
47
48impl Hash for TryCastExpr {
49 fn hash<H: std::hash::Hasher>(&self, state: &mut H) {
50 self.expr.hash(state);
51 self.cast_type.hash(state);
52 }
53}
54
55impl TryCastExpr {
56 pub fn new(expr: Arc<dyn PhysicalExpr>, cast_type: DataType) -> Self {
58 Self { expr, cast_type }
59 }
60
61 pub fn expr(&self) -> &Arc<dyn PhysicalExpr> {
63 &self.expr
64 }
65
66 pub fn cast_type(&self) -> &DataType {
68 &self.cast_type
69 }
70}
71
72impl fmt::Display for TryCastExpr {
73 fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
74 write!(f, "TRY_CAST({} AS {})", self.expr, self.cast_type)
75 }
76}
77
78impl PhysicalExpr for TryCastExpr {
79 fn data_type(&self, _input_schema: &Schema) -> Result<DataType> {
80 Ok(self.cast_type.clone())
81 }
82
83 fn nullable(&self, _input_schema: &Schema) -> Result<bool> {
84 Ok(true)
85 }
86
87 fn evaluate(&self, batch: &RecordBatch) -> Result<ColumnarValue> {
88 let value = self.expr.evaluate(batch)?;
89 let options = CastOptions {
90 safe: true,
91 format_options: DEFAULT_FORMAT_OPTIONS,
92 };
93 value.cast_to(&self.cast_type, Some(&options))
94 }
95
96 fn return_field(&self, input_schema: &Schema) -> Result<FieldRef> {
97 self.expr
98 .return_field(input_schema)
99 .map(|f| f.as_ref().clone().with_data_type(self.cast_type.clone()))
100 .map(Arc::new)
101 }
102
103 fn children(&self) -> Vec<&Arc<dyn PhysicalExpr>> {
104 vec![&self.expr]
105 }
106
107 fn with_new_children(
108 self: Arc<Self>,
109 children: Vec<Arc<dyn PhysicalExpr>>,
110 ) -> Result<Arc<dyn PhysicalExpr>> {
111 Ok(Arc::new(TryCastExpr::new(
112 Arc::clone(&children[0]),
113 self.cast_type.clone(),
114 )))
115 }
116
117 fn fmt_sql(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
118 write!(f, "TRY_CAST(")?;
119 self.expr.fmt_sql(f)?;
120 write!(f, " AS {:?})", self.cast_type)
121 }
122
123 #[cfg(feature = "proto")]
124 fn try_to_proto(
125 &self,
126 ctx: &datafusion_physical_expr_common::physical_expr::proto_encode::PhysicalExprEncodeCtx<'_>,
127 ) -> Result<Option<datafusion_proto_models::protobuf::PhysicalExprNode>> {
128 use datafusion_proto_models::protobuf;
129
130 Ok(Some(protobuf::PhysicalExprNode {
131 expr_id: None,
132 expr_type: Some(protobuf::physical_expr_node::ExprType::TryCast(Box::new(
133 protobuf::PhysicalTryCastNode {
134 expr: Some(Box::new(ctx.encode_child(&self.expr)?)),
135 arrow_type: Some(self.cast_type().try_into()?),
136 },
137 ))),
138 }))
139 }
140}
141
142#[cfg(feature = "proto")]
143impl TryCastExpr {
144 pub fn try_from_proto(
146 node: &datafusion_proto_models::protobuf::PhysicalExprNode,
147 ctx: &datafusion_physical_expr_common::physical_expr::proto_decode::PhysicalExprDecodeCtx<'_>,
148 ) -> Result<Arc<dyn PhysicalExpr>> {
149 use datafusion_physical_expr_common::expect_expr_variant;
150 use datafusion_physical_expr_common::physical_expr::proto_decode::require_proto_field;
151 use datafusion_proto_models::protobuf;
152
153 let try_cast = expect_expr_variant!(
154 node,
155 protobuf::physical_expr_node::ExprType::TryCast,
156 "TryCastExpr",
157 );
158 let expr = ctx.decode_required_expression(
159 try_cast.expr.as_deref(),
160 "TryCastExpr",
161 "expr",
162 )?;
163 let arrow_type = require_proto_field(
164 try_cast.arrow_type.as_ref(),
165 "TryCastExpr",
166 "arrow_type",
167 )?;
168 let cast_type: DataType = arrow_type.try_into()?;
169
170 Ok(Arc::new(TryCastExpr::new(expr, cast_type)))
171 }
172}
173
174pub fn try_cast(
179 expr: Arc<dyn PhysicalExpr>,
180 input_schema: &Schema,
181 cast_type: DataType,
182) -> Result<Arc<dyn PhysicalExpr>> {
183 let expr_type = expr.data_type(input_schema)?;
184 if expr_type == cast_type {
185 Ok(Arc::clone(&expr))
186 } else if can_cast_types(&expr_type, &cast_type) {
187 Ok(Arc::new(TryCastExpr::new(expr, cast_type)))
188 } else {
189 not_impl_err!("Unsupported TRY_CAST from {expr_type} to {cast_type}")
190 }
191}
192
193#[cfg(test)]
194mod tests {
195 use super::*;
196 use crate::expressions::col;
197 use arrow::array::{
198 Decimal128Array, Decimal128Builder, StringArray, Time64NanosecondArray,
199 };
200 use arrow::{
201 array::{
202 Array, Float32Array, Float64Array, Int8Array, Int16Array, Int32Array,
203 Int64Array, TimestampNanosecondArray, UInt32Array,
204 },
205 datatypes::*,
206 };
207 use datafusion_physical_expr_common::physical_expr::fmt_sql;
208
209 macro_rules! generic_decimal_to_other_test_cast {
216 ($DECIMAL_ARRAY:ident, $A_TYPE:expr, $TYPEARRAY:ident, $TYPE:expr, $VEC:expr) => {{
217 let schema = Schema::new(vec![Field::new("a", $A_TYPE, true)]);
218 let batch = RecordBatch::try_new(
219 Arc::new(schema.clone()),
220 vec![Arc::new($DECIMAL_ARRAY)],
221 )?;
222 let expression = try_cast(col("a", &schema)?, &schema, $TYPE)?;
224
225 assert_eq!(
227 format!("TRY_CAST(a@0 AS {})", $TYPE),
228 format!("{}", expression)
229 );
230
231 assert_eq!(expression.data_type(&schema)?, $TYPE);
233
234 let result = expression
236 .evaluate(&batch)?
237 .into_array(batch.num_rows())
238 .expect("Failed to convert to array");
239
240 assert_eq!(*result.data_type(), $TYPE);
242
243 let result = result
245 .as_any()
246 .downcast_ref::<$TYPEARRAY>()
247 .expect("failed to downcast");
248
249 for (i, x) in $VEC.iter().enumerate() {
251 match x {
252 Some(x) => assert_eq!(result.value(i), *x),
253 None => assert!(result.is_null(i)),
254 }
255 }
256 }};
257 }
258
259 macro_rules! generic_test_cast {
266 ($A_ARRAY:ident, $A_TYPE:expr, $A_VEC:expr, $TYPEARRAY:ident, $TYPE:expr, $VEC:expr) => {{
267 let schema = Schema::new(vec![Field::new("a", $A_TYPE, true)]);
268 let a_vec_len = $A_VEC.len();
269 let a = $A_ARRAY::from($A_VEC);
270 let batch =
271 RecordBatch::try_new(Arc::new(schema.clone()), vec![Arc::new(a)])?;
272
273 let expression = try_cast(col("a", &schema)?, &schema, $TYPE)?;
275
276 assert_eq!(
278 format!("TRY_CAST(a@0 AS {})", $TYPE),
279 format!("{}", expression)
280 );
281
282 assert_eq!(expression.data_type(&schema)?, $TYPE);
284
285 let result = expression
287 .evaluate(&batch)?
288 .into_array(batch.num_rows())
289 .expect("Failed to convert to array");
290
291 assert_eq!(*result.data_type(), $TYPE);
293
294 assert_eq!(result.len(), a_vec_len);
296
297 let result = result
299 .as_any()
300 .downcast_ref::<$TYPEARRAY>()
301 .expect("failed to downcast");
302
303 for (i, x) in $VEC.iter().enumerate() {
305 match x {
306 Some(x) => assert_eq!(result.value(i), *x),
307 None => assert!(result.is_null(i)),
308 }
309 }
310 }};
311 }
312
313 #[test]
314 fn test_try_cast_decimal_to_decimal() -> Result<()> {
315 let array: Vec<i128> = vec![1234, 2222, 3, 4000, 5000];
317 let decimal_array = create_decimal_array(&array, 10, 3);
318 generic_decimal_to_other_test_cast!(
319 decimal_array,
320 DataType::Decimal128(10, 3),
321 Decimal128Array,
322 DataType::Decimal128(20, 6),
323 [
324 Some(1_234_000),
325 Some(2_222_000),
326 Some(3_000),
327 Some(4_000_000),
328 Some(5_000_000),
329 None
330 ]
331 );
332
333 let decimal_array = create_decimal_array(&array, 10, 3);
334 generic_decimal_to_other_test_cast!(
335 decimal_array,
336 DataType::Decimal128(10, 3),
337 Decimal128Array,
338 DataType::Decimal128(10, 2),
339 [Some(123), Some(222), Some(0), Some(400), Some(500), None]
340 );
341
342 Ok(())
343 }
344
345 #[test]
346 fn test_try_cast_decimal_to_numeric() -> Result<()> {
347 let array: Vec<i128> = vec![1, 2, 3, 4, 5];
350 let decimal_array = create_decimal_array(&array, 10, 0);
351 generic_decimal_to_other_test_cast!(
353 decimal_array,
354 DataType::Decimal128(10, 0),
355 Int8Array,
356 DataType::Int8,
357 [
358 Some(1_i8),
359 Some(2_i8),
360 Some(3_i8),
361 Some(4_i8),
362 Some(5_i8),
363 None
364 ]
365 );
366
367 let decimal_array = create_decimal_array(&array, 10, 0);
369 generic_decimal_to_other_test_cast!(
370 decimal_array,
371 DataType::Decimal128(10, 0),
372 Int16Array,
373 DataType::Int16,
374 [
375 Some(1_i16),
376 Some(2_i16),
377 Some(3_i16),
378 Some(4_i16),
379 Some(5_i16),
380 None
381 ]
382 );
383
384 let decimal_array = create_decimal_array(&array, 10, 0);
386 generic_decimal_to_other_test_cast!(
387 decimal_array,
388 DataType::Decimal128(10, 0),
389 Int32Array,
390 DataType::Int32,
391 [
392 Some(1_i32),
393 Some(2_i32),
394 Some(3_i32),
395 Some(4_i32),
396 Some(5_i32),
397 None
398 ]
399 );
400
401 let decimal_array = create_decimal_array(&array, 10, 0);
403 generic_decimal_to_other_test_cast!(
404 decimal_array,
405 DataType::Decimal128(10, 0),
406 Int64Array,
407 DataType::Int64,
408 [
409 Some(1_i64),
410 Some(2_i64),
411 Some(3_i64),
412 Some(4_i64),
413 Some(5_i64),
414 None
415 ]
416 );
417
418 let array: Vec<i128> = vec![1234, 2222, 3, 4000, 5000];
420 let decimal_array = create_decimal_array(&array, 10, 3);
421 generic_decimal_to_other_test_cast!(
422 decimal_array,
423 DataType::Decimal128(10, 3),
424 Float32Array,
425 DataType::Float32,
426 [
427 Some(1.234_f32),
428 Some(2.222_f32),
429 Some(0.003_f32),
430 Some(4.0_f32),
431 Some(5.0_f32),
432 None
433 ]
434 );
435 let decimal_array = create_decimal_array(&array, 20, 6);
437 generic_decimal_to_other_test_cast!(
438 decimal_array,
439 DataType::Decimal128(20, 6),
440 Float64Array,
441 DataType::Float64,
442 [
443 Some(0.001234_f64),
444 Some(0.002222_f64),
445 Some(0.000003_f64),
446 Some(0.004_f64),
447 Some(0.005_f64),
448 None
449 ]
450 );
451
452 Ok(())
453 }
454
455 #[test]
456 fn test_try_cast_numeric_to_decimal() -> Result<()> {
457 generic_test_cast!(
459 Int8Array,
460 DataType::Int8,
461 vec![1, 2, 3, 4, 5],
462 Decimal128Array,
463 DataType::Decimal128(3, 0),
464 [Some(1), Some(2), Some(3), Some(4), Some(5)]
465 );
466
467 generic_test_cast!(
469 Int16Array,
470 DataType::Int16,
471 vec![1, 2, 3, 4, 5],
472 Decimal128Array,
473 DataType::Decimal128(5, 0),
474 [Some(1), Some(2), Some(3), Some(4), Some(5)]
475 );
476
477 generic_test_cast!(
479 Int32Array,
480 DataType::Int32,
481 vec![1, 2, 3, 4, 5],
482 Decimal128Array,
483 DataType::Decimal128(10, 0),
484 [Some(1), Some(2), Some(3), Some(4), Some(5)]
485 );
486
487 generic_test_cast!(
489 Int64Array,
490 DataType::Int64,
491 vec![1, 2, 3, 4, 5],
492 Decimal128Array,
493 DataType::Decimal128(20, 0),
494 [Some(1), Some(2), Some(3), Some(4), Some(5)]
495 );
496
497 generic_test_cast!(
499 Int64Array,
500 DataType::Int64,
501 vec![1, 2, 3, 4, 5],
502 Decimal128Array,
503 DataType::Decimal128(20, 2),
504 [Some(100), Some(200), Some(300), Some(400), Some(500)]
505 );
506
507 generic_test_cast!(
509 Float32Array,
510 DataType::Float32,
511 vec![1.5, 2.5, 3.0, 1.123_456_8, 5.50],
512 Decimal128Array,
513 DataType::Decimal128(10, 2),
514 [Some(150), Some(250), Some(300), Some(112), Some(550)]
515 );
516
517 generic_test_cast!(
519 Float64Array,
520 DataType::Float64,
521 vec![1.5, 2.5, 3.0, 1.123_456_8, 5.50],
522 Decimal128Array,
523 DataType::Decimal128(20, 4),
524 [
525 Some(15000),
526 Some(25000),
527 Some(30000),
528 Some(11235),
529 Some(55000)
530 ]
531 );
532 Ok(())
533 }
534
535 #[test]
536 fn test_cast_i32_u32() -> Result<()> {
537 generic_test_cast!(
538 Int32Array,
539 DataType::Int32,
540 vec![1, 2, 3, 4, 5],
541 UInt32Array,
542 DataType::UInt32,
543 [
544 Some(1_u32),
545 Some(2_u32),
546 Some(3_u32),
547 Some(4_u32),
548 Some(5_u32)
549 ]
550 );
551 Ok(())
552 }
553
554 #[test]
555 fn test_cast_i32_utf8() -> Result<()> {
556 generic_test_cast!(
557 Int32Array,
558 DataType::Int32,
559 vec![1, 2, 3, 4, 5],
560 StringArray,
561 DataType::Utf8,
562 [Some("1"), Some("2"), Some("3"), Some("4"), Some("5")]
563 );
564 Ok(())
565 }
566
567 #[test]
568 fn test_try_cast_utf8_i32() -> Result<()> {
569 generic_test_cast!(
570 StringArray,
571 DataType::Utf8,
572 vec!["a", "2", "3", "b", "5"],
573 Int32Array,
574 DataType::Int32,
575 [None, Some(2), Some(3), None, Some(5)]
576 );
577 Ok(())
578 }
579
580 #[test]
581 fn test_cast_i64_t64() -> Result<()> {
582 let original = vec![1, 2, 3, 4, 5];
583 let expected: Vec<Option<i64>> = original
584 .iter()
585 .map(|i| Some(Time64NanosecondArray::from(vec![*i]).value(0)))
586 .collect();
587 generic_test_cast!(
588 Int64Array,
589 DataType::Int64,
590 original,
591 TimestampNanosecondArray,
592 DataType::Timestamp(TimeUnit::Nanosecond, None),
593 expected
594 );
595 Ok(())
596 }
597
598 #[test]
599 fn invalid_cast() {
600 let schema = Schema::new(vec![Field::new("a", DataType::Int32, false)]);
602
603 let result = try_cast(
604 col("a", &schema).unwrap(),
605 &schema,
606 DataType::Interval(IntervalUnit::MonthDayNano),
607 );
608 result.expect_err("expected Invalid TRY_CAST");
609 }
610
611 fn create_decimal_array(array: &[i128], precision: u8, scale: i8) -> Decimal128Array {
613 let mut decimal_builder = Decimal128Builder::with_capacity(array.len());
614 for value in array {
615 decimal_builder.append_value(*value);
616 }
617 decimal_builder.append_null();
618 decimal_builder
619 .finish()
620 .with_precision_and_scale(precision, scale)
621 .unwrap()
622 }
623
624 #[test]
625 fn test_fmt_sql() -> Result<()> {
626 let schema = Schema::new(vec![Field::new("a", DataType::Int32, true)]);
627
628 let expr = try_cast(col("a", &schema)?, &schema, DataType::Int64)?;
630 let display_string = expr.to_string();
631 assert_eq!(display_string, "TRY_CAST(a@0 AS Int64)");
632 let sql_string = fmt_sql(expr.as_ref()).to_string();
633 assert_eq!(sql_string, "TRY_CAST(a AS Int64)");
634
635 let schema = Schema::new(vec![Field::new("b", DataType::Utf8, true)]);
637 let expr = try_cast(col("b", &schema)?, &schema, DataType::Int32)?;
638 let display_string = expr.to_string();
639 assert_eq!(display_string, "TRY_CAST(b@0 AS Int32)");
640 let sql_string = fmt_sql(expr.as_ref()).to_string();
641 assert_eq!(sql_string, "TRY_CAST(b AS Int32)");
642
643 Ok(())
644 }
645}
646
647#[cfg(all(test, feature = "proto"))]
648mod proto_tests {
649 use super::*;
650 use crate::expressions::{Column, col};
651 use crate::proto_test_util::{
652 StubDecoder, StubEncoder, UnreachableDecoder, column_node,
653 };
654 use arrow::datatypes::Field;
655 use datafusion_common::DataFusionError;
656 use datafusion_physical_expr_common::physical_expr::proto_decode::PhysicalExprDecodeCtx;
657 use datafusion_physical_expr_common::physical_expr::proto_encode::PhysicalExprEncodeCtx;
658 use datafusion_proto_models::datafusion_common::ArrowType;
659 use datafusion_proto_models::protobuf::{
660 PhysicalExprNode, PhysicalTryCastNode, physical_expr_node,
661 };
662
663 fn try_cast_fixture() -> TryCastExpr {
664 let schema = Schema::new(vec![Field::new("a", DataType::Utf8, true)]);
665 TryCastExpr::new(col("a", &schema).unwrap(), DataType::Int32)
666 }
667
668 fn int32_arrow_type() -> ArrowType {
669 (&DataType::Int32).try_into().unwrap()
670 }
671
672 fn try_cast_node(
673 expr: Option<Box<PhysicalExprNode>>,
674 arrow_type: Option<ArrowType>,
675 ) -> PhysicalExprNode {
676 PhysicalExprNode {
677 expr_id: None,
678 expr_type: Some(physical_expr_node::ExprType::TryCast(Box::new(
679 PhysicalTryCastNode { expr, arrow_type },
680 ))),
681 }
682 }
683
684 #[test]
685 fn try_to_proto_encodes_try_cast_expr() {
686 let try_cast = try_cast_fixture();
687 let encoder = StubEncoder::ok();
688 let ctx = PhysicalExprEncodeCtx::new(&encoder);
689
690 let node = try_cast
691 .try_to_proto(&ctx)
692 .unwrap()
693 .expect("TryCastExpr should encode to Some(node)");
694
695 assert!(node.expr_id.is_none());
696 let try_cast_node = match node.expr_type {
697 Some(physical_expr_node::ExprType::TryCast(boxed)) => *boxed,
698 other => panic!("expected a TryCastExpr node, got {other:?}"),
699 };
700 assert!(try_cast_node.expr.is_some());
701
702 let arrow_type = try_cast_node
703 .arrow_type
704 .as_ref()
705 .expect("try cast type should be encoded");
706 let data_type: DataType = arrow_type.try_into().unwrap();
707 assert_eq!(data_type, DataType::Int32);
708 }
709
710 #[test]
711 fn try_to_proto_propagates_child_encode_error() {
712 let try_cast = try_cast_fixture();
713 let encoder = StubEncoder::failing_on(1);
714 let ctx = PhysicalExprEncodeCtx::new(&encoder);
715 let err = try_cast.try_to_proto(&ctx).unwrap_err();
716 assert!(matches!(err, DataFusionError::Internal(msg) if msg.contains("call 1")));
717 }
718
719 #[test]
720 fn try_from_proto_decodes_try_cast_expr() {
721 let node =
722 try_cast_node(Some(Box::new(column_node("a"))), Some(int32_arrow_type()));
723 let schema = Schema::empty();
724 let decoder = StubDecoder::ok();
725 let ctx = PhysicalExprDecodeCtx::new(&schema, &decoder);
726
727 let decoded = TryCastExpr::try_from_proto(&node, &ctx).unwrap();
728 let try_cast = decoded
729 .downcast_ref::<TryCastExpr>()
730 .expect("decoded expr should be a TryCastExpr");
731
732 assert_eq!(try_cast.cast_type(), &DataType::Int32);
733 assert!(try_cast.expr().downcast_ref::<Column>().is_some());
734 }
735
736 #[test]
737 fn try_from_proto_rejects_non_try_cast_node() {
738 let node = column_node("a");
739 let schema = Schema::empty();
740 let decoder = UnreachableDecoder;
741 let ctx = PhysicalExprDecodeCtx::new(&schema, &decoder);
742
743 let err = TryCastExpr::try_from_proto(&node, &ctx).unwrap_err();
744 assert!(
745 matches!(err, DataFusionError::Internal(msg) if msg.contains("PhysicalExprNode is not a TryCastExpr"))
746 );
747 }
748
749 #[test]
750 fn try_from_proto_rejects_missing_expr() {
751 let node = try_cast_node(None, Some(int32_arrow_type()));
752 let schema = Schema::empty();
753 let decoder = UnreachableDecoder;
754 let ctx = PhysicalExprDecodeCtx::new(&schema, &decoder);
755
756 let err = TryCastExpr::try_from_proto(&node, &ctx).unwrap_err();
757 assert!(
758 matches!(err, DataFusionError::Internal(msg) if msg.contains("TryCastExpr is missing required field 'expr'"))
759 );
760 }
761
762 #[test]
763 fn try_from_proto_rejects_missing_arrow_type() {
764 let node = try_cast_node(Some(Box::new(column_node("a"))), None);
765 let schema = Schema::empty();
766 let decoder = StubDecoder::ok();
767 let ctx = PhysicalExprDecodeCtx::new(&schema, &decoder);
768
769 let err = TryCastExpr::try_from_proto(&node, &ctx).unwrap_err();
770 assert!(
771 matches!(err, DataFusionError::Internal(msg) if msg.contains("TryCastExpr is missing required field 'arrow_type'"))
772 );
773 }
774
775 #[test]
776 fn try_from_proto_propagates_child_decode_error() {
777 let node =
778 try_cast_node(Some(Box::new(column_node("a"))), Some(int32_arrow_type()));
779 let schema = Schema::empty();
780 let decoder = StubDecoder::failing_on(1);
781 let ctx = PhysicalExprDecodeCtx::new(&schema, &decoder);
782 let err = TryCastExpr::try_from_proto(&node, &ctx).unwrap_err();
783 assert!(matches!(err, DataFusionError::Internal(msg) if msg.contains("call 1")));
784 }
785}