datafusion_physical_expr/expressions/
column.rs

1// Licensed to the Apache Software Foundation (ASF) under one
2// or more contributor license agreements.  See the NOTICE file
3// distributed with this work for additional information
4// regarding copyright ownership.  The ASF licenses this file
5// to you under the Apache License, Version 2.0 (the
6// "License"); you may not use this file except in compliance
7// with the License.  You may obtain a copy of the License at
8//
9//   http://www.apache.org/licenses/LICENSE-2.0
10//
11// Unless required by applicable law or agreed to in writing,
12// software distributed under the License is distributed on an
13// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
14// KIND, either express or implied.  See the License for the
15// specific language governing permissions and limitations
16// under the License.
17
18//! Physical column reference: [`Column`]
19
20use std::hash::Hash;
21use std::sync::Arc;
22
23use crate::physical_expr::PhysicalExpr;
24use arrow::datatypes::FieldRef;
25use arrow::{
26    datatypes::{DataType, Schema, SchemaRef},
27    record_batch::RecordBatch,
28};
29use datafusion_common::tree_node::{Transformed, TreeNode};
30use datafusion_common::{Result, internal_err, plan_err};
31use datafusion_expr::ColumnarValue;
32use datafusion_expr_common::placement::ExpressionPlacement;
33
34/// Represents the column at a given index in a RecordBatch
35///
36/// This is a physical expression that represents a column at a given index in an
37/// arrow [`Schema`] / [`RecordBatch`].
38///
39/// Unlike the [logical `Expr::Column`], this expression is always resolved by schema index,
40/// even though it does have a name. This is because the physical plan is always
41/// resolved to a specific schema and there is no concept of "relation"
42///
43/// # Example:
44///  If the schema is `a`, `b`, `c` the `Column` for `b` would be represented by
45///  index 1, since `b` is the second column in the schema.
46///
47/// ```
48/// # use datafusion_physical_expr::expressions::Column;
49/// # use arrow::datatypes::{DataType, Field, Schema};
50/// // Schema with columns a, b, c
51/// let schema = Schema::new(vec![
52///     Field::new("a", DataType::Int32, false),
53///     Field::new("b", DataType::Int32, false),
54///     Field::new("c", DataType::Int32, false),
55/// ]);
56///
57/// // reference to column b is index 1
58/// let column_b = Column::new_with_schema("b", &schema).unwrap();
59/// assert_eq!(column_b.index(), 1);
60///
61/// // reference to column c is index 2
62/// let column_c = Column::new_with_schema("c", &schema).unwrap();
63/// assert_eq!(column_c.index(), 2);
64/// ```
65/// [logical `Expr::Column`]: https://docs.rs/datafusion/latest/datafusion/logical_expr/enum.Expr.html#variant.Column
66#[derive(Debug, Hash, PartialEq, Eq, Clone)]
67pub struct Column {
68    /// The name of the column (used for debugging and display purposes)
69    name: String,
70    /// The index of the column in its schema
71    index: usize,
72}
73
74impl Column {
75    /// Create a new column expression which references the
76    /// column with the given index in the schema.
77    pub fn new(name: &str, index: usize) -> Self {
78        Self {
79            name: name.to_owned(),
80            index,
81        }
82    }
83
84    /// Create a new column expression which references the
85    /// column with the given name in the schema
86    pub fn new_with_schema(name: &str, schema: &Schema) -> Result<Self> {
87        Ok(Column::new(name, schema.index_of(name)?))
88    }
89
90    /// Get the column's name
91    pub fn name(&self) -> &str {
92        &self.name
93    }
94
95    /// Get the column's schema index
96    pub fn index(&self) -> usize {
97        self.index
98    }
99}
100
101impl std::fmt::Display for Column {
102    fn fmt(&self, f: &mut std::fmt::Formatter) -> std::fmt::Result {
103        write!(f, "{}@{}", self.name, self.index)
104    }
105}
106
107impl PhysicalExpr for Column {
108    /// Get the data type of this expression, given the schema of the input
109    fn data_type(&self, input_schema: &Schema) -> Result<DataType> {
110        self.bounds_check(input_schema)?;
111        Ok(input_schema.field(self.index).data_type().clone())
112    }
113
114    /// Decide whether this expression is nullable, given the schema of the input
115    fn nullable(&self, input_schema: &Schema) -> Result<bool> {
116        self.bounds_check(input_schema)?;
117        Ok(input_schema.field(self.index).is_nullable())
118    }
119
120    /// Evaluate the expression
121    fn evaluate(&self, batch: &RecordBatch) -> Result<ColumnarValue> {
122        self.bounds_check(batch.schema().as_ref())?;
123        Ok(ColumnarValue::Array(Arc::clone(batch.column(self.index))))
124    }
125
126    fn return_field(&self, input_schema: &Schema) -> Result<FieldRef> {
127        self.bounds_check(input_schema)?;
128        Ok(input_schema.field(self.index).clone().into())
129    }
130
131    fn children(&self) -> Vec<&Arc<dyn PhysicalExpr>> {
132        vec![]
133    }
134
135    fn with_new_children(
136        self: Arc<Self>,
137        _children: Vec<Arc<dyn PhysicalExpr>>,
138    ) -> Result<Arc<dyn PhysicalExpr>> {
139        Ok(self)
140    }
141
142    fn fmt_sql(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
143        write!(f, "{}", self.name)
144    }
145
146    fn placement(&self) -> ExpressionPlacement {
147        ExpressionPlacement::Column
148    }
149
150    #[cfg(feature = "proto")]
151    fn try_to_proto(
152        &self,
153        _ctx: &datafusion_physical_expr_common::physical_expr::proto_encode::PhysicalExprEncodeCtx<'_>,
154    ) -> Result<Option<datafusion_proto_models::protobuf::PhysicalExprNode>> {
155        use datafusion_proto_models::protobuf;
156        Ok(Some(protobuf::PhysicalExprNode {
157            expr_id: None,
158            expr_type: Some(protobuf::physical_expr_node::ExprType::Column(self.into())),
159        }))
160    }
161}
162
163#[cfg(feature = "proto")]
164impl From<&datafusion_proto_models::protobuf::PhysicalColumn> for Column {
165    fn from(c: &datafusion_proto_models::protobuf::PhysicalColumn) -> Self {
166        Column::new(&c.name, c.index as usize)
167    }
168}
169
170#[cfg(feature = "proto")]
171impl From<&Column> for datafusion_proto_models::protobuf::PhysicalColumn {
172    fn from(c: &Column) -> Self {
173        Self {
174            name: c.name.clone(),
175            index: c.index as u32,
176        }
177    }
178}
179
180#[cfg(feature = "proto")]
181impl Column {
182    /// Reconstruct a [`Column`] from its protobuf representation.
183    ///
184    /// Takes the whole [`PhysicalExprNode`] — the exact inverse of what
185    /// [`PhysicalExpr::try_to_proto`] produces — so every expression's
186    /// `try_from_proto` shares one signature. The decode context is currently
187    /// unused, but is threaded through so that future expressions with child
188    /// sub-expressions can recurse via [`PhysicalExprDecodeCtx::decode`].
189    ///
190    /// [`PhysicalExprNode`]: datafusion_proto_models::protobuf::PhysicalExprNode
191    /// [`PhysicalExpr::try_to_proto`]: datafusion_physical_expr_common::physical_expr::PhysicalExpr::try_to_proto
192    /// [`PhysicalExprDecodeCtx::decode`]: datafusion_physical_expr_common::physical_expr::proto_decode::PhysicalExprDecodeCtx::decode
193    pub fn try_from_proto(
194        node: &datafusion_proto_models::protobuf::PhysicalExprNode,
195        _ctx: &datafusion_physical_expr_common::physical_expr::proto_decode::PhysicalExprDecodeCtx<'_>,
196    ) -> Result<Arc<dyn PhysicalExpr>> {
197        use datafusion_physical_expr_common::expect_expr_variant;
198        use datafusion_proto_models::protobuf;
199        let column = expect_expr_variant!(
200            node,
201            protobuf::physical_expr_node::ExprType::Column,
202            "Column",
203        );
204        Ok(Arc::new(Column::from(column)))
205    }
206}
207
208impl Column {
209    fn bounds_check(&self, input_schema: &Schema) -> Result<()> {
210        if self.index < input_schema.fields.len() {
211            Ok(())
212        } else {
213            internal_err!(
214                "PhysicalExpr Column references column '{}' at index {} (zero-based) but input schema only has {} columns: {:?}",
215                self.name,
216                self.index,
217                input_schema.fields.len(),
218                input_schema
219                    .fields()
220                    .iter()
221                    .map(|f| f.name())
222                    .collect::<Vec<_>>()
223            )
224        }
225    }
226}
227
228/// Create a column expression
229pub fn col(name: &str, schema: &Schema) -> Result<Arc<dyn PhysicalExpr>> {
230    Ok(Arc::new(Column::new_with_schema(name, schema)?))
231}
232
233/// Rewrites an expression according to new schema; i.e. changes the columns it
234/// refers to with the column at corresponding index in the new schema. Returns
235/// an error if the given schema has fewer columns than the original schema.
236/// Note that the resulting expression may not be valid if data types in the
237/// new schema is incompatible with expression nodes.
238pub fn with_new_schema(
239    expr: Arc<dyn PhysicalExpr>,
240    schema: &SchemaRef,
241) -> Result<Arc<dyn PhysicalExpr>> {
242    Ok(expr
243        .transform_up(|expr| {
244            if let Some(col) = expr.downcast_ref::<Column>() {
245                let idx = col.index();
246                let Some(field) = schema.fields().get(idx) else {
247                    return plan_err!(
248                        "New schema has fewer columns than original schema"
249                    );
250                };
251                let new_col = Column::new(field.name(), idx);
252                Ok(Transformed::yes(Arc::new(new_col) as _))
253            } else {
254                Ok(Transformed::no(expr))
255            }
256        })?
257        .data)
258}
259
260#[cfg(test)]
261mod test {
262    use super::Column;
263    use crate::physical_expr::PhysicalExpr;
264
265    use arrow::array::StringArray;
266    use arrow::datatypes::{DataType, Field, Schema};
267    use arrow::record_batch::RecordBatch;
268
269    use std::sync::Arc;
270
271    #[test]
272    fn out_of_bounds_data_type() {
273        let schema = Schema::new(vec![Field::new("foo", DataType::Utf8, true)]);
274        let col = Column::new("id", 9);
275        let error = col.data_type(&schema).expect_err("error").strip_backtrace();
276        assert!("Internal error: PhysicalExpr Column references column 'id' at index 9 (zero-based) \
277             but input schema only has 1 columns: [\"foo\"].\nThis issue was likely caused by a bug \
278             in DataFusion's code. Please help us to resolve this by filing a bug report \
279             in our issue tracker: https://github.com/apache/datafusion/issues".starts_with(&error))
280    }
281
282    #[test]
283    fn out_of_bounds_nullable() {
284        let schema = Schema::new(vec![Field::new("foo", DataType::Utf8, true)]);
285        let col = Column::new("id", 9);
286        let error = col.nullable(&schema).expect_err("error").strip_backtrace();
287        assert!("Internal error: PhysicalExpr Column references column 'id' at index 9 (zero-based) \
288             but input schema only has 1 columns: [\"foo\"].\nThis issue was likely caused by a bug \
289             in DataFusion's code. Please help us to resolve this by filing a bug report \
290             in our issue tracker: https://github.com/apache/datafusion/issues".starts_with(&error));
291    }
292
293    #[test]
294    fn out_of_bounds_evaluate() {
295        let schema = Schema::new(vec![Field::new("foo", DataType::Utf8, true)]);
296        let data: StringArray = vec!["data"].into();
297        let batch = RecordBatch::try_new(Arc::new(schema), vec![Arc::new(data)]).unwrap();
298        let col = Column::new("id", 9);
299        let error = col.evaluate(&batch).expect_err("error").strip_backtrace();
300        assert!("Internal error: PhysicalExpr Column references column 'id' at index 9 (zero-based) \
301             but input schema only has 1 columns: [\"foo\"].\nThis issue was likely caused by a bug \
302             in DataFusion's code. Please help us to resolve this by filing a bug report \
303             in our issue tracker: https://github.com/apache/datafusion/issues".starts_with(&error));
304    }
305}
datafusion_physical_expr/expressions/column.rs

datafusion_physical_expr/expressions/
column.rs
