vortex_array/expr/
mod.rs

1// SPDX-License-Identifier: Apache-2.0
2// SPDX-FileCopyrightText: Copyright the Vortex contributors
3
4//! Vortex's expression language: scalar operations over [arrays](crate::ArrayRef).
5//!
6//! An [`Expression`] is a tree of scalar operations rooted at a scope (see [`root`]). Expressions
7//! are the common currency of scans: a scan takes a *filter* expression that resolves to a boolean
8//! and a *projection* expression that shapes the output. All expressions are serializable and own
9//! their own wire format, so they can be pushed down to remote sources and reconstructed on workers.
10//!
11//! # Scalar functions
12//!
13//! Each node references a scalar function defined by a
14//! [`ScalarFnVTable`](crate::scalar_fn::ScalarFnVTable). The vtable declares the function signature,
15//! properties such as strictness, and the logic that executes it over input arrays. Built-in
16//! functions live in [`crate::scalar_fn`]; integration and plugin crates supply additional,
17//! use-case-specific functions.
18//!
19//! # Deferred execution
20//!
21//! Applying an expression to an array does not compute the result eagerly. Instead it builds a
22//! [`ScalarFnArray`](crate::arrays::ScalarFnArray) representing the deferred application, letting
23//! downstream encodings push the computation into compressed data, or fuse several expressions
24//! together, before any data is materialized. The deferred tree is executed toward canonical form
25//! only when a result is actually required.
26//!
27//! # Type checking
28//!
29//! Expressions are strictly typed: an input array's dtype must match the function signature exactly,
30//! so callers perform any required casts themselves before building the expression. The one
31//! relaxation is nullability—for example, equality may compare a `u32` against a `u32?`, but never
32//! a `u32` against an `i32`.
33//!
34//! Scans decompose bound filter expressions into independent conjuncts so that they can evaluate
35//! and reorder the most selective predicates first.
36//!
37//! The implementation takes inspiration from [Postgres] and [Apache Datafusion].
38//!
39//! [Postgres]: https://www.postgresql.org/docs/current/sql-expressions.html
40//! [Apache Datafusion]: https://github.com/apache/datafusion/tree/5fac581efbaffd0e6a9edf931182517524526afd/datafusion/expr
41
42pub mod aliases;
43pub mod analysis;
44#[cfg(feature = "arbitrary")]
45pub mod arbitrary;
46pub mod bound_expression;
47pub mod display;
48pub(crate) mod expression;
49mod exprs;
50pub(crate) mod field;
51pub mod forms;
52mod optimize;
53pub mod proto;
54pub mod scope;
55pub mod stats;
56pub mod transform;
57pub mod traversal;
58
59pub use analysis::*;
60pub use bound_expression::*;
61pub use expression::*;
62pub use exprs::and;
63pub use exprs::and_collect;
64pub use exprs::between;
65pub use exprs::binary;
66pub use exprs::bound;
67pub use exprs::byte_length;
68pub use exprs::case_when;
69pub use exprs::case_when_no_else;
70pub use exprs::cast;
71pub use exprs::checked_add;
72pub use exprs::col;
73pub use exprs::dynamic;
74pub use exprs::dynamic_with_options;
75pub use exprs::eq;
76pub use exprs::ext_storage;
77pub use exprs::fill_null;
78pub use exprs::get_item;
79pub use exprs::gt;
80pub use exprs::gt_eq;
81pub use exprs::ilike;
82pub use exprs::in_list;
83pub use exprs::is_nan;
84pub use exprs::is_not_null;
85pub use exprs::is_null;
86pub use exprs::is_root;
87pub use exprs::like;
88pub use exprs::list_contains;
89pub use exprs::list_contains_opts;
90pub use exprs::list_length;
91pub use exprs::list_sum;
92pub use exprs::list_sum_opts;
93pub use exprs::lit;
94pub use exprs::lt;
95pub use exprs::lt_eq;
96pub use exprs::mask;
97pub use exprs::merge;
98pub use exprs::merge_opts;
99pub use exprs::nested_case_when;
100pub use exprs::not;
101pub use exprs::not_eq;
102pub use exprs::not_ilike;
103pub use exprs::not_like;
104pub use exprs::or;
105pub use exprs::or_collect;
106pub use exprs::pack;
107pub use exprs::replace_time_zone;
108pub use exprs::root;
109pub use exprs::select;
110pub use exprs::select_exclude;
111pub use exprs::variant_get;
112pub use exprs::zip_expr;
113pub use scope::*;
114
115#[cfg(feature = "_test-harness")]
116pub mod test_harness {
117    use crate::dtype::DType;
118    use crate::dtype::Nullability;
119    use crate::dtype::PType;
120    use crate::dtype::StructFields;
121
122    pub fn struct_dtype() -> DType {
123        DType::Struct(
124            StructFields::new(
125                ["a", "col1", "col2", "bool1", "bool2"].into(),
126                vec![
127                    DType::Primitive(PType::I32, Nullability::NonNullable),
128                    DType::Primitive(PType::U16, Nullability::Nullable),
129                    DType::Primitive(PType::U16, Nullability::Nullable),
130                    DType::Bool(Nullability::NonNullable),
131                    DType::Bool(Nullability::NonNullable),
132                ],
133            ),
134            Nullability::NonNullable,
135        )
136    }
137}
138
139#[cfg(test)]
140mod tests {
141    use vortex_array::expr::eq;
142    use vortex_array::expr::lit;
143    use vortex_array::expr::root;
144
145    use super::*;
146    use crate::dtype::DType;
147    use crate::dtype::FieldName;
148    use crate::dtype::FieldNames;
149    use crate::dtype::Nullability;
150    use crate::dtype::PType;
151    use crate::dtype::StructFields;
152    use crate::expr::and;
153    use crate::expr::bound;
154    use crate::expr::case_when;
155    use crate::expr::col;
156    use crate::expr::gt;
157    use crate::expr::gt_eq;
158    use crate::expr::lt;
159    use crate::expr::lt_eq;
160    use crate::expr::not;
161    use crate::expr::not_eq;
162    use crate::expr::or;
163    use crate::expr::select;
164    use crate::expr::select_exclude;
165    use crate::scalar::Scalar;
166    use crate::scalar_fn::fns::literal::Literal;
167
168    #[test]
169    fn bound_constructors_preserve_order_and_types() -> vortex_error::VortexResult<()> {
170        let value_dtype = DType::Primitive(PType::I32, Nullability::NonNullable);
171        let scope = DType::Struct(
172            StructFields::from_iter([("value", value_dtype.clone())]),
173            Nullability::NonNullable,
174        );
175
176        let root = bound::root(scope.clone());
177        let value = bound::get_item("value", root);
178        let literal = bound::lit(5i32);
179        let condition = bound::gt(value.clone(), literal.clone());
180        assert_eq!(condition.dtype(), &DType::Bool(Nullability::NonNullable));
181        assert_eq!(condition.children(), &[value.clone(), literal.clone()]);
182
183        let case = bound::case_when(condition.clone(), value.clone(), literal.clone());
184        assert_eq!(case.dtype(), &value_dtype);
185        assert_eq!(case.children(), &[condition.clone(), value, literal]);
186
187        let packed = bound::pack(
188            [("condition", condition.clone()), ("value", case.clone())],
189            Nullability::NonNullable,
190        );
191        assert_eq!(packed.children(), &[condition, case.clone()]);
192        assert_eq!(
193            packed.dtype(),
194            &DType::Struct(
195                StructFields::from_iter([
196                    ("condition", DType::Bool(Nullability::NonNullable)),
197                    ("value", value_dtype),
198                ]),
199                Nullability::NonNullable,
200            )
201        );
202
203        let unbound = case_when(gt(col("value"), lit(5i32)), col("value"), lit(5i32));
204        assert_eq!(unbound.bind(&scope)?, case);
205        Ok(())
206    }
207
208    #[test]
209    fn expr_display() {
210        assert_eq!(col("a").to_string(), "$.a");
211        assert_eq!(root().to_string(), "$");
212
213        let col1: Expression = col("col1");
214        let col2: Expression = col("col2");
215        assert_eq!(
216            and(col1.clone(), col2.clone()).to_string(),
217            "($.col1 and $.col2)"
218        );
219        assert_eq!(
220            or(col1.clone(), col2.clone()).to_string(),
221            "($.col1 or $.col2)"
222        );
223        assert_eq!(
224            eq(col1.clone(), col2.clone()).to_string(),
225            "($.col1 = $.col2)"
226        );
227        assert_eq!(
228            not_eq(col1.clone(), col2.clone()).to_string(),
229            "($.col1 != $.col2)"
230        );
231        assert_eq!(
232            gt(col1.clone(), col2.clone()).to_string(),
233            "($.col1 > $.col2)"
234        );
235        assert_eq!(
236            gt_eq(col1.clone(), col2.clone()).to_string(),
237            "($.col1 >= $.col2)"
238        );
239        assert_eq!(
240            lt(col1.clone(), col2.clone()).to_string(),
241            "($.col1 < $.col2)"
242        );
243        assert_eq!(
244            lt_eq(col1.clone(), col2.clone()).to_string(),
245            "($.col1 <= $.col2)"
246        );
247
248        assert_eq!(
249            or(lt(col1.clone(), col2.clone()), not_eq(col1.clone(), col2),).to_string(),
250            "(($.col1 < $.col2) or ($.col1 != $.col2))"
251        );
252
253        assert_eq!(not(col1).to_string(), "vortex.not($.col1)");
254
255        assert_eq!(
256            select(vec![FieldName::from("col1")], root()).to_string(),
257            "${col1}"
258        );
259        assert_eq!(
260            select(
261                vec![FieldName::from("col1"), FieldName::from("col2")],
262                root()
263            )
264            .to_string(),
265            "${col1, col2}"
266        );
267        assert_eq!(
268            select_exclude(
269                vec![FieldName::from("col1"), FieldName::from("col2")],
270                root()
271            )
272            .to_string(),
273            "${~ col1, col2}"
274        );
275
276        assert_eq!(lit(Scalar::from(0u8)).to_string(), "0u8");
277        assert_eq!(lit(Scalar::from(0.0f32)).to_string(), "0f32");
278        assert_eq!(
279            lit(Scalar::from(i64::MAX)).to_string(),
280            "9223372036854775807i64"
281        );
282        assert_eq!(lit(Scalar::from(true)).to_string(), "true");
283        assert_eq!(
284            lit(Scalar::null(DType::Bool(Nullability::Nullable))).to_string(),
285            "null"
286        );
287
288        assert_eq!(
289            lit(Scalar::struct_(
290                DType::Struct(
291                    StructFields::new(
292                        FieldNames::from(["dog", "cat"]),
293                        vec![
294                            DType::Primitive(PType::U32, Nullability::NonNullable),
295                            DType::Utf8(Nullability::NonNullable)
296                        ],
297                    ),
298                    Nullability::NonNullable
299                ),
300                vec![Scalar::from(32_u32), Scalar::from("rufus".to_string())]
301            ))
302            .to_string(),
303            "{dog: 32u32, cat: \"rufus\"}"
304        );
305    }
306
307    #[test]
308    fn expr_contains() {
309        let expression = &eq(root(), lit(3u64));
310        assert!(expression.contains::<Literal>().unwrap());
311        let expression = root();
312        assert!(!expression.contains::<Literal>().unwrap());
313    }
314}
vortex_array/expr/mod.rs

vortex_array/expr/
mod.rs
