Skip to main content

vortex_array/expr/
mod.rs

1// SPDX-License-Identifier: Apache-2.0
2// SPDX-FileCopyrightText: Copyright the Vortex contributors
3
4//! Vortex's expression language: scalar operations over [arrays](crate::ArrayRef).
5//!
6//! An [`Expression`] is a tree of scalar operations rooted at a scope (see [`root`]). Expressions
7//! are the common currency of scans: a scan takes a *filter* expression that resolves to a boolean
8//! and a *projection* expression that shapes the output. All expressions are serializable and own
9//! their own wire format, so they can be pushed down to remote sources and reconstructed on workers.
10//!
11//! # Scalar functions
12//!
13//! Each node references a scalar function defined by a
14//! [`ScalarFnVTable`](crate::scalar_fn::ScalarFnVTable). The vtable declares the function signature,
15//! properties such as strictness, and the logic that executes it over input arrays. Built-in
16//! functions live in [`crate::scalar_fn`]; integration and plugin crates supply additional,
17//! use-case-specific functions.
18//!
19//! # Deferred execution
20//!
21//! Applying an expression to an array does not compute the result eagerly. Instead it builds a
22//! [`ScalarFnArray`](crate::arrays::ScalarFnArray) representing the deferred application, letting
23//! downstream encodings push the computation into compressed data, or fuse several expressions
24//! together, before any data is materialized. The deferred tree is executed toward canonical form
25//! only when a result is actually required.
26//!
27//! # Type checking
28//!
29//! Expressions are strictly typed: an input array's dtype must match the function signature exactly,
30//! so callers perform any required casts themselves before building the expression. The one
31//! relaxation is nullability—for example, equality may compare a `u32` against a `u32?`, but never
32//! a `u32` against an `i32`.
33//!
34//! Scans decompose bound filter expressions into independent conjuncts so that they can evaluate
35//! and reorder the most selective predicates first.
36//!
37//! The implementation takes inspiration from [Postgres] and [Apache Datafusion].
38//!
39//! [Postgres]: https://www.postgresql.org/docs/current/sql-expressions.html
40//! [Apache Datafusion]: https://github.com/apache/datafusion/tree/5fac581efbaffd0e6a9edf931182517524526afd/datafusion/expr
41
42pub mod aliases;
43#[cfg(feature = "arbitrary")]
44pub mod arbitrary;
45pub mod bound_expression;
46pub mod display;
47pub(crate) mod expression;
48mod exprs;
49pub(crate) mod field;
50pub mod lambda;
51mod optimize;
52pub mod proto;
53pub mod scope;
54pub mod stats;
55pub mod traversal;
56
57pub use bound_expression::*;
58pub use expression::*;
59pub use exprs::and;
60pub use exprs::and_collect;
61pub use exprs::between;
62pub use exprs::binary;
63pub use exprs::bound;
64pub use exprs::byte_length;
65pub use exprs::case_when;
66pub use exprs::case_when_no_else;
67pub use exprs::cast;
68pub use exprs::checked_add;
69pub use exprs::col;
70pub use exprs::dynamic;
71pub use exprs::dynamic_expression;
72pub use exprs::dynamic_with_options;
73pub use exprs::eq;
74pub use exprs::ext_storage;
75pub use exprs::fill_null;
76pub use exprs::get_item;
77pub use exprs::gt;
78pub use exprs::gt_eq;
79pub use exprs::ilike;
80pub use exprs::in_list;
81pub use exprs::in_list_of;
82pub use exprs::is_nan;
83pub use exprs::is_not_null;
84pub use exprs::is_null;
85pub use exprs::is_root;
86pub use exprs::like;
87pub use exprs::list_contains;
88pub use exprs::list_contains_opts;
89pub use exprs::list_length;
90pub use exprs::list_sum;
91pub use exprs::list_sum_opts;
92pub use exprs::list_transform;
93pub use exprs::lit;
94pub use exprs::lt;
95pub use exprs::lt_eq;
96pub use exprs::mask;
97pub use exprs::merge;
98pub use exprs::merge_opts;
99pub use exprs::nested_case_when;
100pub use exprs::not;
101pub use exprs::not_eq;
102pub use exprs::not_ilike;
103pub use exprs::not_like;
104pub use exprs::or;
105pub use exprs::or_collect;
106pub use exprs::pack;
107pub use exprs::replace_time_zone;
108pub use exprs::root;
109pub use exprs::select;
110pub use exprs::select_exclude;
111pub use exprs::var;
112pub use exprs::variant_get;
113pub use exprs::zip_expr;
114pub use lambda::*;
115pub use scope::*;
116
117#[cfg(feature = "_test-harness")]
118pub mod test_harness {
119    use crate::dtype::DType;
120    use crate::dtype::Nullability;
121    use crate::dtype::PType;
122    use crate::dtype::StructFields;
123
124    pub fn struct_dtype() -> DType {
125        DType::Struct(
126            StructFields::new(
127                ["a", "col1", "col2", "bool1", "bool2"].into(),
128                vec![
129                    DType::Primitive(PType::I32, Nullability::NonNullable),
130                    DType::Primitive(PType::U16, Nullability::Nullable),
131                    DType::Primitive(PType::U16, Nullability::Nullable),
132                    DType::Bool(Nullability::NonNullable),
133                    DType::Bool(Nullability::NonNullable),
134                ],
135            ),
136            Nullability::NonNullable,
137        )
138    }
139}
140
141#[cfg(test)]
142mod tests {
143    use vortex_array::expr::eq;
144    use vortex_array::expr::lit;
145    use vortex_array::expr::root;
146
147    use super::*;
148    use crate::dtype::DType;
149    use crate::dtype::FieldName;
150    use crate::dtype::FieldNames;
151    use crate::dtype::Nullability;
152    use crate::dtype::PType;
153    use crate::dtype::StructFields;
154    use crate::expr::and;
155    use crate::expr::bound;
156    use crate::expr::case_when;
157    use crate::expr::col;
158    use crate::expr::gt;
159    use crate::expr::gt_eq;
160    use crate::expr::lt;
161    use crate::expr::lt_eq;
162    use crate::expr::not;
163    use crate::expr::not_eq;
164    use crate::expr::or;
165    use crate::expr::select;
166    use crate::expr::select_exclude;
167    use crate::scalar::Scalar;
168    use crate::scalar_fn::fns::literal::Literal;
169
170    #[test]
171    fn bound_constructors_preserve_order_and_types() -> vortex_error::VortexResult<()> {
172        let value_dtype = DType::Primitive(PType::I32, Nullability::NonNullable);
173        let scope = DType::Struct(
174            StructFields::from_iter([("value", value_dtype.clone())]),
175            Nullability::NonNullable,
176        );
177
178        let root = bound::root(scope.clone());
179        let value = bound::get_item("value", root);
180        let literal = bound::lit(5i32);
181        let condition = bound::gt(value.clone(), literal.clone());
182        assert_eq!(condition.dtype(), &DType::Bool(Nullability::NonNullable));
183        assert_eq!(condition.children(), &[value.clone(), literal.clone()]);
184
185        let case = bound::case_when(condition.clone(), value.clone(), literal.clone());
186        assert_eq!(case.dtype(), &value_dtype);
187        assert_eq!(case.children(), &[condition.clone(), value, literal]);
188
189        let packed = bound::pack(
190            [("condition", condition.clone()), ("value", case.clone())],
191            Nullability::NonNullable,
192        );
193        assert_eq!(packed.children(), &[condition, case.clone()]);
194        assert_eq!(
195            packed.dtype(),
196            &DType::Struct(
197                StructFields::from_iter([
198                    ("condition", DType::Bool(Nullability::NonNullable)),
199                    ("value", value_dtype),
200                ]),
201                Nullability::NonNullable,
202            )
203        );
204
205        let unbound = case_when(gt(col("value"), lit(5i32)), col("value"), lit(5i32));
206        assert_eq!(unbound.bind(&scope)?, case);
207        Ok(())
208    }
209
210    #[test]
211    fn expr_display() {
212        assert_eq!(col("a").to_string(), "$.a");
213        assert_eq!(root().to_string(), "$");
214
215        let col1: Expression = col("col1");
216        let col2: Expression = col("col2");
217        assert_eq!(
218            and(col1.clone(), col2.clone()).to_string(),
219            "($.col1 and $.col2)"
220        );
221        assert_eq!(
222            or(col1.clone(), col2.clone()).to_string(),
223            "($.col1 or $.col2)"
224        );
225        assert_eq!(
226            eq(col1.clone(), col2.clone()).to_string(),
227            "($.col1 = $.col2)"
228        );
229        assert_eq!(
230            not_eq(col1.clone(), col2.clone()).to_string(),
231            "($.col1 != $.col2)"
232        );
233        assert_eq!(
234            gt(col1.clone(), col2.clone()).to_string(),
235            "($.col1 > $.col2)"
236        );
237        assert_eq!(
238            gt_eq(col1.clone(), col2.clone()).to_string(),
239            "($.col1 >= $.col2)"
240        );
241        assert_eq!(
242            lt(col1.clone(), col2.clone()).to_string(),
243            "($.col1 < $.col2)"
244        );
245        assert_eq!(
246            lt_eq(col1.clone(), col2.clone()).to_string(),
247            "($.col1 <= $.col2)"
248        );
249
250        assert_eq!(
251            or(lt(col1.clone(), col2.clone()), not_eq(col1.clone(), col2),).to_string(),
252            "(($.col1 < $.col2) or ($.col1 != $.col2))"
253        );
254
255        assert_eq!(not(col1).to_string(), "vortex.not($.col1)");
256
257        assert_eq!(
258            select(vec![FieldName::from("col1")], root()).to_string(),
259            "${col1}"
260        );
261        assert_eq!(
262            select(
263                vec![FieldName::from("col1"), FieldName::from("col2")],
264                root()
265            )
266            .to_string(),
267            "${col1, col2}"
268        );
269        assert_eq!(
270            select_exclude(
271                vec![FieldName::from("col1"), FieldName::from("col2")],
272                root()
273            )
274            .to_string(),
275            "${~ col1, col2}"
276        );
277
278        assert_eq!(lit(Scalar::from(0u8)).to_string(), "0u8");
279        assert_eq!(lit(Scalar::from(0.0f32)).to_string(), "0f32");
280        assert_eq!(
281            lit(Scalar::from(i64::MAX)).to_string(),
282            "9223372036854775807i64"
283        );
284        assert_eq!(lit(Scalar::from(true)).to_string(), "true");
285        assert_eq!(
286            lit(Scalar::null(DType::Bool(Nullability::Nullable))).to_string(),
287            "null"
288        );
289
290        assert_eq!(
291            lit(Scalar::struct_(
292                DType::Struct(
293                    StructFields::new(
294                        FieldNames::from(["dog", "cat"]),
295                        vec![
296                            DType::Primitive(PType::U32, Nullability::NonNullable),
297                            DType::Utf8(Nullability::NonNullable)
298                        ],
299                    ),
300                    Nullability::NonNullable
301                ),
302                vec![Scalar::from(32_u32), Scalar::from("rufus".to_string())]
303            ))
304            .to_string(),
305            "{dog: 32u32, cat: \"rufus\"}"
306        );
307    }
308
309    #[test]
310    fn expr_contains() {
311        let expression = &eq(root(), lit(3u64));
312        assert!(expression.contains::<Literal>().unwrap());
313        let expression = root();
314        assert!(!expression.contains::<Literal>().unwrap());
315    }
316}