Skip to main content

vortex_array/scalar_fn/fns/
pack.rs

1// SPDX-License-Identifier: Apache-2.0
2// SPDX-FileCopyrightText: Copyright the Vortex contributors
3
4use std::fmt::Display;
5use std::fmt::Formatter;
6use std::hash::Hash;
7use std::sync::Arc;
8
9use itertools::Itertools as _;
10use prost::Message;
11use vortex_error::VortexResult;
12use vortex_session::VortexSession;
13use vortex_session::registry::CachedId;
14
15use crate::ArrayRef;
16use crate::ExecutionCtx;
17use crate::IntoArray;
18use crate::arrays::StructArray;
19use crate::dtype::DType;
20use crate::dtype::FieldName;
21use crate::dtype::FieldNames;
22use crate::dtype::Nullability;
23use crate::dtype::StructFields;
24use crate::expr::display::ExprDisplay;
25use crate::proto::expr as pb;
26use crate::scalar_fn::Arity;
27use crate::scalar_fn::ChildName;
28use crate::scalar_fn::ExecutionArgs;
29use crate::scalar_fn::ReduceNode;
30use crate::scalar_fn::ReduceNodeValidity;
31use crate::scalar_fn::ScalarFnId;
32use crate::scalar_fn::ScalarFnVTable;
33use crate::validity::Validity;
34
35/// Pack zero or more expressions into a structure with named fields.
36#[derive(Clone)]
37pub struct Pack;
38
39#[derive(Debug, Clone, PartialEq, Eq, Hash)]
40pub struct PackOptions {
41    pub names: FieldNames,
42    pub nullability: Nullability,
43}
44
45impl Display for PackOptions {
46    fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result {
47        write!(
48            f,
49            "names: [{}], nullability: {:#}",
50            self.names.iter().join(", "),
51            self.nullability
52        )
53    }
54}
55
56impl ScalarFnVTable for Pack {
57    type Options = PackOptions;
58
59    fn id(&self) -> ScalarFnId {
60        static ID: CachedId = CachedId::new("vortex.pack");
61        *ID
62    }
63
64    fn serialize(&self, instance: &Self::Options) -> VortexResult<Option<Vec<u8>>> {
65        Ok(Some(
66            pb::PackOpts {
67                paths: instance.names.iter().map(|n| n.to_string()).collect(),
68                nullable: instance.nullability.into(),
69            }
70            .encode_to_vec(),
71        ))
72    }
73
74    fn deserialize(
75        &self,
76        _metadata: &[u8],
77        _session: &VortexSession,
78    ) -> VortexResult<Self::Options> {
79        let opts = pb::PackOpts::decode(_metadata)?;
80        let names: FieldNames = opts
81            .paths
82            .iter()
83            .map(|name| FieldName::from(name.as_str()))
84            .collect();
85        Ok(PackOptions {
86            names,
87            nullability: opts.nullable.into(),
88        })
89    }
90
91    fn arity(&self, options: &Self::Options) -> Arity {
92        Arity::Exact(options.names.len())
93    }
94
95    fn child_name(&self, instance: &Self::Options, child_idx: usize) -> ChildName {
96        match instance.names.get(child_idx) {
97            Some(name) => ChildName::from(Arc::clone(name.inner())),
98            None => unreachable!(
99                "Invalid child index {} for Pack expression with {} fields",
100                child_idx,
101                instance.names.len()
102            ),
103        }
104    }
105
106    fn fmt_sql(
107        &self,
108        options: &Self::Options,
109        expr: &dyn ExprDisplay,
110        f: &mut Formatter<'_>,
111    ) -> std::fmt::Result {
112        write!(f, "pack(")?;
113        for (i, name) in options.names.iter().enumerate() {
114            write!(f, "{}: ", name)?;
115            Display::fmt(expr.display_child(i), f)?;
116            if i + 1 < options.names.len() {
117                write!(f, ", ")?;
118            }
119        }
120        write!(f, "){}", options.nullability)
121    }
122
123    fn return_dtype(&self, options: &Self::Options, arg_dtypes: &[DType]) -> VortexResult<DType> {
124        Ok(DType::Struct(
125            StructFields::new(options.names.clone(), arg_dtypes.to_vec()),
126            options.nullability,
127        ))
128    }
129
130    fn validity<T: ReduceNode>(
131        &self,
132        _options: &Self::Options,
133        node: &T,
134    ) -> VortexResult<ReduceNodeValidity<T>> {
135        Ok(ReduceNodeValidity::Reduced(node.new_constant(true.into())))
136    }
137
138    fn execute(
139        &self,
140        options: &Self::Options,
141        args: &dyn ExecutionArgs,
142        ctx: &mut ExecutionCtx,
143    ) -> VortexResult<ArrayRef> {
144        let len = args.row_count();
145        let value_arrays: Vec<ArrayRef> = (0..args.num_inputs())
146            .map(|i| args.get(i))
147            .collect::<VortexResult<_>>()?;
148        let validity: Validity = options.nullability.into();
149        StructArray::try_new(options.names.clone(), value_arrays, len, validity)?
150            .into_array()
151            .execute(ctx)
152    }
153
154    fn is_strict(&self, _instance: &Self::Options) -> bool {
155        // A null field value does not force the packed struct row to be null.
156        false
157    }
158
159    fn is_infallible(&self, _instance: &Self::Options) -> bool {
160        true
161    }
162}
163
164#[cfg(test)]
165mod tests {
166    use std::sync::LazyLock;
167
168    use vortex_buffer::buffer;
169    use vortex_error::VortexResult;
170    use vortex_error::vortex_bail;
171    use vortex_session::VortexSession;
172
173    use super::Pack;
174    use super::PackOptions;
175    use crate::ArrayRef;
176    use crate::IntoArray;
177    use crate::VortexSessionExecute;
178    use crate::arrays::PrimitiveArray;
179    use crate::arrays::struct_::StructArrayExt;
180    use crate::assert_arrays_eq;
181    use crate::dtype::Nullability;
182    use crate::expr::col;
183    use crate::expr::pack;
184    use crate::scalar_fn::ScalarFnVTableExt;
185    use crate::scalar_fn::fns::pack::StructArray;
186    use crate::validity::Validity;
187
188    static SESSION: LazyLock<VortexSession> = LazyLock::new(VortexSession::empty);
189
190    fn test_array() -> ArrayRef {
191        StructArray::from_fields(&[
192            ("a", buffer![0, 1, 2].into_array()),
193            ("b", buffer![4, 5, 6].into_array()),
194        ])
195        .unwrap()
196        .into_array()
197    }
198
199    fn field_array(array: &ArrayRef, field_path: &[&str]) -> VortexResult<ArrayRef> {
200        let mut ctx = SESSION.create_execution_ctx();
201        let mut field_path = field_path.iter();
202
203        let Some(field) = field_path.next() else {
204            vortex_bail!("empty field path");
205        };
206
207        let mut array = array
208            .clone()
209            .execute::<StructArray>(&mut ctx)?
210            .unmasked_field_by_name(field)?
211            .clone();
212        for field in field_path {
213            let next = array
214                .clone()
215                .execute::<StructArray>(&mut ctx)?
216                .unmasked_field_by_name(field)?
217                .clone();
218            array = next;
219        }
220        Ok(array)
221    }
222
223    #[test]
224    pub fn test_empty_pack() {
225        let mut ctx = SESSION.create_execution_ctx();
226        let expr = Pack.new_expr(
227            PackOptions {
228                names: Default::default(),
229                nullability: Default::default(),
230            },
231            [],
232        );
233
234        let test_array = test_array();
235        let actual_array = test_array.clone().apply(&expr).unwrap();
236        assert_eq!(actual_array.len(), test_array.len());
237        let nfields = actual_array
238            .execute::<StructArray>(&mut ctx)
239            .unwrap()
240            .struct_fields()
241            .nfields();
242        assert_eq!(nfields, 0);
243    }
244
245    #[test]
246    pub fn test_simple_pack() {
247        let mut ctx = SESSION.create_execution_ctx();
248        let expr = Pack.new_expr(
249            PackOptions {
250                names: ["one", "two", "three"].into(),
251                nullability: Nullability::NonNullable,
252            },
253            [col("a"), col("b"), col("a")],
254        );
255
256        let actual_array = test_array()
257            .apply(&expr)
258            .unwrap()
259            .execute::<StructArray>(&mut ctx)
260            .unwrap();
261
262        assert_eq!(actual_array.names(), ["one", "two", "three"]);
263        assert!(matches!(actual_array.validity(), Ok(Validity::NonNullable)));
264
265        assert_arrays_eq!(
266            field_array(&actual_array.clone().into_array(), &["one"]).unwrap(),
267            PrimitiveArray::from_iter([0i32, 1, 2]),
268            &mut ctx
269        );
270        assert_arrays_eq!(
271            field_array(&actual_array.clone().into_array(), &["two"]).unwrap(),
272            PrimitiveArray::from_iter([4i32, 5, 6]),
273            &mut ctx
274        );
275        assert_arrays_eq!(
276            field_array(&actual_array.into_array(), &["three"]).unwrap(),
277            PrimitiveArray::from_iter([0i32, 1, 2]),
278            &mut ctx
279        );
280    }
281
282    #[test]
283    pub fn test_nested_pack() {
284        let mut ctx = SESSION.create_execution_ctx();
285        let expr = Pack.new_expr(
286            PackOptions {
287                names: ["one", "two", "three"].into(),
288                nullability: Nullability::NonNullable,
289            },
290            [
291                col("a"),
292                Pack.new_expr(
293                    PackOptions {
294                        names: ["two_one", "two_two"].into(),
295                        nullability: Nullability::NonNullable,
296                    },
297                    [col("b"), col("b")],
298                ),
299                col("a"),
300            ],
301        );
302
303        let actual_array = test_array()
304            .apply(&expr)
305            .unwrap()
306            .execute::<StructArray>(&mut ctx)
307            .unwrap();
308
309        assert_eq!(actual_array.names(), ["one", "two", "three"]);
310
311        assert_arrays_eq!(
312            field_array(&actual_array.clone().into_array(), &["one"]).unwrap(),
313            PrimitiveArray::from_iter([0i32, 1, 2]),
314            &mut ctx
315        );
316        assert_arrays_eq!(
317            field_array(&actual_array.clone().into_array(), &["two", "two_one"]).unwrap(),
318            PrimitiveArray::from_iter([4i32, 5, 6]),
319            &mut ctx
320        );
321        assert_arrays_eq!(
322            field_array(&actual_array.clone().into_array(), &["two", "two_two"]).unwrap(),
323            PrimitiveArray::from_iter([4i32, 5, 6]),
324            &mut ctx
325        );
326        assert_arrays_eq!(
327            field_array(&actual_array.into_array(), &["three"]).unwrap(),
328            PrimitiveArray::from_iter([0i32, 1, 2]),
329            &mut ctx
330        );
331    }
332
333    #[test]
334    pub fn test_pack_nullable() {
335        let mut ctx = SESSION.create_execution_ctx();
336        let expr = Pack.new_expr(
337            PackOptions {
338                names: ["one", "two", "three"].into(),
339                nullability: Nullability::Nullable,
340            },
341            [col("a"), col("b"), col("a")],
342        );
343
344        let actual_array = test_array()
345            .apply(&expr)
346            .unwrap()
347            .execute::<StructArray>(&mut ctx)
348            .unwrap();
349
350        assert_eq!(actual_array.names(), ["one", "two", "three"]);
351        assert!(matches!(actual_array.validity(), Ok(Validity::AllValid)));
352        let expected = StructArray::try_new(
353            ["one", "two", "three"].into(),
354            vec![
355                buffer![0i32, 1, 2].into_array(),
356                buffer![4i32, 5, 6].into_array(),
357                buffer![0i32, 1, 2].into_array(),
358            ],
359            3,
360            Validity::AllValid,
361        )
362        .unwrap();
363        assert_arrays_eq!(actual_array, expected, &mut ctx);
364    }
365
366    #[test]
367    pub fn test_display() {
368        let expr = pack(
369            [("id", col("user_id")), ("name", col("username"))],
370            Nullability::NonNullable,
371        );
372        assert_eq!(expr.to_string(), "pack(id: $.user_id, name: $.username)");
373
374        let expr2 = Pack.new_expr(
375            PackOptions {
376                names: ["x", "y"].into(),
377                nullability: Nullability::Nullable,
378            },
379            [col("a"), col("b")],
380        );
381        assert_eq!(expr2.to_string(), "pack(x: $.a, y: $.b)?");
382    }
383}