Skip to main content

vortex_array/scalar_fn/fns/
get_item.rs

1// SPDX-License-Identifier: Apache-2.0
2// SPDX-FileCopyrightText: Copyright the Vortex contributors
3
4use std::fmt::Display;
5use std::fmt::Formatter;
6use std::slice;
7
8use prost::Message;
9use vortex_error::VortexResult;
10use vortex_error::vortex_err;
11use vortex_session::VortexSession;
12use vortex_session::registry::CachedId;
13
14use crate::ArrayRef;
15use crate::ExecutionCtx;
16use crate::IntoArray;
17use crate::arrays::ConstantArray;
18use crate::arrays::ScalarFnArray;
19use crate::arrays::StructArray;
20use crate::arrays::struct_::StructArrayExt;
21use crate::builtins::ArrayBuiltins;
22use crate::dtype::DType;
23use crate::dtype::FieldName;
24use crate::dtype::Nullability;
25use crate::expr::display::ExprDisplay;
26use crate::proto::expr as pb;
27use crate::scalar::Scalar;
28use crate::scalar_fn::Arity;
29use crate::scalar_fn::ChildName;
30use crate::scalar_fn::EmptyOptions;
31use crate::scalar_fn::ExecutionArgs;
32use crate::scalar_fn::ReduceNode;
33use crate::scalar_fn::ScalarFnId;
34use crate::scalar_fn::ScalarFnVTable;
35use crate::scalar_fn::ScalarFnVTableExt;
36use crate::scalar_fn::fns::mask::Mask;
37use crate::scalar_fn::fns::pack::Pack;
38use crate::scalar_fn::fns::select::FieldSelection;
39use crate::scalar_fn::fns::select::Select;
40
41#[derive(Clone)]
42pub struct GetItem;
43
44impl GetItem {
45    /// Creates a lazy projection of `field_name` from `input`.
46    ///
47    /// # Errors
48    ///
49    /// Returns an error if `input` is not a struct or does not contain `field_name`.
50    pub fn try_new(
51        input: ArrayRef,
52        field_name: impl Into<FieldName>,
53    ) -> VortexResult<ScalarFnArray> {
54        ScalarFnArray::try_new(GetItem.bind(field_name.into()), [input])
55    }
56
57    /// Read `field_name` out of `struct_node`, without a `get_item` node where possible.
58    ///
59    /// Rules that read a field out of their own child use this instead of building the
60    /// `get_item` node themselves. The optimizer visits children before their parent, so a
61    /// `get_item` that a rule introduces is never simplified again. Removing it here keeps the
62    /// output of the rule a fixed point.
63    pub(crate) fn apply_now<T: ReduceNode>(
64        struct_node: &T,
65        field_name: &FieldName,
66    ) -> VortexResult<T> {
67        if let Some(field) = Self::maybe_apply_now(struct_node, field_name)? {
68            return Ok(field);
69        }
70
71        struct_node.new_node(
72            GetItem.bind(field_name.clone()),
73            slice::from_ref(struct_node),
74        )
75    }
76
77    /// Extract a constant field, bypass a selection, or read a field directly from a pack.
78    fn maybe_apply_now<T: ReduceNode>(
79        struct_node: &T,
80        field_name: &FieldName,
81    ) -> VortexResult<Option<T>> {
82        if let Some(scalar) = struct_node.as_constant() {
83            return Ok(Some(
84                struct_node.new_constant(Self::constant_field(scalar, field_name)?),
85            ));
86        }
87
88        if let Some(selection) = struct_node
89            .scalar_fn()
90            .and_then(|scalar_fn| scalar_fn.as_opt::<Select>())
91        {
92            let included = match selection {
93                FieldSelection::Include(fields) => fields.as_ref().contains(field_name),
94                FieldSelection::Exclude(fields) => !fields.as_ref().contains(field_name),
95            };
96            if included {
97                return Self::apply_now(&struct_node.child(0), field_name).map(Some);
98            }
99            return Ok(None);
100        }
101
102        let Some(pack) = struct_node
103            .scalar_fn()
104            .and_then(|scalar_fn| scalar_fn.as_opt::<Pack>())
105        else {
106            return Ok(None);
107        };
108        let Some(idx) = pack.names.find(field_name) else {
109            return Ok(None);
110        };
111
112        let field = struct_node.child(idx);
113        if !pack.nullability.is_nullable() {
114            return Ok(Some(field));
115        }
116
117        // A nullable pack intersects its validity into the field, so mask the field with an
118        // all-true constant to make its dtype nullable.
119        Ok(Some(struct_node.new_node(
120            Mask.bind(EmptyOptions),
121            &[field, struct_node.new_constant(true.into())],
122        )?))
123    }
124
125    fn constant_field(scalar: &Scalar, field_name: &FieldName) -> VortexResult<Scalar> {
126        let dtype = Self.return_dtype(field_name, &[scalar.dtype().clone()])?;
127        if scalar.is_null() {
128            return Ok(Scalar::null(dtype));
129        }
130        scalar
131            .as_struct()
132            .field(field_name)
133            .ok_or_else(|| {
134                vortex_err!(
135                    "Field '{}' missing from constant struct array {}",
136                    field_name,
137                    scalar.dtype()
138                )
139            })?
140            .cast(&dtype)
141    }
142}
143
144impl ScalarFnVTable for GetItem {
145    type Options = FieldName;
146
147    fn id(&self) -> ScalarFnId {
148        static ID: CachedId = CachedId::new("vortex.get_item");
149        *ID
150    }
151
152    fn serialize(&self, instance: &Self::Options) -> VortexResult<Option<Vec<u8>>> {
153        Ok(Some(
154            pb::GetItemOpts {
155                path: instance.to_string(),
156            }
157            .encode_to_vec(),
158        ))
159    }
160
161    fn deserialize(
162        &self,
163        _metadata: &[u8],
164        _session: &VortexSession,
165    ) -> VortexResult<Self::Options> {
166        let opts = pb::GetItemOpts::decode(_metadata)?;
167        Ok(FieldName::from(opts.path))
168    }
169
170    fn arity(&self, _field_name: &FieldName) -> Arity {
171        Arity::Exact(1)
172    }
173
174    fn child_name(&self, _instance: &Self::Options, child_idx: usize) -> ChildName {
175        match child_idx {
176            0 => ChildName::from("input"),
177            _ => unreachable!("Invalid child index {} for GetItem expression", child_idx),
178        }
179    }
180
181    fn fmt_sql(
182        &self,
183        field_name: &FieldName,
184        expr: &dyn ExprDisplay,
185        f: &mut Formatter<'_>,
186    ) -> std::fmt::Result {
187        Display::fmt(expr.display_child(0), f)?;
188        write!(f, ".{}", field_name)
189    }
190
191    fn return_dtype(&self, field_name: &FieldName, arg_dtypes: &[DType]) -> VortexResult<DType> {
192        let struct_dtype = &arg_dtypes[0];
193        let field_dtype = struct_dtype
194            .as_struct_fields_opt()
195            .and_then(|st| st.field(field_name))
196            .ok_or_else(|| {
197                vortex_err!("Couldn't find the {} field in the input scope", field_name)
198            })?;
199
200        // Match here to avoid cloning the dtype if nullability doesn't need to change
201        if matches!(
202            (struct_dtype.nullability(), field_dtype.nullability()),
203            (Nullability::Nullable, Nullability::NonNullable)
204        ) {
205            return Ok(field_dtype.with_nullability(Nullability::Nullable));
206        }
207
208        Ok(field_dtype)
209    }
210
211    fn execute(
212        &self,
213        field_name: &FieldName,
214        args: &dyn ExecutionArgs,
215        ctx: &mut ExecutionCtx,
216    ) -> VortexResult<ArrayRef> {
217        let input = args.get(0)?;
218        if let Some(scalar) = input.as_constant() {
219            return Ok(
220                ConstantArray::new(Self::constant_field(scalar, field_name)?, input.len())
221                    .into_array(),
222            );
223        }
224
225        let input = input.execute::<StructArray>(ctx)?;
226        let field = input.unmasked_field_by_name(field_name).cloned()?;
227
228        match input.dtype().nullability() {
229            Nullability::NonNullable => Ok(field),
230            Nullability::Nullable => field.mask(input.validity()?.to_array(input.len())),
231        }
232    }
233
234    fn reduce<T: ReduceNode>(&self, field_name: &FieldName, node: &T) -> VortexResult<Option<T>> {
235        Self::maybe_apply_now(&node.child(0), field_name)
236    }
237
238    fn is_strict(&self, _field_name: &FieldName) -> bool {
239        true
240    }
241
242    fn is_infallible(&self, _field_name: &FieldName) -> bool {
243        // If this type-checks, it is infallible.
244        true
245    }
246}
247
248#[cfg(test)]
249mod tests {
250    use std::sync::LazyLock;
251
252    use rstest::rstest;
253    use vortex_buffer::buffer;
254    use vortex_error::VortexResult;
255    use vortex_session::VortexSession;
256
257    use super::GetItem;
258    use crate::ArrayRef;
259    use crate::IntoArray;
260    use crate::VortexSessionExecute;
261    use crate::arrays::Constant;
262    use crate::arrays::ConstantArray;
263    use crate::arrays::Filter;
264    use crate::arrays::Primitive;
265    use crate::arrays::PrimitiveArray;
266    use crate::arrays::ScalarFn;
267    use crate::arrays::ScalarFnArray;
268    use crate::arrays::Slice;
269    use crate::assert_arrays_eq;
270    use crate::builtins::ArrayBuiltins;
271    use crate::dtype::DType;
272    use crate::dtype::FieldName;
273    use crate::dtype::FieldNames;
274    use crate::dtype::Nullability;
275    use crate::dtype::Nullability::NonNullable;
276    use crate::dtype::PType;
277    use crate::dtype::StructFields;
278    use crate::expr::Expression;
279    use crate::expr::checked_add;
280    use crate::expr::get_item;
281    use crate::expr::lit;
282    use crate::expr::pack;
283    use crate::expr::root;
284    use crate::expr::select;
285    use crate::expr::select_exclude;
286    use crate::optimizer::ArrayOptimizer;
287    use crate::scalar::Scalar;
288    use crate::scalar_fn::ScalarFnVTableExt;
289    use crate::scalar_fn::fns::get_item::StructArray;
290    use crate::scalar_fn::fns::select::FieldSelection;
291    use crate::scalar_fn::fns::select::Select;
292    use crate::validity::Validity;
293
294    static SESSION: LazyLock<VortexSession> = LazyLock::new(VortexSession::empty);
295
296    fn test_array() -> StructArray {
297        StructArray::from_fields(&[
298            ("a", buffer![0i32, 1, 2].into_array()),
299            ("b", buffer![4i64, 5, 6].into_array()),
300        ])
301        .unwrap()
302    }
303
304    #[test]
305    fn get_item_by_name() {
306        let st = test_array();
307        let get_item = get_item("a", root());
308        assert!(
309            get_item
310                .as_scalar()
311                .is_some_and(|f| f.signature().is_strict())
312        );
313        let item = st.into_array().apply(&get_item).unwrap();
314        assert_eq!(item.dtype(), &DType::from(PType::I32));
315        assert_arrays_eq!(
316            item,
317            buffer![0i32, 1, 2].into_array(),
318            &mut SESSION.create_execution_ctx()
319        );
320    }
321
322    #[test]
323    fn get_item_by_name_none() {
324        let st = test_array();
325        let get_item = get_item("c", root());
326        assert!(st.into_array().apply(&get_item).is_err());
327    }
328
329    #[test]
330    fn get_nullable_field() {
331        let st = StructArray::try_new(
332            FieldNames::from(["a"]),
333            vec![buffer![1i32].into_array()],
334            1,
335            Validity::AllInvalid,
336        )
337        .unwrap()
338        .into_array();
339
340        let get_item_expr = get_item("a", root());
341        let item = st.apply(&get_item_expr).unwrap();
342        // The dtype should be nullable since it inherits struct validity
343        assert_eq!(
344            item.dtype(),
345            &DType::Primitive(PType::I32, Nullability::Nullable)
346        );
347        assert_arrays_eq!(
348            item,
349            PrimitiveArray::from_option_iter([None::<i32>]),
350            &mut SESSION.create_execution_ctx()
351        );
352    }
353
354    #[test]
355    fn get_non_nullable_field_from_all_valid_nullable_struct() -> VortexResult<()> {
356        let st = StructArray::try_new(
357            FieldNames::from(["a"]),
358            vec![buffer![1i32].into_array()],
359            1,
360            Validity::AllValid,
361        )?
362        .into_array();
363
364        let item = st.apply(&get_item("a", root()))?;
365        assert_eq!(
366            item.dtype(),
367            &DType::Primitive(PType::I32, Nullability::Nullable)
368        );
369        assert_arrays_eq!(
370            item,
371            PrimitiveArray::from_option_iter([Some(1i32)]),
372            &mut SESSION.create_execution_ctx()
373        );
374        Ok(())
375    }
376
377    #[test]
378    fn execute_constant_struct_stays_constant() -> VortexResult<()> {
379        let field_count = 128usize;
380        let selected_idx = 97i32;
381        let names = (0..field_count)
382            .map(|idx| FieldName::from(format!("f{idx}")))
383            .collect::<Vec<_>>();
384        let dtypes = vec![DType::Primitive(PType::I32, NonNullable); field_count];
385        let fields = StructFields::new(FieldNames::from(names), dtypes);
386        let scalar = Scalar::struct_(
387            DType::Struct(fields, NonNullable),
388            (0..128).map(|idx| Scalar::primitive(idx, NonNullable)),
389        );
390        let array = ConstantArray::new(scalar, 1_000_000).into_array();
391        let mut ctx = SESSION.create_execution_ctx();
392
393        let item: ArrayRef = GetItem::try_new(array, "f97")?
394            .into_array()
395            .execute(&mut ctx)?;
396
397        let constant = item.as_::<Constant>();
398        assert_eq!(constant.len(), 1_000_000);
399        assert_eq!(
400            constant.scalar(),
401            &Scalar::primitive(selected_idx, NonNullable)
402        );
403        Ok(())
404    }
405
406    #[rstest]
407    #[case::include(FieldSelection::Include(FieldNames::from(["a"])))]
408    #[case::exclude(FieldSelection::Exclude(FieldNames::from(["b"])))]
409    fn get_item_reduces_through_select(
410        #[case] selection: FieldSelection,
411        #[values(
412            Validity::NonNullable,
413            Validity::AllValid,
414            Validity::AllInvalid,
415            Validity::from_iter([true, false, true])
416        )]
417        validity: Validity,
418    ) -> VortexResult<()> {
419        let array = StructArray::try_new(
420            FieldNames::from(["a", "b"]),
421            vec![
422                ConstantArray::new(42i32, 3).into_array(),
423                buffer![1i32, 2, 3].into_array(),
424            ],
425            3,
426            validity,
427        )?
428        .into_array();
429        let expected = array.get_item("a")?;
430        let selected = ScalarFnArray::try_new(Select.bind(selection), [array])?.into_array();
431        let item = GetItem::try_new(selected, "a")?.into_array().optimize()?;
432
433        assert_eq!(item.encoding_id(), expected.encoding_id());
434        assert_eq!(item.dtype(), expected.dtype());
435        assert_arrays_eq!(item, expected, &mut SESSION.create_execution_ctx());
436        Ok(())
437    }
438
439    #[rstest]
440    #[case::include(select(["a"], root()))]
441    #[case::exclude(select_exclude(["b"], root()))]
442    fn get_item_select_expression_reduces(#[case] selected: Expression) -> VortexResult<()> {
443        let dtype = test_array().dtype().clone();
444        let result = get_item("a", selected).bind(&dtype)?.optimize_recursive()?;
445        assert_eq!(result, get_item("a", root()).bind(&dtype)?);
446        Ok(())
447    }
448
449    #[rstest]
450    #[case::include(select(["b"], root()))]
451    #[case::exclude(select_exclude(["a"], root()))]
452    fn get_item_select_rejects_removed_field(#[case] selected: Expression) {
453        assert!(
454            test_array()
455                .into_array()
456                .apply(&get_item("a", selected))
457                .is_err()
458        );
459    }
460
461    #[rstest]
462    fn get_item_constant_reduces(
463        #[values(false, true)] null: bool,
464        #[values(0, 3)] len: usize,
465    ) -> VortexResult<()> {
466        let dtype = DType::Struct(
467            StructFields::new(FieldNames::from(["a"]), vec![DType::from(PType::I32)]),
468            Nullability::Nullable,
469        );
470        let scalar = if null {
471            Scalar::null(dtype)
472        } else {
473            Scalar::struct_(dtype, [Scalar::from(42i32)])
474        };
475        let item = ConstantArray::new(scalar, len)
476            .into_array()
477            .apply(&get_item("a", root()))?;
478        let expected = if null {
479            Scalar::null(DType::Primitive(PType::I32, Nullability::Nullable))
480        } else {
481            Scalar::primitive(42i32, Nullability::Nullable)
482        };
483
484        assert_eq!(item.as_::<Constant>().scalar(), &expected);
485        assert_eq!(item.len(), len);
486        Ok(())
487    }
488
489    #[test]
490    fn get_item_pushes_through_filter() -> VortexResult<()> {
491        use vortex_mask::Mask;
492
493        let filtered = test_array()
494            .into_array()
495            .filter(Mask::from_iter([true, false, true]))?;
496
497        let item = filtered.get_item("a")?;
498
499        assert!(item.is::<Filter>());
500        assert!(!item.is::<ScalarFn>());
501        assert_arrays_eq!(
502            item,
503            buffer![0i32, 2].into_array(),
504            &mut SESSION.create_execution_ctx()
505        );
506        Ok(())
507    }
508
509    #[test]
510    fn get_item_pushes_through_slice() -> VortexResult<()> {
511        let sliced = test_array().into_array().slice(1..3)?;
512
513        let item = sliced.get_item("a")?;
514
515        assert!(item.is::<Primitive>());
516        assert!(!item.is::<Slice>());
517        assert!(!item.is::<ScalarFn>());
518        assert_arrays_eq!(
519            item,
520            buffer![1i32, 2].into_array(),
521            &mut SESSION.create_execution_ctx()
522        );
523        Ok(())
524    }
525
526    #[test]
527    fn get_item_pushes_through_masked() -> VortexResult<()> {
528        let masked = test_array()
529            .into_array()
530            .mask(Validity::from_iter([true, false, true]).to_array(3))?;
531
532        let item = masked.get_item("a")?;
533
534        assert!(item.is::<Primitive>());
535        assert_eq!(
536            item.dtype(),
537            &DType::Primitive(PType::I32, Nullability::Nullable)
538        );
539        assert!(!item.is::<ScalarFn>());
540        assert_arrays_eq!(
541            item,
542            PrimitiveArray::from_option_iter([Some(0i32), None, Some(2)]),
543            &mut SESSION.create_execution_ctx()
544        );
545        Ok(())
546    }
547
548    #[test]
549    fn test_pack_get_item_rule() {
550        // Create: pack(a: lit(1), b: lit(2)).get_item("b")
551        let pack_expr = pack([("a", lit(1)), ("b", lit(2))], NonNullable);
552        let get_item_expr = get_item("b", pack_expr);
553
554        let dtype = DType::Struct(StructFields::empty(), NonNullable);
555        let result = get_item_expr
556            .bind(&dtype)
557            .and_then(|expr| expr.optimize_recursive())
558            .unwrap();
559
560        assert_eq!(result, lit(2).bind(&dtype).unwrap());
561    }
562
563    #[test]
564    fn test_multi_level_pack_get_item_simplify() {
565        let inner_pack = pack([("a", lit(1)), ("b", lit(2))], NonNullable);
566        let get_a = get_item("a", inner_pack);
567
568        let outer_pack = pack([("x", get_a), ("y", lit(3)), ("z", lit(4))], NonNullable);
569        let get_z = get_item("z", outer_pack);
570
571        let dtype = DType::Primitive(PType::I32, NonNullable);
572
573        let result = get_z.bind(&dtype).unwrap().optimize_recursive().unwrap();
574        assert_eq!(result, lit(4).bind(&dtype).unwrap());
575    }
576
577    #[test]
578    fn test_deeply_nested_pack_get_item() {
579        let innermost = pack([("a", lit(42))], NonNullable);
580        let get_a = get_item("a", innermost);
581
582        let level2 = pack([("b", get_a)], NonNullable);
583        let get_b = get_item("b", level2);
584
585        let level3 = pack([("c", get_b)], NonNullable);
586        let get_c = get_item("c", level3);
587
588        let outermost = pack([("final", get_c)], NonNullable);
589        let get_final = get_item("final", outermost);
590
591        let dtype = DType::Primitive(PType::I32, NonNullable);
592
593        let result = get_final
594            .bind(&dtype)
595            .unwrap()
596            .optimize_recursive()
597            .unwrap();
598        assert_eq!(result, lit(42).bind(&dtype).unwrap());
599    }
600
601    #[test]
602    fn test_partial_pack_get_item_simplify() {
603        let inner_pack = pack([("x", lit(1)), ("y", lit(2))], NonNullable);
604        let get_x = get_item("x", inner_pack);
605        let add_expr = checked_add(get_x, lit(10));
606
607        let outer_pack = pack([("result", add_expr)], NonNullable);
608        let get_result = get_item("result", outer_pack);
609
610        let dtype = DType::Primitive(PType::I32, NonNullable);
611
612        let result = get_result
613            .bind(&dtype)
614            .unwrap()
615            .optimize_recursive()
616            .unwrap();
617        let expected = checked_add(lit(1), lit(10)).bind(&dtype).unwrap();
618        assert_eq!(&result, &expected);
619    }
620
621    #[test]
622    fn get_item_filter_list_field() {
623        let mut ctx = SESSION.create_execution_ctx();
624        use vortex_mask::Mask;
625
626        use crate::arrays::BoolArray;
627        use crate::arrays::FilterArray;
628        use crate::arrays::ListArray;
629
630        let list = ListArray::try_new(
631            buffer![0f32, 1., 2., 3., 4., 5., 6., 7., 8., 9., 10., 11.].into_array(),
632            buffer![2u64, 4, 6, 8, 10, 12].into_array(),
633            Validity::Array(BoolArray::from_iter([true, true, false, true, true]).into_array()),
634            &mut ctx,
635        )
636        .unwrap();
637
638        let filtered = FilterArray::try_new(
639            list.into_array(),
640            Mask::from_iter([true, true, false, false, false]),
641        )
642        .unwrap();
643
644        let st = StructArray::try_new(
645            FieldNames::from(["data"]),
646            vec![filtered.into_array()],
647            2,
648            Validity::AllValid,
649        )
650        .unwrap();
651
652        let result = st.into_array().apply(&get_item("data", root())).unwrap();
653        let expected = ListArray::try_new(
654            buffer![2f32, 3., 4., 5.].into_array(),
655            buffer![0u64, 2, 4].into_array(),
656            Validity::AllValid,
657            &mut ctx,
658        )
659        .unwrap();
660        assert_arrays_eq!(result, expected, &mut SESSION.create_execution_ctx());
661    }
662}