1use std::fmt::Display;
5use std::fmt::Formatter;
6use std::slice;
7
8use prost::Message;
9use vortex_error::VortexResult;
10use vortex_error::vortex_err;
11use vortex_session::VortexSession;
12use vortex_session::registry::CachedId;
13
14use crate::ArrayRef;
15use crate::ExecutionCtx;
16use crate::IntoArray;
17use crate::arrays::ConstantArray;
18use crate::arrays::ScalarFnArray;
19use crate::arrays::StructArray;
20use crate::arrays::struct_::StructArrayExt;
21use crate::builtins::ArrayBuiltins;
22use crate::dtype::DType;
23use crate::dtype::FieldName;
24use crate::dtype::Nullability;
25use crate::expr::display::ExprDisplay;
26use crate::proto::expr as pb;
27use crate::scalar::Scalar;
28use crate::scalar_fn::Arity;
29use crate::scalar_fn::ChildName;
30use crate::scalar_fn::EmptyOptions;
31use crate::scalar_fn::ExecutionArgs;
32use crate::scalar_fn::ReduceNode;
33use crate::scalar_fn::ScalarFnId;
34use crate::scalar_fn::ScalarFnVTable;
35use crate::scalar_fn::ScalarFnVTableExt;
36use crate::scalar_fn::fns::mask::Mask;
37use crate::scalar_fn::fns::pack::Pack;
38use crate::scalar_fn::fns::select::FieldSelection;
39use crate::scalar_fn::fns::select::Select;
40
41#[derive(Clone)]
42pub struct GetItem;
43
44impl GetItem {
45 pub fn try_new(
51 input: ArrayRef,
52 field_name: impl Into<FieldName>,
53 ) -> VortexResult<ScalarFnArray> {
54 ScalarFnArray::try_new(GetItem.bind(field_name.into()), [input])
55 }
56
57 pub(crate) fn apply_now<T: ReduceNode>(
64 struct_node: &T,
65 field_name: &FieldName,
66 ) -> VortexResult<T> {
67 if let Some(field) = Self::maybe_apply_now(struct_node, field_name)? {
68 return Ok(field);
69 }
70
71 struct_node.new_node(
72 GetItem.bind(field_name.clone()),
73 slice::from_ref(struct_node),
74 )
75 }
76
77 fn maybe_apply_now<T: ReduceNode>(
79 struct_node: &T,
80 field_name: &FieldName,
81 ) -> VortexResult<Option<T>> {
82 if let Some(scalar) = struct_node.as_constant() {
83 return Ok(Some(
84 struct_node.new_constant(Self::constant_field(scalar, field_name)?),
85 ));
86 }
87
88 if let Some(selection) = struct_node
89 .scalar_fn()
90 .and_then(|scalar_fn| scalar_fn.as_opt::<Select>())
91 {
92 let included = match selection {
93 FieldSelection::Include(fields) => fields.as_ref().contains(field_name),
94 FieldSelection::Exclude(fields) => !fields.as_ref().contains(field_name),
95 };
96 if included {
97 return Self::apply_now(&struct_node.child(0), field_name).map(Some);
98 }
99 return Ok(None);
100 }
101
102 let Some(pack) = struct_node
103 .scalar_fn()
104 .and_then(|scalar_fn| scalar_fn.as_opt::<Pack>())
105 else {
106 return Ok(None);
107 };
108 let Some(idx) = pack.names.find(field_name) else {
109 return Ok(None);
110 };
111
112 let field = struct_node.child(idx);
113 if !pack.nullability.is_nullable() {
114 return Ok(Some(field));
115 }
116
117 Ok(Some(struct_node.new_node(
120 Mask.bind(EmptyOptions),
121 &[field, struct_node.new_constant(true.into())],
122 )?))
123 }
124
125 fn constant_field(scalar: &Scalar, field_name: &FieldName) -> VortexResult<Scalar> {
126 let dtype = Self.return_dtype(field_name, &[scalar.dtype().clone()])?;
127 if scalar.is_null() {
128 return Ok(Scalar::null(dtype));
129 }
130 scalar
131 .as_struct()
132 .field(field_name)
133 .ok_or_else(|| {
134 vortex_err!(
135 "Field '{}' missing from constant struct array {}",
136 field_name,
137 scalar.dtype()
138 )
139 })?
140 .cast(&dtype)
141 }
142}
143
144impl ScalarFnVTable for GetItem {
145 type Options = FieldName;
146
147 fn id(&self) -> ScalarFnId {
148 static ID: CachedId = CachedId::new("vortex.get_item");
149 *ID
150 }
151
152 fn serialize(&self, instance: &Self::Options) -> VortexResult<Option<Vec<u8>>> {
153 Ok(Some(
154 pb::GetItemOpts {
155 path: instance.to_string(),
156 }
157 .encode_to_vec(),
158 ))
159 }
160
161 fn deserialize(
162 &self,
163 _metadata: &[u8],
164 _session: &VortexSession,
165 ) -> VortexResult<Self::Options> {
166 let opts = pb::GetItemOpts::decode(_metadata)?;
167 Ok(FieldName::from(opts.path))
168 }
169
170 fn arity(&self, _field_name: &FieldName) -> Arity {
171 Arity::Exact(1)
172 }
173
174 fn child_name(&self, _instance: &Self::Options, child_idx: usize) -> ChildName {
175 match child_idx {
176 0 => ChildName::from("input"),
177 _ => unreachable!("Invalid child index {} for GetItem expression", child_idx),
178 }
179 }
180
181 fn fmt_sql(
182 &self,
183 field_name: &FieldName,
184 expr: &dyn ExprDisplay,
185 f: &mut Formatter<'_>,
186 ) -> std::fmt::Result {
187 Display::fmt(expr.display_child(0), f)?;
188 write!(f, ".{}", field_name)
189 }
190
191 fn return_dtype(&self, field_name: &FieldName, arg_dtypes: &[DType]) -> VortexResult<DType> {
192 let struct_dtype = &arg_dtypes[0];
193 let field_dtype = struct_dtype
194 .as_struct_fields_opt()
195 .and_then(|st| st.field(field_name))
196 .ok_or_else(|| {
197 vortex_err!("Couldn't find the {} field in the input scope", field_name)
198 })?;
199
200 if matches!(
202 (struct_dtype.nullability(), field_dtype.nullability()),
203 (Nullability::Nullable, Nullability::NonNullable)
204 ) {
205 return Ok(field_dtype.with_nullability(Nullability::Nullable));
206 }
207
208 Ok(field_dtype)
209 }
210
211 fn execute(
212 &self,
213 field_name: &FieldName,
214 args: &dyn ExecutionArgs,
215 ctx: &mut ExecutionCtx,
216 ) -> VortexResult<ArrayRef> {
217 let input = args.get(0)?;
218 if let Some(scalar) = input.as_constant() {
219 return Ok(
220 ConstantArray::new(Self::constant_field(scalar, field_name)?, input.len())
221 .into_array(),
222 );
223 }
224
225 let input = input.execute::<StructArray>(ctx)?;
226 let field = input.unmasked_field_by_name(field_name).cloned()?;
227
228 match input.dtype().nullability() {
229 Nullability::NonNullable => Ok(field),
230 Nullability::Nullable => field.mask(input.validity()?.to_array(input.len())),
231 }
232 }
233
234 fn reduce<T: ReduceNode>(&self, field_name: &FieldName, node: &T) -> VortexResult<Option<T>> {
235 Self::maybe_apply_now(&node.child(0), field_name)
236 }
237
238 fn is_strict(&self, _field_name: &FieldName) -> bool {
239 true
240 }
241
242 fn is_infallible(&self, _field_name: &FieldName) -> bool {
243 true
245 }
246}
247
248#[cfg(test)]
249mod tests {
250 use std::sync::LazyLock;
251
252 use rstest::rstest;
253 use vortex_buffer::buffer;
254 use vortex_error::VortexResult;
255 use vortex_session::VortexSession;
256
257 use super::GetItem;
258 use crate::ArrayRef;
259 use crate::IntoArray;
260 use crate::VortexSessionExecute;
261 use crate::arrays::Constant;
262 use crate::arrays::ConstantArray;
263 use crate::arrays::Filter;
264 use crate::arrays::Primitive;
265 use crate::arrays::PrimitiveArray;
266 use crate::arrays::ScalarFn;
267 use crate::arrays::ScalarFnArray;
268 use crate::arrays::Slice;
269 use crate::assert_arrays_eq;
270 use crate::builtins::ArrayBuiltins;
271 use crate::dtype::DType;
272 use crate::dtype::FieldName;
273 use crate::dtype::FieldNames;
274 use crate::dtype::Nullability;
275 use crate::dtype::Nullability::NonNullable;
276 use crate::dtype::PType;
277 use crate::dtype::StructFields;
278 use crate::expr::Expression;
279 use crate::expr::checked_add;
280 use crate::expr::get_item;
281 use crate::expr::lit;
282 use crate::expr::pack;
283 use crate::expr::root;
284 use crate::expr::select;
285 use crate::expr::select_exclude;
286 use crate::optimizer::ArrayOptimizer;
287 use crate::scalar::Scalar;
288 use crate::scalar_fn::ScalarFnVTableExt;
289 use crate::scalar_fn::fns::get_item::StructArray;
290 use crate::scalar_fn::fns::select::FieldSelection;
291 use crate::scalar_fn::fns::select::Select;
292 use crate::validity::Validity;
293
294 static SESSION: LazyLock<VortexSession> = LazyLock::new(VortexSession::empty);
295
296 fn test_array() -> StructArray {
297 StructArray::from_fields(&[
298 ("a", buffer![0i32, 1, 2].into_array()),
299 ("b", buffer![4i64, 5, 6].into_array()),
300 ])
301 .unwrap()
302 }
303
304 #[test]
305 fn get_item_by_name() {
306 let st = test_array();
307 let get_item = get_item("a", root());
308 assert!(
309 get_item
310 .as_scalar()
311 .is_some_and(|f| f.signature().is_strict())
312 );
313 let item = st.into_array().apply(&get_item).unwrap();
314 assert_eq!(item.dtype(), &DType::from(PType::I32));
315 assert_arrays_eq!(
316 item,
317 buffer![0i32, 1, 2].into_array(),
318 &mut SESSION.create_execution_ctx()
319 );
320 }
321
322 #[test]
323 fn get_item_by_name_none() {
324 let st = test_array();
325 let get_item = get_item("c", root());
326 assert!(st.into_array().apply(&get_item).is_err());
327 }
328
329 #[test]
330 fn get_nullable_field() {
331 let st = StructArray::try_new(
332 FieldNames::from(["a"]),
333 vec![buffer![1i32].into_array()],
334 1,
335 Validity::AllInvalid,
336 )
337 .unwrap()
338 .into_array();
339
340 let get_item_expr = get_item("a", root());
341 let item = st.apply(&get_item_expr).unwrap();
342 assert_eq!(
344 item.dtype(),
345 &DType::Primitive(PType::I32, Nullability::Nullable)
346 );
347 assert_arrays_eq!(
348 item,
349 PrimitiveArray::from_option_iter([None::<i32>]),
350 &mut SESSION.create_execution_ctx()
351 );
352 }
353
354 #[test]
355 fn get_non_nullable_field_from_all_valid_nullable_struct() -> VortexResult<()> {
356 let st = StructArray::try_new(
357 FieldNames::from(["a"]),
358 vec![buffer![1i32].into_array()],
359 1,
360 Validity::AllValid,
361 )?
362 .into_array();
363
364 let item = st.apply(&get_item("a", root()))?;
365 assert_eq!(
366 item.dtype(),
367 &DType::Primitive(PType::I32, Nullability::Nullable)
368 );
369 assert_arrays_eq!(
370 item,
371 PrimitiveArray::from_option_iter([Some(1i32)]),
372 &mut SESSION.create_execution_ctx()
373 );
374 Ok(())
375 }
376
377 #[test]
378 fn execute_constant_struct_stays_constant() -> VortexResult<()> {
379 let field_count = 128usize;
380 let selected_idx = 97i32;
381 let names = (0..field_count)
382 .map(|idx| FieldName::from(format!("f{idx}")))
383 .collect::<Vec<_>>();
384 let dtypes = vec![DType::Primitive(PType::I32, NonNullable); field_count];
385 let fields = StructFields::new(FieldNames::from(names), dtypes);
386 let scalar = Scalar::struct_(
387 DType::Struct(fields, NonNullable),
388 (0..128).map(|idx| Scalar::primitive(idx, NonNullable)),
389 );
390 let array = ConstantArray::new(scalar, 1_000_000).into_array();
391 let mut ctx = SESSION.create_execution_ctx();
392
393 let item: ArrayRef = GetItem::try_new(array, "f97")?
394 .into_array()
395 .execute(&mut ctx)?;
396
397 let constant = item.as_::<Constant>();
398 assert_eq!(constant.len(), 1_000_000);
399 assert_eq!(
400 constant.scalar(),
401 &Scalar::primitive(selected_idx, NonNullable)
402 );
403 Ok(())
404 }
405
406 #[rstest]
407 #[case::include(FieldSelection::Include(FieldNames::from(["a"])))]
408 #[case::exclude(FieldSelection::Exclude(FieldNames::from(["b"])))]
409 fn get_item_reduces_through_select(
410 #[case] selection: FieldSelection,
411 #[values(
412 Validity::NonNullable,
413 Validity::AllValid,
414 Validity::AllInvalid,
415 Validity::from_iter([true, false, true])
416 )]
417 validity: Validity,
418 ) -> VortexResult<()> {
419 let array = StructArray::try_new(
420 FieldNames::from(["a", "b"]),
421 vec![
422 ConstantArray::new(42i32, 3).into_array(),
423 buffer![1i32, 2, 3].into_array(),
424 ],
425 3,
426 validity,
427 )?
428 .into_array();
429 let expected = array.get_item("a")?;
430 let selected = ScalarFnArray::try_new(Select.bind(selection), [array])?.into_array();
431 let item = GetItem::try_new(selected, "a")?.into_array().optimize()?;
432
433 assert_eq!(item.encoding_id(), expected.encoding_id());
434 assert_eq!(item.dtype(), expected.dtype());
435 assert_arrays_eq!(item, expected, &mut SESSION.create_execution_ctx());
436 Ok(())
437 }
438
439 #[rstest]
440 #[case::include(select(["a"], root()))]
441 #[case::exclude(select_exclude(["b"], root()))]
442 fn get_item_select_expression_reduces(#[case] selected: Expression) -> VortexResult<()> {
443 let dtype = test_array().dtype().clone();
444 let result = get_item("a", selected).bind(&dtype)?.optimize_recursive()?;
445 assert_eq!(result, get_item("a", root()).bind(&dtype)?);
446 Ok(())
447 }
448
449 #[rstest]
450 #[case::include(select(["b"], root()))]
451 #[case::exclude(select_exclude(["a"], root()))]
452 fn get_item_select_rejects_removed_field(#[case] selected: Expression) {
453 assert!(
454 test_array()
455 .into_array()
456 .apply(&get_item("a", selected))
457 .is_err()
458 );
459 }
460
461 #[rstest]
462 fn get_item_constant_reduces(
463 #[values(false, true)] null: bool,
464 #[values(0, 3)] len: usize,
465 ) -> VortexResult<()> {
466 let dtype = DType::Struct(
467 StructFields::new(FieldNames::from(["a"]), vec![DType::from(PType::I32)]),
468 Nullability::Nullable,
469 );
470 let scalar = if null {
471 Scalar::null(dtype)
472 } else {
473 Scalar::struct_(dtype, [Scalar::from(42i32)])
474 };
475 let item = ConstantArray::new(scalar, len)
476 .into_array()
477 .apply(&get_item("a", root()))?;
478 let expected = if null {
479 Scalar::null(DType::Primitive(PType::I32, Nullability::Nullable))
480 } else {
481 Scalar::primitive(42i32, Nullability::Nullable)
482 };
483
484 assert_eq!(item.as_::<Constant>().scalar(), &expected);
485 assert_eq!(item.len(), len);
486 Ok(())
487 }
488
489 #[test]
490 fn get_item_pushes_through_filter() -> VortexResult<()> {
491 use vortex_mask::Mask;
492
493 let filtered = test_array()
494 .into_array()
495 .filter(Mask::from_iter([true, false, true]))?;
496
497 let item = filtered.get_item("a")?;
498
499 assert!(item.is::<Filter>());
500 assert!(!item.is::<ScalarFn>());
501 assert_arrays_eq!(
502 item,
503 buffer![0i32, 2].into_array(),
504 &mut SESSION.create_execution_ctx()
505 );
506 Ok(())
507 }
508
509 #[test]
510 fn get_item_pushes_through_slice() -> VortexResult<()> {
511 let sliced = test_array().into_array().slice(1..3)?;
512
513 let item = sliced.get_item("a")?;
514
515 assert!(item.is::<Primitive>());
516 assert!(!item.is::<Slice>());
517 assert!(!item.is::<ScalarFn>());
518 assert_arrays_eq!(
519 item,
520 buffer![1i32, 2].into_array(),
521 &mut SESSION.create_execution_ctx()
522 );
523 Ok(())
524 }
525
526 #[test]
527 fn get_item_pushes_through_masked() -> VortexResult<()> {
528 let masked = test_array()
529 .into_array()
530 .mask(Validity::from_iter([true, false, true]).to_array(3))?;
531
532 let item = masked.get_item("a")?;
533
534 assert!(item.is::<Primitive>());
535 assert_eq!(
536 item.dtype(),
537 &DType::Primitive(PType::I32, Nullability::Nullable)
538 );
539 assert!(!item.is::<ScalarFn>());
540 assert_arrays_eq!(
541 item,
542 PrimitiveArray::from_option_iter([Some(0i32), None, Some(2)]),
543 &mut SESSION.create_execution_ctx()
544 );
545 Ok(())
546 }
547
548 #[test]
549 fn test_pack_get_item_rule() {
550 let pack_expr = pack([("a", lit(1)), ("b", lit(2))], NonNullable);
552 let get_item_expr = get_item("b", pack_expr);
553
554 let dtype = DType::Struct(StructFields::empty(), NonNullable);
555 let result = get_item_expr
556 .bind(&dtype)
557 .and_then(|expr| expr.optimize_recursive())
558 .unwrap();
559
560 assert_eq!(result, lit(2).bind(&dtype).unwrap());
561 }
562
563 #[test]
564 fn test_multi_level_pack_get_item_simplify() {
565 let inner_pack = pack([("a", lit(1)), ("b", lit(2))], NonNullable);
566 let get_a = get_item("a", inner_pack);
567
568 let outer_pack = pack([("x", get_a), ("y", lit(3)), ("z", lit(4))], NonNullable);
569 let get_z = get_item("z", outer_pack);
570
571 let dtype = DType::Primitive(PType::I32, NonNullable);
572
573 let result = get_z.bind(&dtype).unwrap().optimize_recursive().unwrap();
574 assert_eq!(result, lit(4).bind(&dtype).unwrap());
575 }
576
577 #[test]
578 fn test_deeply_nested_pack_get_item() {
579 let innermost = pack([("a", lit(42))], NonNullable);
580 let get_a = get_item("a", innermost);
581
582 let level2 = pack([("b", get_a)], NonNullable);
583 let get_b = get_item("b", level2);
584
585 let level3 = pack([("c", get_b)], NonNullable);
586 let get_c = get_item("c", level3);
587
588 let outermost = pack([("final", get_c)], NonNullable);
589 let get_final = get_item("final", outermost);
590
591 let dtype = DType::Primitive(PType::I32, NonNullable);
592
593 let result = get_final
594 .bind(&dtype)
595 .unwrap()
596 .optimize_recursive()
597 .unwrap();
598 assert_eq!(result, lit(42).bind(&dtype).unwrap());
599 }
600
601 #[test]
602 fn test_partial_pack_get_item_simplify() {
603 let inner_pack = pack([("x", lit(1)), ("y", lit(2))], NonNullable);
604 let get_x = get_item("x", inner_pack);
605 let add_expr = checked_add(get_x, lit(10));
606
607 let outer_pack = pack([("result", add_expr)], NonNullable);
608 let get_result = get_item("result", outer_pack);
609
610 let dtype = DType::Primitive(PType::I32, NonNullable);
611
612 let result = get_result
613 .bind(&dtype)
614 .unwrap()
615 .optimize_recursive()
616 .unwrap();
617 let expected = checked_add(lit(1), lit(10)).bind(&dtype).unwrap();
618 assert_eq!(&result, &expected);
619 }
620
621 #[test]
622 fn get_item_filter_list_field() {
623 let mut ctx = SESSION.create_execution_ctx();
624 use vortex_mask::Mask;
625
626 use crate::arrays::BoolArray;
627 use crate::arrays::FilterArray;
628 use crate::arrays::ListArray;
629
630 let list = ListArray::try_new(
631 buffer![0f32, 1., 2., 3., 4., 5., 6., 7., 8., 9., 10., 11.].into_array(),
632 buffer![2u64, 4, 6, 8, 10, 12].into_array(),
633 Validity::Array(BoolArray::from_iter([true, true, false, true, true]).into_array()),
634 &mut ctx,
635 )
636 .unwrap();
637
638 let filtered = FilterArray::try_new(
639 list.into_array(),
640 Mask::from_iter([true, true, false, false, false]),
641 )
642 .unwrap();
643
644 let st = StructArray::try_new(
645 FieldNames::from(["data"]),
646 vec![filtered.into_array()],
647 2,
648 Validity::AllValid,
649 )
650 .unwrap();
651
652 let result = st.into_array().apply(&get_item("data", root())).unwrap();
653 let expected = ListArray::try_new(
654 buffer![2f32, 3., 4., 5.].into_array(),
655 buffer![0u64, 2, 4].into_array(),
656 Validity::AllValid,
657 &mut ctx,
658 )
659 .unwrap();
660 assert_arrays_eq!(result, expected, &mut SESSION.create_execution_ctx());
661 }
662}