1use std::fmt::Display;
5use std::fmt::Formatter;
6
7use itertools::Itertools;
8use prost::Message;
9use vortex_error::VortexError;
10use vortex_error::VortexExpect;
11use vortex_error::VortexResult;
12use vortex_error::vortex_bail;
13use vortex_error::vortex_err;
14use vortex_session::VortexSession;
15use vortex_session::registry::CachedId;
16use vortex_utils::aliases::hash_set::HashSet;
17
18use crate::ArrayRef;
19use crate::ExecutionCtx;
20use crate::IntoArray;
21use crate::arrays::Constant;
22use crate::arrays::ConstantArray;
23use crate::arrays::StructArray;
24use crate::arrays::struct_::StructArrayExt;
25use crate::dtype::DType;
26use crate::dtype::FieldName;
27use crate::dtype::FieldNames;
28use crate::expr::BoundExpression;
29use crate::expr::display::ExprDisplay;
30use crate::expr::field::DisplayFieldNames;
31use crate::proto::expr::FieldNames as ProtoFieldNames;
32use crate::proto::expr::SelectOpts;
33use crate::proto::expr::select_opts::Opts;
34use crate::scalar_fn::Arity;
35use crate::scalar_fn::ChildName;
36use crate::scalar_fn::ExecutionArgs;
37use crate::scalar_fn::ScalarFnId;
38use crate::scalar_fn::ScalarFnVTable;
39use crate::scalar_fn::ScalarFnVTableExt;
40use crate::scalar_fn::fns::pack::Pack;
41use crate::scalar_fn::fns::pack::PackOptions;
42
43#[derive(Debug, Clone, PartialEq, Eq, Hash)]
44pub enum FieldSelection {
45 Include(FieldNames),
46 Exclude(FieldNames),
47}
48
49#[derive(Clone)]
50pub struct Select;
51
52impl ScalarFnVTable for Select {
53 type Options = FieldSelection;
54
55 fn id(&self) -> ScalarFnId {
56 static ID: CachedId = CachedId::new("vortex.select");
57 *ID
58 }
59
60 fn serialize(&self, instance: &FieldSelection) -> VortexResult<Option<Vec<u8>>> {
61 let opts = match instance {
62 FieldSelection::Include(fields) => Opts::Include(ProtoFieldNames {
63 names: fields.iter().map(|f| f.to_string()).collect(),
64 }),
65 FieldSelection::Exclude(fields) => Opts::Exclude(ProtoFieldNames {
66 names: fields.iter().map(|f| f.to_string()).collect(),
67 }),
68 };
69
70 let select_opts = SelectOpts { opts: Some(opts) };
71 Ok(Some(select_opts.encode_to_vec()))
72 }
73
74 fn deserialize(
75 &self,
76 _metadata: &[u8],
77 _session: &VortexSession,
78 ) -> VortexResult<FieldSelection> {
79 let prost_metadata = SelectOpts::decode(_metadata)?;
80
81 let select_opts = prost_metadata
82 .opts
83 .ok_or_else(|| vortex_err!("SelectOpts missing opts field"))?;
84
85 let field_selection = match select_opts {
86 Opts::Include(field_names) => FieldSelection::Include(FieldNames::from_iter(
87 field_names.names.iter().map(|s| s.as_str()),
88 )),
89 Opts::Exclude(field_names) => FieldSelection::Exclude(FieldNames::from_iter(
90 field_names.names.iter().map(|s| s.as_str()),
91 )),
92 };
93
94 Ok(field_selection)
95 }
96
97 fn arity(&self, _options: &FieldSelection) -> Arity {
98 Arity::Exact(1)
99 }
100
101 fn child_name(&self, _instance: &FieldSelection, child_idx: usize) -> ChildName {
102 match child_idx {
103 0 => ChildName::from("child"),
104 _ => unreachable!(),
105 }
106 }
107
108 fn fmt_sql(
109 &self,
110 selection: &FieldSelection,
111 expr: &dyn ExprDisplay,
112 f: &mut Formatter<'_>,
113 ) -> std::fmt::Result {
114 Display::fmt(expr.display_child(0), f)?;
115 match selection {
116 FieldSelection::Include(fields) => {
117 write!(f, "{{{}}}", DisplayFieldNames(fields))
118 }
119 FieldSelection::Exclude(fields) => {
120 write!(f, "{{~ {}}}", DisplayFieldNames(fields))
121 }
122 }
123 }
124
125 fn return_dtype(
126 &self,
127 selection: &FieldSelection,
128 arg_dtypes: &[DType],
129 ) -> VortexResult<DType> {
130 let child_dtype = &arg_dtypes[0];
131 let child_struct_dtype = child_dtype
132 .as_struct_fields_opt()
133 .ok_or_else(|| vortex_err!("Select child not a struct dtype"))?;
134
135 let projected = match selection {
136 FieldSelection::Include(fields) => child_struct_dtype.project(fields.as_ref())?,
137 FieldSelection::Exclude(fields) => child_struct_dtype
138 .names()
139 .iter()
140 .cloned()
141 .zip_eq(child_struct_dtype.fields())
142 .filter(|(name, _)| !fields.as_ref().contains(name))
143 .collect(),
144 };
145
146 Ok(DType::Struct(projected, child_dtype.nullability()))
147 }
148
149 fn execute(
150 &self,
151 selection: &FieldSelection,
152 args: &dyn ExecutionArgs,
153 ctx: &mut ExecutionCtx,
154 ) -> VortexResult<ArrayRef> {
155 let child = args.get(0)?;
156 if let Some(constant) = child.as_opt::<Constant>() {
157 let child_struct_dtype = child
158 .dtype()
159 .as_struct_fields_opt()
160 .ok_or_else(|| vortex_err!("Select child not a struct dtype"))?;
161 let included = selection.normalize_to_included_fields(child_struct_dtype.names())?;
162 let scalar = constant.scalar().as_struct().project(included.as_ref())?;
163
164 return Ok(ConstantArray::new(scalar, child.len()).into_array());
165 }
166
167 let child = child.execute::<StructArray>(ctx)?;
168
169 let result = match selection {
170 FieldSelection::Include(f) => child.project(f.as_ref()),
171 FieldSelection::Exclude(names) => {
172 let included_names = child
173 .names()
174 .iter()
175 .filter(|&f| !names.as_ref().contains(f))
176 .cloned()
177 .collect::<Vec<_>>();
178 child.project(included_names.as_slice())
179 }
180 }?;
181
182 result.into_array().execute(ctx)
183 }
184
185 fn simplify(
186 &self,
187 selection: &FieldSelection,
188 expr: &BoundExpression,
189 ) -> VortexResult<Option<BoundExpression>> {
190 let child_struct = expr.child(0);
191 let struct_dtype = child_struct.dtype();
192 let struct_nullability = struct_dtype.nullability();
193
194 let struct_fields = struct_dtype.as_struct_fields_opt().ok_or_else(|| {
195 vortex_err!(
196 "Select child must return a struct dtype, however it was a {}",
197 struct_dtype
198 )
199 })?;
200
201 let included_fields = selection.normalize_to_included_fields(struct_fields.names())?;
203 let all_included_fields_are_nullable = included_fields.iter().all(|name| {
204 struct_fields
205 .field(name)
206 .vortex_expect(
207 "`normalize_to_included_fields` checks that the included fields already exist \
208 in `struct_fields`",
209 )
210 .is_nullable()
211 });
212
213 if included_fields.is_empty() {
218 return Ok(Some(Pack.try_new_bound_expr(
219 PackOptions {
220 names: FieldNames::empty(),
221 nullability: struct_nullability,
222 },
223 [],
224 )?));
225 }
226
227 let would_intersect_validity =
237 struct_nullability.is_nullable() && !all_included_fields_are_nullable;
238
239 if let Some(child_pack) = child_struct.as_opt::<Pack>()
240 && !would_intersect_validity
241 {
242 let fields: Vec<BoundExpression> = included_fields
247 .iter()
248 .map(|name| {
249 let idx = child_pack.names.find(name).ok_or_else(|| {
250 vortex_err!(
251 "Cannot find field {} in pack fields {:?}",
252 name,
253 child_pack.names
254 )
255 })?;
256
257 Ok(child_struct.child(idx).clone())
258 })
259 .try_collect::<_, _, VortexError>()?;
260
261 return Ok(Some(Pack.try_new_bound_expr(
262 PackOptions {
263 names: included_fields,
264 nullability: struct_nullability,
265 },
266 fields,
267 )?));
268 }
269
270 Ok(None)
271 }
272
273 fn is_strict(&self, _options: &FieldSelection) -> bool {
274 true
275 }
276
277 fn is_infallible(&self, _instance: &FieldSelection) -> bool {
278 true
280 }
281}
282
283impl FieldSelection {
284 pub fn include(columns: FieldNames) -> Self {
285 assert_eq!(columns.iter().unique().collect_vec().len(), columns.len());
286 Self::Include(columns)
287 }
288
289 pub fn exclude(columns: FieldNames) -> Self {
290 assert_eq!(columns.iter().unique().collect_vec().len(), columns.len());
291 Self::Exclude(columns)
292 }
293
294 pub fn is_include(&self) -> bool {
295 matches!(self, Self::Include(_))
296 }
297
298 pub fn is_exclude(&self) -> bool {
299 matches!(self, Self::Exclude(_))
300 }
301
302 pub fn field_names(&self) -> &FieldNames {
303 let (FieldSelection::Include(fields) | FieldSelection::Exclude(fields)) = self;
304
305 fields
306 }
307
308 pub fn normalize_to_included_fields(
309 &self,
310 available_fields: &FieldNames,
311 ) -> VortexResult<FieldNames> {
312 let available: HashSet<&FieldName> = available_fields.iter().collect();
314
315 if self.field_names().iter().any(|f| !available.contains(f)) {
317 vortex_bail!(
318 "Select fields {:?} must be a subset of child fields {:?}",
319 self,
320 available_fields
321 );
322 }
323
324 match self {
325 FieldSelection::Include(fields) => Ok(fields.clone()),
326 FieldSelection::Exclude(exc_fields) => {
327 let excluded: HashSet<&FieldName> = exc_fields.iter().collect();
328 Ok(available_fields
329 .iter()
330 .filter(|f| !excluded.contains(f))
331 .cloned()
332 .collect())
333 }
334 }
335 }
336}
337
338impl Display for FieldSelection {
339 fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result {
340 match self {
341 FieldSelection::Include(fields) => write!(f, "{{{}}}", DisplayFieldNames(fields)),
342 FieldSelection::Exclude(fields) => write!(f, "~{{{}}}", DisplayFieldNames(fields)),
343 }
344 }
345}
346
347#[cfg(test)]
348mod tests {
349 use std::sync::LazyLock;
350
351 use vortex_buffer::buffer;
352 use vortex_error::VortexResult;
353 use vortex_session::VortexSession;
354
355 use crate::ArrayRef;
356 use crate::IntoArray;
357 use crate::VortexSessionExecute;
358 use crate::arrays::Constant;
359 use crate::arrays::ConstantArray;
360 use crate::arrays::ScalarFnArray;
361 use crate::assert_arrays_eq;
362 use crate::dtype::DType;
363 use crate::dtype::FieldName;
364 use crate::dtype::FieldNames;
365 use crate::dtype::Nullability;
366 use crate::dtype::Nullability::Nullable;
367 use crate::dtype::PType::I32;
368 use crate::dtype::StructFields;
369 use crate::expr::root;
370 use crate::expr::select;
371 use crate::expr::select_exclude;
372 use crate::expr::test_harness;
373 use crate::scalar::Scalar;
374 use crate::scalar_fn::ScalarFnVTableExt;
375 use crate::scalar_fn::fns::select::FieldSelection;
376 use crate::scalar_fn::fns::select::Select;
377 use crate::scalar_fn::fns::select::StructArray;
378
379 static SESSION: LazyLock<VortexSession> = LazyLock::new(VortexSession::empty);
380
381 fn test_array() -> StructArray {
382 StructArray::from_fields(&[
383 ("a", buffer![0, 1, 2].into_array()),
384 ("b", buffer![4, 5, 6].into_array()),
385 ])
386 .unwrap()
387 }
388
389 #[test]
390 pub fn include_columns() {
391 let mut ctx = SESSION.create_execution_ctx();
392 let st = test_array();
393 let select = select(vec![FieldName::from("a")], root());
394 let selected = st.into_array().apply(&select).unwrap();
395 let selected_names = selected.dtype().as_struct_fields().names();
396 assert_eq!(selected_names.as_ref(), &["a"]);
397 assert_arrays_eq!(
398 selected,
399 StructArray::from_fields(&[("a", buffer![0i32, 1, 2].into_array())]).unwrap(),
400 &mut ctx
401 );
402 }
403
404 #[test]
405 pub fn exclude_columns() {
406 let mut ctx = SESSION.create_execution_ctx();
407 let st = test_array();
408 let select = select_exclude(vec![FieldName::from("a")], root());
409 let selected = st.into_array().apply(&select).unwrap();
410 let selected_names = selected.dtype().as_struct_fields().names();
411 assert_eq!(selected_names.as_ref(), &["b"]);
412 assert_arrays_eq!(
413 selected,
414 StructArray::from_fields(&[("b", buffer![4i32, 5, 6].into_array())]).unwrap(),
415 &mut ctx
416 );
417 }
418
419 #[test]
420 fn dtype() {
421 let dtype = test_harness::struct_dtype();
422
423 let select_expr = select(vec![FieldName::from("a")], root());
424 let expected_dtype = DType::Struct(
425 dtype
426 .as_struct_fields_opt()
427 .unwrap()
428 .project(&["a".into()])
429 .unwrap(),
430 Nullability::NonNullable,
431 );
432 assert_eq!(select_expr.return_dtype(&dtype).unwrap(), expected_dtype);
433
434 let select_expr_exclude = select_exclude(
435 vec![
436 FieldName::from("col1"),
437 FieldName::from("col2"),
438 FieldName::from("bool1"),
439 FieldName::from("bool2"),
440 ],
441 root(),
442 );
443 assert_eq!(
444 select_expr_exclude.return_dtype(&dtype).unwrap(),
445 expected_dtype
446 );
447
448 let select_expr_exclude = select_exclude(
449 vec![FieldName::from("col1"), FieldName::from("col2")],
450 root(),
451 );
452 assert_eq!(
453 select_expr_exclude.return_dtype(&dtype).unwrap(),
454 DType::Struct(
455 dtype
456 .as_struct_fields_opt()
457 .unwrap()
458 .project(&["a".into(), "bool1".into(), "bool2".into()])
459 .unwrap(),
460 Nullability::NonNullable
461 )
462 );
463 }
464
465 #[test]
466 fn test_as_include_names() {
467 let field_names = FieldNames::from(["a", "b", "c"]);
468 for expr in [select(["a"], root()), select_exclude(["b", "c"], root())] {
469 assert_eq!(
470 expr.as_::<Select>()
471 .normalize_to_included_fields(&field_names)
472 .unwrap(),
473 FieldNames::from(["a"])
474 );
475 }
476 }
477
478 #[test]
479 fn execute_constant_struct_stays_constant() -> VortexResult<()> {
480 let field_count = 128usize;
481 let names = (0..field_count)
482 .map(|idx| FieldName::from(format!("f{idx}")))
483 .collect::<Vec<_>>();
484 let dtypes = vec![I32.into(); field_count];
485 let fields = StructFields::new(FieldNames::from(names), dtypes);
486 let scalar = Scalar::struct_(
487 DType::Struct(fields, Nullability::NonNullable),
488 (0..128).map(Scalar::from),
489 );
490 let array = ConstantArray::new(scalar, 1_000_000).into_array();
491 let mut ctx = SESSION.create_execution_ctx();
492
493 let item: ArrayRef = ScalarFnArray::try_new(
494 Select.bind(FieldSelection::Include(FieldNames::from(["f97", "f3"]))),
495 vec![array],
496 )?
497 .into_array()
498 .execute(&mut ctx)?;
499
500 let constant = item.as_::<Constant>();
501 assert_eq!(constant.len(), 1_000_000);
502 assert_eq!(
503 constant.scalar(),
504 &Scalar::struct_(
505 DType::Struct(
506 StructFields::new(
507 FieldNames::from(["f97", "f3"]),
508 vec![I32.into(), I32.into()],
509 ),
510 Nullability::NonNullable,
511 ),
512 [Scalar::from(97), Scalar::from(3)],
513 )
514 );
515 Ok(())
516 }
517
518 #[test]
519 fn test_remove_select_rule() {
520 let dtype = DType::Struct(
521 StructFields::new(["a", "b"].into(), vec![I32.into(), I32.into()]),
522 Nullable,
523 );
524 let e = select(["a", "b"], root());
525
526 let result = e.bind(&dtype).unwrap().optimize_recursive().unwrap();
527
528 assert_eq!(result.dtype(), &dtype);
529 }
530
531 #[test]
532 fn test_remove_select_rule_exclude_fields() {
533 use crate::expr::select_exclude;
534
535 let dtype = DType::Struct(
536 StructFields::new(
537 ["a", "b", "c"].into(),
538 vec![I32.into(), I32.into(), I32.into()],
539 ),
540 Nullable,
541 );
542 let e = select_exclude(["c"], root());
543
544 let result = e.bind(&dtype).unwrap().optimize_recursive().unwrap();
545
546 let result_dtype = result.dtype();
548 assert!(result_dtype.is_nullable());
549 let fields = result_dtype.as_struct_fields_opt().unwrap();
550 assert_eq!(fields.names().as_ref(), &["a", "b"]);
551 assert_eq!(
552 result_dtype,
553 &DType::struct_([("a", I32), ("b", I32)], Nullable)
554 );
555 }
556}