1use std::fmt::Display;
5use std::fmt::Formatter;
6use std::hash::Hash;
7use std::sync::Arc;
8
9use itertools::Itertools as _;
10use prost::Message;
11use vortex_error::VortexResult;
12use vortex_session::VortexSession;
13use vortex_session::registry::CachedId;
14
15use crate::ArrayRef;
16use crate::ExecutionCtx;
17use crate::IntoArray;
18use crate::arrays::StructArray;
19use crate::dtype::DType;
20use crate::dtype::FieldName;
21use crate::dtype::FieldNames;
22use crate::dtype::Nullability;
23use crate::dtype::StructFields;
24use crate::expr::display::ExprDisplay;
25use crate::proto::expr as pb;
26use crate::scalar_fn::Arity;
27use crate::scalar_fn::ChildName;
28use crate::scalar_fn::ExecutionArgs;
29use crate::scalar_fn::ReduceNode;
30use crate::scalar_fn::ReduceNodeValidity;
31use crate::scalar_fn::ScalarFnId;
32use crate::scalar_fn::ScalarFnVTable;
33use crate::validity::Validity;
34
35#[derive(Clone)]
37pub struct Pack;
38
39#[derive(Debug, Clone, PartialEq, Eq, Hash)]
40pub struct PackOptions {
41 pub names: FieldNames,
42 pub nullability: Nullability,
43}
44
45impl Display for PackOptions {
46 fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result {
47 write!(
48 f,
49 "names: [{}], nullability: {:#}",
50 self.names.iter().join(", "),
51 self.nullability
52 )
53 }
54}
55
56impl ScalarFnVTable for Pack {
57 type Options = PackOptions;
58
59 fn id(&self) -> ScalarFnId {
60 static ID: CachedId = CachedId::new("vortex.pack");
61 *ID
62 }
63
64 fn serialize(&self, instance: &Self::Options) -> VortexResult<Option<Vec<u8>>> {
65 Ok(Some(
66 pb::PackOpts {
67 paths: instance.names.iter().map(|n| n.to_string()).collect(),
68 nullable: instance.nullability.into(),
69 }
70 .encode_to_vec(),
71 ))
72 }
73
74 fn deserialize(
75 &self,
76 _metadata: &[u8],
77 _session: &VortexSession,
78 ) -> VortexResult<Self::Options> {
79 let opts = pb::PackOpts::decode(_metadata)?;
80 let names: FieldNames = opts
81 .paths
82 .iter()
83 .map(|name| FieldName::from(name.as_str()))
84 .collect();
85 Ok(PackOptions {
86 names,
87 nullability: opts.nullable.into(),
88 })
89 }
90
91 fn arity(&self, options: &Self::Options) -> Arity {
92 Arity::Exact(options.names.len())
93 }
94
95 fn child_name(&self, instance: &Self::Options, child_idx: usize) -> ChildName {
96 match instance.names.get(child_idx) {
97 Some(name) => ChildName::from(Arc::clone(name.inner())),
98 None => unreachable!(
99 "Invalid child index {} for Pack expression with {} fields",
100 child_idx,
101 instance.names.len()
102 ),
103 }
104 }
105
106 fn fmt_sql(
107 &self,
108 options: &Self::Options,
109 expr: &dyn ExprDisplay,
110 f: &mut Formatter<'_>,
111 ) -> std::fmt::Result {
112 write!(f, "pack(")?;
113 for (i, name) in options.names.iter().enumerate() {
114 write!(f, "{}: ", name)?;
115 Display::fmt(expr.display_child(i), f)?;
116 if i + 1 < options.names.len() {
117 write!(f, ", ")?;
118 }
119 }
120 write!(f, "){}", options.nullability)
121 }
122
123 fn return_dtype(&self, options: &Self::Options, arg_dtypes: &[DType]) -> VortexResult<DType> {
124 Ok(DType::Struct(
125 StructFields::new(options.names.clone(), arg_dtypes.to_vec()),
126 options.nullability,
127 ))
128 }
129
130 fn validity<T: ReduceNode>(
131 &self,
132 _options: &Self::Options,
133 node: &T,
134 ) -> VortexResult<ReduceNodeValidity<T>> {
135 Ok(ReduceNodeValidity::Reduced(node.new_constant(true.into())))
136 }
137
138 fn execute(
139 &self,
140 options: &Self::Options,
141 args: &dyn ExecutionArgs,
142 ctx: &mut ExecutionCtx,
143 ) -> VortexResult<ArrayRef> {
144 let len = args.row_count();
145 let value_arrays: Vec<ArrayRef> = (0..args.num_inputs())
146 .map(|i| args.get(i))
147 .collect::<VortexResult<_>>()?;
148 let validity: Validity = options.nullability.into();
149 StructArray::try_new(options.names.clone(), value_arrays, len, validity)?
150 .into_array()
151 .execute(ctx)
152 }
153
154 fn is_strict(&self, _instance: &Self::Options) -> bool {
155 false
157 }
158
159 fn is_infallible(&self, _instance: &Self::Options) -> bool {
160 true
161 }
162}
163
164#[cfg(test)]
165mod tests {
166 use std::sync::LazyLock;
167
168 use vortex_buffer::buffer;
169 use vortex_error::VortexResult;
170 use vortex_error::vortex_bail;
171 use vortex_session::VortexSession;
172
173 use super::Pack;
174 use super::PackOptions;
175 use crate::ArrayRef;
176 use crate::IntoArray;
177 use crate::VortexSessionExecute;
178 use crate::arrays::PrimitiveArray;
179 use crate::arrays::struct_::StructArrayExt;
180 use crate::assert_arrays_eq;
181 use crate::dtype::Nullability;
182 use crate::expr::col;
183 use crate::expr::pack;
184 use crate::scalar_fn::ScalarFnVTableExt;
185 use crate::scalar_fn::fns::pack::StructArray;
186 use crate::validity::Validity;
187
188 static SESSION: LazyLock<VortexSession> = LazyLock::new(VortexSession::empty);
189
190 fn test_array() -> ArrayRef {
191 StructArray::from_fields(&[
192 ("a", buffer![0, 1, 2].into_array()),
193 ("b", buffer![4, 5, 6].into_array()),
194 ])
195 .unwrap()
196 .into_array()
197 }
198
199 fn field_array(array: &ArrayRef, field_path: &[&str]) -> VortexResult<ArrayRef> {
200 let mut ctx = SESSION.create_execution_ctx();
201 let mut field_path = field_path.iter();
202
203 let Some(field) = field_path.next() else {
204 vortex_bail!("empty field path");
205 };
206
207 let mut array = array
208 .clone()
209 .execute::<StructArray>(&mut ctx)?
210 .unmasked_field_by_name(field)?
211 .clone();
212 for field in field_path {
213 let next = array
214 .clone()
215 .execute::<StructArray>(&mut ctx)?
216 .unmasked_field_by_name(field)?
217 .clone();
218 array = next;
219 }
220 Ok(array)
221 }
222
223 #[test]
224 pub fn test_empty_pack() {
225 let mut ctx = SESSION.create_execution_ctx();
226 let expr = Pack.new_expr(
227 PackOptions {
228 names: Default::default(),
229 nullability: Default::default(),
230 },
231 [],
232 );
233
234 let test_array = test_array();
235 let actual_array = test_array.clone().apply(&expr).unwrap();
236 assert_eq!(actual_array.len(), test_array.len());
237 let nfields = actual_array
238 .execute::<StructArray>(&mut ctx)
239 .unwrap()
240 .struct_fields()
241 .nfields();
242 assert_eq!(nfields, 0);
243 }
244
245 #[test]
246 pub fn test_simple_pack() {
247 let mut ctx = SESSION.create_execution_ctx();
248 let expr = Pack.new_expr(
249 PackOptions {
250 names: ["one", "two", "three"].into(),
251 nullability: Nullability::NonNullable,
252 },
253 [col("a"), col("b"), col("a")],
254 );
255
256 let actual_array = test_array()
257 .apply(&expr)
258 .unwrap()
259 .execute::<StructArray>(&mut ctx)
260 .unwrap();
261
262 assert_eq!(actual_array.names(), ["one", "two", "three"]);
263 assert!(matches!(actual_array.validity(), Ok(Validity::NonNullable)));
264
265 assert_arrays_eq!(
266 field_array(&actual_array.clone().into_array(), &["one"]).unwrap(),
267 PrimitiveArray::from_iter([0i32, 1, 2]),
268 &mut ctx
269 );
270 assert_arrays_eq!(
271 field_array(&actual_array.clone().into_array(), &["two"]).unwrap(),
272 PrimitiveArray::from_iter([4i32, 5, 6]),
273 &mut ctx
274 );
275 assert_arrays_eq!(
276 field_array(&actual_array.into_array(), &["three"]).unwrap(),
277 PrimitiveArray::from_iter([0i32, 1, 2]),
278 &mut ctx
279 );
280 }
281
282 #[test]
283 pub fn test_nested_pack() {
284 let mut ctx = SESSION.create_execution_ctx();
285 let expr = Pack.new_expr(
286 PackOptions {
287 names: ["one", "two", "three"].into(),
288 nullability: Nullability::NonNullable,
289 },
290 [
291 col("a"),
292 Pack.new_expr(
293 PackOptions {
294 names: ["two_one", "two_two"].into(),
295 nullability: Nullability::NonNullable,
296 },
297 [col("b"), col("b")],
298 ),
299 col("a"),
300 ],
301 );
302
303 let actual_array = test_array()
304 .apply(&expr)
305 .unwrap()
306 .execute::<StructArray>(&mut ctx)
307 .unwrap();
308
309 assert_eq!(actual_array.names(), ["one", "two", "three"]);
310
311 assert_arrays_eq!(
312 field_array(&actual_array.clone().into_array(), &["one"]).unwrap(),
313 PrimitiveArray::from_iter([0i32, 1, 2]),
314 &mut ctx
315 );
316 assert_arrays_eq!(
317 field_array(&actual_array.clone().into_array(), &["two", "two_one"]).unwrap(),
318 PrimitiveArray::from_iter([4i32, 5, 6]),
319 &mut ctx
320 );
321 assert_arrays_eq!(
322 field_array(&actual_array.clone().into_array(), &["two", "two_two"]).unwrap(),
323 PrimitiveArray::from_iter([4i32, 5, 6]),
324 &mut ctx
325 );
326 assert_arrays_eq!(
327 field_array(&actual_array.into_array(), &["three"]).unwrap(),
328 PrimitiveArray::from_iter([0i32, 1, 2]),
329 &mut ctx
330 );
331 }
332
333 #[test]
334 pub fn test_pack_nullable() {
335 let mut ctx = SESSION.create_execution_ctx();
336 let expr = Pack.new_expr(
337 PackOptions {
338 names: ["one", "two", "three"].into(),
339 nullability: Nullability::Nullable,
340 },
341 [col("a"), col("b"), col("a")],
342 );
343
344 let actual_array = test_array()
345 .apply(&expr)
346 .unwrap()
347 .execute::<StructArray>(&mut ctx)
348 .unwrap();
349
350 assert_eq!(actual_array.names(), ["one", "two", "three"]);
351 assert!(matches!(actual_array.validity(), Ok(Validity::AllValid)));
352 let expected = StructArray::try_new(
353 ["one", "two", "three"].into(),
354 vec![
355 buffer![0i32, 1, 2].into_array(),
356 buffer![4i32, 5, 6].into_array(),
357 buffer![0i32, 1, 2].into_array(),
358 ],
359 3,
360 Validity::AllValid,
361 )
362 .unwrap();
363 assert_arrays_eq!(actual_array, expected, &mut ctx);
364 }
365
366 #[test]
367 pub fn test_display() {
368 let expr = pack(
369 [("id", col("user_id")), ("name", col("username"))],
370 Nullability::NonNullable,
371 );
372 assert_eq!(expr.to_string(), "pack(id: $.user_id, name: $.username)");
373
374 let expr2 = Pack.new_expr(
375 PackOptions {
376 names: ["x", "y"].into(),
377 nullability: Nullability::Nullable,
378 },
379 [col("a"), col("b")],
380 );
381 assert_eq!(expr2.to_string(), "pack(x: $.a, y: $.b)?");
382 }
383}