1mod kernel;
5
6pub use kernel::*;
7use vortex_error::VortexResult;
8use vortex_error::vortex_bail;
9use vortex_error::vortex_ensure;
10use vortex_error::vortex_err;
11use vortex_session::VortexSession;
12use vortex_session::registry::CachedId;
13
14use crate::AnyColumnar;
15use crate::ArrayRef;
16use crate::Canonical;
17use crate::CanonicalView;
18use crate::ColumnarView;
19use crate::ExecutionCtx;
20use crate::IntoArray;
21use crate::arrays::Bool;
22use crate::arrays::Constant;
23use crate::arrays::Decimal;
24use crate::arrays::Primitive;
25use crate::arrays::ScalarFnArray;
26use crate::arrays::VarBinView;
27use crate::dtype::DType;
28use crate::expr::BoundExpression;
29use crate::optimizer::ArrayOptimizer;
30use crate::require_input;
31use crate::scalar::Scalar;
32use crate::scalar_fn::Arity;
33use crate::scalar_fn::ChildName;
34use crate::scalar_fn::EmptyOptions;
35use crate::scalar_fn::ExecutionArgs;
36use crate::scalar_fn::ReduceNode;
37use crate::scalar_fn::ReduceNodeValidity;
38use crate::scalar_fn::ScalarFnExecutionResult;
39use crate::scalar_fn::ScalarFnId;
40use crate::scalar_fn::ScalarFnVTable;
41use crate::scalar_fn::ScalarFnVTableExt;
42use crate::scalar_fn::is_not_null_node;
43
44#[derive(Clone)]
46pub struct FillNull;
47
48impl FillNull {
49 pub fn try_new(input: ArrayRef, fill_value: ArrayRef) -> VortexResult<ScalarFnArray> {
55 ScalarFnArray::try_new(FillNull.bind(EmptyOptions), [input, fill_value])
56 }
57
58 pub(crate) fn supports_dtype(dtype: &DType) -> bool {
64 matches!(
65 dtype,
66 DType::Bool(_)
67 | DType::Primitive(..)
68 | DType::Decimal(..)
69 | DType::Utf8(_)
70 | DType::Binary(_)
71 )
72 }
73}
74
75impl ScalarFnVTable for FillNull {
76 type Options = EmptyOptions;
77
78 fn id(&self) -> ScalarFnId {
79 static ID: CachedId = CachedId::new("vortex.fill_null");
80 *ID
81 }
82
83 fn serialize(&self, _options: &Self::Options) -> VortexResult<Option<Vec<u8>>> {
84 Ok(Some(vec![]))
85 }
86
87 fn deserialize(
88 &self,
89 _metadata: &[u8],
90 _session: &VortexSession,
91 ) -> VortexResult<Self::Options> {
92 Ok(EmptyOptions)
93 }
94
95 fn arity(&self, _options: &Self::Options) -> Arity {
96 Arity::Exact(2)
97 }
98
99 fn child_name(&self, _options: &Self::Options, child_idx: usize) -> ChildName {
100 match child_idx {
101 0 => ChildName::from("input"),
102 1 => ChildName::from("fill_value"),
103 _ => unreachable!("Invalid child index {} for FillNull expression", child_idx),
104 }
105 }
106
107 fn return_dtype(&self, _options: &Self::Options, arg_dtypes: &[DType]) -> VortexResult<DType> {
108 vortex_ensure!(
109 arg_dtypes[0].eq_ignore_nullability(&arg_dtypes[1]),
110 "fill_null requires input and fill value to have the same base type, got {} and {}",
111 arg_dtypes[0],
112 arg_dtypes[1]
113 );
114 Ok(arg_dtypes[0]
116 .clone()
117 .with_nullability(arg_dtypes[1].nullability()))
118 }
119
120 fn execute(
121 &self,
122 _options: &Self::Options,
123 args: &dyn ExecutionArgs,
124 ctx: &mut ExecutionCtx,
125 ) -> VortexResult<ArrayRef> {
126 let input = args.get(0)?;
127 let fill_value = args.get(1)?;
128 let fill_was_lazy = !fill_value.is::<Constant>();
131 let fill_value = if fill_was_lazy {
132 fill_value.execute_until::<Constant>(ctx)?
133 } else {
134 fill_value
135 };
136
137 let Some(fill_scalar) = fill_value.as_constant() else {
138 vortex_ensure!(
141 input.is_empty(),
142 "fill_null fill_value must be a constant/scalar"
143 );
144 let dtype = input
145 .dtype()
146 .with_nullability(fill_value.dtype().nullability());
147 return Ok(Canonical::empty(&dtype).into_array());
148 };
149
150 vortex_ensure!(
151 !fill_scalar.is_null(),
152 "fill_null requires a non-null fill value"
153 );
154
155 let Some(columnar) = input.as_opt::<AnyColumnar>() else {
156 if fill_was_lazy {
157 return FillNull::try_new(input, fill_value)?
160 .into_array()
161 .optimize_ctx(ctx.session());
162 }
163 let input = input.execute::<ArrayRef>(ctx)?;
164 return Ok(FillNull::try_new(input, fill_value)?.into_array());
165 };
166
167 match columnar {
168 ColumnarView::Canonical(canonical) => fill_null_canonical(canonical, fill_scalar, ctx),
169 ColumnarView::Constant(constant) => fill_null_constant(constant, fill_scalar),
170 }
171 }
172
173 fn execute_step(
174 &self,
175 options: &Self::Options,
176 args: &dyn ExecutionArgs,
177 ctx: &mut ExecutionCtx,
178 ) -> VortexResult<ScalarFnExecutionResult> {
179 if args.input_matches(1, |fill| fill.is::<Constant>())? {
182 require_input!(args, 0 => AnyColumnar);
183 }
184 self.execute(options, args, ctx)
185 .map(ScalarFnExecutionResult::done)
186 }
187
188 fn simplify(
189 &self,
190 _options: &Self::Options,
191 expr: &BoundExpression,
192 ) -> VortexResult<Option<BoundExpression>> {
193 if !expr.child(0).dtype().is_nullable() {
194 return Ok(Some(expr.child(0).clone()));
195 }
196
197 Ok(None)
198 }
199
200 fn validity<T: ReduceNode>(
201 &self,
202 _options: &Self::Options,
203 node: &T,
204 ) -> VortexResult<ReduceNodeValidity<T>> {
205 Ok(ReduceNodeValidity::Reduced(is_not_null_node(
208 &node.child(1),
209 )?))
210 }
211
212 fn is_strict(&self, _options: &Self::Options) -> bool {
213 false
215 }
216
217 fn is_infallible(&self, _options: &Self::Options) -> bool {
218 true
219 }
220}
221
222fn fill_null_canonical(
226 canonical: CanonicalView<'_>,
227 fill_value: &Scalar,
228 ctx: &mut ExecutionCtx,
229) -> VortexResult<ArrayRef> {
230 let arr = canonical.to_array_ref();
231 if let Some(result) = short_circuit(&arr, fill_value)? {
232 return Ok(result);
233 }
234 match canonical {
235 CanonicalView::Bool(a) => <Bool as FillNullKernel>::fill_null(a, fill_value, ctx)?
236 .ok_or_else(|| vortex_err!("FillNullKernel for BoolArray returned None")),
237 CanonicalView::Primitive(a) => {
238 <Primitive as FillNullKernel>::fill_null(a, fill_value, ctx)?
239 .ok_or_else(|| vortex_err!("FillNullKernel for PrimitiveArray returned None"))
240 }
241 CanonicalView::Decimal(a) => <Decimal as FillNullKernel>::fill_null(a, fill_value, ctx)?
242 .ok_or_else(|| vortex_err!("FillNullKernel for DecimalArray returned None")),
243 CanonicalView::VarBinView(a) => {
244 <VarBinView as FillNullKernel>::fill_null(a, fill_value, ctx)?
245 .ok_or_else(|| vortex_err!("FillNullKernel for VarBinViewArray returned None"))
246 }
247 other => vortex_bail!(
248 "No FillNullKernel for canonical array {}",
249 other.to_array_ref().encoding_id()
250 ),
251 }
252}
253
254#[cfg(test)]
255mod tests {
256 use std::sync::Arc;
257 use std::sync::LazyLock;
258
259 use vortex_buffer::buffer;
260 use vortex_error::VortexExpect;
261 use vortex_session::VortexSession;
262
263 use super::*;
264 use crate::Canonical;
265 use crate::IntoArray;
266 use crate::VortexSessionExecute;
267 use crate::arrays::BoolArray;
268 use crate::arrays::Chunked;
269 use crate::arrays::ChunkedArray;
270 use crate::arrays::Constant;
271 use crate::arrays::ConstantArray;
272 use crate::arrays::Primitive;
273 use crate::arrays::PrimitiveArray;
274 use crate::arrays::ScalarFnArray;
275 use crate::arrays::StructArray;
276 use crate::arrays::scalar_fn::ExactScalarFn;
277 use crate::assert_arrays_eq;
278 use crate::builders::builder_with_capacity_in;
279 use crate::builtins::ArrayBuiltins;
280 use crate::dtype::DType;
281 use crate::dtype::DecimalDType;
282 use crate::dtype::MapDType;
283 use crate::dtype::Nullability;
284 use crate::dtype::PType;
285 use crate::dtype::StructFields;
286 use crate::expr::fill_null;
287 use crate::expr::get_item;
288 use crate::expr::lit;
289 use crate::expr::root;
290 use crate::extension::datetime::Date;
291 use crate::extension::datetime::TimeUnit;
292 use crate::kernel::ExecuteParentKernel;
293 use crate::matcher::Matcher;
294 use crate::optimizer::rules::ArrayParentReduceRule;
295 use crate::scalar::Scalar;
296 use crate::scalar_fn::fns::binary::Binary;
297 use crate::scalar_fn::fns::operators::Operator;
298
299 static SESSION: LazyLock<VortexSession> = LazyLock::new(VortexSession::empty);
300
301 fn lazy_fill_42(len: usize) -> VortexResult<ArrayRef> {
303 Ok(ScalarFnArray::try_new(
304 Binary.bind(Operator::Add),
305 vec![
306 ConstantArray::new(40i32, len).into_array(),
307 ConstantArray::new(2i32, len).into_array(),
308 ],
309 )?
310 .into_array())
311 }
312
313 #[test]
314 fn dtype() {
315 let dtype = DType::Primitive(PType::I32, Nullability::Nullable);
316 assert_eq!(
317 fill_null(root(), lit(0i32)).return_dtype(&dtype).unwrap(),
318 DType::Primitive(PType::I32, Nullability::NonNullable)
319 );
320 }
321
322 #[test]
323 fn replace_children() {
324 let expr = fill_null(root(), lit(0i32));
325 let replaced = expr
326 .with_children(vec![get_item("a", root()), lit(42i32)])
327 .vortex_expect("operation should succeed in test");
328 assert_eq!(replaced, fill_null(get_item("a", root()), lit(42i32)));
329 }
330
331 #[test]
332 fn evaluate() {
333 let mut ctx = SESSION.create_execution_ctx();
334 let test_array =
335 PrimitiveArray::from_option_iter([Some(1i32), None, Some(3), None, Some(5)])
336 .into_array();
337
338 let expr = fill_null(root(), lit(42i32));
339 let result = test_array.apply(&expr).unwrap();
340
341 assert_eq!(
342 result.dtype(),
343 &DType::Primitive(PType::I32, Nullability::NonNullable)
344 );
345 assert_arrays_eq!(
346 result,
347 PrimitiveArray::from_iter([1i32, 42, 3, 42, 5]),
348 &mut ctx
349 );
350 }
351
352 #[test]
353 fn evaluate_lazy_constant_fill() -> VortexResult<()> {
354 let mut ctx = SESSION.create_execution_ctx();
355 let input = PrimitiveArray::from_option_iter([Some(1i32), None, Some(3)]).into_array();
356 let fill = lazy_fill_42(input.len())?;
357 assert!(fill.as_constant().is_none());
358
359 let result = FillNull::try_new(input, fill)?
360 .into_array()
361 .execute::<ArrayRef>(&mut ctx)?;
362
363 assert_eq!(
364 result.dtype(),
365 &DType::Primitive(PType::I32, Nullability::NonNullable)
366 );
367 assert_arrays_eq!(result, PrimitiveArray::from_iter([1i32, 42, 3]), &mut ctx);
368 Ok(())
369 }
370
371 #[test]
372 fn evaluate_lazy_fill_preserves_chunked_input() -> VortexResult<()> {
373 let mut ctx = SESSION.create_execution_ctx();
374 let input = ChunkedArray::try_new(
375 vec![
376 PrimitiveArray::from_option_iter([Some(1i32), None]).into_array(),
377 PrimitiveArray::from_option_iter([Some(3), None]).into_array(),
378 ],
379 DType::Primitive(PType::I32, Nullability::Nullable),
380 )?
381 .into_array();
382 let fill = lazy_fill_42(input.len())?;
383
384 let result = FillNull::try_new(input, fill)?
385 .into_array()
386 .execute::<ArrayRef>(&mut ctx)?;
387
388 assert!(result.is::<Chunked>());
389 assert_arrays_eq!(
390 result,
391 PrimitiveArray::from_iter([1i32, 42, 3, 42]),
392 &mut ctx
393 );
394 Ok(())
395 }
396
397 #[test]
398 fn non_constant_fill_declines_and_errors() -> VortexResult<()> {
399 let mut ctx = SESSION.create_execution_ctx();
400 let varying = PrimitiveArray::from_iter([1i32, 2, 3]).into_array();
401
402 let constant_input = ConstantArray::new(Scalar::null_native::<i32>(), 3).into_array();
403 let constant_parent =
404 FillNull::try_new(constant_input.clone(), varying.clone())?.into_array();
405 let reduced = FillNullReduceAdaptor(Constant).reduce_parent(
406 constant_input
407 .as_opt::<Constant>()
408 .vortex_expect("constant input"),
409 ExactScalarFn::<FillNull>::try_match(&constant_parent)
410 .vortex_expect("fill_null parent"),
411 0,
412 )?;
413 assert!(reduced.is_none());
414
415 let primitive_input =
416 PrimitiveArray::from_option_iter([Some(1i32), None, Some(3)]).into_array();
417 let primitive_parent = FillNull::try_new(primitive_input.clone(), varying)?.into_array();
418 let executed = FillNullExecuteAdaptor(Primitive).execute_parent(
419 primitive_input
420 .as_opt::<Primitive>()
421 .vortex_expect("primitive input"),
422 ExactScalarFn::<FillNull>::try_match(&primitive_parent)
423 .vortex_expect("fill_null parent"),
424 0,
425 &mut ctx,
426 )?;
427 assert!(executed.is_none());
428
429 let err = primitive_parent.execute::<ArrayRef>(&mut ctx).unwrap_err();
430 assert!(
431 err.to_string()
432 .contains("fill_null fill_value must be a constant/scalar")
433 );
434 Ok(())
435 }
436
437 #[test]
440 fn repeated_fill_over_empty_chunk() -> VortexResult<()> {
441 let mut ctx = SESSION.create_execution_ctx();
442 let input = ChunkedArray::try_new(
443 vec![
444 PrimitiveArray::from_option_iter(Vec::<Option<u16>>::new()).into_array(),
445 PrimitiveArray::from_option_iter([Some(7u16), None]).into_array(),
446 ],
447 DType::Primitive(PType::U16, Nullability::Nullable),
448 )?
449 .into_array();
450
451 let result = input
452 .fill_null(Scalar::primitive(1u16, Nullability::Nullable))?
453 .fill_null(Scalar::primitive(2u16, Nullability::NonNullable))?;
454
455 assert_arrays_eq!(result, PrimitiveArray::from_iter([7u16, 1]), &mut ctx);
456 Ok(())
457 }
458
459 #[test]
460 fn evaluate_struct_field() {
461 let mut ctx = SESSION.create_execution_ctx();
462 let test_array = StructArray::from_fields(&[(
463 "a",
464 PrimitiveArray::from_option_iter([Some(1i32), None, Some(3)]).into_array(),
465 )])
466 .unwrap()
467 .into_array();
468
469 let expr = fill_null(get_item("a", root()), lit(0i32));
470 let result = test_array.apply(&expr).unwrap();
471
472 assert_eq!(
473 result.dtype(),
474 &DType::Primitive(PType::I32, Nullability::NonNullable)
475 );
476 assert_arrays_eq!(result, PrimitiveArray::from_iter([1i32, 0, 3]), &mut ctx);
477 }
478
479 #[test]
480 fn evaluate_non_nullable_input() {
481 let mut ctx = SESSION.create_execution_ctx();
482 let test_array = buffer![1i32, 2, 3].into_array();
483 let expr = fill_null(root(), lit(0i32));
484 let result = test_array.apply(&expr).unwrap();
485 assert_arrays_eq!(result, PrimitiveArray::from_iter([1i32, 2, 3]), &mut ctx);
486 }
487
488 #[test]
491 fn supports_dtype_matches_kernels() -> VortexResult<()> {
492 let mut ctx = SESSION.create_execution_ctx();
493 let i32_dtype = DType::Primitive(PType::I32, Nullability::NonNullable);
494 let dtypes = [
495 DType::Bool(Nullability::Nullable),
496 DType::Primitive(PType::I32, Nullability::Nullable),
497 DType::Decimal(DecimalDType::new(19, 2), Nullability::Nullable),
498 DType::Utf8(Nullability::Nullable),
499 DType::Binary(Nullability::Nullable),
500 DType::List(Arc::new(i32_dtype.clone()), Nullability::Nullable),
501 DType::FixedSizeList(Arc::new(i32_dtype.clone()), 2, Nullability::Nullable),
502 DType::Struct(
503 StructFields::from_iter([("value", i32_dtype.clone())]),
504 Nullability::Nullable,
505 ),
506 DType::Map(
507 MapDType::try_new(i32_dtype.clone(), i32_dtype, false)?,
508 Nullability::Nullable,
509 ),
510 DType::Extension(Date::new(TimeUnit::Days, Nullability::Nullable).erased()),
511 ];
512
513 let supported = [
514 true, true, true, true, true, false, false, false, false, false,
515 ];
516 for (dtype, supported) in dtypes.into_iter().zip(supported) {
517 assert_eq!(FillNull::supports_dtype(&dtype), supported, "{dtype}");
518 let fill = Scalar::default_value(&dtype.as_nonnullable());
519 let mut builder = builder_with_capacity_in(&dtype, 2, ctx.allocator());
520 builder.append_scalar(&fill.cast(&dtype)?)?;
521 builder.append_null();
522 let filled = builder
523 .finish()
524 .fill_null(fill.clone())?
525 .execute::<Canonical>(&mut ctx);
526
527 if supported {
528 let filled = filled?.into_array();
529 assert_eq!(filled.dtype(), fill.dtype(), "{dtype}");
530 assert_eq!(filled.execute_scalar(1, &mut ctx)?, fill, "{dtype}");
531 assert_arrays_eq!(filled, ConstantArray::new(fill, 2), &mut ctx);
532 } else {
533 assert!(filled.is_err(), "{dtype} executes but is not supported");
534 }
535 }
536 Ok(())
537 }
538
539 #[test]
540 fn test_display() {
541 let expr = fill_null(get_item("value", root()), lit(0i32));
542 assert_eq!(expr.to_string(), "vortex.fill_null($.value, 0i32)");
543 }
544
545 #[test]
547 fn executes_non_columnar_input_iteratively() -> VortexResult<()> {
548 let mut ctx = SESSION.create_execution_ctx();
549 let input = PrimitiveArray::from_option_iter([Some(1i32), None, Some(3)])
550 .into_array()
551 .binary(ConstantArray::new(2i32, 3).into_array(), Operator::Gt)?;
552
553 let result = input.fill_null(Scalar::from(false))?;
554
555 assert_arrays_eq!(result, BoolArray::from_iter([false, false, true]), &mut ctx);
556 Ok(())
557 }
558}