Skip to main content

vortex_array/scalar_fn/fns/fill_null/
mod.rs

1// SPDX-License-Identifier: Apache-2.0
2// SPDX-FileCopyrightText: Copyright the Vortex contributors
3
4mod kernel;
5
6pub use kernel::*;
7use vortex_error::VortexResult;
8use vortex_error::vortex_bail;
9use vortex_error::vortex_ensure;
10use vortex_error::vortex_err;
11use vortex_session::VortexSession;
12use vortex_session::registry::CachedId;
13
14use crate::AnyColumnar;
15use crate::ArrayRef;
16use crate::Canonical;
17use crate::CanonicalView;
18use crate::ColumnarView;
19use crate::ExecutionCtx;
20use crate::IntoArray;
21use crate::arrays::Bool;
22use crate::arrays::Constant;
23use crate::arrays::Decimal;
24use crate::arrays::Primitive;
25use crate::arrays::ScalarFnArray;
26use crate::arrays::VarBinView;
27use crate::dtype::DType;
28use crate::expr::BoundExpression;
29use crate::optimizer::ArrayOptimizer;
30use crate::require_input;
31use crate::scalar::Scalar;
32use crate::scalar_fn::Arity;
33use crate::scalar_fn::ChildName;
34use crate::scalar_fn::EmptyOptions;
35use crate::scalar_fn::ExecutionArgs;
36use crate::scalar_fn::ReduceNode;
37use crate::scalar_fn::ReduceNodeValidity;
38use crate::scalar_fn::ScalarFnExecutionResult;
39use crate::scalar_fn::ScalarFnId;
40use crate::scalar_fn::ScalarFnVTable;
41use crate::scalar_fn::ScalarFnVTableExt;
42use crate::scalar_fn::is_not_null_node;
43
44/// An expression that replaces null values in the input with a fill value.
45#[derive(Clone)]
46pub struct FillNull;
47
48impl FillNull {
49    /// Creates a lazy operation that replaces null input values with `fill_value`.
50    ///
51    /// # Errors
52    ///
53    /// Returns an error if the children have different lengths or incompatible dtypes.
54    pub fn try_new(input: ArrayRef, fill_value: ArrayRef) -> VortexResult<ScalarFnArray> {
55        ScalarFnArray::try_new(FillNull.bind(EmptyOptions), [input, fill_value])
56    }
57
58    /// Returns whether fill_null can execute inputs of `dtype`.
59    ///
60    /// These are the dtypes whose canonical arrays have a [`FillNullKernel`], so update this list
61    /// whenever `fill_null_canonical` gains or loses one. Rewrites into fill_null, such as the
62    /// CASE WHEN simplification, check it first because other dtypes fail at execution time.
63    pub(crate) fn supports_dtype(dtype: &DType) -> bool {
64        matches!(
65            dtype,
66            DType::Bool(_)
67                | DType::Primitive(..)
68                | DType::Decimal(..)
69                | DType::Utf8(_)
70                | DType::Binary(_)
71        )
72    }
73}
74
75impl ScalarFnVTable for FillNull {
76    type Options = EmptyOptions;
77
78    fn id(&self) -> ScalarFnId {
79        static ID: CachedId = CachedId::new("vortex.fill_null");
80        *ID
81    }
82
83    fn serialize(&self, _options: &Self::Options) -> VortexResult<Option<Vec<u8>>> {
84        Ok(Some(vec![]))
85    }
86
87    fn deserialize(
88        &self,
89        _metadata: &[u8],
90        _session: &VortexSession,
91    ) -> VortexResult<Self::Options> {
92        Ok(EmptyOptions)
93    }
94
95    fn arity(&self, _options: &Self::Options) -> Arity {
96        Arity::Exact(2)
97    }
98
99    fn child_name(&self, _options: &Self::Options, child_idx: usize) -> ChildName {
100        match child_idx {
101            0 => ChildName::from("input"),
102            1 => ChildName::from("fill_value"),
103            _ => unreachable!("Invalid child index {} for FillNull expression", child_idx),
104        }
105    }
106
107    fn return_dtype(&self, _options: &Self::Options, arg_dtypes: &[DType]) -> VortexResult<DType> {
108        vortex_ensure!(
109            arg_dtypes[0].eq_ignore_nullability(&arg_dtypes[1]),
110            "fill_null requires input and fill value to have the same base type, got {} and {}",
111            arg_dtypes[0],
112            arg_dtypes[1]
113        );
114        // The result dtype takes the nullability of the fill value.
115        Ok(arg_dtypes[0]
116            .clone()
117            .with_nullability(arg_dtypes[1].nullability()))
118    }
119
120    fn execute(
121        &self,
122        _options: &Self::Options,
123        args: &dyn ExecutionArgs,
124        ctx: &mut ExecutionCtx,
125    ) -> VortexResult<ArrayRef> {
126        let input = args.get(0)?;
127        let fill_value = args.get(1)?;
128        // Parent kernels decline a fill that is not already constant. Materialize it here so a
129        // lazy scalar, such as a literal `ScalarFnArray`, can still fill nulls.
130        let fill_was_lazy = !fill_value.is::<Constant>();
131        let fill_value = if fill_was_lazy {
132            fill_value.execute_until::<Constant>(ctx)?
133        } else {
134            fill_value
135        };
136
137        let Some(fill_scalar) = fill_value.as_constant() else {
138            // Slicing a constant fill to an empty range yields an empty canonical array rather
139            // than a constant. An empty input has no nulls to fill, so there is nothing to do.
140            vortex_ensure!(
141                input.is_empty(),
142                "fill_null fill_value must be a constant/scalar"
143            );
144            let dtype = input
145                .dtype()
146                .with_nullability(fill_value.dtype().nullability());
147            return Ok(Canonical::empty(&dtype).into_array());
148        };
149
150        vortex_ensure!(
151            !fill_scalar.is_null(),
152            "fill_null requires a non-null fill value"
153        );
154
155        let Some(columnar) = input.as_opt::<AnyColumnar>() else {
156            if fill_was_lazy {
157                // Hand the rebuilt node back with a constant fill so the input's FillNull
158                // kernels get a chance before we decode the input.
159                return FillNull::try_new(input, fill_value)?
160                    .into_array()
161                    .optimize_ctx(ctx.session());
162            }
163            let input = input.execute::<ArrayRef>(ctx)?;
164            return Ok(FillNull::try_new(input, fill_value)?.into_array());
165        };
166
167        match columnar {
168            ColumnarView::Canonical(canonical) => fill_null_canonical(canonical, fill_scalar, ctx),
169            ColumnarView::Constant(constant) => fill_null_constant(constant, fill_scalar),
170        }
171    }
172
173    fn execute_step(
174        &self,
175        options: &Self::Options,
176        args: &dyn ExecutionArgs,
177        ctx: &mut ExecutionCtx,
178    ) -> VortexResult<ScalarFnExecutionResult> {
179        // Only once the fill is constant: `execute` materializes a lazy fill first, so the input's
180        // FillNull kernels can still apply before the input is decoded.
181        if args.input_matches(1, |fill| fill.is::<Constant>())? {
182            require_input!(args, 0 => AnyColumnar);
183        }
184        self.execute(options, args, ctx)
185            .map(ScalarFnExecutionResult::done)
186    }
187
188    fn simplify(
189        &self,
190        _options: &Self::Options,
191        expr: &BoundExpression,
192    ) -> VortexResult<Option<BoundExpression>> {
193        if !expr.child(0).dtype().is_nullable() {
194            return Ok(Some(expr.child(0).clone()));
195        }
196
197        Ok(None)
198    }
199
200    fn validity<T: ReduceNode>(
201        &self,
202        _options: &Self::Options,
203        node: &T,
204    ) -> VortexResult<ReduceNodeValidity<T>> {
205        // After fill_null, the result validity depends on the fill value's nullability.
206        // If fill_value is non-nullable, the result is always valid.
207        Ok(ReduceNodeValidity::Reduced(is_not_null_node(
208            &node.child(1),
209        )?))
210    }
211
212    fn is_strict(&self, _options: &Self::Options) -> bool {
213        // This function replaces null input values instead of propagating them.
214        false
215    }
216
217    fn is_infallible(&self, _options: &Self::Options) -> bool {
218        true
219    }
220}
221
222/// Fill nulls on a canonical array by directly dispatching to the appropriate kernel.
223///
224/// Returns the filled array, or bails if no kernel is registered for the canonical type.
225fn fill_null_canonical(
226    canonical: CanonicalView<'_>,
227    fill_value: &Scalar,
228    ctx: &mut ExecutionCtx,
229) -> VortexResult<ArrayRef> {
230    let arr = canonical.to_array_ref();
231    if let Some(result) = short_circuit(&arr, fill_value)? {
232        return Ok(result);
233    }
234    match canonical {
235        CanonicalView::Bool(a) => <Bool as FillNullKernel>::fill_null(a, fill_value, ctx)?
236            .ok_or_else(|| vortex_err!("FillNullKernel for BoolArray returned None")),
237        CanonicalView::Primitive(a) => {
238            <Primitive as FillNullKernel>::fill_null(a, fill_value, ctx)?
239                .ok_or_else(|| vortex_err!("FillNullKernel for PrimitiveArray returned None"))
240        }
241        CanonicalView::Decimal(a) => <Decimal as FillNullKernel>::fill_null(a, fill_value, ctx)?
242            .ok_or_else(|| vortex_err!("FillNullKernel for DecimalArray returned None")),
243        CanonicalView::VarBinView(a) => {
244            <VarBinView as FillNullKernel>::fill_null(a, fill_value, ctx)?
245                .ok_or_else(|| vortex_err!("FillNullKernel for VarBinViewArray returned None"))
246        }
247        other => vortex_bail!(
248            "No FillNullKernel for canonical array {}",
249            other.to_array_ref().encoding_id()
250        ),
251    }
252}
253
254#[cfg(test)]
255mod tests {
256    use std::sync::Arc;
257    use std::sync::LazyLock;
258
259    use vortex_buffer::buffer;
260    use vortex_error::VortexExpect;
261    use vortex_session::VortexSession;
262
263    use super::*;
264    use crate::Canonical;
265    use crate::IntoArray;
266    use crate::VortexSessionExecute;
267    use crate::arrays::BoolArray;
268    use crate::arrays::Chunked;
269    use crate::arrays::ChunkedArray;
270    use crate::arrays::Constant;
271    use crate::arrays::ConstantArray;
272    use crate::arrays::Primitive;
273    use crate::arrays::PrimitiveArray;
274    use crate::arrays::ScalarFnArray;
275    use crate::arrays::StructArray;
276    use crate::arrays::scalar_fn::ExactScalarFn;
277    use crate::assert_arrays_eq;
278    use crate::builders::builder_with_capacity_in;
279    use crate::builtins::ArrayBuiltins;
280    use crate::dtype::DType;
281    use crate::dtype::DecimalDType;
282    use crate::dtype::MapDType;
283    use crate::dtype::Nullability;
284    use crate::dtype::PType;
285    use crate::dtype::StructFields;
286    use crate::expr::fill_null;
287    use crate::expr::get_item;
288    use crate::expr::lit;
289    use crate::expr::root;
290    use crate::extension::datetime::Date;
291    use crate::extension::datetime::TimeUnit;
292    use crate::kernel::ExecuteParentKernel;
293    use crate::matcher::Matcher;
294    use crate::optimizer::rules::ArrayParentReduceRule;
295    use crate::scalar::Scalar;
296    use crate::scalar_fn::fns::binary::Binary;
297    use crate::scalar_fn::fns::operators::Operator;
298
299    static SESSION: LazyLock<VortexSession> = LazyLock::new(VortexSession::empty);
300
301    /// A fill of `40 + 2` that is not a [`ConstantArray`] until it executes.
302    fn lazy_fill_42(len: usize) -> VortexResult<ArrayRef> {
303        Ok(ScalarFnArray::try_new(
304            Binary.bind(Operator::Add),
305            vec![
306                ConstantArray::new(40i32, len).into_array(),
307                ConstantArray::new(2i32, len).into_array(),
308            ],
309        )?
310        .into_array())
311    }
312
313    #[test]
314    fn dtype() {
315        let dtype = DType::Primitive(PType::I32, Nullability::Nullable);
316        assert_eq!(
317            fill_null(root(), lit(0i32)).return_dtype(&dtype).unwrap(),
318            DType::Primitive(PType::I32, Nullability::NonNullable)
319        );
320    }
321
322    #[test]
323    fn replace_children() {
324        let expr = fill_null(root(), lit(0i32));
325        let replaced = expr
326            .with_children(vec![get_item("a", root()), lit(42i32)])
327            .vortex_expect("operation should succeed in test");
328        assert_eq!(replaced, fill_null(get_item("a", root()), lit(42i32)));
329    }
330
331    #[test]
332    fn evaluate() {
333        let mut ctx = SESSION.create_execution_ctx();
334        let test_array =
335            PrimitiveArray::from_option_iter([Some(1i32), None, Some(3), None, Some(5)])
336                .into_array();
337
338        let expr = fill_null(root(), lit(42i32));
339        let result = test_array.apply(&expr).unwrap();
340
341        assert_eq!(
342            result.dtype(),
343            &DType::Primitive(PType::I32, Nullability::NonNullable)
344        );
345        assert_arrays_eq!(
346            result,
347            PrimitiveArray::from_iter([1i32, 42, 3, 42, 5]),
348            &mut ctx
349        );
350    }
351
352    #[test]
353    fn evaluate_lazy_constant_fill() -> VortexResult<()> {
354        let mut ctx = SESSION.create_execution_ctx();
355        let input = PrimitiveArray::from_option_iter([Some(1i32), None, Some(3)]).into_array();
356        let fill = lazy_fill_42(input.len())?;
357        assert!(fill.as_constant().is_none());
358
359        let result = FillNull::try_new(input, fill)?
360            .into_array()
361            .execute::<ArrayRef>(&mut ctx)?;
362
363        assert_eq!(
364            result.dtype(),
365            &DType::Primitive(PType::I32, Nullability::NonNullable)
366        );
367        assert_arrays_eq!(result, PrimitiveArray::from_iter([1i32, 42, 3]), &mut ctx);
368        Ok(())
369    }
370
371    #[test]
372    fn evaluate_lazy_fill_preserves_chunked_input() -> VortexResult<()> {
373        let mut ctx = SESSION.create_execution_ctx();
374        let input = ChunkedArray::try_new(
375            vec![
376                PrimitiveArray::from_option_iter([Some(1i32), None]).into_array(),
377                PrimitiveArray::from_option_iter([Some(3), None]).into_array(),
378            ],
379            DType::Primitive(PType::I32, Nullability::Nullable),
380        )?
381        .into_array();
382        let fill = lazy_fill_42(input.len())?;
383
384        let result = FillNull::try_new(input, fill)?
385            .into_array()
386            .execute::<ArrayRef>(&mut ctx)?;
387
388        assert!(result.is::<Chunked>());
389        assert_arrays_eq!(
390            result,
391            PrimitiveArray::from_iter([1i32, 42, 3, 42]),
392            &mut ctx
393        );
394        Ok(())
395    }
396
397    #[test]
398    fn non_constant_fill_declines_and_errors() -> VortexResult<()> {
399        let mut ctx = SESSION.create_execution_ctx();
400        let varying = PrimitiveArray::from_iter([1i32, 2, 3]).into_array();
401
402        let constant_input = ConstantArray::new(Scalar::null_native::<i32>(), 3).into_array();
403        let constant_parent =
404            FillNull::try_new(constant_input.clone(), varying.clone())?.into_array();
405        let reduced = FillNullReduceAdaptor(Constant).reduce_parent(
406            constant_input
407                .as_opt::<Constant>()
408                .vortex_expect("constant input"),
409            ExactScalarFn::<FillNull>::try_match(&constant_parent)
410                .vortex_expect("fill_null parent"),
411            0,
412        )?;
413        assert!(reduced.is_none());
414
415        let primitive_input =
416            PrimitiveArray::from_option_iter([Some(1i32), None, Some(3)]).into_array();
417        let primitive_parent = FillNull::try_new(primitive_input.clone(), varying)?.into_array();
418        let executed = FillNullExecuteAdaptor(Primitive).execute_parent(
419            primitive_input
420                .as_opt::<Primitive>()
421                .vortex_expect("primitive input"),
422            ExactScalarFn::<FillNull>::try_match(&primitive_parent)
423                .vortex_expect("fill_null parent"),
424            0,
425            &mut ctx,
426        )?;
427        assert!(executed.is_none());
428
429        let err = primitive_parent.execute::<ArrayRef>(&mut ctx).unwrap_err();
430        assert!(
431            err.to_string()
432                .contains("fill_null fill_value must be a constant/scalar")
433        );
434        Ok(())
435    }
436
437    /// Filling a chunked array slices the constant fill per chunk, and an empty slice is a
438    /// canonical array rather than a constant. Stacked fills over an empty chunk must still run.
439    #[test]
440    fn repeated_fill_over_empty_chunk() -> VortexResult<()> {
441        let mut ctx = SESSION.create_execution_ctx();
442        let input = ChunkedArray::try_new(
443            vec![
444                PrimitiveArray::from_option_iter(Vec::<Option<u16>>::new()).into_array(),
445                PrimitiveArray::from_option_iter([Some(7u16), None]).into_array(),
446            ],
447            DType::Primitive(PType::U16, Nullability::Nullable),
448        )?
449        .into_array();
450
451        let result = input
452            .fill_null(Scalar::primitive(1u16, Nullability::Nullable))?
453            .fill_null(Scalar::primitive(2u16, Nullability::NonNullable))?;
454
455        assert_arrays_eq!(result, PrimitiveArray::from_iter([7u16, 1]), &mut ctx);
456        Ok(())
457    }
458
459    #[test]
460    fn evaluate_struct_field() {
461        let mut ctx = SESSION.create_execution_ctx();
462        let test_array = StructArray::from_fields(&[(
463            "a",
464            PrimitiveArray::from_option_iter([Some(1i32), None, Some(3)]).into_array(),
465        )])
466        .unwrap()
467        .into_array();
468
469        let expr = fill_null(get_item("a", root()), lit(0i32));
470        let result = test_array.apply(&expr).unwrap();
471
472        assert_eq!(
473            result.dtype(),
474            &DType::Primitive(PType::I32, Nullability::NonNullable)
475        );
476        assert_arrays_eq!(result, PrimitiveArray::from_iter([1i32, 0, 3]), &mut ctx);
477    }
478
479    #[test]
480    fn evaluate_non_nullable_input() {
481        let mut ctx = SESSION.create_execution_ctx();
482        let test_array = buffer![1i32, 2, 3].into_array();
483        let expr = fill_null(root(), lit(0i32));
484        let result = test_array.apply(&expr).unwrap();
485        assert_arrays_eq!(result, PrimitiveArray::from_iter([1i32, 2, 3]), &mut ctx);
486    }
487
488    /// Inputs with both valid and null rows execute exactly for the dtypes that `supports_dtype`
489    /// accepts, so adding a kernel without updating `supports_dtype` fails here.
490    #[test]
491    fn supports_dtype_matches_kernels() -> VortexResult<()> {
492        let mut ctx = SESSION.create_execution_ctx();
493        let i32_dtype = DType::Primitive(PType::I32, Nullability::NonNullable);
494        let dtypes = [
495            DType::Bool(Nullability::Nullable),
496            DType::Primitive(PType::I32, Nullability::Nullable),
497            DType::Decimal(DecimalDType::new(19, 2), Nullability::Nullable),
498            DType::Utf8(Nullability::Nullable),
499            DType::Binary(Nullability::Nullable),
500            DType::List(Arc::new(i32_dtype.clone()), Nullability::Nullable),
501            DType::FixedSizeList(Arc::new(i32_dtype.clone()), 2, Nullability::Nullable),
502            DType::Struct(
503                StructFields::from_iter([("value", i32_dtype.clone())]),
504                Nullability::Nullable,
505            ),
506            DType::Map(
507                MapDType::try_new(i32_dtype.clone(), i32_dtype, false)?,
508                Nullability::Nullable,
509            ),
510            DType::Extension(Date::new(TimeUnit::Days, Nullability::Nullable).erased()),
511        ];
512
513        let supported = [
514            true, true, true, true, true, false, false, false, false, false,
515        ];
516        for (dtype, supported) in dtypes.into_iter().zip(supported) {
517            assert_eq!(FillNull::supports_dtype(&dtype), supported, "{dtype}");
518            let fill = Scalar::default_value(&dtype.as_nonnullable());
519            let mut builder = builder_with_capacity_in(&dtype, 2, ctx.allocator());
520            builder.append_scalar(&fill.cast(&dtype)?)?;
521            builder.append_null();
522            let filled = builder
523                .finish()
524                .fill_null(fill.clone())?
525                .execute::<Canonical>(&mut ctx);
526
527            if supported {
528                let filled = filled?.into_array();
529                assert_eq!(filled.dtype(), fill.dtype(), "{dtype}");
530                assert_eq!(filled.execute_scalar(1, &mut ctx)?, fill, "{dtype}");
531                assert_arrays_eq!(filled, ConstantArray::new(fill, 2), &mut ctx);
532            } else {
533                assert!(filled.is_err(), "{dtype} executes but is not supported");
534            }
535        }
536        Ok(())
537    }
538
539    #[test]
540    fn test_display() {
541        let expr = fill_null(get_item("value", root()), lit(0i32));
542        assert_eq!(expr.to_string(), "vortex.fill_null($.value, 0i32)");
543    }
544
545    /// A scalar function input is executed through the executor before its nulls are filled.
546    #[test]
547    fn executes_non_columnar_input_iteratively() -> VortexResult<()> {
548        let mut ctx = SESSION.create_execution_ctx();
549        let input = PrimitiveArray::from_option_iter([Some(1i32), None, Some(3)])
550            .into_array()
551            .binary(ConstantArray::new(2i32, 3).into_array(), Operator::Gt)?;
552
553        let result = input.fill_null(Scalar::from(false))?;
554
555        assert_arrays_eq!(result, BoolArray::from_iter([false, false, true]), &mut ctx);
556        Ok(())
557    }
558}